diff --git a/.github/branch-protection.json b/.github/branch-protection.json index 985d4116..4aca6925 100644 --- a/.github/branch-protection.json +++ b/.github/branch-protection.json @@ -3,17 +3,10 @@ "strict": false, "contexts": [ "py3.12 on ubuntu-latest", - "py3.13 on ubuntu-latest", - "py3.14 on ubuntu-latest", - "py3.12 on windows-latest", - "py3.13 on windows-latest", - "py3.14 on windows-latest", - "py3.12 on macos-latest", - "py3.13 on macos-latest", - "py3.14 on macos-latest", "coverage gate (85%)", "black and flake8", "frontend modules", + "all tests", "viewerjs", "real ComfyUI compatibility", "bandit (high severity)" diff --git a/.github/workflows/portability.yml b/.github/workflows/portability.yml new file mode 100644 index 00000000..75122ab0 --- /dev/null +++ b/.github/workflows/portability.yml @@ -0,0 +1,55 @@ +name: Windows and macOS + +permissions: + contents: read + +# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context +# here is a required check. A red run is fixed at its cause before the workspace cutover (#334). +on: + push: + branches: [main] + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +# Every run gets its own group (run_id), so no main commit's run is ever cancelled or dropped as pending. +concurrency: + group: ${{ github.workflow }}-${{ github.run_id }} + cancel-in-progress: false + +env: + PYTHONFAULTHANDLER: "1" + +# Time limit rule as in test.yml: ceil(1.5 x the slowest of three recent green runs on main) + 3 +# minutes, never under 5. Measured 2026-10-26 in test.yml on runs 40257217681 (3be0ada), +# 40240706568 (fb9134f), 40228167328 (f9835aa), before these legs moved here in #420: +# slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min +# (macos 6m14s-8m30s, windows 11m42s-15m54s) +jobs: + test: + name: py${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 27 + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + python-version: ["3.12", "3.13", "3.14"] + + steps: + - uses: actions/checkout@v7 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: requirements-dev.txt + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Run tests + run: python -m tests.run_suite -v diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc7321fa..85194d1a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,10 +28,18 @@ env: # py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min # (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s) # 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs. +# Windows and macOS moved to portability.yml (#420), which keeps this 27-minute limit; this +# Linux-only matrix keeps it too until it is re-measured. # real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min # coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min # perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s, # frontend modules 23s / 19s / 29s -> 5 min (the floor) +# all tests: a result check without setup, 5 min (the floor) +# +# Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend +# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on +# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set. +# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge. jobs: test: name: py${{ matrix.python-version }} on ${{ matrix.os }} @@ -40,8 +48,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.12", "3.13", "3.14"] + os: [ubuntu-latest] + python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.12"]') || fromJSON('["3.12", "3.13", "3.14"]') }} steps: - uses: actions/checkout@v7 @@ -63,6 +71,9 @@ jobs: comfyui: name: real ComfyUI compatibility + # Merge queue, main and manual runs only. A job skipped by its condition reports Success, so this + # required context never blocks a pull request; "all tests" fails if it is skipped anywhere else. + if: github.event_name != 'pull_request' runs-on: ubuntu-latest # Chromium and its system libraries come from the pinned Playwright image (tag = @playwright/test in # tools/vendor/package-lock.json; tests/vendor/browser-image.test.mjs checks it). The runner install @@ -221,7 +232,10 @@ jobs: - uses: actions/setup-python@v7 with: python-version: "3.12" - - run: pip install black flake8 + cache: pip + cache-dependency-path: requirements-dev.txt + # The exact black and flake8 pins from requirements-dev.txt, so a new release cannot turn CI red. + - run: pip install $(grep -E '^(black|flake8)==' requirements-dev.txt) - run: black --check . - run: flake8 . @@ -235,6 +249,8 @@ jobs: - uses: actions/setup-node@v7 with: node-version: "24" + cache: npm + cache-dependency-path: tools/vendor/package-lock.json - name: Check shipped extension and page modules parse run: | @@ -248,3 +264,32 @@ jobs: - name: Run JS tests run: node --test tests/js/*.test.js + + # The required context "all tests". A dropped matrix leg never reports a check at all, so the eight + # legs that pull requests skip cannot be required by name; this gate requires them instead. Every + # job must succeed; real ComfyUI must be skipped on a pull request and succeed on every other event. + all-tests: + name: all tests + needs: [test, comfyui, coverage, lint, js-test] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require every test job for this event + env: + EVENT_NAME: ${{ github.event_name }} + TEST_RESULT: ${{ needs.test.result }} + COMFYUI_RESULT: ${{ needs.comfyui.result }} + COVERAGE_RESULT: ${{ needs.coverage.result }} + LINT_RESULT: ${{ needs.lint.result }} + JS_RESULT: ${{ needs.js-test.result }} + run: | + queue_only=success + if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi + echo "event: $EVENT_NAME (real ComfyUI must be $queue_only)" + echo "test matrix: $TEST_RESULT" + echo "comfyui: $COMFYUI_RESULT" + echo "coverage: $COVERAGE_RESULT" + echo "lint: $LINT_RESULT" + echo "js-test: $JS_RESULT" + test "$TEST_RESULT" = success && test "$COVERAGE_RESULT" = success && test "$LINT_RESULT" = success && test "$JS_RESULT" = success && test "$COMFYUI_RESULT" = "$queue_only" diff --git a/.github/workflows/viewerjs-check.yml b/.github/workflows/viewerjs-check.yml index 8582a67d..783e1b92 100644 --- a/.github/workflows/viewerjs-check.yml +++ b/.github/workflows/viewerjs-check.yml @@ -165,6 +165,8 @@ jobs: browser-workflows: name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2) + # Merge queue, main and manual runs only (#420); pull requests run the Chromium/touch coverage shards. + if: github.event_name != 'pull_request' runs-on: ubuntu-latest # Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the # same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json; @@ -219,8 +221,9 @@ jobs: retention-days: 14 # Branch protection requires the exact context "viewerjs". This job keeps that - # name and fails unless every frontend job above succeeded (every shard, the - # coverage merge and every browser shard), including when one was cancelled or skipped. + # name and fails unless every coverage shard and the coverage merge succeeded and + # the browser shards succeeded, or, on a pull request only, were skipped (#420). + # Any failure, cancellation or other skip fails it. viewerjs: needs: [frontend-coverage, frontend-coverage-merge, browser-workflows] if: always() @@ -229,11 +232,15 @@ jobs: steps: - name: Require every frontend job to succeed env: + EVENT_NAME: ${{ github.event_name }} COVERAGE_RESULT: ${{ needs.frontend-coverage.result }} MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }} WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }} run: | + queue_only=success + if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi + echo "event: $EVENT_NAME (browser-workflows must be $queue_only)" echo "frontend-coverage: $COVERAGE_RESULT" echo "frontend-coverage-merge: $MERGE_RESULT" echo "browser-workflows: $WORKFLOWS_RESULT" - test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success + test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = "$queue_only" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca3295df..9106e4fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,9 +13,11 @@ repos: language: system types: [python] + # Fails at the severity CI enforces (code-quality.yml: -lll, high only), never stricter. + # pre-commit appends the staged files, so no shell wrapper can drop the first one. - id: bandit name: bandit - entry: bash -c 'bandit "$@" -ll -q 2>&1; true' + entry: bandit -lll -q language: system types: [python] exclude: ^(tests/|reference/) diff --git a/README.md b/README.md index 7a918ef9..6142e6fc 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Both nodes include the complete PromptManager feature set: - **๐Ÿ“Š Analytics**: Track prompt usage patterns and effectiveness over time - **๐Ÿ”ฌ Workflow Analysis**: Extract and analyze ComfyUI workflow data from PNG images - **๐Ÿ“‹ Metadata Viewer**: Standalone tool for analyzing ComfyUI-generated images -- **๐Ÿ› ๏ธ System Management**: Built-in diagnostics, backup/restore, and maintenance tools +- **๐Ÿ› ๏ธ System Management**: Built-in diagnostics, backup/restore, and maintenance tools in System - **๐Ÿท๏ธ AI AutoTag**: Automatically tag your image collection using WD14 or JoyCaption vision models - **๐Ÿ”— LoRA Manager Integration**: Import LoRA metadata and preview images from [ComfyUI-Lora-Manager](https://github.com/willmiao/ComfyUI-Lora-Manager) @@ -206,14 +206,11 @@ match Classic and Focused. - **Responsive Design**: Works on desktop, tablet, and mobile devices - **Real-time Search**: Instant search results as you type - **Bulk Operations**: Edit multiple prompts simultaneously with batch tools -- **Settings Panel**: Configure behavior and display options -- **System Management**: Complete toolset for database operations: +- **Settings and System tools live in System**: the dashboard's **Settings** opens System โ€บ Overview, **Maintenance** opens System โ€บ Maintenance, **Backup DB** and **Restore DB** open System โ€บ Backup & export, and **Diagnostics** and **View Logs** open System โ€บ Diagnostics & logs, each in a new tab. System reviews every destructive operation before it runs it as an Activity job. +- **Page options**: the list header's **Page options** keeps the prompt list's Infinite Scrolling default +- **Dashboard tools**: - **๐Ÿ” Metadata Viewer**: Launch standalone PNG analysis tool - - **๐Ÿ“ธ Scan Images**: Bulk scan output directories for ComfyUI images - - **๐Ÿ’พ Backup/Restore**: Database backup and recovery operations - - **๐Ÿ”ง Maintenance**: Database optimization and cleanup tools including orphaned prompt removal - - **๐Ÿ“‹ Logs**: Comprehensive logging system with download capability - - **๐Ÿ” Diagnostics**: System health checks and troubleshooting + - **๐Ÿ“ธ Scan Images**: Bulk scan output directories for ComfyUI images (with a quick backup first) - **Export Tools**: Download prompts and metadata in various formats - **Progress Tracking**: Real-time progress indicators for long operations - **Error Handling**: Graceful error display with recovery suggestions @@ -333,14 +330,11 @@ The comprehensive gallery system provides complete access to all your ComfyUI ou #### **โš™๏ธ Gallery Settings** -Configure your viewing experience in the settings panel: +The gallery's **Settings** opens System โ€บ Appearance in a new tab: image quality, details on cards, collapsed viewer details, video autoplay/mute/loop, metadata loading and file paths are set there and apply to this page too. -- **Display Settings**: Default view, items per page, grid columns, media info, infinite scrolling -- **Performance Settings**: Image quality, lazy loading -- **Video Settings**: Autoplay, mute by default, loop videos -- **Thumbnail Management**: Generate optimized thumbnails for faster loading (runs on the configured worker threads; a run that stops at the per-run cap tells you how many remain) -- **Find Duplicate Images**: Hashes every file on the worker threads with a progress bar; close the window and it keeps running, results wait in a pill -- **Rescan Output Folder**: Drops the server-side listing cache and rewalks the folders +The gallery's **Page options** keep the page's own settings: the thumbnail check at startup, lazy loading, items per page, grid columns, infinite scrolling, debug mode and API timeout. Reset puts back only these. + +Thumbnail generation, the server thumbnail and browser cache clears, refreshing the file listing and finding duplicate images are in System โ€บ Sources & scans and System โ€บ Maintenance (Page options links both); each runs as an Activity job you can follow after closing the page. #### **๐Ÿ–ผ๏ธ Image Viewing Experience** @@ -398,7 +392,7 @@ Import existing ComfyUI images into your database: - Any errors or issues encountered 5. **Access imported data** through the normal gallery interface -The scan reads PNG text chunks and A1111-style parameters, and, when `ffprobe` is installed, the prompt ComfyUI's video savers embed in MP4/MOV/WebM files. Files are read on the number of **worker threads** set in Settings โ†’ Performance (default: half your cores), which makes a large library several times faster. +The scan reads PNG text chunks and A1111-style parameters, and, when `ffprobe` is installed, the prompt ComfyUI's video savers embed in MP4/MOV/WebM files. Files are read on the number of **worker threads** set in System โ€บ Sources & scans (default: half your cores), which makes a large library several times faster. ### ๐Ÿท๏ธ AI AutoTag @@ -414,7 +408,7 @@ Automatically tag your entire image collection using AI vision models: 1. **AutoTag (Batch Mode)**: Tag your entire collection automatically - Click **"๐Ÿท๏ธ AutoTag"** in the admin dashboard - - Choose your model type (FP16 or GGUF) + - Choose your model type (FP16 or GGUF). A model that is not downloaded yet shows **Manage in System**: download (or unload) it in System โ€บ Auto-tagging, then reopen the dialog - Select how to handle already-tagged images: - **Skip images with existing tags**: Preserve your meticulous manual tagging work - **Re-tag all images**: Overwrite existing tags with fresh AI analysis @@ -445,7 +439,7 @@ Adjust the system prompt to match your tagging style: #### **Requirements** - Sufficient VRAM for the chosen model (FP16 requires more, GGUF is lighter) -- Models are downloaded automatically on first use +- Models are downloaded in System โ€บ Auto-tagging - Feature requests and improvements are welcome! ### ๐Ÿ“ Folder Filter @@ -470,18 +464,18 @@ If you use [ComfyUI-Lora-Manager](https://github.com/willmiao/ComfyUI-Lora-Manag #### **Enabling the Integration** -1. Open the admin dashboard and click **Settings** -2. Scroll to the **Integrations** section โ€” PromptManager will auto-detect if LoRA Manager is installed +1. Open System โ€บ Integrations (the admin dashboard's **Settings** opens System) +2. PromptManager auto-detects whether LoRA Manager is installed ![LoRA Enabled](images/pm-lora-enabled.png) 3. Toggle the **LoraManager** switch to enable 4. Optionally enable **Auto-inject trigger words** to automatically append LoRA trigger words when `` is detected in your prompts -5. Click **Save Settings** (requires a ComfyUI restart to take effect) +5. Click **Save preferences** (requires a ComfyUI restart to take effect) #### **Importing LoRA Data** -Click **Import LoRA Data** in the settings panel to start the import. A progress popup shows real-time status as each LoRA is processed: +Click **Review import** in System โ€บ Integrations; Start runs the import as an Activity job: - Reads LoRA Manager's own settings the way LoRA Manager does: from the folder in `LORA_MANAGER_SETTINGS_DIR`, from the plugin's `settings.json` in portable mode, or from your user config folder, and only the keys that say where its files are - Finds each LoRA's metadata in both storage modes: beside the model (the default) or in the central sidecar folder, and imports only LoRAs whose model file still exists in a LoRA folder (the review says how many metadata files it skipped) @@ -502,7 +496,7 @@ A CivitAI API key is **required if your library contains NSFW LoRAs** โ€” CivitA To add your key: 1. Get your API key from [civitai.com/user/account](https://civitai.com/user/account) -2. Paste it into the **CivitAI API Key** field in the Integrations settings +2. Paste it into the **CivitAI API key** field in System โ€บ Integrations 3. Save and re-import to download any previously skipped images The key is sent only over HTTPS to CivitAI's own hosts. CivitAI serves images through a redirect to its storage host; PromptManager follows such a redirect only over HTTPS to a host on CivitAI's own `civitai.com`, `civitai.red` or `civitai.green` domains, at most three hops, and never sends the key with it. When downloads fail, the import's status line says how many; when none could be downloaded, it starts with a warning. The server log has one warning per failed image with the reason (HTTP status, refused redirect, size cap, unreadable image, timeout or network error). @@ -517,23 +511,20 @@ The comprehensive web interface provides: - **Filter Options**: Filter by category, tags, rating, and date - **Bulk Operations**: Select multiple prompts for batch editing - **Export Tools**: Download your prompt collection -- **Settings Panel**: Configure auto-save and display options -- **Diagnostics**: System health checks and troubleshooting +- **Settings**: System (new tab) for settings, maintenance, backups and diagnostics; **Page options** for the page's own display settings ### ๐Ÿ”ง Database Maintenance -Keep your database optimized with built-in maintenance tools: +Keep your database optimized in System โ€บ Maintenance (the admin dashboard's **Maintenance** opens it in a new tab): -1. **Open the admin dashboard** and click **"๐Ÿ”ง Maintenance"** -2. **Select maintenance operations**: - - **Remove Duplicates**: Find and merge duplicate prompts while preserving images - - **Vacuum Database**: Optimize database file size and performance - - **Clean Orphaned Images**: Remove image records for missing files - - **๐Ÿšฎ Prune Orphaned Prompts**: Remove prompts that have no linked images - - **Check Consistency**: Verify database integrity and relationships - - **Database Statistics**: Generate detailed database reports -3. **Run selected operations** and monitor progress in real-time -4. **Review results** showing detailed information about each operation +- **Duplicate prompt text**: merge duplicate prompts while preserving images +- **Optimize database**: reclaim unused SQLite space +- **Missing-file links**: remove image records for missing files +- **Unused prompt components**: remove prompts nothing uses +- **Run health checks**: verify database integrity and relationships (the Health report) +- **Database size and prompt counts**: System โ€บ Diagnostics & logs (Database) and the Overview's Statistics + +Each operation opens a review that shows what it will change and the safety backup it takes, then runs as an Activity job. ### Database Location @@ -835,8 +826,8 @@ Before upgrading or changing this setting: 1. Identify the database your current installation actually uses and confirm its library contents. Check both the configured path and the prior fallback `/prompts.db`; keep track of which existing file you intend to use. -2. Preserve a consistent backup of the active database with the dashboard's - Backup/Restore backup action or SQLite's online backup API. Do not copy only +2. Preserve a consistent backup of the active database with + System โ€บ Backup & export's Back up or SQLite's online backup API. Do not copy only the main file of a live SQLite database: committed data may still be in its `-wal` file. For a filesystem copy, stop ComfyUI cleanly first and preserve any remaining `-wal` file together with the main database. diff --git a/TESTING.md b/TESTING.md index 2c845b9e..a0a11167 100644 --- a/TESTING.md +++ b/TESTING.md @@ -181,10 +181,12 @@ check, so run it before pushing a new or deleted frontend file instead of waiting for the `frontend coverage` jobs. The `viewerjs` check runs on every PR, including backend-only changes. It is an -aggregate job that passes only when every frontend job succeeds: the three -`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four -`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped -job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public +aggregate job. It requires the three `frontend coverage (shard N/3)` jobs and +`frontend coverage (merge)` to succeed on every event. The four +`page workflows (firefox|webkit, shard N/2)` jobs run only in the merge queue, +on `main` and on `workflow_dispatch` (see "Pull requests and the merge queue"). +`viewerjs` requires them to be skipped on a pull request and to succeed +everywhere else. Any other failed, cancelled or skipped job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3` (shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3 @@ -271,6 +273,13 @@ gh run view --json jobs # shows a job that alr gh run view --job --log-failed # that job's failed log, at once ``` +Python 3.13/3.14, real ComfyUI and Firefox/WebKit failures first show up in the +merge queue, on the merge-group commit rather than the +pushed one. Find that run with `gh run list --event merge_group --workflow test.yml` +(or `viewerjs-check.yml`) and match its head branch +(`gh-readonly-queue/main/pr--...`). Windows and macOS failures show up only +after the merge, in `gh run list --workflow portability.yml`. + Start fixing as soon as a failed log shows the cause, but push the fix only after the run for that commit has finished: pull-request runs use `cancel-in-progress`, so a push cancels the running jobs and can hide a second failure. @@ -313,6 +322,26 @@ from the tests' package unload (`cleanup_package`), and it removes its own ComfyUI-Manager restart through `os.execv`, a closed Windows console, or ComfyUI Desktop stopping its server. SQLite's WAL journal covers those. +Jobs running at exit (#376): before any `atexit` callback, Python joins every +executor thread (`concurrent.futures` registers that join with +`threading._register_atexit`). The package registers its own callback there +(`utils/exit_signal.install`, after `concurrent.futures`, so it runs first): +it sets the exit signal and kills the running encoder processes +(`shutdown_encoders`, the ffmpeg that never sees Ctrl+C on Windows). The call in +flight then stops at its next safe point: `Job.cancelled` is True, so every +check a job already makes between files stops it, `map_parallel` starts no +further item (`ExitRequested`), the scan walk stops between folders, LoRA +import staging stops before its next sidecar (also for the legacy scan) and a +tagging model download stops before its next chunk where huggingface_hub takes +`tqdm_class` (1.x removes its temp file; 0.x `hf_hub_download` does not take it +and finishes the file). Xet transfers ignore that checkpoint, so the hook also +calls hub's own `abort_xet_session` when huggingface_hub is loaded. The +trigger-word cache load (small JSON sidecars) has no checkpoint. No check sits +inside a library write. That hook is +CPython-internal; without it (or when it is refused) `install` returns False and +exit waits for the call in flight, as before. `tests/test_exit_cancel.py` runs a +child process with three such calls running and requires its exit within 15 s. + Stop contract: `PromptTracker.shutdown(timeout)`, `ImageGenerationHandler.shutdown(timeout)` (restartable), `ImageGenerationHandler.close(timeout)` (final: `schedule()` refuses and a @@ -362,6 +391,10 @@ A spec performance budget gets two tests: `test_filter_budget` (Images filters, spec D12: p95 < 150 ms at 50,000 files and 20,000 links). +A hot SQLite read that returns many rows aggregates them into one step +(`json_group_array`) instead of stepping once per row, so it does not wait for +the GIL once per row while ComfyUI generates (#354, `get_image_link_matches`). + The `perf budget` job in `.github/workflows/test.yml` is the reference measurement: `ubuntu-latest`, one Python, no coverage, only the wall-clock tests. It runs on every push to `main` and on `workflow_dispatch`, never on pull @@ -519,29 +552,95 @@ Before changing paths, SQLite, streams, restore or cleanup: open with `FirstWriterHold`) and give peers that must all succeed the ordinary busy timeout; cross-thread handoffs wait on events with generous ceilings, never a short wall clock (#242). -- Run native Windows CI early for these changes and include a Windows impact and - evidence note in the PR. All twelve OS/Python combinations and every applicable +- Run native Windows CI early for these changes. Pull requests and the merge + queue run only Linux, so start Windows and macOS on the branch with + `gh workflow run portability.yml --ref `. + Include a Windows impact and evidence note in the PR. All six Windows/macOS + combinations in that run, every Linux leg and every applicable exact-head check must pass before merge. Portable `ntpath`/`PureWindowsPath` controls do not establish native filesystem, locking or cleanup success. +## Pull requests and the merge queue + +Each PR runs the full matrix once, in the merge queue (#420). The event decides +what runs: + +| Job | `pull_request` | `merge_group`, push to `main`, `workflow_dispatch` | +| --- | --- | --- | +| `py3.12 on ubuntu-latest` | runs | runs | +| `py3.13 on ubuntu-latest`, `py3.14 on ubuntu-latest` | not created | run | +| `real ComfyUI compatibility` | skipped | runs | +| `coverage gate (85%)`, `black and flake8`, `frontend modules`, `bandit (high severity)` | run | run | +| `frontend coverage (shard N/3)`, `frontend coverage (merge)` (vendor pins and tooling tests on shard 3) | run | run | +| `page workflows (firefox\|webkit, shard N/2)` | skipped | run | +| `all tests`, `viewerjs` (gates) | run | run | +| `perf budget` | skipped | push and dispatch only (not in the queue) | +| `py3.12-3.14 on windows-latest/macos-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required | + +GitHub has one required set for pull requests and the queue. A job skipped by +its `if:` reports Success. A dropped matrix leg never reports, so it would stay +pending. That is why `all tests` (in `test.yml`) and `viewerjs` carry the +guarantee: +- On a pull request they require the subset to succeed and the queue-only jobs + to be skipped. +- On every other event they require every job to succeed. + +So a merge group cannot turn green unless all three Linux Python legs, real +ComfyUI and the four browser shards passed. + +Windows and macOS (Python 3.12-3.14) are out of the blocking path for now +(maintainer, 2026-10-28). They run in `portability.yml` on every push to `main`, +nightly and on demand, and no context there is required. A red portability run +is still a failure: fix it at its cause, not by rerunning, and have every one +green before the workspace cutover (#334). `tests/vendor/ci-workflow.test.mjs` pins the +event split and runs both gate scripts for each event. + ## Required merge checks -`.github/branch-protection.json` records the required checks: all twelve Python/OS -combinations, Python coverage, formatting, frontend modules, `viewerjs`, `real ComfyUI compatibility`, -and `bandit (high severity)`. -The branch must be current with main, and administrators follow the same checks. +`.github/branch-protection.json` records the required checks: +`py3.12 on ubuntu-latest`, `coverage gate (85%)`, `black and flake8`, +`frontend modules`, `all tests`, `viewerjs`, `real ComfyUI compatibility` and +`bandit (high severity)`. Each one reports on a pull request. +The same set lives in two places, and both must change together: the `main` +ruleset (25960526, which also holds the merge queue) and the classic branch +protection on `main`, which the manifest mirrors. GitHub requires the union of +the two, so a context left in either one blocks every PR that does not report it. It does not require an additional human approval. A repository administrator applies the configuration after the new workflow is passing, then verifies it: +```sh +gh api repos/ComfyAssets/ComfyUI_PromptManager/rulesets/25960526 \ + | jq '.rules[] | select(.type == "required_status_checks") | [.parameters.required_status_checks[].context]' +``` + +A PUT replaces the whole `rules` array, so edit the current ruleset in place. +Keep the `deletion`, `non_fast_forward` and `merge_queue` rules, and give every +context `integration_id` 15368 (GitHub Actions), so no other integration can +post a required check. Save the current ruleset first; it is the rollback: + +```sh +gh api repos/ComfyAssets/ComfyUI_PromptManager/rulesets/25960526 > ruleset-before.json +jq '{name, target, enforcement, conditions, bypass_actors, rules: [.rules[] + | if .type == "required_status_checks" + then .parameters.required_status_checks = [$ARGS.positional[] | {context: ., integration_id: 15368}] + else . end]}' ruleset-before.json --args "py3.12 on ubuntu-latest" "..." > ruleset-new.json +gh api --method PUT repos/ComfyAssets/ComfyUI_PromptManager/rulesets/25960526 --input ruleset-new.json +``` + +Afterwards, read it back. Check that the `merge_queue` rule is still there and +that every context keeps its `integration_id`. Then apply the manifest to the +classic protection and read both back: + ```sh gh api --method PUT repos/ComfyAssets/ComfyUI_PromptManager/branches/main/protection \ --input .github/branch-protection.json gh api repos/ComfyAssets/ComfyUI_PromptManager/branches/main/protection/required_status_checks +gh api repos/ComfyAssets/ComfyUI_PromptManager/rules/branches/main ``` - -Update the manifest whenever required job names change. Keep the frontend check + Update the manifest whenever +required job names change. Keep the frontend check free of path filters so a required check is never silently skipped. ## Unified workspace @@ -794,6 +893,25 @@ unattributed stage, when it is a plain first stage: not a detailer, no prompt ov set, and no listed sampler in its own latent ancestry (else it stays unattributed) (`tests/test_resolver_saver_anchor.py`, shapes in `tests/saver_anchor_graphs.py` from the maintainer's `gpu_unlock_bench`, dataset and ANIMA runs; browser parity through the `anchor-*` cases, which carry `__file_name`). +Recorded run history (#386): both inspect routes (Images and a history occurrence) +ask `database.run_attribution.recorded_output` inside the file's stat window. It +answers only for a file that exactly one recorded run reported and that was written +during that run (the #342 rules): the one output node that run reported it from (none +when several did) and, per sampler anchor, the effective texts PromptManager recorded +at execution (a role only inferred from the graph is none). Only when the file alone +leaves the prompt open (`ambiguous` or `unresolved`) the inspector resolves again +anchored on that node (`recorded_saver_id`, walked like a file-name match; a file +name naming a different saver contradicts history and anchors nothing), and keeps +that result only when history's saver chose its samplers (`recorded_anchor`) and it +names the positive (resolved, or a guider with no negative); its `graph-inferred` +sides read `recorded-run` ("Recorded run" badge, ranked below every "Workflow +inferred" source). Still open at the one sampler that saver leads to, with that +sampler's positive text recorded (a language model's output feeding PromptManager), +the recorded texts show as `execution`. Outputs and texts are read in one +transaction. A result the file resolves alone never changes; a lookup failure is +logged and judges the file alone; local Inspect and ingestion are unchanged +(`tests/test_recorded_saver_anchor.py`, `tests/test_run_attribution.py`, +`tests/test_api_history_outputs.py`). `tools/inspection_corpus.py --root DIR --out JSONL [--workers N] [--baseline JSONL] [--summary TXT] [--code TREE]` measures a media folder read-only against the #240 @@ -1227,7 +1345,7 @@ How the earlier workspace Images filters moved (old โ†’ new). The legacy `/galle | Folder | "Refine this view" โ†’ Folder (D14); shown in the scope chip | | Media (All media / Images / Videos) | "Refine this view" โ†’ Media (D14) | | Sort: Newest first / Oldest first / Filename Aโ€“Z / Filename Zโ€“A | toolbar sort: Recently added / Oldest first / Name Aโ€“Z / Name Zโ€“A, plus Highest rated (D13, D14) | -| (none) | toolbar category and "Refine this view" โ†’ Tags, through the linked positive (D13, D14) | +| (none) | toolbar category (the effective category, D141) and "Refine this view" โ†’ Tags, through the linked positive (D13, D14) | | Refresh | Import โ†’ "Scan an output folder" โ†’ Rescan (D15) | | Grid / List / Masonry, Comfortable / Large / Compact | the same choices in the concept filter row | | Card image โ†’ side panel; preview icon โ†’ viewer | image โ†’ viewer; Info โ†’ side panel (D19) | @@ -1312,7 +1430,8 @@ Recorded deviations on this page (spec ยง8): `overscroll-behavior: contain`. - **D35**: counts pluralize ("1 positive prompt", "1 image", "1 more image loaded"). - **D36**: the category list is the Library's, sorted alphabetically (case-insensitive) - by the page, whatever order the server sends. + by the page, whatever order the server sends (amended by D141: Images adds its + image categories). - **D37**: production loads 40 images per page (the concept 12), so the continuation and toast use 40; after Escape production returns focus to the opening image. - **D44** (#249 PR E): Filters reads "Filters ยท N" while N of Source, Folder, Media and @@ -1398,15 +1517,46 @@ Recorded deviations on this page (spec ยง8): - **D39** (ruled 2026-10-21: fallback): a recipe-linked image shows its recipe's own category when set, otherwise the linked positive prompt's, and its recipe's own tags when it has any, otherwise the positive's; each field falls back on its own and the - two tag sets are never merged. Set category (#259) writes the positive's category and - the Images category and Tags filters match the positive's, so the panel now agrees - with them whenever the recipe adds nothing. The inspection response names each + two tag sets are never merged. The Images Tags filter matches the positive's tags. + The category filter matches the same inherited category as the panel (D142, #332), and + an image's own category overrides both (D141, D146). The inspection response names each field's source (`library.category_scope`, `library.tags_scope`: `pair`, `positive` or null); the category chip's visually hidden prefix (D21) says "Recipe category: " or "Positive prompt category: " from it, and tag chips are Tags filter buttons only when they are the positive's (#260 D2). History's media inspector uses the same rule. The concept shows the prompt's category and tags for every image and the concept fixture's recipes have neither, so the inspector comparison needs no mapping for it. +- **D141** (#332, amends D13 and D36): the Images category filter matches each image's + effective category: its own category (Images Set category, PR 3) when it has one, + else its inherited one (D142), else none. An image whose own category differs never + matches its prompt's category, and an unlinked or ambiguous image with an own + category is listed under it (the members-only gate now applies to the Tags filter + only). "Select all N matching" and saved views use the same value. The caption's + positive-prompt count still counts the linked positives of the images shown. With a + category set the listing reads the own categories once per source (a primary-key + range); a failed read is the existing 500 `listing-failed`, never an unfiltered page. + The link read then keeps only links whose raw recipe or positive category is the + value (a superset of those that inherit it; Python decides), and reads the links of + files whose own category is the value by path, so their rating and positive count. + `test_filter_budget` seeds 2,000 own categories on its 50,000-file fixture. + The option list adds the image categories of listed files + (`GET /prompt_manager/images/categories`, PR 3). +- **D142** (#332 MC4, amends D39's filter note): the inherited category is one rule, + `inherited_category` in `utils/inspection_response.py`: a linked recipe's own + category when it is not blank, else the linked positive's. Before #332 the filter + matched the positive's even when the chip showed the recipe's. The link reads select + the raw recipe and positive categories and never decide inheritance in SQL + (`tests/test_api_workspace_image_categories.py` checks the filter and the chip agree + on blank values stored as spaces, tabs, U+00A0 and BLOB). The concept has no recipe + categories, so this is behavior only. +- **D145** (#332, DC4): History's media inspector shows an image's own category with + the chip rule of the side panel and viewer. Only the inspection request reads it, once, + inside the History snapshot: the file is the symlink-resolved output folder plus the + receipt's subfolder and filename, keyed with `image_category_key` over the configured + output sources, as Images keys it. A temp output or a file no source holds has none; + a failed read hides the chip (DC5). The outputs list and media requests read nothing + new. The chip's hidden prefix reads "Image category: " for an own category; the + visible origin text and its comparison are D146 (PR 3). - **D42**: the concept never shows a toast and a page alert together. Production keeps the concept's fixed toast (centred, 22px from the bottom, 12px at 700px and below) and, when an Import โ†’ Rescan toast leaves the Images discovery alert @@ -2672,6 +2822,14 @@ production hub `#/overview` is unchanged (D28); its System card opens the pressed section button and disposes the previous section's scope, so its reads, listeners and dialogs never outlive it. +System is the home for settings and maintenance (#330 System S13). The legacy pages keep +their prompt list, tags, Scan Images, auto-tag batch and review and gallery browsing; their +Settings, Diagnostics, View Logs, Maintenance, Backup DB, Restore DB, the gallery's +thumbnail, cache, rescan and duplicate tools and the auto-tag model downloads open the +System section that owns them in a new tab (`legacy-system-links.spec.mjs`). Each page +keeps only its own settings in Page options. What left the legacy pages is listed in +"Legacy System UI removed by S13". + Drafts (`system-state.mjs`) live for the page session (Q10). Save writes the dirty browser fields, then the host fields (`POST /prompt_manager/settings` with `expected_revision`, #220), then the runtime fields. A failed domain keeps its drafts and @@ -4216,7 +4374,30 @@ modals use the same routes and keep working. other: each file's identity is checked just before its unlink. - **Clear all logs** (`POST /logs/truncate`, also the admin page's Clear) empties the active log through its handler, deletes the rotated copies (Rule R) and clears the - buffer; other files and links are kept. Response keys are unchanged. + buffer; other files and links are kept. Response keys are unchanged. It deletes or + empties no more than its confirmation showed (#389): + - `/logs/files` rows carry `identity`, the string `"::"` (a + string: `mtime_ns` and many inodes are beyond a browser's exact integers). The + workspace sends `{"expected": [{name, identity}]}` with the active and rotated rows + its dialog was built from. Under the files lock and every handler lock each listed + rotated copy must still match (`_matches_review`) and the listed active log must be + the same inode, or nothing changes: 409 `logs-changed`. Only listed copies are + deleted; one that appeared after the confirmation is kept. A malformed list is 400. + - Without a body (the legacy admin page, whose confirm lists nothing; also the + workspace when the file list could not be read and the dialog said "deletes its + rotated copies") every rotated copy present at the locked rescan is deleted. + - Either way, before anything changes (best effort, not a cross-process lock: a + write before our own last write, or any while file logging is off here, is not + seen): a second writer of the active log (our + handler's flushed stream is behind the file's size, or its file is now a rotated + copy: another ComfyUI sharing the folder) is 409 `logs-foreign-writer`; an active + log with more than one name (`st_nlink > 1` by `lstat` or the handler's `fstat`, + NTFS included) is 409 `logs-hard-link`, never truncated (replacing it instead would + rename a file our handler holds open, which Windows refuses). Each copy is + `lstat`-checked again just before its unlink; one that changed stops the clear there + and keeps it, the copies after it and the active log, which is emptied last. + - The browser fixture grows the active log through the handler's own stream, since a + second handle appending to it is exactly what Clear refuses. - **Runtime settings** (`GET/POST /logs/config`): the level (DEBUGโ€ฆCRITICAL) and console logging apply until the server restarts; the retention values are shown read-only in the workspace and the API keeps accepting them. @@ -4263,6 +4444,9 @@ auto-tag / review", which the Images selection bar owns, not a System section. A is not `covered` names none (`โ€”`). Test is the exact title of a test that runs, then its file. Anchors are line numbers on `main` at this PR. +S13 (#330) removed the legacy copy of the `covered` rows the legacy pages linked to System; +"Legacy System UI removed by S13" below lists every removed element against its row. + `tests/vendor/legacy-coverage.test.mjs` fails on a row with any of these problems: - no owner, or a status outside the three; - a malformed Native control cell; @@ -4336,6 +4520,62 @@ list stay unscoped: any row's control proves them. | API timeout `apiTimeout` | `gallery-settings.js:37` | Legacy gallery (S12-10; read by no request) | kept in legacy | โ€” | defaults are a complete, sane settings object (gallery-settings.test.js) | | config.json-only keys | `py/config.py:212โ€“234,722โ€“739`; `py/api/diagnostics_report.py:185` | S12 report (read-only) | covered | `diagnostics` ยท button ยท "Download diagnostic report" | test_report_lists_config_only_keys (test_api_diagnostics_report.py) | +### Legacy System UI removed by S13 + +System S13 (#330) points the legacy pages' System buttons at native System (a link that +opens `workspace#/system/` in a new tab) and removes the legacy copy of a capability +only when its row above is `covered`. One row per removed capability: Capability is the +exact Capability cell of its coverage row, Page is `admin` or `gallery`, Removed ids and +Removed methods list what went (`โ€”` for none), and Native test copies that row's Test +cell. An id that stays on a System link is not listed, only its handler. The gallery's +Default View Mode left Page options too: System โ€บ Appearance's Default layout is the one +default-layout control. Nothing writes the legacy `defaultViewMode` any more; a stored +value still picks the view the legacy gallery opens in (grid otherwise), and Page options +Save and Reset keep it. + +`tests/vendor/legacy-coverage.test.mjs` fails when a row cites a capability that is not +`covered` or copies another test, when an id or method is listed twice for one page, or +when a listed id is still on its page (`id="โ€ฆ"` in the markup or the script, a +`getElementById`/`querySelector('#โ€ฆ')` lookup, an entry of a modal-id list) or a listed +method is still defined or called (`.name(`). A deletion listed nowhere is caught by the +kept-flow tests (`kept admin flows still work` in `legacy-system-links.spec.mjs` and the +legacy page workflows), not by this check. + +| Capability | Page | Removed ids | Removed methods | Native test | +| --- | --- | --- | --- | --- | +| Diagnostics run | admin | `#diagnosticsModal`, `#runDiagnosticsBtn`, `#diagnosticsContent` | `showDiagnosticsModal()`, `closeDiagnostics()`, `runDiagnostics()`, `renderDiagnostics()` | Run checks opens the shared Health report (workspace-system-diagnostics.spec.mjs) | +| Synthetic image-link test | admin | `#testImageLinkBtn` | `testImageLink()` | Create test link confirms first and names the prompt (workspace-system-diagnostics.spec.mjs) | +| View logs with level and limit | admin | `#logsModal`, `#logLevel`, `#logLimit`, `#refreshLogsBtn`, `#logsContainer` | `showLogsModal()`, `closeLogs()`, `refreshLogs()`, `displayLogs()` | level filter is minimum-level over all six choices (workspace-system-diagnostics.spec.mjs) | +| Log statistics | admin | `#logStatsTotal`, `#logStatsErrors`, `#logStatsWarnings`, `#logStatsLevel`, `#logStatsSize` | `loadLogStats()`, `formatBytes()` | log statistics show entries, errors, warnings, files and size (workspace-system-diagnostics.spec.mjs) | +| Log files list and download | admin | `#downloadLogsBtn`, `#logFilesList` | `loadLogFiles()`, `downloadLogs()`, `downloadLogFile()` | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) | +| Clear/truncate logs | admin | `#clearLogsBtn` | `clearLogs()` | Clear all logs states both counts, confirms first and reads everything again (workspace-system-diagnostics.spec.mjs) | +| Set log level (incl. CRITICAL), console logging | admin | `#updateLogConfigBtn`, `#setLogLevel`, `#consoleLogging` | `loadLogConfig()`, `updateLogConfig()` | runtime level (including CRITICAL) and console logging save and say they apply until the server restarts (workspace-system-diagnostics.spec.mjs) | +| Image scan directories add/remove | admin | `#scanPathsList`, `#addScanPathBtn` | `renderScanPaths()`, `addScanPath()`, `removeScanPath()`, `_collectScanPaths()` | Add source validates on the server, then marks the host draft dirty; Save persists (workspace-system-sources.spec.mjs) | +| Currently monitoring list + Refresh | admin | `#refreshMonitoringStatus`, `#monitoringStatus` | `updateMonitoringStatus()` | watching only with a live observer; restart pending otherwise (workspace-system-sources.spec.mjs) | +| Search results auto-hide `result_timeout` | admin | `#resultTimeout` | โ€” | host fields round-trip through /settings with revision (workspace-system.spec.mjs) | +| Web UI display mode | admin | `#webuiDisplayMode` | โ€” | host fields round-trip through /settings with revision (workspace-system.spec.mjs) | +| Worker threads | admin | `#workerThreads`, `#workerThreadsValue` | `renderWorkerThreads()` | worker range max is cpu_count (workspace-system-sources.spec.mjs) | +| LoraManager badge, enable, path, trigger words, CivitAI key, clear key | admin | `#loraDetectionBadge`, `#loraEnabled`, `#loraSettings`, `#loraManagerPath`, `#loraTriggerWords`, `#civitaiApiKey`, `#clearCivitaiKey` | `detectLoraManager()`, `_bindLoraEvents()`, `saveLoraSettings()` | LoRA rows show detected / enabled / cache-loaded separately (workspace-system-integrations.spec.mjs) | +| LoRA data import with progress | admin | `#loraImportModal`, `#loraImportBtn`, `#loraImportStatus`, `#loraImportPercent`, `#loraImportBar`, `#loraImportProcessed`, `#loraImportImported` | `runLoraImport()` | Review import shows replacement, blocked items and network effects with the backup box; Start opens the job (D107) (workspace-system-integrations.spec.mjs) | +| Maintenance cleanup_duplicates, vacuum, cleanup_orphaned_images, prune_orphaned_prompts | admin | `#maintenanceModal`, `#cleanup_duplicates`, `#vacuum`, `#cleanup_orphaned_images`, `#prune_orphaned_prompts`, `#runMaintenanceBtn`, `#maintenanceResults` | `showMaintenanceModal()`, `closeMaintenance()`, `runMaintenance()`, `renderMaintenanceResults()` | Apply reviewed operation closes, toasts, opens the job in Activity (workspace-system-maintenance.spec.mjs) | +| Maintenance check_hash_duplicates, check_consistency | admin | `#check_hash_duplicates`, `#check_consistency` | โ€” | Run health checks opens Health report and updates metric 4 (workspace-system.spec.mjs) | +| Maintenance statistics (database info) | admin | `#statistics` | โ€” | meta-grid shows real database and source status, the pinned Viewer.js version and the host renderers (workspace-system-diagnostics.spec.mjs) | +| Maintenance select all / clear all / run | admin | `#selectAllMaintenanceBtn`, `#clearAllMaintenanceBtn` | `selectAllMaintenance()`, `clearAllMaintenance()` | plan dialog shows the server's scope, protected, backup policy and media lines with the backup box (D100); Cancel writes nothing (workspace-system-maintenance.spec.mjs) | +| Database backup | admin | โ€” | `backupDatabase()` | Back up downloads through the api client and revokes the object URL (D103) (workspace-system-backups.spec.mjs) | +| Restore from file | admin | `#restoreModal`, `#restoreFileInput`, `#confirmRestore`, `#cancelRestore` | `showRestoreModal()`, `confirmRestore()` | Review restore stages the chosen file and shows the real summary with the backup box (D105) (workspace-system-backups.spec.mjs) | +| Gallery thumbnail generation | gallery | `#generateThumbnailsBtn`, `#thumbnailStatus`, `#thumbnailProgress`, `#progressText`, `#progressPercent`, `#cancelThumbnailsBtn`, `#progressBar`, `#progressDetails`, `#progressETA` | `generateThumbnails()`, `generateThumbnailsWithProgress()` | Start closes the dialog, toasts, and opens the job in Activity (workspace-system-sources.spec.mjs) | +| Default view mode, image quality | gallery | `#defaultViewModeSelect`, `#imageQualitySelect` | โ€” | defaults apply to a new visit but not to the current view or a saved view (workspace-system.spec.mjs) | +| Image info on cards; sidebar collapsed | gallery | `#showImageInfoToggle`, `#sidebarCollapsedByDefaultToggle` | โ€” | added rows reach their consumers (workspace-system.spec.mjs) | +| Video autoplay, mute, loop | gallery | `#videoAutoplayToggle`, `#videoMuteToggle`, `#videoLoopToggle` | โ€” | video defaults reach the viewer (workspace-system.spec.mjs) | +| Auto-load metadata, cache metadata, show file paths | gallery | `#autoLoadMetadataToggle`, `#cacheMetadataToggle`, `#showFilePathsToggle` | โ€” | metadata off shows an explicit load and loads once on request (workspace-system.spec.mjs) | +| Browser cache clear | gallery | `#clearCacheBtn` | `clearCache()` | Metadata cache reviews and clears only owned browser entries when there are some (D102) (workspace-system-maintenance.spec.mjs) | +| Clear server thumbnails | gallery | โ€” | โ€” | Thumbnail cache opens the S5 dialog (workspace-system-maintenance.spec.mjs) | +| Refresh file listing | gallery | `#rescanFolderBtn` | `rescanFolder()` | Refresh files calls /gallery/rescan and toasts its counts (workspace-system-sources.spec.mjs) | +| Duplicate image scan, review, delete | gallery | `#scanDuplicatesBtn`, `#dupScanPill`, `#dupScanPillText`, `#duplicatesModal`, `#closeDuplicatesBtn`, `#duplicatesScanStatus`, `#dupProgressText`, `#dupProgressBar`, `#duplicatesContent`, `#duplicatesList`, `#duplicatesFooter`, `#duplicatesCount`, `#selectAllBtn`, `#removeDuplicatesBtn` | `scanDuplicates()`, `attachDuplicateStream()`, `resumeDuplicateScan()`, `applyDuplicateEvent()`, `renderDuplicateProgress()`, `setDuplicateButton()`, `showDuplicatePill()`, `hideDuplicatePill()`, `openDuplicatesModal()`, `showDuplicatesModal()`, `hideDuplicatesModal()`, `displayDuplicates()`, `selectAllDuplicates()`, `removeDuplicates()` | Find duplicates starts a job; its view links Review groups; Keep radios build a duplicate_files plan (D101) (workspace-system-maintenance.spec.mjs) | +| Auto-tag model download | admin | `#autoTagDownloadModal`, `#downloadModelName`, `#downloadStatus`, `#downloadProgressPercent`, `#downloadProgressBar`, `#cancelDownloadBtn` | `downloadModel()`, `cancelDownload()` | Download starts the job and Activity shows it without Cancel (workspace-system-tagging.spec.mjs) | +| Auto-tag model download | gallery | `#autoTagDownloadModal`, `#downloadModelName`, `#downloadStatus`, `#downloadProgressPercent`, `#downloadProgressBar`, `#cancelDownloadBtn` | `downloadModel()`, `cancelDownload()` | Download starts the job and Activity shows it without Cancel (workspace-system-tagging.spec.mjs) | +| Unload model | gallery | โ€” | `unloadModel()` | Unload is enabled only while a model is loaded, and a held gate's refusal names the operation (workspace-system-tagging.spec.mjs) | + ### Browser cache ownership (#222) The legacy gallery's Clear cache used to delete every localStorage key starting `pm_` or @@ -4353,17 +4593,15 @@ page writes a key that is neither a preference nor inventoried, writes one it ca CacheStorage or IndexedDB; `the storage scan reports every way of writing browser storage` pins those patterns. It skips `web/lib` (third-party builds) and the pinned Toolkit build (`web/js/shared/toolkit`, never hand-edited, verified by `npm run check`; -its storage belongs to the Toolkit repository and a new pin is reviewed on its own). The -gallery's server thumbnail clear stays its own request and runs first: refused or -unreachable, it changes nothing and says so (the old code toasted success when the -request failed); refused browser storage is reported, never hidden -(`tests/js/legacy-refusals.test.js`; the browser case `the gallery cache clear keeps -preferences and other tools caches` in `workflows.spec.mjs`). +its storage belongs to the Toolkit repository and a new pin is reviewed on its own). Since +System S13 (#330) the legacy gallery no longer offers Clear cache: System โ€บ Maintenance's +Metadata cache and Thumbnail cache reviews own both parts (removal table above), and +`cache-ownership.js` serves the native review. -The legacy duplicates dialog's View Full now names its source (`?root=`, with the -path URL-encoded) when the gallery's output folder is a listed source; before, the serve -route opened the first source holding a file at that path -(`test_view_full_names_the_gallery_folder_source`). +The duplicates dialog's View Full named its source (`?root=`, with the path +URL-encoded) when the gallery's output folder is a listed source +(`test_view_full_names_the_gallery_folder_source`); S13 removed the legacy dialog, and the +route keeps that behaviour for System โ€บ Maintenance's duplicate review. ### Library location contract (#249 System S6b) @@ -4655,8 +4893,18 @@ Filters (`mode=workspace`; anything invalid is 400 "Invalid gallery filters"): rating 5 to 1, then linked but unrated, then unlinked and ambiguous; ties break newest first. A signed cursor covers every filter and the sort, so reusing it with a change is 400. -- One link read (`get_image_link_matches`, streamed and bounded to the listed - paths) serves the filters, the rating sort and the count. +- One link read (`get_image_link_matches`) serves the filters, the rating sort + and the count. SQLite returns its rows as one JSON value in one step (#354): + Python's `sqlite3` releases and retakes the GIL around every step, and while + ComfyUI generates each retake can wait on the busy thread, so a read that + stepped once per linked row took seconds. The result keeps only the listed + paths; peak memory grows with the database's matching linked rows (about + 530 bytes each, freed on return). When the JSON step fails (a SQLite without + JSON functions, a BLOB value, a JSON text over SQLite's length limit) the read + streams the rows as before and logs one warning per process; a missing JSON + function stops the attempt for the rest of the process. An interrupted read + (Move library) raises as before. `tests/test_database_link_matches.py` proves both reads + equal on every filter and on odd values. `positive_count` is the number of distinct linked positives over the full matching set, not the page: @@ -4707,6 +4955,70 @@ concept fixture stars the concept's favorites (each other prompt's first image). restore. Old backups without the table restore, and backup plus restore keeps WAL-resident favorites and closes every handle (`tests/test_database_backup.py`). +Image categories (#332, store and routes): an image file may carry its own +category; without one it shows the category its link supplies. + +- The table is `image_categories(root_key, relative_path, category, updated_at)` + with primary key `(root_key, relative_path)` and the index + `idx_image_categories_category`. `migrate_image_categories` creates both empty + inside a savepoint, with no backfill. It reads only `sqlite_master` and its own + table's PRAGMAs, and writes only its two objects. `tests/test_image_categories.py` + checks this with an SQLite authorizer on a 23,000-link library, and checks that + `prompts` and `prompt_pairs` `(id, category, updated_at)` are byte-for-byte equal + afterwards. +- The **file key** is `image_category_key(sources, absolute path)` in + `py/api/images.py`: the deepest output source that contains the path (compared + lexically, never resolving the path, so a symlinked subfolder keeps its listed + spelling) and the path below it, keyed with the favorites helpers. A source + matches by its real path or its configured spelling (a detected output folder + can be a symlink). Without nested sources it equals a favorite's key. Every write and read uses it. A path + that no source contains has no key, so it has no own category. +- `image_categories_schema_error` checks the structure at startup (a value saved + under older rules never stops the library opening; a table without its index, + which this code never leaves, is repaired by the install). Restore also checks the + values: text columns, keys within the favorites bound, `category` 1 to 100 + characters with no character below U+0020 and no leading or trailing space, and + `updated_at` of at most 64 bytes. Backups made before #332 restore without the + table; a migrated restore stage must have it (`require_library_state`). +- `PUT /prompt_manager/images/category` `{items, category}` answers + `{success, category, updated, unchanged, not_found}`. + `DELETE /prompt_manager/images/category` `{items}` answers `{success, cleared}`. + `GET /prompt_manager/images/categories` answers `{success, categories}`: the + sorted distinct own values whose file is in some source's listing. +- `items` are 1 to 500 Images targets `{root_index, root_scope, path}`, checked by + `py/api/media_target.py` (shared with Collections, #331). Refusals: + - 400: a malformed body, a field other than `items` and `category`, a + `root_index` outside the output sources (a LoRA folder), or a category that is + blank after stripping, over 100 characters, or holds a control character; + - 403: traversal, a drive, a backslash, a non-media name ("Only media files can + have a category") or a key over the stored bound; + - 409 `source-changed`: a stale `root_scope`; + - 413: a body over 1 MiB. + Any invalid item fails the whole request before a write. Targets that fold to one + key are written and counted once. Instead of `items` a request may send `filter`, + the Images listing's own query ("Select all N matching", #392, keys as in + `parse_listing_filter`), resolved now: over 500 matches is a 400 + `too-many-images` ("Select at most 500 images."), a renumbered source a 409 + `source-changed`. +- PUT writes only files the source's listing holds and counts the rest in + `not_found`; when none is listed it answers 404 `files-not-found` and writes + nothing. A file that already has the value counts as `unchanged` and keeps its + `updated_at`. DELETE accepts any well-formed target, so a missing file's + category can still be removed. An unexpected failure is a logged 500 ("The + category could not be changed." / "Image categories could not be read.") without + the exception text. The routes never read or write `prompts`, `prompt_pairs` or + `generated_images`; `POST /prompt_manager/bulk/category` is unchanged. +- Duplicate cleanup (System โ€บ Maintenance): the kept copy's own category wins. A + kept copy without one receives the removed copy's (`COPY_SQL`, part of the + cleanup's query-plan check). The removed copy's row is never deleted, so a + give-back (the kept copy changed) and a failed delete lose nothing. The job log + says "Kept N image categories on the kept copies". + `tests/test_duplicate_image_categories.py` covers nested sources in both orders. +- A moved, renamed or deleted file's row stays and is hidden; a file that later + appears at the same path shows it. The real-host check is the last test in + `tests/comfyui/image-attribution.spec.mjs`: PUT, list, DELETE on a generated + output. + Image annotations schema (#249, `database/annotation_schema.py`): the group `media_annotation_metadata` (one row: schema version, `image_index_revision`), `media_annotation_files` (one private locator per physical file, @@ -4829,7 +5141,23 @@ files, oldest forgotten first, keyed by 16-byte digests: 8.6 MiB measured at the cap). A file whose inspection raises is logged, counted in `review_errors` and reported `unresolved`; the scan goes on. An older version's pending, undecided item of bytes now untraced leaves the queue -(`settle_stale_review_items`), so an item never stays stuck at 409. The 16-candidate limited +(`settle_stale_review_items`), so an item never stays stuck at 409. +Saver anchor (#390, `tests/test_review_saver_anchor.py`): the scan and Apply +resolve with the file's name as the workspace inspector does, so the one saver +whose `filename_prefix` names the file anchors its branch. A file is never both +unlinked and unqueued (maintainer ruling): when only that anchor resolves it +(the named inspection run resolves, the unnamed one does not) and neither the +default profile nor the parser floor links it, the scan queues it with the +anchor's pair as its one candidate (evidence `anchor: "saver"`; the page +preselects it), never linking it. Such an item counts as a choice for the +untraced settle. Files already linked or resolved by the default path are +unchanged. No match, several matches or an unfinished walk anchor nothing, and +the file is queued as before. `REVIEW_RULE_VERSION` is folded into inspection +item versions, so older inspection items are recorded again once. Apply accepts +the anchored pair and refuses a choice the anchor contradicts. Measured on the +maintainer's 40,890 outputs: the queue is unchanged (249 files); the 242 +`gpu_unlock_bench` files the inspector resolves are linked by the parser +floor. The 16-candidate limited rule and the 8 MiB bound apply unchanged. The queue heals itself: after each batch, the scan removes the pending items of every listed file (read or skipped as known) that a saved row now links, and the monitor does the same for each ambiguous output it links @@ -5220,7 +5548,7 @@ which used to save `null` under a link. Primitive or through `PreviewAny`, for PromptManager, PromptManagerText, Focused, Split and the Pair (positive, negative, both, neither linked, and a Primitive negative). A wildcard Primitive on PromptManager (queued as written) and Focused (one choice) saves - and counts what was queued. The linked text is saved, counted once, recorded as + and counts what was queued. The linked text is saved, counted once per job, recorded as the run's raw positive/negative, and after a scan the run's image links to it. Text only known at run time (`PreviewAny`) cannot be read from the image; the case only requires that it is never linked to another prompt (its attribution from the recorded @@ -5234,6 +5562,39 @@ PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:comfyui PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:executions -- execution-text-link ``` +## Cached runtime-linked runs (#347) + +Text only known at run time (a `PreviewAny` or LLM link, a list node) is counted by the +node when it executes; the queue hook cannot read it. When ComfyUI serves such a node from +its output cache, `utils/cached_runs.py` counts the job instead: every execution records +a receipt (digests of its STRING outputs and hashes of the text it stores, per run key and +queue item), and on `execution_cached` the receipt whose digests equal the cache entry's +text is replayed once. The role (positive, negative, which Pair outputs are used) and what +the hook already saw are recomputed from the job's own graph, and the node and replay +decide through the same `own_counts`. + +- Fail closed: no receipt in this process (after a restart, an external cache provider), + changed cached text, more than 64 list values, a node the job's outputs do not need, or + a host without the queue/executor interfaces: nothing is counted, as before #347. +- A cached node counts at job start like the hook: a job failing later still counts it. + A node that executes after its replay in the same job (RAM-pressure freed the entry) + skips its own count but still consumes the pending first-use catch-up. +- No backfill: earlier cached jobs keep their counts. The history page's "tracked runs" + can stay below the Library count for these prompts (#403). +- Tests: `tests/test_cached_runs.py` (whole jobs with the real nodes, a real database, + fake host caches); `tests/comfyui/execution-text-link.spec.mjs` "#347" cases on the + managed host: PreviewAny into each node kind, 3 jobs changing only the sampler seed, + count 3 in both renderers; in one renderer the Classic, LRU, RAM-pressure and no-cache + modes, A, B, A from the owned `PMCycledText` under LRU, typed text, a downstream + failure, a "run selected output" job that does not need the node, and a rewire to the + negative input. The RAM-pressure eviction path cannot be forced on a real host and is + covered by the Python tests. + +```sh +python3 -m unittest discover -s tests -p "test_cached_runs.py" +PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:executions -- execution-text-link --grep "#347" +``` + ## Durable execution ledger `npm run --prefix tools/vendor test:executions` runs 52 real execution cases @@ -5653,6 +6014,20 @@ rotates the epoch, so a pre-restore `expected_epoch` is refused with `storage-ch Owner rows and their timestamps never change on a tag command; a category apply only touches rows carrying the exact source. +Image categories (#332 ยง7, MC9, MC10, DC7): each `GET categories` row also carries +`image_count`, the stored `image_categories` rows holding that exact value (files +missing on disk included). A value prompts and images share appears once, ordered by +its oldest positive; a value only images hold appears once after every prompt-backed +value (`first_pid` 9223372036854775807 in the keyset cursor), ordered by value, with +`prompt_count` 0. `/library/options` keeps its own query and never lists image-only +values. Category preview and apply report `impact.images` (rows holding the source), +the target exists when an image row holds it, an image-only value can be renamed, the +apply renames the image rows in the same transaction (file keys and rowids kept, so a +merge cannot collide), and the impact token hashes the sorted image rowids, so an image +row gaining or losing the value between preview and apply is 409 +`vocabulary-conflict` with nothing written. Every image-category statement reads +`idx_image_categories_category` (`test_category_reads_use_the_image_category_index`). + Two legacy routes change additively: `GET /tags` now lists unused registry tags, and `/tags/suggest` without context appends unused names at count 0. Unused tags become deletable only through the native Delete tag (vocabulary UI PR), not on the legacy diff --git a/__init__.py b/__init__.py index a398acfd..41c6fba7 100644 --- a/__init__.py +++ b/__init__.py @@ -195,6 +195,26 @@ except Exception as e: f"Failed to register usage hook: {e}" ) +# A cached node counts what its last execution of the same text counted (#347). +try: + from .py.config import server_instance + from .utils.cached_runs import register_cached_runs + + register_cached_runs(server_instance, _db_factory) +except Exception as e: + try: + from .utils.logging_config import get_logger + + get_logger("prompt_manager.init").error( + f"Failed to register cached-run counting: {e}" + ) + except Exception: + import logging + + logging.getLogger("prompt_manager.init").error( + f"Failed to register cached-run counting: {e}" + ) + def _history_db_factory() -> "PromptDatabase": """Initialize history on its worker even if monitoring initialization failed.""" @@ -355,6 +375,44 @@ async def _stop_background_work_on_cleanup(app): _stop_background_work() +def _kill_encoders_at_exit(): + """Run by the exit hook before Python joins the executor threads (#376): + a JPEG XL or ffmpeg encoder (no console on Windows, so Ctrl+C never + reaches it) is killed instead of being waited for.""" + from .py.api.thumbnail_encode import shutdown_encoders + + shutdown_encoders() + + +def _abort_downloads_at_exit(): + """Run by the exit hook (#376): huggingface_hub's Xet transfers ignore + the chunk checkpoint, so they are aborted the way hub aborts on Ctrl+C.""" + import sys + + if "huggingface_hub" not in sys.modules: + return # no download ever ran: import nothing at exit + from .py.autotag import abort_hub_downloads + + abort_hub_downloads() + + +try: + # Before the executor join: running jobs stop at their next file, item or + # chunk (#376). The teardown below still runs from atexit afterwards. + from .utils import exit_signal as _exit_signal + + _exit_signal.install(_kill_encoders_at_exit) + _exit_signal.install(_abort_downloads_at_exit) +except Exception as e: + try: + from .utils.logging_config import get_logger + + get_logger("prompt_manager.init").error( + f"Failed to register the exit signal: {e}" + ) + except Exception: + pass + try: atexit.register(_stop_background_work) from .py.config import server_instance as _teardown_server diff --git a/database/duplicate_copies.py b/database/duplicate_copies.py index ee4c07b4..5399b7e1 100644 --- a/database/duplicate_copies.py +++ b/database/duplicate_copies.py @@ -13,6 +13,8 @@ IMMEDIATE`` transaction, so no row ever names a deleted file: union under the same keys a favorite moves by; - its image tags (``media_image_tags`` of its byte record), joined onto the kept copy's byte record as their union, in order; +- its own image category (#332), copied to the kept copy when that copy has + none; the removed copy's row is kept, so a give-back loses nothing; - its pending Needs review items, settled: only an item still ``pending`` at revision 0 with no decision row is removed (``review._SETTLE_ITEM``), so a decided item is never touched. @@ -24,6 +26,8 @@ import os from .annotation_schema import BYTES_TABLE, FILES_TABLE from .annotations import _BUMP_INDEX +from .image_categories import COPY_SQL as COPY_CATEGORY_SQL +from .image_categories import copy_image_category from .models import normalize_image_path, utc_now_iso from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM from .review_changes import notify_review_change @@ -89,6 +93,7 @@ FILE_LINK_SQL = ( COPY_BYTE_SQL, COPY_TAG_COUNT_SQL, PENDING_COUNT_SQL, + COPY_CATEGORY_SQL, ) @@ -150,9 +155,11 @@ class DuplicateCopyOperations: removed key, kept root key, kept key)`` per listed root holding the file; they move its favorite and its collection memberships), ``removed_locator``, ``kept_locator``, ``sha256`` and the kept copy's - ``kept_size`` and ``kept_mtime_ns``. Returns ``{moved, merged, - favorites, tags, review, collections}`` (``collections``: the - memberships inserted on the kept copy). + ``kept_size`` and ``kept_mtime_ns``; ``categories``: ``(removed file + key, kept file key)`` pairs of the image categories (#332). Returns + ``{moved, merged, favorites, tags, review, collections, categories}`` + (``collections``: the memberships inserted on the kept copy; + ``categories``: the image categories copied to the kept copy). """ counts = dict.fromkeys( ("moved", "merged", "favorites", "tags", "review", "collections"), 0 @@ -191,6 +198,10 @@ class DuplicateCopyOperations: counts["favorites"] += _move_favorite(conn, *favorite) counts["collections"] += _move_memberships(conn, *favorite) counts["tags"] = self._move_copy_tags(conn, move) + counts["categories"] = sum( + copy_image_category(conn, removed, kept) + for removed, kept in move.get("categories", ()) + ) for (item_id,) in conn.execute( _PENDING_OF_LOCATOR, (move["removed_locator"],) ).fetchall(): diff --git a/database/image_categories.py b/database/image_categories.py new file mode 100644 index 00000000..a1f5d5a1 --- /dev/null +++ b/database/image_categories.py @@ -0,0 +1,249 @@ +"""Image categories (#332): an image file's own category. + +An image shows its own category when it has one, otherwise the category its +link supplies (its recipe's, else its positive prompt's); this table holds +only the own values. A row is keyed by the file key: the deepest configured +output source that contains the file (its real, case-normalized path) and the +path below it, case-folded where the OS folds case +(``py/api/images.py`` ``image_category_key``). Without nested sources the key +equals a favorite's key. Nothing follows a moved file: its row stays and is +hidden until a file appears at the same path again. + +Writes validate strictly (the route strips the value and applies +``validate_category``). Startup checks the structure only, so a value saved +under older rules never stops the library opening; restore also checks the +values. Optional in old backups, required once a database is current. +""" + +import re +import sqlite3 +from typing import Iterable + +TABLE = "image_categories" +INDEX = "idx_image_categories_category" +MAX_KEY_LENGTH = 4096 +MAX_CATEGORY_LENGTH = 100 +_DDL = ( + f"CREATE TABLE {TABLE} (root_key TEXT NOT NULL, relative_path TEXT NOT NULL, " + "category TEXT NOT NULL, updated_at TEXT NOT NULL, " + "PRIMARY KEY (root_key, relative_path))" +) +_INDEX_DDL = f"CREATE INDEX {INDEX} ON {TABLE}(category)" +_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1" +_COLUMNS = ( + ("root_key", "TEXT", 1, 1), + ("relative_path", "TEXT", 1, 2), + ("category", "TEXT", 1, 0), + ("updated_at", "TEXT", 1, 0), +) +# A category may not hold a character below U+0020 (validate_category). +_CONTROL_GLOB = "*[" + "".join(chr(code) for code in range(1, 32)) + "]*" + +CATEGORY_OF_SQL = ( + f"SELECT category FROM {TABLE} WHERE root_key = ? AND relative_path = ?" +) +CATEGORIES_OF_ROOT_SQL = ( + f"SELECT relative_path, category FROM {TABLE} WHERE root_key = ?" +) +UPSERT_SQL = ( + f"INSERT INTO {TABLE} (root_key, relative_path, category, updated_at) " + "VALUES (?, ?, ?, ?) ON CONFLICT(root_key, relative_path) " + "DO UPDATE SET category = excluded.category, updated_at = excluded.updated_at" +) +DELETE_SQL = f"DELETE FROM {TABLE} WHERE root_key = ? AND relative_path = ?" +# The duplicate cleanup's copy (#332 MC11): the removed copy's row is copied +# to the kept copy only when the kept copy has none; it is never deleted. +COPY_SQL = ( + f"INSERT OR IGNORE INTO {TABLE} (root_key, relative_path, category, updated_at) " + f"SELECT ?, ?, category, updated_at FROM {TABLE} " + "WHERE root_key = ? AND relative_path = ?" +) +STATEMENTS = (CATEGORY_OF_SQL, CATEGORIES_OF_ROOT_SQL, UPSERT_SQL, DELETE_SQL, COPY_SQL) + + +def _compact(sql: str) -> str: + return re.sub(r"\s+", "", sql or "").lower() + + +def _structure_error( + conn: sqlite3.Connection, allow_missing: bool, allow_missing_index: bool = False +) -> str | None: + objects = conn.execute( + "SELECT type,name,sql FROM sqlite_master WHERE name=? " + "OR (type IN ('index','trigger') AND tbl_name=?)", + (TABLE, TABLE), + ).fetchall() + if not objects: + return None if allow_missing else "Missing image categories schema" + tables = [sql for kind, _name, sql in objects if kind == "table"] + others = {(kind, name) for kind, name, _sql in objects if kind != "table"} + expected = {("index", _AUTOINDEX), ("index", INDEX)} + # Before its install, a table without the category index is repaired by + # CREATE INDEX IF NOT EXISTS rather than refused. + if allow_missing_index and others == {("index", _AUTOINDEX)}: + expected = others + if len(tables) != 1 or others != expected: + return "Image categories name is reserved by an unsupported schema object" + if _compact(tables[0]) != _compact(_DDL): + return "Image categories has an invalid table definition" + index_sql = next((sql for kind, name, sql in objects if name == INDEX), None) + if ("index", INDEX) in expected and _compact(index_sql) != _compact(_INDEX_DDL): + return "Image categories has an invalid index" + autoindex_sql = next(sql for kind, name, sql in objects if name == _AUTOINDEX) + if autoindex_sql is not None: + return "Image categories has an invalid index" + columns = tuple( + (row[1], row[2].upper(), row[3], row[5]) + for row in conn.execute(f"PRAGMA table_info({TABLE})") + ) + if columns != _COLUMNS: + return "Image categories has invalid columns" + return None + + +def _values_error(conn: sqlite3.Connection) -> str | None: + # Bytes bound characters from above (UTF-8 is at most 4 bytes each). + limit = MAX_KEY_LENGTH * 4 + if conn.execute( + f"SELECT 1 FROM {TABLE} WHERE typeof(root_key) != 'text' " # nosec B608 + "OR typeof(relative_path) != 'text' OR typeof(category) != 'text' " + "OR typeof(updated_at) != 'text' " + f"OR length(CAST(root_key AS BLOB)) > {limit} " + f"OR length(CAST(relative_path AS BLOB)) > {limit} " + "OR length(CAST(updated_at AS BLOB)) > 64 " + "OR instr(CAST(category AS BLOB), x'00') > 0 " + f"OR length(category) < 1 OR length(category) > {MAX_CATEGORY_LENGTH} " + "OR category != TRIM(category) OR category GLOB ? LIMIT 1", + (_CONTROL_GLOB,), + ).fetchone(): + return "Image categories contains invalid values" + for root, path in conn.execute( + f"SELECT CAST(root_key AS BLOB), CAST(relative_path AS BLOB) " # nosec B608 + f"FROM {TABLE} WHERE length(CAST(root_key AS BLOB)) > {MAX_KEY_LENGTH} " + f"OR length(CAST(relative_path AS BLOB)) > {MAX_KEY_LENGTH}" + ): + for raw in (root, path): + if len(bytes(raw).decode("utf-8", "replace")) > MAX_KEY_LENGTH: + return "Image categories contains invalid values" + return None + + +def image_categories_schema_error( + conn: sqlite3.Connection, + *, + allow_missing: bool = True, + check_values: bool = True, +) -> str | None: + """Reject an unsupported image categories schema (and, optionally, values). + + Reads only. A missing table is acceptable for backups that predate it + unless ``allow_missing`` is false. ``check_values`` also refuses a stored + value the write path could never have written. + """ + error = _structure_error(conn, allow_missing) + if error is not None or not check_values: + return error + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,) + ).fetchone() + return _values_error(conn) if exists else None + + +def migrate_image_categories(conn: sqlite3.Connection) -> None: + """Install the table and its index atomically; structure is checked + before and after. Reads only ``sqlite_master`` and this table's PRAGMAs + and writes only its own two objects (no backfill).""" + conn.execute("SAVEPOINT image_categories_schema") + try: + error = _structure_error(conn, True, allow_missing_index=True) + if error: + raise sqlite3.DatabaseError(error) + conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1)) + conn.execute( + _INDEX_DDL.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS", 1) + ) + error = image_categories_schema_error( + conn, allow_missing=False, check_values=False + ) + if error: + raise sqlite3.DatabaseError(error) + conn.execute("RELEASE image_categories_schema") + except BaseException: + conn.execute("ROLLBACK TO image_categories_schema") + conn.execute("RELEASE image_categories_schema") + raise + + +def copy_image_category(conn: sqlite3.Connection, removed_key, kept_key) -> int: + """Give the kept copy the removed copy's category when it has none. + + Runs inside the caller's transaction; never deletes the removed copy's + row, so a give-back or a failed delete loses nothing. Returns 1 when a + row was inserted, else 0. + """ + if tuple(removed_key) == tuple(kept_key): + return 0 + return max(conn.execute(COPY_SQL, (*kept_key, *removed_key)).rowcount, 0) + + +class ImageCategoryOperations: + """PromptDatabase mixin: set, clear and read images' own categories. + + Never reads or writes ``prompts``, ``prompt_pairs`` or + ``generated_images``. + """ + + def image_category(self, root_key: str, relative_key: str) -> str | None: + """One file's own category, or None.""" + row = ( + self.model.get_connection() + .execute(CATEGORY_OF_SQL, (root_key, relative_key)) + .fetchone() + ) + return row[0] if row else None + + def own_categories_of(self, root_keys: Iterable[str]) -> dict: + """``{(root key, relative key): category}`` under the given sources. + + One primary-key range read per source. + """ + found = {} + conn = self.model.get_connection() + for root_key in dict.fromkeys(root_keys): + for relative, category in conn.execute(CATEGORIES_OF_ROOT_SQL, (root_key,)): + found[(root_key, relative)] = category + return found + + def set_image_categories(self, file_keys, category: str) -> dict: + """Set *category* on every file key, in one transaction. + + ``file_keys``: ``(root key, relative key)`` pairs from any number of + sources. Returns ``{updated, unchanged}``; a file that already had + exactly this value is ``unchanged`` and keeps its ``updated_at``. + """ + from .models import utc_now_iso # models imports this module's schema + + keys = list(dict.fromkeys(tuple(key) for key in file_keys)) + counts = {"updated": 0, "unchanged": 0} + now = utc_now_iso() + with self._pair_transaction() as conn: + for key in keys: + row = conn.execute(CATEGORY_OF_SQL, key).fetchone() + if row is not None and row[0] == category: + counts["unchanged"] += 1 + continue + conn.execute(UPSERT_SQL, (*key, category, now)) + counts["updated"] += 1 + return counts + + def clear_image_categories(self, file_keys) -> int: + """Remove the own category of every file key (one transaction). + + Returns the number of rows removed. + """ + keys = list(dict.fromkeys(tuple(key) for key in file_keys)) + removed = 0 + with self._pair_transaction() as conn: + for key in keys: + removed += conn.execute(DELETE_SQL, key).rowcount + return removed diff --git a/database/models.py b/database/models.py index fe30685e..805d665f 100644 --- a/database/models.py +++ b/database/models.py @@ -38,6 +38,7 @@ from .review_schema import ( review_without_annotations_error, ) from .saved_views import migrate_saved_views, saved_views_schema_error +from .image_categories import image_categories_schema_error, migrate_image_categories from .workflow_documents import ( DOCUMENT_ID_COLUMNS, migrate_workflow_documents, @@ -355,6 +356,8 @@ class PromptModel: migrate_annotation_state(conn) migrate_review_state(conn) migrate_autotag_state(conn) + # Image categories (#332): independent of every table above. + migrate_image_categories(conn) # Last: the older generated_images rebuilds run only before file_path. migrate_workflow_documents(conn) @@ -1160,8 +1163,9 @@ class PromptModel: Args: path: File to inspect require_library_state: Also require the tables every current - database has (Library state, image favorites, saved views, - collections, image annotations and the review queue), as a migrated + database has (Library state, image favorites, image + categories, saved views, collections, image annotations and + the review queue), as a migrated restore stage must; old backups may lack them. documents: Read every workflow document and refuse a damaged one. A restore checks its upload without it and repairs @@ -1225,6 +1229,11 @@ class PromptModel: ) if favorites_error: return False, favorites_error + categories_error = image_categories_schema_error( + conn, allow_missing=not require_library_state + ) + if categories_error: + return False, categories_error views_error = saved_views_schema_error( conn, allow_missing=not require_library_state ) diff --git a/database/operations.py b/database/operations.py index def1adeb..a40651b4 100644 --- a/database/operations.py +++ b/database/operations.py @@ -7,7 +7,12 @@ import json import math import datetime import os -from typing import Optional, List, Dict, Any, Set, Tuple +from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple + +try: + from ..utils.inspection_response import inherited_category +except ImportError: + from utils.inspection_response import inherited_category from .models import PromptModel, normalize_image_path, utc_now_iso from .pairs import PairDependencyError, PairOperations # noqa: F401 (re-exported) @@ -28,6 +33,7 @@ from .health import HealthOperations, iter_consistency_issues from .duplicate_copies import DuplicateCopyOperations from .workflow_storage import WorkflowStorageOperations from .category_import import CategoryImportOperations, EmptyReplacement # noqa: F401 +from .image_categories import ImageCategoryOperations from .workflow_documents import ( DocumentReader, documents_of_rows, @@ -238,12 +244,17 @@ LEFT JOIN prompt_library_state pls ON pls.prompt_id = pp.positive_prompt_id WHERE gi.image_path IN ({marks}) OR gi.file_path IN ({marks}){fallback}""" # One image row describes a card only when no other row claims the same file # (as in get_image_link_summaries, several rows are ambiguous). The pair scope -# and the linked positive follow _LINK_SUMMARY_SQL; category, rating, tags and -# the searchable text and titles come from that positive (spec D12/D13). Every +# and the linked positive follow _LINK_SUMMARY_SQL; rating, tags and the +# searchable text and titles come from that positive (spec D12/D13). The raw +# recipe and positive categories are read for the inherited category, which +# Python decides (``inherited_category``, #332 MC4); SQL never filters by +# category. Every # writer since the 3.2.4 migration stores file_path, so it is the path key. # The prompt-level conditions are uncorrelated IN sets SQLite builds once, and # the CASE evaluates the per-row ambiguity probe only for admitted rows. -_LINK_FILTER_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id +_LINK_FILTER_SQL = """SELECT gi.file_path AS file_path, p.rating AS rating, + {hit} AS matched, p.id AS positive, pp.id IS NOT NULL AS has_pair, + pp.category AS pair_category, p.category AS category FROM generated_images gi LEFT JOIN prompt_pairs pp ON pp.id = ( SELECT CASE WHEN COUNT(DISTINCT ip.pair_id) = 1 THEN MIN(ip.pair_id) END @@ -252,10 +263,31 @@ JOIN prompts p ON p.id = COALESCE(pp.positive_prompt_id, gi.prompt_id) WHERE gi.file_path IS NOT NULL AND CASE WHEN {admit} THEN NOT EXISTS ( SELECT 1 FROM generated_images other WHERE other.file_path = gi.file_path AND other.id <> gi.id) ELSE 0 END""" +# _LINK_FILTER_SQL's rows as one JSON array, built by SQLite in one step (#354). +# Python's sqlite3 releases and retakes the GIL around every step, so a read +# that steps once per linked row waits on a busy ComfyUI thread thousands of +# times. A BLOB value (no writer stores one) makes the value NULL (or +# json_array refuses it), and the read streams instead. The raw categories are +# put in only for a category read (#332), so plain reads stay as small as #354 +# measured them. +_LINK_MATCHES_JSON_SQL = """SELECT CASE + WHEN max({blob}) THEN NULL + ELSE json_group_array(json_array({columns})) END +FROM ({select})""" +_LINK_PLAIN_COLUMNS = ("file_path", "rating", "matched", "positive") +_LINK_CATEGORY_COLUMNS = _LINK_PLAIN_COLUMNS + ("has_pair", "pair_category", "category") +# The columns a BLOB can reach: matched, positive and has_pair are computed. +_LINK_STORED_COLUMNS = ("file_path", "rating", "pair_category", "category") +# Set when this SQLite has no JSON functions: every later read streams. +_one_step_unsupported = False +# One warning per process when a read falls back to streaming. +_one_step_warned = False # The linked positive of each given path (one unambiguous link, pair scope as -# in _LINK_FILTER_SQL) with its category; {hit} as in link_filter_query. Used -# for the few files whose current image tags meet a Tags filter (#249 B3). -_LINK_FACTS_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id, p.category +# in _LINK_FILTER_SQL) with the raw categories of its inherited category; {hit} +# as in link_filter_query. Used for the few files whose current image tags meet +# a Tags filter (#249 B3). +_LINK_FACTS_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id, +pp.id IS NOT NULL, pp.category, p.category FROM generated_images gi LEFT JOIN prompt_pairs pp ON pp.id = ( SELECT CASE WHEN COUNT(DISTINCT ip.pair_id) = 1 THEN MIN(ip.pair_id) END @@ -284,8 +316,24 @@ _LINK_FILTER_TAG = ( ) +# One unambiguously linked image's facts for the Images filters, a plain tuple +# (hit, rating, positive id, inherited category): built per linked row, so no +# NamedTuple. The category is the inherited one (#332 MC4: the recipe's when +# set, else the positive's), never the image's own category. +LinkMatch = Tuple[bool, Optional[int], int, Any] + + +def _link_match(rating, matched, positive, has_pair, pair_category, category): + inherited, _scope = inherited_category(category, pair_category, bool(has_pair)) + return (bool(matched), rating, positive, inherited) + + def link_filter_query( - q: str, category: Optional[str], tags: List[str], *, every_link: bool = False + q: str, + tags: List[str], + *, + every_link: bool = False, + category: Optional[str] = None, ) -> Tuple[str, Dict[str, Any]]: """The workspace filter SELECT and its named parameters (all bound). @@ -299,7 +347,9 @@ def link_filter_query( params["pattern"] = f"%{escape_like(q)}%" admitted = [] if category is not None: - admitted.append("p.category = :category") + # Only narrows: an inherited category is one of the two raw values; + # which one is decided in Python (inherited_category, #332 MC4). + admitted.append("(p.category = :category OR pp.category = :category)") params["category"] = category for index, tag in enumerate(tags): admitted.append(_LINK_FILTER_TAG.format(index=index)) @@ -310,6 +360,50 @@ def link_filter_query( return sql, params +def link_matches_json_query(sql: str, categories: bool = False) -> str: + """*sql* from link_filter_query, wrapped to return its rows as one JSON value. + + With *categories* each row also carries the raw values the inherited + category is decided from; otherwise only path, rating, hit and positive. + """ + columns = _LINK_CATEGORY_COLUMNS if categories else _LINK_PLAIN_COLUMNS + blob = " OR ".join( + f"typeof({name}) = 'blob'" for name in _LINK_STORED_COLUMNS if name in columns + ) + return _LINK_MATCHES_JSON_SQL.format( + blob=blob, columns=", ".join(columns), select=sql + ) + + +def _link_matches_from_rows( + rows: Iterable[Sequence[Any]], categories: bool, wanted: Optional[Set[str]] +) -> Dict[str, LinkMatch]: + """LinkMatch per wanted path from link rows (one-step or streamed). + + Without *categories* the inherited category is None, as the caller then + never compares it. + """ + matches: Dict[str, LinkMatch] = {} + for path, rating, matched, positive, *raw in rows: + if wanted is None or path in wanted: + if categories: + matches[path] = _link_match(rating, matched, positive, *raw[:3]) + else: + matches[path] = (bool(matched), rating, positive, None) + return matches + + +def _streamed_link_matches( + conn: sqlite3.Connection, + sql: str, + params: Dict[str, Any], + wanted: Optional[Set[str]], + categories: bool = False, +) -> Dict[str, LinkMatch]: + """The link read stepped once per row (the fallback of the one-step read).""" + return _link_matches_from_rows(conn.execute(sql, params), categories, wanted) + + _ASCII_FOLD = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") @@ -370,6 +464,7 @@ class PromptDatabase( DuplicateCopyOperations, WorkflowStorageOperations, CategoryImportOperations, + ImageCategoryOperations, ): """Database operations class for managing prompts.""" @@ -2543,38 +2638,94 @@ class PromptDatabase( def get_image_link_matches( self, q: str, - category: Optional[str], tags: List[str], *, every_link: bool = False, wanted: Optional[Set[str]] = None, - ) -> Dict[str, Tuple[bool, Optional[int], int]]: - """Map normalized paths of unambiguously linked images to - (hit, rating, positive id). + category: Optional[str] = None, + ) -> Dict[str, LinkMatch]: + """Map normalized paths of unambiguously linked images to a LinkMatch + (hit, rating, positive id, inherited category). - Rows are limited to images whose linked positive has *category* and - every tag in *tags* (exact names). ``hit`` says whether *q* occurs in + Rows are limited to images whose linked positive has every tag in + *tags* (exact names). The category filter is applied by the caller to + the effective category (#332). ``hit`` says whether *q* occurs in the positive's text, Library title or a tag name; a pair's own title names its negative variant and is not searched. - Without category or tags only hits are returned, unless *every_link* + Without tags only hits are returned, unless *every_link* asks for every linked image (the rating sort and the positive count). - The positive id is the card's linked positive. One SELECT whose rows - are streamed; with *wanted* (the listing's normalized paths) only those - paths are kept, so memory is bounded by the listing, not the database. + The positive id is the card's linked positive. With *category* only + links whose raw recipe or positive category is that value are read (a + superset of the links that inherit it; the caller compares the + inherited value) and the inherited category is decided per row; + otherwise it is None, which keeps the plain reads cheap. + + One SELECT, returned as one JSON value (#354): SQLite steps once, so + the read takes the GIL back a constant two or three times instead of + once per linked row while ComfyUI generates. Peak memory therefore + grows with the database's matching linked rows (about 530 bytes each, + more with the raw categories of a category read, freed on return); the + result is bounded by *wanted* (the listing's normalized paths) when + given. Where the JSON step fails (no JSON functions, a BLOB value, a + JSON text over SQLite's length limit) the rows are streamed as before. """ - sql, params = link_filter_query(q, category, tags, every_link=every_link) - matches: Dict[str, Tuple[bool, Optional[int], int]] = {} + sql, params = link_filter_query( + q, tags, every_link=every_link, category=category + ) + categories = category is not None with self.model.get_connection() as conn: - for path, rating, matched, positive in conn.execute(sql, params): - if wanted is None or path in wanted: - matches[path] = (bool(matched), rating, positive) - return matches + rows = None + if not _one_step_unsupported: + rows = self._one_step_link_rows(conn, sql, params, categories) + if rows is None: + return _streamed_link_matches(conn, sql, params, wanted, categories) + return _link_matches_from_rows(rows, categories, wanted) + + def _one_step_link_rows( + self, + conn: sqlite3.Connection, + sql: str, + params: Dict[str, Any], + categories: bool = False, + ) -> Optional[List[List[Any]]]: + """*sql*'s rows from one JSON value, or None when this read must stream. + + An interrupted read (Move library retiring the connection) is raised + as before, never retried row by row. + """ + global _one_step_unsupported, _one_step_warned + error: Optional[str] = None + try: + cursor = conn.execute(link_matches_json_query(sql, categories), params) + try: + ((document,),) = cursor.fetchall() + finally: + cursor.close() # resets the statement: its snapshot ends now + except (sqlite3.OperationalError, sqlite3.DataError) as failure: + # DataError: the JSON text is longer than SQLITE_LIMIT_LENGTH. + error = str(failure) + if error == "interrupted": + raise + if error.startswith("no such function"): + _one_step_unsupported = True + else: + if document is not None: + return json.loads(document) + error = "a BLOB value" + if not _one_step_warned: + _one_step_warned = True + self.logger.warning( + "Reading image links row by row (slower while ComfyUI " + "generates); the one-step read failed: %s", + error, + ) + return None def get_image_link_facts( self, paths: List[str], q: str = "" - ) -> Dict[str, Tuple[bool, Optional[int], int, Optional[str], Tuple[str, ...]]]: + ) -> Dict[str, Tuple[bool, Optional[int], int, Any, Tuple[str, ...]]]: """Map each normalized path with one unambiguous link to its positive's - (hit, rating, positive id, category, tag names). + (hit, rating, positive id, inherited category, tag names). The link is resolved as in get_image_link_matches, and ``hit`` says whether *q* occurs in the positive's text, Library title or a tag @@ -2585,7 +2736,7 @@ class PromptDatabase( if q: hit = _LINK_FILTER_HIT params["pattern"] = f"%{escape_like(q)}%" - rows: Dict[str, Tuple[bool, Optional[int], int, Optional[str]]] = {} + rows: Dict[str, LinkMatch] = {} names: Dict[int, List[str]] = {} unique = list(dict.fromkeys(paths)) with self.model.get_connection() as conn: @@ -2594,10 +2745,8 @@ class PromptDatabase( bound = {f"path{i}": path for i, path in enumerate(chunk)} marks = ",".join(f":{key}" for key in bound) sql = _LINK_FACTS_SQL.format(hit=hit, marks=marks) - for path, rating, matched, positive, category in conn.execute( - sql, {**params, **bound} - ): - rows[path] = (bool(matched), rating, positive, category) + for path, *row in conn.execute(sql, {**params, **bound}): + rows[path] = _link_match(*row) positives = sorted({row[2] for row in rows.values()}) for start in range(0, len(positives), _FACTS_CHUNK): chunk = positives[start : start + _FACTS_CHUNK] @@ -2610,20 +2759,16 @@ class PromptDatabase( path: (*row, tuple(names.get(row[2], ()))) for path, row in rows.items() } - def get_image_paths_matching( - self, q: str, category: Optional[str], tags: List[str] - ) -> Set[str]: - """Normalized paths whose linked positive matches *q*, *category* and *tags*. + def get_image_paths_matching(self, q: str, tags: List[str]) -> Set[str]: + """Normalized paths whose linked positive matches *q* and *tags*. See get_image_link_matches; an empty *q* matches every linked image - the category and tags admit. + the tags admit. """ return { path - for path, (hit, _rating, _positive) in self.get_image_link_matches( - q, category, tags - ).items() - if hit or not q + for path, match in self.get_image_link_matches(q, tags).items() + if match[0] or not q } def get_prompt_id_for_image(self, image_path: str) -> Optional[int]: diff --git a/database/review_schema.py b/database/review_schema.py index 0aa8318b..c0c45d76 100644 --- a/database/review_schema.py +++ b/database/review_schema.py @@ -229,13 +229,15 @@ def evidence_offers_choice(evidence: Any) -> bool: """ if not isinstance(evidence, dict): return False + # The saver anchor's one pair (#390) is a suggestion to confirm. + needed = 1 if evidence.get("anchor") == "saver" else MIN_CHOOSABLE if evidence.get("limited"): total = evidence.get("candidate_total") - return type(total) is int and total >= MIN_CHOOSABLE + return type(total) is int and total >= needed candidates = evidence.get("candidates") if not isinstance(candidates, list): return False - return sum(1 for c in candidates if choosable(c)) >= MIN_CHOOSABLE + return sum(1 for c in candidates if choosable(c)) >= needed def evidence_digest(text: str) -> str: diff --git a/database/run_attribution.py b/database/run_attribution.py index b7d4ec58..50a671db 100644 --- a/database/run_attribution.py +++ b/database/run_attribution.py @@ -15,6 +15,10 @@ only when all of this holds: Anything else returns None: the caller leaves the file unlinked. Read-only. +``recorded_output`` applies the first two rules and returns, for the +inspector (#386), the one output node (save node) that run reported the file +from and the texts PromptManager recorded at execution per sampler. + Lookups are batched (``stored_run_prompts``): one parameterized query per chunk of at most ``LOOKUP_CHUNK`` distinct file names, matched in memory. The history schema is left exactly as it is (no extra index), so older builds can @@ -26,6 +30,7 @@ import os import sqlite3 import threading import weakref +from dataclasses import dataclass from typing import Any, Callable, Dict, Iterable, List, Optional try: @@ -45,7 +50,8 @@ LOOKUP_CHUNK = 500 _RUNS_FOR_NAMES = ( "SELECT DISTINCT o.filename, o.subfolder, o.root_key, o.path_style, o.run_id, " - "r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at) " + "r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at), " + "o.output_node_id " "FROM execution_run_outputs o JOIN execution_runs r ON r.run_id = o.run_id " "WHERE o.folder_type = 'output' AND o.filename IN ({marks})" ) @@ -138,9 +144,10 @@ def stored_run_prompt( def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]: - """``{(filename, subfolder, root_key, style): {run_id: (start, end)}}``. + """``{(filename, subfolder, root_key, style): {run_id: (start, end, nodes)}}``. - One query per ``LOOKUP_CHUNK`` names; None without history tables. + ``nodes`` is the set of output node ids that reported the location in that + run. One query per ``LOOKUP_CHUNK`` names; None without history tables. """ found: Dict[tuple, dict] = {} conn = db.model.get_connection() @@ -151,12 +158,49 @@ def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]: rows = conn.execute(query, chunk).fetchall() # nosec B608 except sqlite3.Error: return None # no history tables (a library without history) - for name, subfolder, root_key, style, run_id, began, ended in rows: + for name, subfolder, root_key, style, run_id, began, ended, node in rows: runs = found.setdefault((name, subfolder, root_key, style), {}) - runs[run_id] = (began, ended) + runs.setdefault(run_id, (began, ended, set()))[2].add(node) return found +def _recorded_runs( + db: Any, + image_paths: Iterable[str], + output_root: Optional[str], + path_style: Optional[str], +) -> Dict[str, tuple]: + """``{path: (run_id, output node ids)}`` for each file history attributes. + + A file is attributed when exactly one recorded run reported its location + and it was last written while that run executed. + """ + paths = list(dict.fromkeys(path for path in image_paths if path)) + if not paths or not output_root: + return {} + style = path_style or ("windows" if os.name == "nt" else "posix") + located = {} + for path in paths: + location = _location(path, output_root, style) + if location is not None: + located[path] = location + if not located: + return {} + names = sorted({filename for _key, _sub, filename in located.values()}) + runs = _runs_by_location(db, names) + if not runs: + return {} + attributed = {} + for path, (root_key, subfolder, filename) in located.items(): + reported = runs.get((filename, subfolder, root_key, style), {}) + if len(reported) != 1: + continue + ((run_id, (began, ended, nodes)),) = reported.items() + if _written_during(path, began, ended): + attributed[path] = (run_id, nodes) + return attributed + + def _named_prompt(db: Any, run_id: str) -> Optional[int]: contexts = db.get_execution_contexts(run_id).get("contexts") or [] named = { @@ -183,35 +227,115 @@ def stored_run_prompts( """ paths = list(dict.fromkeys(path for path in image_paths if path)) result: Dict[str, Optional[int]] = dict.fromkeys(paths) - if not paths or not output_root: - return result - style = path_style or ("windows" if os.name == "nt" else "posix") - located = {} - for path in paths: - location = _location(path, output_root, style) - if location is not None: - located[path] = location - if not located: - return result - names = sorted({filename for _key, _sub, filename in located.values()}) - runs = _runs_by_location(db, names) - if not runs: - return result prompts: Dict[str, Optional[int]] = {} - for path, (root_key, subfolder, filename) in located.items(): - reported = runs.get((filename, subfolder, root_key, style), {}) - if len(reported) != 1: - continue - ((run_id, (began, ended)),) = reported.items() - if not _written_during(path, began, ended): - continue + for path, (run_id, _nodes) in _recorded_runs( + db, paths, output_root, path_style + ).items(): if run_id not in prompts: prompts[run_id] = _named_prompt(db, run_id) result[path] = prompts[run_id] return result +@dataclass(frozen=True) +class RecordedOutput: + """What recorded history knows about the run that wrote one file (#386). + + ``saver_id`` is the one output node the run reported the file from (None + when several did). ``texts`` maps each sampler anchor of the run to the + effective texts PromptManager recorded at execution for its roles + (``{"positive": text or None, "negative": text or None}``); a role whose + text was only inferred from the graph is None. + """ + + saver_id: Optional[str] + texts: Dict[str, Dict[str, Optional[str]]] + + +def _recorded_texts(db: Any, run_id: str) -> Dict[str, Dict[str, Optional[str]]]: + texts = {} + for context in db.get_execution_contexts(run_id).get("contexts") or []: + evidence = context.get("evidence") or {} + roles = {} + for role in ("positive", "negative"): + values = context.get(role) or {} + proof = (evidence.get(role) or {}).get("effective") or {} + value = values.get("effective") + recorded = proof.get("provenance") == "recorded" + roles[role] = value if recorded and isinstance(value, str) else None + texts[context["anchor_id"]] = roles + return texts + + +def recorded_output( + db: Any, + image_path: str, + output_root: Optional[str], + path_style: Optional[str] = None, +) -> Optional[RecordedOutput]: + """The recorded run facts for ``image_path``, else None (#386). + + The file must be attributed to one run (``_recorded_runs``: one run + reported its location and it was written during that run). Read-only; it + never links anything. + """ + if not image_path: + return None + conn = db.model.get_connection() + own_transaction = not conn.in_transaction + if own_transaction: # outputs and texts from one state of history + conn.execute("BEGIN") + try: + found = _recorded_runs(db, [image_path], output_root, path_style) + if image_path not in found: + return None + run_id, nodes = found[image_path] + saver = next(iter(nodes)) if len(nodes) == 1 else None + return RecordedOutput(saver, _recorded_texts(db, run_id)) + finally: + if own_transaction: + conn.rollback() + + _LOGGER = logging.getLogger("prompt_manager.run_attribution") + + +def recorded_output_lookup( + db: Any, *, close: bool = False, reference: Optional[dict] = None +) -> Callable[[Any], Any]: + """``lookup(path) -> RecordedOutput or None`` for the inspect routes (#386). + + Uses ComfyUI's output folder. Any failure (no history, a locked library) + is logged and names no node, so the file is judged from its own metadata. + ``close`` closes the calling thread's connection afterwards, for routes + that close theirs after reading. ``reference`` (a history occurrence's + output reference) spells the file as history recorded it: the route's + target is case-normalized on Windows, and history matches names exactly. + """ + + def lookup(path: Any) -> Optional[RecordedOutput]: + root = host_output_directory() + if reference is not None: + if reference.get("folder_type") != "output" or not root: + return None + path = os.path.join( + root, reference.get("subfolder") or "", reference.get("filename") or "" + ) + try: + return recorded_output(db, str(path), root) + except (sqlite3.Error, OSError, ValueError): + _LOGGER.warning("Recorded run lookup failed", exc_info=True) + return None + except Exception: + _LOGGER.exception("Recorded run lookup failed unexpectedly") + return None + finally: + if close: + db.model.close() + + return lookup + + _SEALED_LISTENERS: List[weakref.WeakMethod] = [] _SEALED_LOCK = threading.Lock() diff --git a/database/vocabulary.py b/database/vocabulary.py index 5bca3fb3..9dd5f42a 100644 --- a/database/vocabulary.py +++ b/database/vocabulary.py @@ -13,7 +13,9 @@ same connection before commit. Owner rows and their timestamps never change. Category consolidation is previewed in one ``library_snapshot`` and applied in one ``_pair_transaction`` that recomputes the owner projection and requires the -preview's ``impact_token``; only rows carrying the exact source change. +preview's ``impact_token``; only rows carrying the exact source change. Image +rows holding the source (``image_categories``, #332) are owners too: they are +counted, hashed into the token and renamed with the prompts and pairs. """ import sqlite3 @@ -97,16 +99,30 @@ def _validate_tag_id(tag_id: object) -> None: raise ValueError("Invalid tag ID") -# The /library/options universe: nonblank values on eligible parents or on -# active pairs, grouped exactly (whitespace and case variants stay distinct). -_CATEGORY_GROUPS = ( +# Category values in Tags & categories (#332 ยง7): the prompt-backed values +# (nonblank values on eligible parents or on active pairs, grouped exactly, so +# whitespace and case variants stay distinct; /library/options lists the same +# prompt-backed set with its own query) plus the values only image rows hold. +# A prompt-backed value is ordered by its oldest owning positive; an image-only +# value sorts after all of them (first_pid is SQLite's INTEGER maximum, above +# any prompt id), then by value. ``image_count`` counts stored rows, including +# files missing on disk. No FULL OUTER JOIN, so it runs on SQLite 3.31+. +_IMAGE_ONLY_FIRST_PID = 2**63 - 1 +_CATEGORY_ROWS = ( "WITH owners(value,pid) AS (" "SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL " "AND TRIM(p.category)!='' AND " + _ELIGIBLE + " " "UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp " "WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!='')," - "grouped AS (SELECT value,MIN(pid) AS first_pid," - "COUNT(DISTINCT pid) AS prompt_count FROM owners GROUP BY value) " + "image_values(value,image_count) AS (" + "SELECT category,COUNT(*) FROM image_categories GROUP BY category)," + "grouped(value,first_pid,prompt_count) AS (" + "SELECT value,MIN(pid),COUNT(DISTINCT pid) FROM owners GROUP BY value " + "UNION ALL SELECT value," + str(_IMAGE_ONLY_FIRST_PID) + ",0 FROM image_values " + "WHERE value NOT IN (SELECT value FROM owners))," + "counted AS (SELECT g.value,g.first_pid,g.prompt_count," + "COALESCE(i.image_count,0) AS image_count FROM grouped g " + "LEFT JOIN image_values i ON i.value=g.value) " ) @@ -117,15 +133,23 @@ _CATEGORY_COMPONENTS = ( "WHERE p.category=? ORDER BY p.id" ) _CATEGORY_PAIRS = "SELECT id,archived FROM prompt_pairs WHERE category=? ORDER BY id" +# Image rows holding one exact value (#332 DC7), an index search. +_IMAGE_CATEGORY_ROWIDS = ( + "SELECT rowid FROM image_categories WHERE category=? ORDER BY rowid" +) _CATEGORY_EXISTS = ( "SELECT EXISTS(SELECT 1 FROM prompts WHERE category=:v)" " OR EXISTS(SELECT 1 FROM prompt_pairs WHERE category=:v)" + " OR EXISTS(SELECT 1 FROM image_categories WHERE category=:v)" ) -# Pairs after components, so a failing pair update rolls both back. +# Pairs after components and image rows last, so a failing update rolls every +# earlier one back. An image row is keyed by its file, never by its value, so +# merging into a value other files already hold cannot conflict. _CATEGORY_UPDATES = ( "UPDATE prompts SET category=?,updated_at=? WHERE category=?", "UPDATE prompt_pairs SET category=?,updated_at=? WHERE category=?", + "UPDATE image_categories SET category=?,updated_at=? WHERE category=?", ) @@ -136,8 +160,15 @@ def category_impact_token( target_exists: bool, components: list, pairs: list, + *, + images: list = (), ) -> str: - """Hash a category projection like ``library_edit_token`` (no secret).""" + """Hash a category projection like ``library_edit_token`` (no secret). + + ``images`` are the rowids of the image rows holding ``source`` (#332 DC7). + With none the preimage has no ``images`` key, so a prompt-only token keeps + the value it had before image categories existed. + """ preimage = { "epoch": epoch, "source": source, @@ -146,6 +177,8 @@ def category_impact_token( "components": sorted([int(i), bool(flag)] for i, flag in components), "pairs": sorted([int(i), bool(flag)] for i, flag in pairs), } + if images: + preimage["images"] = sorted(int(i) for i in images) return canonical_token(_CATEGORY_DOMAIN, preimage) @@ -325,7 +358,8 @@ class VocabularyOperations: pairs = [ [row[0], bool(row[1])] for row in conn.execute(_CATEGORY_PAIRS, (source,)) ] - if not components and not pairs: + images = [row[0] for row in conn.execute(_IMAGE_CATEGORY_ROWIDS, (source,))] + if not components and not pairs and not images: raise LibraryCommandError("vocabulary-conflict", epoch) target_exists = bool( conn.execute(_CATEGORY_EXISTS, {"v": target}).fetchone()[0] @@ -339,9 +373,10 @@ class VocabularyOperations: "other_prompts": len(components) - eligible, "negative_variants": len(pairs) - archived, "archived_negative_variants": archived, + "images": len(images), }, "impact_token": category_impact_token( - epoch, source, target, target_exists, components, pairs + epoch, source, target, target_exists, components, pairs, images=images ), } @@ -365,7 +400,8 @@ class VocabularyOperations: def apply_vocabulary_category( self, *, expected_epoch: str, source: str, target: str, impact_token: str ) -> dict: - """Set the exact ``source`` to ``target`` on every component and pair. + """Set the exact ``source`` to ``target`` on every component, pair + and image row. The recomputed projection must still match the preview's token; a mismatch is ``vocabulary-conflict`` with nothing written. @@ -422,15 +458,15 @@ class VocabularyOperations: _limit(limit) key = _category_after(after) total = conn.execute( - _CATEGORY_GROUPS + "SELECT COUNT(*) FROM grouped" + _CATEGORY_ROWS + "SELECT COUNT(*) FROM counted" ).fetchone()[0] where, params = "", [] if key is not None: where = " WHERE first_pid>? OR (first_pid=? AND value>?)" params = [key[0], key[0], key[1]] rows = conn.execute( - _CATEGORY_GROUPS - + "SELECT value,first_pid,prompt_count FROM grouped" + _CATEGORY_ROWS + + "SELECT value,first_pid,prompt_count,image_count FROM counted" + where + " ORDER BY first_pid,value LIMIT ?", [*params, limit + 1], @@ -440,7 +476,11 @@ class VocabularyOperations: return { "library_epoch": epoch, "categories": [ - {"value": row["value"], "prompt_count": row["prompt_count"]} + { + "value": row["value"], + "prompt_count": row["prompt_count"], + "image_count": row["image_count"], + } for row in rows ], "total": total, diff --git a/prompt_manager.py b/prompt_manager.py index b9b3f5ed..2dcf0031 100644 --- a/prompt_manager.py +++ b/prompt_manager.py @@ -268,6 +268,7 @@ class PromptManager(PromptManagerBase, ComfyNodeABC): record_runtime_context( type(self).__name__, text, prepend_text, append_text, encoding_text ) + self._record_cached_item(unique_id, (encoding_text,), storage_text) return result @classmethod diff --git a/prompt_manager_base.py b/prompt_manager_base.py index 3a655318..62977b1f 100644 --- a/prompt_manager_base.py +++ b/prompt_manager_base.py @@ -26,6 +26,13 @@ try: from .utils.prompt_graph import resolve_text, run_prompt_nodes from .utils.run_keys import executing_run_id, prompt_node_key from .utils.usage_tracking import PENDING_FIRST_USE + from .utils.cached_runs import ( + CACHED_RUNS, + Role, + Seen, + own_counts, + stored_facts, + ) except ImportError: import sys @@ -38,6 +45,13 @@ except ImportError: from utils.prompt_graph import resolve_text, run_prompt_nodes from utils.run_keys import executing_run_id, prompt_node_key from utils.usage_tracking import PENDING_FIRST_USE + from utils.cached_runs import ( + CACHED_RUNS, + Role, + Seen, + own_counts, + stored_facts, + ) class PromptManagerBase: @@ -71,8 +85,10 @@ class PromptManagerBase: """Save the prompt to the SQLite database. Runs are normally counted by the queue hook (utils/usage_tracking.py), so a - save only counts one when asked to (batch items whose text the hook could not - see) or when the hook queued this prompt before it existed. + save only counts one when asked to (text the hook could not see, unless + this job already counted the node from ComfyUI's cache) or when the hook + queued this prompt before it existed. The two are independent: the + pending catch-up is written even when ``count_run`` is False. Args: text: The prompt text @@ -167,8 +183,19 @@ class PromptManagerBase: batch_item = ( role["known"] and role["is_run_prompt"] and not role["seen_at_queue"] ) + counts = own_counts( + type(self).__name__, + stored_facts(text), + Role( + frozenset({"text"} if role["known"] and role["is_run_prompt"] else ()) + ), + Seen(text=role["seen_at_queue"]), + ) prompt_id = self._save_prompt_to_database( - text=text, category=category, tags=tags, count_run=batch_item + text=text, + category=category, + tags=tags, + count_run=bool(counts) and not self._replayed(unique_id), ) if role["is_run_prompt"] and not prompt_id: # The save failed: an older entry of this key must not stand in (#342) @@ -182,6 +209,34 @@ class PromptManagerBase: ) return prompt_id + def _cached_runs(self) -> Any: + return getattr(self, "cached_runs", CACHED_RUNS) + + def _replayed(self, unique_id: Any) -> bool: + """This job already counted the node from ComfyUI's cache (#347). + + A RAM-pressure cache can free the entry after ``execution_cached``, so + the node executes in a job that replayed it: its own count is skipped. + """ + try: + return unique_id is not None and self._cached_runs().replayed(unique_id) + except Exception as error: + self.logger.debug(f"Could not check the cached-run replay: {error}") + return False + + def _record_cached_item( + self, unique_id: Any, outputs: tuple, text: Any, negative_text: Any = None + ) -> None: + """Remember this execution's outputs and stored text for cached jobs (#347).""" + if unique_id is None: + return + try: + self._cached_runs().record_item( + unique_id, outputs, stored_facts(text, negative_text) + ) + except Exception as error: + self.logger.debug(f"Could not record the cached-run receipt: {error}") + @staticmethod def _run_identity(prompt_graph: Any, unique_id: Any) -> Dict[str, Any]: """The tracker's proof that an image belongs to this execution (#342).""" diff --git a/prompt_manager_pair.py b/prompt_manager_pair.py index 5c903338..df62d633 100644 --- a/prompt_manager_pair.py +++ b/prompt_manager_pair.py @@ -12,6 +12,14 @@ else: try: from .prompt_manager import IO, InputTypeDict, PromptManagerFocused + from .utils.cached_runs import ( + PAIR, + Role, + Seen, + own_counts, + pair_components, + stored_facts, + ) from .utils.hashing import generate_prompt_hash from .utils.pair_execution import record_pair_execution from .utils.prompt_graph import intrinsic_pair_nodes, run_prompt_components @@ -23,6 +31,14 @@ try: ) except ImportError: from prompt_manager import IO, InputTypeDict, PromptManagerFocused + from utils.cached_runs import ( + PAIR, + Role, + Seen, + own_counts, + pair_components, + stored_facts, + ) from utils.hashing import generate_prompt_hash from utils.pair_execution import record_pair_execution from utils.prompt_graph import intrinsic_pair_nodes, run_prompt_components @@ -165,6 +181,9 @@ class PromptManagerPair(PromptManagerFocused): record_runtime_context( type(self).__name__, negative_text, "", "", negative_text, role="negative" ) + self._record_cached_item( + unique_id, (positive, negative_text), text, negative_text + ) return result def _save_and_track_pair( @@ -209,36 +228,35 @@ class PromptManagerPair(PromptManagerFocused): and generate_prompt_hash(resolved_negative) == generate_prompt_hash(negative_text) ) + negative_seen = resolved_negative is not None and generate_prompt_hash( + resolved_negative + ) == generate_prompt_hash(negative_text) positive_pending = getattr(self, "pending_first_use", PENDING_FIRST_USE) pair_pending = getattr(self, "pending_pair_first_use", PENDING_PAIR_FIRST_USE) - counted = set() - for field, component_text, resolved_text, component_id in ( - ("text", text, resolved_positive, prompt_id), - ( - "negative_text", - negative_text, - resolved_negative, - pair["negative_prompt_id"], - ), - ): - component_hash = generate_prompt_hash(component_text) - if ( - field not in used_fields - or not component_id - or component_hash in counted - ): + facts = stored_facts(text, negative_text) + role = Role(frozenset(used_fields), pair_used) + # Own counts are skipped when this job already counted the node from + # ComfyUI's cache (#347); the pending first-use catch-up never is. + counts = ( + [] + if self._replayed(unique_id) + else own_counts( + PAIR, facts, role, Seen(positive_seen, negative_seen, pair_seen) + ) + ) + component_ids = {"text": prompt_id, "negative_text": pair["negative_prompt_id"]} + for field, component_hash in pair_components(facts, role): + component_id = component_ids[field] + if not component_id: continue - counted.add(component_hash) - seen_at_queue = ( - resolved_text is not None - and generate_prompt_hash(resolved_text) == component_hash + runs = positive_pending.consume(component_hash) + int( + ("prompt", component_hash) in counts ) - runs = positive_pending.consume(component_hash) + int(not seen_at_queue) if runs: self.db.record_prompt_use(component_id, times=runs) if pair_used: runs = pair_pending.consume(pair_run_key(text, negative_text)) + int( - not pair_seen + ("pair", facts.positive_hash, facts.negative_hash) in counts ) if runs: self.db.record_prompt_pair_use(pair_id, times=runs) diff --git a/prompt_manager_text.py b/prompt_manager_text.py index b09945df..6e316f98 100644 --- a/prompt_manager_text.py +++ b/prompt_manager_text.py @@ -239,6 +239,7 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC): record_runtime_context( type(self).__name__, text, prepend_text, append_text, final_text ) + self._record_cached_item(unique_id, (final_text,), storage_text) return result @classmethod diff --git a/py/api/__init__.py b/py/api/__init__.py index ed94ecea..76fd0bb1 100644 --- a/py/api/__init__.py +++ b/py/api/__init__.py @@ -73,6 +73,7 @@ from .diagnostics_report import DiagnosticsReportMixin from .autotag_routes import AutotagRoutesMixin from .lora_integration import LoraIntegrationMixin from .lora_import_routes import LoraImportRoutesMixin +from .image_categories import ImageCategoryRoutesMixin try: from ...database.operations import PromptDatabase @@ -338,6 +339,7 @@ class PromptManagerAPI( AutotagRoutesMixin, LoraIntegrationMixin, LoraImportRoutesMixin, + ImageCategoryRoutesMixin, ): """REST API handler for PromptManager operations and web interface. @@ -703,6 +705,7 @@ class PromptManagerAPI( self._register_autotag_routes(routes) self._register_lora_routes(routes) self._register_lora_import_routes(routes) + self._register_image_category_routes(routes) # Register gzip compression middleware (once) global _gzip_registered diff --git a/py/api/admin.py b/py/api/admin.py index ec3f6db7..a96d07d2 100644 --- a/py/api/admin.py +++ b/py/api/admin.py @@ -34,6 +34,11 @@ try: except ImportError: from utils.parallel import map_parallel +try: + from ...utils import exit_signal +except ImportError: + from utils import exit_signal + try: from ...database.pair_ingestion import blank_positive, ingest_resolved_pair from ...database.run_attribution import ( @@ -118,6 +123,7 @@ def _collect_media_files(output_dirs, extensions): if not os.path.isdir(root): continue for current, dirnames, filenames in os.walk(root, followlinks=False): + exit_signal.checkpoint() # #376: stop between directories at exit depth = 0 if current == root else len(Path(current).relative_to(root).parts) dirnames[:] = sorted( d @@ -1630,7 +1636,14 @@ class AdminRoutesMixin: if self._review_file_linked(media_file, roots): outcome = "unchanged" # already linked: not a review case else: - self._record_scanned_review(media_file, roots, review) + # The records name the item versions that count; the + # saver anchor (#390) is checked on the captured bytes. + self._record_scanned_review( + media_file, + roots, + review, + scanned=(records.get("canonical"), None), + ) elif untraced and outcome == "unresolved": # Untraced by the default profile: the inspection profile may # still offer a choice (queued for review, never linked). @@ -1691,9 +1704,9 @@ class AdminRoutesMixin: ``"ambiguous"`` (nothing linked it) is a Needs review case. An ambiguous result with nothing to choose (``reviewable`` is False: fewer than two candidates carry a prompt) that nothing links returns - ``"unresolved"`` instead, and ``media_file`` goes to ``untraced``; - its parsed records go to ``records["canonical"]`` when ``records`` is - a dict. + ``"unresolved"`` instead, and ``media_file`` goes to ``untraced``. + The parsed records of either ambiguous file go to + ``records["canonical"]`` when ``records`` is a dict. """ if not metadata: # No prompt metadata (--disable-metadata, bare savers): only the @@ -1723,6 +1736,8 @@ class AdminRoutesMixin: records["canonical"] = canonical elif resolution["state"] == "ambiguous" and ambiguous is not None: ambiguous.append(media_file) + if records is not None: + records["canonical"] = canonical if resolution["state"] == "resolved": result = ingest_resolved_pair( self.db, str(media_file), canonical, resolution diff --git a/py/api/backup_restore.py b/py/api/backup_restore.py index edd60d14..a83ecbfa 100644 --- a/py/api/backup_restore.py +++ b/py/api/backup_restore.py @@ -483,13 +483,59 @@ class BackupRestoreRoutesMixin: self.logger.info(f"Removed {len(removed)} old safety backups") async def _publish(self, model, staged): - """Publish ``staged``; on a page-copy failure prune and re-raise.""" + """Publish ``staged``; on a page-copy failure prune and re-raise. + + A cancel (server stop) waits for the publishing thread (#360): the + caller's cleanup and its registry admission must outlive it. When the + copy then succeeded the library was replaced, so it is adopted + (``_after_publish``, itself run to its end) before the cancel goes + on; a copy that failed is handled as usual, then the cancel goes on. + """ + future = self._in_executor_task(model.publish_staged, staged) + cancelled = await _outlive_cancel(future) try: - return await self._run_in_executor(model.publish_staged, staged) + backup_path = future.result() except sqlite3.OperationalError as exc: if getattr(exc, "_prompt_manager_restore_backup_created", False) is True: await self._prune_after(model.db_path, failed=True) + if cancelled: + raise asyncio.CancelledError from exc raise + except Exception as exc: + if cancelled: + raise asyncio.CancelledError from exc + raise + if cancelled: + self.logger.info("The restore finished while the server was stopping") + # A second cancel (the loop's own shutdown) must not cut it short. + adopted = asyncio.ensure_future(self._after_publish(model.db_path)) + await _outlive_cancel(adopted) + if not adopted.cancelled() and adopted.exception() is not None: + self.logger.error( + "Could not adopt the restored library", + exc_info=adopted.exception(), + ) + raise asyncio.CancelledError + return backup_path + + async def _stage_to_end(self, model, temp_path): + """Stage ``temp_path``; a cancel waits for the worker (#360). + + The upload stays until the staging thread is done with it, and a + stage it finished after the cancel is removed before the cancel goes + on. + """ + future = self._in_executor_task(model.stage_restore, temp_path) + if await _outlive_cancel(future): + if not future.cancelled() and future.exception() is None: + await self._discard_held([_Orphan(future.result())]) + raise asyncio.CancelledError + return future.result() + + def _in_executor_task(self, func, *args): + """``_run_in_executor(func, *args)`` as a task ``_outlive_cancel`` can + wait for (shielded, the executor call is never cancelled).""" + return asyncio.ensure_future(self._run_in_executor(func, *args)) async def _after_publish(self, db_path): """Everything after the copy replaced the library: best effort, logged. @@ -609,7 +655,7 @@ class BackupRestoreRoutesMixin: {"success": False, "error": REQUESTS_BUSY}, status=409 ) model = self.db.model - staged = await self._run_in_executor(model.stage_restore, temp_path) + staged = await self._stage_to_end(model, temp_path) try: backup_path = await self._publish(model, staged) finally: @@ -927,6 +973,30 @@ class BackupRestoreRoutesMixin: job.update(progress=100, status_line=line) +async def _outlive_cancel(future): + """Await ``future`` (an executor call's task) to its end, even through + cancels (#360). + + A cancelled task cannot stop the worker thread, so the files it uses and + the registry admission it runs under must outlive it: the cancel is held + until the worker returned. Returns whether the task was cancelled + meanwhile; the caller reads the outcome from ``future`` and re-raises + ``CancelledError``. A future cancelled itself (an executor shut down) is + not a cancel of the task: ``future.result()`` raises it. Never catches + ``GeneratorExit``: a closed loop still closes the coroutine. + """ + cancelled = False + while not future.done(): + try: + await asyncio.shield(future) + except asyncio.CancelledError: + if not future.cancelled(): + cancelled = True + except Exception: + pass # the caller reads it from the future + return cancelled + + class _Orphan: """A stage with no store entry, in the shape ``_discard_held`` takes.""" diff --git a/py/api/duplicate_files.py b/py/api/duplicate_files.py index 9cf101f5..34e15ed4 100644 --- a/py/api/duplicate_files.py +++ b/py/api/duplicate_files.py @@ -591,6 +591,7 @@ class DuplicateFilesMixin: totals = dict.fromkeys( ("moved", "merged", "favorites", "tags", "review", "collections"), 0 ) + totals["categories"] = 0 # image categories kept on the kept copies (#332) for done, target in enumerate(targets, start=1): if failure is not None: self._record_target(job, "kept", target, NOT_ATTEMPTED) @@ -642,6 +643,9 @@ class DuplicateFilesMixin: job.log_line( f"Added {_plural(totals['tags'], 'image tag')} to the kept copies" ) + if totals.get("categories"): + kept = _plural(totals["categories"], "image category", "image categories") + job.log_line(f"Kept {kept} on the kept copies") if totals["review"]: job.log_line( f"Settled {_plural(totals['review'], 'Needs review item')} " diff --git a/py/api/duplicate_spellings.py b/py/api/duplicate_spellings.py index 3653e5de..7157e994 100644 --- a/py/api/duplicate_spellings.py +++ b/py/api/duplicate_spellings.py @@ -18,7 +18,7 @@ Blocking: each lookup stats the file's folders once (cached). import os from pathlib import Path -from .images import _favorite_relative_key, _favorite_root_key +from .images import _favorite_relative_key, _favorite_root_key, image_category_key def _identity(path): @@ -78,6 +78,15 @@ class Spellings: ) return spelled + @property + def listed_roots(self): + """The listed roots (the output sources), in order.""" + return [root for root, _identity in self._listed] + + def category_key(self, path): + """*path*'s image category file key (#332), or None outside every source.""" + return image_category_key(self.listed_roots, path) + def favorites(self, path): """``{listed root: path below it (POSIX)}`` for each listed root holding it.""" path = os.path.abspath(str(path)) @@ -126,9 +135,23 @@ def copy_move(target, kept_stat, spellings): "targets": targets, "kept_path": target["keeper_abs"], "favorites": favorites, + "categories": _category_pairs(target, spellings), "removed_locator": target["locator"], "kept_locator": target["keeper_locator"], "sha256": target["hash"], "kept_size": kept_stat.st_size, "kept_mtime_ns": kept_stat.st_mtime_ns, } + + +def _category_pairs(target, spellings): + """``[(removed file key, kept file key)]`` for the image category (#332). + + From the two absolute paths, not the favorites tuples: those follow the + listed root, which differs from the file key when sources nest. + """ + removed = spellings.category_key(target["abs"]) + kept = spellings.category_key(target["keeper_abs"]) + if removed is None or kept is None: + return [] + return [(removed, kept)] diff --git a/py/api/history.py b/py/api/history.py index f50960fb..9426ee0c 100644 --- a/py/api/history.py +++ b/py/api/history.py @@ -5,6 +5,7 @@ import base64 import binascii from functools import wraps import json +import os import re import sqlite3 from typing import Callable @@ -27,10 +28,12 @@ from .history_current import ( snapshot_current_output_scope, ) from .inspection_service import CurrentFileChanged, inspect_current_file +from .images import image_category_key try: from ...database.execution_history import HistoryEpochChanged from ...database.history_gallery import read_history_gallery + from ...database.run_attribution import recorded_output_lookup from ...database.history_reads import ( HistoryCaptureChanged, OutputPosition, @@ -38,10 +41,15 @@ try: WindowPosition, ) from ...utils.history_read_fence import HistoryReadFence - from ...utils.inspection_response import encode_history_inspection_response + from ...utils.inspection_response import ( + OWN_CATEGORY_UNREAD, + apply_own_category, + encode_history_inspection_response, + ) except ImportError: from database.execution_history import HistoryEpochChanged from database.history_gallery import read_history_gallery + from database.run_attribution import recorded_output_lookup from database.history_reads import ( HistoryCaptureChanged, OutputPosition, @@ -49,7 +57,11 @@ except ImportError: WindowPosition, ) from utils.history_read_fence import HistoryReadFence - from utils.inspection_response import encode_history_inspection_response + from utils.inspection_response import ( + OWN_CATEGORY_UNREAD, + apply_own_category, + encode_history_inspection_response, + ) _RUN_MAX_BYTES = 8 * 1024 * 1024 @@ -675,6 +687,32 @@ class HistoryRoutesMixin: ) -> web.Response: return await self._history_current(request, "inspection") + def _history_own_category(self, reference: dict, scope): + """The recorded output's own category (#332), inside the snapshot. + + The path is the symlink-resolved output folder plus the receipt's + subfolder and filename, keyed by ``image_category_key`` over the + configured output sources as Images keys it. A temp output or a path + no source holds has none. A failed read is OWN_CATEGORY_UNREAD. + """ + if ( + reference.get("folder_type") != "output" + or scope.canonical_output is None + or candidate_locator(reference, scope) is None + ): + return None + path = os.path.join( + str(scope.canonical_output), reference["subfolder"], reference["filename"] + ) + try: + key = image_category_key(self._get_all_output_dirs(), path) + return None if key is None else self.db.image_category(*key) + except Exception: + self.logger.warning( + "History image category could not be read", exc_info=True + ) + return OWN_CATEGORY_UNREAD + async def _history_current( self, request: web.Request, kind: str ) -> web.StreamResponse: @@ -699,6 +737,7 @@ class HistoryRoutesMixin: if scope.current_scope != current_scope: raise _current_stale() result = db.get_history_occurrence(run_id, node, ordinal) + own = None if result is not None: receipt = result["receipt"] path = candidate_locator(receipt["reference"], scope) @@ -706,6 +745,10 @@ class HistoryRoutesMixin: db.model.get_connection(), (path,) if path is not None else (), ) + if kind == "inspection": + own = self._history_own_category( + receipt["reference"], scope + ) finally: db.model.close() if result is None: @@ -724,9 +767,19 @@ class HistoryRoutesMixin: resolved.canonical_target, source_label=resolved.public["source_label"], saved=saved.get("saved"), - library=saved.get( - "library", - {"status": "unavailable" if saved_unavailable else "none"}, + library=apply_own_category( + saved.get( + "library", + { + "status": ( + "unavailable" if saved_unavailable else "none" + ) + }, + ), + own, + ), + recorded=recorded_output_lookup( + db, close=True, reference=receipt["reference"] ), ) _reauthorize_current_target(resolved, scope) diff --git a/py/api/image_categories.py b/py/api/image_categories.py new file mode 100644 index 00000000..ff37ecd4 --- /dev/null +++ b/py/api/image_categories.py @@ -0,0 +1,374 @@ +"""Image category routes (#332): an image file's own category. + +``PUT /prompt_manager/images/category`` sets the own category of up to 500 +listed files; ``DELETE`` removes it (listed or not, so a missing file's +category can still be removed); ``GET /prompt_manager/images/categories`` +lists the own-category values of files the sources list. Targets are the +Images listing identity (``{root_index, root_scope, path}``, checked by +``media_target``) of an output source; LoRA folders are not sources here. +Instead of ``items`` a request may name ``filter``, the Images listing's own +query ("Select all N matching", #392), resolved now up to 500 images. +Each file is stored under its file key (``images.image_category_key``). +These routes never read or write prompts, recipes or image links. +""" + +import os +from pathlib import Path + +from aiohttp import web + +from .image_tag_jobs import MAX_JOB_TARGETS, _parse_targets +from .images import ( + _favorite_relative_key, + _favorite_root_key, + _forbidden, + _json_body_or_error, + image_category_key, +) +from .prompts import bad_request + +try: + from ...database.image_categories import MAX_CATEGORY_LENGTH, MAX_KEY_LENGTH + from ...database.models import normalize_image_path + from ...utils.validators import validate_category +except ImportError: + from database.image_categories import MAX_CATEGORY_LENGTH, MAX_KEY_LENGTH + from database.models import normalize_image_path + from utils.validators import validate_category + +CATEGORY_MEDIA_MESSAGE = "Only media files can have a category" +_PUT_FIELDS = frozenset({"category"}) +_DELETE_FIELDS = frozenset() +TOO_MANY = f"Select at most {MAX_JOB_TARGETS} images." +SOURCE_CHANGED_MESSAGE = "The sources changed. Refresh the images and try again." +WRITE_FAILED = "The category could not be changed." +READ_FAILED = "Image categories could not be read." + + +def _coded(status: int, code: str, message: str) -> web.Response: + return web.json_response( + {"success": False, "code": code, "error": message}, + status=status, + headers={"Cache-Control": "no-store"}, + ) + + +def _category_value(raw) -> tuple[str | None, web.Response | None]: + """The stripped category, or a 400 (an empty value is a 400: use DELETE).""" + if not isinstance(raw, str): + return None, bad_request("category must be a string") + value = raw.strip() + if not value: + return None, bad_request("category must not be empty") + if len(value) > MAX_CATEGORY_LENGTH: + return None, bad_request( + f"Category cannot exceed {MAX_CATEGORY_LENGTH} characters" + ) + try: + value.encode("utf-8") + validate_category(value) + except UnicodeEncodeError: + return None, bad_request("category must be valid text") + except ValueError as exc: + return None, bad_request(str(exc)) + return value, None + + +class _KeyRefused(Exception): + """A target has no storable file key (outside every source, or too long).""" + + def __init__(self, message: str = "Access denied"): + super().__init__(message) + self.message = message + + +def _file_key(roots, root, relative) -> tuple[str, str]: + """The target's file key (blocking: resolves the sources).""" + key = image_category_key(roots, Path(root) / relative) + if key is None: + raise _KeyRefused() + if max(len(key[0]), len(key[1])) > MAX_KEY_LENGTH: + raise _KeyRefused("Path is too long") + return key + + +def _nested_sources(root: Path, sources) -> list[tuple[str, str]]: + """``[(relative key prefix, source root key)]`` of sources inside *root*. + + Deepest first, so the first prefix that matches is the file's source. + """ + base = os.path.join(os.path.normcase(os.path.realpath(str(root))), "") + nested = [] + for source in sources: + real = os.path.realpath(str(source)) + if not os.path.normcase(real).startswith(base): + continue + relative = os.path.relpath(real, os.path.realpath(str(root))) + nested.append( + ( + _favorite_relative_key(relative.replace(os.sep, "/")), + _favorite_root_key(source), + ) + ) + nested.sort(key=lambda item: item[0].count("/"), reverse=True) + return nested + + +def _source_prefix(relative_key: str, nested) -> str | None: + """The prefix of the deepest nested source holding *relative_key*.""" + for prefix, _source_key in nested: + if relative_key.startswith(prefix + "/"): + return prefix + return None + + +class ImageCategoryRoutesMixin: + """Routes for images' own categories (#332).""" + + def _register_image_category_routes(self, routes): + # Fixed paths beside DELETE /prompt_manager/images/{image_id}, as the + # favorite routes are. Registered after it, they still win: aiohttp + # (3.10+) looks fixed paths up before dynamic resources + # (test_delete_category_never_reaches_delete_image). + @routes.put("/prompt_manager/images/category") + async def set_image_category_route(request): + return await self.set_image_category(request) + + @routes.delete("/prompt_manager/images/category") + async def clear_image_category_route(request): + return await self.clear_image_category(request) + + @routes.get("/prompt_manager/images/categories") + async def list_image_categories_route(request): + return await self.list_image_categories(request) + + # -- reading --------------------------------------------------------------- + + def _own_category_map(self, sources, listings) -> dict: + """``{normalized listed path: own category}`` for the listed files (blocking). + + One primary-key range read per source; each stored row is matched to + the listing of every selected root that holds its source, so the work + follows the stored rows, not the files. A file under a nested source + is keyed by the deepest one (``image_category_key``). A row whose file + is gone matches nothing. A failed read raises. + """ + sources = list(sources) + rows = self.db.own_categories_of(_favorite_root_key(s) for s in sources) + if not rows: + return {} + by_source: dict[str, dict[str, str]] = {} + for (root_key, relative), category in rows.items(): + by_source.setdefault(root_key, {})[relative] = category + found = {} + for _index, root, files in listings: + listing = self._favorite_listing(root, files) + root_key = _favorite_root_key(root) + # normalize_image_path(root / listed) without a call per file: the + # listed spellings are clean relative POSIX paths. Windows abspath + # also drops a component's trailing dots and spaces, so it keeps + # the call there. + base = os.path.join(normalize_image_path(str(root)), "") + + def normalized(listed, base=base, root=root): + if os.name == "nt": + return normalize_image_path(str(root / listed)) + return base + os.path.normcase(listed.replace("/", os.sep)) + + nested = [ + item for item in _nested_sources(root, sources) if item[1] != root_key + ] + for relative, category in by_source.get(root_key, {}).items(): + listed = listing.get(relative) + if listed is not None and _source_prefix(relative, nested) is None: + found[normalized(listed)] = category + for prefix, source_key in nested: + for relative, category in by_source.get(source_key, {}).items(): + full = f"{prefix}/{relative}" + listed = listing.get(full) + if listed is not None and _source_prefix(full, nested) == prefix: + found[normalized(listed)] = category + return found + + async def _source_listings(self, roots) -> list[tuple]: + return [ + (index, root, await self._get_gallery_files(root)) + for index, root in enumerate(roots) + ] + + async def list_image_categories(self, request): + """GET: the own-category values of listed files, sorted.""" + try: + roots = list(self._get_all_output_dirs()) + listings = await self._source_listings(roots) + found = await self._run_in_executor(self._own_category_map, roots, listings) + except Exception: + self.logger.exception("Image categories read failed") + return _coded(500, "categories-failed", READ_FAILED) + return web.json_response( + {"success": True, "categories": sorted(set(found.values()))}, + headers={"Cache-Control": "no-store"}, + ) + + # -- writing --------------------------------------------------------------- + + async def _filter_targets(self, raw): + """``(targets, None)`` for an Images listing filter, else ``(None, error)``. + + "Select all N matching" (#392) sends the listing's own query; it is + resolved now, up to 500 images (a 400 ``too-many-images`` above). + """ + from .images import ListingSourceChanged, parse_listing_filter + + try: + selection = await self.resolve_listing_filter( + parse_listing_filter(raw), MAX_JOB_TARGETS + ) + except ListingSourceChanged: + return None, _coded(409, "source-changed", SOURCE_CHANGED_MESSAGE) + except ValueError as exc: + return None, bad_request(str(exc)) + if selection.truncated: + return None, _coded(400, "too-many-images", TOO_MANY) + return tuple(selection.targets), None + + async def _category_request(self, request, fields: frozenset): + """``(data, targets, roots, None)`` or ``(None, None, None, error)``. + + The images are ``items`` (1 to 500 targets) or ``filter`` (all images + an Images listing filter admits), besides *fields*. + """ + data, error = await _json_body_or_error(request) + if error is not None: + return None, None, None, error + names = set(data) - fields + if set(data) & fields != fields or names not in ({"items"}, {"filter"}): + return ( + None, + None, + None, + bad_request( + "Request must contain " + + ", ".join(sorted(fields)) + + (" and " if fields else "") + + "either items or filter" + ), + ) + if "filter" in data: + targets, error = await self._filter_targets(data["filter"]) + if error is not None: + return None, None, None, error + return data, targets, list(self._get_all_output_dirs()), None + items = data["items"] + if not isinstance(items, list) or not 1 <= len(items) <= MAX_JOB_TARGETS: + return ( + None, + None, + None, + bad_request(f"items must list 1 to {MAX_JOB_TARGETS} images"), + ) + targets, error = _parse_targets(items, media_message=CATEGORY_MEDIA_MESSAGE) + if error is not None: + return None, None, None, error + try: + for _index, _scope, relative in targets: + relative.encode("utf-8") + except UnicodeEncodeError: + return None, None, None, bad_request("path must be valid text") + roots = list(self._get_all_output_dirs()) + for index, scope in dict.fromkeys((t[0], t[1]) for t in targets): + if index >= len(roots): + return None, None, None, bad_request("Invalid gallery root") + error = self._gallery_scope_error( + {"root_scope": scope, "root": str(index)}, roots + ) + if error is not None: + if error.status == 409: + error = _coded(409, "source-changed", SOURCE_CHANGED_MESSAGE) + return None, None, None, error + return data, targets, roots, None + + def _listed_file_keys(self, roots, targets, files) -> tuple[list, list]: + """``(listed keys, unlisted keys)`` of the targets (blocking). + + A target counts as listed when its source's listing holds it; its key + then follows the listed spelling. Targets folding to one key count + once, and a key that is listed through any target is listed. + """ + listings, listed, missing = {}, {}, {} + for index, _scope, relative in targets: + root = roots[index] + if index not in listings: + listings[index] = self._favorite_listing(root, files[index]) + spelled = listings[index].get(_favorite_relative_key(relative)) + key = _file_key(roots, root, spelled or relative) + (missing if spelled is None else listed)[key] = True + return list(listed), [key for key in missing if key not in listed] + + async def set_image_category(self, request): + """PUT ``{items, category}``: set the own category of listed files.""" + data, targets, roots, error = await self._category_request(request, _PUT_FIELDS) + if error is not None: + return error + category, error = _category_value(data["category"]) + if error is not None: + return error + try: + files = {} + for index in dict.fromkeys(t[0] for t in targets): + files[index] = await self._get_gallery_files(roots[index]) + listed_keys, missing_keys = await self._run_in_executor( + self._listed_file_keys, roots, targets, files + ) + except _KeyRefused as refused: + return _forbidden(refused.message) + except Exception: + self.logger.exception("Image category update failed") + return _coded(500, "category-failed", WRITE_FAILED) + try: + if not listed_keys: + return _coded( + 404, + "files-not-found", + "None of the selected images could be found.", + ) + counts = await self._run_in_executor( + self.db.set_image_categories, listed_keys, category + ) + except Exception: + self.logger.exception("Image category update failed") + return _coded(500, "category-failed", WRITE_FAILED) + return web.json_response( + { + "success": True, + "category": category, + "updated": counts["updated"], + "unchanged": counts["unchanged"], + "not_found": len(missing_keys), + }, + headers={"Cache-Control": "no-store"}, + ) + + async def clear_image_category(self, request): + """DELETE ``{items}``: remove the own category, listed or not.""" + _data, targets, roots, error = await self._category_request( + request, _DELETE_FIELDS + ) + if error is not None: + return error + try: + keys = await self._run_in_executor( + lambda: list( + dict.fromkeys(_file_key(roots, roots[i], r) for i, _s, r in targets) + ) + ) + cleared = await self._run_in_executor(self.db.clear_image_categories, keys) + except _KeyRefused as refused: + return _forbidden(refused.message) + except Exception: + self.logger.exception("Image category removal failed") + return _coded(500, "category-failed", WRITE_FAILED) + return web.json_response( + {"success": True, "cleared": cleared}, + headers={"Cache-Control": "no-store"}, + ) diff --git a/py/api/image_tag_jobs.py b/py/api/image_tag_jobs.py index 66f148f5..61c79422 100644 --- a/py/api/image_tag_jobs.py +++ b/py/api/image_tag_jobs.py @@ -195,13 +195,17 @@ class _Spec: return scope -def _parse_targets(raw): - """Unique validated targets in request order, or ``(None, error)``.""" +def _parse_targets(raw, **target_options): + """Unique validated targets in request order, or ``(None, error)``. + + ``target_options`` reach the target check (the image category routes + pass their own ``media_message``, #332). + """ if not isinstance(raw, list) or not 1 <= len(raw) <= MAX_JOB_TARGETS: return None, bad_request(f"targets must list 1 to {MAX_JOB_TARGETS} images") seen, targets = set(), [] for item in raw: - target, error = _target(item) + target, error = _target(item, **target_options) if error is not None: return None, error if target not in seen: diff --git a/py/api/images.py b/py/api/images.py index 10e70070..614287b9 100644 --- a/py/api/images.py +++ b/py/api/images.py @@ -62,10 +62,13 @@ try: from ...database.image_favorites import MAX_KEY_LENGTH from ...database.library import display_title from ...database.models import normalize_image_path + from ...database.run_attribution import recorded_output_lookup from ...utils.card_seed import card_seed, header_params, parsed_positive from ...utils.parallel import map_parallel from ...utils.json_strict import dumps_strict from ...utils.inspection_response import ( + OWN_CATEGORY_UNREAD, + apply_own_category, encode_inspection_response, library_metadata, ) @@ -83,10 +86,13 @@ except ImportError: from database.image_favorites import MAX_KEY_LENGTH from database.library import display_title from database.models import normalize_image_path + from database.run_attribution import recorded_output_lookup from utils.card_seed import card_seed, header_params, parsed_positive from utils.parallel import map_parallel from utils.json_strict import dumps_strict from utils.inspection_response import ( + OWN_CATEGORY_UNREAD, + apply_own_category, encode_inspection_response, library_metadata, ) @@ -580,7 +586,8 @@ class _ImageTagMatch(NamedTuple): tags: bool # every filter tag is in its image tags or linked positive tags query: bool # ``q`` is a case-folded substring of one image tag name - link: tuple | None # (hit, rating, positive id) of its unambiguous link + # (hit, rating, positive id, inherited category) of its unambiguous link + link: tuple | None def _current_fingerprint(path: Path) -> tuple[int, int] | None: @@ -817,6 +824,53 @@ def _favorite_root_key(root) -> str: return _favorite_relative_key(os.path.realpath(str(root))) +def _source_depth(source: str) -> int: + return len([part for part in re.split(r"[\\/]+", source) if part]) + + +def deepest_source(sources, absolute_path): + """``(source, POSIX path below it)`` of the deepest source holding the path. + + A source matches by its real path or by its spelling as configured (a + detected output folder can be a symlink), and the deepest real path wins. + The path is compared lexically after ``normpath`` and never resolved, so a + symlinked subfolder keeps its listed spelling. Containment folds case + where the OS does. *source* is the real path. None when no source + contains the path. + """ + path = os.path.normpath(str(absolute_path)) + folded = os.path.normcase(path) + best = None + for source in sources: + real = os.path.normpath(os.path.realpath(str(source))) + for spelling in dict.fromkeys((real, os.path.normpath(str(source)))): + prefix = os.path.join(os.path.normcase(spelling), "") + if not folded.startswith(prefix): + continue + if best is None or _source_depth(real) > _source_depth(best[0]): + best = (real, spelling) + break + if best is None: + return None + relative = os.path.relpath(path, best[1]) + return best[0], relative.replace(os.sep, "/") + + +def image_category_key(sources, absolute_path): + """The file key of an image's own category (#332), or None. + + ``(root key, relative key)``: the deepest output source that contains the + file and the path below it, keyed with the favorites helpers. Without + nested sources it equals the file's favorite key. A path no source + contains has no key (and so no own category). + """ + found = deepest_source(sources, absolute_path) + if found is None: + return None + source, relative = found + return _favorite_root_key(source), _favorite_relative_key(relative) + + def _name_suffix(name): """``Path(name).suffix`` as Python 3.10-3.13 compute it. @@ -1194,20 +1248,27 @@ def _admitted_entries( links: dict | None, members: frozenset | None = None, image_tags: dict | None = None, + own_categories: dict | None = None, ): """Yield ``(position, entry, link match)`` for each entry the filters admit. - *links* maps normalized paths to (hit, rating, positive id) for every - unambiguously linked image (see PromptDatabase.get_image_link_matches); a - category or tag filter admits only those, and ``q`` admits a filename match - or a linked-positive hit. *members*, when given, admits only those listed - paths (the favorites of ``favorites=1`` or a ``collection``'s members). - *image_tags* maps normalized paths to their _ImageTagMatch: a tag filter - also admits a file whose current image tags (with its linked positive's - tags) carry every tag, and ``q`` also admits an image tag name hit. + *links* maps normalized paths to (hit, rating, positive id, inherited + category) for every unambiguously linked image (see + PromptDatabase.get_image_link_matches); a tag filter admits only those, + and ``q`` admits a filename match or a linked-positive hit. A category + admits a file whose effective category equals it (#332 MC5): its own + category (*own_categories*, ``{normalized path: category}``) when it has one, + else the inherited one, so an unlinked file with its own category is + admitted. *members*, when given, admits only those listed paths (the + favorites of ``favorites=1`` or a ``collection``'s members). *image_tags* + maps normalized paths to their _ImageTagMatch: a tag filter also admits a + file whose current image tags (with its linked positive's tags) carry every + tag, and ``q`` also admits an image tag name hit. """ query, media, folder = filters["query"], filters["media"], filters["folder"] - members_only = filters["category"] is not None or bool(filters["tags"]) + category = filters["category"] + members_only = bool(filters["tags"]) + own = own_categories or {} native_folder = folder.replace("/", os.sep) nested = native_folder + os.sep exact = filters.get("path", "") @@ -1226,6 +1287,12 @@ def _admitted_entries( image = None if image_tags is None else image_tags.get(entry.normalized) if members_only and match is None and not (image and image.tags): continue + if category is not None: + effective = own.get(entry.normalized) if own else None + if effective is None and match is not None: + effective = match[3] + if effective != category: + continue if ( query and query not in entry.folded @@ -1244,6 +1311,7 @@ def _filtered_page( links: dict | None, members: frozenset | None = None, image_tags: dict | None = None, + own_categories: dict | None = None, ) -> tuple[int, list, int | None]: """Count the *entries* the filters admit and take *size* from *start* on. @@ -1253,7 +1321,7 @@ def _filtered_page( """ total, page, positives = 0, [], set() for position, entry, match in _admitted_entries( - entries, filters, links, members, image_tags + entries, filters, links, members, image_tags, own_categories ): total += 1 if match is not None: @@ -2196,9 +2264,10 @@ class ImageRoutesMixin: stat'ed (never every listed file), and a candidate's tags count only while a byte record is verified at its present ``(size, mtime_ns)`` (spec amendment S1). Under a Tags filter the candidates' linked - positives are read too, so their tags and category combine. + positives are read too, so their tags combine; the category is checked + once, on the effective category, by _admitted_entries (#332). """ - tags, query, category = filters["tags"], filters["query"], filters["category"] + tags, query = filters["tags"], filters["query"] wanted = set(tags) located = self._workspace_locators(listings) # The tag index answers which files carry a relevant name (#379); no @@ -2228,11 +2297,9 @@ class ImageRoutesMixin: link = facts.get(entry.normalized) carried = set(names).union(link[4] if link else ()) matches[entry.normalized] = _ImageTagMatch( - bool(wanted) - and wanted <= carried - and (category is None or (link is not None and link[3] == category)), + bool(wanted) and wanted <= carried, bool(query) and any(query in name.casefold() for name in names), - link[:3] if link else None, + tuple(link[:4]) if link else None, ) return matches @@ -2241,12 +2308,16 @@ class ImageRoutesMixin: ) -> tuple: """The ordered entries and what admits them (blocking; run in executor). - Returns ``(entries, key, links, image_tags, members)``: the listing in - the filters' order, its keyset key, and the link, image-tag and member - maps _admitted_entries reads (members: a collection's, else the - favorites, else None). A collection deleted since the route checked it - raises CollectionMissing. One SELECT (get_image_link_matches) reads - every linked positive when *read_links*. A failed read raises when the + Returns ``(entries, key, links, image_tags, members, own)``: the + listing in the filters' order, its keyset key, and the link, image-tag, + member and own-category maps _admitted_entries reads (members: a + collection's, else the favorites, else None). A collection deleted since + the route checked it raises CollectionMissing. One SELECT + (get_image_link_matches) reads every linked positive when *read_links*; + under a category it keeps only links whose raw recipe or positive + category is that value (the inherited one is decided in Python), and + the links of files whose own category is that value are read by path, + so their rating and positive still count (#332). A failed read raises when the filters need it (category, tags, the rating sort) or *strict*; else the listing stays usable without links (a search then matches filenames only). A failed image-tag read raises under a Tags filter; a search then @@ -2258,10 +2329,10 @@ class ImageRoutesMixin: try: links = self.db.get_image_link_matches( filters["text"], - category, tags, every_link=True, wanted=self._workspace_wanted(listings), + category=category, ) except Exception: if strict or category is not None or tags or order == "rating": @@ -2275,6 +2346,13 @@ class ImageRoutesMixin: ), exc_info=True, ) + own = None + if category is not None: + # A failed read raises: an unfiltered page must never pass for the + # category (DC6). + own = self._own_category_map(self._get_all_output_dirs(), listings) + if links is not None: + links = self._own_category_links(links, own, filters) image_tags = None if filters["query"] or tags: try: @@ -2324,8 +2402,32 @@ class ImageRoutesMixin: links, image_tags, self._workspace_members(listings, filters), + own, ) + def _own_category_links(self, links: dict, own: dict, filters: dict) -> dict: + """*links* plus the links of files whose own category is the filter's. + + Their inherited category differs, so the category's link read left + them out; read by path (get_image_link_facts), kept only when their + positive has every filter tag, as the link read would. + """ + category, wanted = filters["category"], set(filters["tags"]) + missing = [ + path + for path, value in own.items() + if value == category and path not in links + ] + if not missing: + return links + facts = self.db.get_image_link_facts(missing, filters["text"]) + extra = { + path: tuple(fact[:4]) + for path, fact in facts.items() + if wanted <= set(fact[4]) + } + return {**links, **extra} if extra else links + def _select_workspace_page( self, listings: list[tuple], @@ -2353,7 +2455,7 @@ class ImageRoutesMixin: or bool(filters["tags"]) or order == "rating" ) - entries, key, links, image_tags, members = self._workspace_admission( + entries, key, links, image_tags, members, own = self._workspace_admission( listings, filters, read_links=needed or after is None ) start = 0 @@ -2364,7 +2466,7 @@ class ImageRoutesMixin: else: start = bisect_right(entries, after, key=key) total, page, positive_count = _filtered_page( - entries, start, limit + 1, filters, links, members, image_tags + entries, start, limit + 1, filters, links, members, image_tags, own ) if not needed and after is not None: positive_count = _COUNT_OMITTED @@ -2386,12 +2488,12 @@ class ImageRoutesMixin: The links are read strictly: the positives must be the listing's own. """ - entries, _key, links, image_tags, members = self._workspace_admission( + entries, _key, links, image_tags, members, own = self._workspace_admission( listings, filters, read_links=True, strict=True ) targets, positives, unlinked, scopes = [], {}, 0, {} for _position, entry, match in _admitted_entries( - entries, filters, links, members, image_tags + entries, filters, links, members, image_tags, own ): if len(targets) == cap: return ListingSelection(targets, list(positives), unlinked, True) @@ -3745,17 +3847,25 @@ class ImageRoutesMixin: except Exception: # The link stays known; only its title is withheld. self.logger.warning("Inspection link title could not be read") + # An output file's own category (#332); LoRA roots have none. + own = ( + self._read_own_category(outputs, path) + if index < len(outputs) + else None + ) try: observed = inspect_current_file( path, source_label=_public_path(roots[index]), saved=saved, - library=self._inspection_library( - saved, unavailable=unavailable + library=apply_own_category( + self._inspection_library(saved, unavailable=unavailable), + own, ), image_tags=self._inspection_image_tags( roots[index], Path(relative).parts ), + recorded=recorded_output_lookup(self.db), ) except CurrentFileChanged: return None @@ -3818,6 +3928,19 @@ class ImageRoutesMixin: return read + def _read_own_category(self, sources, path): + """One file's own category, None, or OWN_CATEGORY_UNREAD (blocking). + + Keyed by ``image_category_key`` over the output sources, as Images + writes it. A failed read is logged and hides the category (DC5). + """ + try: + key = image_category_key(sources, path) + return None if key is None else self.db.image_category(*key) + except Exception: + self.logger.warning("Image category could not be read", exc_info=True) + return OWN_CATEGORY_UNREAD + def _inspection_library(self, saved: dict | None, *, unavailable: bool) -> dict: """The inspector's Library facts; the route sets the title afterwards.""" result = { diff --git a/py/api/inspection_service.py b/py/api/inspection_service.py index 5c8e3e9c..3a99af86 100644 --- a/py/api/inspection_service.py +++ b/py/api/inspection_service.py @@ -3,7 +3,7 @@ import os import stat from pathlib import Path -from typing import Callable +from typing import Any, Callable try: from ...utils.prompt_inspector import ( @@ -100,18 +100,23 @@ def inspect_current_file( saved: dict | None, library: dict, image_tags: Callable[[int, int], dict] | None = None, + recorded: Callable[[Path], Any] | None = None, ) -> dict: """Current-file evidence read between two equal stats of *path*. *image_tags*, when given, is called once with that stable ``(st_size, st_mtime_ns)`` and its facet is returned as ``image_tags`` (spec amendment S1: tags of a byte record verified at exactly that stat). + *recorded*, when given, is asked within the same stat window for what + recorded run history knows about *path* (``RecordedOutput`` or None: the + save node that wrote it and the texts PromptManager recorded, #386). """ before = _current_file_facts(path) try: inspection = read_inspection_file(path) except MetadataReadError: inspection = None + run = recorded(path) if recorded is not None and inspection is not None else None after = _current_file_facts(path) fields = ("st_dev", "st_ino", "st_mode", "st_size", "st_mtime_ns", "st_ctime_ns") if any(getattr(before, key) != getattr(after, key) for key in fields): @@ -129,9 +134,15 @@ def inspect_current_file( profile = ( PREVIEW_INSPECTION_PROFILE if _in_host_temp(path) else INSPECTION_PROFILE ) - # The file name anchors on the saver that wrote it (#381). + # The file name anchors on the saver that wrote it (#381); recorded run + # history names that saver when the file alone cannot (#386). evidence = inspect_prompt_evidence( - inspection.metadata, role_saved, profile=profile, file_name=path.name + inspection.metadata, + role_saved, + profile=profile, + file_name=path.name, + recorded_saver_id=run.saver_id if run is not None else None, + recorded_texts=run.texts if run is not None else None, ) return { "file": { diff --git a/py/api/jobs.py b/py/api/jobs.py index 39b74b36..de92fba8 100644 --- a/py/api/jobs.py +++ b/py/api/jobs.py @@ -47,6 +47,11 @@ import time import uuid from pathlib import PurePosixPath, PureWindowsPath +try: + from ...utils import exit_signal +except ImportError: # tests import the package top-level + from utils import exit_signal + logger = logging.getLogger(__name__) # Start and finish order: wall-clock times can tie (coarse Windows clocks). _sequence = itertools.count() @@ -195,11 +200,21 @@ class Job: self.counters = dict.fromkeys(groups, 0) self.outcomes = {group: [] for group in groups} self.log = [] - self.cancelled = False + self._cancel_requested = False self.last_event = None self.task = None self._subscribers = set() + @property + def cancelled(self): + """Cancel was requested, or the interpreter is exiting (#376): every + check a job already makes between files also stops it at exit.""" + return self._cancel_requested or exit_signal.requested() + + @cancelled.setter + def cancelled(self, value): + self._cancel_requested = bool(value) + @property def record_changing(self): return self.admission == "records" diff --git a/py/api/logging_routes.py b/py/api/logging_routes.py index 8918986c..ed13096a 100644 --- a/py/api/logging_routes.py +++ b/py/api/logging_routes.py @@ -11,6 +11,11 @@ import stat from aiohttp import web +try: + from ...utils.logging_config import ClearRefused, parse_confirmed +except ImportError: # loaded as the top-level ``py`` package (tests) + from utils.logging_config import ClearRefused, parse_confirmed + # Keys PromptManagerLogger.update_config() understands. Anything else is # rejected so a client cannot inject arbitrary entries into the logging config. LOG_CONFIG_KEYS = frozenset( @@ -263,10 +268,43 @@ class LoggingRoutesMixin: ) async def truncate_logs(self, request): - """Truncate all log files.""" + """Clear all logs: what ``{"expected": [...]}`` lists, or all (#389). + + ``expected`` holds the ``{name, identity}`` rows of ``/logs/files`` the + confirmation showed. Without a body (the legacy admin page, whose + confirmation lists nothing) every rotated copy present now is deleted. + A malformed list is 400; a clear that would empty or delete what was + not confirmed changes nothing and is 409 with the reason's ``code``. + """ try: + args = () + if request.can_read_body: + data, error_response = await self._read_json_body(request) + if error_response is not None: + return error_response + if "expected" in data: + try: + parse_confirmed(data["expected"]) + except ValueError: + return web.json_response( + { + "success": False, + "error": "expected must list name and identity" + " rows from /logs/files", + }, + status=400, + ) + args = (data["expected"],) logger_manager = self._get_logger_manager() - results = await self._run_in_executor(logger_manager.truncate_logs) + try: + results = await self._run_in_executor( + logger_manager.truncate_logs, *args + ) + except ClearRefused as refused: + return web.json_response( + {"success": False, "error": str(refused), "code": refused.code}, + status=409, + ) return web.json_response( { diff --git a/py/api/review.py b/py/api/review.py index 6f81e545..a07c02ce 100644 --- a/py/api/review.py +++ b/py/api/review.py @@ -677,7 +677,14 @@ def _verify_and_read(store, capture, item, root, need_metadata: bool): facts = store.verify(root, relative) captured = None if need_metadata: - captured = captured_review(store, capture, item_profile(item["evidence"])) + # The file's own name, so an inspection item resolves anchored on its + # saver as it was recorded (#390). + captured = captured_review( + store, + capture, + item_profile(item["evidence"]), + relative.rsplit("/", 1)[-1], + ) return relative, facts, captured @@ -723,6 +730,9 @@ def _candidate_still_offered(resolution: dict, choice: str) -> bool: that rewords the top-level reason or reorders candidates leaves a pending item applicable. """ + if isinstance(resolution, dict) and resolution.get("anchor") == "saver": + # Only the saver anchor resolves the bytes (#390): its pair alone. + resolution = {"state": "ambiguous", "candidates": [resolution]} if not isinstance(resolution, dict) or resolution.get("state") != "ambiguous": return False evidence, _count, limited = review_evidence(resolution) diff --git a/py/api/review_recording.py b/py/api/review_recording.py index 6de59641..141cf14a 100644 --- a/py/api/review_recording.py +++ b/py/api/review_recording.py @@ -88,7 +88,8 @@ class ReviewRecordingMixin: def _record_scanned_review(self, media_file, roots, review=None, scanned=None): """Record one ambiguous, unlinked scanned file (logged and counted). - ``scanned`` is the scan's ``(metadata, inspection_resolution)``. + ``scanned`` is the scan's ``(metadata, inspection_resolution)``; the + resolution is None when the scan did not run the inspection profile. """ try: outcome = self._record_review_sync( @@ -199,7 +200,7 @@ class ReviewRecordingMixin: key = _memory_key(locator) if state is not None and memory.get(key) == state: return None - resolution = review_capture.inspection_choice(canonical) + resolution = review_capture.inspection_choice(canonical, media_file.name) if resolution is not None: memory.pop(key, None) return canonical, resolution diff --git a/py/api/thumbnail_auto.py b/py/api/thumbnail_auto.py index bfe38315..3f425dd9 100644 --- a/py/api/thumbnail_auto.py +++ b/py/api/thumbnail_auto.py @@ -88,9 +88,14 @@ from .thumbnail_encode import shutdown_encoders, writer_settings from .thumbnail_jobs import ThumbnailOptions try: - from ...utils.parallel import map_parallel + from ...utils.parallel import map_parallel, note_refusal, refused_at_shutdown except ImportError: # tests import the package top-level - from utils.parallel import map_parallel + from utils.parallel import map_parallel, note_refusal, refused_at_shutdown + +try: + from ...utils.exit_signal import ExitRequested +except ImportError: # tests import the package top-level + from utils.exit_signal import ExitRequested # The placeholder frame a video without a preview is answered with (JPEG # bytes, never written to disk); written previews follow the settings. @@ -410,7 +415,8 @@ class AutoThumbnailer: def enqueue(self, src, dst, is_video): """Queue *dst*'s preview for the worker; False when already queued, - full or closed. Never blocks on a write.""" + full, closed or its worker is refused at interpreter exit. Never + blocks on a write.""" key = _dst_key(dst) with self._changed: if self._closed or key in self._queued: @@ -420,34 +426,47 @@ class AutoThumbnailer: self._queued.add(key) self._queue.append((key, Path(src), Path(dst), bool(is_video))) self._busy = True + started = True if self._start_worker and not self._worker_running: - self._start_worker_locked() + started = self._start_worker_locked() + if not started: + # Refused at interpreter exit: no worker will take it (#375). + self._queue.pop() + self._queued.discard(key) self._changed.notify_all() - return True + return started def _start_worker_locked(self): - """Start the worker (lock held). A start that fails leaves the file - queued and the next enqueue tries again.""" + """Start the worker (lock held); False only when the interpreter is + exiting (#375: logged once at DEBUG, the caller drops the file). + Another start failure leaves the file queued and the next enqueue + tries again.""" worker = threading.Thread(target=self._run, name=WORKER_NAME, daemon=True) worker.pm_preview_token = _MODULE_TOKEN worker.pm_preview_close = self.close self._worker_running = True try: worker.start() - except RuntimeError: + except RuntimeError as error: self._worker_running = False self._busy = False + if refused_at_shutdown(error): + note_refusal(self._logger, "The automatic preview worker") + return False self._logger.warning( "The automatic preview worker could not start; the next output " "tries again", exc_info=True, ) - return + return True self._worker = worker + return True def _run(self): try: self._drain() + except ExitRequested: + pass # #376: the interpreter is exiting; queued files are dropped except Exception: self._logger.warning("The automatic preview worker stopped", exc_info=True) finally: @@ -790,8 +809,8 @@ class AutoThumbnailMixin: Returns ``queued``, ``fresh`` (its preview is current) or ``skipped`` (not a media file under a current root, a link, a preview itself, - already queued, or after the app's cleanup). Never writes on the - caller's thread. + already queued, after the app's cleanup, or when its worker is refused + at interpreter exit). Never writes on the caller's thread. """ if self._auto_thumbnails_shut: return "skipped" diff --git a/py/autotag.py b/py/autotag.py index 8938ed5f..d002f626 100644 --- a/py/autotag.py +++ b/py/autotag.py @@ -7,6 +7,7 @@ Refactored from standalone_tagger.py for integration with PromptManager API. import errno import gc +import inspect import os import sys from pathlib import Path @@ -32,6 +33,54 @@ try: except ImportError: from py.tagging_defaults import DEFAULT_TAG_TEMPLATE +try: + from ..utils import exit_signal +except ImportError: + from utils import exit_signal + + +def abort_hub_downloads() -> None: + """Abort huggingface_hub's Xet transfers (run by the exit hook, #376). + + Xet downloads report progress from native code, which ignores an + exception raised in ``tqdm.update``; huggingface_hub aborts them itself + on Ctrl+C with ``abort_xet_session``. Nothing happens when + huggingface_hub was never imported or has no Xet support. + """ + if "huggingface_hub" not in sys.modules: + return + try: + from huggingface_hub.utils import _xet + except ImportError: + return + abort = getattr(_xet, "abort_xet_session", None) + if callable(abort): + abort() + + +def _stop_at_exit(download) -> Dict[str, Any]: + """Keyword arguments that stop a huggingface_hub download at its next + chunk once the interpreter is exiting (#376), or none when this + huggingface_hub does not take ``tqdm_class``. + + huggingface_hub 1.x removes its partial temp file on any failure; older + versions keep their resumable ``.incomplete`` file. The model's own path + is only written by the final rename either way. + """ + try: + if "tqdm_class" not in inspect.signature(download).parameters: + return {} + from huggingface_hub.utils import tqdm as hub_tqdm + except (ImportError, TypeError, ValueError): + return {} + + class StopAtExit(hub_tqdm): + def update(self, n=1): + exit_signal.checkpoint() + return super().update(n) + + return {"tqdm_class": StopAtExit} + def _autotag_config(): """The saved defaults; imported late, as config needs ComfyUI's server.""" @@ -397,6 +446,7 @@ class AutoTagService: filename=filename, local_dir=str(model_dir), local_dir_use_symlinks=False, + **_stop_at_exit(hf_hub_download), ) if progress_callback: @@ -541,6 +591,13 @@ class AutoTagService: else: return self._download_hf_model(progress_callback) except Exception as e: + if exit_signal.requested(): + # ExitRequested between chunks, or the Xet session aborted + # by the exit hook (abort_hub_downloads). + self.logger.info( + f"Model download stopped: ComfyUI is exiting ({model_type})" + ) + return False self.logger.error(f"Download failed: {e}") if progress_callback: progress_callback(f"Error: {str(e)}", 0) @@ -570,6 +627,7 @@ class AutoTagService: filename=config["filename"], local_dir=str(gguf_dir), local_dir_use_symlinks=False, + **_stop_at_exit(hf_hub_download), ) self.logger.info("GGUF model downloaded") @@ -587,6 +645,7 @@ class AutoTagService: filename=config["mmproj_filename"], local_dir=str(gguf_dir), local_dir_use_symlinks=False, + **_stop_at_exit(hf_hub_download), ) self.logger.info("mmproj downloaded") @@ -613,6 +672,7 @@ class AutoTagService: repo_id=config["repo"], local_dir=str(model_dir), local_dir_use_symlinks=False, + **_stop_at_exit(snapshot_download), ) self.logger.info("HF model downloaded") diff --git a/py/lora_import.py b/py/lora_import.py index 23f053cf..c906eb8d 100644 --- a/py/lora_import.py +++ b/py/lora_import.py @@ -44,8 +44,10 @@ from .lora_manager_layout import ( ) try: + from ..utils import exit_signal from ..utils.hashing import generate_prompt_hash except ImportError: # pragma: no cover - flat test imports + from utils import exit_signal from utils.hashing import generate_prompt_hash LORA_CATEGORY = "lora-manager" @@ -189,14 +191,15 @@ def stage( downloads its CivitAI example images. A sidecar that cannot be read, or whose record fails to build, is skipped by its name, as are the centralized sidecars discovery could not parse. ``cancelled()`` is asked - before each sidecar (raises ImportCancelled); ``on_record(index, + before each sidecar, and so is the exit signal (#376: the legacy scan's + ``cancelled`` does not read it); either raises ImportCancelled. ``on_record(index, record_or_None, staged)`` follows each one. """ sidecars = discovery.sidecars staged = StagedImport(total=len(sidecars)) staged.skipped.extend(discovery.unreadable) for index, sidecar in enumerate(sidecars): - if cancelled(): + if cancelled() or exit_signal.requested(): raise ImportCancelled() record = None path = sidecar.metadata_path diff --git a/requirements-dev.txt b/requirements-dev.txt index 43e2130a..6db215e2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,8 +8,8 @@ coverage>=7.16.2 numpy>=2.2.6 # WD14 engine tests # Code quality -black>=26.10.1 -flake8>=7.4.1 +black==26.10.1 +flake8==7.4.1 # Security scanning bandit[toml]>=1.9.4 diff --git a/tests/browser/images-concept.spec.mjs b/tests/browser/images-concept.spec.mjs index f8a45f44..fe69a751 100644 --- a/tests/browser/images-concept.spec.mjs +++ b/tests/browser/images-concept.spec.mjs @@ -289,7 +289,8 @@ const continuationText=page=>page.locator('.infinite-continuation > p').first(). async function searchOf(page,term) { const before=await countText(page); await page.locator('#search').fill(term); - await expect.poll(()=>countText(page)).not.toBe(before); + // Production's caption reads "Loading imagesโ€ฆ" until the search's listing answers (#391); the concept's is instant. + await expect.poll(async()=>{const text=await countText(page);return text!==before&&text!=='Loading imagesโ€ฆ';}).toBe(true); await expect.poll(()=>page.locator(CARDS).count()).toBeLessThan(FIXTURE_IMAGES); await settle(page); const result={count:await countText(page),continuation:await continuationText(page),cards:await page.locator(`${CARDS} .image-open`).evaluateAll(nodes=>nodes.map(node=>node.getAttribute('aria-label')))}; diff --git a/tests/browser/inspector-concept.spec.mjs b/tests/browser/inspector-concept.spec.mjs index 45c1b16a..a60bc9a6 100644 --- a/tests/browser/inspector-concept.spec.mjs +++ b/tests/browser/inspector-concept.spec.mjs @@ -248,7 +248,8 @@ async function productionSendOf(page,context) { async function tagOf(page) { const before=await page.locator('#result-count').textContent(); await panelOf(page).locator('.detail-inner > .tags').last().getByRole('button',{name:'cinematic',exact:true}).click(); - await expect.poll(()=>page.locator('#result-count').textContent()).not.toBe(before); + // Production's caption reads "Loading imagesโ€ฆ" until the tag's listing answers (#391); the concept's search is instant. + await expect.poll(async()=>{const text=await page.locator('#result-count').textContent();return text!==before&&text!=='Loading imagesโ€ฆ';}).toBe(true); await settle(page); return {count:await page.locator('#result-count').textContent(), cards:await page.locator(`${CARDS} .image-open`).evaluateAll(nodes=>nodes.map(node=>node.getAttribute('aria-label')))}; @@ -348,6 +349,25 @@ test('a recipe-linked image without recipe metadata shows its positive prompt me await expect(panelOf(page).locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Prompt history','Collectionsโ€ฆ']); }); +// #391: a tag chip replaces the collection; until its listing answers the caption says it is loading, never that the +// filter matched nothing ("0 of 0 images" was painted before the response and read by a fast reader). +test('a tag chip collection reads as loading, not empty, until its listing answers (#391)',async({page})=>{ + await page.setViewportSize({width:1600,height:1100}); + await openProductionPanel(page,'dark'); + let release,arrived; + const held=new Promise(resolve=>{release=resolve;}),reached=new Promise(resolve=>{arrived=resolve;}); + await page.route(url=>url.pathname==='/prompt_manager/images/output'&&url.searchParams.get('tags')==='cinematic', + async route=>{arrived();await held;await route.continue();},{times:1}); + const count=page.locator('#result-count'); + await panelOf(page).locator('.detail-inner > .tags').last().getByRole('button',{name:'cinematic',exact:true}).click(); + await reached; + await expect(count).toHaveText('Loading imagesโ€ฆ'); + await expect(page.locator(CARDS)).toHaveCount(0); + release(); + await expect(count).toHaveText(countCopy('8 of 8 images ยท 2 positive prompts')); + await expect(page.locator(CARDS)).toHaveCount(8); +}); + // Task C3 (D88, #249 PR C): the concept's chip row is the image's own tags (`i.tags`, app.js a402). With those names // saved as the file's current image tags, production draws the row from them (Tags filter buttons in the panel, plain // in the viewer, the linked positive's equal tags not repeated) and still equals the concept, in both sizes. diff --git a/tests/browser/legacy-autotag.spec.mjs b/tests/browser/legacy-autotag.spec.mjs index ff96f681..089dafd2 100644 --- a/tests/browser/legacy-autotag.spec.mjs +++ b/tests/browser/legacy-autotag.spec.mjs @@ -1,11 +1,9 @@ -// #328: the legacy admin and gallery auto-tag tools keep working beside System โ€บ Auto-tagging. The model download is -// the `model_download` Activity job; the pages' stream attaches to it. No model is ever downloaded: the fixture's -// fake download waits until the test releases it (`/__test__/tagging/download`). +// #328: the legacy admin and gallery auto-tag batch and review keep working beside System โ€บ Auto-tagging. Since #330 +// the pages download and unload models only through System (legacy-system-links.spec.mjs); a batch never downloads +// one. The fixture's fake download (`/__test__/tagging/download`) is held so nothing is ever fetched. import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; import { test, expect } from './fixtures.mjs'; -const TITLE = 'Tagging model download ยท WD14 ViT'; - async function post(request, url, data) { await expect(await request.post(url, data === undefined ? {} : { data })).toBeOK(); } @@ -33,61 +31,6 @@ test.afterEach(async ({ page, request, browserName }) => { await post(request, '/__test__/jobs/restart'); }); -async function startDownload(page, request) { - await page.locator('#autoTagBtn').click(); - const status = page.locator('#wd14VitModelStatus'); - await status.getByRole('button', { name: 'Download', exact: true }).click(); - const modal = page.locator('#autoTagDownloadModal'); - await expect(modal).toBeVisible(); - await expect(page.locator('#downloadStatus')).toHaveText('Downloadingโ€ฆ'); - await expect(page.locator('#downloadProgressBar')).toHaveClass(/animate-pulse/); - await expect(modal).toContainText('The download continues in Activity.'); - await expect(page.locator('#cancelDownloadBtn')).toHaveText('Close'); - const running = (await downloads(request)).filter(job => job.state === 'running'); - expect(running.map(job => [job.title, job.cancellable])).toEqual([[TITLE, false]]); - return { modal, status }; -} - -test('admin download modal closes on complete and Activity lists the job without Cancel', async ({ page, request }) => { - await page.goto('/prompt_manager/admin'); - const { modal, status } = await startDownload(page, request); - - await page.goto('/prompt_manager/workspace#/activity'); - const item = page.locator('#workspace-main section.surface > .activity-item') - .filter({ has: page.getByRole('link', { name: new RegExp(`${TITLE}$`) }) }); - await expect(item.locator('.chip')).toHaveText('Running'); - await expect(item.locator('progress')).not.toHaveAttribute('value'); - await expect(item.getByRole('button', { name: 'Cancel job', exact: true })).toHaveCount(0); - - // Back on the admin page a second download of the same model attaches to the running job. - await page.goto('/prompt_manager/admin'); - await startDownload(page, request); - await post(request, '/__test__/tagging/download', { mode: 'succeed' }); - await expect(modal).toBeHidden(); - await expect(status).toContainText('Downloaded'); - expect((await downloads(request)).map(job => [job.title, job.state])).toEqual([[TITLE, 'finished']]); -}); - -test('gallery download modal closes on complete', async ({ page, request }) => { - await page.goto('/prompt_manager/gallery'); - const { modal, status } = await startDownload(page, request); - await post(request, '/__test__/tagging/download', { mode: 'succeed' }); - await expect(modal).toBeHidden(); - await expect(page.locator('#autoTagModal')).toBeVisible(); - await expect(status).toContainText('Downloaded'); - expect((await downloads(request)).map(job => job.state)).toEqual(['finished']); -}); - -test('a failed download closes the modal and says so', async ({ page, request }) => { - await page.goto('/prompt_manager/gallery'); - const { modal, status } = await startDownload(page, request); - await post(request, '/__test__/tagging/download', { mode: 'fail' }); - await expect(modal).toBeHidden(); - await expect(page.getByText('Download failed: Download failed')).toBeVisible(); - await expect(status.getByRole('button', { name: 'Download', exact: true })).toBeVisible(); - expect((await downloads(request)).map(job => job.state)).toEqual(['failed']); -}); - // (wd14-vit is not downloaded in these tests: beforeEach marks it so the download tests can offer it.) // The gallery's batch reader keyed on `data.status` values the server never sends, so its modal never moved or // closed (#328 Problem 5). It now reads `type` frames as the admin page does. @@ -171,25 +114,6 @@ test('admin review opens the first image', async ({ page }) => { await expect(page.getByText('Failed to start review mode')).toHaveCount(0); }); -test('gallery unload works and is refused while the gate is held', async ({ page, request }) => { - await page.goto('/prompt_manager/gallery'); - await runGalleryBatch(page); // keeps the model in memory - await page.locator('#autoTagBtn').click(); - const unload = page.locator('#unloadModelBtn'); - await expect(unload).toBeVisible(); - - // A held download owns the model gate: Unload is refused with the server's error. - const held = await request.post('/prompt_manager/jobs/model-download', { data: { model_type: 'gguf' } }); - expect(held.status()).toBe(202); - await unload.click(); - await expect(page.getByText(/^Failed to unload model: A tagging model is busy/)).toBeVisible(); - - await post(request, '/__test__/tagging/download', { mode: 'succeed' }); - await expect.poll(async () => (await downloads(request)).map(job => job.state)).toEqual(['finished']); - await unload.click(); - await expect(page.getByText('WD14-SWINV2 model unloaded successfully')).toBeVisible(); -}); - /** The admin review lists every row, videos too: Skip any image that is not one until the confirmation shows. */ async function untilAutoTagged(page, confirm) { // Each image's skip check answers before its confirmation or its error shows. diff --git a/tests/browser/legacy-system-links.spec.mjs b/tests/browser/legacy-system-links.spec.mjs new file mode 100644 index 00000000..37fe9e9e --- /dev/null +++ b/tests/browser/legacy-system-links.spec.mjs @@ -0,0 +1,286 @@ +// System S13 (#330): the legacy admin and gallery pages open native System for their System tools. Each replaced +// control is a plain link (`workspace#/system/`, a new tab), and the legacy flows S13 keeps still work. +import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; +import { test, expect } from './fixtures.mjs'; + +const PAGES = Object.freeze({ admin: '/prompt_manager/admin', gallery: '/prompt_manager/gallery' }); +const LABELS = Object.freeze({ + overview: 'Overview', + appearance: 'Appearance', + sources: 'Sources & scans', + integrations: 'Integrations', + tagging: 'Auto-tagging', + maintenance: 'Maintenance', + backups: 'Backup & export', + diagnostics: 'Diagnostics & logs', +}); + +async function openPageOptions(page) { + await page.locator('#pageOptionsBtn').click(); + await expect(page.locator('#settingsModal')).toBeVisible(); +} + +/** The auto-tag modal with its model rows read from /autotag/models. */ +async function openAutoTag(page) { + const read = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/models')); + await page.locator('#autoTagBtn').click(); + await read; + await expect(page.locator('#autoTagModal')).toBeVisible(); +} + +const notDownloaded = (page, request) => post(request, '/__test__/tagging/models', { unavailable: ['gguf'] }); +const modelLoaded = (page, request) => post(request, '/__test__/tagging/loaded', { model: 'wd14-vit' }); + +// One row per System link of the design's ยง2 map: the page, the control's label, how to find it, the section it opens +// and what a test must arrange before the control shows. +const LINKS = Object.freeze([ + { page: 'admin', label: 'Settings', locate: page => page.locator('#settingsBtn'), section: 'overview' }, + { page: 'admin', label: 'Maintenance', locate: page => page.locator('#maintenanceBtn'), section: 'maintenance' }, + { page: 'admin', label: 'Backup DB', locate: page => page.locator('#backupBtn'), section: 'backups' }, + { page: 'admin', label: 'Restore DB', locate: page => page.locator('#restoreBtn'), section: 'backups' }, + { page: 'gallery', label: 'Settings', locate: page => page.locator('#settingsBtn'), section: 'appearance' }, + { page: 'admin', label: 'Manage in System (model row)', arrange: notDownloaded, reveal: openAutoTag, + locate: page => page.locator('#ggufModelStatus a'), section: 'tagging' }, + { page: 'gallery', label: 'Manage in System (model row)', arrange: notDownloaded, reveal: openAutoTag, + locate: page => page.locator('#ggufModelStatus a'), section: 'tagging' }, + { page: 'gallery', label: 'Manage in System (Unload Model)', arrange: modelLoaded, reveal: openAutoTag, + locate: page => page.locator('#unloadModelBtn'), section: 'tagging' }, + { page: 'gallery', label: 'Page options Maintenance', reveal: openPageOptions, + locate: page => page.locator('#pageOptionsMaintenanceLink'), section: 'maintenance' }, + { page: 'gallery', label: 'Page options Sources & scans', reveal: openPageOptions, + locate: page => page.locator('#pageOptionsSourcesLink'), section: 'sources' }, + { page: 'admin', label: 'Diagnostics', locate: page => page.locator('#diagnosticsBtn'), section: 'diagnostics' }, + { page: 'admin', label: 'View Logs', locate: page => page.locator('#logsBtn'), section: 'diagnostics' }, +]); + +async function post(request, url, data) { + await expect(await request.post(url, data === undefined ? {} : { data })).toBeOK(); +} + +test.beforeEach(async ({ page, request, browserName }) => { + await startCoverage(page, browserName); + await post(request, '/__test__/reset'); + await post(request, '/__test__/system/reset'); + await post(request, '/__test__/tagging/reset'); + // The 3.2.0 update notice would cover the admin page's toolbar. + await page.addInitScript(() => { localStorage.setItem('pm_update_notice_dismissed', '3.2.0'); }); + page.legacyErrors = []; + page.on('pageerror', error => page.legacyErrors.push(error.message)); +}); + +test.afterEach(async ({ page, request, browserName }) => { + await saveCoverage(page, browserName); + await post(request, '/__test__/tagging/reset'); + await post(request, '/__test__/system/reset'); + expect(page.legacyErrors).toEqual([]); +}); + +/** Opens the legacy page with the link showing; returns the link. */ +async function openLink(page, request, link) { + if (link.arrange) await link.arrange(page, request); + await page.goto(PAGES[link.page]); + if (link.reveal) await link.reveal(page); + const control = link.locate(page); + await expect(control).toBeVisible(); + return control; +} + +// Each row opens its section in a new tab and sends no request from the legacy page (the design's `a System link sends +// no request from the legacy page`, checked per row so one slow page never times out the whole table). +for (const link of LINKS) { + test(`legacy ${link.page} ${link.label} opens System ${link.section}`, async ({ page, context, request }) => { + // The gallery's thumbnail startup check (K9) reads its status on a timer of its own; it is off here. + await page.addInitScript(() => { + localStorage.setItem('gallerySettings', JSON.stringify({ checkThumbnailsAtStartup: false })); + }); + const control = await openLink(page, request, link); + const legacyUrl = page.url(); + await expect(control).toHaveAttribute('href', `workspace#/system/${link.section}`); + await expect(control).toHaveAttribute('target', '_blank'); + await expect(control).toHaveAttribute('rel', 'noopener'); + // The page's own start-up reads (list, film strips, settings) finish first; then the click must add none. + // Lazy thumbnails may still load whenever the layout changes; they are not the click's. + await page.waitForLoadState('networkidle'); + const sent = []; + const record = item => { if (item.resourceType() !== 'image') sent.push(`${item.method()} ${item.url()}`); }; + page.on('request', record); + const opened = context.waitForEvent('page'); + await control.click(); + const system = await opened; + await expect(system).toHaveURL(new RegExp(`/prompt_manager/workspace#/system/${link.section}$`)); + page.off('request', record); + expect(sent, 'a System link sends no request from the legacy page').toEqual([]); + // System's page heading is "System"; the open section is the pressed button of its section switcher. + const main = system.locator('#workspace-main'); + await expect(main.getByRole('heading', { level: 1, name: 'System', exact: true })).toBeVisible(); + await expect(main.getByRole('group', { name: 'System sections', exact: true }) + .getByRole('button', { name: LABELS[link.section], exact: true })).toHaveAttribute('aria-pressed', 'true'); + expect(page.url()).toBe(legacyUrl); + await system.close(); + }); +} + +test('kept admin flows still work', async ({ page, context }) => { + await page.goto(PAGES.admin); + const texts = page.locator('#resultsList .prompt-text'); + await expect(texts.first()).toBeVisible(); + + // K1: sorting reorders the list. + for (const [value, order] of [['text_asc', 1], ['text_desc', -1]]) { + const listed = page.waitForResponse(item => /\/prompt_manager\/(recent|search)\?/.test(item.url()) && item.url().includes(`sort=${value}`)); + await page.locator('#sortBy').selectOption(value); + await listed; + await expect.poll(async () => { + const shown = await texts.allTextContents(); + // The server sorts case-insensitively (SQLite NOCASE), not by locale. + const key = text => text.toLowerCase(); + const sorted = [...shown].sort((a, b) => order * (key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0)); + return shown.length > 1 && shown.join('\n') === sorted.join('\n'); + }).toBe(true); + } + // K3: Add Prompt saves a prompt the list then shows. + await page.locator('#addPromptBtn').click(); + await page.locator('#addPromptText').fill('Kept flow lighthouse'); + const saved = page.waitForResponse(item => item.request().method() === 'POST' && item.url().endsWith('/prompt_manager/save')); + await page.locator('#saveNewPromptBtn').click(); + expect((await saved).ok()).toBe(true); + await expect(page.locator('#addPromptModal')).toBeHidden(); + await expect(texts.filter({ hasText: 'Kept flow lighthouse' })).toBeVisible(); + + // K1: a search narrows the list, and a selection enables the bulk buttons. + const all = await texts.allTextContents(); + const word = all[0].trim().split(/\s+/)[0]; + const searched = page.waitForResponse(item => item.url().includes('/prompt_manager/search?') && item.url().includes('text=')); + await page.locator('#searchText').fill(word); + await page.locator('#searchBtn').click(); + await searched; + await expect.poll(async () => { + const shown = await texts.allTextContents(); + return shown.length > 0 && shown.length < all.length && shown.every(text => text.toLowerCase().includes(word.toLowerCase())); + }).toBe(true); + await expect(page.locator('#bulkDeleteBtn')).toBeDisabled(); + await page.locator('#resultsList .prompt-checkbox').first().check(); + for (const id of ['#bulkDeleteBtn', '#bulkTagBtn', '#bulkCategoryBtn']) await expect(page.locator(id)).toBeEnabled(); + + // K4: Stats reads the statistics again; Export downloads the prompts. + const stats = page.waitForResponse(item => item.url().endsWith('/prompt_manager/stats') && item.ok()); + await page.locator('#statsBtn').click(); + await stats; + await expect(page.getByText('Stats refreshed successfully!')).toBeVisible(); + const download = page.waitForEvent('download'); + await page.locator('#exportBtn').click(); + expect((await download).suggestedFilename()).toMatch(/^prompts-\d{4}-\d{2}-\d{2}\.json$/); + + // K5: Scan Images opens its dialog (with the quick backup) without starting a scan. + await page.locator('#scanBtn').click(); + await expect(page.locator('#scanModal')).toBeVisible(); + await expect(page.locator('#quickBackupBtn')).toBeVisible(); + await page.locator('#cancelScan').click(); + await expect(page.locator('#scanModal')).toBeHidden(); + + // K7: Metadata and Gallery open their pages in a new tab. + for (const [id, path] of [['#metadataBtn', '/prompt_manager/gallery.html'], ['#galleryBtn', '/prompt_manager/gallery']]) { + const opened = context.waitForEvent('page'); + await page.locator(id).click(); + const tab = await opened; + await tab.waitForLoadState('domcontentloaded'); + expect(new URL(tab.url()).pathname).toBe(path); + await tab.close(); + } +}); + +test('Page options keeps the page-only settings admin', async ({ page, request }) => { + const before = (await (await request.get('/prompt_manager/settings')).json()).settings; + await page.goto(PAGES.admin); + await expect(page.locator('#resultsList .prompt-text').first()).toBeVisible(); + await page.locator('#pageOptionsBtn').click(); + const dialog = page.locator('#settingsModal'); + await expect(dialog).toBeVisible(); + await expect(dialog.locator('h3')).toHaveText('Page options'); + // Exactly the page-only setting (G5): the list's Infinite Scrolling default. + expect(await dialog.locator('input, select, textarea').evaluateAll(fields => fields.map(field => field.id))) + .toEqual(['infiniteScrollDefault']); + await expect(page.locator('#infiniteScrollDefault')).not.toBeChecked(); + await page.locator('#infiniteScrollDefault').check(); + const saved = page.waitForRequest(item => item.method() === 'POST' && item.url().endsWith('/prompt_manager/settings')); + await page.locator('#saveSettings').click(); + expect((await saved).postDataJSON()).toEqual({ infinite_scroll: true }); + await expect(dialog).toBeHidden(); + // The list header follows the saved default at once; nothing else changed. + await expect(page.locator('#infiniteScrollSession')).toBeChecked(); + const after = (await (await request.get('/prompt_manager/settings')).json()).settings; + expect(after).toEqual({ ...before, infinite_scroll: true }); + await page.reload(); + // The dialog shows the saved default once the page has read it (the header follows it first). + await expect(page.locator('#infiniteScrollSession')).toBeChecked(); + await page.locator('#pageOptionsBtn').click(); + await expect(page.locator('#infiniteScrollDefault')).toBeChecked(); +}); + +test('Page options keeps the page-only settings gallery', async ({ page }) => { + // Native Appearance keys and the startup prompt's own keys are in the shared object before the page opens. + const stored = { imageQuality: 'high', showImageInfo: false, videoMute: false, + autoLoadMetadata: false, workspaceLayout: 'masonry', thumbnailPromptDismissed: 'sig', thumbnailsGenerated: true, + checkThumbnailsAtStartup: false, defaultViewMode: 'list' }; + await page.addInitScript(value => { + if (!sessionStorage.getItem('seeded')) { + sessionStorage.setItem('seeded', '1'); + localStorage.setItem('gallerySettings', value); + } + }, JSON.stringify(stored)); + const listed = page.waitForResponse(item => item.url().includes('/prompt_manager/images/output?')); + await page.goto(PAGES.gallery); + await listed; + // With no Page options control, a stored default view still opens the page in it. + await expect(page.locator('#listViewBtn')).toHaveClass(/bg-pm-accent/); + await openPageOptions(page); + const dialog = page.locator('#settingsModal'); + await expect(dialog.locator('h3')).toHaveText('Page options'); + // Exactly the page-only settings (G5); the default view is System โ€บ Appearance's Default layout. + expect(await dialog.locator('input, select, textarea').evaluateAll(fields => fields.map(field => field.id))).toEqual([ + 'checkThumbnailsAtStartup', 'lazyLoadingToggle', 'defaultLimitSelect', 'gridColumnsSelect', + 'infiniteScrollToggle', 'debugModeToggle', 'apiTimeoutInput']); + await expect(dialog.getByText('Default View Mode')).toHaveCount(0); + await page.locator('#defaultLimitSelect').selectOption('50'); + await page.locator('#debugModeToggle').check(); + await page.locator('#saveSettingsBtn').click(); + await expect(dialog).toBeHidden(); + const read = () => page.evaluate(() => JSON.parse(localStorage.getItem('gallerySettings'))); + let saved = await read(); + expect(saved).toMatchObject({ ...stored, defaultLimit: 50, debugMode: true }); + + // Reset puts back only the page options' defaults; the Appearance, startup-prompt and default-view keys stay. + await openPageOptions(page); + page.once('dialog', dialog => dialog.accept()); + await page.locator('#resetSettingsBtn').click(); + await expect(page.getByText('Page options reset to defaults')).toBeVisible(); + await expect(page.locator('#defaultLimitSelect')).toHaveValue('100'); + saved = await read(); + expect(saved).toMatchObject({ defaultViewMode: 'list', imageQuality: 'high', showImageInfo: false, videoMute: false, + autoLoadMetadata: false, workspaceLayout: 'masonry', thumbnailPromptDismissed: 'sig', thumbnailsGenerated: true, + defaultLimit: 100, debugMode: false, checkThumbnailsAtStartup: true, lazyLoading: true, gridColumns: 8, + infiniteScroll: false, apiTimeout: 30 }); +}); + +test('the legacy auto-tag modal stays open after Manage in System', async ({ page, context, request }) => { + for (const legacy of ['admin', 'gallery']) { + await post(request, '/__test__/tagging/models', { unavailable: ['gguf'] }); + await page.goto(PAGES[legacy]); + await openAutoTag(page); + const link = page.locator('#ggufModelStatus').getByRole('link', { name: 'Manage in System', exact: true }); + await expect(link).toHaveAttribute('title', 'Opens System โ€บ Auto-tagging in a new tab'); + // A downloaded model keeps its status text. + await expect(page.locator('#wd14VitModelStatus')).toHaveText('\u2713 Downloaded'); + const opened = context.waitForEvent('page'); + await link.click(); + const system = await opened; + await expect(system).toHaveURL(/#\/system\/tagging$/); + await expect(page.locator('#autoTagModal')).toBeVisible(); + await system.close(); + // Downloaded in System meanwhile: reopening the modal reads the models again (CUJ 4). + await post(request, '/__test__/tagging/models', { unavailable: [] }); + await page.goto(PAGES[legacy]); + await openAutoTag(page); + await expect(page.locator('#ggufModelStatus')).toHaveText('\u2713 Downloaded'); + } +}); diff --git a/tests/browser/server.py b/tests/browser/server.py index 0054a55b..eb135b43 100644 --- a/tests/browser/server.py +++ b/tests/browser/server.py @@ -272,6 +272,7 @@ def create_app(root: Path) -> web.Application: "media_image_tags", "media_annotation_bytes", "media_annotation_files", + "image_categories", "image_prompt_pairs", "prompt_pair_tags", "prompt_pairs", diff --git a/tests/browser/unseen-restore.mjs b/tests/browser/unseen-restore.mjs new file mode 100644 index 00000000..aabd8962 --- /dev/null +++ b/tests/browser/unseen-restore.mjs @@ -0,0 +1,54 @@ +// A restore the open workspace page has not noticed yet (#413). +// +// Specs that restore through the API while a page is open check what the page does with a restore it did not see: +// its next write is refused as "Prompt storage was restored since this dialog opened". But the page's Activity store +// reads the job list every 15 s (activity-store.mjs POLL_MS), and a read the server answers while the restore job runs +// makes the page follow it: when it finishes, the page is shown again from the restored data (S9, index.mjs +// onRestoreFinished), which closes any open dialog. On a slow runner a restore can take longer than the time left to +// the next read (CI run 40646112668: the restore took 16 s), so whether the page saw it depended on timing. +// +// `armUnseenRestores(page)`, before the page loads, passes every job-list read of the page through one route, so it +// knows which reads the server has not answered yet. `unseenRestore(page, restoring)` then closes the gate, waits +// for those reads' answers (sent before the restore began, they cannot list it), runs `restoring()`, and only after +// it has answered lets the held reads go on: they read the restore as already ended, as a tab that read its job list +// just before and just after the restore does. A restore that is never seen running never reloads the page. +const JOBS = '**/prompt_manager/jobs'; +const gates = new WeakMap(); + +/** Route `page`'s job-list reads through the restore gate. Call before the page's first navigation. */ +export async function armUnseenRestores(page) { + const gate = { closed: null, inFlight: 0, idle: [] }; + gates.set(page, gate); + await page.route(JOBS, async route => { + if (gate.closed) { + await gate.closed; + return route.fallback(); + } + gate.inFlight += 1; + try { + const response = await route.fetch(); + await route.fulfill({ response }).catch(() => {}); + } catch { + await route.abort().catch(() => {}); + } finally { + gate.inFlight -= 1; + if (gate.inFlight === 0) for (const resolve of gate.idle.splice(0)) resolve(); + } + }); +} + +/** Run `restoring()` (an API restore) unseen by `page`'s Activity store; resolves with what it resolves with. */ +export async function unseenRestore(page, restoring) { + const gate = gates.get(page); + if (!gate) throw new Error('armUnseenRestores(page) must run before the page loads'); + let open; + gate.closed = new Promise(resolve => { open = resolve; }); + try { + // Reads already on their way were sent before the restore: their answers cannot list it. + if (gate.inFlight > 0) await new Promise(resolve => gate.idle.push(resolve)); + return await restoring(); + } finally { + gate.closed = null; + open(); + } +} diff --git a/tests/browser/workflows.spec.mjs b/tests/browser/workflows.spec.mjs index e822d998..849839b1 100644 --- a/tests/browser/workflows.spec.mjs +++ b/tests/browser/workflows.spec.mjs @@ -104,33 +104,6 @@ for (const leave of ['beforeunload', 'pagehide']) { }); } -// #222: the gallery's Clear cache clears the server's thumbnails by its own request and keeps this browser's -// preferences and other tools' storage; nothing of Prompt Manager's is cached in a browser, and the toast says so. -test('the gallery cache clear keeps preferences and other tools caches', async ({ page }) => { - await page.addInitScript(() => { - if (sessionStorage.getItem('seeded')) return; - sessionStorage.setItem('seeded', '1'); - localStorage.setItem('pm_workspace_theme', 'light'); - localStorage.setItem('other-tool-setting', 'keep'); - localStorage.setItem('pm_other_tool', 'keep'); - }); - await page.goto('/prompt_manager/gallery'); - await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2); - await page.evaluate(async () => { await (await caches.open('other-tool-gallery-thumbnails')).put('/x', new Response('x')); }); - const clears = []; - page.on('request', req => { if (req.url().includes('/clear-thumbnails')) clears.push(req.method()); }); - page.once('dialog', dialog => dialog.accept()); - await page.locator('#settingsBtn').click(); - await page.locator('#clearCacheBtn').click(); - await expect(page.getByText("Thumbnails cleared; original images untouched. Nothing of Prompt Manager's is cached in this browser.")) - .toBeVisible(); - expect(clears).toEqual(['POST']); - expect(await page.evaluate(async () => ({ - theme: localStorage.getItem('pm_workspace_theme'), other: localStorage.getItem('other-tool-setting'), - prefixed: localStorage.getItem('pm_other_tool'), notice: localStorage.getItem('pm_update_notice_dismissed'), - caches: await caches.keys(), - }))).toEqual({ theme: 'light', other: 'keep', prefixed: 'keep', notice: '3.2.0', caches: ['other-tool-gallery-thumbnails'] }); -}); // #379: the reviewed tags are that image's own tags; no prompt changes. test('actual gallery review generates and persists tags for the selected root\'s image', async ({ page, request }) => { @@ -226,7 +199,7 @@ test('gallery infinite scroll appends a real backend page while retaining existi await expect(await request.post('/__test__/reset', { data: { extra_images: 108 } })).toBeOK(); await page.goto('/prompt_manager/gallery'); await expect(page.locator('#galleryGrid .image-item')).toHaveCount(100); - await page.locator('#settingsBtn').click(); + await page.locator('#pageOptionsBtn').click(); await page.locator('#infiniteScrollToggle').check(); await page.locator('#saveSettingsBtn').click(); await expect(page.locator('#settingsModal')).toBeHidden(); @@ -246,37 +219,6 @@ async function promptSnapshot(request) { })).sort((a, b) => a.id - b.id); } -async function downloadBackup(page) { - const downloading = page.waitForEvent('download'); - await page.locator('#backupBtn').click(); - const download = await downloading; - expect(download.suggestedFilename()).toMatch(/^prompts_backup_.*\.db$/); - expect(await download.failure()).toBeNull(); - const chunks = []; - for await (const chunk of await download.createReadStream()) chunks.push(chunk); - const buffer = Buffer.concat(chunks); - expect(buffer.subarray(0, 16).toString()).toBe('SQLite format 3\0'); - return { name: download.suggestedFilename(), mimeType: 'application/octet-stream', buffer }; -} - -async function chooseRestore(page, file) { - await page.locator('#restoreBtn').click(); - await expect(page.locator('#restoreModal')).toBeVisible(); - await page.locator('#restoreFileInput').setInputFiles(file); -} - -// Confirms the chosen restore and waits for the server's answer, not for a timer: a restore is disk-bound and can take -// longer than an expect timeout on a slow runner (#359). The modal closes only after that answer. -async function confirmRestoreAndWait(page) { - const restored = page.waitForResponse(response => response.url().endsWith('/prompt_manager/restore')); - await page.locator('#confirmRestore').click(); - const response = await restored; - const body = await response.text(); - expect(response.status(), body).toBe(200); - expect(JSON.parse(body)).toMatchObject({ success: true, prompt_count: 2, backup_created: expect.any(String) }); - await expect(page.locator('#restoreModal')).toBeHidden(); -} - function confirmNextDialog(page, accept, message) { page.once('dialog', async dialog => { expect(dialog.type()).toBe('confirm'); @@ -292,88 +234,6 @@ async function deletePromptButton(page, text) { return page.locator(`[data-action="delete"][data-prompt-id="${id}"]`); } -test('downloaded backup restores deleted prompts, tags, and image links through the actual upload form', async ({ page, request }) => { - await page.goto('/prompt_manager/admin'); - await expect(page.locator('.prompt-text')).toHaveCount(2); - const before = await promptSnapshot(request); - expect(before.map(prompt => prompt.tags)).toEqual([['cats'], ['dogs']]); - expect(before.every(prompt => prompt.images.length === 1)).toBe(true); - const backup = await downloadBackup(page); - const button = await deletePromptButton(page, 'Cat portrait'); - confirmNextDialog(page, true, 'delete this prompt'); - await button.click(); - await expect(page.locator('.prompt-text')).toHaveCount(1); - await expect(await request.post('/prompt_manager/save', { data: { text: 'Added after backup', tags: ['temporary'] } })).toBeOK(); - expect(await promptSnapshot(request)).not.toEqual(before); - await chooseRestore(page, backup); - confirmNextDialog(page, true, backup.name); - await confirmRestoreAndWait(page); - await expect(page.locator('.prompt-text')).toHaveCount(2); - await page.reload(); - await expect(page.locator('.prompt-text').filter({ hasText: 'Cat portrait' })).toBeVisible(); - await expect(page.locator('.prompt-text').filter({ hasText: 'Added after backup' })).toHaveCount(0); - expect(await promptSnapshot(request)).toEqual(before); - await page.goto('/prompt_manager/admin#/tags'); - await page.locator('[data-tag-name="cats"]').click(); - await expect(page.locator('#tagPromptsGrid')).toContainText('Cat portrait'); - await page.locator('#tagPromptsGrid img').first().click(); - await expect(page.locator('.viewer-canvas img')).toBeVisible(); - await expect.poll(() => page.locator('.viewer-canvas img').evaluate(image => image.complete && image.naturalWidth > 0)).toBe(true); -}); - -test('restore validation and cancellation preserve live data and allow a valid retry', async ({ page, request }) => { - await page.goto('/prompt_manager/admin'); - const before = await promptSnapshot(request); - const backup = await downloadBackup(page); - let uploads = 0; - page.on('request', request => { if (request.url().endsWith('/prompt_manager/restore')) uploads++; }); - await page.locator('#restoreBtn').click(); - await page.locator('#confirmRestore').click(); - await expect(page.getByText('Please select a database file to restore', { exact: true })).toBeVisible(); - await page.locator('#restoreFileInput').setInputFiles({ ...backup, name: 'backup.txt' }); - await page.locator('#confirmRestore').click(); - await expect(page.getByText('Please select a valid .db file', { exact: true })).toBeVisible(); - expect(uploads).toBe(0); - await page.locator('#restoreFileInput').setInputFiles(backup); - confirmNextDialog(page, false, 'replace your current database'); - await page.locator('#confirmRestore').click(); - await expect(page.locator('#restoreModal')).toBeVisible(); - expect(uploads).toBe(0); - await page.locator('#restoreFileInput').setInputFiles({ name: 'corrupt.db', mimeType: 'application/octet-stream', buffer: Buffer.from('not a SQLite database') }); - confirmNextDialog(page, true, 'corrupt.db'); - const rejected = page.waitForResponse(response => response.url().endsWith('/prompt_manager/restore')); - await page.locator('#confirmRestore').click(); - expect((await rejected).status()).toBe(400); - await expect(page.getByText(/Restore failed:/)).toBeVisible(); - await expect(page.locator('#restoreModal')).toBeVisible(); - await expect(page.locator('#confirmRestore')).toBeEnabled(); - expect(await promptSnapshot(request)).toEqual(before); - await page.locator('#restoreFileInput').setInputFiles(backup); - confirmNextDialog(page, true, backup.name); - await confirmRestoreAndWait(page); - expect(await promptSnapshot(request)).toEqual(before); -}); - -test('backup and restore recover from interrupted network requests without losing data', async ({ page, request }) => { - await page.goto('/prompt_manager/admin'); - const before = await promptSnapshot(request); - await page.route('**/prompt_manager/backup', route => route.abort('failed'), { times: 1 }); - await page.locator('#backupBtn').click(); - await expect(page.getByText(/Backup failed:/)).toBeVisible(); - const backup = await downloadBackup(page); - await chooseRestore(page, backup); - await page.route('**/prompt_manager/restore', route => route.abort('failed'), { times: 1 }); - confirmNextDialog(page, true, backup.name); - await page.locator('#confirmRestore').click(); - await expect(page.getByText(/Restore failed:/)).toBeVisible(); - await expect(page.locator('#confirmRestore')).toBeEnabled(); - await expect(page.locator('#restoreModal')).toBeVisible(); - expect(await promptSnapshot(request)).toEqual(before); - confirmNextDialog(page, true, backup.name); - await confirmRestoreAndWait(page); - expect(await promptSnapshot(request)).toEqual(before); -}); - test('prompt deletion requires confirmation and can retry after a network failure', async ({ page, request }) => { await page.goto('/prompt_manager/admin'); const before = await promptSnapshot(request); @@ -401,98 +261,3 @@ test('prompt deletion requires confirmation and can retry after a network failur await expect(await request.get(url)).toBeOK(); } }); - -// The duplicate scan reads the output root only, but the fixture's input root holds a different -// shared/same.png. A bare /images/serve/ URL tries every gallery root in turn, so once the -// output copy is deleted it answers 200 with the input copy: check each file in its own root. -const serveIn = (root, path) => `/prompt_manager/images/serve/${path.split(/[\\/]/).map(encodeURIComponent).join('/')}?root=${root}`; -const rootOf = url => new URL(url, 'http://gallery.test').searchParams.get('root'); - -// Which copy the scan keeps depends on file times: the fixture stamps shared/same.png with the -// wall clock, and the kernel can stamp its bulk copies a fraction of a millisecond earlier. -// listOrder rewrites the finished scan's groups so a test can pick the order it needs. -async function openDuplicateScan(page, request, listOrder = null) { - await expect(await request.post('/__test__/reset', { data: { extra_images: 2 } })).toBeOK(); - if (listOrder) { - await page.route('**/prompt_manager/scan_duplicates/stream', async route => { - const response = await route.fetch(); - const body = (await response.text()).replace(/^data: (.*)$/gm, (line, json) => { - const event = JSON.parse(json); - if (event.type !== 'complete') return line; - for (const group of event.duplicates) group.images = listOrder(group.images); - return `data: ${JSON.stringify(event)}`; - }); - await route.fulfill({ response, body }); - }); - } - await page.goto('/prompt_manager/gallery'); - await expect(page.locator('#galleryGrid .image-item')).toHaveCount(4); - const originals = await page.locator('#galleryGrid img[data-original]').evaluateAll(images => images.map(image => image.dataset.original)); - const output = rootOf(originals.find(url => url.includes('/bulk/'))); - const input = rootOf(originals.find(url => url.includes('/shared/same.png') && rootOf(url) !== output)); - expect(output).not.toBeNull(); - expect(input).not.toBeNull(); - await page.locator('#settingsBtn').click(); - await page.locator('#scanDuplicatesBtn').click(); - await expect(page.locator('#duplicatesCount')).toHaveText('1'); - const candidates = page.locator('.duplicate-checkbox'); - await expect(candidates).toHaveCount(2); - return { originals, output, input, candidates }; -} - -async function expectOnlySelectedGone(page, request, { originals, output, input }, selected, retained) { - expect((await request.get(serveIn(output, selected))).status()).toBe(404); - await expect(await request.get(serveIn(output, retained))).toBeOK(); - await expect(await request.get(serveIn(input, 'shared/same.png'))).toBeOK(); - await page.reload(); - await expect(page.locator('#galleryGrid .image-item')).toHaveCount(3); - const remaining = await page.locator('#galleryGrid img[data-original]').evaluateAll(images => images.map(image => image.dataset.original)); - expect(remaining).toHaveLength(originals.length - 1); - for (const url of remaining) { - expect(originals).toContain(url); - await expect(await request.get(url)).toBeOK(); - } -} - -test('duplicate cleanup cancels safely and deletes only the selected file after a retry', async ({ page, request }) => { - const scan = await openDuplicateScan(page, request); - const { output, candidates } = scan; - const selected = await candidates.first().getAttribute('data-image-path'); - const retained = await candidates.last().getAttribute('data-image-path'); - await candidates.first().check(); - let deletes = 0; - page.on('request', request => { if (request.url().endsWith('/delete_duplicate_images')) deletes++; }); - confirmNextDialog(page, false, 'permanently delete 1 duplicate image files'); - await page.locator('#removeDuplicatesBtn').click(); - expect(deletes).toBe(0); - await expect(await request.get(serveIn(output, selected))).toBeOK(); - await page.route('**/prompt_manager/delete_duplicate_images', route => route.abort('failed'), { times: 1 }); - confirmNextDialog(page, true, 'permanently delete 1 duplicate image files'); - await page.locator('#removeDuplicatesBtn').click(); - await expect(page.getByText('Failed to delete duplicate images', { exact: true })).toBeVisible(); - await expect(page.locator('#removeDuplicatesBtn')).toBeEnabled(); - await expect(await request.get(serveIn(output, selected))).toBeOK(); - confirmNextDialog(page, true, 'permanently delete 1 duplicate image files'); - const deleted = page.waitForResponse(response => response.url().endsWith('/delete_duplicate_images')); - await page.locator('#removeDuplicatesBtn').click(); - expect((await (await deleted).json())).toMatchObject({ success: true, deleted_count: 1, failed_count: 0 }); - await expect(page.locator('#duplicatesModal')).toBeHidden(); - await expectOnlySelectedGone(page, request, scan, selected, retained); -}); - -test('deleting a duplicate whose path another gallery root also holds removes only the scanned copy', async ({ page, request }) => { - // The order CI saw: a bulk copy stamped first, so shared/same.png is the first Delete candidate. - const ciOrder = ['bulk/0000.png', 'shared/same.png', 'bulk/0001.png']; - const rank = image => ciOrder.indexOf(image.path.replace(/\\/g, '/')); - const scan = await openDuplicateScan(page, request, images => [...images].sort((a, b) => rank(a) - rank(b))); - const selected = await scan.candidates.first().getAttribute('data-image-path'); - const retained = await scan.candidates.last().getAttribute('data-image-path'); - expect(selected.replace(/\\/g, '/')).toBe('shared/same.png'); - await scan.candidates.first().check(); - confirmNextDialog(page, true, 'permanently delete 1 duplicate image files'); - const deleted = page.waitForResponse(response => response.url().endsWith('/delete_duplicate_images')); - await page.locator('#removeDuplicatesBtn').click(); - expect((await (await deleted).json())).toMatchObject({ success: true, deleted_count: 1, failed_count: 0 }); - await expect(page.locator('#duplicatesModal')).toBeHidden(); - await expectOnlySelectedGone(page, request, scan, selected, retained); -}); diff --git a/tests/browser/workspace-history.spec.mjs b/tests/browser/workspace-history.spec.mjs index 48f81074..f912a892 100644 --- a/tests/browser/workspace-history.spec.mjs +++ b/tests/browser/workspace-history.spec.mjs @@ -1,5 +1,6 @@ import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; import {test,expect} from './fixtures.mjs'; +import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs'; import {readFile,writeFile} from 'node:fs/promises'; import {createHash} from 'node:crypto'; @@ -16,13 +17,13 @@ const marker=(page,id,anchor='10')=>recorded(page).locator(`button[data-run-id=" const exact=async locator=>locator.evaluate(node=>node.textContent); test.beforeEach(async({page,request,browserName})=>{ - await startCoverage(page,browserName);page.historyErrors=[];page.on('pageerror',error=>page.historyErrors.push(error.message)); + await startCoverage(page,browserName);await armUnseenRestores(page);page.historyErrors=[];page.on('pageerror',error=>page.historyErrors.push(error.message)); const reset=await request.post('/__test__/reset',{data:{chronological_history:true}});await expect(reset).toBeOK();page.fixture=await reset.json(); }); test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.historyErrors).toEqual([]);}); const command=async(request,data)=>{const response=await request.post('/__test__/history/commands',{data});await expect(response).toBeOK();return response.json();}; -const restoreSnapshot=async(request,bytes)=>{const response=await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'history.db',mimeType:'application/octet-stream',buffer:bytes}}});await expect(response).toBeOK();expect((await response.json()).success).toBe(true);}; +const restoreSnapshot=async(page,request,bytes)=>{const response=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'history.db',mimeType:'application/octet-stream',buffer:bytes}}}));await expect(response).toBeOK();expect((await response.json()).success).toBe(true);}; const historyRead=async(request,id)=>{const response=await request.get(`/prompt_manager/library/${id}/history`);await expect(response).toBeOK();return response.json();}; const frozenContexts=contexts=>contexts.map(({navigation,positive_prompt_id,negative_prompt_id,pair_id,...frozen})=>frozen); const immutableReceipts=items=>items.map(({current_candidate,...receipt})=>receipt); @@ -62,7 +63,7 @@ for(const boundary of ['window','detail','candidate','inspection','media']) test await command(request,{action:'arm',boundary});const pending=request.get(target); try { await expect.poll(async()=> (await(await request.get('/__test__/history/checkpoint')).json()).entered).toBe(true); - await restoreSnapshot(request,bytes); + await restoreSnapshot(page,request,bytes); } finally {await command(request,{action:'release'});} const retired=await pending;expect(retired.status()).toBe(409);expect((await retired.json()).code).toBe('history-changed'); for(const url of [`/prompt_manager/library/${f.detail_id}/history?limit=1&cursor=${encodeURIComponent(firstWindow.next_cursor)}`,`${base}?epoch=${epoch}`,`${base}/outputs?epoch=${epoch}`,`${base}/outputs?epoch=${epoch}&cursor=${encodeURIComponent(outputs.next_cursor)}`,media.url,media.thumbnail_url,media.inspection_url]) { @@ -130,7 +131,7 @@ test('archive delete merge and prune preserve frozen evidence without repairing test('actual restore with reused numeric owner rejects stale history and offers explicit saved recovery',async({page,request})=>{ const f=page.fixture;await open(page,f.detail_id);const original=await exact(saved(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text')); - const source=await request.get('/__test__/history/reused-snapshot');await expect(source).toBeOK();await restoreSnapshot(request,await source.body()); + const source=await request.get('/__test__/history/reused-snapshot');await expect(source).toBeOK();await restoreSnapshot(page,request,await source.body()); await marker(page,f.runs.shared).click();await expect(dialog(page)).toContainText('Recorded evidence changed');await page.keyboard.press('Escape'); await expect(page.locator('.library-parent-status')).toContainText('changed');expect(await exact(saved(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'))).toBe(original); await expect(recorded(page).getByRole('button',{name:'Reload recorded history',exact:true})).toBeVisible(); @@ -142,12 +143,12 @@ for(const outcome of ['success','error'])test(`delayed actual old browser ${outc const f=page.fixture;await open(page,f.detail_id);const snapshot=await request.get('/__test__/history/snapshot');const bytes=await snapshot.body(); let entered,release;const started=new Promise(resolve=>{entered=resolve;}),held=new Promise(resolve=>{release=resolve;}); await page.route(new RegExp(`/history/runs/${f.runs.shared}\\?`),async route=>{ - if(outcome==='error')await restoreSnapshot(request,bytes); + if(outcome==='error')await restoreSnapshot(page,request,bytes); const response=await route.fetch();expect(response.status()).toBe(outcome==='success'?200:409);entered();await held; await route.fulfill({response}).catch(()=>{}); }); try { - await marker(page,f.runs.shared).click();await started;if(outcome==='success')await restoreSnapshot(request,bytes); + await marker(page,f.runs.shared).click();await started;if(outcome==='success')await restoreSnapshot(page,request,bytes); await page.evaluate(()=>{location.hash='#/overview';});await expect(page.locator('#workspace-main h1')).toHaveText('Overview');release(); await page.unroute(new RegExp(`/history/runs/${f.runs.shared}\\?`));await open(page,f.detail_id); await expect(recorded(page).locator('button[data-run-id]')).toHaveCount(0); @@ -258,7 +259,7 @@ test('inspection first discovers a real restore and immediately retires every su if(changed)window.recordedRecoveryStates.push({receipts:run.querySelectorAll('[data-receipt]').length,active:[...run.querySelectorAll('[data-send-role],[data-copy-role],[data-view-receipt]')].filter(node=>!node.disabled&&node.checkVisibility()).length}); });observer.observe(run,{subtree:true,childList:true,attributes:true,characterData:true}); }); - await restoreSnapshot(request,await snapshot.body()); + await restoreSnapshot(page,request,await snapshot.body()); } finally {await command(request,{action:'release'});} const response=await inspectionRead;expect(response.status()).toBe(409);expect((await response.json()).code).toBe('history-changed'); await expect(dialog(page).locator('.recorded-run-status')).toContainText('Recorded evidence changed'); diff --git a/tests/browser/workspace-library-editor.spec.mjs b/tests/browser/workspace-library-editor.spec.mjs index cac57771..2cc85b25 100644 --- a/tests/browser/workspace-library-editor.spec.mjs +++ b/tests/browser/workspace-library-editor.spec.mjs @@ -1,6 +1,7 @@ import {writeFile} from 'node:fs/promises'; import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; import {test,expect} from './fixtures.mjs'; +import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs'; @@ -13,7 +14,7 @@ const RATINGS=[['5','5 / 5'],['4','4 / 5'],['3','3 / 5'],['2','2 / 5'],['1','1 / const editorDialog=page=>page.locator('dialog.library-editor'); test.beforeEach(async({page,request,browserName})=>{ - await startCoverage(page,browserName); + await startCoverage(page,browserName);await armUnseenRestores(page); const reset=await request.post('/__test__/library-editor/reset',{data:{}});await expect(reset).toBeOK();page.fixture=await reset.json(); page.editorErrors=[];page.on('pageerror',error=>page.editorErrors.push(error.message)); page.writes=[];page.on('request',item=>{if(item.method()!=='GET')page.writes.push({method:item.method(),url:new URL(item.url()).pathname,body:item.postDataJSON()});}); @@ -260,7 +261,8 @@ test('recovery states render inline and never replay a write',async({page,reques page.writes.length=0; dialog=await openEditor(page,{mode:'new'});await ready(dialog); const snapshot=await request.get('/__test__/library-editor/snapshot');await expect(snapshot).toBeOK(); - const restore=await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:await snapshot.body()}}}); + const buffer=await snapshot.body(); + const restore=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer}}})); await expect(restore).toBeOK();page.writes.length=0; await field(dialog,'Title').fill('After restore');await field(dialog,'Positive prompt').fill('Restored storage prompt');await button(dialog,'Save').click(); await expect(note(dialog)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was saved.'); @@ -588,7 +590,7 @@ test('saved refresh failure retries only reads',async({page})=>{ test('restored storage after a confirmed save or before an archive offers an explicit Library reload',async({page,request})=>{ const f=page.fixture;const snapshot=await(await request.get('/__test__/library-editor/snapshot')).body(); - const restore=()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}}); + const restore=()=>unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}})); await openHistory(page,f.positive_id); await page.route(`**/prompt_manager/library/${f.positive_id}`,async route=>{ if(route.request().method()!=='PATCH')return route.continue(); @@ -615,7 +617,7 @@ test('restored storage after a confirmed save or before an archive offers an exp const DISCARDS='Leaving this dialog discards your entries.'; test('restored storage before an Edit or Add negative save offers only the Library reload exit',async({page,request})=>{ const f=page.fixture;const snapshot=await(await request.get('/__test__/library-editor/snapshot')).body(); - const restore=()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}}); + const restore=()=>unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}})); await openHistory(page,f.positive_id); await page.getByRole('button',{name:'Edit metadata',exact:true}).click();let dialog=editorDialog(page);await ready(dialog); await expect(await restore()).toBeOK(); diff --git a/tests/browser/workspace-system-backups.spec.mjs b/tests/browser/workspace-system-backups.spec.mjs index 6856490f..edc369d1 100644 --- a/tests/browser/workspace-system-backups.spec.mjs +++ b/tests/browser/workspace-system-backups.spec.mjs @@ -47,6 +47,17 @@ async function serverJson(request, url) { return response.json(); } +/** + * The restore job's own end (#361): its progress stream closes when the job takes its final state, however long the + * disk-bound publish takes (bounded by the test, not by a 5 s expect), and the job finished. The toast the page shows + * for it is then the page's reaction alone, checked with the default timeout. + */ +async function restoreFinished(request, id) { + await expect(await request.get(`/prompt_manager/jobs/${id}/events`)).toBeOK(); + const { job } = await serverJson(request, `/prompt_manager/jobs/${id}`); + expect([job.kind, job.state]).toEqual(['restore', 'finished']); +} + async function openBackups(page) { // A fresh page each time: the drafts live for the page session (Q10), and a hash-only goto would not reload. if (page.url().endsWith('#/system/backups')) await page.reload(); @@ -216,6 +227,7 @@ test('Restore starts the job, opens Activity; views reload after it finishes', a const { job } = await (await applied).json(); expect([job.kind, job.title, job.cancellable]).toEqual(['restore', 'Database restore', false]); await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`); + await restoreFinished(request, job.id); await expect(toast(page)).toHaveText(/^Database restore finished ยท Restored \d+ prompts?, /); // The library is the backup's again, and the page says so without a reload: the sidebar counts were read again. expect(await libraryTotal(request)).toBe(restoredTotal); @@ -244,7 +256,8 @@ test('a finished restore shows the open page again from the restored data', asyn const card = page.locator('#workspace-main article.prompt-card').filter({ hasText: 'S9 prompt the restore removes' }); await expect(card).toHaveCount(1); await expect(await request.post('/__test__/release')).toBeOK(); - await applied; + const { job } = await (await applied).json(); + await restoreFinished(request, job.id); await expect(toast(page)).toHaveText(/^Database restore finished ยท /); // The shelf was read again from the restored library, on the same route. await expect(card).toHaveCount(0); @@ -382,14 +395,15 @@ test('a failed settings read leaves the policy disabled with the reason', async await expect(row.locator('input')).toBeDisabled(); }); -test('the legacy admin Backup and Restore keep working', async ({ page, request }) => { +test('the legacy Backup and Restore routes keep working and the admin page links to System', async ({ page, request }) => { const bytes = await backupBytes(request); const restored = await request.post('/prompt_manager/restore', { multipart: { database_file: { name: 'prompts_backup.db', mimeType: 'application/octet-stream', buffer: bytes } } }); await expect(restored).toBeOK(); expect((await restored.json()).success).toBe(true); + // #330: the admin page's Backup DB and Restore DB open System โ€บ Backup & export. await page.goto('/prompt_manager/admin'); - await expect(page.locator('#restoreFileInput')).toHaveCount(1); + for (const id of ['#backupBtn', '#restoreBtn']) await expect(page.locator(id)).toHaveAttribute('href', 'workspace#/system/backups'); }); /** Start a fake running job (a metadata scan another tab started), which an exclusive restore must wait for. */ @@ -422,6 +436,7 @@ test('a refused restore reopens its review: Restore retries it, Discard releases await dialog(page).getByRole('button', { name: 'Restore', exact: true }).click(); const { job } = await (await applied).json(); await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`); + await restoreFinished(request, job.id); await expect(toast(page)).toHaveText(/^Database restore finished ยท /); expect(await stagesOf(request)).toBe(0); }); diff --git a/tests/browser/workspace-system-diagnostics.spec.mjs b/tests/browser/workspace-system-diagnostics.spec.mjs index c0bcb963..9f811362 100644 --- a/tests/browser/workspace-system-diagnostics.spec.mjs +++ b/tests/browser/workspace-system-diagnostics.spec.mjs @@ -516,7 +516,7 @@ test('a rollover after the review deletes nothing and Activity offers Review aga test('Clear all logs states both counts, confirms first and reads everything again', async ({ page, request }) => { await seedLogs(request, LOG_FOLDER); const truncates = []; - page.on('request', item => { if (item.url().endsWith('/prompt_manager/logs/truncate')) truncates.push(item.url()); }); + page.on('request', item => { if (item.url().endsWith('/prompt_manager/logs/truncate')) truncates.push(item.postDataJSON()); }); await openDiagnostics(page); await expect(lines(page)).toContainText('Database backup verified'); const details = await openFiles(page); @@ -535,6 +535,10 @@ test('Clear all logs states both counts, confirms first and reads everything aga await dialog(page).getByRole('button', { name: 'Clear', exact: true }).click(); await expect(toast(page)).toHaveText('Logs cleared ยท 3 rotated copies deleted.'); expect(truncates).toHaveLength(1); + // #389: Clear sends back exactly the files the dialog was built from, each with its identity string. + expect(truncates[0].expected.map(row => row.name).sort()).toEqual(['prompt_manager.log', 'prompt_manager.log.1', + 'prompt_manager.log.2', 'prompt_manager.log.3']); + for (const row of truncates[0].expected) expect(row.identity).toMatch(/^\d+:\d+:\d+$/); // Read again: the viewer, the file line and the list. await expect(lines(page)).not.toContainText('Database backup verified'); await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 0$/); @@ -564,6 +568,23 @@ test('Clear all logs reads the files again before it states what it deletes', as expect(await names(request)).toEqual(['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3']); }); +test('a rollover while Clear all logs is open deletes nothing and says why (#389)', async ({ page, request }) => { + await seedLogs(request, { rotated: [1024, 1024], active: 100 }); + await openDiagnostics(page); + await openFiles(page); + await operation(page, 'Clear all logs').getByRole('button', { name: 'Clear all logsโ€ฆ', exact: true }).click(); + await expect(dialog(page).locator('#modal-body > p')).toHaveText(/^Empties prompt_manager\.log, deletes 2 rotated copies /); + // The log rolls over after the confirmation was built: the copies it listed are not the ones there now. + await expect(await request.post('/__test__/logs/rotate')).toBeOK(); + const rolled = await names(request); + const answer = page.waitForResponse(response => response.url().endsWith('/prompt_manager/logs/truncate')); + await dialog(page).getByRole('button', { name: 'Clear', exact: true }).click(); + expect((await answer).status()).toBe(409); + await expect(toast(page)).toHaveText('The log files changed since you confirmed; nothing was cleared. ' + + 'Read the list again and clear again.'); + expect(await names(request)).toEqual(rolled); +}); + test('each listed file downloads by its returned name and a link has no Download', async ({ page, request }) => { await seedLogs(request, LOG_FOLDER); await openDiagnostics(page); diff --git a/tests/browser/workspace-vocabulary.spec.mjs b/tests/browser/workspace-vocabulary.spec.mjs index 44248694..b9b82da2 100644 --- a/tests/browser/workspace-vocabulary.spec.mjs +++ b/tests/browser/workspace-vocabulary.spec.mjs @@ -1,6 +1,7 @@ import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; import {writeFile} from 'node:fs/promises'; import {test,expect} from './fixtures.mjs'; +import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs'; import {captureCanonicalState} from './history-canonical-comparison.mjs'; import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs'; @@ -40,7 +41,7 @@ async function expectLibrary(page,{tags='',category='',count}) { let manifest; test.beforeEach(async({page,request,browserName})=>{ - await startCoverage(page,browserName);manifest=await seed(request); + await startCoverage(page,browserName);await armUnseenRestores(page);manifest=await seed(request); page.vocabularyErrors=[];page.on('pageerror',error=>page.vocabularyErrors.push(error.message)); }); test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.vocabularyErrors).toEqual([]);}); @@ -264,7 +265,8 @@ const toast=page=>page.locator('#workspace-toast'); const actionNames=scope=>scope.locator('.dialog-actions button').evaluateAll(nodes=>nodes.map(node=>node.textContent.trim())); const snapshot=async request=>{const response=await request.get('/__test__/vocabulary/snapshot');await expect(response).toBeOK();return response.json();}; const backup=async request=>{const response=await request.get('/__test__/vocabulary/backup');await expect(response).toBeOK();return response.body();}; -const restore=async(request,buffer)=>expect(await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'vocabulary.db',mimeType:'application/octet-stream',buffer}}})).toBeOK(); +// A restore the open page has not seen, so its dialog stays to report it (#413). +const restore=async(page,request,buffer)=>expect(await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'vocabulary.db',mimeType:'application/octet-stream',buffer}}}))).toBeOK(); const tagNames=rows=>rows.tags.map(([,name])=>name); function trackWrites(page) { const writes=[];page.on('request',item=>{if(item.method()!=='GET')writes.push({method:item.method(),url:new URL(item.url()).pathname,body:item.postDataJSON()});});return writes; @@ -482,7 +484,7 @@ test('conflict and restore states keep the dialog and write nothing',async({page const before=await snapshot(request); box=await openDialog(page,main(page).getByRole('button',{name:'+ Create tag',exact:true}),'Create tag'); - await restore(request,saved); + await restore(page,request,saved); await field(box,'Tag name').fill('sunrise');await button(box,'Apply').click(); await expect(note(box)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was changed.'); await expect(button(box,'Apply')).toBeDisabled(); @@ -622,7 +624,7 @@ test('delete confirmation read failure, missing tag and restore keep Delete disa await expect(button(box,'Delete')).toBeDisabled();await expect(tagRow(page,'calm').locator('td:nth-child(2)')).toHaveText('2'); await button(box,'Cancel').click(); - box=await openRename(page,'mist');await restore(request,saved); + box=await openRename(page,'mist');await restore(page,request,saved); await button(box,'Delete tagโ€ฆ').click(); await expect(note(box)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was changed.'); await expect(button(box,'Delete')).toBeDisabled();expect(await actionNames(box)).toEqual(['Reload','Cancel','Delete']); diff --git a/tests/browser/workspace_concept_fixture.py b/tests/browser/workspace_concept_fixture.py index 55da0ee0..44d77ca3 100644 --- a/tests/browser/workspace_concept_fixture.py +++ b/tests/browser/workspace_concept_fixture.py @@ -183,6 +183,7 @@ _TABLES = ( "media_image_tags", "media_annotation_bytes", "media_annotation_files", + "image_categories", "image_prompt_pairs", "prompt_pair_tags", "prompt_pairs", diff --git a/tests/browser/workspace_diagnostics_fixture.py b/tests/browser/workspace_diagnostics_fixture.py index b55cf343..bdbc1193 100644 --- a/tests/browser/workspace_diagnostics_fixture.py +++ b/tests/browser/workspace_diagnostics_fixture.py @@ -123,6 +123,34 @@ def _touch(path: Path, size: int, moment: datetime.datetime) -> None: os.utime(path, (stamp, stamp)) +def _append_active(manager, log_dir: Path, size: int) -> None: + """Grow the active log as this process's handler does (#389). + + A second handle appending to the file the handler writes is what Clear + all logs refuses as another program writing to it, so the bytes go + through the handler's own stream; with file logging off, to the file. + """ + + def same(path) -> bool: + return os.path.normcase(os.path.realpath(path)) == os.path.normcase( + os.path.realpath(log_dir / ACTIVE) + ) + + for handler in manager._file_handlers(): + stream = getattr(handler, "stream", None) + if stream is None or not same(handler.baseFilename): + continue + handler.acquire() + try: + stream.write("x" * size) + stream.flush() + finally: + handler.release() + return + with open(log_dir / ACTIVE, "ab") as handle: + handle.write(b"x" * size) + + def _bounded(value, name, top=MAX_SEED_BYTES): if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= top: raise web.HTTPBadRequest(text=f"{name} must be an integer 0..{top}") @@ -169,8 +197,7 @@ def register_diagnostics_commands(app, routes, api, root: Path): for index, (time_text, level, message) in enumerate(CONCEPT_RECORDS): record = _record(time_text, level, message, index) manager.add_to_buffer(record, record.getMessage()) - with open(log_dir / ACTIVE, "ab") as handle: - handle.write(b"x" * CONCEPT_ACTIVE) + _append_active(manager, log_dir, CONCEPT_ACTIVE) for number, size in enumerate(CONCEPT_ROTATED, start=1): moment = ROTATED_TIME - datetime.timedelta(hours=number) _touch(log_dir / f"{ACTIVE}.{number}", size, moment) @@ -194,8 +221,7 @@ def register_diagnostics_commands(app, routes, api, root: Path): manager.add_to_buffer(record, record.getMessage()) active = _bounded(data.get("active", 0), "active") if active: - with open(log_dir / ACTIVE, "ab") as handle: - handle.write(b"x" * active) + _append_active(manager, log_dir, active) rotated = data.get("rotated", []) if not isinstance(rotated, list) or len(rotated) > 20: raise web.HTTPBadRequest(text="rotated must be a list of sizes") diff --git a/tests/browser/workspace_library_editor_fixture.py b/tests/browser/workspace_library_editor_fixture.py index 6f2f7626..25d06b08 100644 --- a/tests/browser/workspace_library_editor_fixture.py +++ b/tests/browser/workspace_library_editor_fixture.py @@ -35,6 +35,7 @@ _TABLES = ( "media_image_tags", "media_annotation_bytes", "media_annotation_files", + "image_categories", "image_prompt_pairs", "prompt_pair_tags", "prompt_pairs", diff --git a/tests/browser/workspace_review_fixture.py b/tests/browser/workspace_review_fixture.py index 66d7d9c7..da5847be 100644 --- a/tests/browser/workspace_review_fixture.py +++ b/tests/browser/workspace_review_fixture.py @@ -87,6 +87,7 @@ _TABLES = ( "media_image_tags", "media_annotation_bytes", "media_annotation_files", + "image_categories", "image_prompt_pairs", "prompt_pair_tags", "prompt_pairs", diff --git a/tests/browser/workspace_system_fixture.py b/tests/browser/workspace_system_fixture.py index 9900abc8..c7d87af3 100644 --- a/tests/browser/workspace_system_fixture.py +++ b/tests/browser/workspace_system_fixture.py @@ -35,7 +35,8 @@ read. These per-server hooks only change what a System test needs on top: previews for Convert outdated; - ``POST /__test__/system/jxl-encoder`` ``{available}`` (#317) makes the server read as having (``cjxl``) or lacking a JPEG XL encoder without - probing; reset restores the thumbnail settings and the real detection. + probing, whatever the host has installed (#371: ``false`` stands in for the + detector itself); reset restores the thumbnail settings and the real detection. - ``POST /__test__/system/duplicate-prompts`` (S7) adds two prompts whose text differs only in case and spacing (a duplicate prompt text group); @@ -118,6 +119,11 @@ def register_system_commands(app, routes, api, root: Path): } from py.api import thumbnail_encode + real_detect = thumbnail_encode._detect + + def _no_encoder(): + return None + monitor = getattr(image_monitor, "_monitor_instance", None) failures: dict[str, int] = {} @@ -147,6 +153,7 @@ def register_system_commands(app, routes, api, root: Path): api._health_report = None for (owner, name), value in host_values.items(): setattr(owner, name, value) + thumbnail_encode._detect = real_detect thumbnail_encode.reset_encoders() image_monitor._monitor_instance = monitor api.__dict__.pop("_library_resolver", None) @@ -298,7 +305,12 @@ def register_system_commands(app, routes, api, root: Path): available = data.get("available") if not isinstance(available, bool): raise web.HTTPBadRequest() - # Test-only: stand in the cached detection (no probe). + # Test-only: the fixture, not the host, decides (#371). Without an encoder the + # detector itself stands in, so a re-probe (a "none found" is probed again on + # every settings read) cannot find the host's real encoder; with one the cached + # answer stands, and a failed write still probes the real detector as before. + # Reset restores the real detector. + thumbnail_encode._detect = real_detect if available else _no_encoder with thumbnail_encode._detect_lock: thumbnail_encode._detected = "cjxl" if available else None return web.json_response({"success": True}) diff --git a/tests/browser/workspace_tagging_fixture.py b/tests/browser/workspace_tagging_fixture.py index fa8ac0e0..ab1d5240 100644 --- a/tests/browser/workspace_tagging_fixture.py +++ b/tests/browser/workspace_tagging_fixture.py @@ -53,6 +53,7 @@ _TABLES = ( "media_image_tags", "media_annotation_bytes", "media_annotation_files", + "image_categories", "image_prompt_pairs", "prompt_pair_tags", "prompt_pairs", @@ -249,6 +250,22 @@ def register_tagging_commands( model.unavailable = set(data["unavailable"]) return web.json_response({"success": True}) + @routes.post("/__test__/tagging/loaded") + async def loaded(request: web.Request) -> web.Response: + """Mark one fake model loaded (``{"model": ""}``), as a batch that + kept it in memory leaves it; ``/autotag/models`` then reports + ``model_loaded``. Reset unloads it.""" + data = await request.json() if request.can_read_body else None + if ( + request.query + or not isinstance(data, dict) + or set(data) != {"model"} + or data["model"] not in MODELS + ): + raise web.HTTPBadRequest() + model.loaded = data["model"] + return web.json_response({"success": True}) + @routes.post("/__test__/tagging/download") async def download_mode(request: web.Request) -> web.Response: """Switch the model's download to a fake (never a real download). diff --git a/tests/browser/workspace_visual_fixture.py b/tests/browser/workspace_visual_fixture.py index 64ff6729..728e509d 100644 --- a/tests/browser/workspace_visual_fixture.py +++ b/tests/browser/workspace_visual_fixture.py @@ -121,6 +121,7 @@ def seed_workspace_visual(api: PromptManagerAPI, root: Path) -> None: "media_image_tags", "media_annotation_bytes", "media_annotation_files", + "image_categories", "image_prompt_pairs", "prompt_pair_tags", "prompt_pairs", diff --git a/tests/browser/workspace_vocabulary_fixture.py b/tests/browser/workspace_vocabulary_fixture.py index b4fc7ec1..32fb801c 100644 --- a/tests/browser/workspace_vocabulary_fixture.py +++ b/tests/browser/workspace_vocabulary_fixture.py @@ -88,6 +88,7 @@ _TABLES = ( "media_image_tags", "media_annotation_bytes", "media_annotation_files", + "image_categories", "image_prompt_pairs", "prompt_pair_tags", "prompt_pairs", diff --git a/tests/comfyui/execution-text-link.spec.mjs b/tests/comfyui/execution-text-link.spec.mjs index 8eefb6da..20e60de6 100644 --- a/tests/comfyui/execution-text-link.spec.mjs +++ b/tests/comfyui/execution-text-link.spec.mjs @@ -11,7 +11,7 @@ import { installHostHooks } from './host-fixture.mjs'; // The worker's owned host must be up before the shared host hooks open the page. test.beforeEach(async ({ execution }) => { expect(execution.databasePath).toBeTruthy(); }); -installHostHooks(test, ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerPair', 'PMHistoryInputs']); +installHostHooks(test, ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerPair', 'PMHistoryInputs', 'PMCycledText']); // `primitive`: a core string node the queue-time resolver follows. `legacy`: the frontend's virtual Primitive node, // whose value is queued as literal text. `preview`: PrimitiveString -> PreviewAny, the shape of the maintainer's @@ -37,6 +37,7 @@ function build(page, { kind, positive, negative, prefix }) { primitive.widgets[0].value = side.text; return; } + if (side.via === 'cycled') { connect(add('PMCycledText', 0, y), 0, prompt, name); return; } const text = add('PrimitiveString', 0, y); text.widgets.find(widget => widget.name === 'value').value = side.text; let source = text; @@ -75,9 +76,9 @@ function build(page, { kind, positive, negative, prefix }) { }, { kind, positive, negative, prefix }); } -async function run(execution, page, testCase) { +async function run(execution, page, testCase, built) { const prefix = `textlink/${randomUUID()}`; - const { output, workflow, promptNode } = await build(page, { ...testCase, prefix }); + const { output, workflow, promptNode } = built ?? await build(page, { ...testCase, prefix }); const id = randomUUID(); const response = await execution.request.post('/prompt', { data: { prompt: output, prompt_id: id, extra_data: { extra_pnginfo: { workflow } } } }); expect(response.ok(), await response.text()).toBe(true); @@ -193,3 +194,164 @@ for (const [label, linked, negativeVia] of [['positive', ['text'], 'primitive'], await expectImagesLinked(execution, evidence, sides.text.via === 'preview'); }); } + +// #347: a cached PromptManager node counts every job, as an executing one does. Jobs after the first change only +// the sampler seed, so ComfyUI serves the PromptManager node (and its PreviewAny source) from its output cache. +const COUNTS = `import json, sqlite3, sys, pathlib +db = sqlite3.connect(pathlib.Path(sys.argv[1]).resolve().as_uri() + "?mode=ro", uri=True) +one = lambda sql, key: (db.execute(sql, (int(key),)).fetchone() or [None])[0] if key != "-" else None +print(json.dumps({"positive": one("SELECT run_count FROM prompts WHERE id=?", sys.argv[2]), + "negative": one("SELECT run_count FROM prompts WHERE id=?", sys.argv[3]), + "pair": one("SELECT run_count FROM prompt_pairs WHERE id=?", sys.argv[4])}))`; +async function runCounts(execution, context) { + const key = value => value == null ? '-' : String(value); + const { stdout } = await promisify(execFile)(process.env.PM_TEST_PYTHON || 'python', + ['-c', COUNTS, execution.databasePath, key(context.positive_prompt_id), key(context.negative_prompt_id), key(context.pair_id)], { timeout: 5000 }); + return JSON.parse(stdout); +} + +async function setCache(execution, mode) { + expect((await execution.request.post('/pm_execution_fixture/cache', { data: { mode } })).ok()).toBe(true); +} + +// Queue an already built workflow again with another sampler seed; returns the job's history item and cached nodes. +async function queueAgain(execution, first, { seed, edit, targets, status = 'success' } = {}) { + const output = structuredClone(first.output); + const sampler = Object.values(output).find(node => node.class_type === 'KSampler'); + sampler.inputs.seed = seed; + edit?.(output); + const id = randomUUID(); + const data = { prompt: output, prompt_id: id, extra_data: { extra_pnginfo: { workflow: first.workflow } } }; + if (targets) data.partial_execution_targets = targets; + const response = await execution.request.post('/prompt', { data }); + expect(response.ok(), await response.text()).toBe(true); + let item; + await expect.poll(async () => { + item = (await (await execution.request.get(`/history/${id}`)).json())[id]; + return item?.status?.status_str ?? null; + }, { timeout: 20000 }).toBe(status); + const cached = item.status.messages.find(([event]) => event === 'execution_cached')?.[1]?.nodes ?? []; + return { id, item, cached }; +} + +// The recorded context of one job, once its capture reached SQLite. +async function capturedContext(execution, id) { + let result; + await expect.poll(async () => { + result = await readExecutionRuns(execution.databasePath, id); + const row = result.rows[0]; + return row ? result.captures[row.run_id].state : null; + }, { timeout: 10000 }).toMatch(/^(complete|partial)$/); + return result.captures[result.rows[0].run_id].contexts[0]; +} + +async function firstRun(execution, page, testCase) { + const prefix = `textlink/${randomUUID()}`; + const built = await build(page, { ...testCase, prefix }); + const evidence = await run(execution, page, testCase, built); + return { ...evidence, workflow: built.workflow }; +} + +for (const kind of ['PromptManager', 'PromptManagerFocused', 'PromptManagerSplit', 'PromptManagerText', 'PromptManagerPair']) { + test(`${kind} counts each cached re-run of text linked from PreviewAny (#347)`, async ({ execution, page }) => { + test.setTimeout(90000); + await setCache(execution, 'RAM_PRESSURE'); + const pair = kind === 'PromptManagerPair'; + const testCase = { kind, positive: { typed: 'Typed under the link', text: unique(`Cached ${kind}`), via: 'preview' }, + negative: pair ? { typed: unique('Typed negative') } : undefined }; + const first = await firstRun(execution, page, testCase); + for (const seed of [2, 3]) { + const again = await queueAgain(execution, first, { seed }); + expect(again.cached, 'ComfyUI served the PromptManager node from its cache').toContain(first.promptNode); + } + const counts = await runCounts(execution, first.context); + expect(counts.positive).toBe(3); + if (pair) expect(counts).toMatchObject({ negative: 0, pair: 3 }); + }); +} + +test.describe('cached runs in one renderer (#347)', () => { + // Cache behavior does not depend on the renderer: one project runs these. + test.beforeEach(({}, testInfo) => { test.skip(testInfo.project.name !== 'classic-start', 'covered in classic-start'); }); + const linked = () => ({ kind: 'PromptManager', positive: { typed: 'Typed under the link', text: unique('Cached'), via: 'preview' } }); + + for (const mode of ['CLASSIC', 'LRU', 'RAM_PRESSURE', 'NONE']) { + test(`counts 3 jobs under the ${mode} cache`, async ({ execution, page }) => { + test.setTimeout(90000); + await setCache(execution, mode); + try { + const first = await firstRun(execution, page, linked()); + for (const seed of [2, 3]) { + const again = await queueAgain(execution, first, { seed }); + if (mode === 'NONE') expect(again.cached).not.toContain(first.promptNode); + else expect(again.cached).toContain(first.promptNode); + } + expect((await runCounts(execution, first.context)).positive).toBe(3); + } finally { await setCache(execution, 'RAM_PRESSURE'); } + }); + } + + test('alternating upstream values A, B, A under LRU each count their own prompt', async ({ execution, page }) => { + test.setTimeout(90000); + await setCache(execution, 'LRU'); + try { + const [a, b] = [unique('Cycled A'), unique('Cycled B')]; + const cycle = async value => expect((await execution.request.post('/pm_execution_fixture/cycled', { data: { value } })).ok()).toBe(true); + await cycle(a); + const first = await firstRun(execution, page, { kind: 'PromptManager', positive: { typed: 'Typed under the link', text: a, via: 'cycled' } }); + expect(first.context.positive.raw).toBe(a); + await cycle(b); + const second = await queueAgain(execution, first, { seed: 2 }); + expect(second.cached).not.toContain(first.promptNode); + await cycle(a); + const third = await queueAgain(execution, first, { seed: 3 }); + expect(third.cached, 'LRU kept A').toContain(first.promptNode); + expect((await runCounts(execution, first.context)).positive).toBe(2); + const bContext = await capturedContext(execution, second.id); + expect(bContext.positive.raw).toBe(b); + expect((await runCounts(execution, bContext)).positive).toBe(1); + } finally { await setCache(execution, 'RAM_PRESSURE'); } + }); + + test('typed text served from the cache still counts once per job', async ({ execution, page }) => { + test.setTimeout(90000); + await setCache(execution, 'RAM_PRESSURE'); + const first = await firstRun(execution, page, { kind: 'PromptManager', positive: { typed: unique('Typed cached') } }); + for (const seed of [2, 3]) expect((await queueAgain(execution, first, { seed })).cached).toContain(first.promptNode); + expect((await runCounts(execution, first.context)).positive).toBe(3); + }); + + test('a job failing downstream of a cached node counts it once; a job that does not need it, never', async ({ execution, page }) => { + test.setTimeout(90000); + await setCache(execution, 'RAM_PRESSURE'); + const first = await firstRun(execution, page, linked()); + const failing = await queueAgain(execution, first, { seed: 2, status: 'error', edit: output => { + output.pm_failure = { class_type: 'PMHistoryOutput', inputs: { value: [Object.keys(output).find(id => output[id].class_type === 'KSampler'), 0], mode: 'failure' } }; + } }); + expect(failing.cached).toContain(first.promptNode); + expect((await runCounts(execution, first.context)).positive).toBe(2); + // "Run selected output": only an unrelated output node; ComfyUI still reports the cached PromptManager node + const unrelated = await queueAgain(execution, first, { seed: 2, targets: ['pm_unrelated'], edit: output => { + output.pm_unrelated = { class_type: 'PMHistoryCached', inputs: { value: unique('unrelated') } }; + } }); + expect(unrelated.cached).toContain(first.promptNode); + expect((await runCounts(execution, first.context)).positive).toBe(2); + }); + + test('a cached node rewired to the negative input does not count the positive prompt', async ({ execution, page }) => { + test.setTimeout(90000); + await setCache(execution, 'RAM_PRESSURE'); + const first = await firstRun(execution, page, linked()); + const rewired = await queueAgain(execution, first, { seed: 2, edit: output => { + const sampler = Object.values(output).find(node => node.class_type === 'KSampler'); + [sampler.inputs.positive, sampler.inputs.negative] = [sampler.inputs.negative, sampler.inputs.positive]; + } }); + expect(rewired.cached).toContain(first.promptNode); + expect((await runCounts(execution, first.context)).positive).toBe(1); + // The history capture of a cached job is unchanged: effective text from the cache, no stored-prompt hint + const cachedContext = await capturedContext(execution, (await queueAgain(execution, first, { seed: 3 })).id); + expect(cachedContext.positive.effective).toBe(first.context.positive.effective); + expect(cachedContext.positive_prompt_id).toBeNull(); + expect((await runCounts(execution, first.context)).positive).toBe(2); + }); +}); diff --git a/tests/comfyui/fixtures/ledger_nodes.py b/tests/comfyui/fixtures/ledger_nodes.py index 844249af..a0c57444 100644 --- a/tests/comfyui/fixtures/ledger_nodes.py +++ b/tests/comfyui/fixtures/ledger_nodes.py @@ -201,6 +201,39 @@ NODE_CLASS_MAPPINGS.update( {"PMHistoryImages": PMHistoryImages, "PMHistoryImageOutput": PMHistoryImageOutput} ) +# #347: text whose value (and cache signature) follows a value the test sets, so +# one queued graph can serve A, then B, then A again (each cached under LRU). +_CYCLED = {"value": "cycled"} + + +class PMCycledText: + @classmethod + def INPUT_TYPES(cls) -> dict: + return {"required": {}} + + RETURN_TYPES = ("STRING",) + FUNCTION = "execute" + CATEGORY = "testing/execution-history" + + @classmethod + def IS_CHANGED(cls, **kwargs: object) -> str: + return _CYCLED["value"] + + def execute(self) -> tuple: + return (_CYCLED["value"],) + + +NODE_CLASS_MAPPINGS["PMCycledText"] = PMCycledText + + +@PromptServer.instance.routes.post("/pm_execution_fixture/cycled") +async def cycled_text(request: web.Request) -> web.Response: + value = (await request.json())["value"] + if type(value) is not str: + raise web.HTTPBadRequest() + _CYCLED["value"] = value + return web.json_response({"value": value}) + @PromptServer.instance.routes.post("/pm_execution_fixture/metadata") async def metadata(request: web.Request) -> web.Response: diff --git a/tests/comfyui/image-attribution.spec.mjs b/tests/comfyui/image-attribution.spec.mjs index 64f14e68..65ffd655 100644 --- a/tests/comfyui/image-attribution.spec.mjs +++ b/tests/comfyui/image-attribution.spec.mjs @@ -140,3 +140,26 @@ test('an output saved without metadata links once its recorded run names a store expect((await request.post('/pm_execution_fixture/metadata', { data: { disabled: false } })).ok()).toBe(true); } }); + +// #332 on the real host: the migration ran at startup and the image category routes +// set, list and remove an own category on a generated output. +test('an output gets its own image category and loses it again', async ({ execution }, testInfo) => { + const request = execution.request; + const filename = await saved(request, graph({ text: `category host ${randomUUID()}`, prefix: 'pm332/category' })); + let image; + await expect.poll(async () => { + const listing = await (await request.get('/prompt_manager/images/output?mode=workspace&q=pm332&rescan=1')).json(); + image = (listing.images || []).find(entry => entry.filename === filename || entry.path?.endsWith(`/${filename}`)); + return Boolean(image); + }, { timeout: 20000, message: `${filename} is listed by Images` }).toBe(true); + const items = [{ root_index: image.root_index, root_scope: image.root_scope, path: image.path }]; + const value = `PM332 host ${testInfo.project.name}`; + const set = await request.put('/prompt_manager/images/category', { data: { items, category: value } }); + expect(set.status()).toBe(200); + expect(await set.json()).toEqual({ success: true, category: value, updated: 1, unchanged: 0, not_found: 0 }); + expect((await (await request.get('/prompt_manager/images/categories')).json()).categories).toContain(value); + const cleared = await request.delete('/prompt_manager/images/category', { data: { items } }); + expect(cleared.status()).toBe(200); + expect(await cleared.json()).toEqual({ success: true, cleared: 1 }); + expect((await (await request.get('/prompt_manager/images/categories')).json()).categories).not.toContain(value); +}); diff --git a/tests/disk_waits.py b/tests/disk_waits.py new file mode 100644 index 00000000..26a1f879 --- /dev/null +++ b/tests/disk_waits.py @@ -0,0 +1,16 @@ +"""Hang guards for waits on disk-bound work (tests only). + +A test that waits for a job or handoff whose work is file and SQLite I/O +(a safety backup with fsync, a restore, hashing and unlinking files) waits +on that state; the ceiling below only ends a real hang. Hosted Windows +runners stall disk I/O for tens of seconds at a time: a restore took over +10 s (merge queue, #382), and on PR #405 every test setup in a 50 s window +took up to 9 s against 0.1-0.6 s around it while the test bodies stayed +fast, so a one-file duplicate cleanup missed a 10 s bound (#409). + +The ceiling is per wait and stays under the suite's 120 s per-test hang +watchdog. A test with several such waits in a row can reach the watchdog +first; either way a real hang fails, the watchdog with every thread's stack. +""" + +DISK_BOUND_SECONDS = 90 diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 25b61bc9..7e1476a2 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -113,3 +113,19 @@ by their own suites. The expectations start as the observed baseline. A change that alters an outcome edits the vector in the same commit. `tests/js/legacy-metadata-characterization.test.js` pins what the legacy viewers show on the same files. + +# Legacy contract snapshot + +`legacy_contract_snapshot.json` records what the legacy admin and gallery pages +talk to while System S13 (#330) moves their tools to native System: every +registered route as `" "` (`routes`) and the key paths of +`GET /prompt_manager/settings` to depth 2 (`settings_keys`; list items give the +union of their keys as `name[].key`). `tests/test_legacy_contract_snapshot.py` +compares a temp-database API against it. Record it again only for an intended +change, and say why in the PR body: + +```sh +PM_RECORD_LEGACY_SNAPSHOT=1 python3 -m unittest tests.test_legacy_contract_snapshot +``` + +Cutover (#334) removes the test and this fixture. diff --git a/tests/fixtures/legacy_contract_snapshot.json b/tests/fixtures/legacy_contract_snapshot.json new file mode 100644 index 00000000..27568173 --- /dev/null +++ b/tests/fixtures/legacy_contract_snapshot.json @@ -0,0 +1,209 @@ +{ + "routes": [ + "DELETE /prompt_manager/collections/items", + "DELETE /prompt_manager/collections/{collection_id}", + "DELETE /prompt_manager/delete/{prompt_id}", + "DELETE /prompt_manager/image-annotations/capture/{token}", + "DELETE /prompt_manager/images/category", + "DELETE /prompt_manager/images/favorite", + "DELETE /prompt_manager/images/{image_id}", + "DELETE /prompt_manager/pairs/{pair_id}", + "DELETE /prompt_manager/prompts/{prompt_id}/tags", + "DELETE /prompt_manager/restore/preview/{token}", + "DELETE /prompt_manager/saved-views/{view_id}", + "DELETE /prompt_manager/tags/{tag_name}", + "GET /prompt_manager/admin", + "GET /prompt_manager/autotag/models", + "GET /prompt_manager/backup", + "GET /prompt_manager/backup/status", + "GET /prompt_manager/categories", + "GET /prompt_manager/collections", + "GET /prompt_manager/collections/membership", + "GET /prompt_manager/diagnostics", + "GET /prompt_manager/export", + "GET /prompt_manager/gallery", + "GET /prompt_manager/gallery.html", + "GET /prompt_manager/gallery/subfolders", + "GET /prompt_manager/history/runs/{run_id}", + "GET /prompt_manager/history/runs/{run_id}/output/current/inspection", + "GET /prompt_manager/history/runs/{run_id}/output/current/media", + "GET /prompt_manager/history/runs/{run_id}/output/current/thumbnail", + "GET /prompt_manager/history/runs/{run_id}/outputs", + "GET /prompt_manager/image-annotations/capture/{token}", + "GET /prompt_manager/images/all", + "GET /prompt_manager/images/categories", + "GET /prompt_manager/images/inspect/{image_path}", + "GET /prompt_manager/images/output", + "GET /prompt_manager/images/prompt/{image_path}", + "GET /prompt_manager/images/recent", + "GET /prompt_manager/images/search", + "GET /prompt_manager/images/serve/{filepath}", + "GET /prompt_manager/images/thumbnails/plan", + "GET /prompt_manager/images/thumbnails/status", + "GET /prompt_manager/images/{image_id}/file", + "GET /prompt_manager/jobs", + "GET /prompt_manager/jobs/{job_id}", + "GET /prompt_manager/jobs/{job_id}/events", + "GET /prompt_manager/jobs/{job_id}/report", + "GET /prompt_manager/js/{filepath}", + "GET /prompt_manager/lib/{filepath}", + "GET /prompt_manager/library", + "GET /prompt_manager/library/export", + "GET /prompt_manager/library/options", + "GET /prompt_manager/library/{positive_id}", + "GET /prompt_manager/library/{positive_id}/history", + "GET /prompt_manager/library/{positive_id}/history/pairs/{pair_id}/newest", + "GET /prompt_manager/library/{positive_id}/images", + "GET /prompt_manager/logs", + "GET /prompt_manager/logs/config", + "GET /prompt_manager/logs/download/{filename}", + "GET /prompt_manager/logs/files", + "GET /prompt_manager/logs/stats", + "GET /prompt_manager/lora/detect", + "GET /prompt_manager/lora/status", + "GET /prompt_manager/lora/trigger-words", + "GET /prompt_manager/maintenance/duplicates/{job_id}", + "GET /prompt_manager/pairs", + "GET /prompt_manager/pairs/export", + "GET /prompt_manager/pairs/{pair_id}", + "GET /prompt_manager/prompts/{prompt_id}/images", + "GET /prompt_manager/recent", + "GET /prompt_manager/review/events", + "GET /prompt_manager/review/items", + "GET /prompt_manager/review/items/{item_id}", + "GET /prompt_manager/saved-views", + "GET /prompt_manager/scan/status", + "GET /prompt_manager/scan_duplicates", + "GET /prompt_manager/scan_duplicates/status", + "GET /prompt_manager/scan_output_dir", + "GET /prompt_manager/search", + "GET /prompt_manager/settings", + "GET /prompt_manager/stats", + "GET /prompt_manager/subfolders", + "GET /prompt_manager/system/diagnostics", + "GET /prompt_manager/system/health", + "GET /prompt_manager/system/library", + "GET /prompt_manager/system/overview", + "GET /prompt_manager/system/statistics", + "GET /prompt_manager/tags", + "GET /prompt_manager/tags/filter", + "GET /prompt_manager/tags/stats", + "GET /prompt_manager/tags/suggest", + "GET /prompt_manager/tags/{tag_name}/prompts", + "GET /prompt_manager/test", + "GET /prompt_manager/vocabulary/categories", + "GET /prompt_manager/vocabulary/tags", + "GET /prompt_manager/vocabulary/tags/{tag_id}/impact", + "GET /prompt_manager/web", + "GET /prompt_manager/workspace", + "PATCH /prompt_manager/collections/{collection_id}", + "PATCH /prompt_manager/library/{positive_id}", + "PATCH /prompt_manager/pairs/{pair_id}", + "PATCH /prompt_manager/saved-views/{view_id}", + "PATCH /prompt_manager/vocabulary/tags/{tag_id}", + "POST /prompt_manager/autotag/apply", + "POST /prompt_manager/autotag/batch/preview", + "POST /prompt_manager/autotag/download/{model_type}", + "POST /prompt_manager/autotag/single", + "POST /prompt_manager/autotag/start", + "POST /prompt_manager/autotag/unload", + "POST /prompt_manager/bulk/category", + "POST /prompt_manager/bulk/delete", + "POST /prompt_manager/bulk/tags", + "POST /prompt_manager/cleanup", + "POST /prompt_manager/collections/items", + "POST /prompt_manager/delete_duplicate_images", + "POST /prompt_manager/diagnostics/test-link", + "POST /prompt_manager/gallery/rescan", + "POST /prompt_manager/image-annotations/observe", + "POST /prompt_manager/image-annotations/suggest", + "POST /prompt_manager/images/clear-thumbnails", + "POST /prompt_manager/images/generate-thumbnails", + "POST /prompt_manager/images/generate-thumbnails/progress", + "POST /prompt_manager/images/link", + "POST /prompt_manager/images/reveal", + "POST /prompt_manager/jobs/autotag", + "POST /prompt_manager/jobs/duplicates", + "POST /prompt_manager/jobs/image-tags", + "POST /prompt_manager/jobs/model-download", + "POST /prompt_manager/jobs/scan", + "POST /prompt_manager/jobs/thumbnails", + "POST /prompt_manager/jobs/{job_id}/cancel", + "POST /prompt_manager/library", + "POST /prompt_manager/library/{positive_id}/variants", + "POST /prompt_manager/logs/config", + "POST /prompt_manager/logs/truncate", + "POST /prompt_manager/lora/enable", + "POST /prompt_manager/lora/import/apply", + "POST /prompt_manager/lora/import/preview", + "POST /prompt_manager/lora/refresh-cache", + "POST /prompt_manager/lora/scan", + "POST /prompt_manager/maintenance", + "POST /prompt_manager/maintenance/apply", + "POST /prompt_manager/maintenance/preview", + "POST /prompt_manager/pairs", + "POST /prompt_manager/prompts/tags", + "POST /prompt_manager/prompts/{prompt_id}/tags", + "POST /prompt_manager/restore", + "POST /prompt_manager/restore/apply", + "POST /prompt_manager/restore/preview", + "POST /prompt_manager/review/items/{item_id}/decision", + "POST /prompt_manager/save", + "POST /prompt_manager/saved-views", + "POST /prompt_manager/scan", + "POST /prompt_manager/scan/cancel", + "POST /prompt_manager/scan_duplicates/stream", + "POST /prompt_manager/settings", + "POST /prompt_manager/settings/validate-source", + "POST /prompt_manager/system/health", + "POST /prompt_manager/system/library/keep", + "POST /prompt_manager/system/library/move", + "POST /prompt_manager/tags/merge", + "POST /prompt_manager/vocabulary/categories/apply", + "POST /prompt_manager/vocabulary/categories/preview", + "POST /prompt_manager/vocabulary/tags", + "POST /prompt_manager/vocabulary/tags/{tag_id}/delete", + "PUT /prompt_manager/image-annotations/tags", + "PUT /prompt_manager/images/category", + "PUT /prompt_manager/images/favorite", + "PUT /prompt_manager/prompts/{prompt_id}", + "PUT /prompt_manager/prompts/{prompt_id}/rating", + "PUT /prompt_manager/tags/{tag_name}" + ], + "settings_keys": [ + "monitor", + "monitor.observer_alive", + "monitor.present", + "monitor.scheduled", + "revision", + "settings", + "settings.autotag_character_threshold", + "settings.autotag_existing", + "settings.autotag_general_threshold", + "settings.autotag_model_type", + "settings.autotag_suggestion_limit", + "settings.autotag_template", + "settings.cpu_count", + "settings.gallery_root_path", + "settings.gallery_root_paths", + "settings.infinite_scroll", + "settings.monitored_directories", + "settings.monitoring_enabled", + "settings.result_timeout", + "settings.safety_backup", + "settings.thumbnail_edge", + "settings.thumbnail_format", + "settings.thumbnail_quality", + "settings.webui_display_mode", + "settings.worker_threads", + "sources", + "sources.auto_detected", + "sources.configured", + "sources.effective", + "success", + "thumbnail_encoders", + "thumbnail_encoders.jpeg", + "thumbnail_encoders.jxl", + "thumbnail_encoders.webp" + ] +} diff --git a/tests/js/gallery-settings.test.js b/tests/js/gallery-settings.test.js index a8fbf658..7f85fd9a 100644 --- a/tests/js/gallery-settings.test.js +++ b/tests/js/gallery-settings.test.js @@ -76,35 +76,101 @@ test("the workspace keys have their own defaults and validation", () => { assert.equal(normalizeGallerySettings({ defaultViewMode: "masonry" }).defaultViewMode, "grid"); }); -test("a legacy gallery Save keeps the workspace keys and every other stored key", () => { +// #330 S13: the legacy gallery's Page options edit only the page-only keys (PAGE_OPTION_KEYS); System โ€บ Appearance owns the rest, so a Save +// or a Reset from the legacy page keeps every other stored key. +function legacyGallery(stored, controls, extra = {}) { const fs = require("node:fs"); const path = require("node:path"); const vm = require("node:vm"); const settings = require("../../web/js/gallery-settings.js"); - const storage = new Map([["gallerySettings", JSON.stringify({ workspaceLayout: "masonry", workspaceDensity: "compact", someOtherTool: 1 })]]); - const controls = { - lazyLoadingToggle: { checked: true }, imageQualitySelect: { value: "high" }, defaultViewModeSelect: { value: "list" }, - defaultLimitSelect: { value: "200" }, gridColumnsSelect: { value: "6" }, showImageInfoToggle: { checked: false }, - autoLoadMetadataToggle: { checked: true }, cacheMetadataToggle: { checked: true }, showFilePathsToggle: { checked: false }, - sidebarCollapsedByDefaultToggle: { checked: true }, infiniteScrollToggle: { checked: false }, debugModeToggle: { checked: false }, - apiTimeoutInput: { value: "30" }, checkThumbnailsAtStartup: { checked: true }, videoAutoplayToggle: { checked: true }, - videoMuteToggle: { checked: false }, videoLoopToggle: { checked: true }, - }; + const storage = new Map([["gallerySettings", JSON.stringify(stored)]]); const context = vm.createContext({ document: { addEventListener() {}, getElementById: id => controls[id] }, window: {}, console: { debug() {}, warn() {}, error() {} }, GallerySettings: settings, - localStorage: { getItem: key => storage.get(key) ?? null, setItem: (key, value) => storage.set(key, value) }, + localStorage: { getItem: key => storage.get(key) ?? null, setItem: (key, value) => storage.set(key, value), + removeItem: key => storage.delete(key) }, + ...extra, }); const file = path.join(__dirname, "../../web/js/gallery.js"); const Manager = vm.runInContext(`${fs.readFileSync(file, "utf8")}\nGalleryManager;`, context, { filename: file }); const manager = Object.create(Manager.prototype); - Object.assign(manager, { showNotification() {}, hideSettings() {}, applySettings() {} }); + const calls = []; + Object.assign(manager, { showNotification: (message, type) => calls.push([type, message]), hideSettings() {}, + applySettings: applied => calls.push(["apply", applied.lazyLoading]) }); + return { manager, storage, calls, saved: () => JSON.parse(storage.get("gallerySettings")) }; +} + +const PAGE_OPTION_CONTROLS = () => ({ + checkThumbnailsAtStartup: { checked: false }, lazyLoadingToggle: { checked: false }, + defaultLimitSelect: { value: "200" }, + gridColumnsSelect: { value: "6" }, infiniteScrollToggle: { checked: true }, debugModeToggle: { checked: true }, + apiTimeoutInput: { value: "45" }, +}); + +test("a legacy gallery Save keeps the workspace keys and every other stored key", () => { + const { manager, saved } = legacyGallery({ workspaceLayout: "masonry", workspaceDensity: "compact", someOtherTool: 1, + imageQuality: "high", showImageInfo: false, videoMute: false, thumbnailPromptDismissed: "sig", defaultViewMode: "list" }, + PAGE_OPTION_CONTROLS()); manager.saveSettings(); - const saved = JSON.parse(storage.get("gallerySettings")); - assert.equal(saved.workspaceLayout, "masonry"); - assert.equal(saved.workspaceDensity, "compact"); - assert.equal(saved.someOtherTool, 1); - assert.equal(saved.defaultViewMode, "list"); - assert.equal(saved.showImageInfo, false); - assert.equal(saved.videoMute, false); + const result = saved(); + assert.equal(result.workspaceLayout, "masonry"); + assert.equal(result.workspaceDensity, "compact"); + assert.equal(result.someOtherTool, 1); + assert.equal(result.imageQuality, "high"); + assert.equal(result.showImageInfo, false); + assert.equal(result.videoMute, false); + assert.equal(result.thumbnailPromptDismissed, "sig"); + // Page options no longer edit the default view (S13 G5), so a stored one stays. + assert.equal(result.defaultViewMode, "list"); + // The page-only keys come from the dialog. + assert.equal(result.lazyLoading, false); + assert.equal(result.debugMode, true); + assert.equal(result.checkThumbnailsAtStartup, false); + assert.equal(result.defaultLimit, 200); + assert.equal(result.gridColumns, 6); + assert.equal(result.infiniteScroll, true); + assert.equal(result.apiTimeout, 45); +}); + +test("a legacy gallery Reset restores only the page options and keeps every other stored key", () => { + const controls = PAGE_OPTION_CONTROLS(); + const { manager, saved, calls } = legacyGallery({ workspaceLayout: "masonry", imageQuality: "high", videoMute: false, + thumbnailPromptDismissed: "sig", thumbnailsGenerated: true, lazyLoading: false, debugMode: true, defaultLimit: 200, + gridColumns: 6, infiniteScroll: true, apiTimeout: 45, checkThumbnailsAtStartup: false, defaultViewMode: "list" }, controls, + { confirm: message => (calls.push(["confirm", message]), true) }); + manager.resetSettings(); + const result = saved(); + const { DEFAULT_SETTINGS } = require("../../web/js/gallery-settings.js"); + for (const key of ["checkThumbnailsAtStartup", "lazyLoading", "defaultLimit", "gridColumns", + "infiniteScroll", "debugMode", "apiTimeout"]) { + assert.equal(result[key], DEFAULT_SETTINGS[key], key); + } + // The default view is not a page option (S13 G5): Reset keeps a stored one. + assert.equal(result.defaultViewMode, "list"); + assert.equal(result.workspaceLayout, "masonry"); + assert.equal(result.imageQuality, "high"); + assert.equal(result.videoMute, false); + assert.equal(result.thumbnailPromptDismissed, "sig"); + assert.equal(result.thumbnailsGenerated, true); + // The dialog shows the defaults again and the page applies them. + assert.equal(controls.lazyLoadingToggle.checked, DEFAULT_SETTINGS.lazyLoading); + assert.equal(controls.apiTimeoutInput.value, DEFAULT_SETTINGS.apiTimeout); + assert.deepEqual(calls, [["confirm", "Reset these page options to their defaults?"], ["apply", DEFAULT_SETTINGS.lazyLoading], + ["success", "Page options reset to defaults"]]); +}); + +test("Page options no longer hold the default view; System โ€บ Appearance's Default layout is the one control", () => { + const { manager } = legacyGallery({}, {}); + assert.deepEqual([...manager.constructor.PAGE_OPTION_KEYS], ["checkThumbnailsAtStartup", "lazyLoading", "defaultLimit", + "gridColumns", "infiniteScroll", "debugMode", "apiTimeout"]); +}); + +test("a viewer mute or loop click remembers only that key and never reads the Page options dialog", () => { + const { manager, saved, calls } = legacyGallery({ defaultLimit: 200, lazyLoading: false, videoMute: true }, {}); + manager.storeSetting("videoMute", false); + const result = saved(); + assert.equal(result.videoMute, false); + assert.equal(result.defaultLimit, 200); + assert.equal(result.lazyLoading, false); + assert.deepEqual(calls, []); }); diff --git a/tests/js/gallery-thumbnails.test.js b/tests/js/gallery-thumbnails.test.js index cb135161..118306a2 100644 --- a/tests/js/gallery-thumbnails.test.js +++ b/tests/js/gallery-thumbnails.test.js @@ -112,11 +112,12 @@ test('saving gallery settings retains the remembered thumbnail dismissal', () => }); // #249 System S5: the stream's work is now an Activity job that outlives the page's connection, so the gallery's -// Cancel asks the server to stop the job. The stream's first frame, `job {id, started}`, names the job; the gallery +// Cancel asks the server to stop the job. Since #330 the startup dialog is the gallery's only thumbnail stream (System +// โ€บ Sources & scans generates the rest). The stream's first frame, `job {id, started}`, names the job; the gallery // cancels by that id, only a job this stream started, and a Cancel before the id waits for it (review I1). const JOB = 'b'.repeat(32); -function streamFixture({ modal = false, postFails = false } = {}) { +function streamFixture({ postFails = false } = {}) { const posts = []; const warnings = []; const gets = []; @@ -146,18 +147,20 @@ function streamFixture({ modal = false, postFails = false } = {}) { return { success: true }; } } }); const emit = (type, data) => (listeners[type] || []).forEach(handler => handler({ data: JSON.stringify(data) })); + // The dialog's Cancel runs the handler and from then on reads as cancelled, as the startup dialog does. + let cancelled = false; const ui = { addStatusMessage() {}, progressText: element(), progressPercent: element(), progressBar: element(), - progressDetails: element(), progressETA: element(), isCancelled: () => false, - onCancel: handler => { ui.cancel = handler; } }; - const result = modal ? manager.generateThumbnailsWithProgressInModal(ui) - : manager.generateThumbnailsWithProgress({ quality: 'medium' }); + progressDetails: element(), progressETA: element(), isCancelled: () => cancelled, + onCancel: handler => { ui.cancel = () => { cancelled = true; handler(); }; } }; + const result = manager.generateThumbnailsWithProgressInModal(ui); const outcome = result.then(value => ({ value }), error => ({ error: error.message })); - const cancel = () => (modal ? ui.cancel() : elements.cancelThumbnailsBtn.click()); + const cancel = () => ui.cancel(); const settle = () => new Promise(resolve => setImmediate(resolve)); return { emit, source, posts, gets, warnings, outcome, cancel, settle }; } const cancelUrl = id => `/prompt_manager/jobs/${id}/cancel`; +const CANCELLED = 'Thumbnail generation cancelled; previews already written are kept'; test('Cancel before the job id arrives cancels that job once it does', async () => { const f = streamFixture(); @@ -165,8 +168,8 @@ test('Cancel before the job id arrives cancels that job once it does', async () await f.settle(); assert.deepEqual(f.posts, []); assert.equal(f.source.closed, 0); // kept open until the stream names its job - assert.deepEqual(await f.outcome, { error: 'Thumbnail generation cancelled by user' }); - f.emit('status', { phase: 'scanning', message: 'Scanningโ€ฆ' }); + assert.deepEqual(await f.outcome, { error: CANCELLED }); + // The job frame is the stream's first (the dialog closes the stream on any later frame once cancelled). f.emit('job', { id: JOB, started: true }); await f.settle(); assert.equal(JSON.stringify(f.posts), JSON.stringify([[cancelUrl(JOB), {}]])); @@ -185,7 +188,7 @@ test('Cancel after the id cancels it at once, and a double Cancel posts once', a await f.settle(); assert.equal(JSON.stringify(f.posts), JSON.stringify([[cancelUrl(JOB), {}]])); assert.equal(f.source.closed, 1); - assert.deepEqual(await f.outcome, { error: 'Thumbnail generation cancelled by user' }); + assert.deepEqual(await f.outcome, { error: CANCELLED }); assert.deepEqual(f.gets, []); }); @@ -216,11 +219,11 @@ test('a cancel post that fails is reported, not thrown', async () => { await f.settle(); assert.equal(f.posts.length, 1); assert.equal(f.warnings.length, 1); - assert.deepEqual(await f.outcome, { error: 'Thumbnail generation cancelled by user' }); + assert.deepEqual(await f.outcome, { error: CANCELLED }); }); test('the startup dialog cancels its own job by id too', async () => { - const f = streamFixture({ modal: true }); + const f = streamFixture(); f.cancel(); await f.settle(); assert.deepEqual(f.posts, []); diff --git a/tests/js/legacy-refusals.test.js b/tests/js/legacy-refusals.test.js index c06daedc..78d15c9b 100644 --- a/tests/js/legacy-refusals.test.js +++ b/tests/js/legacy-refusals.test.js @@ -65,109 +65,12 @@ function load(file, className, fetchUrl, extra = {}, dom = elements()) { const BUSY = 'Scan for metadata already running'; -test('gallery cache clear reports a refused server clear and never claims success', async () => { - const calls = []; - const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async url => { - calls.push(url); - return jsonResponse(409, { success: false, error: BUSY }); - }); - page.getSettings = () => ({ thumbnailsGenerated: true }); - dom.getElementById('thumbnailStatus').textContent = 'Generated'; - - await page.clearCache(); - - assert.deepEqual(calls, ['/prompt_manager/images/clear-thumbnails']); - assert.deepEqual(notifications, [['error', `Thumbnails were not cleared: ${BUSY}`]]); - assert.equal(dom.nodes.thumbnailStatus.textContent, 'Generated'); - assert.equal(storage.get('gallerySettings'), undefined); // thumbnailsGenerated is not reset -}); - -test('gallery cache clear says what it cleared and keeps preferences and other data (#222)', async () => { - const calls = []; - const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async url => { - calls.push(url); - return jsonResponse(200, { success: true, cleared_count: 3 }); - }); - page.getSettings = () => ({ thumbnailsGenerated: true }); - - await page.clearCache(); - - // The server's thumbnails are cleared by their own explicit call; nothing Prompt Manager owns is cached in this - // browser, so no browser entry is removed and the toast says exactly that. - assert.deepEqual(calls, ['/prompt_manager/images/clear-thumbnails']); - assert.deepEqual(notifications, [['success', - "Thumbnails cleared; original images untouched. Nothing of Prompt Manager's is cached in this browser."]]); - assert.equal(dom.nodes.thumbnailStatus.textContent, 'Not generated'); - assert.equal(JSON.parse(storage.get('gallerySettings')).thumbnailsGenerated, false); - assert.equal(storage.get('userData'), 'keep'); - assert.equal(storage.get('pm_cached'), '1'); - assert.equal(storage.get('pm_workspace_theme'), 'light'); - assert.equal(storage.get('pm_update_notice_dismissed'), '3.3.0'); -}); - -test("gallery cache clear never deletes another tool's gallery or thumbnail caches (#222)", async () => { - const names = new Set(['comfy-gallery', 'thumbnail-cache']); - const caches = { keys: async () => [...names], delete: async name => names.delete(name) }; - const { page, notifications } = load('gallery.js', 'GalleryManager', - async () => jsonResponse(200, { success: true }), { caches, window: { caches } }); - page.getSettings = () => ({}); - - await page.clearCache(); - - assert.deepEqual([...names], ['comfy-gallery', 'thumbnail-cache']); - assert.equal(notifications.at(-1)[0], 'success'); -}); - -test('gallery cache clear reports a lost server clear instead of success', async () => { - const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async () => { - throw new TypeError('Failed to fetch'); - }); - page.getSettings = () => ({ thumbnailsGenerated: true }); - dom.getElementById('thumbnailStatus').textContent = 'Generated'; - - await page.clearCache(); - - assert.deepEqual(notifications, [['error', 'Thumbnails were not cleared: the server could not be reached.']]); - assert.equal(dom.nodes.thumbnailStatus.textContent, 'Generated'); - assert.equal(storage.get('gallerySettings'), undefined); -}); - -test('gallery cache clear says so when browser storage is denied', async () => { - const denied = { get length() { throw new Error('SecurityError'); }, key() { throw new Error('SecurityError'); }, - getItem() { throw new Error('SecurityError'); }, setItem() { throw new Error('SecurityError'); }, - removeItem() { throw new Error('SecurityError'); } }; - const { page, notifications } = load('gallery.js', 'GalleryManager', - async () => jsonResponse(200, { success: true }), { localStorage: denied }); - page.getSettings = () => ({}); - - await page.clearCache(); - - assert.deepEqual(notifications, [['warning', - 'Thumbnails cleared; original images untouched. This browser refused access to its storage, so nothing there was cleared.']]); -}); - -test('admin LoRA import shows a refusal frame and closes its modal', async () => { - const { page, dom, timers, notifications } = load('admin.js', 'PromptAdmin', - async () => sseResponse([{ type: 'error', status: BUSY, progress: 0, message: BUSY }])); - page.saveLoraSettings = async () => true; - page.loadStatistics = () => assert.fail('a refused import must not reload statistics'); - - await page.runLoraImport(); - - assert.equal(dom.nodes.loraImportStatus.textContent, `Error: ${BUSY}`); - assert.deepEqual(notifications.filter(([type]) => type === 'error'), [['error', `LoRA import failed: ${BUSY}`]]); - assert.equal(timers.length, 1); - timers[0](); - assert.deepEqual(notifications.at(-1), ['hide', 'loraImportModal']); -}); - test('admin settings save shows why config.json refused it', async () => { const message = 'config.json cannot be read, so nothing was saved. Fix or remove the file, then save again.'; const { page, notifications } = load('admin.js', 'PromptAdmin', async () => jsonResponse(409, { success: false, persisted: false, code: 'config-unreadable', error: message })); page.settings = {}; - page._collectScanPaths = () => []; - page.saveLoraSettings = async () => assert.fail('nothing else is saved after a refusal'); + page.setInfiniteScroll = () => assert.fail('nothing changes after a refusal'); await page.saveSettings(); @@ -178,27 +81,30 @@ test('admin settings save keeps its plain message when the reply has no reason', const { page, notifications } = load('admin.js', 'PromptAdmin', async () => ({ ok: false, status: 500, json: async () => { throw new SyntaxError('not json'); } })); page.settings = {}; - page._collectScanPaths = () => []; await page.saveSettings(); assert.deepEqual(notifications, [['error', 'Failed to save settings']]); }); -test('admin maintenance names the job that refused it', async () => { - const { page, dom, notifications } = load('admin.js', 'PromptAdmin', - async () => jsonResponse(409, { success: false, error: BUSY }), { - escapeHtml: text => String(text), - document: { readyState: 'loading', addEventListener() {}, getElementById: id => dom.getElementById(id), - querySelectorAll: selector => (selector === '.maintenance-option:checked' ? [{ id: 'vacuum' }] : []) }, - }); +// #330 S13-9: the admin page's reduced Page options save sends only the list's scroll default, so a stale legacy form +// can no longer overwrite scan folders or other settings edited in System. +test('admin Page options save posts only infinite_scroll', async () => { + const sent = []; + const { page, dom, notifications } = load('admin.js', 'PromptAdmin', async (url, init) => { + sent.push([url, init.method, JSON.parse(init.body)]); + return jsonResponse(200, { success: true, restart_required: true }); + }); + page.settings = { infiniteScroll: false }; + const scrolled = []; + page.setInfiniteScroll = (on, options) => scrolled.push([on, { ...options }]); + dom.getElementById('infiniteScrollDefault').checked = true; - await page.runMaintenance(); + await page.saveSettings(); - const results = dom.nodes.maintenanceResults.innerHTML; - assert.match(results, /Maintenance Failed/); - assert.match(results, new RegExp(BUSY)); - assert.doesNotMatch(results, /HTTP 409/); - assert.deepEqual(notifications, [['error', 'โŒ Maintenance failed']]); - assert.equal(dom.nodes.runMaintenanceBtn.disabled, false); + assert.deepEqual(sent, [['/prompt_manager/settings', 'POST', { infinite_scroll: true }]]); + assert.deepEqual(scrolled, [[true, { session: false }]]); + assert.equal(page.settings.infiniteScroll, true); + assert.deepEqual(notifications, [['success', 'Page options saved'], ['hide', 'settingsModal']]); }); + diff --git a/tests/js/workspace-file-inspector.test.js b/tests/js/workspace-file-inspector.test.js index b89f2c43..a6eb836a 100644 --- a/tests/js/workspace-file-inspector.test.js +++ b/tests/js/workspace-file-inspector.test.js @@ -185,6 +185,26 @@ test('legacy fill applies to local records', async () => { assert.notEqual(zeroed.prompts.positive.text, ''); }); +test('a branch named by recorded run history shows the Recorded run badge', async () => { + // #386: the server labels sides whose save node recorded history named; Execution recorded stays stronger. + const base = await localRecord(new File([fs.readFileSync(new URL('f09.png', PARITY))], 'f09.png')); + const recorded = {...base, prompts:{positive:{...base.prompts.positive, source:'recorded-run'}, negative:{...base.prompts.negative, source:'recorded-run'}}}; + let view = renderFull(recorded); + try { + assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Recorded run']); + assert.ok(view.texts(n => n.tagName === 'p').includes('Prompt evidence: recorded run')); + } finally { view.done(); } + view = renderFull({...recorded, prompts:{...recorded.prompts, negative:{...recorded.prompts.negative, source:'execution'}}}); + try { + assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Execution recorded']); + } finally { view.done(); } + // A recorded branch whose negative left out a run-time part is still inferred from the workflow. + view = renderFull({...recorded, prompts:{...recorded.prompts, negative:{...recorded.prompts.negative, source:'runtime-parts-omitted'}}}); + try { + assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Workflow inferred']); + } finally { view.done(); } +}); + // #249 PR C Task C3 (D88): one chip row, the file's current image tags first, then D21's linked tags not already shown. function tagged(imageTags, library = {status:'linked', tags:['Minimal', 'warm_light', 'editorial'], tags_scope:'positive'}) { return {...record(), imageTags, library}; @@ -642,3 +662,32 @@ test('focus starts on the field, moves to the first checkbox once listed, and fa assert.equal(none.find(n => n.tagName === 'input' && n.type !== 'checkbox')[0].focused, true); } finally { none.done(); } }); + +test('category chip names its origin and shows an own category for any link status (#332)', async () => { + const base = await localRecord(localPng('a.png')); + const chipTexts = record => { + const {f, done} = renderFull(record); + try { + return f.walk(f.container, n => /(^| )chip( |$)/.test(n.className ?? '')).map(n => n.textContent); + } finally { done(); } + }; + const cases = [ + [{status:'none', category:'Scans', category_scope:'image'}, 'Image category: Scans'], + [{status:'ambiguous', category:'Scans', category_scope:'image'}, 'Image category: Scans'], + [{status:'linked', category:'Portraits', category_scope:'image'}, 'Image category: Portraits'], + [{status:'linked', category:'Landscapes', category_scope:'positive'}, 'Positive prompt category: Landscapes'], + [{status:'linked', category:'Recipes', category_scope:'pair'}, 'Recipe category: Recipes'], + ]; + for (const [library, text] of cases) { + const chips = chipTexts({...base, library:{...base.library, tags:[], tags_scope:null, ...library}}); + assert.ok(chips.includes(text), `${JSON.stringify(library)} -> ${chips}`); + } + for (const library of [ + {status:'none', category:'Inherited', category_scope:'positive'}, + {status:'linked', category:null, category_scope:null}, + {status:'none', category:null, category_scope:null}, + ]) { + const chips = chipTexts({...base, library:{...base.library, tags:[], tags_scope:null, ...library}}); + assert.equal(chips.some(chip => /category: /.test(chip)), false, JSON.stringify(library)); + } +}); diff --git a/tests/js/workspace-images-state.test.js b/tests/js/workspace-images-state.test.js index 23c02675..84b0fd1b 100644 --- a/tests/js/workspace-images-state.test.js +++ b/tests/js/workspace-images-state.test.js @@ -250,6 +250,23 @@ test('the count line reads N of M images and K positive prompts only when known' assert.equal(countLine({loaded:1200,total:3400,positiveCount:0}),'1,200 of 3,400 images ยท 0 positive prompts'); }); +test('the count line reads as loading, never as an empty result, until a first page answers (#391)', async () => { + const {countLine,createImageStream}=await source; + assert.equal(countLine({loaded:0,total:0,positiveCount:null,loading:true,error:null}),'Loading imagesโ€ฆ'); + // A failed first page and a settled empty one say what the server said; a later page keeps the loaded count. + assert.equal(countLine({loaded:0,total:0,positiveCount:null,loading:false,error:new Error('x')}),'0 of 0 images'); + assert.equal(countLine({loaded:0,total:0,positiveCount:0,loading:false,error:null}),'0 of 0 images ยท 0 positive prompts'); + assert.equal(countLine({loaded:40,total:102,positiveCount:2,loading:true,error:null}),'40 of 102 images ยท 2 positive prompts'); + // While a reset's first page is in flight the line reads as loading (the cleared state before it is replaced in + // the same task, so it is never painted). + let answer;const lines=[]; + const stream=createImageStream({api:{get:()=>new Promise(resolve=>{answer=resolve;})},onChange:state=>lines.push(countLine(state))}); + const loading=stream.reset({tags:['cinematic']}); + assert.equal(lines.at(-1),'Loading imagesโ€ฆ'); + answer({images:[{path:'a.png',root_index:0}],total:1,has_more:false,next_cursor:null,positive_count:1});await loading; + assert.equal(lines.at(-1),'1 of 1 image ยท 1 positive prompt'); +}); + test('the caption date names the newest result day under Newest first only', async () => { const {captionDate}=await source; const now=new Date(2026,9,17,15,0); diff --git a/tests/js/workspace-inspection-state.test.js b/tests/js/workspace-inspection-state.test.js index 5bdd7d89..520dcbe4 100644 --- a/tests/js/workspace-inspection-state.test.js +++ b/tests/js/workspace-inspection-state.test.js @@ -278,6 +278,7 @@ test('record rejects root path media and invalid versioned fields with retryable data => { data.resolution.sampler_id = {}; }, data => { data.library.tags_scope = 'recipe'; }, data => { data.library.category_scope = 'recipe'; }, + data => { data.library.tags_scope = 'image'; }, data => { delete data.library.category_scope; }, data => { delete data.library.tags_scope; }, ]) { @@ -942,3 +943,14 @@ test("names the server quotes (parentheses, '!*') still match their original and assert.equal(record.download.url, entry.url); assert.deepEqual(record.preview, { url: entry.thumbnail_url }); }); + +test('record accepts an image\'s own category on any link status (#332)', () => { + const toRecord = contract('toInspectionRecord'); + for (const status of ['linked', 'none', 'ambiguous', 'unavailable']) { + const data = envelope(); + Object.assign(data.library, { status, category: 'Scans', category_scope: 'image' }); + const record = toRecord(data, image(), extractGenerationFacts); + assert.equal(record.library.category, 'Scans'); + assert.equal(record.library.category_scope, 'image'); + } +}); diff --git a/tests/js/workspace-system-diagnostics.test.js b/tests/js/workspace-system-diagnostics.test.js index 65b0bddb..161c1bfa 100644 --- a/tests/js/workspace-system-diagnostics.test.js +++ b/tests/js/workspace-system-diagnostics.test.js @@ -175,6 +175,28 @@ test('Clear all logs states both counts and its toast says what was done', async assert.equal(clearToast(undefined), 'Logs cleared ยท 0 rotated copies deleted.'); }); +test('Clear all logs sends back exactly the files its confirmation showed (#389)', async () => { + const { clearBody, clearFailureText, COPY } = await diagnostics; + const files = [ + { filename: 'prompt_manager.log', kind: 'active', identity: '10:1761000000123456789:9007199254740993' }, + { filename: 'prompt_manager.log.1', kind: 'rotated', identity: '20:1761000000123456789:42' }, + { filename: 'prompt_manager.log.4', kind: 'link', identity: '5:1:7' }, + { filename: 'prompt_manager.log.bak', kind: 'other', identity: '1:1:8' }, + { filename: 'prompt_manager.log.2', kind: 'rotated' }, + ]; + // Identities stay strings: mtime_ns and large inodes are not exact JS numbers. + assert.deepEqual(clearBody(files), { expected: [ + { name: 'prompt_manager.log', identity: '10:1761000000123456789:9007199254740993' }, + { name: 'prompt_manager.log.1', identity: '20:1761000000123456789:42' }, + ] }); + assert.deepEqual(clearBody([]), { expected: [] }); + assert.deepEqual(clearBody(null), {}); + const refused = Object.assign(new Error('The log files changed since you confirmed; nothing was cleared.'), { status: 409 }); + assert.equal(clearFailureText(refused), refused.message); + assert.equal(clearFailureText(Object.assign(new Error('boom'), { status: 500 })), COPY.clearFailed); + assert.equal(clearFailureText(undefined), COPY.clearFailed); +}); + test('the test-link dialog names the prompt and how Missing-file links removes the record', async () => { const { testLinkSentence, promptName } = await diagnostics; assert.equal(testLinkSentence({ id: 812, text: 'Misty harbor' }), 'Writes a placeholder image record ' diff --git a/tests/private_temp.py b/tests/private_temp.py new file mode 100644 index 00000000..dee79ce9 --- /dev/null +++ b/tests/private_temp.py @@ -0,0 +1,24 @@ +"""A temp SQLite database in a folder no other test process shares (#346). + +``PromptManagerAPI`` sweeps its database's folder at startup for the stages, +uploads and safety-backup copies an interrupted restore or backup left. A +fixture database directly in the system temp folder made every other test +process that started an API (another worktree's suite, a pre-push hook) delete +this process's in-flight restore files, and left safety backups behind there. +""" + +import tempfile + + +def private_temp_db(case): + """Return a closed ``NamedTemporaryFile`` handle for an empty ``.db`` file. + + The file lives in a new folder that ``case`` removes at cleanup, after its + own teardown has closed and unlinked the database (cleanups run after + ``tearDown``), together with anything a restore or backup left next to it. + """ + folder = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + case.addCleanup(folder.cleanup) + handle = tempfile.NamedTemporaryFile(delete=False, suffix=".db", dir=folder.name) + handle.close() + return handle diff --git a/tests/test_api.py b/tests/test_api.py index 051881a8..efcb433a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,7 +8,6 @@ without requiring a full ComfyUI server. import json import os import sys -import tempfile import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -18,6 +17,7 @@ from aiohttp.test_utils import AioHTTPTestCase from database.operations import PromptDatabase from py.api import PromptManagerAPI +from tests.private_temp import private_temp_db from utils.hashing import generate_prompt_hash @@ -25,8 +25,7 @@ class APITestCase(AioHTTPTestCase): """Base class that stands up a test aiohttp app with PromptManager routes.""" async def get_application(self): - self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db") - self._temp_db.close() + self._temp_db = private_temp_db(self) app = web.Application() routes = web.RouteTableDef() diff --git a/tests/test_api_autotag.py b/tests/test_api_autotag.py index e2d2dffc..cfe3f47a 100644 --- a/tests/test_api_autotag.py +++ b/tests/test_api_autotag.py @@ -35,6 +35,7 @@ from aiohttp.test_utils import AioHTTPTestCase from database.operations import PromptDatabase from py.api import PromptManagerAPI +from tests.private_temp import private_temp_db from py.api.autotag_routes import MODEL_BUSY_ERROR from py.api.jobs import JobBusy from utils.hashing import generate_prompt_hash @@ -143,8 +144,7 @@ class AutotagAPITestCase(AioHTTPTestCase): fake engine.""" async def get_application(self): - self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db") - self._temp_db.close() + self._temp_db = private_temp_db(self) self.tmp_root = Path(tempfile.mkdtemp()) self.output_dir = self.tmp_root / "output" self.output_dir.mkdir() diff --git a/tests/test_api_backup_restore.py b/tests/test_api_backup_restore.py index e9b451cf..32da612d 100644 --- a/tests/test_api_backup_restore.py +++ b/tests/test_api_backup_restore.py @@ -264,6 +264,41 @@ class TestRestoreRoute(BackupRestoreTestCase): texts = sorted(p["text"] for p in self.api.db.search_prompts()) self.assertEqual(texts, ["restored a", "restored b"]) + async def test_restore_round_trip_keeps_tags_and_image_links(self): + """A downloaded backup brings back every prompt with its tags and images. + + The legacy admin page's Restore form left for System (#330); this keeps + its browser test's data check at the route. + """ + db = self.api.db + originals = {} + for text, tags, image in ( + ("round trip cat", ["cats", "pets"], "cat.png"), + ("round trip dog", ["dogs"], "dog.png"), + ): + prompt_id = db.save_prompt( + text=text, tags=tags, prompt_hash=generate_prompt_hash(text) + ) + path = self._write(image, b"\x89PNG\r\n\x1a\n") + self.assertTrue(db.link_image_to_prompt(str(prompt_id), path)) + originals[text] = (prompt_id, sorted(tags), [path]) + + resp = await self.client.get("/prompt_manager/backup") + self.assertEqual(resp.status, 200) + backup = self._write("downloaded.db", await resp.read()) + self.assertTrue(db.delete_prompt(originals["round trip dog"][0])) + + resp = await self._restore(backup) + self.assertEqual(resp.status, 200, await resp.text()) + + restored = {p["text"]: p for p in self.api.db.search_prompts()} + self.assertEqual(sorted(restored), sorted(originals)) + for text, (prompt_id, tags, paths) in originals.items(): + self.assertEqual(restored[text]["id"], prompt_id) + self.assertEqual(sorted(restored[text]["tags"]), tags) + images = self.api.db.get_prompt_images(str(prompt_id)) + self.assertEqual([image["image_path"] for image in images], paths) + async def test_restore_leaves_no_temp_files_behind(self): upload = self._make_db_file(["restored"]) created = [] diff --git a/tests/test_api_duplicate_files.py b/tests/test_api_duplicate_files.py index a708e229..9350a491 100644 --- a/tests/test_api_duplicate_files.py +++ b/tests/test_api_duplicate_files.py @@ -18,15 +18,19 @@ from pathlib import Path from unittest.mock import patch try: + from tests.disk_waits import DISK_BOUND_SECONDS from tests.open_handles import open_handles_under from tests.test_api_maintenance import ApplyTestCase, _digest except ImportError: # discovered with tests/ as the top-level directory + from disk_waits import DISK_BOUND_SECONDS from open_handles import open_handles_under from test_api_maintenance import ApplyTestCase, _digest from py.api import duplicate_files # noqa: E402 from py.config import GalleryConfig, PromptManagerConfig # noqa: E402 +# Gates and server-state polls. Waits on a find or cleanup job's task use +# DISK_BOUND_SECONDS: those jobs hash, back up and unlink files (#409). WAIT = 10 FIND = "/prompt_manager/jobs/duplicates" GROUPS = "/prompt_manager/maintenance/duplicates/" @@ -83,7 +87,7 @@ class DuplicateFilesTestCase(ApplyTestCase): self.assertEqual(response.status, 202, text) self.assertNotIn(self.tmpdir, text) job = self.api._jobs.get(json.loads(text)["job"]["id"]) - await asyncio.wait_for(asyncio.shield(job.task), WAIT) + await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS) return job async def groups(self, job_id, status=200, **query): @@ -127,7 +131,7 @@ class DuplicateFilesTestCase(ApplyTestCase): plan = await self.review(job.id, [self.keep_of(group, path)]) body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) return plan, cleanup async def until(self, predicate): @@ -178,7 +182,7 @@ class TestFindJob(DuplicateFilesTestCase): gate.set() body = await (await legacy).text() job = self.api._jobs.get(summary["id"]) - await asyncio.wait_for(asyncio.shield(job.task), WAIT) + await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS) events = _sse_payloads(body) done = events[-1] self.assertEqual(done["type"], "complete") @@ -241,7 +245,7 @@ class TestFindJob(DuplicateFilesTestCase): self.assertTrue((await cancel.json())["cancellation_requested"]) gate.set() job = self.api._jobs.get(job_id) - await asyncio.wait_for(asyncio.shield(job.task), WAIT) + await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS) self.assertEqual(job.state, "cancelled") last = _sse_payloads(await (await stream).text())[-1] self.assertEqual(last, {"type": "error", "message": "Duplicate scan cancelled"}) @@ -341,6 +345,8 @@ class TestReview(DuplicateFilesTestCase): self.assertEqual(plan["protected"], {"count": 0, "reasons": []}) async def test_cannot_delete_keeper_or_outside_group(self): + # About which files may go, not the backup: skip its fsync-heavy copy. + PromptManagerConfig.SAFETY_BACKUP = False self.trio() self.write("e.png", b"other pair") self.write("f.png", b"other pair") @@ -372,7 +378,7 @@ class TestReview(DuplicateFilesTestCase): self.assertEqual([row["label"] for row in plan["details"]], ["b.png"]) body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.counters["removed"], 1) self.assertTrue((self.output_dir / "a.png").is_file()) self.assertTrue((self.output_dir / "sub" / "c.png").is_file()) @@ -496,7 +502,7 @@ class TestCleanupJob(DuplicateFilesTestCase): plan = await self.review(job.id, keep) body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertTrue(kept.is_file()) self.assertFalse(gone.exists()) self.assertFalse(video_thumb.exists()) @@ -508,7 +514,7 @@ class TestCleanupJob(DuplicateFilesTestCase): self.write("b.png", b"new bytes") # changed since the review body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.counters, {"removed": 1, "kept": 2, "errors": 0}) self.assertEqual((self.output_dir / "b.png").read_bytes(), b"new bytes") self.assertFalse((self.output_dir / "sub" / "c.png").exists()) @@ -522,7 +528,7 @@ class TestCleanupJob(DuplicateFilesTestCase): self.write("a.png", b"edited keeper") body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.counters["removed"], 0) self.assertTrue((self.output_dir / "b.png").is_file()) self.assertTrue((self.output_dir / "sub" / "c.png").is_file()) @@ -549,7 +555,7 @@ class TestCleanupJob(DuplicateFilesTestCase): with patch.object(duplicate_files, "_unlink", side_effect=unlink): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.state, "failed") self.assertEqual(cleanup.counters, {"removed": 1, "kept": 1, "errors": 1}) self.assertEqual( @@ -584,7 +590,7 @@ class TestCleanupJob(DuplicateFilesTestCase): ): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.counters, {"removed": 0, "kept": 3, "errors": 1}) self.assertEqual( cleanup.outcomes["errors"][0]["detail"], duplicate_files.DELETE_FAILED @@ -619,7 +625,7 @@ class TestCleanupJob(DuplicateFilesTestCase): with context: body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) finally: if handle is not None: # pragma: no cover - Windows handle.close() @@ -642,7 +648,7 @@ class TestCleanupJob(DuplicateFilesTestCase): with patch.object(duplicate_files, "_unlink", side_effect=unlink): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.counters["removed"], 2) self.assertEqual(open_at_unlink, []) @@ -736,7 +742,7 @@ class TestCleanupJob(DuplicateFilesTestCase): with patch.object(duplicate_files, "_unlink", side_effect=unlink): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) # Its records moved to the identical kept copy before the unlink failed. [image] = self.api.db.get_prompt_images(prompt) self.assertTrue(image["image_path"].endswith("a.png")) @@ -752,7 +758,7 @@ class TestCleanupJob(DuplicateFilesTestCase): self.assertEqual((busy["code"], busy["running"]["id"]), ("busy", find["id"])) gate.set() await asyncio.wait_for( - asyncio.shield(self.api._jobs.get(find["id"]).task), WAIT + asyncio.shield(self.api._jobs.get(find["id"]).task), DISK_BOUND_SECONDS ) # The token survived the refusal; the cleanup holds off a new find. release = asyncio.Event() @@ -770,7 +776,7 @@ class TestCleanupJob(DuplicateFilesTestCase): self.assertEqual(legacy.status, 409) release.set() cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) async def test_cleanup_refused_up_front_when_the_backup_does_not_fit(self): from py.api import maintenance @@ -1001,7 +1007,7 @@ class TestReviewHelpers(DuplicateFilesTestCase): GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.second_source())] body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.state, "failed") self.assertEqual(cleanup.last_event["code"], "sources-changed") self.assertEqual(self.backups(), []) @@ -1021,7 +1027,7 @@ class TestReviewHelpers(DuplicateFilesTestCase): ): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) self.assertEqual(cleanup.state, "failed") self.assertEqual( cleanup.outcomes["errors"][-1]["detail"], duplicate_files.RECORDS_FAILED diff --git a/tests/test_api_duplicate_safety.py b/tests/test_api_duplicate_safety.py index 02115143..8ae50edb 100644 --- a/tests/test_api_duplicate_safety.py +++ b/tests/test_api_duplicate_safety.py @@ -19,17 +19,17 @@ from pathlib import Path from unittest.mock import patch try: + from tests.disk_waits import DISK_BOUND_SECONDS from tests.open_handles import open_handles_under from tests import test_api_duplicate_files as dup_tests except ImportError: # discovered with tests/ as the top-level directory import test_api_duplicate_files as dup_tests + from disk_waits import DISK_BOUND_SECONDS from open_handles import open_handles_under from py.api import duplicate_files # noqa: E402 from py.config import GalleryConfig # noqa: E402 -WAIT = 10 - def _locator(path): return os.path.normcase(os.path.realpath(str(path))) @@ -49,7 +49,7 @@ class SafetyTestCase(dup_tests.DuplicateFilesTestCase): async def apply_plan(self, plan): body = await self.apply(plan["token"]) cleanup = self.api._jobs.get(body["job"]["id"]) - await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS) return cleanup diff --git a/tests/test_api_history.py b/tests/test_api_history.py index 67476d7a..dc952332 100644 --- a/tests/test_api_history.py +++ b/tests/test_api_history.py @@ -1169,3 +1169,47 @@ class HistoryHostCleanupTests(unittest.IsolatedAsyncioTestCase): fence.validate(epoch, generation) with self.assertRaises(sqlite3.Error): api._history_fence() + + +class SharedTempFolderRestoreTests(ImageAPITestCase): + """#346: a restore survives another suite starting in the shared temp folder. + + Every ``PromptManagerAPI`` sweeps its database's folder for stages and + uploads an interrupted restore left. A fixture database directly in the + shared temp folder let any other test process (another worktree's suite, + a pre-push hook) delete this restore's files mid-request. The shared + folder here is a scratch stand-in for the system temp folder, so the test + never sweeps the real one. + """ + + async def get_application(self): + shared = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + # Cleanups run last-in first-out: this one after the fixture's own. + self.addCleanup(shared.cleanup) + self.shared_tmp = shared.name + with patch.object(tempfile, "tempdir", shared.name): + return await super().get_application() + + async def test_restore_survives_another_suite_sweeping_the_shared_folder(self): + from database import restore_stage + from database.models import PromptModel + + self.api.db.save_prompt("kept by the upload") + source = self.comfy_dir / "upload.db" + self.assertTrue(self.api.db.model.backup_database(str(source))) + stage = PromptModel.stage_restore + swept = [] + + def stage_then_other_suite_starts(model, src_path): + staged = stage(model, src_path) + # What another suite's PromptManagerAPI does at startup. + other = os.path.join(self.shared_tmp, "other-suite.db") + swept.extend(restore_stage.sweep_restore_temps(other)) + return staged + + form = FormData() + form.add_field("database_file", source.read_bytes(), filename="upload.db") + with patch.object(PromptModel, "stage_restore", stage_then_other_suite_starts): + response = await self.client.post("/prompt_manager/restore", data=form) + self.assertEqual(response.status, 200, await response.text()) + self.assertEqual(swept, [], "another suite's sweep reached this restore") diff --git a/tests/test_api_history_outputs.py b/tests/test_api_history_outputs.py index c250cc27..c43f43f1 100644 --- a/tests/test_api_history_outputs.py +++ b/tests/test_api_history_outputs.py @@ -21,6 +21,8 @@ from tests.test_api_images import ImageAPITestCase, make_png from tests.test_api_prompt_inspector import graph, png from tests.test_execution_context_database import bundle as context_bundle from tests.test_execution_history_database import JOB, SESSION +from tests.saver_anchor_graphs import BRANCH_TEXTS, shared_prefix +from tests.test_recorded_saver_anchor import LLM_TEXT, text_generate_run from utils.execution_contexts import ContextBundle from utils.execution_history_events import make_finish, make_start from utils.execution_outputs import OutputBundle, OutputReference, snapshot_output_roots @@ -1505,3 +1507,88 @@ class HistoryOutputAPITests(ImageAPITestCase): body = await response.json() self.assertEqual(response.status, 200, body) self.assertEqual(body["prompts"]["positive"]["text"], "Readable portrait") + + async def recorded_shared_prefix_run(self): + """#386: the dataset shape, branch 3's saver wrote the file in a run.""" + path = png(self.output_dir / "IMG_GEN_00371_.png", prompt=shared_prefix()) + # make_start/make_finish observe the run at 0 ms: the file was written then. + os.utime(path, ns=(1_000_000, 1_000_000)) + start = self.capture( + [self.ref(node="33", filename=path.name), self.ref("43", 1, "other.png")] + ) + return start + + async def test_both_inspect_routes_name_the_branch_recorded_history_saved(self): + start = await self.recorded_shared_prefix_run() + _, body = await self.outputs(start) + descriptor = next( + item["current_candidate"]["media"] + for item in body["items"] + if item["occurrence"]["output_node_id"] == "33" + ) + response = await self.current(descriptor["inspection_url"]) + recorded = await response.json() + self.assertEqual(response.status, 200, recorded) + response = await self.client.get( + "/prompt_manager/images/inspect/IMG_GEN_00371_.png", params={"root": "0"} + ) + images = await response.json() + self.assertEqual(response.status, 200, images) + for result in (recorded, images): + self.assertEqual(result["resolution"]["state"], "resolved") + self.assertEqual( + result["prompts"]["positive"], + { + "status": "resolved", + "text": BRANCH_TEXTS[2], + "source": "recorded-run", + }, + ) + + async def test_a_file_rewritten_after_its_run_keeps_several_prompts(self): + await self.recorded_shared_prefix_run() + os.utime(self.output_dir / "IMG_GEN_00371_.png") # written now: not that run + response = await self.client.get( + "/prompt_manager/images/inspect/IMG_GEN_00371_.png", params={"root": "0"} + ) + body = await response.json() + self.assertEqual(response.status, 200, body) + self.assertEqual(body["resolution"]["state"], "ambiguous") + self.assertEqual(body["prompts"]["positive"]["status"], "ambiguous") + + async def test_a_failing_history_lookup_judges_the_file_alone(self): + await self.recorded_shared_prefix_run() + with ( + patch( + "database.run_attribution.recorded_output", + side_effect=sqlite3.OperationalError("locked"), + ), + self.assertLogs("prompt_manager.run_attribution", "WARNING"), + ): + response = await self.client.get( + "/prompt_manager/images/inspect/IMG_GEN_00371_.png", + params={"root": "0"}, + ) + body = await response.json() + self.assertEqual(response.status, 200, body) + self.assertEqual(body["resolution"]["state"], "ambiguous") + + async def test_a_language_model_text_shows_what_promptmanager_recorded(self): + path = png(self.output_dir / "moss_00026_.png", prompt=text_generate_run()) + os.utime(path, ns=(1_000_000, 1_000_000)) + start = self.capture([self.ref(node="9", filename=path.name)]) + self.api.db.write_execution_contexts( + start.run_id, + start.storage_epoch, + context_bundle("", LLM_TEXT, anchor="7"), + ) + response = await self.client.get( + "/prompt_manager/images/inspect/moss_00026_.png", params={"root": "0"} + ) + body = await response.json() + self.assertEqual(response.status, 200, body) + self.assertEqual( + body["prompts"]["positive"], + {"status": "resolved", "text": LLM_TEXT, "source": "execution"}, + ) + self.assertEqual(body["prompts"]["negative"]["text"], "") diff --git a/tests/test_api_image_categories.py b/tests/test_api_image_categories.py new file mode 100644 index 00000000..02905267 --- /dev/null +++ b/tests/test_api_image_categories.py @@ -0,0 +1,375 @@ +"""Image category routes (#332): PUT/DELETE /images/category, GET /images/categories.""" + +import json +import ntpath +import sqlite3 +from unittest.mock import patch + +try: + from test_api_images import ImageAPITestCase, make_png + from open_handles import assert_closed + from test_image_categories import category_checksum +except ImportError: + from tests.test_api_images import ImageAPITestCase, make_png + from tests.open_handles import assert_closed + from tests.test_image_categories import category_checksum + +from database.image_categories import MAX_KEY_LENGTH, TABLE +from py.api.images import _favorite_root_key, image_category_key + +CATEGORY = "/prompt_manager/images/category" +CATEGORIES = "/prompt_manager/images/categories" + + +class ImageCategoryTestCase(ImageAPITestCase): + async def tearDownAsync(self): + self.api.db.close_all() + assert_closed(self, self._temp_db.name, self._output_tmp.name) + await super().tearDownAsync() + + def target(self, path, index=0): + roots = self.api._get_all_output_dirs() + return { + "root_index": index, + "root_scope": self.api._gallery_root_scope(roots[index], index), + "path": path, + } + + async def send(self, method, body, status=200): + response = await self.client.request(method, CATEGORY, json=body) + text = await response.text() + self.assertEqual(response.status, status, text) + return json.loads(text) + + async def put(self, paths, category, status=200): + items = [p if isinstance(p, dict) else self.target(p) for p in paths] + return await self.send("PUT", {"items": items, "category": category}, status) + + async def delete(self, paths, status=200): + items = [p if isinstance(p, dict) else self.target(p) for p in paths] + return await self.send("DELETE", {"items": items}, status) + + async def listed(self, status=200): + response = await self.client.get(CATEGORIES) + text = await response.text() + self.assertEqual(response.status, status, text) + return json.loads(text) + + def rows(self): + return [ + tuple(row) + for row in self.api.db.model.get_connection().execute( + f"SELECT root_key, relative_path, category FROM {TABLE} " + "ORDER BY root_key, relative_path" + ) + ] + + def key(self, relative, root=None): + roots = self.api._get_all_output_dirs() + return image_category_key(roots, (root or self.output_dir) / relative) + + +class TestSetAndClear(ImageCategoryTestCase): + async def test_put_sets_listed_and_counts_missing(self): + make_png(self.output_dir / "a.png") + make_png(self.output_dir / "sub" / "b.png") + data = await self.put(["a.png", "sub/b.png", "missing.png"], " Portraits ") + self.assertEqual( + data, + { + "success": True, + "category": "Portraits", + "updated": 2, + "unchanged": 0, + "not_found": 1, + }, + ) + root_key = _favorite_root_key(self.output_dir) + self.assertEqual( + self.rows(), + [(root_key, "a.png", "Portraits"), (root_key, "sub/b.png", "Portraits")], + ) + data = await self.put(["a.png", "sub/b.png"], "Portraits") + self.assertEqual((data["updated"], data["unchanged"]), (0, 2)) + data = await self.put(["a.png"], "Scans") + self.assertEqual((data["updated"], data["unchanged"]), (1, 0)) + self.assertEqual(self.api.db.image_category(*self.key("a.png")), "Scans") + + async def test_put_with_no_listed_file_is_404_and_writes_nothing(self): + data = await self.put(["missing.png", "gone/x.png"], "Scans", status=404) + self.assertEqual(data["code"], "files-not-found") + self.assertEqual(self.rows(), []) + + async def test_duplicate_targets_are_written_and_counted_once(self): + make_png(self.output_dir / "a.png") + item = self.target("a.png") + data = await self.put([item, dict(item)], "Scans") + self.assertEqual((data["updated"], data["not_found"]), (1, 0)) + + async def test_delete_clears_listed_and_missing_files(self): + make_png(self.output_dir / "a.png") + await self.put(["a.png"], "Scans") + self.api.db.set_image_categories([self.key("gone.png")], "Scans") + data = await self.delete(["a.png", "gone.png", "never.png"]) + self.assertEqual(data, {"success": True, "cleared": 2}) + self.assertEqual(self.rows(), []) + self.assertEqual((await self.delete(["a.png"]))["cleared"], 0) + + async def test_put_never_writes_prompts(self): + make_png(self.output_dir / "a.png") + make_png(self.output_dir / "b.png") + positive = self._save_prompt("castle at dusk", category="Landscapes") + self.api.db.link_image_to_prompt(positive, str(self.output_dir / "a.png")) + negative = self._save_prompt("blurry", category="Negatives") + self.api.db.save_prompt_pair("castle at dusk", "blurry", category="Recipes") + conn = self.api.db.model.get_connection() + before = category_checksum(conn) + links = conn.execute("SELECT * FROM generated_images").fetchall() + await self.put(["a.png", "b.png"], "Portraits") + await self.listed() + await self.delete(["a.png"]) + self.assertEqual(category_checksum(conn), before) + self.assertEqual( + conn.execute("SELECT * FROM generated_images").fetchall(), links + ) + self.assertIsNotNone(negative) + + async def test_two_sources_in_one_request(self): + second = self.comfy_dir / "renders" + make_png(self.output_dir / "a.png") + make_png(second / "a.png") + self.api._get_all_output_dirs = lambda: [self.output_dir, second] + data = await self.put([self.target("a.png", 0), self.target("a.png", 1)], "X") + self.assertEqual(data["updated"], 2) + self.assertEqual( + {row[0] for row in self.rows()}, + {_favorite_root_key(self.output_dir), _favorite_root_key(second)}, + ) + self.assertEqual((await self.delete([self.target("a.png", 1)]))["cleared"], 1) + + async def test_nested_sources_key_by_the_deepest(self): + inner = self.output_dir / "nested" + make_png(inner / "a.png") + self.api._get_all_output_dirs = lambda: [self.output_dir, inner] + # Named through the outer source, stored under the inner one. + data = await self.put([self.target("nested/a.png", 0)], "Scans") + self.assertEqual(data["updated"], 1) + self.assertEqual(self.rows(), [(_favorite_root_key(inner), "a.png", "Scans")]) + # The same file through the inner source is the same key. + data = await self.put([self.target("a.png", 1)], "Scans") + self.assertEqual(data["unchanged"], 1) + self.assertEqual((await self.listed())["categories"], ["Scans"]) + self.assertEqual((await self.delete([self.target("a.png", 1)]))["cleared"], 1) + + async def test_video_files_can_have_a_category(self): + (self.output_dir / "clip.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42") + data = await self.put(["clip.mp4"], "Clips") + self.assertEqual(data["updated"], 1) + + async def test_windows_case_folds_the_key(self): + make_png(self.output_dir / "Sub" / "A.png") + with patch("py.api.images._path_flavor", ntpath): + data = await self.put(["Sub/A.png"], "Scans") + self.assertEqual(data["updated"], 1) + data = await self.put(["sub/a.PNG"], "Scans") + self.assertEqual(data["unchanged"], 1) + self.assertEqual(self.rows()[0][1], "sub/a.png") + self.assertEqual((await self.delete(["SUB/A.png"]))["cleared"], 1) + + async def test_a_key_beyond_the_stored_bound_is_403(self): + make_png(self.output_dir / "a.png") + too_long = "/" + "r" * MAX_KEY_LENGTH + with patch("py.api.images._favorite_root_key", return_value=too_long): + data = await self.put(["a.png"], "Scans", status=403) + self.assertNotIn(too_long, json.dumps(data)) + await self.delete(["a.png"], status=403) + self.assertEqual(self.rows(), []) + + async def test_an_unexpected_failure_is_a_logged_500(self): + make_png(self.output_dir / "a.png") + + def broken(*_args, **_kwargs): + raise sqlite3.OperationalError("secret detail") + + self.api.db.set_image_categories = broken + self.api.db.clear_image_categories = broken + with self.assertLogs("prompt_manager", level="ERROR"): + data = await self.put(["a.png"], "Scans", status=500) + self.assertEqual(data["error"], "The category could not be changed.") + self.assertNotIn("secret", json.dumps(data)) + with self.assertLogs("prompt_manager", level="ERROR"): + data = await self.delete(["a.png"], status=500) + self.assertNotIn("secret", json.dumps(data)) + + async def test_a_symlinked_output_folder_keys_like_its_favorites(self): + """A detected output folder can be a symlink: keys use its real path.""" + real = self.comfy_dir / "real-output" + make_png(real / "a.png") + link = self.comfy_dir / "linked-output" + self._make_symlink(real, link, target_is_directory=True) + self.api._get_all_output_dirs = lambda: [link] + data = await self.put(["a.png"], "Scans") + self.assertEqual(data["updated"], 1) + self.assertEqual(self.rows(), [(_favorite_root_key(link), "a.png", "Scans")]) + self.assertEqual((await self.listed())["categories"], ["Scans"]) + self.assertEqual((await self.delete(["a.png"]))["cleared"], 1) + + async def test_lone_surrogates_are_400_not_500(self): + make_png(self.output_dir / "a.png") + await self.send( + "PUT", {"items": [self.target("a.png")], "category": "x\ud800"}, 400 + ) + await self.send("DELETE", {"items": [self.target("a\ud800.png")]}, 400) + self.assertEqual(self.rows(), []) + + async def test_delete_category_never_reaches_delete_image(self): + make_png(self.output_dir / "a.png") + calls = [] + + async def recorded(request): + calls.append(request.match_info) + raise AssertionError("delete_image reached") + + self.api.delete_image = recorded + await self.put(["a.png"], "Scans") + data = await self.delete(["a.png"]) + self.assertEqual(data["cleared"], 1) + self.assertEqual(calls, []) + + +class TestRefusals(ImageCategoryTestCase): + async def test_malformed_bodies_are_400(self): + make_png(self.output_dir / "a.png") + item = self.target("a.png") + cases = [ + {"items": [item]}, + {"category": "x"}, + {"items": [item], "category": "x", "extra": 1}, + {"items": item, "category": "x"}, + {"items": [], "category": "x"}, + {"items": [item] * 501, "category": "x"}, + {"items": [item], "category": 3}, + {"items": [item], "category": None}, + {"items": [item], "category": " \t "}, + {"items": [item], "category": "x" * 101}, + {"items": [item], "category": "a\x07b"}, + {"items": [{**item, "root_index": "0"}], "category": "x"}, + {"items": [{**item, "root_scope": "nothex"}], "category": "x"}, + {"items": [{"root_index": 0, "path": "a.png"}], "category": "x"}, + {"items": [{**item, "root_index": 1}], "category": "x"}, + ] + for body in cases: + with self.subTest(body=str(body)[:80]): + data = await self.send("PUT", body, status=400) + self.assertFalse(data["success"]) + await self.send("DELETE", {"items": [item], "category": "x"}, status=400) + await self.send("DELETE", {"items": []}, status=400) + response = await self.client.request( + "PUT", CATEGORY, data="not json", headers={"Content-Type": "text/plain"} + ) + self.assertEqual(response.status, 400) + self.assertEqual(self.rows(), []) + + async def test_unsafe_paths_are_403(self): + (self.output_dir / "notes.txt").write_text("not media") + for path in ("../outside.png", "/a.png", "C:/a.png", "a\\b.png", "a/./b.png"): + with self.subTest(path=path): + await self.put([self.target(path)], "x", status=403) + data = await self.put([self.target("notes.txt")], "x", status=403) + self.assertEqual(data["error"], "Only media files can have a category") + await self.delete([self.target("notes.txt")], status=403) + + async def test_a_stale_scope_is_409(self): + make_png(self.output_dir / "a.png") + item = {**self.target("a.png"), "root_scope": "0" * 64} + data = await self.put([item], "x", status=409) + self.assertEqual(data["code"], "source-changed") + data = await self.delete([item], status=409) + self.assertEqual(data["code"], "source-changed") + + async def test_an_oversized_body_is_413(self): + response = await self.client.request( + "PUT", + CATEGORY, + data=b'{"items": [], "category": "' + b"x" * (2 * 1024 * 1024) + b'"}', + headers={"Content-Type": "application/json"}, + ) + self.assertEqual(response.status, 413) + + +class TestCategoryList(ImageCategoryTestCase): + async def test_lists_only_values_whose_file_is_listed(self): + make_png(self.output_dir / "a.png") + make_png(self.output_dir / "c.png") + await self.put(["a.png"], "B") + await self.put(["c.png"], "A") + self.api.db.set_image_categories([self.key("gone.png")], "Z") + self.assertEqual( + await self.listed(), {"success": True, "categories": ["A", "B"]} + ) + + async def test_empty_library_lists_nothing(self): + self.assertEqual((await self.listed())["categories"], []) + + async def test_a_failed_read_is_500(self): + make_png(self.output_dir / "a.png") + + def broken(*_args, **_kwargs): + raise sqlite3.OperationalError("secret detail") + + self.api.db.own_categories_of = broken + with self.assertLogs("prompt_manager", level="ERROR"): + data = await self.listed(status=500) + self.assertEqual(data["error"], "Image categories could not be read.") + self.assertNotIn("secret", json.dumps(data)) + + +class TestAllMatchingFilter(ImageCategoryTestCase): + """#392 "Select all N matching": the listing filter, resolved now.""" + + async def test_filter_sets_and_clears_every_match(self): + for name in ("bulk/a.png", "bulk/b.png", "other/c.png"): + make_png(self.output_dir / name) + data = await self.send( + "PUT", {"filter": {"subfolder": "bulk"}, "category": "Scans"} + ) + self.assertEqual((data["updated"], data["not_found"]), (2, 0)) + self.assertEqual( + sorted(row[1] for row in self.rows()), ["bulk/a.png", "bulk/b.png"] + ) + data = await self.send("DELETE", {"filter": {"subfolder": "bulk"}}) + self.assertEqual(data["cleared"], 2) + data = await self.send( + "PUT", {"filter": {"subfolder": "none"}, "category": "Scans"}, 404 + ) + self.assertEqual(data["code"], "files-not-found") + self.assertEqual( + (await self.send("DELETE", {"filter": {"q": "zz"}}))["cleared"], 0 + ) + + async def test_filter_refusals(self): + make_png(self.output_dir / "a.png") + item = self.target("a.png") + for body in ( + {"filter": {}, "items": [item], "category": "x"}, + {"filter": {"unknown": "1"}, "category": "x"}, + {"filter": "q=a", "category": "x"}, + {"filter": {"root": "0", "root_scope": "nothex"}, "category": "x"}, + ): + with self.subTest(body=body): + await self.send("PUT", body, 400) + data = await self.send( + "PUT", + {"filter": {"root": "0", "root_scope": "0" * 64}, "category": "x"}, + 409, + ) + self.assertEqual(data["code"], "source-changed") + self.assertEqual(self.rows(), []) + + async def test_more_than_500_matching_is_refused(self): + for number in range(501): + make_png(self.output_dir / f"many/{number:03}.png", size=(1, 1)) + data = await self.send("PUT", {"filter": {}, "category": "x"}, 400) + self.assertEqual(data["code"], "too-many-images") + self.assertEqual(data["error"], "Select at most 500 images.") + self.assertEqual(self.rows(), []) diff --git a/tests/test_api_images.py b/tests/test_api_images.py index 08713ddb..cc74c6cf 100644 --- a/tests/test_api_images.py +++ b/tests/test_api_images.py @@ -42,6 +42,7 @@ from py import lora_utils # noqa: E402 from py import lora_manager_layout as layout_module # noqa: E402 from py.api.thumbnail_settings import ThumbnailSettings # noqa: E402 from py.api import PromptManagerAPI # noqa: E402 +from tests.private_temp import private_temp_db # noqa: E402 from py.api.prompts import MAX_PAGE_LIMIT # noqa: E402 from py.config import GalleryConfig # noqa: E402 from utils.hashing import generate_prompt_hash # noqa: E402 @@ -60,8 +61,7 @@ class ImageAPITestCase(AioHTTPTestCase): """App with PromptManager routes, a temp DB and a temp output directory.""" async def get_application(self): - self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db") - self._temp_db.close() + self._temp_db = private_temp_db(self) self._output_tmp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) # A fake ComfyUI tree: /output is the gallery root and is the diff --git a/tests/test_api_log_cleanup.py b/tests/test_api_log_cleanup.py index d9287088..9622e599 100644 --- a/tests/test_api_log_cleanup.py +++ b/tests/test_api_log_cleanup.py @@ -415,5 +415,74 @@ class TestApply(LogCleanupTestCase): self.assertEqual(job.status_line, "The log cleanup failed.") +TRUNCATE = "/prompt_manager/logs/truncate" +FILES = "/prompt_manager/logs/files" + + +class TestClearAllLogsConfirmed(LogCleanupTestCase): + """#389: POST /logs/truncate deletes no more than the confirmation listed.""" + + async def confirmed(self): + response = await self.client.request("GET", FILES) + data = await response.json() + return [ + {"name": row["filename"], "identity": row["identity"]} + for row in data["files"] + if row["kind"] in ("active", "rotated") + ] + + def handler(self): + return next( + h + for h in self.manager.logger.handlers + if isinstance(h, logging.FileHandler) + ) + + async def test_a_rollover_after_the_confirmation_is_refused(self): + self.write("prompt_manager.log.1", b"a" * 100) + self.write("prompt_manager.log.2", b"b" * 200) + expected = await self.confirmed() + self.app_logger.info("written after the confirmation") + self.handler().doRollover() + before = self.snapshot() + body = await self.post(TRUNCATE, {"expected": expected}, 409) + self.assertEqual(body["success"], False) + self.assertEqual(body["code"], "logs-changed") + self.assertIn("nothing was cleared", body["error"]) + self.assertEqual(self.snapshot(), before) + + async def test_the_confirmed_files_are_cleared(self): + self.write("prompt_manager.log.1", b"a" * 100) + expected = await self.confirmed() + body = await self.post(TRUNCATE, {"expected": expected}, 200) + self.assertEqual( + sorted(body["results"]["truncated"]), + ["prompt_manager.log", "prompt_manager.log.1"], + ) + self.assertEqual(self.names(), ["prompt_manager.log"]) + + async def test_the_legacy_call_without_a_body_still_clears(self): + self.write("prompt_manager.log.1", b"a" * 100) + response = await self.client.request("POST", TRUNCATE) + self.assertEqual(response.status, 200, await response.text()) + self.assertEqual(self.names(), ["prompt_manager.log"]) + + async def test_a_foreign_writer_is_refused_with_a_clear_message(self): + self.write("prompt_manager.log.1", b"a" * 100) + with open(self.path("prompt_manager.log"), "ab") as other: + other.write(b"line from another ComfyUI\n") + body = await self.post(TRUNCATE, {"expected": await self.confirmed()}, 409) + self.assertEqual(body["code"], "logs-foreign-writer") + self.assertIn("Another program is writing", body["error"]) + self.assertIn("prompt_manager.log.1", self.names()) + + async def test_a_malformed_confirmation_is_400(self): + self.write("prompt_manager.log.1", b"a" * 100) + for expected in ("all", [{"name": "prompt_manager.log.1", "identity": 5}]): + with self.subTest(expected=expected): + await self.post(TRUNCATE, {"expected": expected}, 400) + self.assertIn("prompt_manager.log.1", self.names()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_api_lora.py b/tests/test_api_lora.py index 8fda0562..99446360 100644 --- a/tests/test_api_lora.py +++ b/tests/test_api_lora.py @@ -36,6 +36,7 @@ from aiohttp.test_utils import AioHTTPTestCase from database.operations import PromptDatabase from py.api import PromptManagerAPI +from tests.private_temp import private_temp_db from py import config_store, lora_import, lora_utils from py.config import IntegrationConfig, PromptManagerConfig from py.api import settings_routes as settings_module @@ -57,8 +58,7 @@ class LoraAPITestCase(AioHTTPTestCase): """Stands up the PromptManager routes with a temp DB and neutral LoRA config.""" async def get_application(self): - self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db") - self._temp_db.close() + self._temp_db = private_temp_db(self) self.tmp_root = Path(tempfile.mkdtemp()) self.custom_nodes = self.tmp_root / "custom_nodes" self.custom_nodes.mkdir() diff --git a/tests/test_api_prompts.py b/tests/test_api_prompts.py index c4af9c68..c8625c25 100644 --- a/tests/test_api_prompts.py +++ b/tests/test_api_prompts.py @@ -5,7 +5,6 @@ Route tests for the prompt API (py/api/prompts.py). import json import os import sys -import tempfile import unittest from unittest.mock import MagicMock, patch @@ -29,6 +28,7 @@ except ImportError: # discovered with tests/ as the top-level directory from database.operations import PromptDatabase # noqa: E402 from py.api import PromptManagerAPI # noqa: E402 +from tests.private_temp import private_temp_db # noqa: E402 import py.api.prompts as prompts_module # noqa: E402 from py.api.prompts import ( # noqa: E402 BULK_CATEGORY_MAX_PROMPTS, @@ -43,8 +43,7 @@ class PromptAPITestCase(AioHTTPTestCase): """App with PromptManager routes and a temporary SQLite database.""" async def get_application(self): - self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db") - self._temp_db.close() + self._temp_db = private_temp_db(self) app = web.Application() routes = web.RouteTableDef() diff --git a/tests/test_api_review.py b/tests/test_api_review.py index 6eeaa89d..0e19c6ed 100644 --- a/tests/test_api_review.py +++ b/tests/test_api_review.py @@ -1040,8 +1040,8 @@ class TestApplyRefusals(ReviewRoutesTestCase): _path, item, detail, observed, cat = await self._opened() real = review_capture_module.resolve_prompt_pair - def reworded(metadata): - resolution = real(metadata) + def reworded(metadata, **kwargs): + resolution = real(metadata, **kwargs) return { **resolution, "reason": "a reworded reason from a newer resolver", @@ -1065,8 +1065,8 @@ class TestApplyRefusals(ReviewRoutesTestCase): _path, item, detail, observed, cat = await self._opened() real = review_capture_module.resolve_prompt_pair - def without_cat(metadata): - resolution = real(metadata) + def without_cat(metadata, **kwargs): + resolution = real(metadata, **kwargs) kept = [ c for c in resolution["candidates"] diff --git a/tests/test_api_vocabulary.py b/tests/test_api_vocabulary.py index 062c77d5..9b6a23f8 100644 --- a/tests/test_api_vocabulary.py +++ b/tests/test_api_vocabulary.py @@ -62,12 +62,16 @@ class TestVocabularyAPI(ImageAPITestCase): }, ) self.assertEqual( - categories["categories"], [{"value": "Animal", "prompt_count": 1}] + categories["categories"], + [{"value": "Animal", "prompt_count": 1, "image_count": 0}], ) _, rest = await self.get( "categories", limit=1, cursor=categories["next_cursor"] ) - self.assertEqual(rest["categories"], [{"value": "Pet", "prompt_count": 1}]) + self.assertEqual( + rest["categories"], + [{"value": "Pet", "prompt_count": 1, "image_count": 0}], + ) self.assertFalse(rest["has_more"]) async def test_vocabulary_cursor_is_signed_epoch_scoped(self): @@ -488,6 +492,7 @@ class TestVocabularyAPI(ImageAPITestCase): "other_prompts": 0, "negative_variants": 1, "archived_negative_variants": 0, + "images": 0, } self.assertEqual( preview, @@ -528,6 +533,73 @@ class TestVocabularyAPI(ImageAPITestCase): self.assertEqual(response.status, 409) self.assertEqual(replay["code"], "vocabulary-conflict") + async def test_category_rows_report_images_and_image_only_values(self): + self._save_prompt("Owner", category="Prompt value") + _, options_before = await self.get_options() + self.api.db.set_image_categories( + [("root", "a.png"), ("root", "b.png")], "Prompt value" + ) + self.api.db.set_image_categories([("root", "c.png")], "Image value") + response, first = await self.get("categories", limit=1) + self.assertEqual(response.status, 200) + self.assertEqual( + first["categories"], + [{"value": "Prompt value", "prompt_count": 1, "image_count": 2}], + ) + self.assertEqual(first["total"], 2) + _, rest = await self.get("categories", limit=1, cursor=first["next_cursor"]) + self.assertEqual( + rest["categories"], + [{"value": "Image value", "prompt_count": 0, "image_count": 1}], + ) + self.assertFalse(rest["has_more"]) + # The Prompt library's category list does not gain image values. + _, options_after = await self.get_options() + self.assertEqual(options_after, options_before) + self.assertEqual(options_after["categories"], ["Prompt value"]) + + async def get_options(self): + response = await self.client.get("/prompt_manager/library/options") + return response, await response.json() + + async def test_category_routes_rename_image_only_value(self): + epoch = self.api.db.get_execution_epoch() + self._save_prompt("Bystander", category="Kept") + self.api.db.set_image_categories( + [("root", "a.png"), ("second", "b.png")], "Image value" + ) + prompts_before = self.category_rows() + body = {"expected_epoch": epoch, "source": "Image value", "target": "Moved"} + response, preview = await self.send("POST", "categories/preview", body) + self.assertEqual(response.status, 200, preview) + impact = { + "positive_prompts": 0, + "other_prompts": 0, + "negative_variants": 0, + "archived_negative_variants": 0, + "images": 2, + } + self.assertEqual(preview["impact"], impact) + stale = preview["impact_token"] + self.api.db.set_image_categories([("root", "late.png")], "Image value") + response, refused = await self.send( + "POST", "categories/apply", {**body, "impact_token": stale} + ) + self.assertEqual(response.status, 409) + self.assertEqual(refused["code"], "vocabulary-conflict") + self.assertEqual(self.api.db.image_category("root", "a.png"), "Image value") + _, preview = await self.send("POST", "categories/preview", body) + response, applied = await self.send( + "POST", + "categories/apply", + {**body, "impact_token": preview["impact_token"]}, + ) + self.assertEqual(response.status, 200, applied) + self.assertEqual(applied["impact"], {**impact, "images": 3}) + for key in (("root", "a.png"), ("second", "b.png"), ("root", "late.png")): + self.assertEqual(self.api.db.image_category(*key), "Moved") + self.assertEqual(self.category_rows(), prompts_before) + async def test_preview_validation(self): epoch = self.api.db.get_execution_epoch() self._save_prompt("Owner", category="Scenery") diff --git a/tests/test_api_workspace_image_categories.py b/tests/test_api_workspace_image_categories.py new file mode 100644 index 00000000..f85b7913 --- /dev/null +++ b/tests/test_api_workspace_image_categories.py @@ -0,0 +1,262 @@ +"""Images filters by the effective category (#332 MC4, MC5, DC6). + +An image's effective category is its own category when it has one, else the +category its link supplies (the recipe's when set, else the positive's), else +none. The category filter, "select all N matching" and saved views match it. +""" + +import os +import sqlite3 +from unittest.mock import patch + +try: + import test_api_workspace_images as workspace + from test_image_categories import category_checksum +except ImportError: + from tests import test_api_workspace_images as workspace + from tests.test_image_categories import category_checksum + +from py.api.images import image_category_key, parse_listing_filter + + +class WorkspaceCategoryCase(workspace.ImageAPITestCase): + """Listing helpers plus own-category writes keyed as the routes key them.""" + + media = workspace.TestWorkspaceImages.media + get = workspace.TestWorkspaceImages.get + tag = workspace.TestWorkspaceImageTags.tag + names = workspace.TestWorkspaceImageTags.names + + async def tearDownAsync(self): + self.api.db.close_all() + workspace.assert_closed(self, self._temp_db.name, self._output_tmp.name) + await super().tearDownAsync() + + def own(self, path, value, sources=None): + sources = sources or self.api._get_all_output_dirs() + self.api.db.set_image_categories([image_category_key(sources, path)], value) + + async def listed(self, **params): + response, body = await self.get(sort="name", **params) + self.assertEqual(response.status, 200, body) + return self.names(body), body + + +class TestEffectiveCategoryFilter(WorkspaceCategoryCase): + def batch(self): + """Four images of "castle at dusk" (Landscapes); one is overridden.""" + castle = self._save_prompt("castle at dusk", category="Landscapes", rating=4) + files = [self.media(f"castle_{n}.png", mtime=100 + n) for n in range(4)] + for path in files: + self.api.db.link_image_to_prompt(castle, str(path)) + self.own(files[0], "Portraits") + return castle, files + + async def test_category_filter_prefers_own_category(self): + castle, _files = self.batch() + conn = self.api.db.model.get_connection() + before = category_checksum(conn) + names, body = await self.listed(category="Portraits") + self.assertEqual(names, ["castle_0.png"]) + # The caption still counts the prompt behind the image shown. + self.assertEqual(body["positive_count"], 1) + names, body = await self.listed(category="Landscapes") + self.assertEqual(names, ["castle_1.png", "castle_2.png", "castle_3.png"]) + self.assertEqual(body["total"], 3) + names, _ = await self.listed() + self.assertEqual(len(names), 4) + self.assertEqual(category_checksum(conn), before) + self.assertEqual(self.api.db.get_prompt_by_id(castle)["category"], "Landscapes") + + async def test_category_filter_lists_unlinked_with_own(self): + scans = [self.media(f"scan_{n}.png", mtime=100 + n) for n in range(2)] + self.media("plain.png", mtime=50) + for path in scans: + self.own(path, "Scans") + names, body = await self.listed(category="Scans") + self.assertEqual(names, ["scan_0.png", "scan_1.png"]) + self.assertEqual(body["positive_count"], 0) + + async def test_recipe_category_is_inherited(self): + """MC4: a recipe's own category, not the positive's, is inherited.""" + db = self.api.db + self._save_prompt("Pair positive", category="Night") + pair = db.save_prompt_pair("Pair positive", "Noise", category="Recipes") + blank = db.save_prompt_pair("Pair positive", "Other noise", category=" ") + recipe = self.media("recipe.png", mtime=103) + blank_recipe = self.media("blank_recipe.png", mtime=102) + positive = db.get_prompt_pair(pair)["positive_prompt_id"] + for path, pair_id in ((recipe, pair), (blank_recipe, blank)): + image = db.link_image_to_prompt(positive, str(path)) + db.link_image_to_pair(pair_id, image, {"source": "fixture"}) + self.assertEqual((await self.listed(category="Recipes"))[0], ["recipe.png"]) + # A blank recipe category falls back to the positive's. + self.assertEqual((await self.listed(category="Night"))[0], ["blank_recipe.png"]) + # An own category still wins over the recipe's. + self.own(recipe, "Mine") + self.assertEqual((await self.listed(category="Recipes"))[0], []) + self.assertEqual((await self.listed(category="Mine"))[0], ["recipe.png"]) + + async def test_ambiguous_files_inherit_nothing_but_keep_their_own(self): + first = self._save_prompt("Owner one", category="Night") + second = self._save_prompt("Owner two", category="Night") + ambiguous = self.media("ambiguous.png") + for owner in (first, second): + self.api.db.link_image_to_prompt(owner, str(ambiguous)) + self.assertEqual((await self.listed(category="Night"))[0], []) + self.own(ambiguous, "Night") + self.assertEqual((await self.listed(category="Night"))[0], ["ambiguous.png"]) + + async def test_combines_with_tags_search_and_rating(self): + stone = self._save_prompt( + "castle at dusk", category="Landscapes", tags=["stone"], rating=4 + ) + files = [self.media(f"castle_{n}.png", mtime=100 + n) for n in range(2)] + for path in files: + self.api.db.link_image_to_prompt(stone, str(path)) + self.own(files[0], "Portraits") + other = self._save_prompt("harbor at noon", category="Portraits", rating=2) + harbor = self.media("harbor.png", mtime=90) + self.api.db.link_image_to_prompt(other, str(harbor)) + names, _ = await self.listed(category="Portraits", tags="stone") + self.assertEqual(names, ["castle_0.png"]) + names, _ = await self.listed(category="Portraits", q="harbor") + self.assertEqual(names, ["harbor.png"]) + names, _ = await self.listed(category="Portraits", q="castle_0") + self.assertEqual(names, ["castle_0.png"]) + response, body = await self.get(category="Portraits", sort="rating") + self.assertEqual(response.status, 200, body) + self.assertEqual(self.names(body), ["castle_0.png", "harbor.png"]) + self.assertEqual(body["positive_count"], 2) + + async def test_image_tag_admitted_file_keeps_its_inherited_category(self): + night = self._save_prompt("Night owl", category="Night", tags=["a"]) + tagged = self.media("tagged.png", mtime=103) + unlinked = self.media("unlinked.png", mtime=102) + self.api.db.link_image_to_prompt(night, str(tagged)) + self.tag(tagged, ["b"]) + self.tag(unlinked, ["a", "b"]) + names, _ = await self.listed(tags="a,b", category="Night") + self.assertEqual(names, ["tagged.png"]) + self.own(unlinked, "Night") + names, _ = await self.listed(tags="a,b", category="Night") + self.assertEqual(names, ["tagged.png", "unlinked.png"]) + + async def test_cursor_pages_walk_only_the_category(self): + files = [self.media(f"img_{n:02}.png", mtime=100 + n) for n in range(7)] + for path in files[::2]: + self.own(path, "Even") + seen, cursor = [], None + while True: + params = {"category": "Even", "limit": 2, "sort": "name"} + if cursor: + params["cursor"] = cursor + response, body = await self.get(**params) + self.assertEqual(response.status, 200, body) + seen += self.names(body) + self.assertEqual(body["total"], 4) + cursor = body["next_cursor"] + if not cursor: + break + self.assertEqual(seen, ["img_00.png", "img_02.png", "img_04.png", "img_06.png"]) + + async def test_an_outer_source_lists_a_nested_sources_own_category(self): + inner = self.output_dir / "nested" + nested = self.media("nested/a.png") + self.media("b.png") + self.api._get_all_output_dirs = lambda: [self.output_dir, inner] + self.own(nested, "Scans") + names, _ = await self.listed(category="Scans", root="0") + self.assertEqual(names, ["nested/a.png"]) + names, _ = await self.listed(category="Scans", root="1") + self.assertEqual(names, ["a.png"]) + # The outer source's own key for that path is not the file's key. + from py.api.images import _favorite_root_key + + self.api.db.clear_image_categories([image_category_key([inner], nested)]) + self.api.db.set_image_categories( + [(_favorite_root_key(self.output_dir), "nested/a.png")], "Stale" + ) + self.assertEqual((await self.listed(category="Stale"))[0], []) + + async def test_a_gone_files_category_is_not_listed(self): + gone = self.media("gone.png") + self.own(gone, "Scans") + os.unlink(gone) + self.api.invalidate_gallery_cache() + self.assertEqual((await self.listed(category="Scans"))[0], []) + + async def test_a_failed_own_category_read_is_listing_failed(self): + self.media("a.png") + + def broken(*_args, **_kwargs): + raise sqlite3.OperationalError("secret detail") + + with patch.object(self.api.db, "own_categories_of", side_effect=broken): + with self.assertLogs("prompt_manager", level="ERROR"): + response, body = await self.get(category="Scans") + self.assertEqual(response.status, 500) + self.assertNotIn("secret", str(body)) + # Without a category nothing new is read. + response, body = await self.get() + self.assertEqual(response.status, 200, body) + + async def test_select_all_matching_uses_the_effective_category(self): + _castle, files = self.batch() + resolved = await self.api.resolve_listing_filter( + parse_listing_filter({"category": "Portraits"}), 100 + ) + self.assertEqual([t[2] for t in resolved.targets], ["castle_0.png"]) + resolved = await self.api.resolve_listing_filter( + parse_listing_filter({"category": "Landscapes"}), 100 + ) + self.assertEqual(len(resolved.targets), 3) + self.assertEqual(len(files), 4) + + async def test_saved_view_category_means_the_effective_category(self): + """CUJ 8: a saved view's category is the listing's query value.""" + _castle, files = self.batch() + self.own(files[1], "Portraits") + names, _ = await self.listed(category="Portraits") + self.assertEqual(names, ["castle_0.png", "castle_1.png"]) + + +class TestFilterAndChipAgreeOnBlanks(WorkspaceCategoryCase): + """The filter and the inspector use one inherited-category rule.""" + + async def chip(self, path): + response = await self.client.get( + f"/prompt_manager/images/inspect/{path}", params={"root": "0"} + ) + body = await response.json() + self.assertEqual(response.status, 200, body) + return body["library"]["category"] + + async def test_blank_recipe_values_agree(self): + db = self.api.db + positive = self._save_prompt("Pair positive", category="Night") + blanks = {"spaces": " ", "tab": "\t", "nbsp": "ย ", "set": "Recipes"} + for index, (label, value) in enumerate(blanks.items()): + pair = db.save_prompt_pair("Pair positive", f"noise {label}") + path = self.media(f"{label}.png", mtime=100 + index) + image = db.link_image_to_prompt(positive, str(path)) + db.link_image_to_pair(pair, image, {"source": "fixture"}) + with db.model.get_connection() as conn: + conn.execute( + "UPDATE prompt_pairs SET category=? WHERE id=?", (value, pair) + ) + blob = db.save_prompt_pair("Pair positive", "noise blob") + path = self.media("blob.png", mtime=90) + image = db.link_image_to_prompt(positive, str(path)) + db.link_image_to_pair(blob, image, {"source": "fixture"}) + with db.model.get_connection() as conn: + conn.execute( + "UPDATE prompt_pairs SET category=CAST(' ' AS BLOB) WHERE id=?", + (blob,), + ) + for name in ("spaces", "tab", "nbsp", "set", "blob"): + with self.subTest(name=name): + shown = await self.chip(f"{name}.png") + for value in ("Night", "Recipes"): + listed = f"{name}.png" in (await self.listed(category=value))[0] + self.assertEqual(listed, shown == value, (name, value, shown)) diff --git a/tests/test_api_workspace_images.py b/tests/test_api_workspace_images.py index 26211a5b..cd998722 100644 --- a/tests/test_api_workspace_images.py +++ b/tests/test_api_workspace_images.py @@ -1347,7 +1347,7 @@ class TestWorkspaceImages(ImageAPITestCase): _, data = await self.get(q="harbor") self.assertEqual(self._names(data), ["night_a.png"]) self.assertEqual( - self.api.db.get_image_paths_matching("zebra", None, []), + self.api.db.get_image_paths_matching("zebra", []), { normalize_image_path(str(files[name])) for name in ("night_ab", "day_ab", "titled") @@ -1407,8 +1407,10 @@ class TestWorkspaceImages(ImageAPITestCase): for query in statements if query.lstrip().upper().startswith(("SELECT", "WITH")) ] - # Plus the one stream of tagged image records q and tags read (B3). - self.assertEqual(len(selects), 4, selects) + # Plus the one stream of tagged image records q and tags read (B3), + # and the category's own-category read of the one source (#332). + self.assertEqual(len(selects), 5, selects) + self.assertEqual(sum("image_categories" in q for q in selects), 1, selects) self.assertEqual(sum("image_favorites" in q for q in selects), 1, selects) self.assertEqual(sum("media_image_tags" in q for q in selects), 1, selects) @@ -1528,15 +1530,16 @@ class TestWorkspaceImages(ImageAPITestCase): elsewhere = self.api.db.get_prompt_id_for_image("/elsewhere/7.png") self.assertEqual( self.api.db.get_image_link_matches( - "", "Night", [], every_link=True, wanted=wanted + "", [], every_link=True, wanted=wanted, category="Night" ), { normalize_image_path(str(files["night_ab"])): ( True, 5, self.api.db.get_prompt_id_for_image(str(files["night_ab"])), + "Night", ), - "/elsewhere/7.png": (True, 5, elsewhere), + "/elsewhere/7.png": (True, 5, elsewhere, "Night"), }, ) @@ -1588,6 +1591,24 @@ class TestWorkspaceImages(ImageAPITestCase): for i in range(links) ], ) + # 2,000 images (one in 25 at 50,000 files) carry their own category + # (#332), so a category filter also reads and matches those. + from py.api.images import _favorite_root_key + + root_key = _favorite_root_key(self.output_dir) + with self.api.db.model.get_connection() as conn: + conn.executemany( + "INSERT INTO image_categories VALUES(?,?,?,?)", + [ + ( + root_key, + listed[i][0].relative_to(self.output_dir).as_posix(), + f"cat-{i % 10}", + "2026-10-27T00:00:00+00:00", + ) + for i in range(1, files, 25)[:2_000] + ], + ) return listed _BUDGET_REQUESTS = ( @@ -1615,7 +1636,7 @@ class TestWorkspaceImages(ImageAPITestCase): import math from py.api import images as images_module - from database.operations import link_filter_query + from database.operations import link_filter_query, link_matches_json_query listed = await asyncio.get_running_loop().run_in_executor( None, self._budget_fixture, 5_000, 2_000 @@ -1678,7 +1699,13 @@ class TestWorkspaceImages(ImageAPITestCase): # q and tags also stream the tagged image records once # (#249 B3); none are tagged here, so nothing is stat'ed. streamed = "q" in params or "tags" in params - self.assertEqual(len(found), 2 if streamed else 1, found) + # A category also reads the own categories, one primary + # key range per source (#332). + own = "category" in params + self.assertEqual(len(found), 1 + streamed + own, found) + self.assertEqual( + sum("image_categories" in query for query in found), int(own) + ) self.assertEqual( sum("media_image_tags" in query for query in found), int(streamed), @@ -1738,13 +1765,42 @@ class TestWorkspaceImages(ImageAPITestCase): self.assertEqual(self.api._workspace_listed_cache, {}) self.assertEqual(self.api._workspace_order_cache, {}) # The per-row ambiguity probe is an index search, never a scan. - sql, params = link_filter_query("lantern", "cat-3", ["tag-1"], every_link=True) - with self.api.db.model.get_connection() as conn: - plan = [row[3] for row in conn.execute("EXPLAIN QUERY PLAN " + sql, params)] - self.assertIn( - "SEARCH other USING COVERING INDEX idx_image_file_path (file_path=?)", plan + # The read runs it wrapped in one JSON aggregate (#354): same plan. + sql, params = link_filter_query("lantern", ["tag-1"], every_link=True) + for query in (sql, link_matches_json_query(sql)): + with self.subTest(wrapped=query is not sql): + with self.api.db.model.get_connection() as conn: + plan = [ + row[3] + for row in conn.execute("EXPLAIN QUERY PLAN " + query, params) + ] + self.assertIn( + "SEARCH other USING COVERING INDEX idx_image_file_path " + "(file_path=?)", + plan, + ) + self.assertFalse( + [step for step in plan if step.startswith("SCAN other")] + ) + # The category's narrowing clause (#332) keeps every join an index + # search, plain and wrapped with the raw categories. + sql, params = link_filter_query( + "", ["tag-1"], every_link=True, category="cat-3" ) - self.assertFalse([step for step in plan if step.startswith("SCAN other")]) + for query in (sql, link_matches_json_query(sql, categories=True)): + with self.subTest(category_wrapped=query is not sql): + with self.api.db.model.get_connection() as conn: + plan = [ + row[3] + for row in conn.execute("EXPLAIN QUERY PLAN " + query, params) + ] + self.assertIn( + "SEARCH other USING COVERING INDEX idx_image_file_path " + "(file_path=?)", + plan, + ) + scans = [step for step in plan if step.startswith("SCAN")] + self.assertTrue(all(step.startswith("SCAN gi") for step in scans), plan) @unittest.skipUnless( PERF_BUDGET and not TRACED, @@ -2577,6 +2633,42 @@ class TestWorkspaceCollectionFilter(ImageAPITestCase): self.assertEqual(self._names(data), expected) self.assertEqual(data["total"], len(expected)) + async def test_collection_filter_combines_with_an_images_own_category(self): + """#331 with #332: a member is admitted by its effective category.""" + from py.api.images import image_category_key + + night = self._save_prompt("Neon zebra", category="Night") + files = { + name: self.media(name, 100 + index) + for index, name in enumerate( + ("linked.png", "overridden.png", "own.png", "outside.png") + ) + } + for name in ("linked.png", "overridden.png"): + self.api.db.link_image_to_prompt(night, str(files[name])) + sources = self.api._get_all_output_dirs() + + def own(name, value): + key = image_category_key(sources, files[name]) + self.api.db.set_image_categories([key], value) + + own("overridden.png", "Portraits") + own("own.png", "Night") # unlinked: only its own category + own("outside.png", "Night") # not a member + members = [(self.output_dir, name) for name in files if name != "outside.png"] + grids = str(self.collect("Grids", *members)) + for category, expected in ( + ("Night", ["linked.png", "own.png"]), + ("Portraits", ["overridden.png"]), + ): + with self.subTest(category=category): + response, data = await self.get( + collection=grids, category=category, sort="name" + ) + self.assertEqual(response.status, 200, data) + self.assertEqual(self._names(data), expected) + self.assertEqual(data["total"], len(expected)) + async def test_invalid_collection_is_a_400(self): for value in ("x", "0", "-1", "1.5", "1234567890", " 1"): with self.subTest(value=value): diff --git a/tests/test_cached_runs.py b/tests/test_cached_runs.py new file mode 100644 index 00000000..d6ea4954 --- /dev/null +++ b/tests/test_cached_runs.py @@ -0,0 +1,823 @@ +"""Cached re-runs of runtime-linked prompt text count once per job (#347). + +Jobs are simulated end to end: the queue hook (submit + dequeue), the +``execution_cached`` replay, the real node executions and ComfyUI's output +cache (fakes of ``execution`` and ``comfy_execution.caching``, as in +tests/test_execution_context_host.py). +""" + +import logging +import os +import sys +import tempfile +import threading +import types +import unittest +from collections import namedtuple +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +try: + from tests.open_handles import assert_closed +except ImportError: # discovered with tests/ as the top-level directory + from open_handles import assert_closed + +from database.operations import PromptDatabase +from utils import cached_runs as module +from utils.cached_runs import ( + CachedRuns, + Facts, + Role, + Seen, + graph_role, + graph_seen, + own_counts, + register_cached_runs, + stored_facts, +) +from utils.execution_context_host import cached_string_outputs +from utils.hashing import generate_prompt_hash +from utils.run_keys import prompt_node_key, upstream_closure, upstream_nodes +from utils.usage_tracking import ( + PendingFirstUse, + QueuedRuns, + find_pair_id, + find_pair_id_by_hashes, + handle_queued_prompt, + on_dequeue, +) + +Entry = namedtuple("CacheEntry", "ui outputs") +LLM = "a lighthouse the LLM wrote" + + +class _Cache: + def __init__(self): + self.entries = {} + + def get_local(self, node_id): + return self.entries.get(node_id) + + +class HierarchicalCache(_Cache): + pass + + +class LRUCache(_Cache): + pass + + +class RAMPressureCache(LRUCache): + pass + + +class NullCache(_Cache): + pass + + +def host_modules(): + """Fake ``execution`` and ``comfy_execution.caching`` for the cache reader.""" + execution = types.ModuleType("execution") + execution.CacheEntry = Entry + + class PromptExecutor: + def add_message(self, event, data, broadcast): + return None + + execution.PromptExecutor = PromptExecutor + caching = types.ModuleType("comfy_execution.caching") + for cls in (HierarchicalCache, LRUCache, RAMPressureCache, NullCache): + setattr(caching, cls.__name__, cls) + package = types.ModuleType("comfy_execution") + package.caching = caching + return { + "execution": execution, + "comfy_execution": package, + "comfy_execution.caching": caching, + } + + +def graph( + kind="PromptManager", + text=None, + role="positive", + sampler=True, + negative_text="", + seed=1, +): + """PrimitiveString -> PreviewAny -> ``kind`` -> KSampler (runtime text). + + ``text`` set: typed text instead of the runtime link. ``role`` "negative": + the node feeds the sampler's negative input. Pair: its negative editor holds + ``negative_text`` and it feeds both sampler inputs. + """ + g = { + "str": {"class_type": "PrimitiveString", "inputs": {"value": "seed text"}}, + "llm": {"class_type": "PreviewAny", "inputs": {"source": ["str", 0]}}, + "ckpt": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt": "m"}}, + "pm": { + "class_type": kind, + "inputs": {"text": ["llm", 0] if text is None else text}, + }, + "other": { + "class_type": "CLIPTextEncode", + "inputs": {"text": "", "clip": ["ckpt", 1]}, + }, + "save": {"class_type": "SaveImage", "inputs": {"images": ["ks", 0]}}, + } + if kind != "PromptManagerText": + g["pm"]["inputs"]["clip"] = ["ckpt", 1] + if kind == "PromptManagerPair": + g["pm"]["inputs"]["negative_text"] = negative_text + source = ["pm", 0] + if kind == "PromptManagerText": + g["enc"] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": ["pm", 0], "clip": ["ckpt", 1]}, + } + source = ["enc", 0] + positive, negative = (source, ["other", 0]) + if role == "negative": + positive, negative = negative, positive + if kind == "PromptManagerPair": + positive, negative = ["pm", 0], ["pm", 1] + if sampler: + g["ks"] = { + "class_type": "KSampler", + "inputs": { + "model": ["ckpt", 0], + "positive": positive, + "negative": negative, + "seed": seed, + }, + } + else: + g["save"]["inputs"]["images"] = positive + return g + + +class JobCase(unittest.TestCase): + """A database, the queue hook stashes and one ``CachedRuns`` per test.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.addCleanup(assert_closed, self, self.tmp.name) + self.db = PromptDatabase(os.path.join(self.tmp.name, "runs.db")) + self.addCleanup(self.db.close_all) + patcher = mock.patch.dict(sys.modules, host_modules()) + patcher.start() + self.addCleanup(patcher.stop) + self.queued, self.pair_queued = QueuedRuns(), QueuedRuns() + self.pending, self.pair_pending = PendingFirstUse(), PendingFirstUse() + self.context = (None, None) + self.runs = CachedRuns(context=lambda: self.context) + self.queue = object() + self.executor = types.SimpleNamespace( + server=types.SimpleNamespace(prompt_queue=self.queue), + caches=types.SimpleNamespace(outputs=HierarchicalCache()), + ) + self.item_id = 0 + self.trackers = [] + + # -- nodes ---------------------------------------------------------------- + + def node(self, kind): + if kind == "PromptManagerPair": + from prompt_manager_pair import PromptManagerPair as cls + elif kind == "PromptManagerText": + from prompt_manager_text import PromptManagerText as cls + else: + import prompt_manager + + cls = getattr(prompt_manager, kind) + node = cls.__new__(cls) + node.db = self.db + node.logger = logging.getLogger("test.cached_runs") + node.prompt_tracker = mock.Mock() + self.trackers.append(node.prompt_tracker) + node.comfyui_integration = mock.Mock() + node._inject_lora_trigger_words = lambda text: text + node.pending_first_use = self.pending + node.pending_pair_first_use = self.pair_pending + node.cached_runs = self.runs + return node + + def execute(self, g, node_id="pm", text=LLM, negative="", index=None): + """Run the node as ComfyUI would; returns its STRING outputs.""" + kind = g[node_id]["class_type"] + node = self.node(kind) + self.context = (self.context[0], index) + clip = mock.Mock() + if kind == "PromptManagerText": + return node.process_text(text, prompt=g, unique_id=node_id) + if kind == "PromptManagerPair": + return node.encode_prompt( + clip, text, negative_text=negative, prompt=g, unique_id=node_id + ) + return node.encode_prompt(clip, text, prompt=g, unique_id=node_id) + + def cache(self, node_id, *results): + """Store executed results (one per list index) as ComfyUI's cache entry.""" + width = len(results[0]) + outputs = [[result[slot] for result in results] for slot in range(width)] + self.executor.caches.outputs.entries[node_id] = Entry({}, outputs) + + # -- jobs ----------------------------------------------------------------- + + def job(self, g, cached=(), run=None, outputs=("save",), prompt_id=None): + """One queued job: hook, dequeue, ``execution_cached``, executions, done.""" + self.item_id += 1 + prompt_id = prompt_id or f"job-{self.item_id}" + request = handle_queued_prompt( + {"prompt": g, "prompt_id": prompt_id}, + self.queued, + pair_queued=self.pair_queued, + ) + on_dequeue( + request["prompt_id"], + self.queued, + lambda: self.db, + self.pending, + pair_queued=self.pair_queued, + pair_pending=self.pair_pending, + ) + self.runs.begin_attempt( + ((self.item_id, prompt_id, g, {}, list(outputs), {}), self.item_id) + ) + self.context = (prompt_id, None) + self.runs.replay(self.executor, prompt_id, list(cached), lambda: self.db) + result = run() if run else None + self.runs.end_attempt(self.item_id) + self.context = (None, None) + return result + + def first(self, g, text=LLM, negative="", node_id="pm"): + """A job that executes the node and fills the cache.""" + result = self.job( + g, run=lambda: self.execute(g, node_id, text=text, negative=negative) + ) + self.cache(node_id, result) + return result + + def count(self, text): + row = self.db.get_prompt_by_hash(generate_prompt_hash(text)) + return row["run_count"] if row else None + + def pair_count(self, positive, negative): + pair_id = find_pair_id(self.db, positive, negative) + return self.db.get_prompt_pair(pair_id)["run_count"] if pair_id else None + + +class TestCachedJobsCount(JobCase): + def test_runtime_text_counts_every_cached_job_for_each_node_kind(self): + for kind in ( + "PromptManager", + "PromptManagerFocused", + "PromptManagerSplit", + "PromptManagerText", + ): + with self.subTest(kind=kind): + text = f"{LLM} for {kind}" + g = graph(kind) + self.first(g, text=text) + self.job(graph(kind, seed=2), cached=["pm"]) + self.job(graph(kind, seed=3), cached=["pm"]) + self.assertEqual(self.count(text), 3) + + def test_typed_text_still_counts_once_per_job(self): + g = graph(text="typed castle") + self.first(g, text="typed castle") + self.job(g, cached=["pm"]) + self.job(g, cached=["pm"]) + self.assertEqual(self.count("typed castle"), 3) + + def test_rewired_to_negative_while_cached_counts_nothing(self): + self.first(graph()) + self.job(graph(role="negative"), cached=["pm"]) + self.assertEqual(self.count(LLM), 1) + # Rewired back to positive: the receipt still proves the text + self.job(graph(), cached=["pm"]) + self.assertEqual(self.count(LLM), 2) + + def test_no_sampler_counts_every_node_on_replay(self): + g = graph(sampler=False) + self.first(g) + self.job(g, cached=["pm"]) + self.assertEqual(self.count(LLM), 2) + + def test_empty_editor_replays_nothing(self): + g = graph() + self.first(g, text="") + self.job(g, cached=["pm"]) + self.assertEqual(self.db.search_prompts(), []) + + def test_alternating_upstream_values_each_count_their_own_prompt(self): + g = graph() + self.first(g, text="value A") + self.first(g, text="value B") + self.cache("pm", self.execute_outputs("value A")) + self.job(g, cached=["pm"]) + self.assertEqual((self.count("value A"), self.count("value B")), (2, 1)) + + def execute_outputs(self, text): + return (mock.ANY, text) + + def test_list_items_replay_each_item_in_order(self): + g = graph() + results = [] + + def run(): + for index, text in enumerate(("one", "two", "three")): + results.append(self.execute(g, text=text, index=index)) + + self.job(g, run=run) + self.cache("pm", *results) + self.job(g, cached=["pm"]) + self.assertEqual([self.count(t) for t in ("one", "two", "three")], [2, 2, 2]) + + def test_a_repeated_id_or_message_never_counts_a_node_twice(self): + g = graph() + self.first(g) + self.job(g, cached=["pm", "pm"]) + self.assertEqual(self.count(LLM), 2) + self.item_id += 1 + self.runs.begin_attempt(((1, "again", g, {}, ["save"], {}), self.item_id)) + for _ in range(2): + self.runs.replay(self.executor, "again", ["pm"], lambda: self.db) + self.assertEqual(self.count(LLM), 3) + + def test_a_shorter_list_keeps_its_receipt_after_a_longer_one(self): + g = graph() + self.first(g, text="alpha") + results = [] + + def run(): + for index, text in enumerate(("alpha", "beta")): + results.append(self.execute(g, text=text, index=index)) + + self.job(g, run=run) + self.cache("pm", (mock.ANY, "alpha")) + self.job(g, cached=["pm"]) + self.assertEqual((self.count("alpha"), self.count("beta")), (3, 1)) + + def test_a_job_reusing_a_prompt_id_counts_and_executes_normally(self): + g = graph() + self.first(g) + self.job(g, cached=["pm"], prompt_id="same") + self.job(g, run=lambda: self.execute(g), prompt_id="same") + self.job(g, cached=["pm"], prompt_id="same") + self.assertEqual(self.count(LLM), 4) + + +class TestFailClosed(JobCase): + def setUp(self): + super().setUp() + self.g = graph() + self.first(self.g) + + def assertNotCounted(self): + self.assertEqual(self.count(LLM), 1) + + def test_changed_cached_text_is_not_counted(self): + self.cache("pm", (mock.ANY, LLM + " edited")) + self.job(self.g, cached=["pm"]) + self.assertNotCounted() + + def test_no_receipt_in_this_process_is_not_counted(self): + self.runs = CachedRuns(context=lambda: self.context) + self.job(self.g, cached=["pm"]) + self.assertNotCounted() + + def test_missing_cache_entry_is_not_counted(self): + self.executor.caches.outputs.entries.clear() + self.job(self.g, cached=["pm"]) + self.assertNotCounted() + + def test_unsupported_cache_is_not_counted(self): + entries = self.executor.caches.outputs.entries + self.executor.caches.outputs = NullCache() + self.executor.caches.outputs.entries = entries + self.job(self.g, cached=["pm"]) + self.assertNotCounted() + + def test_more_list_values_than_the_bound_are_not_counted(self): + self.cache("pm", *[(mock.ANY, LLM)] * 65) + self.job(self.g, cached=["pm"]) + self.assertNotCounted() + + def test_missing_list_index_is_not_counted(self): + self.cache("pm", (mock.ANY, LLM), (mock.ANY, LLM)) + self.job(self.g, cached=["pm"]) + self.assertNotCounted() + + def test_node_the_job_does_not_run_is_not_counted(self): + self.job(self.g, cached=["pm"], outputs=("other",)) + self.assertNotCounted() + + def test_message_for_another_job_is_not_counted(self): + self.item_id += 1 + self.runs.begin_attempt(((1, "this", self.g, {}, ["save"], {}), self.item_id)) + self.runs.replay(self.executor, "another", ["pm"], lambda: self.db) + self.assertNotCounted() + + def test_unkeyable_node_is_not_counted(self): + g = graph() + g["pm"]["inputs"]["extra"] = ["missing", 0] + self.job(g, cached=["pm"]) + self.assertIsNone(prompt_node_key(g, "pm")) + self.assertNotCounted() + + def test_deleted_prompt_is_skipped(self): + self.db.delete_prompt( + self.db.get_prompt_by_hash(generate_prompt_hash(LLM))["id"] + ) + self.job(self.g, cached=["pm"]) + self.assertIsNone(self.count(LLM)) + + def test_database_errors_never_raise(self): + def broken(): + raise RuntimeError("database unavailable") + + self.item_id += 1 + self.runs.begin_attempt(((1, "j", self.g, {}, ["save"], {}), self.item_id)) + self.runs.replay(self.executor, "j", ["pm"], broken) + self.assertNotCounted() + + def test_malformed_messages_and_results_are_ignored(self): + self.runs.begin_attempt(None) + self.runs.begin_attempt((("x",), "not an int")) + self.runs.replay(self.executor, None, ["pm"], lambda: self.db) + self.item_id += 1 + self.runs.begin_attempt(((1, "j", self.g, {}, ["save"], {}), self.item_id)) + self.runs.replay(self.executor, "j", "pm", lambda: self.db) + self.runs.replay(self.executor, "j", ["pm"] * 501, lambda: self.db) + self.runs.replay(self.executor, "j", [7, None], lambda: self.db) + self.assertNotCounted() + + +class TestReplayThenExecution(JobCase): + def test_execution_after_replay_in_the_same_job_is_not_counted_again(self): + # RAM-pressure frees the entry after execution_cached: the node executes + g = graph() + self.first(g) + self.job(g, cached=["pm"], run=lambda: self.execute(g)) + self.assertEqual(self.count(LLM), 2) + # The suppressed execution still leaves a receipt: the next job counts + self.job(g, cached=["pm"]) + self.assertEqual(self.count(LLM), 3) + + def test_suppression_keeps_the_pending_catch_up_and_the_tracker_queue(self): + g = graph() + self.first(g) + prompt_id = self.db.get_prompt_by_hash(generate_prompt_hash(LLM))["id"] + + def run(): + # The replay counted, then the prompt was deleted and re-saved + self.db.delete_prompt(prompt_id) + self.pending.add(generate_prompt_hash(LLM)) + return self.execute(g) + + self.job(g, cached=["pm"], run=run) + self.assertEqual(self.count(LLM), 1) # the catch-up, not the own count + push = self.trackers[-1].set_current_prompt.call_args.kwargs["push_to_queue"] + self.assertTrue(push) + + def test_without_host_context_the_node_counts_as_before(self): + g = graph() + self.runs = CachedRuns(context=lambda: (None, None)) + self.first(g) + self.job(g, cached=["pm"]) + self.job(g, run=lambda: self.execute(g)) + self.assertEqual(self.count(LLM), 2) + + +class TestPairReplay(JobCase): + def test_runtime_positive_counts_positive_and_pair_every_job(self): + g = graph("PromptManagerPair", negative_text="blurry") + self.first(g, negative="blurry") + self.job(graph("PromptManagerPair", negative_text="blurry", seed=2), ["pm"]) + self.job(graph("PromptManagerPair", negative_text="blurry", seed=3), ["pm"]) + self.assertEqual(self.count(LLM), 3) + self.assertEqual(self.pair_count(LLM, "blurry"), 3) + self.assertEqual(self.count("blurry"), 0) + + def test_empty_negative_pair_replays_its_pair_count(self): + g = graph("PromptManagerPair") + self.first(g) + self.job(g, cached=["pm"]) + self.assertEqual(self.pair_count(LLM, ""), 2) + self.assertEqual(self.count(LLM), 2) + + def test_pair_execution_after_replay_is_not_counted_again(self): + g = graph("PromptManagerPair", negative_text="blurry") + self.first(g, negative="blurry") + self.job(g, ["pm"], run=lambda: self.execute(g, negative="blurry")) + self.assertEqual(self.count(LLM), 2) + self.assertEqual(self.pair_count(LLM, "blurry"), 2) + + def test_pair_role_follows_the_current_graph(self): + g = graph("PromptManagerPair", negative_text="blurry") + self.first(g, negative="blurry") + rewired = graph("PromptManagerPair", negative_text="blurry") + rewired["ks"]["inputs"]["positive"] = ["other", 0] + self.job(rewired, cached=["pm"]) + self.assertEqual(self.count(LLM), 1) + self.assertEqual(self.pair_count(LLM, "blurry"), 1) + + +class TestOwnCounts(unittest.TestCase): + """The node path and replay decide through the same function.""" + + def test_single_node_table(self): + facts = Facts("p") + for used, seen, expected in ( + ({"text"}, False, [("prompt", "p")]), + ({"text"}, True, []), + (set(), False, []), + ): + with self.subTest(used=used, seen=seen): + counts = own_counts( + "PromptManager", facts, Role(frozenset(used)), Seen(text=seen) + ) + self.assertEqual(counts, expected) + self.assertEqual( + own_counts("PromptManager", Facts(None), Role(frozenset({"text"})), Seen()), + [], + ) + + def test_pair_table(self): + both = Role(frozenset({"text", "negative_text"}), True) + self.assertEqual( + own_counts("PromptManagerPair", Facts("p", "n"), both, Seen()), + [("prompt", "p"), ("prompt", "n"), ("pair", "p", "n")], + ) + self.assertEqual( + own_counts("PromptManagerPair", Facts("p", "p"), both, Seen()), + [("prompt", "p"), ("pair", "p", "p")], + ) + self.assertEqual( + own_counts("PromptManagerPair", Facts("p"), both, Seen(text=True)), + [("pair", "p", None)], + ) + self.assertEqual( + own_counts( + "PromptManagerPair", Facts("p", "n"), both, Seen(True, True, True) + ), + [], + ) + + def test_node_and_replay_agree_on_role_and_seen(self): + for g, node_id in ( + (graph(), "pm"), + (graph(text="typed"), "pm"), + (graph(role="negative"), "pm"), + (graph(sampler=False), "pm"), + (graph("PromptManagerPair", negative_text="n"), "pm"), + (graph("PromptManagerPair", text="typed", negative_text="n"), "pm"), + ): + kind = g[node_id]["class_type"] + text = "typed" if isinstance(g[node_id]["inputs"]["text"], str) else LLM + facts = stored_facts(text, g[node_id]["inputs"].get("negative_text")) + with self.subTest(kind=kind, inputs=g[node_id]["inputs"]): + replay = own_counts( + kind, + facts, + graph_role(g, node_id, kind), + graph_seen(g, node_id, kind, facts), + ) + self.assertEqual(replay, self.node_counts(g, node_id, kind, text)) + + @staticmethod + def node_counts(g, node_id, kind, text): + """What the executing node counted itself, from a recording database.""" + db = mock.Mock() + db.get_prompt_by_hash.side_effect = lambda h: {"id": h} + db.save_prompt_pair.return_value = 1 + negative = g[node_id]["inputs"].get("negative_text", "") + db.get_prompt_pair.return_value = { + "positive_prompt_id": generate_prompt_hash(text), + "negative_prompt_id": ( + generate_prompt_hash(negative) if negative.strip() else None + ), + } + case = JobCase() + case.db = db + case.pending, case.pair_pending = PendingFirstUse(), PendingFirstUse() + case.runs = CachedRuns(context=lambda: (None, None)) + case.trackers = [] + case.context = (None, None) + if kind == "PromptManagerPair": + node = case.node(kind) + node.encode_prompt( + mock.Mock(), text, negative_text=negative, prompt=g, unique_id=node_id + ) + else: + node = case.node(kind) + node._track_prompt_execution(text, text, None, None, {}, g, node_id) + counts = [("prompt", c.args[0]) for c in db.record_prompt_use.call_args_list] + if kind == "PromptManagerPair": + counts += [ + ("pair", generate_prompt_hash(text), db.get_prompt_pair()[k]) + for k in ["negative_prompt_id"] + if db.record_prompt_pair_use.called + ] + return counts + + +class TestReceipts(unittest.TestCase): + def setUp(self): + self.context = ("j1", None) + self.runs = CachedRuns( + context=lambda: self.context, max_keys=2, max_receipts=2, max_items=3 + ) + self.g = { + name: {"class_type": "PromptManager", "inputs": {"text": name}} + for name in ("a", "b", "c") + } + + def begin(self, item_id, prompt_id="j1"): + self.context = (prompt_id, None) + self.runs.begin_attempt( + ((item_id, prompt_id, self.g, {}, ["a", "b", "c"], {}), item_id) + ) + + def keys(self): + return list(self.runs._receipts) + + def test_least_recently_used_run_keys_are_dropped(self): + self.begin(1) + for name in ("a", "b", "c"): + self.runs.record_item(name, (name,), Facts("h")) + self.assertEqual(self.keys(), [prompt_node_key(self.g, n) for n in ("b", "c")]) + + def test_receipts_per_key_are_bounded_and_equal_outputs_replace(self): + key = prompt_node_key(self.g, "a") + for item_id, text in ((1, "x"), (2, "y"), (3, "z"), (4, "y")): + self.begin(item_id) + self.runs.record_item("a", (text,), Facts(text)) + self.runs.end_attempt(item_id) + receipts = self.runs._receipts[key] + self.assertEqual([r.item_id for r in receipts], [4, 3]) + + def test_list_indices_past_the_bound_are_not_recorded(self): + self.begin(1) + self.context = ("j1", 3) + self.runs.record_item("a", ("x",), Facts("x")) + self.assertEqual(self.keys(), []) + + def test_non_string_outputs_and_unknown_nodes_are_not_recorded(self): + self.begin(1) + self.runs.record_item("a", (None,), Facts("x")) + self.runs.record_item("zzz", ("x",), Facts("x")) + self.assertEqual(self.keys(), []) + + def test_task_done_clears_only_its_own_attempt(self): + self.begin(1) + self.runs.end_attempt(2) + self.runs.record_item("a", ("x",), Facts("x")) + self.runs.end_attempt(1) + self.runs.record_item("b", ("x",), Facts("x")) + self.assertEqual(self.keys(), [prompt_node_key(self.g, "a")]) + + def test_job_nodes_follow_the_outputs_to_execute(self): + self.runs.begin_attempt(((1, "j1", self.g, {}, ["b"], {}), 1)) + self.assertEqual(list(self.runs._attempt.nodes), ["b"]) + + +class TestCacheReader(unittest.TestCase): + def setUp(self): + patcher = mock.patch.dict(sys.modules, host_modules()) + patcher.start() + self.addCleanup(patcher.stop) + self.cache = LRUCache() + self.executor = types.SimpleNamespace( + caches=types.SimpleNamespace(outputs=self.cache) + ) + + def read(self, outputs, kind="PromptManager"): + self.cache.entries["n"] = Entry({}, outputs) + return cached_string_outputs(self.executor, "n", kind) + + def test_reads_each_string_slot_in_order(self): + self.assertEqual(self.read([["c"], ["a", "b"]]), (("a", "b"),)) + self.assertEqual(self.read([["t"]], "PromptManagerText"), (("t",),)) + self.assertEqual( + self.read([[1], [2], ["p"], ["n"]], "PromptManagerPair"), (("p",), ("n",)) + ) + + def test_fails_closed(self): + for outputs, kind in ( + ([["c"], ["a", 1]], "PromptManager"), + ([["c"]], "PromptManager"), + ([["c"], ["a"] * 65], "PromptManager"), + ([[1], [2], ["p", "q"], ["n"]], "PromptManagerPair"), + ([["c"], "text"], "PromptManager"), + ([["c"], ["a"]], "CLIPTextEncode"), + ([[]] * 5, "PromptManager"), + ): + with self.subTest(outputs=outputs, kind=kind): + self.assertIsNone(self.read(outputs, kind)) + self.cache.entries["m"] = ({}, [["c"], ["a"]]) # not a CacheEntry + self.assertIsNone(cached_string_outputs(self.executor, "m", "PromptManager")) + self.executor.caches.outputs = NullCache() + self.assertIsNone(self.read([["c"], ["a"]])) + + +class TestGraphHelpers(unittest.TestCase): + def test_upstream_nodes_and_closure(self): + g = graph() + self.assertEqual( + set(upstream_nodes(g, ["save"])), + {"save", "ks", "pm", "llm", "str", "ckpt", "other"}, + ) + self.assertEqual(upstream_nodes(g, ["missing"]), []) + self.assertEqual(upstream_nodes(None, ["save"]), []) + self.assertIsNone(upstream_closure(g, ["save"], limit=2)) + self.assertIsNone(upstream_closure({"a": {"inputs": {"x": ["b", 0]}}}, ["a"])) + + def test_find_pair_by_hashes_matches_the_text_lookup(self): + with tempfile.TemporaryDirectory() as tmp: + db = PromptDatabase(os.path.join(tmp, "p.db")) + try: + pair_id = db.save_prompt_pair("cat", "") + self.assertEqual( + find_pair_id_by_hashes(db, generate_prompt_hash("cat"), None), + pair_id, + ) + self.assertIsNone( + find_pair_id_by_hashes( + db, generate_prompt_hash("cat"), generate_prompt_hash("x") + ) + ) + finally: + db.close_all() + assert_closed(self, tmp) + + +class TestRegistration(unittest.TestCase): + def setUp(self): + self.modules = host_modules() + patcher = mock.patch.dict(sys.modules, self.modules) + patcher.start() + self.addCleanup(patcher.stop) + flag = mock.patch.object(module, "_registered", threading.Event()) + flag.start() + self.addCleanup(flag.stop) + + class Queue: + def __init__(self): + self.done = [] + + def get(self, timeout=None): + return self.result + + def task_done(self, item_id, *args, **kwargs): + self.done.append(item_id) + + def test_registers_once_and_follows_the_job(self): + queue = self.Queue() + server = types.SimpleNamespace(prompt_queue=queue) + runs = mock.Mock() + self.assertTrue(register_cached_runs(server, lambda: "db", runs)) + self.assertFalse(register_cached_runs(server, lambda: "db", runs)) + queue.result = (("item",), 5) + queue.get() + runs.begin_attempt.assert_called_once_with((("item",), 5)) + executor = self.modules["execution"].PromptExecutor() + executor.server = server + executor.add_message("execution_start", {"prompt_id": "j"}, broadcast=False) + executor.add_message( + "execution_cached", {"prompt_id": "j", "nodes": ["n"]}, broadcast=False + ) + runs.replay.assert_called_once() + self.assertEqual(runs.replay.call_args.args[1:3], ("j", ["n"])) + other = self.modules["execution"].PromptExecutor() + other.server = types.SimpleNamespace(prompt_queue=object()) + other.add_message("execution_cached", {"prompt_id": "j", "nodes": []}, False) + self.assertEqual(runs.replay.call_count, 1) + queue.task_done(5, None, status=None) + runs.end_attempt.assert_called_once_with(5) + + def test_hosts_without_the_interfaces_register_nothing(self): + class NoTaskDone: + def get(self): + return None + + self.assertFalse( + register_cached_runs( + types.SimpleNamespace(prompt_queue=NoTaskDone()), lambda: None + ) + ) + self.assertFalse(register_cached_runs(types.SimpleNamespace(), lambda: None)) + del self.modules["execution"].PromptExecutor.add_message + self.assertFalse( + register_cached_runs( + types.SimpleNamespace(prompt_queue=self.Queue()), lambda: None + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_database_link_matches.py b/tests/test_database_link_matches.py new file mode 100644 index 00000000..8b50b3e3 --- /dev/null +++ b/tests/test_database_link_matches.py @@ -0,0 +1,456 @@ +"""The prompt-link read of the workspace Images page (#354). + +``PromptDatabase.get_image_link_matches`` returns every row of its filter +SELECT in one SQLite step (one JSON value) instead of one step per row, so the +read does not have to win the GIL back once per linked image while ComfyUI +generates. Its result must equal the streamed read it replaces, which stays as +the fallback, on every filter and on odd values. +""" + +import itertools +import os +import sqlite3 +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from database import operations # noqa: E402 +from database.operations import PromptDatabase, link_filter_query # noqa: E402 + +try: + from tests.open_handles import assert_closed # noqa: E402 +except ImportError: # discovered with tests/ as the top-level directory + from open_handles import assert_closed # noqa: E402 + +LOGGER = "prompt_manager.database" +MISSING_FUNCTION_SQL = ( + "SELECT pm_missing_function(json_group_array(json_array(" + "file_path, rating, matched, positive))) FROM ({select})" +) +QUERIES = ("", "lantern", "LANTERN title", "no-such-text") +CATEGORIES = (None, "cat-a", "cat-b", "cat-none") +TAG_SETS = ([], ["t1"], ["t1", "t2"], ["t-none"]) + +PROMPTS = ( + # id, text, category, rating, tags, library title + (1, "red lantern at dusk", "cat-a", 5, ("t1", "t2"), None), + (2, "blue harbor", "cat-a", None, ("t1",), None), + (3, "green field", "cat-b", 3, (), "Lantern title"), + (4, "pair positive", "cat-b", 4, ("t2",), None), + (5, "pair negative lantern", "cat-a", 1, ("t1",), None), +) +QUOTED = '/out/it\'s "quoted".png' +BACKSLASHED = "C:\\out\\back\\slash.png" +UNICODE = "/out/รผnรฏcรธdรฉ ็”ปๅƒ.png" +NEWLINE = "/out/new\nline.png" +IMAGES = ( + # prompt id, file path + (1, "/out/a.png"), + (2, QUOTED), + (2, BACKSLASHED), + (3, UNICODE), + (1, NEWLINE), + (1, "/out/ambiguous.png"), + (2, "/out/ambiguous.png"), + (5, "/out/pair.png"), # linked through pair 1: its positive is prompt 4 + (3, "/out/two-pairs.png"), # two pairs: no single pair, so prompt 3 + (1, None), # no file path: never listed +) + + +class LinkMatchesTestCase(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(os.path.realpath(self.temp.name)) + self.db = PromptDatabase(str(self.root / "prompts.db")) + for name in ("_one_step_unsupported", "_one_step_warned"): + patcher = patch.object(operations, name, False) + patcher.start() + self.addCleanup(patcher.stop) + self.statements = [] + original = self.db.model.get_connection + + def traced_connection(): + connection = original() + connection.set_trace_callback(self.statements.append) + return connection + + patcher = patch.object( + self.db.model, "get_connection", side_effect=traced_connection + ) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + self.db.close_all() + assert_closed(self, self.root) + self.temp.cleanup() + + def seed(self): + with self.db.model.get_connection() as conn: + for prompt_id, text, category, rating, tags, title in PROMPTS: + conn.execute( + "INSERT INTO prompts(id,text,category,rating,hash) " + "VALUES(?,?,?,?,?)", + (prompt_id, text, category, rating, f"hash-{prompt_id}"), + ) + for tag in tags: + conn.execute("INSERT OR IGNORE INTO tags(name) VALUES(?)", (tag,)) + conn.execute( + "INSERT INTO prompt_tags(prompt_id,tag_id) " + "SELECT ?, id FROM tags WHERE name=?", + (prompt_id, tag), + ) + if title: + conn.execute( + "INSERT INTO prompt_library_state(prompt_id,title) " + "VALUES(?,?)", + (prompt_id, title), + ) + ids = {} + for prompt_id, path in IMAGES: + cursor = conn.execute( + "INSERT INTO generated_images" + "(prompt_id,image_path,filename,file_path) VALUES(?,?,?,?)", + (prompt_id, path or "/elsewhere/x.png", "x.png", path), + ) + ids.setdefault(path, cursor.lastrowid) + conn.execute( + "INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) " + "VALUES(1,4,5),(2,3,5),(3,3,NULL)" + ) + conn.executemany( + "INSERT INTO image_prompt_pairs(image_id,pair_id,provenance) " + "VALUES(?,?,'{}')", + [ + (ids["/out/pair.png"], 1), + (ids["/out/two-pairs.png"], 2), + (ids["/out/two-pairs.png"], 3), + ], + ) + + def add_image(self, prompt_id, path): + with self.db.model.get_connection() as conn: + conn.execute( + "INSERT INTO generated_images(prompt_id,image_path,filename,file_path)" + " VALUES(?,?,?,?)", + (prompt_id, "/blob/x.png", "x.png", path), + ) + + def streamed(self, q, category, tags, every_link=False, wanted=None): + """Today's read, called directly (the fallback).""" + sql, params = link_filter_query( + q, tags, every_link=every_link, category=category + ) + with self.db.model.get_connection() as conn: + return operations._streamed_link_matches( + conn, sql, params, wanted, category is not None + ) + + def selects(self): + found = [s for s in self.statements if s.lstrip().upper().startswith("SELECT")] + self.statements.clear() + return found + + +class TestOneStepLinkRead(LinkMatchesTestCase): + def test_plain_read_returns_every_unambiguous_link(self): + self.seed() + matches = self.db.get_image_link_matches("", [], every_link=True) + self.assertEqual( + matches, + { + "/out/a.png": (True, 5, 1, None), + QUOTED: (True, None, 2, None), + BACKSLASHED: (True, None, 2, None), + UNICODE: (True, 3, 3, None), + NEWLINE: (True, 5, 1, None), + "/out/pair.png": (True, 4, 4, None), + "/out/two-pairs.png": (True, 3, 3, None), + }, + ) + self.assertIsNone(matches[QUOTED][1]) + self.assertIsInstance(matches["/out/a.png"][1], int) + self.assertIsInstance(matches["/out/a.png"][2], int) + + def test_search_hits_follow_text_title_and_tags(self): + self.seed() + self.assertEqual( + self.db.get_image_link_matches("lantern", []), + { + "/out/a.png": (True, 5, 1, None), + NEWLINE: (True, 5, 1, None), + UNICODE: (True, 3, 3, None), + "/out/two-pairs.png": (True, 3, 3, None), + }, + ) + self.assertEqual( + self.db.get_image_link_matches("lantern", [], category="cat-a"), + { + "/out/a.png": (True, 5, 1, "cat-a"), + NEWLINE: (True, 5, 1, "cat-a"), + QUOTED: (False, None, 2, "cat-a"), + BACKSLASHED: (False, None, 2, "cat-a"), + }, + ) + + def test_every_filter_equals_the_streamed_read(self): + self.seed() + wanted_sets = (None, {"/out/a.png", UNICODE, "/out/pair.png", "/absent.png"}) + for q, category, tags, every_link, wanted in itertools.product( + QUERIES, CATEGORIES, TAG_SETS, (False, True), wanted_sets + ): + with self.subTest( + q=q, category=category, tags=tags, every=every_link, wanted=wanted + ): + expected = self.streamed(q, category, tags, every_link, wanted) + self.statements.clear() + actual = self.db.get_image_link_matches( + q, + list(tags), + every_link=every_link, + wanted=wanted, + category=category, + ) + self.assertEqual(actual, expected) + self.assertEqual( + {path: tuple(map(type, row)) for path, row in actual.items()}, + {path: tuple(map(type, row)) for path, row in expected.items()}, + ) + found = self.selects() + self.assertEqual(len(found), 1, found) + self.assertIn("json_group_array", found[0]) + + def test_empty_database_returns_nothing_in_one_statement(self): + self.assertEqual(self.db.get_image_link_matches("", [], every_link=True), {}) + self.assertEqual(self.db.get_image_link_matches("x", ["t"], category="c"), {}) + found = self.selects() + self.assertEqual(len(found), 2, found) + self.assertTrue(all("json_group_array" in query for query in found)) + + def test_wanted_limits_the_result_to_the_listing(self): + self.seed() + self.assertEqual( + self.db.get_image_link_matches( + "", [], every_link=True, wanted={"/out/pair.png", "/x.png"} + ), + {"/out/pair.png": (True, 4, 4, None)}, + ) + self.assertEqual( + self.db.get_image_link_matches("", [], every_link=True, wanted=set()), + {}, + ) + + def test_category_read_carries_the_inherited_category_in_one_step(self): + # A recipe's own category wins over its positive's (#332 MC4); the + # read only narrows to links holding the value, the caller compares. + self.seed() + with self.db.model.get_connection() as conn: + conn.execute("UPDATE prompt_pairs SET category='cat-a' WHERE id=1") + self.statements.clear() + for category, expected in ( + ("cat-a", {"/out/pair.png": (True, 4, 4, "cat-a")}), + ("cat-b", {"/out/pair.png": (True, 4, 4, "cat-a")}), + ): + with self.subTest(category=category): + actual = self.db.get_image_link_matches( + "", [], every_link=True, category=category, wanted={"/out/pair.png"} + ) + self.assertEqual(actual, expected) + self.assertEqual( + actual, self.streamed("", category, [], True, {"/out/pair.png"}) + ) + found = [query for query in self.selects() if "json_group_array" in query] + self.assertEqual(len(found), 2, found) + self.assertTrue(all("pair_category" in query for query in found)) + + def test_plain_read_leaves_the_categories_out_of_the_json(self): + self.seed() + self.statements.clear() + self.db.get_image_link_matches("", [], every_link=True) + (query,) = self.selects() + aggregate = query[query.index("json_array(") :].split(")", 1)[0] + self.assertEqual(aggregate, "json_array(file_path, rating, matched, positive") + + def test_read_releases_its_snapshot(self): + # A statement left open after the read would pin its WAL snapshot, and + # a TRUNCATE checkpoint from another connection would report busy. + self.seed() + self.assertTrue(self.db.get_image_link_matches("", [], every_link=True)) + other = sqlite3.connect(str(self.root / "prompts.db")) + try: + other.execute("UPDATE prompts SET rating=2 WHERE id=1") + other.commit() + busy, _log, _done = other.execute( + "PRAGMA wal_checkpoint(TRUNCATE)" + ).fetchone() + finally: + other.close() + self.assertEqual(busy, 0) + self.assertEqual(os.path.getsize(self.root / "prompts.db-wal"), 0) + + def test_wrapped_query_keeps_the_ambiguity_probe_an_index_search(self): + self.seed() + sql, params = link_filter_query( + "lantern", ["t1"], every_link=True, category="cat-a" + ) + with self.db.model.get_connection() as conn: + plan = [ + row[3] + for row in conn.execute( + "EXPLAIN QUERY PLAN " + + operations.link_matches_json_query(sql, categories=True), + params, + ) + ] + self.assertIn( + "SEARCH other USING COVERING INDEX idx_image_file_path (file_path=?)", plan + ) + self.assertFalse([step for step in plan if step.startswith("SCAN other")]) + + +class TestStreamedFallback(LinkMatchesTestCase): + def test_missing_json_functions_stream_and_stop_trying(self): + self.seed() + expected = self.streamed("", None, [], True) + self.statements.clear() + wrapper = operations.link_matches_json_query + with ( + patch.object(operations, "_LINK_MATCHES_JSON_SQL", MISSING_FUNCTION_SQL), + patch.object( + operations, "link_matches_json_query", side_effect=wrapper + ) as attempts, + self.assertLogs(LOGGER, "WARNING") as logged, + ): + first = self.db.get_image_link_matches("", [], every_link=True) + found = self.selects() + self.assertEqual(attempts.call_count, 1) + second = self.db.get_image_link_matches("", [], every_link=True) + again = self.selects() + self.assertEqual(first, expected) + self.assertEqual(second, expected) + self.assertEqual(len(logged.records), 1, logged.output) + self.assertIn("no such function: pm_missing_function", logged.output[0]) + self.assertTrue(operations._one_step_unsupported) + # SQLite refuses the wrapped statement before it runs (not traced). + self.assertEqual(len(found), 1, found) + self.assertNotIn("json_group_array", found[0]) + # The next read skips the attempt for the rest of the process. + self.assertEqual(attempts.call_count, 1) + self.assertEqual(len(again), 1, again) + self.assertNotIn("json_group_array", again[0]) + + def test_blob_path_streams_that_read_and_keeps_trying(self): + self.seed() + self.add_image(2, b"/out/blob.png") + expected = self.streamed("", None, [], True) + self.assertEqual(expected[b"/out/blob.png"], (True, None, 2, None)) + self.statements.clear() + with self.assertLogs(LOGGER, "WARNING") as logged: + first = self.db.get_image_link_matches("", [], every_link=True) + second = self.db.get_image_link_matches("", [], every_link=True) + self.assertEqual(first, expected) + self.assertEqual(second, expected) + self.assertEqual(len(logged.records), 1, logged.output) + self.assertFalse(operations._one_step_unsupported) + found = self.selects() + self.assertEqual(sum("json_group_array" in query for query in found), 2) + self.assertEqual(len(found), 4, found) + + def test_blob_that_reads_as_json_still_streams(self): + # SQLite 3.45+ reads a BLOB that is valid JSONB as JSON (b"\x00" is + # null) instead of refusing it; the streamed read keeps the bytes. + self.seed() + self.add_image(2, b"\x00") + expected = self.streamed("", None, [], True) + self.assertIn(b"\x00", expected) + with self.assertLogs(LOGGER, "WARNING"): + self.assertEqual( + self.db.get_image_link_matches("", [], every_link=True), expected + ) + self.assertFalse(operations._one_step_unsupported) + + def test_blob_rating_streams(self): + self.seed() + with self.db.model.get_connection() as conn: + # Only a database written around the CHECK can hold one. + conn.execute("PRAGMA ignore_check_constraints=ON") + conn.execute("UPDATE prompts SET rating=? WHERE id=3", (b"\x00",)) + conn.execute("PRAGMA ignore_check_constraints=OFF") + expected = self.streamed("", None, [], True) + self.assertEqual(expected[UNICODE], (True, b"\x00", 3, None)) + with self.assertLogs(LOGGER, "WARNING"): + self.assertEqual( + self.db.get_image_link_matches("", [], every_link=True), expected + ) + + def test_blob_category_streams_a_category_read(self): + self.seed() + with self.db.model.get_connection() as conn: + # Admitted by its recipe's text category; the positive's is a BLOB. + conn.execute("UPDATE prompts SET category=? WHERE id=4", (b"cat-b",)) + conn.execute("UPDATE prompt_pairs SET category='cat-b' WHERE id=1") + expected = self.streamed("", "cat-b", [], True) + self.assertEqual(expected["/out/pair.png"], (True, 4, 4, "cat-b")) + with self.assertLogs(LOGGER, "WARNING") as logged: + actual = self.db.get_image_link_matches( + "", [], every_link=True, category="cat-b" + ) + self.assertEqual(actual, expected) + self.assertIn("BLOB", logged.output[0]) + self.assertFalse(operations._one_step_unsupported) + + def test_json_text_over_the_length_limit_streams(self): + self.seed() + expected = self.streamed("", None, [], True) + conn = self.db.model.get_connection() + limit = conn.setlimit(sqlite3.SQLITE_LIMIT_LENGTH, 200) + try: + with self.assertLogs(LOGGER, "WARNING") as logged: + actual = self.db.get_image_link_matches("", [], every_link=True) + finally: + conn.setlimit(sqlite3.SQLITE_LIMIT_LENGTH, limit) + self.assertEqual(actual, expected) + self.assertIn("too big", logged.output[0]) + self.assertFalse(operations._one_step_unsupported) + + def test_an_interrupted_read_raises_without_a_fallback(self): + # Move library interrupts a retired connection: the read fails as + # before, and the one warning is kept for a real fallback. + self.seed() + conn = self.db.model.get_connection() + conn.set_progress_handler(lambda: 1, 1) + self.statements.clear() + try: + with ( + self.assertNoLogs(LOGGER, "WARNING"), + self.assertRaisesRegex(sqlite3.OperationalError, "^interrupted$"), + ): + self.db.get_image_link_matches("", [], every_link=True) + finally: + conn.set_progress_handler(None, 1) + self.assertFalse(operations._one_step_warned) + self.assertFalse(operations._one_step_unsupported) + self.assertFalse([s for s in self.statements if "json_group_array" not in s]) + + def test_the_maintenance_link_rows_reader_is_not_shadowed(self): + from database.maintenance import MaintenanceOperations + + self.assertIs(PromptDatabase._link_rows, MaintenanceOperations._link_rows) + + def test_a_failed_streamed_read_still_raises(self): + with ( + patch.object(operations, "_LINK_MATCHES_JSON_SQL", MISSING_FUNCTION_SQL), + patch.object(operations, "_LINK_FILTER_SQL", "SELECT broken FROM nowhere"), + self.assertLogs(LOGGER, "WARNING"), + ): + with self.assertRaises(sqlite3.OperationalError): + self.db.get_image_link_matches("", [], every_link=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_duplicate_image_categories.py b/tests/test_duplicate_image_categories.py new file mode 100644 index 00000000..97492454 --- /dev/null +++ b/tests/test_duplicate_image_categories.py @@ -0,0 +1,184 @@ +"""Duplicate cleanup keeps images' own categories (#332 MC11). + +The kept copy's own category wins; a kept copy without one receives the +removed copy's. The removed copy's row is never deleted, so a give-back +(the kept copy changed) and a failed delete lose nothing. +""" + +import asyncio +import os +from unittest.mock import patch + +try: + from tests.test_api_duplicate_files import WAIT, DuplicateFilesTestCase +except ImportError: # discovered with tests/ as the top-level directory + from test_api_duplicate_files import WAIT, DuplicateFilesTestCase + +from py.api import duplicate_files # noqa: E402 +from py.api.images import image_category_key # noqa: E402 +from py.config import GalleryConfig # noqa: E402 + + +class TestDuplicateCleanupKeepsCategories(DuplicateFilesTestCase): + def key(self, path): + return image_category_key(self.api._get_all_output_dirs(), path) + + def category(self, path): + return self.api.db.image_category(*self.key(path)) + + def set_category(self, path, value): + self.api.db.set_image_categories([self.key(path)], value) + + async def run_cleanup(self, keep="a.png"): + job, group = await self.found_group() + plan = await self.review(job.id, [self.keep_of(group, keep)]) + body = await self.apply(plan["token"]) + cleanup = self.api._jobs.get(body["job"]["id"]) + await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + return cleanup + + async def test_kept_copy_without_a_category_receives_the_removed_one(self): + self.trio() + b = self.output_dir / "b.png" + self.set_category(b, "Portraits") + cleanup = await self.run_cleanup() + self.assertFalse(b.exists()) + self.assertEqual(self.category(self.output_dir / "a.png"), "Portraits") + # The removed copy's row stays (hidden while its file is gone). + self.assertEqual(self.category(b), "Portraits") + self.assertIn( + "Kept 1 image category on the kept copies", "\n".join(cleanup.log) + ) + + async def test_kept_copy_keeps_its_own_category(self): + self.trio() + self.set_category(self.output_dir / "a.png", "Landscapes") + self.set_category(self.output_dir / "b.png", "Portraits") + self.set_category(self.output_dir / "sub" / "c.png", "Scans") + cleanup = await self.run_cleanup() + self.assertEqual(self.category(self.output_dir / "a.png"), "Landscapes") + self.assertNotIn("image categor", "\n".join(cleanup.log)) + + async def test_two_removed_copies_give_the_first_category(self): + self.trio() + self.set_category(self.output_dir / "b.png", "Portraits") + self.set_category(self.output_dir / "sub" / "c.png", "Scans") + cleanup = await self.run_cleanup() + self.assertIn(self.category(self.output_dir / "a.png"), {"Portraits", "Scans"}) + self.assertIn( + "Kept 1 image category on the kept copies", "\n".join(cleanup.log) + ) + + async def test_give_back_loses_no_category(self): + self.trio() + self.set_category(self.output_dir / "b.png", "Portraits") + with patch.object(duplicate_files, "still_kept", return_value=False): + cleanup = await self.run_cleanup() + self.assertEqual(cleanup.counters["removed"], 0) + rows = {row["path"]: row["detail"] for row in cleanup.outcomes["kept"]} + self.assertEqual( + rows["b.png"], duplicate_files.ROW_DETAILS[duplicate_files.KEEPER_GONE] + ) + self.assertTrue((self.output_dir / "b.png").is_file()) + self.assertEqual(self.category(self.output_dir / "b.png"), "Portraits") + + async def test_a_failed_delete_keeps_both_categories(self): + self.trio() + self.set_category(self.output_dir / "b.png", "Portraits") + real = duplicate_files._unlink + + def unlink(path): + if os.path.basename(path) == "b.png": + raise PermissionError(13, "denied") + real(path) + + with patch.object(duplicate_files, "_unlink", side_effect=unlink): + await self.run_cleanup() + self.assertTrue((self.output_dir / "b.png").is_file()) + self.assertEqual(self.category(self.output_dir / "b.png"), "Portraits") + self.assertEqual(self.category(self.output_dir / "a.png"), "Portraits") + + async def test_nested_sources_use_the_deepest_source_keys(self): + self.write("sub/a.png", b"same", mtime=1) + self.write("sub/b.png", b"same", mtime=2) + # The subfolder is listed first, so the cleanup's listed root is the + # file key's source; the outer source still holds the files. + GalleryConfig.MONITORING_DIRECTORIES = [ + self._root(self.output_dir / "sub"), + self._root(self.output_dir), + ] + sub_key = duplicate_files.favorite_root_key(self.output_dir / "sub") + self.api.db.set_image_categories([(sub_key, "b.png")], "Portraits") + await self.run_cleanup() + self.assertFalse((self.output_dir / "sub" / "b.png").exists()) + self.assertEqual(self.api.db.image_category(sub_key, "a.png"), "Portraits") + outer_key = duplicate_files.favorite_root_key(self.output_dir) + self.assertIsNone(self.api.db.image_category(outer_key, "sub/a.png")) + + async def test_outer_source_listed_first_still_keys_by_the_deepest(self): + self.write("sub/a.png", b"same", mtime=1) + self.write("sub/b.png", b"same", mtime=2) + GalleryConfig.MONITORING_DIRECTORIES = [ + self._root(self.output_dir), + self._root(self.output_dir / "sub"), + ] + sub_key = duplicate_files.favorite_root_key(self.output_dir / "sub") + self.api.db.set_image_categories([(sub_key, "b.png")], "Portraits") + job, group = await self.found_group() + [kept] = [f for f in group["files"] if f["path"].endswith("a.png")][:1] + plan = await self.review( + job.id, [self.keep_of(group, kept["path"], kept["root_index"])] + ) + body = await self.apply(plan["token"]) + cleanup = self.api._jobs.get(body["job"]["id"]) + await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT) + self.assertFalse((self.output_dir / "sub" / "b.png").exists()) + self.assertEqual(self.api.db.image_category(sub_key, "a.png"), "Portraits") + + async def test_a_symlinked_output_folder_still_moves_the_category(self): + link = self.output_dir.parent / "linked-output" + try: + os.symlink(self.output_dir, link, target_is_directory=True) + except (OSError, NotImplementedError): + self.skipTest("symlinks are not available") + self.trio() + with patch.object(self.api, "_get_all_output_dirs", return_value=[link]): + key = image_category_key([link], link / "b.png") + self.api.db.set_image_categories([key], "Portraits") + await self.run_cleanup() + kept = image_category_key([link], link / "a.png") + self.assertFalse((self.output_dir / "b.png").exists()) + self.assertEqual(self.api.db.image_category(*kept), "Portraits") + + async def test_a_copy_with_a_collection_and_a_category_moves_both(self): + # #331 + #332: one removed copy carries both records; one cleanup + # moves its membership and copies its category onto the kept copy. + self.trio() + db = self.api.db + root_key = duplicate_files.favorite_root_key(self.output_dir) + b = self.output_dir / "b.png" + grids = db.add_to_collection(name="LoRA grids", items=[(root_key, "b.png")]) + grids = grids["collection"]["id"] + self.set_category(b, "Portraits") + seen = {} + real_log = duplicate_files.DuplicateFilesMixin._log_moved_records + + def log_moved_records(job, totals): + seen.update(totals) + real_log(job, totals) + + with patch.object( + duplicate_files.DuplicateFilesMixin, + "_log_moved_records", + staticmethod(log_moved_records), + ): + cleanup = await self.run_cleanup() + self.assertFalse(b.exists()) + self.assertEqual(db.collection_members(grids, root_key), {"a.png"}) + self.assertEqual(db.file_collections(root_key, "b.png"), []) + self.assertEqual(self.category(self.output_dir / "a.png"), "Portraits") + self.assertEqual(seen["collections"], 1) + self.assertEqual(seen["categories"], 1) + log = "\n".join(cleanup.log) + self.assertIn("Moved 1 collection membership to the kept copies", log) + self.assertIn("Kept 1 image category on the kept copies", log) diff --git a/tests/test_execution_history_writer.py b/tests/test_execution_history_writer.py index 1d93dcbd..c0b11633 100644 --- a/tests/test_execution_history_writer.py +++ b/tests/test_execution_history_writer.py @@ -31,6 +31,12 @@ from utils.execution_history_writer import HistoryWriter # 0.19 s on macOS py3.13 (PR #351). NON_BLOCKING_CEILING = 1.0 +# Saturation tests block the worker while they queue 1023 more items. The +# block must hold until the test releases it (#346/#352): a 3 s timer let a +# loaded runner unblock the worker mid-fill and drain it. This is only a hang +# guard; the test, or cleanup, always sets the release. +SATURATION_HOLD_SECONDS = 60 + class ManualClock: def __init__(self): @@ -84,6 +90,26 @@ class HistoryWriterTests(unittest.TestCase): threading.Event().wait(0.005) self.fail("finite deadline expired waiting for worker behavior") + def wait_for_worker_pass(self, writer): + """Wait until the idle worker has run one whole loop pass from now. + + A worker with no accepted work ends every pass by parking on its wake + event. The first park counted may end a pass already under way; the + second ends a pass that began after this call, so whatever the worker + decides at the current clock value has happened (state, not a + wall-clock pause that a loaded runner can outlast). + """ + wake = writer._wake + park = wake.wait + parks = [] + + def counted(timeout=None): + parks.append(timeout) + return park(timeout) + + with patch.object(wake, "wait", counted): + self.wait_for(lambda: len(parks) >= 2) + def writer(self, factory=None, ready=True): writer = HistoryWriter(factory or (lambda: self.db), clock=self.clock) self.writers.append(writer) @@ -301,7 +327,7 @@ class HistoryWriterTests(unittest.TestCase): def write(command): entered.set() - release.wait(3) + release.wait(SATURATION_HOLD_SECONDS) return original(command) with patch.object(self.db, "write_execution_start", write): @@ -348,9 +374,12 @@ class HistoryWriterTests(unittest.TestCase): first = self.sealed(writer) charge = first.lease.byte_count + left = threading.Event() + def write(*args, **kwargs): entered.set() release.wait(3) + left.set() return original(*args, **kwargs) with patch.object(self.db, "write_execution_contexts_for_finish", write): @@ -396,8 +425,21 @@ class HistoryWriterTests(unittest.TestCase): before = monotonic() self.assertFalse(writer.close(0.02)) elapsed = monotonic() - before + # close() waits until its own monotonic deadline, so the lower + # bound holds on any runner. self.assertGreaterEqual(elapsed, 0.02) - self.assertLess(elapsed, 0.2) + if early_join: + # Manual clock: only the patched join/sleep advance it, so this + # exact bound proves early joins do not spin forever. + self.assertLess(elapsed, 0.2) + else: + # Real clock: a loaded runner may run late, but close() must + # not wait anywhere near the 3 s the blocked write holds. + self.assertLess(elapsed, 0.02 + NON_BLOCKING_CEILING) + # Bounded on the real clock too: close() gave up while the + # in-flight write was still blocked, instead of waiting it out. + self.assertFalse(left.is_set()) + self.assertTrue(writer.status().alive) self.assertEqual(pool.charged_bytes, charge) replacement = HistoryWriter(lambda: self.db, context_budget=pool) self.writers.append(replacement) @@ -771,7 +813,7 @@ class HistoryWriterTests(unittest.TestCase): self.wait_for(lambda: not writer.status().accepting) self.assertFalse(writer.submit(fresh)) self.clock.advance(4.99) - threading.Event().wait(0.08) + self.wait_for_worker_pass(writer) self.assertEqual(probes, [0]) self.clock.advance(0.01) self.wait_for(lambda: writer.status().accepting) @@ -794,12 +836,12 @@ class HistoryWriterTests(unittest.TestCase): writer = self.writer(factory, ready=False) self.wait_for(lambda: len(calls) == 1) self.clock.advance(4.99) - threading.Event().wait(0.08) + self.wait_for_worker_pass(writer) self.assertEqual(calls, [0]) self.clock.advance(0.01) self.wait_for(lambda: len(calls) == 2) self.clock.advance(4.99) - threading.Event().wait(0.08) + self.wait_for_worker_pass(writer) self.assertEqual(calls, [0, 5]) self.clock.advance(0.01) self.wait_for(lambda: writer.status().accepting) @@ -853,7 +895,7 @@ class HistoryWriterTests(unittest.TestCase): ) writer.invalidate() release.set() - threading.Event().wait(0.08) + self.wait_for_worker_pass(writer) self.assertFalse(writer.status().accepting) self.assertIsNone(writer.status().epoch) self.assertFalse(writer.submit(self.start_command(epoch=stale_epoch))) @@ -871,7 +913,7 @@ class HistoryWriterTests(unittest.TestCase): lock.execute("BEGIN IMMEDIATE") before = time.monotonic() self.assertTrue(writer.submit(command)) - self.assertLess(time.monotonic() - before, 0.1) + self.assertLess(time.monotonic() - before, NON_BLOCKING_CEILING) self.wait_for(lambda: not writer.status().accepting) lock.rollback() self.clock.advance(5) diff --git a/tests/test_execution_output_writer.py b/tests/test_execution_output_writer.py index 06475d24..b274e535 100644 --- a/tests/test_execution_output_writer.py +++ b/tests/test_execution_output_writer.py @@ -18,6 +18,7 @@ from utils.execution_output_capture import seal_outputs from utils.execution_outputs import snapshot_output_roots from utils.execution_history_writer import HistoryWriter from tests.history_waits import drain_writer +from tests.disk_waits import DISK_BOUND_SECONDS try: from tests import test_execution_history_writer as fixtures @@ -28,11 +29,17 @@ except ImportError: # Upper bound a blocked fake write waits for its release. Tests prove ordering # with events, so this only caps a failing run and never decides a pass. BLOCK_SECONDS = 10 + +# Saturation tests block the worker while they queue 1023 more items. The +# block must hold until the test releases it (#346/#352): a 3 s timer let a +# loaded runner unblock the worker mid-fill and drain it. This is only a hang +# guard; the test, or cleanup, always sets the release. +SATURATION_HOLD_SECONDS = 60 + # A restore is a full verify/migrate/backup/replace cycle and is bound by the # disk: a busy Windows runner took over 10 s (merge queue, #382). Waits that -# span a restore use this ceiling, which stays under the runner's 120 s hang -# watchdog, so a real hang still fails with the thread dump. -RESTORE_SECONDS = 90 +# span a restore use the shared disk-bound hang guard (tests/disk_waits.py). +RESTORE_SECONDS = DISK_BOUND_SECONDS class OutputWriterTests(fixtures.unittest.TestCase): @@ -591,7 +598,7 @@ class OutputWriterTests(fixtures.unittest.TestCase): def write(command): entered.set() - release.wait(3) + release.wait(SATURATION_HOLD_SECONDS) return original(command) with patch.object(self.db, "write_execution_start", write): diff --git a/tests/test_exit_cancel.py b/tests/test_exit_cancel.py new file mode 100644 index 00000000..e3fb5ba5 --- /dev/null +++ b/tests/test_exit_cancel.py @@ -0,0 +1,463 @@ +"""Work running at Ctrl+C stops at its next safe point (#376). + +ComfyUI catches Ctrl+C in ``main.py`` and returns without closing its event +loop. Python then joins every executor thread (``concurrent.futures``'s +``_python_exit``, a ``threading._register_atexit`` callback) before any +``atexit`` callback runs, so a call still running on the default executor +used to keep the process alive until it returned. The package registers its +own ``threading._register_atexit`` hook, which runs before that join: it sets +the exit signal that ``Job.cancelled`` and ``map_parallel`` read, and kills +the encoder processes. + +The child-process test starts three long calls on the default executor (a +registry job checking ``job.cancelled`` between files, a ``map_parallel`` +batch and an encoder process that ignores the exit, as ffmpeg does on +Windows), returns from its main module and must exit far sooner than any of +them would end. +""" + +import json +import os +import subprocess +import sys +import tempfile +import textwrap +import threading +import unittest +from unittest import mock + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +try: + from utils import exit_signal # noqa: E402 +except ImportError: # before #376: only the child-process test can run + exit_signal = None + +# The parent never waits longer than this for the child (the hang watchdog +# allows 120 s per test). Each fake job alone would run 300 s or more. +CHILD_CEILING_SECONDS = 60 +# From the child's return to its last atexit callback: the stopped calls end +# at their next file, item or chunk (the encoder is killed). +EXIT_BOUND_SECONDS = 15 +JOB_SECONDS = 600 +# The fake encoder outlives the ceiling, yet a failing run leaks it only +# briefly (the parent cannot reach the child's own child to kill it). +ENCODER_SECONDS = 90 +REPORT_PREFIX = "PM_EXIT_REPORT " + +CHILD = textwrap.dedent(""" + import asyncio + import atexit + import contextlib + import importlib + import io + import json + import os + import sys + import threading + import time + + root, tmp = sys.argv[1], sys.argv[2] + job_seconds, encoder_seconds = float(sys.argv[3]), int(sys.argv[4]) + sys.path.insert(0, root) + state = {"stopped": {}} + + def report(): + returned = state.get("returned") + print("PM_EXIT_REPORT " + json.dumps({ + "exit_seconds": None if returned is None else time.monotonic() - returned, + "stopped": state["stopped"], + })) + sys.stdout.flush() + + # Registered first, so it runs after every other atexit callback, and + # every atexit callback runs after the executor threads were joined. + atexit.register(report) + + from tests.test_init_package import _import_package, fake_server_module + + output = os.path.join(tmp, "output") + with contextlib.redirect_stdout(io.StringIO()): + package, _ = _import_package( + "pm_exit_child", + os.path.join(tmp, "prompts.db"), + output, + fake_server_module(queue_task_done=True), + ) + parallel = importlib.import_module("pm_exit_child.utils.parallel") + encode = importlib.import_module("pm_exit_child.py.api.thumbnail_encode") + registry = package.api._jobs + entered = {name: threading.Event() for name in ("job", "batch", "encoder")} + + def job_work(job): + # A job's thread work: one "file" at a time, checking its cancel + # between files the way the thumbnail and import jobs do. + entered["job"].set() + deadline = time.monotonic() + job_seconds + while time.monotonic() < deadline: + if job.cancelled: + state["stopped"]["job"] = True + return + time.sleep(0.05) + + def batch_item(index): + entered["batch"].set() + time.sleep(0.5) + return index + + def batch_work(): + try: + parallel.map_parallel(batch_item, range(int(job_seconds)), 2) + except Exception as error: + state["stopped"]["batch"] = type(error).__name__ + + def encoder_work(): + # Stands for ffmpeg on Windows, which runs without a console and + # never sees Ctrl+C: only a kill ends it before its timeout. + cmd = [sys.executable, "-c", "import time; time.sleep(%d)" % encoder_seconds] + try: + encode.run_encoder( + cmd, + timeout=encoder_seconds, + track=lambda _process: entered["encoder"].set(), + ) + except Exception as error: + state["stopped"]["encoder"] = type(error).__name__ + + async def begin(): + loop = asyncio.get_running_loop() + + async def runner(job): + await loop.run_in_executor(None, job_work, job) + + registry.start("exit-test", "Exit test", admission="read", runner=runner) + loop.run_in_executor(None, batch_work) + loop.run_in_executor(None, encoder_work) + for name, event in entered.items(): + if not await loop.run_in_executor(None, event.wait, 30): + sys.exit("not started: " + name) + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(begin()) + state["returned"] = time.monotonic() + # Return with the three calls running and the loop open, as ComfyUI does + # after Ctrl+C. + """) + + +class TestExitStopsRunningWork(unittest.TestCase): + def test_running_executor_work_stops_at_exit(self): + with tempfile.TemporaryDirectory() as tmp: + os.makedirs(os.path.join(tmp, "output")) + script = os.path.join(tmp, "child.py") + with open(script, "w", encoding="utf-8") as handle: + handle.write(CHILD) + env = dict(os.environ) + env.pop("PROMPT_MANAGER_CONFIG_PATH", None) + env["PROMPT_MANAGER_DATA_DIR"] = tmp + env["PYTHONIOENCODING"] = "utf-8" + try: + completed = subprocess.run( + [ + sys.executable, + script, + ROOT, + tmp, + str(JOB_SECONDS), + str(ENCODER_SECONDS), + ], + cwd=ROOT, + env=env, + capture_output=True, + timeout=CHILD_CEILING_SECONDS, + encoding="utf-8", + errors="replace", + ) + except subprocess.TimeoutExpired as hung: + self.fail( + f"the child did not exit within {CHILD_CEILING_SECONDS} s\n" + f"stdout:\n{hung.stdout}\nstderr:\n{hung.stderr}" + ) + detail = f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + self.assertEqual(completed.returncode, 0, detail) + reports = [ + line[len(REPORT_PREFIX) :] + for line in completed.stdout.splitlines() + if line.startswith(REPORT_PREFIX) + ] + self.assertEqual(len(reports), 1, detail) + report = json.loads(reports[0]) + self.assertLess(report["exit_seconds"], EXIT_BOUND_SECONDS, detail) + self.assertEqual( + report["stopped"], + {"job": True, "batch": "ExitRequested", "encoder": "EncoderClosed"}, + detail, + ) + for text in ("Traceback", "Background work not stopped in time"): + self.assertNotIn(text, completed.stderr, detail) + + +class _Isolated(unittest.TestCase): + """Each test starts with the signal clear and no hook installed.""" + + def setUp(self): + exit_signal.reset() + self.addCleanup(exit_signal.reset) + + +class TestInstall(_Isolated): + def test_the_hook_is_registered_once_before_the_executor_join(self): + calls = [] + with mock.patch.object( + exit_signal.threading, + "_register_atexit", + side_effect=lambda func: calls.append(func), + create=True, + ): + self.assertTrue(exit_signal.install()) + self.assertTrue(exit_signal.install()) + self.assertEqual(len(calls), 1) + self.assertFalse(exit_signal.requested()) + calls[0]() + self.assertTrue(exit_signal.requested()) + + def test_a_missing_internal_hook_degrades_to_the_old_exit(self): + with mock.patch.object(exit_signal, "threading") as fake: + del fake._register_atexit + self.assertFalse(exit_signal.install()) + self.assertFalse(exit_signal.requested()) + + def test_a_refused_registration_degrades_to_the_old_exit(self): + refuse = RuntimeError("can't register atexit after shutdown") + with mock.patch.object( + exit_signal.threading, "_register_atexit", side_effect=refuse, create=True + ): + self.assertFalse(exit_signal.install()) + + def test_a_failing_exit_action_never_escapes_the_hook(self): + # An exception out of a threading atexit callback would abort + # threading._shutdown and skip the executor join itself. + calls = [] + + def boom(): + raise KeyboardInterrupt # a second Ctrl+C + + with mock.patch.object( + exit_signal.threading, + "_register_atexit", + side_effect=lambda func: calls.append(func), + create=True, + ): + exit_signal.install(boom) + with self.assertLogs(exit_signal.logger, "ERROR"): + calls[0]() + self.assertTrue(exit_signal.requested()) + + +class TestCheckpoints(_Isolated): + def test_checked_runs_until_the_signal_then_raises(self): + seen = [] + record = exit_signal.checked(seen.append) + record(1) + exit_signal.request() + with self.assertRaises(exit_signal.ExitRequested): + record(2) + self.assertEqual(seen, [1]) + + def test_map_parallel_starts_no_item_after_the_signal(self): + from utils.parallel import map_parallel + + started = [] + gate = threading.Event() + + def item(index): + started.append(index) + if index == 0: + exit_signal.request() + gate.set() + return index + + with self.assertRaises(exit_signal.ExitRequested): + map_parallel(item, range(50), 1) + self.assertTrue(gate.is_set()) + self.assertEqual(started, [0]) + + def test_job_cancelled_follows_the_signal_and_its_own_cancel(self): + from py.api.jobs import Job + + job = Job("scan", "Scan", admission="records") + self.assertFalse(job.cancelled) + job.cancelled = True + self.assertTrue(job.cancelled) + other = Job("scan", "Scan", admission="records", cancellable=False) + exit_signal.request() + self.assertTrue(other.cancelled) + + def test_the_scan_walk_stops_between_directories(self): + from py.api.admin import _collect_media_files + + with tempfile.TemporaryDirectory() as tmp: + for name in ("a", "b"): + os.makedirs(os.path.join(tmp, name)) + with open(os.path.join(tmp, name, "x.png"), "wb") as image: + image.write(b"x") + self.assertEqual(len(_collect_media_files([tmp], [".png"])), 2) + exit_signal.request() + with self.assertRaises(exit_signal.ExitRequested): + _collect_media_files([tmp], [".png"]) + + +class TestLoraImportStopsAtExit(_Isolated): + def test_staging_stops_before_the_next_sidecar_whatever_the_caller_checks(self): + import types + + from py import lora_import + + discovery = types.SimpleNamespace( + sidecars=[object(), object()], unreadable=[], layout=None + ) + exit_signal.request() + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(lora_import.ImportCancelled): + lora_import.stage( + discovery, + tmp, + media_extensions=frozenset(), + cancelled=lambda: False, # the legacy scan's own check + ) + + +class _FakeTqdm: + """huggingface_hub.utils.tqdm as far as a download uses it.""" + + def __init__(self, *args, **kwargs): + self.n = 0 + + def update(self, n=1): + self.n += n + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def _fake_hub(chunks_before_exit): + """A ``huggingface_hub`` whose download writes chunks to a temp file and, + like huggingface_hub 1.x, removes it on any failure before the rename.""" + import types + + def hf_hub_download( + repo_id, filename, local_dir=None, local_dir_use_symlinks=None, tqdm_class=None + ): + final = os.path.join(local_dir, filename) + partial = final + ".abc.incomplete" + bar = (tqdm_class or _FakeTqdm)(total=10) + try: + with open(partial, "wb") as handle: + for index in range(10): + if index == chunks_before_exit: + exit_signal.request() + bar.update(1) + handle.write(b"x") + os.replace(partial, final) + finally: + if os.path.exists(partial): + os.unlink(partial) + return final + + hub = types.ModuleType("huggingface_hub") + hub.hf_hub_download = hf_hub_download + hub.snapshot_download = lambda **kwargs: None + utils = types.ModuleType("huggingface_hub.utils") + utils.tqdm = _FakeTqdm + hub.utils = utils + return {"huggingface_hub": hub, "huggingface_hub.utils": utils} + + +class TestModelDownloadStopsAtExit(_Isolated): + def test_a_download_stops_between_chunks_and_leaves_no_file(self): + from py.autotag import AutoTagService + + with tempfile.TemporaryDirectory() as tmp: + service = AutoTagService(models_dir=tmp) + with mock.patch.dict(sys.modules, _fake_hub(chunks_before_exit=3)): + with self.assertLogs(service.logger, "INFO") as logs: + self.assertFalse(service.download_model("wd14-vit")) + folder = os.path.join(tmp, "WD14-ViT") + self.assertEqual(os.listdir(folder), []) + self.assertTrue(any("exiting" in line for line in logs.output), logs.output) + self.assertFalse(any("Download failed" in line for line in logs.output)) + + def test_a_download_without_the_signal_finishes(self): + from py.autotag import AutoTagService + + with tempfile.TemporaryDirectory() as tmp: + service = AutoTagService(models_dir=tmp) + with mock.patch.dict(sys.modules, _fake_hub(chunks_before_exit=None)): + self.assertTrue(service.download_model("wd14-vit")) + folder = os.path.join(tmp, "WD14-ViT") + self.assertEqual( + sorted(os.listdir(folder)), ["model.onnx", "selected_tags.csv"] + ) + + def test_xet_transfers_are_aborted_the_way_hub_aborts_on_ctrl_c(self): + import types + + from py.autotag import abort_hub_downloads + + aborted = [] + xet = types.ModuleType("huggingface_hub.utils._xet") + xet.abort_xet_session = lambda: aborted.append(True) + hub = _fake_hub(chunks_before_exit=None) + hub["huggingface_hub.utils._xet"] = xet + hub["huggingface_hub.utils"]._xet = xet + with mock.patch.dict(sys.modules, hub): + abort_hub_downloads() + self.assertEqual(aborted, [True]) + with mock.patch.dict(sys.modules): + sys.modules.pop("huggingface_hub", None) + abort_hub_downloads() # never imported: nothing to abort + self.assertEqual(aborted, [True]) + + def test_an_aborted_xet_download_is_logged_as_stopped(self): + from py.autotag import AutoTagService + + hub = _fake_hub(chunks_before_exit=None) + + def aborted(*args, **kwargs): + exit_signal.request() # the hook ran, then hub raised + raise RuntimeError("xet session aborted") + + hub["huggingface_hub"].hf_hub_download = aborted + with tempfile.TemporaryDirectory() as tmp: + service = AutoTagService(models_dir=tmp) + with mock.patch.dict(sys.modules, hub): + with self.assertLogs(service.logger, "INFO") as logs: + self.assertFalse(service.download_model("wd14-vit")) + self.assertTrue(any("exiting" in line for line in logs.output), logs.output) + self.assertFalse(any("Download failed" in line for line in logs.output)) + + def test_an_old_hub_without_tqdm_class_downloads_as_before(self): + from py.autotag import AutoTagService + + hub = _fake_hub(chunks_before_exit=None) + full = hub["huggingface_hub"].hf_hub_download + + def old_download( + repo_id, filename, local_dir=None, local_dir_use_symlinks=None + ): + return full(repo_id, filename, local_dir, local_dir_use_symlinks) + + hub["huggingface_hub"].hf_hub_download = old_download + with tempfile.TemporaryDirectory() as tmp: + service = AutoTagService(models_dir=tmp) + with mock.patch.dict(sys.modules, hub): + self.assertTrue(service.download_model("wd14-vit")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_exit_window_threads.py b/tests/test_exit_window_threads.py new file mode 100644 index 00000000..1323f4db --- /dev/null +++ b/tests/test_exit_window_threads.py @@ -0,0 +1,266 @@ +"""A thread or pool refused at interpreter shutdown is quiet (#375). + +When the interpreter exits, ``threading._shutdown`` first shuts every +``ThreadPoolExecutor`` (``submit`` then raises "cannot schedule new futures +after interpreter shutdown") and joins the non-daemon threads; only then do +the ``atexit`` callbacks (the package teardown) run. Our daemon threads keep +working meanwhile. ``Thread.start`` is refused ("can't create new thread at +interpreter shutdown") only later, once the interpreter is finalizing; the +unit tests cover that case, the child process the pool case. Such a refusal +is logged once at DEBUG, with no traceback, and leaves nothing pending. + +``threading.Thread`` is replaced only in the module under test (a copy of +the ``threading`` namespace), never process-wide. The child-process test runs +the real exit: a daemon thread queues previews while the main thread returns. +""" + +import json +import logging +import os +import subprocess +import sys +import tempfile +import textwrap +import threading +import types +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest import mock + +from py.api import thumbnail_auto +from utils import image_monitor, parallel + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +CHILD_CEILING_SECONDS = 60 # the hang watchdog allows 120 s per test +REPORT_PREFIX = "PM_EXIT_REPORT " +THREAD_REFUSAL = "can't create new thread at interpreter shutdown" +POOL_REFUSAL = "cannot schedule new futures after interpreter shutdown" + + +def _refusing_threading(): + """A ``threading`` namespace whose ``Thread.start`` is refused.""" + + class RefusedThread(threading.Thread): + def start(self): + raise RuntimeError(THREAD_REFUSAL) + + namespace = types.SimpleNamespace(**vars(threading)) + namespace.Thread = RefusedThread + return namespace + + +class _ShutPool(ThreadPoolExecutor): + def submit(self, *args, **kwargs): + raise RuntimeError(POOL_REFUSAL) + + +class _Capture(logging.Handler): + def __init__(self): + super().__init__(logging.DEBUG) + self.records = [] + + def emit(self, record): + self.records.append(record) + + +def _capture(testcase, name): + parallel._noted.clear() # "once" is per process: start each test afresh + logger = logging.getLogger(name) + handler = _Capture() + previous = logger.level + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + + def restore(): + logger.removeHandler(handler) + logger.setLevel(previous) + + testcase.addCleanup(restore) + return handler.records + + +def _assert_one_debug_line(testcase, records): + """Exactly one refusal line, at DEBUG and without a traceback.""" + refusals = [r for r in records if "interpreter is exiting" in r.getMessage()] + testcase.assertEqual(len(refusals), 1, [r.getMessage() for r in records]) + testcase.assertEqual(refusals[0].levelno, logging.DEBUG) + testcase.assertEqual([r for r in records if r.levelno > logging.DEBUG], []) + testcase.assertTrue(all(r.exc_info is None for r in records)) + + +class TestScheduleRefusedAtShutdown(unittest.TestCase): + def setUp(self): + self.handler = image_monitor.ImageGenerationHandler( + mock.MagicMock(), mock.MagicMock() + ) + self.addCleanup(self.handler.close) + + def test_schedule_returns_false_and_leaves_nothing_pending(self): + records = _capture(self, "prompt_manager.image_monitor") + with mock.patch.object(image_monitor, "threading", _refusing_threading()): + self.assertFalse(self.handler.schedule("/out/a.png")) + self.assertFalse(self.handler.schedule("/out/b.png")) + self.assertEqual(self.handler.pending_count(), 0) + _assert_one_debug_line(self, records) + + def test_another_start_failure_still_raises_and_leaves_nothing_pending(self): + class Failing(threading.Thread): + def start(self): + raise RuntimeError("can't start new thread") + + namespace = types.SimpleNamespace(**vars(threading)) + namespace.Thread = Failing + with mock.patch.object(image_monitor, "threading", namespace): + with self.assertRaises(RuntimeError): + self.handler.schedule("/out/a.png") + self.assertEqual(self.handler.pending_count(), 0) + + def test_schedule_before_shutdown_is_unchanged(self): + self.handler.processing_delay = 3600 + self.assertTrue(self.handler.schedule("/out/a.png")) + self.assertEqual(self.handler.pending_count(), 1) + + +class TestPreviewQueueRefusedAtShutdown(unittest.TestCase): + def thumbnailer(self, logger): + made = thumbnail_auto.AutoThumbnailer( + lambda *a, **k: {"action": "generated"}, lambda: True, logger + ) + self.addCleanup(made.close) + return made + + def test_enqueue_is_refused_quietly_and_drops_the_file(self): + logger = logging.getLogger("prompt_manager.test_exit_window") + records = _capture(self, logger.name) + thumbnailer = self.thumbnailer(logger) + src = Path("/out/a.png") + with mock.patch.object(thumbnail_auto, "threading", _refusing_threading()): + self.assertFalse(thumbnailer.enqueue(src, Path("/out/t/a.webp"), False)) + self.assertFalse(thumbnailer.enqueue(src, Path("/out/t/b.webp"), False)) + self.assertFalse(thumbnailer._worker_running) + self.assertEqual(len(thumbnailer._queue), 0) + self.assertEqual(thumbnailer._queued, set()) + self.assertTrue(thumbnailer.wait_idle(0)) + _assert_one_debug_line(self, records) + + +class TestMapParallelRefusedAtShutdown(unittest.TestCase): + def test_a_shut_pool_runs_the_items_on_the_calling_thread(self): + records = _capture(self, "prompt_manager.parallel") + with mock.patch.object(parallel, "ThreadPoolExecutor", _ShutPool): + self.assertEqual( + parallel.map_parallel(lambda n: n * n, [1, 2, 3], 4), [1, 4, 9] + ) + _assert_one_debug_line(self, records) + + def test_another_pool_error_still_raises(self): + class Broken(ThreadPoolExecutor): + def submit(self, *args, **kwargs): + raise RuntimeError("something else") + + with mock.patch.object(parallel, "ThreadPoolExecutor", Broken): + with self.assertRaises(RuntimeError): + parallel.map_parallel(lambda n: n, [1, 2], 2) + + +CHILD = textwrap.dedent(""" + import threading + + # Registered before concurrent.futures is imported, so it runs after the + # pool's own exit hook: from here on, a pool refuses new work. + exiting = threading.Event() + threading._register_atexit(exiting.set) + + import json + import logging + import os + import sys + from pathlib import Path + + root, tmp = sys.argv[1], sys.argv[2] + sys.path.insert(0, root) + + from py.api.thumbnail_auto import AutoThumbnailer + + logging.basicConfig(level=logging.WARNING, stream=sys.stderr) + logger = logging.getLogger("prompt_manager.exit_child") + + def generate(src, dst, is_video, *, stop, guard, track): + Path(dst).write_bytes(b"preview") + return {"action": "generated"} + + thumbnailer = AutoThumbnailer( + generate, lambda: True, logger, workers=lambda: 4 + ) + queued = threading.Event() + + def late_monitor(): + exiting.wait() + # One batch of two: the worker maps it on a pool. + with thumbnailer._changed: + for name in ("a", "b"): + src = Path(tmp, name + ".png") + src.write_bytes(b"x") + thumbnailer.enqueue(src, Path(tmp, name + "_thumb.webp"), False) + queued.set() + + def holder(): + # Non-daemon: the interpreter waits for it, inside the exit window. + exiting.wait() + queued.wait(30) + idle = thumbnailer.wait_idle(30) + written = sorted( + n for n in os.listdir(tmp) if n.endswith("_thumb.webp") + ) + print("PM_EXIT_REPORT " + json.dumps({"idle": idle, "written": written})) + sys.stdout.flush() + + threading.Thread(target=late_monitor, daemon=True).start() + threading.Thread(target=holder).start() + # Return normally, as ComfyUI does after Ctrl+C. + """) + + +class TestExitWindowChild(unittest.TestCase): + def test_previews_queued_in_the_exit_window_log_no_traceback(self): + with tempfile.TemporaryDirectory() as tmp: + script = os.path.join(tmp, "child.py") + with open(script, "w", encoding="utf-8") as handle: + handle.write(CHILD) + env = dict(os.environ) + env.pop("PROMPT_MANAGER_CONFIG_PATH", None) + env["PROMPT_MANAGER_DATA_DIR"] = tmp + env["PYTHONIOENCODING"] = "utf-8" + try: + completed = subprocess.run( + [sys.executable, script, ROOT, tmp], + cwd=ROOT, + env=env, + capture_output=True, + timeout=CHILD_CEILING_SECONDS, + encoding="utf-8", + errors="replace", + ) + except subprocess.TimeoutExpired as hung: + self.fail( + f"the child did not exit within {CHILD_CEILING_SECONDS} s\n" + f"stdout:\n{hung.stdout}\nstderr:\n{hung.stderr}" + ) + detail = f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + self.assertEqual(completed.returncode, 0, detail) + reports = [ + line[len(REPORT_PREFIX) :] + for line in completed.stdout.splitlines() + if line.startswith(REPORT_PREFIX) + ] + self.assertEqual(len(reports), 1, detail) + report = json.loads(reports[0]) + self.assertTrue(report["idle"], detail) + # The batch still ran: on the worker's own thread. + self.assertEqual(report["written"], ["a_thumb.webp", "b_thumb.webp"], detail) + self.assertNotIn("Traceback", completed.stderr, detail) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_image_categories.py b/tests/test_image_categories.py new file mode 100644 index 00000000..9972c70e --- /dev/null +++ b/tests/test_image_categories.py @@ -0,0 +1,523 @@ +"""Image categories (#332): schema, migration, restore check, store and file key.""" + +import hashlib +import ntpath +import os +import sqlite3 +import tempfile +import unittest +from contextlib import closing +from pathlib import Path +from unittest.mock import patch + +try: + from open_handles import assert_closed +except ImportError: + from tests.open_handles import assert_closed + +from database.image_categories import ( + COPY_SQL, + INDEX, + STATEMENTS, + TABLE, + copy_image_category, + image_categories_schema_error, + migrate_image_categories, +) +from database.models import PromptModel +from database.operations import PromptDatabase +from py.api.images import image_category_key + +_DDL = ( + f"CREATE TABLE {TABLE} (root_key TEXT NOT NULL, relative_path TEXT NOT NULL, " + "category TEXT NOT NULL, updated_at TEXT NOT NULL, " + "PRIMARY KEY (root_key, relative_path))" +) +_INDEX_DDL = f"CREATE INDEX {INDEX} ON {TABLE}(category)" + + +def category_checksum(conn) -> tuple: + """sha256 of prompts and prompt_pairs ``(id, category, updated_at)``.""" + sums = [] + for table in ("prompts", "prompt_pairs"): + digest = hashlib.sha256() + for row in conn.execute( # nosec B608 - fixed table names + f"SELECT id, category, updated_at FROM {table} ORDER BY id" + ): + digest.update(repr(tuple(row)).encode()) + sums.append(digest.hexdigest()) + return tuple(sums) + + +def table_checksums(conn) -> dict: + """``{name: (schema sql, sha256 of rows)}`` of every table but ours.""" + found = {} + for name, sql in conn.execute( + "SELECT name, sql FROM sqlite_master WHERE type='table' AND name != ?", + (TABLE,), + ).fetchall(): + digest = hashlib.sha256() + rows = conn.execute(f'SELECT * FROM "{name}"') # nosec B608 + for text in sorted(repr(tuple(row)) for row in rows): + digest.update(text.encode()) + found[name] = (sql, digest.hexdigest()) + return found + + +class _TempDir(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.path = os.path.join(self.tmpdir, "categories.db") + + def tearDown(self): + PromptModel.close_all_instances() + assert_closed(self, self.tmpdir) + for name in os.listdir(self.tmpdir): + os.unlink(os.path.join(self.tmpdir, name)) + os.rmdir(self.tmpdir) + + +class TestImageCategoriesSchema(_TempDir): + def test_migration_creates_table_and_index(self): + db = PromptDatabase(self.path) + db.close_all() + PromptModel.close_all_instances() + with closing(sqlite3.connect(self.path)) as conn: + names = { + (kind, name) + for kind, name in conn.execute( + "SELECT type, name FROM sqlite_master WHERE tbl_name=?", (TABLE,) + ) + } + self.assertEqual( + names, + { + ("table", TABLE), + ("index", INDEX), + ("index", f"sqlite_autoindex_{TABLE}_1"), + }, + ) + self.assertEqual( + conn.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0], 0 + ) + self.assertIsNone(image_categories_schema_error(conn, allow_missing=False)) + + def test_migration_leaves_prompt_categories_unchanged(self): + """23,000 links; the migration reads and writes only its own objects.""" + db = PromptDatabase(self.path) + db.close_all() + PromptModel.close_all_instances() + with closing(sqlite3.connect(self.path)) as conn: + conn.execute(f"DROP TABLE {TABLE}") # a library from before #332 + categories = ["Landscapes", "Portraits", None, " ", "Scans"] + conn.executemany( + "INSERT INTO prompts (text, category, hash, updated_at) " + "VALUES (?, ?, ?, ?)", + [ + (f"p{n}", categories[n % 5], f"h{n}", f"2026-01-01 00:00:{n % 60}") + for n in range(200) + ], + ) + conn.executemany( + "INSERT INTO prompt_pairs (positive_prompt_id, negative_prompt_id, " + "category) VALUES (?, ?, ?)", + [(n, n + 100, categories[(n + 1) % 5]) for n in range(1, 50)], + ) + conn.executemany( + "INSERT INTO generated_images (prompt_id, image_path, filename, " + "file_path) VALUES (?, ?, ?, ?)", + [ + (n % 200 + 1, f"/out/{n}.png", f"{n}.png", f"/out/{n}.png") + for n in range(23000) + ], + ) + conn.commit() + before = table_checksums(conn) + prompts_before = category_checksum(conn) + seen = [] + + def authorize(action, arg1, arg2, _db, _trigger): + seen.append((action, arg1, arg2)) + return sqlite3.SQLITE_OK + + conn.set_authorizer(authorize) + migrate_image_categories(conn) + conn.commit() + conn.set_authorizer(None) + own = {TABLE, "sqlite_master", "sqlite_schema", INDEX} + touched = set() + for action, arg1, arg2 in seen: + if action in (sqlite3.SQLITE_READ, sqlite3.SQLITE_INSERT): + touched.add(arg1) + if action == sqlite3.SQLITE_READ: + self.assertIn(arg1, own, (action, arg1, arg2)) + elif action in ( + sqlite3.SQLITE_INSERT, + sqlite3.SQLITE_UPDATE, + sqlite3.SQLITE_DELETE, + ): + self.assertIn(arg1, own, (action, arg1, arg2)) + elif action == sqlite3.SQLITE_CREATE_TABLE: + self.assertEqual(arg1, TABLE) + elif action == sqlite3.SQLITE_CREATE_INDEX: + self.assertIn(arg1, (INDEX, f"sqlite_autoindex_{TABLE}_1")) + self.assertEqual(arg2, TABLE) + elif action == sqlite3.SQLITE_PRAGMA: + self.assertEqual((arg1, arg2), ("table_info", TABLE)) + self.assertIn(TABLE, touched) # the authorizer saw the work + self.assertEqual(table_checksums(conn), before) + self.assertEqual(category_checksum(conn), prompts_before) + schema = conn.execute( + "SELECT sql FROM sqlite_master ORDER BY name" + ).fetchall() + migrate_image_categories(conn) # a second run changes nothing + conn.commit() + self.assertEqual( + conn.execute("SELECT sql FROM sqlite_master ORDER BY name").fetchall(), + schema, + ) + # The whole startup sequence keeps them too. + PromptDatabase(self.path).close_all() + PromptModel.close_all_instances() + with closing(sqlite3.connect(self.path)) as conn: + self.assertEqual(category_checksum(conn), prompts_before) + self.assertEqual( + conn.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0], 0 + ) + + def test_structure_accepted_and_refused(self): + variants = { + "missing": None, + "extra column": ( + f"CREATE TABLE {TABLE} (root_key TEXT NOT NULL, relative_path TEXT " + "NOT NULL, category TEXT NOT NULL, updated_at TEXT NOT NULL, x TEXT, " + "PRIMARY KEY (root_key, relative_path))", + _INDEX_DDL, + ), + "wrong index": (_DDL, f"CREATE INDEX {INDEX} ON {TABLE}(updated_at)"), + "extra index": ( + _DDL, + _INDEX_DDL, + f"CREATE INDEX extra ON {TABLE}(root_key)", + ), + "trigger": ( + _DDL, + _INDEX_DDL, + f"CREATE TRIGGER t AFTER INSERT ON {TABLE} BEGIN SELECT 1; END", + ), + "wrong ddl": ( + f"CREATE TABLE {TABLE} (root_key TEXT, relative_path TEXT, " + "category TEXT, updated_at TEXT, " + "PRIMARY KEY (root_key, relative_path))", + _INDEX_DDL, + ), + "view": (f"CREATE VIEW {TABLE} AS SELECT 1",), + } + for label, statements in variants.items(): + with self.subTest(label), closing(sqlite3.connect(":memory:")) as conn: + for sql in statements or (): + conn.execute(sql) + error = image_categories_schema_error(conn) + if statements is None: + self.assertIsNone(error) + self.assertIsNotNone( + image_categories_schema_error(conn, allow_missing=False) + ) + continue + self.assertIsNotNone(error) + self.assertIsNotNone( + image_categories_schema_error(conn, check_values=False) + ) + with self.assertRaises(sqlite3.DatabaseError): + migrate_image_categories(conn) + # A table without its category index (never written by this code) is + # refused by the check but repaired by the install. + with closing(sqlite3.connect(":memory:")) as conn: + conn.execute(_DDL) + conn.execute(f"INSERT INTO {TABLE} VALUES ('r', 'a.png', 'Scans', 'now')") + self.assertIsNotNone(image_categories_schema_error(conn)) + migrate_image_categories(conn) + self.assertIsNone(image_categories_schema_error(conn, allow_missing=False)) + self.assertEqual( + conn.execute(f"SELECT COUNT(*) FROM {TABLE}").fetchone()[0], 1 + ) + with closing(sqlite3.connect(":memory:")) as conn: + migrate_image_categories(conn) + migrate_image_categories(conn) # idempotent + self.assertIsNone(image_categories_schema_error(conn, allow_missing=False)) + + def test_values_refused_but_startup_checks_structure_only(self): + bad = { + "blank": "", + "101 characters": "x" * 101, + "control character": "a\tb", + "nul": "a\x00b", + "leading space": " Landscapes", + "trailing space": "Landscapes ", + } + for label, value in bad.items(): + with self.subTest(label), closing(sqlite3.connect(":memory:")) as conn: + migrate_image_categories(conn) + conn.execute( + f"INSERT INTO {TABLE} VALUES ('r', 'a.png', ?, 'now')", (value,) + ) + self.assertIsNotNone(image_categories_schema_error(conn)) + self.assertIsNone( + image_categories_schema_error(conn, check_values=False) + ) + migrate_image_categories(conn) # startup still opens it + with closing(sqlite3.connect(":memory:")) as conn: + migrate_image_categories(conn) + conn.execute( + f"INSERT INTO {TABLE} VALUES ('r', ?, 'Scans', 'now')", ("x" * 5000,) + ) + self.assertIsNotNone(image_categories_schema_error(conn)) + conn.execute(f"DELETE FROM {TABLE}") + conn.execute(f"INSERT INTO {TABLE} VALUES ('r', x'00ff', 'Scans', 'now')") + self.assertIsNotNone(image_categories_schema_error(conn)) + conn.execute(f"DELETE FROM {TABLE}") + for value in ("x" * 100, "Scans", "Cafรฉ ยท ้ขจๆ™ฏ", "a b"): + conn.execute( + f"INSERT OR REPLACE INTO {TABLE} VALUES ('r', 'a.png', ?, 'now')", + (value,), + ) + self.assertIsNone(image_categories_schema_error(conn), value) + + def test_restore_check(self): + PromptDatabase(self.path).close_all() + PromptModel.close_all_instances() + ok, reason = PromptModel.verify_database_file( + self.path, require_library_state=True + ) + self.assertTrue(ok, reason) + with closing(sqlite3.connect(self.path)) as conn: + conn.execute(f"DROP TABLE {TABLE}") + conn.commit() + # A backup from before #332 is accepted ... + ok, reason = PromptModel.verify_database_file(self.path) + self.assertTrue(ok, reason) + # ... but a current database must have the table. + ok, reason = PromptModel.verify_database_file( + self.path, require_library_state=True + ) + self.assertFalse(ok) + self.assertIn("image categories", reason.lower()) + with closing(sqlite3.connect(self.path)) as conn: + conn.execute(_DDL) + conn.execute(_INDEX_DDL) + conn.execute(f"INSERT INTO {TABLE} VALUES ('r', 'a.png', ' x', 'now')") + conn.commit() + ok, reason = PromptModel.verify_database_file(self.path) + self.assertFalse(ok) + self.assertIn("image categories", reason.lower()) + + +class TestImageCategoryStore(_TempDir): + def setUp(self): + super().setUp() + self.db = PromptDatabase(self.path) + + def tearDown(self): + self.db.close_all() + super().tearDown() + + def rows(self): + return ( + self.db.model.get_connection() + .execute( + f"SELECT root_key, relative_path, category, updated_at FROM {TABLE} " + "ORDER BY root_key, relative_path" + ) + .fetchall() + ) + + def test_set_counts_updated_and_unchanged(self): + keys = [("/out", "a.png"), ("/other", "a.png"), ("/out", "a.png")] + self.assertEqual( + self.db.set_image_categories(keys, "Portraits"), + {"updated": 2, "unchanged": 0}, + ) + stamps = {(r[0], r[1]): r[3] for r in self.rows()} + self.assertEqual( + self.db.set_image_categories( + [("/out", "a.png"), ("/out", "b.png")], "Portraits" + ), + {"updated": 1, "unchanged": 1}, + ) + # An unchanged file keeps its updated_at. + self.assertEqual( + {(r[0], r[1]): r[3] for r in self.rows()}[("/out", "a.png")], + stamps[("/out", "a.png")], + ) + self.assertEqual( + self.db.set_image_categories([("/out", "a.png")], "Scans"), + {"updated": 1, "unchanged": 0}, + ) + self.assertEqual(self.db.image_category("/out", "a.png"), "Scans") + self.assertEqual(self.db.image_category("/other", "a.png"), "Portraits") + self.assertIsNone(self.db.image_category("/out", "missing.png")) + self.assertEqual( + self.db.own_categories_of(["/out", "/out", "/none"]), + {("/out", "a.png"): "Scans", ("/out", "b.png"): "Portraits"}, + ) + + def test_clear_counts_removed_rows(self): + self.db.set_image_categories([("/out", "a.png"), ("/b", "c.png")], "Scans") + self.assertEqual( + self.db.clear_image_categories( + [("/out", "a.png"), ("/b", "c.png"), ("/out", "gone.png")] + ), + 2, + ) + self.assertEqual(self.rows(), []) + self.assertEqual(self.db.clear_image_categories([("/out", "a.png")]), 0) + + def test_a_failed_write_writes_nothing(self): + self.db.set_image_categories([("/out", "a.png")], "Scans") + keys = [("/out", "b.png"), ("/out", None)] # NOT NULL fails the second + with self.assertRaises(sqlite3.IntegrityError): + self.db.set_image_categories(keys, "Portraits") + self.assertEqual([r[:3] for r in self.rows()], [("/out", "a.png", "Scans")]) + + def test_store_never_touches_prompt_categories(self): + conn = self.db.model.get_connection() + with conn: + conn.execute( + "INSERT INTO prompts (text, category, hash) VALUES ('p', 'L', 'h')" + ) + before = category_checksum(conn) + seen = set() + conn.set_authorizer( + lambda action, arg1, *_: seen.add(arg1) or sqlite3.SQLITE_OK + ) + try: + self.db.set_image_categories([("/out", "a.png")], "Scans") + self.db.own_categories_of(["/out"]) + self.db.image_category("/out", "a.png") + self.db.clear_image_categories([("/out", "a.png")]) + finally: + conn.set_authorizer(None) + self.assertFalse(seen & {"prompts", "prompt_pairs", "generated_images"}, seen) + self.assertEqual(category_checksum(conn), before) + + def test_copy_keeps_kept_copy_category(self): + self.db.set_image_categories([("/out", "removed.png")], "Scans") + self.db.set_image_categories([("/out", "kept.png")], "Portraits") + with self.db._pair_transaction() as conn: + inserted = copy_image_category( + conn, ("/out", "removed.png"), ("/out", "kept.png") + ) + self.assertEqual(inserted, 0) + self.assertEqual(self.db.image_category("/out", "kept.png"), "Portraits") + with self.db._pair_transaction() as conn: + inserted = copy_image_category( + conn, ("/out", "removed.png"), ("/out", "new.png") + ) + self.assertEqual(inserted, 1) + self.assertEqual(self.db.image_category("/out", "new.png"), "Scans") + # The removed copy's row is never deleted. + self.assertEqual(self.db.image_category("/out", "removed.png"), "Scans") + with self.db._pair_transaction() as conn: + self.assertEqual( + copy_image_category(conn, ("/out", "none.png"), ("/out", "x.png")), 0 + ) + self.assertEqual( + copy_image_category( + conn, ("/out", "removed.png"), ("/out", "removed.png") + ), + 0, + ) + + def test_statements_never_scan_the_table(self): + conn = self.db.model.get_connection() + for sql in STATEMENTS: + with self.subTest(sql): + params = (None,) * sql.count("?") + plan = " ".join( + row[-1] for row in conn.execute(f"EXPLAIN QUERY PLAN {sql}", params) + ) + self.assertNotRegex(plan, rf"SCAN {TABLE}\b", plan) + plan = " ".join( + row[-1] + for row in conn.execute( + f"EXPLAIN QUERY PLAN SELECT rowid FROM {TABLE} WHERE category=? " + "ORDER BY rowid", + ("x",), + ) + ) + self.assertIn(INDEX, plan) + self.assertIn(COPY_SQL, STATEMENTS) + + +class TestImageCategoryKey(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.base = Path(os.path.realpath(self._tmp.name)) + self.outer = self.base / "output" + self.inner = self.outer / "nested" + (self.inner / "sub").mkdir(parents=True) + + def tearDown(self): + self._tmp.cleanup() + + def test_equals_the_favorite_key_without_nesting(self): + from py.api.images import _favorite_relative_key, _favorite_root_key + + key = image_category_key([self.outer], self.outer / "sub" / "a.png") + self.assertEqual( + key, (_favorite_root_key(self.outer), _favorite_relative_key("sub/a.png")) + ) + + def test_deepest_source_wins_in_any_order(self): + from py.api.images import _favorite_root_key + + path = self.inner / "sub" / "a.png" + for sources in ([self.outer, self.inner], [self.inner, self.outer]): + with self.subTest(sources=sources): + self.assertEqual( + image_category_key(sources, path), + (_favorite_root_key(self.inner), "sub/a.png"), + ) + # A file of the outer source only is keyed by it. + self.assertEqual( + image_category_key([self.outer, self.inner], self.outer / "b.png"), + (_favorite_root_key(self.outer), "b.png"), + ) + + def test_outside_every_source_has_no_key(self): + self.assertIsNone(image_category_key([self.inner], self.outer / "a.png")) + self.assertIsNone(image_category_key([], self.outer / "a.png")) + # A sibling sharing a name prefix is not inside. + self.assertIsNone( + image_category_key([self.outer], self.base / "output2" / "a.png") + ) + self.assertIsNone(image_category_key([self.outer], self.outer)) + + def test_symlinked_subfolder_is_not_resolved(self): + target = self.base / "elsewhere" + target.mkdir() + link = self.outer / "linked" + try: + os.symlink(target, link, target_is_directory=True) + except (OSError, NotImplementedError): + self.skipTest("symlinks are not available") + key = image_category_key([self.outer], link / "a.png") + self.assertEqual(key[1], "linked/a.png") + + def test_windows_folds_case_of_the_relative_key(self): + with patch("py.api.images._path_flavor", ntpath): + upper = image_category_key([self.outer], self.outer / "Sub" / "A.PNG") + lower = image_category_key([self.outer], self.outer / "sub" / "a.png") + self.assertEqual(upper[1], "sub/a.png") + self.assertEqual(upper, lower) + + def test_posix_keeps_case_distinct(self): + if os.path.normcase("A") == "a": + self.skipTest("this OS folds case") + upper = image_category_key([self.outer], self.outer / "A.png") + lower = image_category_key([self.outer], self.outer / "a.png") + self.assertNotEqual(upper, lower) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_inspect_image_categories.py b/tests/test_inspect_image_categories.py new file mode 100644 index 00000000..37e2d81a --- /dev/null +++ b/tests/test_inspect_image_categories.py @@ -0,0 +1,277 @@ +"""The inspector and History show an image's own category (#332 ยง5, DC4, DC5).""" + +import os +import sqlite3 +import unittest +from unittest.mock import patch + +from tests import test_api_history_outputs as history +from tests.test_api_images import make_png +from py.api.images import _favorite_root_key, image_category_key +from utils.inspection_response import ( + OWN_CATEGORY_UNREAD, + _library_projection, + apply_own_category, +) + + +class InspectCase(history.ImageAPITestCase): + async def tearDownAsync(self): + self.api.db.close_all() + await super().tearDownAsync() + + def own(self, path, value, sources=None): + sources = sources or self.api._get_all_output_dirs() + self.api.db.set_image_categories([image_category_key(sources, path)], value) + + async def library(self, path, root="0"): + response = await self.client.get( + f"/prompt_manager/images/inspect/{path}", params={"root": root} + ) + body = await response.json() + self.assertEqual(response.status, 200, body) + return body["library"] + + +class TestInspectorOwnCategory(InspectCase): + async def test_inspect_reports_image_scope(self): + path = make_png(self.output_dir / "image.png") + self.own(path, "Portraits") + library = await self.library("image.png") + self.assertEqual(library["status"], "none") + self.assertEqual( + (library["category"], library["category_scope"]), ("Portraits", "image") + ) + + async def test_own_overrides_linked_recipe_and_ambiguous(self): + db = self.api.db + linked = make_png(self.output_dir / "linked.png") + self._link_image(linked, "castle at dusk") + prompt = db.get_prompt_id_for_image(str(linked)) + with db.model.get_connection() as conn: + conn.execute( + "UPDATE prompts SET category='Landscapes' WHERE id=?", (prompt,) + ) + library = await self.library("linked.png") + self.assertEqual( + (library["category"], library["category_scope"]), + ("Landscapes", "positive"), + ) + self.own(linked, "Portraits") + library = await self.library("linked.png") + self.assertEqual(library["status"], "linked") + self.assertEqual( + (library["category"], library["category_scope"]), ("Portraits", "image") + ) + recipe = make_png(self.output_dir / "recipe.png") + self._save_prompt("Pair positive", category="Night") + pair = db.save_prompt_pair("Pair positive", "Noise", category="Recipes") + positive = db.get_prompt_pair(pair)["positive_prompt_id"] + image = db.link_image_to_prompt(positive, str(recipe)) + db.link_image_to_pair(pair, image, {"source": "fixture"}) + library = await self.library("recipe.png") + self.assertEqual( + (library["category"], library["category_scope"]), ("Recipes", "pair") + ) + self.own(recipe, "Mine") + library = await self.library("recipe.png") + self.assertEqual( + (library["category"], library["category_scope"]), ("Mine", "image") + ) + ambiguous = make_png(self.output_dir / "ambiguous.png") + self._link_image(ambiguous, "owner one") + self._link_image(ambiguous, "owner two") + self.own(ambiguous, "Scans") + library = await self.library("ambiguous.png") + self.assertEqual(library["status"], "ambiguous") + self.assertEqual( + (library["category"], library["category_scope"]), ("Scans", "image") + ) + + async def test_a_failed_read_hides_the_category(self): + linked = make_png(self.output_dir / "linked.png") + self._link_image(linked, "castle at dusk") + + def broken(*_args, **_kwargs): + raise sqlite3.OperationalError("secret") + + with patch.object(self.api.db, "image_category", side_effect=broken): + with self.assertLogs("prompt_manager", level="WARNING"): + library = await self.library("linked.png") + self.assertEqual(library["status"], "linked") + self.assertIsNone(library["category"]) + self.assertIsNone(library["category_scope"]) + + async def test_a_lora_root_reads_no_own_category(self): + lora = self.comfy_dir / "loras" + make_png(lora / "preview.png") + roots = [self.output_dir, lora] + reads = [] + real = self.api.db.image_category + + def recorded(*args): + reads.append(args) + return real(*args) + + with ( + patch.object(self.api, "_media_roots", lambda outputs: roots), + patch.object(self.api, "_linked_only_error", self._no_error), + patch.object(self.api.db, "image_category", side_effect=recorded), + ): + response = await self.client.get( + "/prompt_manager/images/inspect/preview.png", params={"root": "1"} + ) + body = await response.json() + self.assertEqual(response.status, 200, body) + self.assertEqual(reads, []) + self.assertIsNone(body["library"]["category"]) + + @staticmethod + async def _no_error(*_args): + return None + + +class TestProjection(unittest.TestCase): + def test_scope_image_is_kept_for_category_only(self): + base = {"status": "none", "scope": "saved-library", "tags": []} + projected = _library_projection( + {**base, "category": "Scans", "category_scope": "image"} + ) + self.assertEqual(projected["category_scope"], "image") + projected = _library_projection({**base, "tags_scope": "image"}) + self.assertNotIn("tags_scope", projected) + + def test_apply_own_category_three_cases(self): + library = {"status": "linked", "category": "L", "category_scope": "positive"} + self.assertEqual( + apply_own_category(library, "Mine"), + {"status": "linked", "category": "Mine", "category_scope": "image"}, + ) + self.assertEqual(apply_own_category(library, None), library) + self.assertEqual( + apply_own_category(library, OWN_CATEGORY_UNREAD), + {"status": "linked", "category": None, "category_scope": None}, + ) + self.assertEqual(library["category"], "L") # never mutated + + +class TestHistoryOwnCategory(history.ImageAPITestCase): + """Only the inspection request reads the own category, keyed as Images.""" + + # The History fixture's helpers, without inheriting its tests. + _base = history.HistoryOutputAPITests + ref = _base.ref + capture = _base.capture + outputs = _base.outputs + current = _base.current + media = _base.media + + async def get_application(self): + app = await super().get_application() + self.temp_dir = self.comfy_dir / "temp" + self.temp_dir.mkdir() + self.host_output = str(self.output_dir) + self.host_temp = str(self.temp_dir) + self.patches = history.ExitStack() + self.addCleanup(self.patches.close) + for name, value in ( + ("folder_paths.get_output_directory", lambda: self.host_output), + ("folder_paths.get_temp_directory", lambda: self.host_temp), + ): + self.patches.enter_context(patch(name, value, create=True)) + self.patches.enter_context( + patch.object( + history.GalleryConfig, + "allowed_gallery_parents", + lambda: [str(self.comfy_dir)], + ) + ) + return app + + async def tearDownAsync(self): + self.api.close_history_reads() + self.patches.close() + await super().tearDownAsync() + + async def inspect(self, start): + descriptor = await self.media(start) + response = await self.current(descriptor["inspection_url"]) + body = await response.json() + self.assertEqual(response.status, 200, body) + return body["library"] + + async def test_history_inspection_shows_own_category(self): + path = make_png(self.output_dir / "image.png") + self.api.db.set_image_categories( + [image_category_key([self.output_dir], path)], "Portraits" + ) + start = self.capture([self.ref()]) + library = await self.inspect(start) + self.assertEqual( + (library["category"], library["category_scope"]), ("Portraits", "image") + ) + + async def test_nested_sources_give_the_images_key(self): + inner = self.output_dir / "nested" + path = make_png(inner / "image.png") + self.api._get_all_output_dirs = lambda: [self.output_dir, inner] + self.api.db.set_image_categories( + [(_favorite_root_key(inner), "image.png")], "S" + ) + start = self.capture([self.ref(subfolder="nested")]) + library = await self.inspect(start) + self.assertEqual(library["category"], "S") + self.assertEqual( + image_category_key([self.output_dir, inner], path), + (_favorite_root_key(inner), "image.png"), + ) + + async def test_a_symlinked_output_folder_still_shows_it(self): + real = self.comfy_dir / "real-output" + real.mkdir() + link = self.comfy_dir / "linked-output" + self._make_symlink(real, link, target_is_directory=True) + make_png(real / "image.png") + self.host_output = str(link) + self.api._get_all_output_dirs = lambda: [real] + self.api.db.set_image_categories([(_favorite_root_key(real), "image.png")], "S") + start = self.capture([self.ref()]) + library = await self.inspect(start) + self.assertEqual(library["category"], "S") + + async def test_outputs_and_media_read_no_own_category(self): + make_png(self.output_dir / "image.png") + start = self.capture([self.ref()]) + reads = [] + real = self.api.db.image_category + + def recorded(*args): + reads.append(args) + return real(*args) + + with patch.object(self.api.db, "image_category", side_effect=recorded): + descriptor = await self.media(start) + for field in ("url", "thumbnail_url"): + response = await self.current(descriptor[field]) + await response.read() + self.assertEqual(response.status, 200) + self.assertEqual(reads, []) + await self.current(descriptor["inspection_url"]) + self.assertEqual(len(reads), 1) + + async def test_a_temp_output_has_none(self): + reference = {"folder_type": "temp", "subfolder": "", "filename": "a.png"} + self.assertIsNone(self.api._history_own_category(reference, None)) + + async def test_a_failed_read_hides_it(self): + make_png(self.output_dir / "image.png") + start = self.capture([self.ref()]) + + def broken(*_args): + raise sqlite3.OperationalError("secret") + + with patch.object(self.api.db, "image_category", side_effect=broken): + with self.assertLogs("prompt_manager", level="WARNING"): + library = await self.inspect(start) + self.assertIsNone(library["category"]) + self.assertTrue(os.path.exists(self.output_dir / "image.png")) diff --git a/tests/test_inspection_response.py b/tests/test_inspection_response.py index c3f33551..6b8412be 100644 --- a/tests/test_inspection_response.py +++ b/tests/test_inspection_response.py @@ -1426,3 +1426,32 @@ class TestPerRecordBudgets(unittest.TestCase): self.assertEqual(metadata["record_formats"], {"workflow": "limited"}) self.assertEqual(metadata["workflow_graph"]["status"], "limited") self.assertEqual(metadata["prompt_graph"]["status"], "absent") + + +class TestInheritedCategory(unittest.TestCase): + """#332 MC4: one inherited-category rule for the chip, filter and History.""" + + def test_recipe_set_blank_and_absent(self): + from utils.inspection_response import inherited_category, library_metadata + + cases = [ + (("Night", "Recipes", True), ("Recipes", "pair")), + (("Night", " ", True), ("Night", "positive")), + (("Night", "ย \t", True), ("Night", "positive")), + (("Night", b" ", True), ("Night", "positive")), + (("Night", None, True), ("Night", "positive")), + (("Night", "Recipes", False), ("Night", "positive")), + ((" ", None, False), (" ", "positive")), + ((None, None, False), (None, "positive")), + ] + for args, expected in cases: + with self.subTest(args=args): + self.assertEqual(inherited_category(*args), expected) + positive, recipe, has_recipe = args + metadata = library_metadata( + positive, ["t"], (recipe, []) if has_recipe else None + ) + self.assertEqual( + (metadata["category"], metadata["category_scope"]), expected + ) + self.assertEqual(metadata["tags_scope"], "positive") diff --git a/tests/test_legacy_contract_snapshot.py b/tests/test_legacy_contract_snapshot.py new file mode 100644 index 00000000..1ed7b7a0 --- /dev/null +++ b/tests/test_legacy_contract_snapshot.py @@ -0,0 +1,136 @@ +"""The legacy pages' backend contract does not move under System S13 (#330). + +S13 points the legacy admin and gallery buttons at native System and removes +their duplicate dialogs, changing no backend file. This snapshot pins what the +legacy pages talk to: every registered route (method and path) and the key +paths of ``GET /prompt_manager/settings``, the one route whose legacy request +body S13 shrinks. Record again only with a reason in the PR body: + + PM_RECORD_LEGACY_SNAPSHOT=1 python3 -m unittest tests.test_legacy_contract_snapshot + +Cutover (#334) removes this test and its fixture. +""" + +import json +import os +import unittest +from pathlib import Path + +try: + from tests.test_api_admin import AdminAPITestCase +except ImportError: # discovered with tests/ as the top-level directory + from test_api_admin import AdminAPITestCase + +from py.config import GalleryConfig # noqa: E402 + +SNAPSHOT = ( + Path(__file__).resolve().parent / "fixtures" / "legacy_contract_snapshot.json" +) +RECORD_ENV = "PM_RECORD_LEGACY_SNAPSHOT" +SETTINGS = "/prompt_manager/settings" +DEPTH = 2 +CHANGED = ( + "the legacy contract changed; if the change is intended (a new route, a new" + " settings key), record it with PM_RECORD_LEGACY_SNAPSHOT=1 and say why in" + " the PR body" +) + + +def key_paths(value, prefix="", depth=1): + """The key paths of a JSON object to ``DEPTH`` levels. + + ``{"a": {"b": 1}}`` gives ``a`` and ``a.b``; the items of a list of + objects give the union of their keys as ``a[].b``. + """ + paths = set() + for key, child in value.items(): + path = f"{prefix}{key}" + paths.add(path) + if depth >= DEPTH: + continue + if isinstance(child, dict): + paths |= key_paths(child, f"{path}.", depth + 1) + elif isinstance(child, list): + for item in child: + if isinstance(item, dict): + paths |= key_paths(item, f"{path}[].", depth + 1) + return paths + + +class TestLegacyContractSnapshot(AdminAPITestCase): + """Routes and settings keys against tests/fixtures/legacy_contract_snapshot.json. + + The base case isolates config.json (``PROMPT_MANAGER_CONFIG_PATH`` in the + temp tree), stubs ComfyUI's ``folder_paths`` and resets the gallery roots; + nothing here reads the maintainer's install. + """ + + async def asyncSetUp(self): + await super().asyncSetUp() + self.api._get_all_output_dirs = lambda: [str(self.output_dir)] + GalleryConfig.MONITORING_DIRECTORIES = [str(self.output_dir)] + + def _routes(self): + return sorted( + { + f"{route.method} {route.resource.canonical}" + for route in self.app.router.routes() + if route.method != "HEAD" + } + ) + + async def _settings_keys(self): + resp = await self.client.request("GET", SETTINGS) + body = await resp.json() + self.assertEqual(resp.status, 200, body) + return sorted(key_paths(body)) + + def _snapshot(self): + with open(SNAPSHOT, encoding="utf-8") as f: + return json.load(f) + + def _record(self, key, value): + recorded = self._snapshot() if SNAPSHOT.exists() else {} + recorded[key] = value + with open(SNAPSHOT, "w", encoding="utf-8", newline="\n") as f: + json.dump(recorded, f, indent=2) + f.write("\n") + + async def test_routes_match_the_snapshot(self): + routes = self._routes() + if os.environ.get(RECORD_ENV) == "1": + self._record("routes", routes) + self.assertGreater(len(routes), 50) + self.assertEqual(routes, self._snapshot()["routes"], CHANGED) + + async def test_settings_response_keeps_its_keys(self): + keys = await self._settings_keys() + if os.environ.get(RECORD_ENV) == "1": + self._record("settings_keys", keys) + self.assertIn("settings.infinite_scroll", keys) + self.assertEqual(keys, self._snapshot()["settings_keys"], CHANGED) + + +class TestKeyPaths(unittest.TestCase): + def test_depth_and_list_items(self): + value = { + "success": True, + "settings": {"a": 1, "deep": {"hidden": 1}}, + "rows": [{"path": "x"}, {"label": "y"}], + } + self.assertEqual( + sorted(key_paths(value)), + [ + "rows", + "rows[].label", + "rows[].path", + "settings", + "settings.a", + "settings.deep", + "success", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pair_usage_tracking.py b/tests/test_pair_usage_tracking.py index 9039f29d..f451cad2 100644 --- a/tests/test_pair_usage_tracking.py +++ b/tests/test_pair_usage_tracking.py @@ -85,6 +85,23 @@ class TestPairUsageTracking(unittest.TestCase): self.db.get_prompt_by_id(pair["negative_prompt_id"])["run_count"], 0 ) + def test_execution_records_a_cached_run_receipt_for_both_editors(self): + from utils.cached_runs import Facts + + node = self.node() + node.cached_runs = Mock() + node.cached_runs.replayed.return_value = False + node.encode_prompt( + Mock(), + "cat", + negative_text="", + prompt=pair_graph(negative=""), + unique_id="1", + ) + node.cached_runs.record_item.assert_called_once_with( + "1", ("cat", ""), Facts(usage_tracking.generate_prompt_hash("cat"), None) + ) + def test_validation_rejection_never_counts_or_saves(self): self.submit(pair_graph()) self.assertEqual(self.db.list_prompt_pairs()["total"], 0) diff --git a/tests/test_prompt_usage.py b/tests/test_prompt_usage.py index c4606641..c1dd27f1 100644 --- a/tests/test_prompt_usage.py +++ b/tests/test_prompt_usage.py @@ -278,5 +278,49 @@ class TestNodeRecordsUse(UsageTestCase): self.assertEqual(self.db.get_prompt_by_id(pid)["run_count"], 2) +class TestNodeRecordsCachedRunReceipt(UsageTestCase): + """Each execution leaves a receipt for cached jobs (#347).""" + + GRAPH = { + "1": {"class_type": "PromptManagerText", "inputs": {"text": "typed text"}}, + } + + def _node(self): + from unittest import mock + + from prompt_manager_text import PromptManagerText + + node = PromptManagerText.__new__(PromptManagerText) + node.db = self.db + node.logger = __import__("logging").getLogger("test.prompt_usage") + node.prompt_tracker = mock.Mock() + node.comfyui_integration = mock.Mock() + node._inject_lora_trigger_words = lambda text: text + node.cached_runs = mock.Mock() + node.cached_runs.replayed.return_value = False + return node + + def test_receipt_holds_the_returned_text_and_the_stored_hash(self): + from utils.cached_runs import Facts + + node = self._node() + node.process_text( + "typed text", prepend_text="pre", prompt=self.GRAPH, unique_id="1" + ) + node.cached_runs.record_item.assert_called_once_with( + "1", ("pre typed text",), Facts(generate_prompt_hash("typed text")) + ) + # Hook-visible text: the node counts nothing itself + row = self.db.get_prompt_by_hash(generate_prompt_hash("typed text")) + self.assertEqual(row["run_count"], 0) + + def test_empty_editor_records_empty_facts(self): + from utils.cached_runs import Facts + + node = self._node() + node.process_text(" ", prompt=self.GRAPH, unique_id="1") + node.cached_runs.record_item.assert_called_once_with("1", (" ",), Facts(None)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_recorded_saver_anchor.py b/tests/test_recorded_saver_anchor.py new file mode 100644 index 00000000..3ca852c6 --- /dev/null +++ b/tests/test_recorded_saver_anchor.py @@ -0,0 +1,379 @@ +"""Recorded run history names the saver that wrote a file (#386, #240). + +When a file's graph alone cannot name its prompt (sampler branches that differ +only in their prompt and share one saver prefix), the inspector asks recorded +history which save node wrote the file and walks from that node, as the #381 +file-name anchor does. A result the file alone resolves is never changed; a +recorded saver that names nothing certain leaves the file "several possible +prompts". The resolved sides are labelled as coming from the recorded run. +""" + +import json +import unittest + +from utils import prompt_pair_resolver as resolver +from utils.prompt_inspector import inspect_prompt_evidence + +try: + from tests import saver_anchor_graphs as shapes +except ImportError: # discovered with ``-s tests`` + import saver_anchor_graphs as shapes + + +def _resolve(graph, **options): + return resolver.resolve_prompt_pair( + {"prompt": graph}, profile=resolver.INSPECTION_PROFILE, **options + ) + + +def _positive(result): + side = result["positive"] + return side["text"] if isinstance(side, dict) else None + + +def _evidence(graph, **options): + return inspect_prompt_evidence({"prompt": json.dumps(graph)}, **options) + + +# ``branch_bench``'s savers are node ``N3`` of branch N (10s based). +def _saver(index): + return str(index * 10 + 3) + + +def guider_dataset(): + """The dataset run's shape: SamplerCustomAdvanced branches on BasicGuider + (no negative), every branch saved with one prefix.""" + graph = { + "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "m"}}, + "2": {"class_type": "EmptyLatentImage", "inputs": {"width": 64}}, + "4": {"class_type": "RandomNoise", "inputs": {"noise_seed": 1}}, + "5": {"class_type": "KSamplerSelect", "inputs": {"sampler_name": "euler"}}, + "6": { + "class_type": "BasicScheduler", + "inputs": {"model": ["1", 0], "steps": 8, "scheduler": "simple"}, + }, + } + for index, text in enumerate(shapes.BRANCH_TEXTS, start=1): + base = index * 10 + graph[str(base)] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": text, "clip": ["1", 1]}, + } + graph[str(base + 4)] = { + "class_type": "BasicGuider", + "inputs": {"model": ["1", 0], "conditioning": [str(base), 0]}, + } + graph[str(base + 1)] = { + "class_type": "SamplerCustomAdvanced", + "inputs": { + "noise": ["4", 0], + "guider": [str(base + 4), 0], + "sampler": ["5", 0], + "sigmas": ["6", 0], + "latent_image": ["2", 0], + }, + } + graph[str(base + 2)] = { + "class_type": "VAEDecode", + "inputs": {"samples": [str(base + 1), 0], "vae": ["1", 2]}, + } + graph[str(base + 3)] = { + "class_type": "SaveImage", + "inputs": { + "filename_prefix": "dataset/IMG_GEN", + "images": [str(base + 2), 0], + }, + } + return graph + + +class TestRecordedSaverInResolver(unittest.TestCase): + def test_the_recorded_saver_picks_its_branch_of_a_shared_prefix(self): + for index, text in enumerate(shapes.BRANCH_TEXTS, start=1): + with self.subTest(branch=index): + result = _resolve( + shapes.shared_prefix(), + file_name="IMG_GEN_00371_.png", + recorded_saver_id=_saver(index), + ) + self.assertEqual(result["state"], "resolved") + self.assertEqual(_positive(result), text) + self.assertEqual(result["sampler_id"], str(index * 10 + 1)) + + def test_without_a_recorded_saver_the_shared_prefix_stays_several(self): + result = _resolve(shapes.shared_prefix(), file_name="IMG_GEN_00371_.png") + self.assertEqual(result["state"], "ambiguous") + + def test_a_recorded_saver_the_graph_lacks_anchors_nothing(self): + for node in ("missing", "3", "2"): + with self.subTest(node=node): + result = _resolve(shapes.shared_prefix(), recorded_saver_id=node) + self.assertEqual(result["state"], "ambiguous") + + def test_a_recorded_saver_of_an_unknown_class_anchors(self): + graph = shapes.branch_bench( + prefixes=["dataset/IMG_GEN"] * len(shapes.BRANCH_TEXTS), + saver="VendorSaveImage", + ) + result = _resolve(graph, recorded_saver_id=_saver(2)) + self.assertEqual(_positive(result), shapes.BRANCH_TEXTS[1]) + + def test_the_default_profile_ignores_a_recorded_saver(self): + result = resolver.resolve_prompt_pair( + {"prompt": shapes.shared_prefix()}, recorded_saver_id=_saver(1) + ) + self.assertEqual(result["state"], "ambiguous") + + +class TestRecordedSaverInInspector(unittest.TestCase): + def test_the_inspector_shows_the_recorded_branch_labelled_as_recorded(self): + evidence = _evidence( + shapes.shared_prefix(), + file_name="IMG_GEN_00371_.png", + recorded_saver_id=_saver(3), + ) + self.assertEqual(evidence["resolution"]["state"], "resolved") + positive = evidence["prompts"]["positive"] + self.assertEqual(positive["text"], shapes.BRANCH_TEXTS[2]) + self.assertEqual(positive["source"], "recorded-run") + self.assertEqual(evidence["prompts"]["negative"]["source"], "recorded-run") + + def test_a_guider_branch_without_a_negative_shows_its_positive(self): + self.assertEqual( + _evidence(guider_dataset(), file_name="IMG_GEN_00371_.png")["resolution"][ + "state" + ], + "ambiguous", + ) + evidence = _evidence( + guider_dataset(), file_name="IMG_GEN_00371_.png", recorded_saver_id="23" + ) + self.assertEqual( + evidence["prompts"]["positive"], + { + "status": "resolved", + "text": shapes.BRANCH_TEXTS[1], + "source": "recorded-run", + }, + ) + self.assertEqual(evidence["prompts"]["negative"]["status"], "absent") + + def test_a_file_name_naming_another_saver_contradicts_history(self): + # The bench file name names branch 2's saver; history says branch 4's. + graph = shapes.branch_bench() + for index in (1, 3, 4): + graph[_saver(index)]["inputs"]["filename_prefix"] = "bench/run/shared" + result = _resolve( + graph, file_name="s45_branch_2_00001_.png", recorded_saver_id=_saver(4) + ) + self.assertIs(result["recorded_anchor"], False) + evidence = _evidence( + shapes.shared_prefix(), + file_name="IMG_GEN_00001_.png", + recorded_saver_id=_saver(4), + ) + self.assertEqual( + evidence["prompts"]["positive"]["text"], shapes.BRANCH_TEXTS[3] + ) + + def test_a_file_that_resolves_alone_keeps_its_own_result(self): + # The file name names branch 2; a recorded saver never overrides it. + evidence = _evidence( + shapes.branch_bench(), + file_name="s45_branch_2_00001_.png", + recorded_saver_id=_saver(4), + ) + positive = evidence["prompts"]["positive"] + self.assertEqual(positive["text"], shapes.BRANCH_TEXTS[1]) + self.assertEqual(positive["source"], "graph-inferred") + + def test_a_recorded_saver_that_names_nothing_keeps_several_prompts(self): + evidence = _evidence(shapes.shared_prefix(), recorded_saver_id="missing") + self.assertEqual(evidence["resolution"]["state"], "ambiguous") + self.assertEqual(evidence["prompts"]["positive"]["status"], "ambiguous") + + def test_no_recorded_saver_is_unchanged(self): + self.assertEqual( + _evidence(shapes.shared_prefix(), recorded_saver_id=None), + _evidence(shapes.shared_prefix()), + ) + + +LLM_TEXT = "a red vintage car parked by the sea, golden hour" + + +def text_generate_run(): + """The maintainer's moss run: a language model writes PromptManager's text.""" + return { + "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "m"}}, + "2": {"class_type": "EmptyLatentImage", "inputs": {"width": 64}}, + "17": {"class_type": "PrimitiveStringMultiline", "inputs": {"value": "a car"}}, + "14": { + "class_type": "PrimitiveStringMultiline", + "inputs": {"value": "You are a prompt engineer."}, + }, + "11": { + "class_type": "TextGenerate", + "inputs": { + "prompt": ["17", 0], + "system_prompt": ["14", 0], + "clip": ["1", 1], + "max_length": 4096, + "sampling_mode.seed": 0, + }, + }, + "36": { + "class_type": "PromptManagerPair", + "inputs": { + "text": ["11", 0], + "negative_text": "", + "prepend_text": "", + "append_text": "", + "clip": ["1", 1], + }, + }, + "7": { + "class_type": "KSampler", + "inputs": { + "seed": 1, + "steps": 20, + "cfg": 1.0, + "sampler_name": "euler", + "scheduler": "simple", + "denoise": 1.0, + "model": ["1", 0], + "positive": ["36", 0], + "negative": ["36", 1], + "latent_image": ["2", 0], + }, + }, + "8": { + "class_type": "VAEDecode", + "inputs": {"samples": ["7", 0], "vae": ["1", 2]}, + }, + "9": { + "class_type": "SaveImage", + "inputs": {"filename_prefix": "moss", "images": ["8", 0]}, + }, + } + + +def two_generated_branches(): + """``text_generate_run`` plus a second generated branch saved by an unknown + saver and also previewed.""" + graph = text_generate_run() + graph["31"] = {**graph["11"], "inputs": {**graph["11"]["inputs"]}} + graph["37"] = { + "class_type": "PromptManagerPair", + "inputs": {**graph["36"]["inputs"], "text": ["31", 0]}, + } + graph["20"] = { + "class_type": "KSampler", + "inputs": { + **graph["7"]["inputs"], + "positive": ["37", 0], + "negative": ["37", 1], + }, + } + graph["21"] = { + "class_type": "VAEDecode", + "inputs": {"samples": ["20", 0], "vae": ["1", 2]}, + } + graph["22"] = { + "class_type": "Image Save", + "inputs": {"filename_prefix": "other", "images": ["21", 0]}, + } + graph["23"] = {"class_type": "PreviewImage", "inputs": {"images": ["21", 0]}} + return graph + + +class TestRecordedTexts(unittest.TestCase): + """The texts PromptManager recorded at execution, per sampler of the run.""" + + def test_a_run_time_text_shows_what_the_run_recorded(self): + alone = _evidence(text_generate_run(), file_name="moss_00026_.png") + self.assertEqual(alone["resolution"]["reason_code"], "runtime-value") + evidence = _evidence( + text_generate_run(), + file_name="moss_00026_.png", + recorded_saver_id="9", + recorded_texts={"7": {"positive": LLM_TEXT, "negative": ""}}, + ) + self.assertEqual( + evidence["prompts"], + { + "positive": { + "status": "resolved", + "text": LLM_TEXT, + "source": "execution", + }, + "negative": {"status": "resolved", "text": "", "source": "execution"}, + }, + ) + self.assertEqual(evidence["resolution"]["state"], "resolved") + self.assertEqual(evidence["resolution"]["sampler_id"], "7") + + def test_an_unrecorded_negative_stays_open(self): + evidence = _evidence( + text_generate_run(), + recorded_saver_id="9", + recorded_texts={"7": {"positive": LLM_TEXT, "negative": None}}, + ) + self.assertEqual(evidence["prompts"]["positive"]["text"], LLM_TEXT) + self.assertEqual(evidence["prompts"]["negative"]["status"], "unresolved") + + def test_texts_that_do_not_name_the_open_sampler_change_nothing(self): + alone = _evidence(text_generate_run()) + for texts in ( + {"8": {"positive": LLM_TEXT, "negative": ""}}, + {"7": {"positive": None, "negative": ""}}, + {"7": {"positive": " ", "negative": ""}}, + {"7": "not roles"}, + {}, + ): + with self.subTest(texts=texts): + self.assertEqual( + _evidence( + text_generate_run(), recorded_saver_id="9", recorded_texts=texts + ), + alone, + ) + + def test_texts_need_the_saver_history_recorded_to_lead_to_the_sampler(self): + texts = {"7": {"positive": LLM_TEXT, "negative": ""}} + alone = _evidence(text_generate_run()) + for saver in (None, "missing", "2"): + with self.subTest(saver=saver): + self.assertEqual( + _evidence( + text_generate_run(), + recorded_saver_id=saver, + recorded_texts=texts, + ), + alone, + ) + + def test_texts_follow_the_recorded_saver_not_another_sampler(self): + # Review: the texts are those of the sampler history's saver leads to. + graph = two_generated_branches() + texts = { + "7": {"positive": "STAGE X TEXT", "negative": ""}, + "20": {"positive": "STAGE Y TEXT", "negative": ""}, + } + for saver, text, sampler in ( + ("22", "STAGE Y TEXT", "20"), + ("9", "STAGE X TEXT", "7"), + ): + with self.subTest(saver=saver): + evidence = _evidence( + graph, + file_name="renamed_00001_.png", + recorded_saver_id=saver, + recorded_texts=texts, + ) + self.assertEqual(evidence["prompts"]["positive"]["text"], text) + self.assertEqual(evidence["resolution"]["sampler_id"], sampler) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_restore_shutdown.py b/tests/test_restore_shutdown.py new file mode 100644 index 00000000..2ebf787f --- /dev/null +++ b/tests/test_restore_shutdown.py @@ -0,0 +1,233 @@ +"""A restore cut off by a server stop waits for its worker thread (#360). + +A cancelled handler (or job) task cannot stop the executor thread that +stages or publishes. Its cleanup (the stage, the upload) and the registry's +``restore`` admission must outlive that thread, and the live library ends as +the old or the restored one, never partial, with the safety backup kept. + +Every positive wait is on an event. The one bounded wait (``SETTLE``) is a +negative check: the cancelled task must still be pending while its worker is +held. +""" + +import asyncio +import os +import shutil +import sqlite3 +import threading +from unittest.mock import patch + +from database import restore_stage +from database.models import PromptModel +from py.api import admin as admin_mod +from py.api.jobs import RESTORE_JOB_KIND + +try: + from tests.test_restore_plans import RestoreTestCase, _texts +except ImportError: # discovered with tests/ as the top-level directory + from test_restore_plans import RestoreTestCase, _texts + +WAIT = 60 # a ceiling only a hung test reaches +SETTLE = 0.3 # the negative check's bound: the old code's cleanup takes ms + + +class _Gate: + """Hold one blocking call on its worker thread until released.""" + + def __init__(self, real): + self.real = real + self.entered = threading.Event() + self.release = threading.Event() + self.done = threading.Event() + + def method(self): + """A function to patch onto the class (bound like the real one).""" + + def held(*args): + return self(*args) + + return held + + def __call__(self, *args): + self.entered.set() + self.release.wait(WAIT) + try: + return self.real(*args) + finally: + self.done.set() + + +class ShutdownRestoreCase(RestoreTestCase): + def gate(self, real): + gate = _Gate(real) + # A failed check never leaves the worker held until WAIT. + self._gates = getattr(self, "_gates", []) + [gate] + self.addCleanup(gate.release.set) + return gate + + def stages(self): + """The stage folders (the upload's prefix starts the same way).""" + return [ + name + for name in super().stages() + if os.path.isdir(os.path.join(self.tmpdir, name)) + ] + + def uploads(self): + """The upload files (not the -wal/-shm its readers may add).""" + return sorted( + name + for name in os.listdir(self.tmpdir) + if name.startswith(restore_stage.UPLOAD_PREFIX) and name.endswith(".tmp") + ) + + def fake_upload(self, texts): + """Make ``_receive_restore_upload`` hand over a prepared upload.""" + self.upload(texts, "source.db") + source = os.path.join(self.tmpdir, "uploads", "source.db") + target = os.path.join(self.tmpdir, restore_stage.UPLOAD_PREFIX + "x.tmp") + shutil.copyfile(source, target) + + async def receive(request, uploads): + uploads.append(target) + return target, None + + return patch.object(self.api, "_receive_restore_upload", receive) + + async def wait_thread(self, event): + loop = asyncio.get_running_loop() + self.assertTrue(await loop.run_in_executor(None, event.wait, WAIT)) + + async def assert_still_pending(self, task): + done, _ = await asyncio.wait({task}, timeout=SETTLE) + if done: + for gate in getattr(self, "_gates", ()): + gate.release.set() # let the worker end before teardown + self.fail("the cancelled task ended while its worker still ran") + + +class TestLegacyRestoreAtShutdown(ShutdownRestoreCase): + async def test_cancel_during_publish_waits_for_the_worker(self): + self.save("old") + old_db = self.api.db + gate = self.gate(PromptModel.publish_staged) + with ( + self.fake_upload(["restored"]), + patch.object(PromptModel, "publish_staged", gate.method()), + ): + task = asyncio.ensure_future(self.api.restore_database(None)) + await self.wait_thread(gate.entered) + self.assertEqual(len(self.stages()), 1) + task.cancel() + await self.assert_still_pending(task) + # The worker still has its stage and upload, and runs alone. + self.assertEqual(len(self.stages()), 1) + self.assertEqual(len(self.uploads()), 1) + self.assertIsNotNone(self.api._jobs.active(RESTORE_JOB_KIND)) + gate.release.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, WAIT) + self.assertTrue(gate.done.is_set()) + self.assertEqual(_texts(self.db_path), ["restored"]) + self.assertIsNot(self.api.db, old_db) # the restored library adopted + self.assertEqual(len(self.backups()), 1) # the safety backup is kept + self.assertEqual(self.stages(), []) + self.assertEqual(self.uploads(), []) + self.assertIsNone(self.api._jobs.active(RESTORE_JOB_KIND)) + + async def test_cancel_during_staging_waits_and_changes_nothing(self): + self.save("old") + gate = self.gate(PromptModel.stage_restore) + with ( + self.fake_upload(["restored"]), + patch.object(PromptModel, "stage_restore", gate.method()), + ): + task = asyncio.ensure_future(self.api.restore_database(None)) + await self.wait_thread(gate.entered) + task.cancel() + await self.assert_still_pending(task) + self.assertEqual(len(self.uploads()), 1) # still being read + self.assertIsNotNone(self.api._jobs.active(RESTORE_JOB_KIND)) + gate.release.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, WAIT) + self.assertEqual(_texts(self.db_path), ["old"]) + self.assertEqual(self.backups(), []) + self.assertEqual(self.stages(), []) # the finished stage is removed + self.assertEqual(self.uploads(), []) + self.assertIsNone(self.api._jobs.active(RESTORE_JOB_KIND)) + + async def test_a_publish_that_fails_after_the_cancel_still_cancels(self): + self.save("old") + + def refuse(model, staged): + raise sqlite3.OperationalError("refused") + + gate = self.gate(refuse) + with ( + self.fake_upload(["restored"]), + patch.object(PromptModel, "publish_staged", gate.method()), + ): + task = asyncio.ensure_future(self.api.restore_database(None)) + await self.wait_thread(gate.entered) + task.cancel() + await self.assert_still_pending(task) + gate.release.set() + # Not answered as a 500: the cancel goes on. + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, WAIT) + self.assertEqual(_texts(self.db_path), ["old"]) + self.assertEqual(self.stages(), []) + self.assertEqual(self.uploads(), []) + self.assertIsNone(self.api._jobs.active(RESTORE_JOB_KIND)) + + async def test_a_second_cancel_still_adopts_the_restored_library(self): + self.save("old") + old_db = self.api.db + publish = self.gate(PromptModel.publish_staged) + prune = self.gate(admin_mod._prune_safety_backups) + with ( + self.fake_upload(["restored"]), + patch.object(PromptModel, "publish_staged", publish.method()), + patch.object(admin_mod, "_prune_safety_backups", prune), + ): + task = asyncio.ensure_future(self.api.restore_database(None)) + await self.wait_thread(publish.entered) + task.cancel() + await self.assert_still_pending(task) + publish.release.set() + await self.wait_thread(prune.entered) # adopting the library + task.cancel() # the loop's own shutdown cancels again + await self.assert_still_pending(task) + prune.release.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, WAIT) + self.assertEqual(_texts(self.db_path), ["restored"]) + self.assertIsNot(self.api.db, old_db) + self.assertEqual(len(self.backups()), 1) + self.assertEqual(self.stages(), []) + self.assertIsNone(self.api._jobs.active(RESTORE_JOB_KIND)) + + +class TestReviewedRestoreAtShutdown(ShutdownRestoreCase): + async def test_cancelled_job_waits_for_the_publish(self): + self.save("old") + old_db = self.api.db + body = await self.preview(self.upload(["restored"])) + gate = self.gate(PromptModel.publish_staged) + with patch.object(PromptModel, "publish_staged", gate.method()): + started = await self.apply(body["token"]) + job = self.api._jobs.get(started["job"]["id"]) + await self.wait_thread(gate.entered) + job.task.cancel() + await self.assert_still_pending(job.task) + self.assertEqual(len(self.stages()), 1) + self.assertTrue(job.running) # its exclusive admission holds + gate.release.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(job.task, WAIT) + self.assertEqual(_texts(self.db_path), ["restored"]) + self.assertIsNot(self.api.db, old_db) + self.assertEqual(len(self.backups()), 1) + self.assertEqual(self.stages(), []) + self.assertFalse(job.running) diff --git a/tests/test_review_inspection.py b/tests/test_review_inspection.py index bec27c2a..ea0774a1 100644 --- a/tests/test_review_inspection.py +++ b/tests/test_review_inspection.py @@ -298,19 +298,21 @@ class TestQwenEditBatchIsQueued(InspectionTestCase): class TestDefaultProfileStillDecides(InspectionTestCase): async def test_ambiguous_file_with_texts_keeps_the_default_profile(self): - """A default-ambiguous file with a choice never runs the inspection - profile, even when that profile would offer more candidates.""" + """A default-ambiguous file with a choice is recorded from the + default profile, even when the inspection profile would offer more + candidates. That profile only checks the saver anchor (#390): no + saver names this file, so it anchors nothing.""" self._roots() graph = { **_prefixed(qwen_edit_graph("make it night"), "q"), **unlinkable_graph("cat", "dog"), } - self._write(graph=graph) + self._write("unsaved_00001_.png", graph=graph) with patch.object( review_capture, - "resolve_inspection", - side_effect=AssertionError("inspection profile ran"), + "inspection_choice", + side_effect=AssertionError("inspection choices were asked"), ): await self._scan() @@ -321,17 +323,24 @@ class TestDefaultProfileStillDecides(InspectionTestCase): self.assertEqual(item["resolver_version"], review_capture.RESOLVER_VERSION) self.assertEqual(_texts(evidence), ["cat", "dog"]) - async def test_apply_of_a_default_item_never_runs_the_inspection_profile(self): + async def test_apply_of_a_default_item_inspects_only_for_the_anchor( + self, + ): self._roots() write_png(self.output_dir / "two.png", prompt=unlinkable_graph("cat", "dog")) await self._scan() - with patch.object( - review_capture, - "resolve_inspection", - side_effect=AssertionError("inspection profile ran"), + with ( + _counted(review_capture, "resolve_inspection") as calls, + patch.object( + review_capture, + "inspection_choice", + side_effect=AssertionError("inspection choices were asked"), + ), ): _item, (status, data) = await self._apply("cat") self.assertEqual(status, 200, data) + # The saver anchor check (#390): with the file's name, nothing matches. + self.assertEqual([c[1] for c in calls], ["two.png"]) self.assertEqual(self._linked_text(self.output_dir / "two.png"), "cat") @@ -369,15 +378,16 @@ class TestApplyInspectionCandidate(InspectionTestCase): calls = [] real = review_capture.resolve_inspection - def spy(metadata): - calls.append(metadata) - return real(metadata) + def spy(metadata, file_name=None): + calls.append((metadata, file_name)) + return real(metadata, file_name) with patch.object(review_capture, "resolve_inspection", side_effect=spy): _item, (status, data) = await self._apply("make it night") self.assertEqual(status, 200, data) self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][1], NAME) # anchored on its saver (#390) async def test_table_change_keeps_the_item_applicable(self): """A table change for a class the graph uses moves the item's @@ -853,8 +863,8 @@ class TestInspectionResolverVersion(InspectionTestCase): version = _version() real = review_capture.resolve_inspection - def newer(metadata): - resolution = real(metadata) + def newer(metadata, file_name=None): + resolution = real(metadata, file_name) return { **resolution, "candidates": [ @@ -899,10 +909,10 @@ class TestScanContainment(InspectionTestCase): self._write("b.png", graph=qwen_edit_graph("snow", "rain")) real = review_capture.resolve_inspection - def fails_for_a(metadata): + def fails_for_a(metadata, file_name=None): if "make it night" in json.dumps(metadata): raise RuntimeError("inspection blew up") - return real(metadata) + return real(metadata, file_name) with ( patch.object(review_capture, "resolve_inspection", side_effect=fails_for_a), diff --git a/tests/test_review_saver_anchor.py b/tests/test_review_saver_anchor.py new file mode 100644 index 00000000..c61d8112 --- /dev/null +++ b/tests/test_review_saver_anchor.py @@ -0,0 +1,333 @@ +"""The scan and Needs review anchor a file on the saver that wrote it (#390). + +The workspace inspector (since #383) passes the file's name to the resolver, +which anchors the file on the one saver whose ``filename_prefix`` names it +(``_GraphResolver.saver_stages``). The scan and Needs review use the same +rule. Maintainer ruling: a file never falls through both the link and the +queue, so when only that anchor resolves a file the default profile and the +parser floor do not link, it is queued with the anchor's pair as its one +choice (never linked automatically). No match, several matches or an +unfinished walk anchor nothing, so such a file is queued as before. Apply +re-resolves with the file's name, so a choice the anchor contradicts is +refused as stale evidence. +""" + +import json +from unittest.mock import patch + +try: + from tests.test_review import unlinkable_graph, write_png + from tests.test_review_inspection import ( + PROMPTS, + _counted, + InspectionTestCase, + qwen_edit_graph, + ) +except ImportError: # discovered with tests/ as the top-level directory + from test_review import unlinkable_graph, write_png + from test_review_inspection import ( + PROMPTS, + _counted, + InspectionTestCase, + qwen_edit_graph, + ) + +from utils import review_capture + + +def branch_graph(*prompts, prefixes=None): + """A Qwen-edit batch whose every SaveImage has its own literal prefix.""" + graph = qwen_edit_graph(*prompts) + savers = [n for n in graph.values() if n["class_type"] == "SaveImage"] + names = prefixes or [f"branch_{index}" for index in range(len(savers))] + for saver, prefix in zip(savers, names): + saver["inputs"]["filename_prefix"] = prefix + return graph + + +def saved_unlinkable_graph(*positives): + """``unlinkable_graph`` whose branches save under their own prefix. + + The default profile finds it ambiguous with a choice and the scan's + browser-parser floor links nothing, so it is queued from the default + profile unless the saver that wrote the file anchors its branch. + """ + graph = unlinkable_graph(*positives) + samplers = [ + node_id + for node_id, node in graph.items() + if node["class_type"] == "KSampler" and node_id != "3" + ] + for index, sampler in enumerate(samplers): + graph[f"{sampler}1"] = { + "class_type": "VAEDecode", + "inputs": {"samples": [sampler, 0], "vae": ["1", 2]}, + } + graph[f"{sampler}2"] = { + "class_type": "SaveImage", + "inputs": { + "images": [f"{sampler}1", 0], + "filename_prefix": f"side_{index}", + }, + } + return graph + + +def _without_name(real): + """``resolve_inspection`` as it was before #390: the file name ignored.""" + + def unnamed(metadata, file_name=None): + return real(metadata) + + return unnamed + + +def _only_candidate(test, item): + """The positive text of an anchored item's one candidate.""" + evidence = json.loads(item["evidence"]) + test.assertEqual(evidence.get("anchor"), "saver") + test.assertEqual(evidence.get("profile"), "inspection") + (candidate,) = evidence["candidates"] + return candidate["positive"]["text"] + + +class TestInspectionItemsAnchor(InspectionTestCase): + async def test_file_its_saver_names_is_queued_with_the_anchor_choice(self): + self._roots() + path = self._write("branch_1_00001_.png", graph=branch_graph(*PROMPTS)) + + job = await self._scan() + + (item,) = self._items() + self.assertEqual(_only_candidate(self, item), "add a red hat") + self.assertEqual(item["candidate_count"], 1) + self.assertEqual( + item["resolver_version"], + review_capture.inspection_review_version( + {"prompt": branch_graph(*PROMPTS)} + ), + ) + (row,) = job["outcomes"]["review"] + self.assertEqual(row["path"], "branch_1_00001_.png") + # Queued, never linked: ingestion stays on the default profile. + self.assertIsNone(self._linked_text(path)) + + async def test_anchored_item_stays_on_rescan(self): + self._roots() + self._write("branch_1_00001_.png", graph=branch_graph(*PROMPTS)) + await self._scan() + (item,) = self._items() + + job = await self._scan("all") + + self.assertEqual([i["id"] for i in self._items()], [item["id"]]) + self.assertEqual(job["scope"]["review_untraced"], 0) + + async def test_file_no_saver_names_stays_queued(self): + self._roots() + self._write("elsewhere_00001_.png", graph=branch_graph(*PROMPTS)) + + await self._scan() + + (item,) = self._items() + self.assertEqual(item["candidate_count"], len(PROMPTS)) + self.assertNotIn("anchor", json.loads(item["evidence"])) + + async def test_file_several_savers_name_stays_queued(self): + self._roots() + graph = branch_graph(*PROMPTS, prefixes=["same", "same", "other"]) + self._write("same_00001_.png", graph=graph) + + await self._scan() + + (item,) = self._items() + self.assertEqual(item["candidate_count"], len(PROMPTS)) + + async def test_old_rule_item_is_replaced_by_the_anchor_choice(self): + self._roots() + self._write("branch_1_00001_.png", graph=branch_graph(*PROMPTS)) + real = review_capture.resolve_inspection + with ( + patch.object(review_capture, "REVIEW_RULE_VERSION", 0), + patch.object( + review_capture, "resolve_inspection", side_effect=_without_name(real) + ), + ): + await self._scan() + (old,) = self._items() # queued under the old rule with every branch + self.assertEqual(old["candidate_count"], len(PROMPTS)) + + await self._scan("all") + + (item,) = self._items() + self.assertNotEqual(item["id"], old["id"]) + self.assertEqual(_only_candidate(self, item), "add a red hat") + + async def test_apply_links_the_anchor_choice(self): + self._require_routes() + self._roots() + path = self._write("branch_1_00001_.png", graph=branch_graph(*PROMPTS)) + await self._scan() + + _item, (status, data) = await self._apply("add a red hat") + + self.assertEqual(status, 200, data) + self.assertEqual(self._linked_text(path), "add a red hat") + + async def test_apply_refuses_an_old_choice_the_anchor_contradicts(self): + self._require_routes() + self._roots() + path = self._write("branch_1_00001_.png", graph=branch_graph(*PROMPTS)) + real = review_capture.resolve_inspection + with ( + patch.object(review_capture, "REVIEW_RULE_VERSION", 0), + patch.object( + review_capture, "resolve_inspection", side_effect=_without_name(real) + ), + ): + await self._scan() + item, detail, observed = await self._open() + # branch_1 is the saver of "add a red hat"; choose another branch. + wrong = self._candidate(detail, "make it night") + + status, data = await self._decide( + item["id"], self._body(detail, observed, wrong["hash"]) + ) + + self.assertEqual(status, 409, data) + self.assertEqual(data["code"], "review-conflict", data) + self.assertIsNone(self._linked_text(path)) + + +class TestDefaultItemsAnchor(InspectionTestCase): + async def test_anchor_resolves_and_nothing_links_so_it_is_queued(self): + """Maintainer ruling: never unlinked and unqueued. The anchor resolves + the file, the default profile and the parser floor link nothing, so + it is queued with the anchor's pair as its choice.""" + self._roots() + path = self.output_dir / "side_0_00001_.png" + graph = saved_unlinkable_graph("cat", "dog") + write_png(path, prompt=graph) + self.assertEqual( + review_capture.resolve_prompt_pair({"prompt": graph})["state"], "ambiguous" + ) + + job = await self._scan() + + (item,) = self._items() + self.assertEqual(_only_candidate(self, item), "cat") + (row,) = job["outcomes"]["review"] + self.assertIn("more than one prompt could match", row["detail"]) + self.assertIsNone(self._linked_text(path)) + + async def test_default_ambiguous_file_no_saver_names_stays_queued(self): + self._roots() + write_png( + self.output_dir / "unsaved_00001_.png", + prompt=saved_unlinkable_graph("cat", "dog"), + ) + + await self._scan() + + (item,) = self._items() + evidence = json.loads(item["evidence"]) + self.assertNotIn("profile", evidence) + self.assertNotIn("anchor", evidence) + self.assertEqual(item["resolver_version"], review_capture.RESOLVER_VERSION) + + async def test_file_the_parser_floor_links_is_not_queued(self): + """Already linked by the default path: unchanged by the anchor.""" + self._roots() + graph = saved_unlinkable_graph("cat", "dog") + del graph["3"] # the parser floor now reads a real sampler + path = self.output_dir / "side_0_00001_.png" + write_png(path, prompt=graph) + + await self._scan() + + self.assertEqual(self._items(), []) + self.assertIsNotNone(self._linked_text(path)) + + async def test_apply_refuses_an_old_default_choice_the_anchor_contradicts(self): + self._require_routes() + self._roots() + path = self.output_dir / "side_0_00001_.png" + write_png(path, prompt=saved_unlinkable_graph("cat", "dog")) + with patch.object(review_capture, "_captured_inspection", return_value=None): + await self._scan() # queued from the default profile, as before #390 + item, detail, observed = await self._open() + # side_0 is the saver of "cat"; choose the other branch. + wrong = self._candidate(detail, "dog") + + status, data = await self._decide( + item["id"], self._body(detail, observed, wrong["hash"]) + ) + + self.assertEqual(status, 409, data) + self.assertEqual(data["code"], "review-conflict", data) + self.assertIsNone(self._linked_text(path)) + + async def test_old_default_item_is_replaced_by_the_anchor_choice(self): + self._roots() + write_png( + self.output_dir / "side_0_00001_.png", + prompt=saved_unlinkable_graph("cat", "dog"), + ) + with patch.object(review_capture, "_captured_inspection", return_value=None): + await self._scan() # queued from the default profile, as before #390 + (old,) = self._items() + self.assertEqual(old["resolver_version"], review_capture.RESOLVER_VERSION) + + await self._scan("all") + + (item,) = self._items() + self.assertNotEqual(item["id"], old["id"]) + self.assertEqual(_only_candidate(self, item), "cat") + + with _counted(review_capture, "resolve_inspection") as calls: + await self._scan("all") # now queued under its version: no re-read + self.assertEqual(calls, []) + self.assertEqual([i["id"] for i in self._items()], [item["id"]]) + + async def test_decided_anchored_item_is_kept(self): + self._require_routes() + self._roots() + write_png( + self.output_dir / "side_0_00001_.png", + prompt=saved_unlinkable_graph("cat", "dog"), + ) + await self._scan() + item, detail, observed = await self._open() + status, data = await self._decide( + item["id"], self._body(detail, observed, "unresolved") + ) + self.assertEqual(status, 200, data) + before = (self._items(), self._decisions()) + + await self._scan("all") + + self.assertEqual((self._items(), self._decisions()), before) + + async def test_apply_accepts_an_old_default_choice_the_anchor_names(self): + self._require_routes() + self._roots() + path = self.output_dir / "side_0_00001_.png" + write_png(path, prompt=saved_unlinkable_graph("cat", "dog")) + with patch.object(review_capture, "_captured_inspection", return_value=None): + await self._scan() + + _item, (status, data) = await self._apply("cat") + + self.assertEqual(status, 200, data) + self.assertEqual(self._linked_text(path), "cat") + + +class TestSaverAnchored(InspectionTestCase): + def test_only_the_anchor_counts(self): + anchored = {"prompt": saved_unlinkable_graph("cat", "dog")} + self.assertTrue(review_capture.saver_anchored(anchored, "side_1_00001_.png")) + self.assertFalse(review_capture.saver_anchored(anchored, "nope_00001_.png")) + self.assertFalse(review_capture.saver_anchored(anchored, None)) + # One sampler resolves without any name: the anchor decides nothing. + single = {"prompt": branch_graph("make it night")} + self.assertFalse(review_capture.saver_anchored(single, "branch_0_00001_.png")) diff --git a/tests/test_run_attribution.py b/tests/test_run_attribution.py index ce3c60c3..050d710a 100644 --- a/tests/test_run_attribution.py +++ b/tests/test_run_attribution.py @@ -9,13 +9,22 @@ import unittest from pathlib import Path from unittest import mock -from database.run_attribution import stored_run_prompt +from database.run_attribution import ( + RecordedOutput, + recorded_output, + stored_run_prompt, +) from tests import test_execution_history_database as history_tests from tests.background_stops import close_handler from tests.test_api_admin import AdminAPITestCase -from tests.test_execution_context_database import snapshot +from tests.test_execution_context_database import known, snapshot from tests.test_prompt_attribution import issue_graph, llm_text_graph, make_tracker -from utils.execution_contexts import ContextBundle, RecipeContext +from utils.execution_contexts import ( + ContextBundle, + PromptSnapshot, + RecipeContext, + TextEvidence, +) from utils.execution_outputs import ( OutputBundle, OutputReference, @@ -38,28 +47,34 @@ class RecordedRuns: start = history_tests.HistoryDatabaseTests.start finish = history_tests.HistoryDatabaseTests.finish - def record_run(self, db, output_root, files, positives, item=0, window=None): + def record_run( + self, db, output_root, files, positives, item=0, window=None, nodes=None + ): started, finished = window or (now_ms() - 5000, now_ms() + 5000) start = self.start(db, item, started) db.write_execution_start(start) finish = self.finish(start, time=finished) db.write_execution_finish(finish) - db.write_execution_contexts( - start.run_id, - start.storage_epoch, - ContextBundle( - tuple( - RecipeContext(str(4 + i), snapshot(text), snapshot(""), "inferred") - for i, text in enumerate(positives) + if positives: # a run with no PromptManager node records no contexts + db.write_execution_contexts( + start.run_id, + start.storage_epoch, + ContextBundle( + tuple( + RecipeContext( + str(4 + i), snapshot(text), snapshot(""), "inferred" + ) + for i, text in enumerate(positives) + ), + "complete", + "captured", ), - "complete", - "captured", - ), - ) + ) root = snapshot_output_roots(STYLE, output_root, output_root).output_key + nodes = nodes or ["9"] * len(files) references = tuple( - OutputReference("9", i, "output", sub, name, STYLE, root) - for i, (sub, name) in enumerate(files) + OutputReference(node, i, "output", sub, name, STYLE, root) + for i, (node, (sub, name)) in enumerate(zip(nodes, files)) ) db.write_execution_outputs_for_finish( finish, OutputBundle(references, "complete", "reported") @@ -171,6 +186,186 @@ class TestStoredRunPrompt(RecordedRuns, unittest.TestCase): self.assertEqual(linked, [prompt_id]) +class TestRecordedOutputNode(RecordedRuns, unittest.TestCase): + """#386: the save node history says wrote a file, under the same rules.""" + + close_databases = history_tests.HistoryDatabaseTests.close_databases + database = history_tests.HistoryDatabaseTests.database + setUp = TestStoredRunPrompt.setUp + + def node(self, path=None): + found = recorded_output(self.db, path or self.image, self.output) + return found.saver_id if found is not None else None + + def test_one_run_one_saver_names_the_saver(self): + other = os.path.join(self.output, "sub", "img_00002_.png") + Path(other).write_bytes(b"png") + self.record_run( + self.db, + self.output, + [("sub", "img_00001_.png"), ("sub", "img_00002_.png")], + [], + nodes=["13", "23"], + ) + self.assertEqual(self.node(), "13") + self.assertEqual(self.node(other), "23") + + def test_a_run_without_prompt_manager_contexts_still_names_the_saver(self): + self.record_run(self.db, self.output, [("sub", "img_00001_.png")], []) + self.assertEqual(self.node(), "9") + + def test_two_runs_reporting_the_file_name_nothing(self): + self.record_run( + self.db, self.output, [("sub", "img_00001_.png")], [], nodes=["13"] + ) + self.record_run( + self.db, self.output, [("sub", "img_00001_.png")], [], item=1, nodes=["23"] + ) + self.assertIsNone(self.node()) + + def test_one_run_reporting_the_file_from_two_savers_names_nothing(self): + self.record_run( + self.db, + self.output, + [("sub", "img_00001_.png"), ("sub", "img_00001_.png")], + [], + nodes=["13", "23"], + ) + self.assertIsNone(self.node()) + + def test_a_file_written_outside_the_run_names_nothing(self): + self.record_run( + self.db, + self.output, + [("sub", "img_00001_.png")], + [], + window=(1_000_000, 2_000_000), + ) + self.assertIsNone(self.node()) + + def test_unrecorded_outside_or_unknown_roots_name_nothing(self): + self.record_run(self.db, self.output, [("sub", "img_00001_.png")], []) + other = os.path.join(self.output, "sub", "other.png") + Path(other).write_bytes(b"png") + self.assertIsNone(self.node(other)) + self.assertIsNone(recorded_output(self.db, self.image, None)) + self.assertIsNone(recorded_output(self.db, "", self.output)) + + def test_the_run_texts_recorded_at_execution_come_with_the_saver(self): + self.record_run( + self.db, self.output, [("sub", "img_00001_.png")], ["an llm prompt"] + ) + self.assertEqual( + recorded_output(self.db, self.image, self.output), + RecordedOutput("9", {"4": {"positive": "an llm prompt", "negative": ""}}), + ) + + def test_texts_only_inferred_from_the_graph_are_not_recorded(self): + start = self.start(self.db, 0, now_ms() - 5000) + self.db.write_execution_start(start) + finish = self.finish(start, time=now_ms() + 5000) + self.db.write_execution_finish(finish) + inferred = TextEvidence("graph text", "inferred", "graph_literal", None) + self.db.write_execution_contexts( + start.run_id, + start.storage_epoch, + ContextBundle( + ( + RecipeContext( + "7", + PromptSnapshot(inferred, inferred, known(""), known("")), + snapshot("kept"), + "inferred", + ), + ), + "complete", + "captured", + ), + ) + root = snapshot_output_roots(STYLE, self.output, self.output).output_key + self.db.write_execution_outputs_for_finish( + finish, + OutputBundle( + ( + OutputReference( + "9", 0, "output", "sub", "img_00001_.png", STYLE, root + ), + ), + "complete", + "reported", + ), + ) + self.assertEqual( + recorded_output(self.db, self.image, self.output).texts, + {"7": {"positive": None, "negative": "kept"}}, + ) + + def test_several_savers_name_no_saver_but_keep_the_texts(self): + self.record_run( + self.db, + self.output, + [("sub", "img_00001_.png"), ("sub", "img_00001_.png")], + ["text"], + nodes=["13", "23"], + ) + found = recorded_output(self.db, self.image, self.output) + self.assertIsNone(found.saver_id) + self.assertEqual(found.texts, {"4": {"positive": "text", "negative": ""}}) + + def test_the_route_lookup_logs_a_failure_and_names_nothing(self): + from database.run_attribution import recorded_output_lookup + + broken = mock.Mock() + broken.model.get_connection.return_value.execute.side_effect = ( + sqlite3.OperationalError("locked") + ) + broken.get_execution_contexts.side_effect = sqlite3.OperationalError("x") + with mock.patch( + "database.run_attribution.host_output_directory", return_value=self.output + ): + self.assertIsNone(recorded_output_lookup(broken)(self.image)) + with ( + mock.patch( + "database.run_attribution._recorded_runs", + side_effect=sqlite3.OperationalError("locked"), + ), + self.assertLogs("prompt_manager.run_attribution", "WARNING"), + ): + self.assertIsNone( + recorded_output_lookup(broken, close=True)(self.image) + ) + broken.model.close.assert_called_once_with() + + def test_a_history_reference_spells_the_file_as_history_recorded_it(self): + # On Windows the history route's target is normcased (lower case); + # history matches names exactly, so the route passes its reference. + from database.run_attribution import recorded_output_lookup + + upper = os.path.join(self.output, "sub", "IMG_00001_.png") + Path(upper).write_bytes(b"png") + self.record_run(self.db, self.output, [("sub", "IMG_00001_.png")], []) + target = os.path.join(self.output, "sub", "img_00001_.png") + reference = { + "folder_type": "output", + "subfolder": "sub", + "filename": "IMG_00001_.png", + } + with mock.patch( + "database.run_attribution.host_output_directory", return_value=self.output + ): + found = recorded_output_lookup(self.db, reference=reference)(target) + self.assertEqual(found.saver_id, "9") + temp = {**reference, "folder_type": "temp"} + self.assertIsNone(recorded_output_lookup(self.db, reference=temp)(target)) + + def test_a_library_without_history_tables_names_nothing(self): + broken = mock.Mock() + broken.model.get_connection.return_value.execute.side_effect = ( + sqlite3.OperationalError("no such table") + ) + self.assertIsNone(recorded_output(broken, self.image, self.output)) + + class TestScanUsesHistory(RecordedRuns, AdminAPITestCase): """The output scan applies the same rules as the monitor.""" diff --git a/tests/test_thumbnail_auto.py b/tests/test_thumbnail_auto.py index 71697131..18c92db1 100644 --- a/tests/test_thumbnail_auto.py +++ b/tests/test_thumbnail_auto.py @@ -978,6 +978,27 @@ class TestThumbnailOnCapture(AutoThumbnailCase): (self.output_dir / "thumbnails" / f"{name}_thumb.webp").is_file() ) + async def test_a_worker_refused_at_interpreter_exit_skips_quietly(self): + """#375: refused once the interpreter exits; nothing stays queued.""" + + class Refused(threading.Thread): + def start(self): + raise RuntimeError("can't create new thread at interpreter shutdown") + + namespace = type(sys)("threading_copy") + namespace.__dict__.update(vars(threading)) + namespace.Thread = Refused + thumbnailer = self.api._auto_thumbnails() + path = make_png(self.output_dir / "late.png", size=(32, 32)) + with ( + patch.object(auto, "threading", namespace), + self.assertNoLogs("prompt_manager.api", "INFO"), + ): + self.assertEqual(self.api.queue_capture_thumbnail(str(path)), "skipped") + self.assertFalse(thumbnailer._worker_running) + self.assertEqual(len(thumbnailer._queue), 0) + self.assertTrue(thumbnailer.wait_idle(0)) + def test_queue_is_bounded(self): thumbnailer = auto.AutoThumbnailer( lambda *a: {"action": "generated"}, diff --git a/tests/test_utils_logging_config.py b/tests/test_utils_logging_config.py index 3b07911e..d64ee51e 100644 --- a/tests/test_utils_logging_config.py +++ b/tests/test_utils_logging_config.py @@ -709,7 +709,8 @@ class TestRotatedCopyRule(IsolatedLoggerTestCase): self.assertNotEqual(link["size"], os.path.getsize(target)) for row in rows.values(): self.assertEqual( - set(row), {"filename", "path", "size", "modified", "is_main", "kind"} + set(row), + {"filename", "path", "size", "modified", "is_main", "kind", "identity"}, ) def test_an_unreadable_timestamp_does_not_break_the_scan(self): @@ -763,6 +764,217 @@ class TestRotatedCopyRule(IsolatedLoggerTestCase): self.manager.log_file_entry(name) +class TestClearOnlyWhatWasConfirmed(IsolatedLoggerTestCase): + """#389: Clear all logs empties or deletes no more than its confirmation showed.""" + + def setUp(self): + super().setUp() + self.logger.info("active log line") # the active log exists + self.outside = os.path.join(self.tmp.name, "outside") + os.makedirs(self.outside) + + def _path(self, name): + return os.path.join(self.log_dir, name) + + def _write(self, name, data=b"rotated\n"): + with open(self._path(name), "wb") as handle: + handle.write(data) + + def _read(self, path): + with open(path, "rb") as handle: + return handle.read() + + def _snapshot(self): + return {name: self._read(self._path(name)) for name in os.listdir(self.log_dir)} + + def _assert_unchanged(self, before, growing="prompt_manager.log"): + """Nothing deleted or emptied (the refusal itself logs to ``growing``).""" + after = self._snapshot() + self.assertEqual(sorted(after), sorted(before)) + for name, data in before.items(): + if name == growing: + self.assertTrue(after[name].startswith(data), name) + else: + self.assertEqual(after[name], data, name) + + def _confirmed(self): + """What the workspace dialog showed: the listed active and rotated rows.""" + return [ + {"name": row["filename"], "identity": row["identity"]} + for row in self.manager.get_log_files() + if row["kind"] in ("active", "rotated") + ] + + def _file_handler(self): + return next( + h + for h in self.manager.logger.handlers + if isinstance(h, logging.FileHandler) + ) + + def test_listed_identity_is_a_string_that_survives_json_numbers(self): + # mtime_ns (~1.7e18) and many inodes exceed 2**53: as JSON numbers a + # browser would round them and every confirmed clear would be refused. + self._write("prompt_manager.log.1") + for row in self.manager.get_log_files(): + with self.subTest(name=row["filename"]): + self.assertIsInstance(row["identity"], str) + self.assertRegex(row["identity"], r"^\d+:-?\d+:\d+$") + + def test_a_copy_dated_before_1970_can_be_confirmed(self): + self._write("prompt_manager.log.1") + try: + os.utime(self._path("prompt_manager.log.1"), ns=(0, -(10**9))) + except (OSError, OverflowError, ValueError) as error: + self.skipTest(f"this filesystem cannot store a time before 1970: {error}") + confirmed = self._confirmed() + if not any(":-" in row["identity"] for row in confirmed): + self.skipTest("the filesystem did not keep the negative time") + result = self.manager.truncate_logs(confirmed) + self.assertIn("prompt_manager.log.1", result["truncated"]) + + def test_a_rollover_after_the_confirmation_deletes_nothing(self): + self._write("prompt_manager.log.1", b"one" * 10) + self._write("prompt_manager.log.2", b"two" * 10) + confirmed = self._confirmed() + self.logger.info("written after the confirmation") + self._file_handler().doRollover() # .2 -> .3, .1 -> .2, log -> .1 + before = self._snapshot() + with self.assertRaises(logging_config.ClearRefused) as caught: + self.manager.truncate_logs(confirmed) + self.assertEqual(caught.exception.code, "logs-changed") + self._assert_unchanged(before) + + def test_a_copy_that_appeared_after_the_confirmation_is_kept(self): + self._write("prompt_manager.log.1", b"one") + confirmed = self._confirmed() + self._write("prompt_manager.log.5", b"not shown") + result = self.manager.truncate_logs(confirmed) + self.assertEqual(result["errors"], []) + self.assertEqual( + sorted(result["truncated"]), ["prompt_manager.log", "prompt_manager.log.1"] + ) + self.assertEqual( + sorted(os.listdir(self.log_dir)), + ["prompt_manager.log", "prompt_manager.log.5"], + ) + + def _clear(self, confirmed=None): + """Run a clear and return what it raised (None), so the files are + asserted first: on the old code the test fails on lost bytes.""" + try: + if confirmed is None: + self.manager.truncate_logs() # the legacy page's call + else: + self.manager.truncate_logs(confirmed) + except Exception as error: # noqa: BLE001 - asserted below + return error + return None + + def test_a_foreign_append_to_the_active_log_is_refused(self): + # A second ComfyUI sharing the folder writes the same active log. + self._write("prompt_manager.log.1") + listings = (None, "listed") + for listing in listings: + with self.subTest(listed=listing is not None): + confirmed = self._confirmed() if listing else None + with open(self._path("prompt_manager.log"), "ab") as other: + other.write(b"line from another process\n") + before = self._snapshot() + error = self._clear(confirmed) + self._assert_unchanged(before) + self.assertEqual(getattr(error, "code", None), "logs-foreign-writer") + self.logger.info("our own line") # our stream catches up again + + @unittest.skipIf(os.name == "nt", "Windows cannot rename a log that is open") + def test_a_foreign_rotation_of_our_log_is_refused(self): + # Another process rolled the shared log over: our handler still writes + # the file now named .1, and a new active log is the other's. + os.rename(self._path("prompt_manager.log"), self._path("prompt_manager.log.1")) + self._write("prompt_manager.log", b"the other process's log\n") + before = self._snapshot() + with self.assertRaises(logging_config.ClearRefused) as caught: + self.manager.truncate_logs() + self.assertEqual(caught.exception.code, "logs-foreign-writer") + # Our handler still writes the renamed file, so the refusal lands there. + self._assert_unchanged(before, growing="prompt_manager.log.1") + + def test_a_copy_that_changes_mid_clear_stops_the_clear(self): + # The identity is checked again just before each unlink: another + # process rotating while we delete stops the clear there. + for n in (1, 2): + self._write(f"prompt_manager.log.{n}", b"r" * n) + confirmed = self._confirmed() + real_unlink = os.unlink + calls = [] + + def unlink_then_rotate(path, *args, **kwargs): + real_unlink(path, *args, **kwargs) + if not calls: + with open(self._path("prompt_manager.log.2"), "ab") as handle: + handle.write(b"rotated in by another process") + calls.append(os.path.basename(path)) + + active_before = self._read(self._path("prompt_manager.log")) + with mock.patch.object(logging_config.os, "unlink", unlink_then_rotate): + result = self.manager.truncate_logs(confirmed) + self.assertEqual(calls, ["prompt_manager.log.1"]) + self.assertEqual(result["truncated"], ["prompt_manager.log.1"]) + self.assertEqual(len(result["errors"]), 1) + self.assertIn("prompt_manager.log.2", result["errors"][0]) + self.assertTrue(os.path.exists(self._path("prompt_manager.log.2"))) + # The active log is emptied last, so it is kept too. + self.assertTrue( + self._read(self._path("prompt_manager.log")).startswith(active_before) + ) + + def _require_hard_link(self, source, link): + try: + os.link(source, link) + except (OSError, NotImplementedError, AttributeError) as error: + self.skipTest(f"this filesystem cannot make hard links: {error}") + + def test_a_hard_linked_active_log_is_never_emptied(self): + other_name = os.path.join(self.outside, "kept-by-the-user.log") + self._require_hard_link(self._path("prompt_manager.log"), other_name) + kept = self._read(other_name) + self.assertTrue(kept) + for listed in (False, True): + with self.subTest(listed=listed): + error = self._clear(self._confirmed() if listed else None) + self.assertTrue(self._read(other_name).startswith(kept)) + self.assertEqual(getattr(error, "code", None), "logs-hard-link") + + def test_a_hard_linked_active_log_with_file_logging_off_is_kept(self): + self.manager.update_config({"file_logging": False}) + other_name = os.path.join(self.outside, "kept-by-the-user.log") + self._require_hard_link(self._path("prompt_manager.log"), other_name) + kept = self._read(other_name) + error = self._clear() + self.assertEqual(self._read(other_name), kept) + self.assertEqual(getattr(error, "code", None), "logs-hard-link") + + def test_a_malformed_confirmation_is_refused_before_anything_changes(self): + self._write("prompt_manager.log.1") + before = self._snapshot() + for expected in ( + "prompt_manager.log.1", + [{"name": "prompt_manager.log.1"}], + [{"name": "prompt_manager.log.1", "identity": 7}], + [{"name": "prompt_manager.log.1", "identity": "1:2"}], + [{"name": 1, "identity": "1:2:3"}], + [ + {"name": "prompt_manager.log.1", "identity": "1:2:3"}, + {"name": "prompt_manager.log.1", "identity": "1:2:3"}, + ], + ["prompt_manager.log.1"], + ): + with self.subTest(expected=expected): + with self.assertRaises(ValueError): + self.manager.truncate_logs(expected) + self._assert_unchanged(before) + + class TestConfiguration(IsolatedLoggerTestCase): def test_in_flight_record_cannot_reopen_retired_file_handler(self) -> None: handler = next( diff --git a/tests/test_vocabulary.py b/tests/test_vocabulary.py index d8252eb6..928619be 100644 --- a/tests/test_vocabulary.py +++ b/tests/test_vocabulary.py @@ -13,6 +13,11 @@ from database.library_commands import LibraryCommandError from database.operations import PromptDatabase from utils.hashing import generate_prompt_hash +try: + from test_image_categories import category_checksum +except ImportError: + from tests.test_image_categories import category_checksum + class TestVocabularyReads(unittest.TestCase): def setUp(self): @@ -168,7 +173,8 @@ class TestVocabularyReads(unittest.TestCase): self.assertEqual(page["total"], 4) self.assertFalse(page["has_more"]) for row in page["categories"]: - self.assertEqual(set(row), {"value", "prompt_count"}) + self.assertEqual(set(row), {"value", "prompt_count", "image_count"}) + self.assertEqual(row["image_count"], 0) self.assertEqual( row["prompt_count"], self.db.list_library(category=row["value"])["total"], @@ -238,6 +244,111 @@ class TestVocabularyReads(unittest.TestCase): with self.assertRaises(ValueError): self.read("list_vocabulary_tags", **options) + def own(self, value, *paths, root="root"): + """Give each relative path under ``root`` the own category ``value``.""" + self.db.set_image_categories([(root, path) for path in paths], value) + + def all_categories(self, limit=200): + rows, pages, after = [], [], None + while True: + page = self.read("list_vocabulary_categories", limit=limit, after=after) + pages.append(page) + rows.extend(page["categories"]) + if not page["has_more"]: + return rows, pages + after = page["next_key"] + + def test_category_list_includes_image_only_values_last(self): + self.save("Oldest", category="Zeta") + self.save("Newer", category="Alpha") + self.own("Zeta", "a.png", "b.png") + self.own("Only images B", "c.png") + self.own("Only images A", "d.png", "e.png", "gone.png") + rows, _pages = self.all_categories() + self.assertEqual( + rows, + [ + {"value": "Zeta", "prompt_count": 1, "image_count": 2}, + {"value": "Alpha", "prompt_count": 1, "image_count": 0}, + {"value": "Only images A", "prompt_count": 0, "image_count": 3}, + {"value": "Only images B", "prompt_count": 0, "image_count": 1}, + ], + ) + page = self.read("list_vocabulary_categories") + self.assertEqual(page["total"], 4) + + def test_category_cursor_crosses_into_image_only_values(self): + for index in range(3): + self.save(f"Owner {index}", category=f"P{index}") + self.own("P1", "shared.png") + self.own("I-b", "x.png") + self.own("I-a", "y.png") + self.own("I-c", "z.png", root="other") + for limit in (1, 2, 3, 4): + with self.subTest(limit=limit): + rows, pages = self.all_categories(limit=limit) + self.assertEqual( + [row["value"] for row in rows], + ["P0", "P1", "P2", "I-a", "I-b", "I-c"], + ) + self.assertEqual({page["total"] for page in pages}, {6}) + self.assertEqual( + [row["image_count"] for row in rows], [0, 1, 0, 1, 1, 1] + ) + first = self.read("list_vocabulary_categories", limit=3) + self.assertEqual(first["next_key"], [3, "P2"]) + second = self.read("list_vocabulary_categories", limit=1, after=[3, "P2"]) + self.assertEqual(second["next_key"], [2**63 - 1, "I-a"]) + + def test_image_rows_on_a_prompt_value_count_once(self): + self.save("Owner", category="Shared") + self.db.save_prompt_pair("Owner", "Noise", category="Shared") + self.own("Shared", *[f"{n}.png" for n in range(5)]) + self.own("Shared", "elsewhere.png", root="second") + rows, _pages = self.all_categories() + self.assertEqual( + rows, [{"value": "Shared", "prompt_count": 1, "image_count": 6}] + ) + + def test_library_options_ignore_image_categories(self): + self.save("Owner", category="Prompt value") + before = self.db.get_library_options() + self.own("Image value", "a.png") + self.own("Prompt value", "b.png") + self.assertEqual(self.db.get_library_options(), before) + self.assertEqual(before["categories"], ["Prompt value"]) + + def test_category_reads_use_the_image_category_index(self): + from database import vocabulary + + self.own("Value", "a.png") + statements = [ + vocabulary._CATEGORY_ROWS + "SELECT COUNT(*) FROM counted", + vocabulary._CATEGORY_ROWS + + "SELECT value,first_pid,prompt_count,image_count FROM counted " + "WHERE first_pid>? OR (first_pid=? AND value>?) " + "ORDER BY first_pid,value LIMIT ?", + ] + plans = [] + with self.db.model.get_connection() as conn: + for sql in statements: + params = [1] * sql.count("?") + plans.extend( + row[-1] for row in conn.execute("EXPLAIN QUERY PLAN " + sql, params) + ) + for sql in ( + vocabulary._IMAGE_CATEGORY_ROWIDS, + *vocabulary._CATEGORY_UPDATES, + ): + params = [1] * sql.count("?") + plans.extend( + row[-1] for row in conn.execute("EXPLAIN QUERY PLAN " + sql, params) + ) + image_steps = [step for step in plans if "image_categories" in step] + self.assertTrue(image_steps) + for step in image_steps: + self.assertIn("idx_image_categories_category", step, step) + def test_reads_share_one_snapshot_and_report_epoch(self): self.save("Cat", category="Animal", tags=["cat"]) epoch = self.db.get_execution_epoch() @@ -741,6 +852,7 @@ class TestVocabularyCategoryCommands(unittest.TestCase): "other_prompts": 1, "negative_variants": 2, "archived_negative_variants": 1, + "images": 0, }, ) self.assertFalse(result["target_exists"]) @@ -810,6 +922,7 @@ class TestVocabularyCategoryCommands(unittest.TestCase): "other_prompts": 0, "negative_variants": 0, "archived_negative_variants": 0, + "images": 0, }, ) self.assertEqual(self.rows("prompts")[variant]["category"], "Other") @@ -902,6 +1015,171 @@ class TestVocabularyCategoryCommands(unittest.TestCase): self.apply(bad) self.assertEqual(self.snapshot(), before) + def own(self, value, *paths, root="root"): + self.db.set_image_categories([(root, path) for path in paths], value) + + def image_rows(self): + with self.db.model.get_connection() as conn: + return { + (row[0], row[1]): (row[2], row[3], row[4]) + for row in conn.execute( + "SELECT root_key,relative_path,category,updated_at,rowid " + "FROM image_categories ORDER BY rowid" + ) + } + + def checksum(self): + with self.db.model.get_connection() as conn: + return category_checksum(conn) + + def test_preview_counts_image_rows(self): + self.owners() + self.own("Scenery", "a.png", "b.png", "missing.png") + self.own("Scenery", "c.png", root="second") + self.own("Other", "d.png") + before = self.image_rows() + result = self.preview() + self.assertEqual(result["impact"]["images"], 4) + self.assertEqual(self.image_rows(), before) + self.assertTrue(self.preview(target="Other")["target_exists"]) + + def test_apply_renames_image_categories(self): + self.owners() + bystander = self.save("Bystander", category="Other") + self.own("Scenery", "a.png", "b.png") + self.own("Scenery", "c.png", root="second") + self.own(" Scenery ", "padded.png") + self.own("Other", "d.png") + self.age_rows() + with self.db.model.get_connection() as conn: + conn.execute("UPDATE image_categories SET updated_at=?", (_OLD_STAMP,)) + images_before = self.image_rows() + prompts_before, pairs_before = self.snapshot() + preview = self.preview() + receipt = self.apply(preview["impact_token"]) + self.assertEqual(receipt["impact"]["images"], 3) + images_after = self.image_rows() + self.assertEqual(set(images_after), set(images_before)) + renamed = {("root", "a.png"), ("root", "b.png"), ("second", "c.png")} + for key, (value, stamp, rowid) in images_after.items(): + old_value, old_stamp, old_rowid = images_before[key] + self.assertEqual(rowid, old_rowid, key) + if key in renamed: + self.assertEqual(value, "Landscape") + self.assertNotEqual(stamp, _OLD_STAMP) + else: + self.assertEqual((value, stamp), (old_value, old_stamp)) + prompts_after, pairs_after = self.snapshot() + self.assertEqual(prompts_after[bystander], prompts_before[bystander]) + for before, after in ( + (prompts_before, prompts_after), + (pairs_before, pairs_after), + ): + for row_id, row in after.items(): + if before[row_id]["category"] != "Scenery": + self.assertEqual(row, before[row_id]) + # Merging into a value images already hold keeps every file key. + merge = self.preview(source="Landscape", target="Other") + self.assertTrue(merge["target_exists"]) + self.assertEqual(merge["impact"]["images"], 3) + receipt = self.apply(merge["impact_token"], source="Landscape", target="Other") + self.assertTrue(receipt["consolidated"]) + merged = self.image_rows() + self.assertEqual(set(merged), set(images_before)) + self.assertEqual( + sorted(key for key, row in merged.items() if row[0] == "Other"), + sorted([*renamed, ("root", "d.png")]), + ) + + def test_image_only_value_can_be_renamed(self): + self.save("Bystander", category="Kept") + self.db.save_prompt_pair("Pair", "Noise", category="Kept") + self.own("Image only", "a.png", "b.png") + self.age_rows() + before = self.checksum() + preview = self.preview(source="Image only", target="Renamed") + self.assertEqual( + preview["impact"], + { + "positive_prompts": 0, + "other_prompts": 0, + "negative_variants": 0, + "archived_negative_variants": 0, + "images": 2, + }, + ) + self.assertFalse(preview["target_exists"]) + receipt = self.apply( + preview["impact_token"], source="Image only", target="Renamed" + ) + self.assertFalse(receipt["consolidated"]) + self.assertEqual({row[0] for row in self.image_rows().values()}, {"Renamed"}) + self.assertEqual(self.checksum(), before) + # An image-only target counts as existing. + self.own("Image target", "c.png") + self.assertTrue( + self.preview(source="Renamed", target="Image target")["target_exists"] + ) + # Nothing holds the value any more: refused. + with self.assertRaises(LibraryCommandError) as caught: + self.preview(source="Image only", target="Again") + self.assertEqual(caught.exception.code, "vocabulary-conflict") + + def test_token_refused_after_an_image_row_gains_or_loses_the_value(self): + self.owners() + self.own("Scenery", "a.png", "b.png") + + def gain(): + self.own("Scenery", "late.png") + + def lose(): + self.db.clear_image_categories([("root", "a.png")]) + + def move_away(): + self.own("Elsewhere", "b.png") + + def target_on_image(): + self.own("Landscape", "target.png") + + for mutate in (gain, lose, move_away, target_on_image): + with self.subTest(mutate.__name__): + token = self.preview()["impact_token"] + mutate() + before = self.snapshot(), self.image_rows() + with self.assertRaises(LibraryCommandError) as caught: + self.apply(token) + self.assertEqual(caught.exception.code, "vocabulary-conflict") + self.assertEqual((self.snapshot(), self.image_rows()), before) + + def test_image_category_failure_rolls_back_prompts(self): + self.owners() + self.own("Scenery", "a.png") + token = self.preview()["impact_token"] + with self.db.model.get_connection() as conn: + conn.execute( + "CREATE TRIGGER abort_images BEFORE UPDATE ON image_categories " + "BEGIN SELECT RAISE(ABORT, 'boom'); END" + ) + before = self.snapshot(), self.image_rows() + with self.assertRaises(sqlite3.Error): + self.apply(token) + self.assertEqual((self.snapshot(), self.image_rows()), before) + + def test_category_exists_reads_the_image_index(self): + from database import vocabulary + + with self.db.model.get_connection() as conn: + plan = [ + row[-1] + for row in conn.execute( + "EXPLAIN QUERY PLAN " + vocabulary._CATEGORY_EXISTS, {"v": "x"} + ) + ] + steps = [step for step in plan if "image_categories" in step] + self.assertTrue(steps, plan) + for step in steps: + self.assertIn("idx_image_categories_category", step) + def test_category_apply_failure_rolls_back(self): self.owners() token = self.preview()["impact_token"] @@ -942,6 +1220,18 @@ class TestSharedTokenHashing(unittest.TestCase): ), "cefe7f6a5544b7f6a203f5527810899c6af568aa7abc1b75adffbfb5ae6251ae", ) + with_images = category_impact_token( + "e", "S", "T", True, [(1, True)], [], images=[9, 4] + ) + self.assertRegex(with_images, r"\A[0-9a-f]{64}\Z") + self.assertEqual( + with_images, + category_impact_token("e", "S", "T", True, [(1, True)], [], images=[4, 9]), + ) + self.assertNotEqual( + with_images, + category_impact_token("e", "S", "T", True, [(1, True)], [], images=[4]), + ) if __name__ == "__main__": diff --git a/tests/test_workspace_system_fixture.py b/tests/test_workspace_system_fixture.py index a43ec854..23189560 100644 --- a/tests/test_workspace_system_fixture.py +++ b/tests/test_workspace_system_fixture.py @@ -1,12 +1,26 @@ -"""The System browser hooks only ever reset their own server's config.json (#249 S3).""" +"""The System browser hooks only ever reset their own server's config.json (#249 S3) +and decide JPEG XL availability whatever the host has installed (#371).""" +import asyncio import os +import sys import tempfile +import types import unittest from pathlib import Path from unittest import mock -from tests.browser.workspace_system_fixture import register_system_commands +from aiohttp import web +from aiohttp.test_utils import TestClient, TestServer + +_mock_server = mock.MagicMock() +_mock_server.PromptServer.instance.routes = mock.MagicMock() +sys.modules.setdefault("server", _mock_server) + +from py.api import thumbnail_encode # noqa: E402 +from tests.browser.workspace_system_fixture import ( # noqa: E402 + register_system_commands, +) class TestWorkspaceSystemFixture(unittest.TestCase): @@ -33,6 +47,48 @@ class TestWorkspaceSystemFixture(unittest.TestCase): register_system_commands(None, None, None, self.root) self.assertEqual(elsewhere.read_bytes(), b'{"real": true}') + def test_no_encoder_holds_even_when_the_host_has_one(self): + """``{available: false}`` survives the settings read's re-probe (#371).""" + config = self.root / "config.json" + env = {**os.environ, "PROMPT_MANAGER_CONFIG_PATH": str(config)} + api = types.SimpleNamespace() + app, routes = web.Application(), web.RouteTableDef() + real_detect = thumbnail_encode._detect + with mock.patch.dict(os.environ, env, clear=True): + reset = register_system_commands(app, routes, api, self.root) + app.add_routes(routes) + self.addCleanup(thumbnail_encode.reset_encoders) + self.addCleanup(setattr, thumbnail_encode, "_detect", real_detect) + + async def scenario(): + async with TestClient(TestServer(app)) as client: + + async def post(available): + response = await client.post( + "/__test__/system/jxl-encoder", json={"available": available} + ) + self.assertEqual(response.status, 200) + + await post(False) + status = thumbnail_encode.encoder_status()["jxl"] + self.assertFalse(status["available"]) + self.assertIsNone( + thumbnail_encode.detect_jxl_encoder(recheck_missing=True) + ) + await post(True) + status = thumbnail_encode.encoder_status()["jxl"] + self.assertEqual(status["encoder"], "cjxl") + await post(False) + + # The host's real detector finds an encoder (Pillow can save JPEG XL). + with mock.patch.object(thumbnail_encode, "_pillow_can_save_jxl", lambda: True): + asyncio.run(scenario()) + # Reset gives detection back to the host. + reset() + self.assertIs(thumbnail_encode._detect, real_detect) + status = thumbnail_encode.encoder_status()["jxl"] + self.assertEqual(status["encoder"], "pillow") + if __name__ == "__main__": unittest.main() diff --git a/tests/vendor/ci-workflow.test.mjs b/tests/vendor/ci-workflow.test.mjs index f489fd64..d8ed881a 100644 --- a/tests/vendor/ci-workflow.test.mjs +++ b/tests/vendor/ci-workflow.test.mjs @@ -67,6 +67,45 @@ const browserJobIds = [...allJobs].filter(([, body]) => /npm run --prefix tools\ const mergeJob = () => allJobs.get('frontend-coverage-merge'); const verifyStep = () => steps(mergeJob()).find(step => /name: Verify every shard's raw coverage arrived/.test(step)); +// Jobs that run only in the merge queue, on main and on demand (#420): skipped on pull_request. +const QUEUE_ONLY_IF = /^ {4}if: github\.event_name != 'pull_request'$/m; +const EVENTS = ['pull_request', 'merge_group', 'push', 'workflow_dispatch']; + +function gateVariables(jobText) { + return new Map([...jobText.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.([\w-]+)\.result \}\}$/gm)] + .map(match => [match[2], match[1]])); +} + +function runGate(jobText, env) { + const script = runScript(steps(jobText).find(step => /run: \|/.test(step))); + try { + execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' }); + return true; + } catch { + return false; + } +} + +// A gate passes only when every always-on job succeeded and every queue-only job is skipped on a +// pull request and succeeded on every other event. Anything else fails it. +function assertGateBehaviour(jobText, queueOnly) { + assert.match(jobText, /^ {10}EVENT_NAME: \$\{\{ github\.event_name \}\}$/m, 'the gate knows the event'); + const variables = gateVariables(jobText); + for (const id of queueOnly) assert.ok(variables.has(id), `${id} reaches the gate`); + for (const event of EVENTS) { + const expected = id => (queueOnly.includes(id) && event === 'pull_request' ? 'skipped' : 'success'); + const green = Object.fromEntries([...variables].map(([id, name]) => [name, expected(id)])); + assert.equal(runGate(jobText, { ...green, EVENT_NAME: event }), true, `${event}: the expected results pass`); + for (const [id, name] of variables) { + for (const result of ['success', 'failure', 'cancelled', 'skipped']) { + if (result === expected(id)) continue; + assert.equal(runGate(jobText, { ...green, EVENT_NAME: event, [name]: result }), false, + `${event}: ${id}=${result} must fail the gate`); + } + } + } +} + test('viewerjs keeps its name and needs every frontend job', () => { const viewerjs = allJobs.get('viewerjs'); assert.ok(viewerjs, 'the required context is the job id viewerjs'); @@ -76,32 +115,17 @@ test('viewerjs keeps its name and needs every frontend job', () => { assert.deepEqual([...needs].sort(), [...others].sort(), 'viewerjs needs every other job in the workflow'); assert.deepEqual([...needs].sort(), ['browser-workflows', 'frontend-coverage', 'frontend-coverage-merge']); assert.match(viewerjs, /^ {4}if: always\(\)$/m, 'viewerjs runs (and fails) when a needed job failed or was skipped'); - const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step))); - for (const id of needs) { - const variable = new RegExp(`^ {10}([A-Z_]+): \\$\\{\\{ needs\\.${id}\\.result \\}\\}$`, 'm').exec(viewerjs); - assert.ok(variable, `${id}'s result reaches the check`); - assert.match(script, new RegExp(`test "\\$${variable[1]}" = success`), `${id} must be exactly success`); - } + assert.deepEqual([...gateVariables(viewerjs).keys()].sort(), [...needs].sort(), 'every needed result reaches the check'); }); -test('viewerjs fails unless every needed job succeeded', () => { - const viewerjs = allJobs.get('viewerjs'); - const script = runScript(steps(viewerjs).find(step => /run: \|/.test(step))); - const variables = [...viewerjs.matchAll(/^ {10}([A-Z_]+): \$\{\{ needs\.[\w-]+\.result \}\}$/gm)].map(match => match[1]); - const run = env => { - try { - execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], { env: { PATH: process.env.PATH, ...env }, stdio: 'pipe' }); - return true; - } catch { - return false; - } - }; - const allSuccess = Object.fromEntries(variables.map(name => [name, 'success'])); - assert.equal(run(allSuccess), true); - for (const name of variables) { - for (const result of ['failure', 'cancelled', 'skipped']) { - assert.equal(run({ ...allSuccess, [name]: result }), false, `${name}=${result} must fail viewerjs`); - } +test('viewerjs requires the browser shards everywhere but on a pull request', () => { + assertGateBehaviour(allJobs.get('viewerjs'), ['browser-workflows']); +}); + +test('Firefox/WebKit page workflows run only outside pull requests; coverage runs on every event', () => { + assert.match(allJobs.get('browser-workflows'), QUEUE_ONLY_IF); + for (const id of ['frontend-coverage', 'frontend-coverage-merge']) { + assert.doesNotMatch(allJobs.get(id), /^ {4}if:/m, `${id} runs on every event`); } }); @@ -224,7 +248,8 @@ test('browser job timeouts keep a real margin over measured runs', () => { }); // Every job is bounded (#350): a hang ends at a measured limit instead of GitHub's 6-hour default. -const BOUNDED_WORKFLOWS = ['test.yml', 'code-quality.yml', 'viewerjs-check.yml']; +const GATING_WORKFLOWS = ['test.yml', 'code-quality.yml', 'viewerjs-check.yml']; +const BOUNDED_WORKFLOWS = [...GATING_WORKFLOWS, 'portability.yml']; const readWorkflow = name => readFileSync(new URL(`../../.github/workflows/${name}`, import.meta.url), 'utf8'); test('every job in the CI workflows has a numeric timeout-minutes', () => { @@ -247,3 +272,102 @@ test('the Python matrix and coverage run the suite through the watchdog runner', assert.doesNotMatch(found.get(id), /unittest discover/, `${id} no longer runs the bare unittest discover`); } }); + +// CI hygiene (#352): lint tools are pinned, the bandit hook can fail, the JS job caches npm. +const readRepoFile = path => readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8'); + +test('black and flake8 are pinned exactly and the lint job installs those pins', () => { + const requirements = readRepoFile('requirements-dev.txt'); + assert.match(requirements, /^black==\d+\.\d+\.\d+$/m, 'black is pinned exactly'); + assert.match(requirements, /^flake8==\d+\.\d+\.\d+$/m, 'flake8 is pinned exactly'); + const lint = jobs(readWorkflow('test.yml')).get('lint'); + assert.match(lint, /pip install \$\(grep -E '\^\(black\|flake8\)==' requirements-dev\.txt\)/); + assert.doesNotMatch(lint, /pip install black flake8/, 'lint no longer installs the latest releases'); +}); + +test('the frontend modules job caches npm keyed on the vendor lockfile', () => { + const setupNode = steps(jobs(readWorkflow('test.yml')).get('js-test')) + .find(step => /uses: actions\/setup-node@v\d+/.test(step)); + assert.match(setupNode, /^ {10}cache: npm$/m); + assert.match(setupNode, /^ {10}cache-dependency-path: tools\/vendor\/package-lock\.json$/m); +}); + +test('the bandit pre-commit hook fails at the severity CI enforces and passes every file', () => { + const quality = readWorkflow('code-quality.yml'); + assert.match(quality, /^ {8}run: bandit -r \. -lll -q --exclude /m, 'CI fails on high severity'); + const hook = /- id: bandit\n([\s\S]*?)(?=\n {6}(?:- id:|#)|\n*$)/.exec(readRepoFile('.pre-commit-config.yaml'))?.[1]; + assert.ok(hook, 'the bandit hook exists'); + assert.match(hook, /^ {8}entry: bandit -lll -q$/m, 'the hook enforces the same -lll level as CI'); + assert.doesNotMatch(hook, /true|exit-zero|bash -c/, 'nothing masks the exit code or drops a file'); + assert.doesNotMatch(hook, /pass_filenames: false/, 'the hook scans the staged files'); +}); + +// Event split (#420): pull requests run a fast subset, the merge queue and main run everything. +const testJobs = () => jobs(readWorkflow('test.yml')); + +test('the Python matrix is Linux 3.12 on a pull request and Linux 3.12-3.14 on every other event', () => { + const body = testJobs().get('test'); + assert.deepEqual(matrixList(body, 'os'), ['ubuntu-latest'], 'Windows and macOS never gate a merge'); + const match = /^ {8}python-version: \$\{\{ github\.event_name == 'pull_request' && fromJSON\('(\[[^\]]*\])'\) \|\| fromJSON\('(\[[^\]]*\])'\) \}\}$/m.exec(body); + assert.ok(match, 'python-version is chosen by event'); + assert.deepEqual({ pr: JSON.parse(match[1]), other: JSON.parse(match[2]) }, { pr: ['3.12'], other: ['3.12', '3.13', '3.14'] }); + assert.match(body, /^ {6}fail-fast: false$/m); + assert.match(body, /^ {4}name: py\$\{\{ matrix\.python-version \}\} on \$\{\{ matrix\.os \}\}$/m); +}); + +test('Windows and macOS run 3.12-3.14 on main, nightly and on demand, never on a PR or in the queue', () => { + const text = readWorkflow('portability.yml'); + const triggers = text.slice(text.indexOf('\non:'), text.indexOf('\nconcurrency:')); + for (const event of ['push', 'schedule', 'workflow_dispatch']) assert.match(triggers, new RegExp(`^ {2}${event}:`, 'm')); + for (const event of ['pull_request', 'merge_group']) assert.doesNotMatch(triggers, new RegExp(`^ {2}${event}`, 'm')); + assert.match(triggers, /^ {4}branches: \[main\]$/m, 'push runs only for main'); + const body = jobs(text).get('test'); + assert.deepEqual(matrixList(body, 'os'), ['windows-latest', 'macos-latest']); + assert.deepEqual(matrixList(body, 'python-version'), ['"3.12"', '"3.13"', '"3.14"']); + assert.match(body, /^ {6}fail-fast: false$/m); + assert.match(body, /^ {8}run: python -m tests\.run_suite -v$/m); + assert.doesNotMatch(text, /cancel-in-progress: true|cancel-in-progress: \$/, 'no portability run is cancelled'); +}); + +test('real ComfyUI runs only outside pull requests; the other test.yml checks run on every event', () => { + const found = testJobs(); + assert.match(found.get('comfyui'), QUEUE_ONLY_IF); + for (const id of ['test', 'coverage', 'lint', 'js-test']) { + assert.doesNotMatch(found.get(id), /^ {4}if:/m, `${id} runs on every event`); + } +}); + +test('all tests needs every gating test.yml job and requires the full set outside pull requests', () => { + const found = testJobs(); + const gate = found.get('all-tests'); + assert.ok(gate, 'the aggregate gate exists'); + assert.match(gate, /^ {4}name: all tests$/m, 'the required context is "all tests"'); + assert.match(gate, /^ {4}if: always\(\)$/m); + const gating = [...found.keys()].filter(id => !['all-tests', 'perf-budget'].includes(id)); + assert.deepEqual([...needsOf(gate)].sort(), gating.sort(), 'the gate needs every job but the non-gating perf budget'); + assert.deepEqual([...gateVariables(gate).keys()].sort(), gating.sort(), 'every needed result reaches the gate'); + assertGateBehaviour(gate, ['comfyui']); +}); + +test('the CI workflows run on pull_request, merge_group and push without path filters', () => { + for (const name of GATING_WORKFLOWS) { + const text = readWorkflow(name); + const triggers = text.slice(text.indexOf('\non:'), text.indexOf('\n', text.indexOf('\non:') + 1) + 400); + for (const event of ['merge_group', 'pull_request', 'push']) { + assert.match(triggers, new RegExp(`^ {2}${event}:`, 'm'), `${name} runs on ${event}`); + } + assert.doesNotMatch(text, /^ {4}paths(-ignore)?:/m, `${name} has no path filter that would leave a required check pending`); + } + assert.doesNotMatch(jobs(readWorkflow('code-quality.yml')).get('security'), /^ {4}if:/m, 'bandit runs on every event'); +}); + +test('every required check reports on a pull request and the gates carry the full matrix', () => { + const contexts = JSON.parse(readRepoFile('.github/branch-protection.json')).required_status_checks.contexts; + assert.deepEqual([...contexts].sort(), [ + 'all tests', 'bandit (high severity)', 'black and flake8', 'coverage gate (85%)', 'frontend modules', + 'py3.12 on ubuntu-latest', 'real ComfyUI compatibility', 'viewerjs', + ].sort()); + assert.equal(contexts.filter(context => /^py/.test(context)).length, 1, + 'a dropped matrix leg never reports on a pull request, so it cannot be required'); + assert.ok(!contexts.some(context => /windows|macos/.test(context)), 'Windows and macOS never gate a merge'); +}); diff --git a/tests/vendor/comfyui-config.test.mjs b/tests/vendor/comfyui-config.test.mjs index c5b00677..2cb9587c 100644 --- a/tests/vendor/comfyui-config.test.mjs +++ b/tests/vendor/comfyui-config.test.mjs @@ -33,15 +33,17 @@ test('execution script discovers only the new matrix and preserves all 78 origin const executions = list('tools/vendor/playwright.execution.config.mjs'); assert.match(executions, /\[classic-start\]/); assert.match(executions, /\[nodes2-start\]/); - // + the fresh-host library location case, + #342 image attribution, + #343 linked text - assert.match(executions, /Total: 90 tests in 5 files/); + // + the fresh-host library location case, + #342 image attribution, + #343 linked text, + // + #332 image category routes on a generated output (image-attribution.spec.mjs, 2 projects), + // + #347 cached runtime-linked runs: five node kinds and eight one-renderer cases (execution-text-link.spec.mjs, 2 projects) + assert.match(executions, /Total: 118 tests in 5 files/); for (const project of ['classic-start', 'nodes2-start']) { const cases = executions.split('\n').filter(line => line.includes(`[${project}]`)); assert.equal(cases.filter(line => line.includes('execution-history.spec.mjs')).length, 15); assert.equal(cases.filter(line => line.includes('execution-outputs.spec.mjs')).length, 8); assert.equal(cases.filter(line => line.includes('library-location.spec.mjs')).length, 1); - assert.equal(cases.filter(line => line.includes('image-attribution.spec.mjs')).length, 2); - assert.equal(cases.filter(line => line.includes('execution-text-link.spec.mjs')).length, 19); + assert.equal(cases.filter(line => line.includes('image-attribution.spec.mjs')).length, 3); + assert.equal(cases.filter(line => line.includes('execution-text-link.spec.mjs')).length, 32); } assert.doesNotMatch(executions, /prompt-manager.spec/); }); diff --git a/tests/vendor/legacy-coverage.test.mjs b/tests/vendor/legacy-coverage.test.mjs index c1bf38d7..4ab74223 100644 --- a/tests/vendor/legacy-coverage.test.mjs +++ b/tests/vendor/legacy-coverage.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { COLUMNS, CONTROL_EXEMPTIONS, STATUSES, declaredTitles, parseControl, parseCoverageTable, readCoverageTable, +import { COLUMNS, CONTROL_EXEMPTIONS, PAGES, REMOVAL_COLUMNS, STATUSES, declaredTitles, duplicateRemovals, idUses, methodUses, + parseControl, parseCoverageTable, parseRemovalTable, readCoverageTable, readPage, readRemovalTable, removalProblems, rowProblems } from '../../tools/vendor/legacy-coverage.mjs'; // TESTING.md "Legacy System capability coverage" (#249 System S12, #329): every legacy System capability names an @@ -114,3 +115,99 @@ test('a covered row names its native control unless it is exempt', () => { assert.deepEqual([row.Status, row['Native control']], ['covered', 'โ€”']); } }); + +// TESTING.md "Legacy System UI removed by S13" (#330): a legacy element is removed only when its capability is +// `covered`, the removal copies that row's native test, and the element is really gone from its page. +const COVERAGE_HEADER = '| Capability | Legacy anchor | Owner | Status | Native control | Test |\n| --- | --- | --- | --- | --- | --- |'; +const REMOVAL_HEADER = '| Capability | Page | Removed ids | Removed methods | Native test |\n| --- | --- | --- | --- | --- |'; +const FIXTURE_COVERAGE = parseCoverageTable(['### Legacy System capability coverage', '', COVERAGE_HEADER, + '| View logs | `admin.html:1` | S12 Logs | covered | `diagnostics` ยท combobox ยท "Level" | level filter works (logs.spec.mjs) |', + '| Infinite scrolling default | `admin.html:2` | Legacy prompt list | kept in legacy | โ€” | scrolls (list-state.test.js) |', + ''].join('\n')); +const removalRow = cells => parseRemovalTable(['### Legacy System UI removed by S13', '', REMOVAL_HEADER, `| ${cells} |`, ''] + .join('\n'))[0]; +const EMPTY_PAGE = () => ({ html: '', script: 'class Admin {\n kept() {}\n}\n' }); + +test('every removed legacy element cites a covered capability and its native test', () => { + const coverage = readCoverageTable(); + const rows = readRemovalTable(); + for (const row of rows) { + const cited = coverage.find(item => item.Capability === row.Capability); + assert.ok(cited, `TESTING.md:${row.line} cites "${row.Capability}"`); + assert.equal(cited.Status, 'covered', `TESTING.md:${row.line} ${row.Capability}`); + assert.equal(row['Native test'], cited.Test, `TESTING.md:${row.line} ${row.Capability}`); + assert.ok(Object.keys(PAGES).includes(row.Page), `TESTING.md:${row.line} page ${row.Page}`); + } + assert.deepEqual(duplicateRemovals(rows), []); + assert.deepEqual(REMOVAL_COLUMNS, ['Capability', 'Page', 'Removed ids', 'Removed methods', 'Native test']); +}); + +test('a removed id or method is gone from its page', () => { + const coverage = readCoverageTable(); + for (const row of readRemovalTable()) { + assert.deepEqual(removalProblems(row, coverage, page => readPage(page)), [], `TESTING.md:${row.line} ${row.Capability}`); + } + // Each way a page can still hold an id or a method is found; a call through another object, or a longer name, is not + // a definition. + const page = { + html: '
', + script: [ + "document.getElementById('logLevel').value;", + "this.showModal('diagnosticsModal');", + '', + 'document.querySelector("#logLimit option");', + "const backdrops = ['settingsModal', 'duplicatesModal'];", + 'class Admin {', + ' async refreshLogs(level) {', + ' }', + '} displayLogs(logs) {', + ' }', + 'this.loadLogStats();', + '// downloadLogs() {', + 'other.formatBytesLater(1);', + ].join('\n'), + }; + assert.deepEqual(idUses('logsModal', page), ['id="logsModal" in the markup']); + assert.deepEqual(idUses('logLevel', page), ["getElementById('logLevel')"]); + assert.deepEqual(idUses('logLimit', page), ["querySelector('#logLimit')"]); + assert.deepEqual(idUses('duplicatesModal', page), ["'duplicatesModal' in a modal list"]); + assert.deepEqual(idUses('logLevelX', page), []); + assert.deepEqual(idUses('diagnosticsModal', page), ["a โ€ฆModal('diagnosticsModal') call"]); + assert.deepEqual(idUses('logsContainer', page), []); + assert.deepEqual(methodUses('closeLogs', page), ['.closeLogs( called in the markup']); + assert.deepEqual(methodUses('refreshLogs', page), ['refreshLogs() defined']); + assert.deepEqual(methodUses('displayLogs', page), ['displayLogs() defined']); + assert.deepEqual(methodUses('loadLogStats', page), ['.loadLogStats( called']); + assert.deepEqual(methodUses('downloadLogs', page), []); + assert.deepEqual(methodUses('formatBytes', page), []); + // One id or method is listed on one row per page. + const twice = [removalRow('View logs | admin | `#logsModal` | โ€” | level filter works (logs.spec.mjs)'), + { ...removalRow('View logs | admin | `#logsModal` | โ€” | level filter works (logs.spec.mjs)'), line: 9 }]; + assert.match(duplicateRemovals(twice).join(), /admin #logsModal is listed on rows/); + assert.deepEqual(removalProblems(removalRow('View logs | admin | `logsModal`, `#ok` | `close` | level filter works (logs.spec.mjs)'), + FIXTURE_COVERAGE, EMPTY_PAGE), ['malformed entry "logsModal"', 'malformed entry "close"']); +}); + +test('a removal citing a kept-in-legacy capability fails', () => { + const row = removalRow('Infinite scrolling default | admin | `#infiniteScrollDefault` | โ€” | scrolls (list-state.test.js)'); + assert.deepEqual(removalProblems(row, FIXTURE_COVERAGE, EMPTY_PAGE), ['cites a kept in legacy capability']); + const unknown = removalRow('Something else | admin | โ€” | โ€” | scrolls (list-state.test.js)'); + assert.deepEqual(removalProblems(unknown, FIXTURE_COVERAGE, EMPTY_PAGE), ['cites no single coverage row "Something else"']); +}); + +test('a removal whose native test differs fails', () => { + const row = removalRow('View logs | admin | `#logsModal` | `showLogsModal()` | another title (logs.spec.mjs)'); + assert.deepEqual(removalProblems(row, FIXTURE_COVERAGE, EMPTY_PAGE), + ['its native test differs from the coverage row\'s "level filter works (logs.spec.mjs)"']); + assert.deepEqual(removalProblems({ ...row, 'Native test': 'level filter works (logs.spec.mjs)' }, FIXTURE_COVERAGE, EMPTY_PAGE), []); +}); + +test('a removed id still on its page fails', () => { + const row = removalRow('View logs | admin | `#logsModal` | `showLogsModal()` | level filter works (logs.spec.mjs)'); + const page = () => ({ html: '', script: 'class Admin {\n showLogsModal() {\n }\n}\n' }); + assert.deepEqual(removalProblems(row, FIXTURE_COVERAGE, page), [ + '#logsModal is still on the admin page: id="logsModal" in the markup', + 'showLogsModal() is still on the admin page: showLogsModal() defined', + ]); + assert.deepEqual(removalProblems({ ...row, Page: 'metadata' }, FIXTURE_COVERAGE, page), ['unknown page "metadata"']); +}); diff --git a/tools/vendor/legacy-coverage.mjs b/tools/vendor/legacy-coverage.mjs index 46a807c7..4da4a74f 100644 --- a/tools/vendor/legacy-coverage.mjs +++ b/tools/vendor/legacy-coverage.mjs @@ -42,25 +42,30 @@ function cells(line) { return out; } -/** The table's rows as objects keyed by COLUMNS, from the section that starts at HEADING. */ -export function parseCoverageTable(markdown) { +/** The rows, keyed by `columns`, of the first Markdown table under `heading` (`label` names it in errors). */ +function parseTable(markdown, heading, columns, label) { const lines = markdown.split('\n'); - const start = lines.findIndex(line => line.trim() === HEADING); - if (start === -1) throw new Error(`TESTING.md has no "${HEADING}" section`); + const start = lines.findIndex(line => line.trim() === heading); + if (start === -1) throw new Error(`TESTING.md has no "${heading}" section`); const header = lines.findIndex((line, index) => index > start && line.trim().startsWith('|')); if (header === -1 || lines.slice(start + 1, header).some(line => /^#{1,3} /.test(line))) { - throw new Error('The coverage section has no table'); + throw new Error(`The ${label} section has no table`); } - const columns = cells(lines[header]); - if (columns.join('|') !== COLUMNS.join('|')) throw new Error(`The coverage table's columns are ${columns.join(', ')}`); + const found = cells(lines[header]); + if (found.join('|') !== columns.join('|')) throw new Error(`The ${label} table's columns are ${found.join(', ')}`); const rows = []; for (let index = header + 2; index < lines.length && lines[index].trim().startsWith('|'); index += 1) { const values = cells(lines[index]); - rows.push({ line: index + 1, ...Object.fromEntries(COLUMNS.map((column, at) => [column, values[at] ?? ''])) }); + rows.push({ line: index + 1, ...Object.fromEntries(columns.map((column, at) => [column, values[at] ?? ''])) }); } return rows; } +/** The table's rows as objects keyed by COLUMNS, from the section that starts at HEADING. */ +export function parseCoverageTable(markdown) { + return parseTable(markdown, HEADING, COLUMNS, 'coverage'); +} + /** * A Native control cell: `` `
` ยท ยท "" ``, optionally ` ยท in ""` when the name is shared * by several rows of the section โ†’ {section, role, name, row}; `โ€”` โ†’ null; else it throws. @@ -213,3 +218,130 @@ export function rowProblems(row, { root = ROOT, locate = file => testFile(file, export function readCoverageTable(root = ROOT) { return parseCoverageTable(readFileSync(join(root, 'TESTING.md'), 'utf8')); } + +// โ”€โ”€ Legacy System UI removed by S13 (#330) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// TESTING.md "Legacy System UI removed by S13": one row per covered capability whose legacy UI S13 removed, with the +// element ids and methods that went. A row must cite a `covered` coverage row and copy its Test cell, and none of its +// ids or methods may still be on its page. +export const REMOVAL_HEADING = '### Legacy System UI removed by S13'; +export const REMOVAL_COLUMNS = Object.freeze(['Capability', 'Page', 'Removed ids', 'Removed methods', 'Native test']); +/** Each legacy page's markup and script. */ +export const PAGES = Object.freeze({ + admin: Object.freeze({ html: 'web/admin.html', script: 'web/js/admin.js' }), + gallery: Object.freeze({ html: 'web/gallery.html', script: 'web/js/gallery.js' }), +}); + +/** The removal table's rows, from the section that starts at REMOVAL_HEADING. */ +export function parseRemovalTable(markdown) { + return parseTable(markdown, REMOVAL_HEADING, REMOVAL_COLUMNS, 'removal'); +} + +/** The removal table read from TESTING.md. */ +export function readRemovalTable(root = ROOT) { + return parseRemovalTable(readFileSync(join(root, 'TESTING.md'), 'utf8')); +} + +/** A Removed ids or Removed methods cell โ†’ its entries without backticks (`โ€”` โ†’ none). */ +export function listCell(cell) { + if (cell.trim() === NONE) return []; + return cell.split(',').map(entry => entry.trim().replace(/^`|`$/g, '')).filter(Boolean); +} + +/** The removed ids of a row (without `#`) and its malformed entries. */ +function removedIds(row) { + const entries = listCell(row['Removed ids']); + return { ids: entries.filter(entry => /^#[\w-]+$/.test(entry)).map(entry => entry.slice(1)), + bad: entries.filter(entry => !/^#[\w-]+$/.test(entry)) }; +} + +/** The removed methods of a row (without `()`) and its malformed entries. */ +function removedMethods(row) { + const entries = listCell(row['Removed methods']); + return { names: entries.filter(entry => /^[A-Za-z_$][\w$]*\(\)$/.test(entry)).map(entry => entry.slice(0, -2)), + bad: entries.filter(entry => !/^[A-Za-z_$][\w$]*\(\)$/.test(entry)) }; +} + +const escapeRegExp = text => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** + * Where `id` is still on a page: `id=""` in its markup or script; in its script a lookup, a `โ€ฆModal("")` call + * (`showModal`, `hideModal`) or a modal-list entry. + */ +export function idUses(id, { html, script }) { + const name = escapeRegExp(id); + const code = stripJsComments(script); + const uses = []; + // `id=`, not `data-id=`: the attribute starts after a space or a tag's `(` call (`this.`, `window.admin.`) + * in its script or markup. + */ +export function methodUses(name, { html, script }) { + const method = escapeRegExp(name); + const code = stripJsComments(script); + const uses = []; + if (new RegExp(`(?:^|[\\s;}])(?:async\\s+)?${method}\\s*\\([^)]*\\)\\s*\\{`, 'm').test(code)) uses.push(`${name}() defined`); + if (new RegExp(`\\.${method}\\s*\\(`).test(code)) uses.push(`.${name}( called`); + if (new RegExp(`\\.${method}\\s*\\(`).test(html)) uses.push(`.${name}( called in the markup`); + return uses; +} + +/** + * Every problem of one removal row (empty: sound). `coverage` is the coverage table's rows; `readPage(page)` returns + * that page's `{html, script}`. + */ +export function removalProblems(row, coverage, readPage) { + const problems = []; + const cited = coverage.filter(item => item.Capability === row.Capability); + if (cited.length !== 1) problems.push(`cites no single coverage row "${row.Capability}"`); + else if (cited[0].Status !== 'covered') problems.push(`cites a ${cited[0].Status} capability`); + else if (cited[0].Test !== row['Native test']) problems.push(`its native test differs from the coverage row's "${cited[0].Test}"`); + if (!Object.hasOwn(PAGES, row.Page)) { + problems.push(`unknown page "${row.Page}"`); + return problems; + } + const { ids, bad: badIds } = removedIds(row); + const { names, bad: badNames } = removedMethods(row); + for (const entry of [...badIds, ...badNames]) problems.push(`malformed entry "${entry}"`); + const page = readPage(row.Page); + for (const id of ids) for (const use of idUses(id, page)) problems.push(`#${id} is still on the ${row.Page} page: ${use}`); + for (const name of names) for (const use of methodUses(name, page)) problems.push(`${name}() is still on the ${row.Page} page: ${use}`); + return problems; +} + +/** Ids and methods listed on more than one row of the same page. */ +export function duplicateRemovals(rows) { + const seen = new Map(); + const duplicates = []; + for (const row of rows) { + for (const entry of [...listCell(row['Removed ids']), ...listCell(row['Removed methods'])]) { + const key = `${row.Page} ${entry}`; + if (seen.has(key)) duplicates.push(`${key} is listed on rows ${seen.get(key)} and ${row.line}`); + else seen.set(key, row.line); + } + } + return duplicates; +} + +/** A page's markup and script from the repository. */ +export function readPage(page, root = ROOT) { + const files = PAGES[page]; + return { html: readFileSync(join(root, files.html), 'utf8'), script: readFileSync(join(root, files.script), 'utf8') }; +} diff --git a/utils/cached_runs.py b/utils/cached_runs.py new file mode 100644 index 00000000..fd73cd2e --- /dev/null +++ b/utils/cached_runs.py @@ -0,0 +1,532 @@ +"""Count a cached PromptManager node like the execution it reuses (#347). + +The queue hook (``usage_tracking``) counts text it can read from the queued +graph. Text only known at run time (a PreviewAny or LLM link, a list node) is +counted by the node itself when it executes. When ComfyUI later serves the node +from its output cache the node does not execute, so without this module that +job is not counted. + +When a PromptManager-family node executes it records a *receipt item*: the +role-independent facts its own count is made from (the stored text hashes) +and digests of the STRING outputs it returned. When ComfyUI reports the node +as cached for a later job (``execution_cached``), the receipt whose digests +equal the text in the cache entry is replayed once for that job. The node's +role (positive, negative, which Pair outputs are used) and what the hook +already saw are recomputed from that job's own graph, and both paths count +through the same ``own_counts`` so they cannot drift. + +Fail closed: when the cached text cannot be matched to a receipt recorded in +this process, nothing is counted (the behavior before #347). In memory only and +bounded; starts no threads. +""" + +import hashlib +import threading +from collections import OrderedDict +from typing import Any, Callable, Dict, FrozenSet, List, NamedTuple, Optional, Tuple + +try: + from .hashing import generate_prompt_hash + from .logging_config import get_logger + from .prompt_graph import ( + PROMPT_MANAGER_TYPES, + intrinsic_pair_nodes, + resolve_text, + run_prompt_components, + run_prompt_nodes, + ) + from .prompt_pair_resolver import resolve_text_input + from .queue_observers import subscribe_dequeue, subscribe_task_done + from .run_keys import prompt_node_key, upstream_nodes + from .usage_tracking import find_pair_id_by_hashes, survive_switch +except ImportError: + from utils.hashing import generate_prompt_hash + from utils.logging_config import get_logger + from utils.prompt_graph import ( + PROMPT_MANAGER_TYPES, + intrinsic_pair_nodes, + resolve_text, + run_prompt_components, + run_prompt_nodes, + ) + from utils.prompt_pair_resolver import resolve_text_input + from utils.queue_observers import subscribe_dequeue, subscribe_task_done + from utils.run_keys import prompt_node_key, upstream_nodes + from utils.usage_tracking import find_pair_id_by_hashes, survive_switch + +logger = get_logger("prompt_manager.cached_runs") + +PAIR = "PromptManagerPair" +MAX_KEYS = 256 # run keys remembered (the same bound as MAX_KEYED_RUNS) +MAX_RECEIPTS = 4 # receipts per run key: a recurring upstream value (A, B, A) +MAX_ITEMS = 64 # list items per receipt (the cache reader's bound) +MAX_CACHED_NODES = 500 # the execution_cached guard the history capture uses +_EMPTY_HASH = generate_prompt_hash("") + +Count = Tuple[Any, ...] # ("prompt", hash) or ("pair", positive, negative or None) + + +class Facts(NamedTuple): + """What one execution stored: hashes of the stored text, None when nothing. + + ``positive_hash`` is None for an empty editor (or saving disabled); + ``negative_hash`` (Pair only) is None for an empty negative, which the pair + stores as ``negative_prompt_id IS NULL``. + """ + + positive_hash: Optional[str] + negative_hash: Optional[str] = None + + +class Role(NamedTuple): + """How a job's graph uses the node: which editors reach a sampler's positive.""" + + used_fields: FrozenSet[str] + pair_used: bool = False + + +class Seen(NamedTuple): + """Which of the node's texts the queue hook resolved, and so already counted.""" + + text: bool = False + negative_text: bool = False + pair: bool = False + + +class _Item(NamedTuple): + outputs: Tuple[str, ...] + facts: Facts + + +class _Receipt: + __slots__ = ("item_id", "items") + + def __init__(self, item_id: int) -> None: + self.item_id = item_id + self.items: Dict[int, _Item] = {} + + def outputs(self) -> Dict[int, Tuple[str, ...]]: + return {index: item.outputs for index, item in self.items.items()} + + +class _Attempt(NamedTuple): + item_id: int + prompt_id: str + graph: Dict[str, Any] + nodes: Dict[str, Tuple[str, str]] # node id -> (class, run key) + replayed: set + + +def text_digest(value: str) -> str: + """Digest of one returned STRING value, exactly as returned.""" + return hashlib.sha256(value.encode("utf-8", "surrogatepass")).hexdigest() + + +def stored_facts(text: Any, negative_text: Any = None) -> Facts: + """The facts of an execution that stores ``text`` (and a Pair's negative).""" + positive = text if isinstance(text, str) and text.strip() else None + negative = ( + negative_text + if isinstance(negative_text, str) and negative_text.strip() + else None + ) + return Facts( + generate_prompt_hash(positive) if positive is not None else None, + generate_prompt_hash(negative) if negative is not None else None, + ) + + +def pair_components(facts: Facts, role: Role) -> List[Tuple[str, str]]: + """``(field, hash)`` of each used, stored Pair component; each hash once.""" + found, counted = [], set() + for field, prompt_hash in ( + ("text", facts.positive_hash), + ("negative_text", facts.negative_hash), + ): + if field not in role.used_fields or prompt_hash is None: + continue + if prompt_hash in counted: + continue + counted.add(prompt_hash) + found.append((field, prompt_hash)) + return found + + +def own_counts(node_type: str, facts: Facts, role: Role, seen: Seen) -> List[Count]: + """What one execution of a node counts itself: text the hook could not see. + + Shared by the executing node and by replay. Excludes the pending first-use + catch-up, which only an execution that saves the prompt can consume. + """ + if facts.positive_hash is None: + return [] + if node_type != PAIR: + if "text" in role.used_fields and not seen.text: + return [("prompt", facts.positive_hash)] + return [] + counts: List[Count] = [ + ("prompt", prompt_hash) + for field, prompt_hash in pair_components(facts, role) + if not getattr(seen, field) + ] + if role.pair_used and not seen.pair: + counts.append(("pair", facts.positive_hash, facts.negative_hash)) + return counts + + +def graph_role(graph: Any, node_id: str, node_type: str) -> Role: + """The node's role in a queued graph, by the functions the nodes use.""" + if node_type != PAIR: + used = {"text"} if node_id in run_prompt_nodes(graph) else set() + return Role(frozenset(used)) + used = { + field for source, field in run_prompt_components(graph) if source == node_id + } + return Role(frozenset(used), node_id in intrinsic_pair_nodes(graph)) + + +def _same(resolved: Optional[str], prompt_hash: Optional[str]) -> bool: + return resolved is not None and generate_prompt_hash(resolved) == prompt_hash + + +def graph_seen(graph: Any, node_id: str, node_type: str, facts: Facts) -> Seen: + """Which texts the queue hook resolved from ``graph`` (as the node decides).""" + if node_type != PAIR: + return Seen(text=_same(resolve_text(graph, node_id), facts.positive_hash)) + positive = _same(resolve_text_input(graph, node_id, "text"), facts.positive_hash) + negative = _same( + resolve_text_input(graph, node_id, "negative_text"), + facts.negative_hash or _EMPTY_HASH, + ) + return Seen(positive, negative, positive and negative) + + +def _host_context() -> Tuple[Optional[str], Optional[int]]: + """(prompt id, list index) of the node executing on this thread, if known.""" + try: + from comfy_execution.utils import get_executing_context + except Exception: + return None, None + try: + context = get_executing_context() + except Exception: + return None, None + prompt_id = getattr(context, "prompt_id", None) + index = getattr(context, "list_index", None) + return ( + prompt_id if isinstance(prompt_id, str) and prompt_id else None, + index if type(index) is int else None, + ) + + +def write_count(db: Any, count: Count) -> bool: + """Record one counted run; False when the prompt or pair no longer exists.""" + + def write() -> bool: + if count[0] == "prompt": + row = db.get_prompt_by_hash(count[1]) + if row: + db.record_prompt_use(row["id"]) + return bool(row) + pair_id = find_pair_id_by_hashes(db, count[1], count[2]) + if pair_id: + db.record_prompt_pair_use(pair_id) + return bool(pair_id) + + return bool(survive_switch(db, write)) + + +class CachedRuns: + """Receipts of executed nodes and the one job currently executing.""" + + def __init__( + self, + *, + max_keys: int = MAX_KEYS, + max_receipts: int = MAX_RECEIPTS, + max_items: int = MAX_ITEMS, + context: Callable[[], Tuple[Optional[str], Optional[int]]] = _host_context, + ) -> None: + self._max_keys = max_keys + self._max_receipts = max_receipts + self._max_items = max_items + self._context = context + self._receipts: "OrderedDict[str, List[_Receipt]]" = OrderedDict() + self._attempt: Optional[_Attempt] = None + self._lock = threading.Lock() + + # -- the job ------------------------------------------------------------ + + def begin_attempt(self, result: Any) -> None: + """A job was dequeued: ``result`` is ``PromptQueue.get``'s return value.""" + attempt = None + try: + item, item_id = result[0], result[1] + prompt_id, graph, outputs = item[1], item[2], item[4] + if type(item_id) is int and isinstance(prompt_id, str): + if isinstance(graph, dict): + attempt = _Attempt( + item_id, prompt_id, graph, _job_nodes(graph, outputs), set() + ) + except Exception as error: + logger.debug(f"Dequeued job not tracked for cached runs: {error}") + with self._lock: + self._attempt = attempt + + def end_attempt(self, item_id: Any) -> None: + """The job finished: its receipts are complete and replace equal older ones. + + Deduplicating only complete receipts keeps a list execution that is + still recording ([A] so far of [A, B]) from dropping an older [A]. + """ + with self._lock: + if self._attempt is not None and self._attempt.item_id == item_id: + self._attempt = None + for run_key, receipts in self._receipts.items(): + if receipts and receipts[0].item_id == item_id: + self._receipts[run_key] = _deduplicated(receipts) + + def _current(self) -> Tuple[Optional[_Attempt], Optional[int]]: + """The attempt the executing node belongs to, and its list index.""" + prompt_id, index = self._context() + with self._lock: + attempt = self._attempt + if attempt is None or prompt_id is None or prompt_id != attempt.prompt_id: + return None, None + return attempt, index + + # -- node execution ----------------------------------------------------- + + def replayed(self, node_id: Any) -> bool: + """Whether this job already counted the executing node by replay.""" + attempt, _ = self._current() + if attempt is None: + return False + with self._lock: + return str(node_id) in attempt.replayed + + def record_item(self, node_id: Any, outputs: Tuple[Any, ...], facts: Facts) -> None: + """Remember what this execution returned and was entitled to count.""" + try: + attempt, index = self._current() + node = attempt.nodes.get(str(node_id)) if attempt else None + if node is None: + return + index = 0 if index is None else index + if not 0 <= index < self._max_items: + return + if any(not isinstance(value, str) for value in outputs): + return + item = _Item(tuple(text_digest(value) for value in outputs), facts) + self._store(node[1], attempt.item_id, index, item) + except Exception as error: + logger.debug(f"Could not record a cached-run receipt: {error}") + + def _store(self, run_key: str, item_id: int, index: int, item: _Item) -> None: + with self._lock: + receipts = self._receipts.pop(run_key, []) + if receipts and receipts[0].item_id == item_id: + receipt = receipts[0] + else: + # The previous job's receipt is complete (its task_done may + # never have fired): it replaces older receipts it equals. + receipts = _deduplicated(receipts) + receipt = _Receipt(item_id) + receipts.insert(0, receipt) + receipt.items[index] = item + self._receipts[run_key] = receipts[: self._max_receipts] + while len(self._receipts) > self._max_keys: + self._receipts.popitem(last=False) + + # -- cached nodes ------------------------------------------------------- + + def replay( + self, + executor: Any, + prompt_id: Any, + nodes: Any, + db_factory: Callable[[], Any], + reader: Optional[Callable[[Any, str, str], Any]] = None, + ) -> None: + """Count the job's cached nodes whose cached text a receipt proves.""" + if reader is None: + reader = _host().cached_string_outputs + with self._lock: + attempt = self._attempt + if attempt is None or prompt_id != attempt.prompt_id: + return + if type(nodes) not in (list, tuple) or len(nodes) > MAX_CACHED_NODES: + return + db = None + for node_id in nodes: + node = attempt.nodes.get(node_id) if type(node_id) is str else None + if node is None: + continue + node_type, run_key = node + items = self._match(run_key, reader(executor, node_id, node_type)) + if items is None: + logger.debug( + f"Cached node {node_id} not counted: no matching execution" + ) + continue + with self._lock: + if node_id in attempt.replayed: + continue # a repeated message or id never counts twice + attempt.replayed.add(node_id) + counts: List[Count] = [] + role = graph_role(attempt.graph, node_id, node_type) + for item in items: + seen = graph_seen(attempt.graph, node_id, node_type, item.facts) + counts.extend(own_counts(node_type, item.facts, role, seen)) + if not counts: + continue + try: + db = db if db is not None else db_factory() + for count in counts: + if not write_count(db, count): + logger.info( + f"Cached node {node_id}: its {count[0]} no longer exists, " + "not counted" + ) + except Exception as error: + logger.warning(f"Could not count cached node {node_id}: {error}") + + def _match(self, run_key: str, values: Any) -> Optional[List[_Item]]: + """The items of the one receipt whose outputs equal ``values``, in order.""" + if not values: + return None + length = len(values[0]) + if length == 0 or length > self._max_items: + return None + if any(len(slot) != length for slot in values): + return None + wanted = { + index: tuple(text_digest(slot[index]) for slot in values) + for index in range(length) + } + with self._lock: + receipts = self._receipts.get(run_key) + if not receipts: + return None + matches = [receipt for receipt in receipts if receipt.outputs() == wanted] + if not matches: + return None + self._receipts.move_to_end(run_key) + facts = { + tuple(receipt.items[index].facts for index in range(length)) + for receipt in matches + } + if len(facts) != 1: + return None + return [matches[0].items[index] for index in range(length)] + + +def _deduplicated(receipts: List[_Receipt]) -> List[_Receipt]: + """The newest receipt replaces older ones with the same outputs.""" + if not receipts: + return receipts + outputs = receipts[0].outputs() + return [receipts[0]] + [r for r in receipts[1:] if r.outputs() != outputs] + + +def _job_nodes(graph: Dict[str, Any], outputs: Any) -> Dict[str, Tuple[str, str]]: + """PromptManager-family nodes the job's outputs need, with their run keys.""" + starts = ( + [str(node) for node in outputs] if isinstance(outputs, (list, tuple)) else [] + ) + needed = upstream_nodes(graph, starts) + nodes = {} + for node_id in needed: + node = graph.get(node_id) + node_type = node.get("class_type") if isinstance(node, dict) else None + if node_type not in PROMPT_MANAGER_TYPES: + continue + run_key = prompt_node_key(graph, node_id) + if run_key is not None: + nodes[node_id] = (node_type, run_key) + return nodes + + +def _host() -> Any: + """The host adapter module (imported late: it needs ComfyUI's modules).""" + try: + from . import execution_context_host + except ImportError: + from utils import execution_context_host + return execution_context_host + + +# Shared between the nodes and the host observers +CACHED_RUNS = CachedRuns() + +_registered = threading.Event() +_register_lock = threading.Lock() + + +def register_cached_runs( + server: Any, db_factory: Callable[[], Any], runs: Optional[CachedRuns] = None +) -> bool: + """Observe dequeue, ``execution_cached`` and ``task_done``; once per process. + + Registers nothing when the host lacks any of them: counting is then as + before #347. + """ + runs = CACHED_RUNS if runs is None else runs + with _register_lock: + if _registered.is_set(): + return False + queue = getattr(server, "prompt_queue", None) + if not callable(getattr(queue, "get", None)) or not callable( + getattr(queue, "task_done", None) + ): + logger.info("Cached runs not counted: the prompt queue is not observable") + return False + try: + from execution import PromptExecutor + except Exception: + logger.info("Cached runs not counted: no ComfyUI executor") + return False + if not callable(getattr(PromptExecutor, "add_message", None)): + logger.info("Cached runs not counted: executor messages unavailable") + return False + subscribe_executor_messages = _host().subscribe_executor_messages + + def on_message(executor: Any, args: tuple, kwargs: dict) -> None: + try: + event = args[0] if args else kwargs.get("event") + if event != "execution_cached": + return + server_of = getattr(executor, "server", None) + if getattr(server_of, "prompt_queue", None) is not queue: + return + data = args[1] if len(args) > 1 else kwargs.get("data") + if type(data) is not dict: + return + runs.replay( + executor, data.get("prompt_id"), data.get("nodes"), db_factory + ) + except Exception as error: + logger.warning(f"Could not count cached nodes: {error}") + + unsubscribers = [] + try: + unsubscribers.append( + subscribe_dequeue(queue, "cached_runs", runs.begin_attempt) + ) + unsubscribers.append( + subscribe_task_done( + queue, + "cached_runs", + lambda args, kwargs: runs.end_attempt( + args[0] if args else kwargs.get("item_id") + ), + ) + ) + unsubscribers.append(subscribe_executor_messages("cached_runs", on_message)) + except Exception: + for unsubscribe in unsubscribers: + unsubscribe() + raise + _registered.set() + logger.info("Counting cached runs of runtime-linked prompt text") + return True diff --git a/utils/execution_context_host.py b/utils/execution_context_host.py index c3d8ba38..fdf02afd 100644 --- a/utils/execution_context_host.py +++ b/utils/execution_context_host.py @@ -75,6 +75,82 @@ def subscribe_executor_messages(key: str, callback: Callable) -> Callable[[], No return state.subscribe(key, callback) +# STRING output slots of each PromptManager-family node, in output order +STRING_SLOTS = { + "PromptManager": (1,), + "PromptManagerFocused": (1,), + "PromptManagerSplit": (1,), + "PromptManagerText": (0,), + "PromptManagerPair": (2, 3), +} +MAX_CACHED_VALUES = 64 + + +def _local_cache_reader(executor: object) -> Callable | None: + """``node_id -> outputs`` of the executor's local output cache, or None. + + Only the supported cache classes, only ``get_local`` (never a provider or + an async lookup), only a real ``CacheEntry`` with at most four outputs. + """ + from comfy_execution import caching + from execution import CacheEntry + + cache = executor.caches.outputs + supported = tuple( + cls + for name in ("HierarchicalCache", "LRUCache", "RAMPressureCache") + if isinstance(cls := getattr(caching, name, None), type) + ) + if type(cache) not in supported: + return None + local = getattr(cache, "get_local", None) + if not callable(local): + return None + + def read(node_id: str) -> list | tuple | None: + entry = local(node_id) + if type(entry) is not CacheEntry: + return None + outputs = entry.outputs + if type(outputs) not in (list, tuple) or not 0 < len(outputs) <= 4: + return None + return outputs + + return read + + +def cached_string_outputs( + executor: object, node_id: str, node_type: str +) -> tuple[tuple[str, ...], ...] | None: + """One tuple per STRING slot of a cached node's local entry, or None (#347). + + None when the cache or entry is not supported, a slot is missing or holds + more than 64 values, any value is not a ``str``, or the slots differ in + length. Never runs a provider or an async lookup. + """ + try: + slots = STRING_SLOTS.get(node_type) + if slots is None or type(node_id) is not str: + return None + local = _local_cache_reader(executor) + outputs = local(node_id) if local is not None else None + if outputs is None or not max(slots) < len(outputs): + return None + found = [] + for slot in slots: + values = outputs[slot] + if type(values) not in (list, tuple) or len(values) > MAX_CACHED_VALUES: + return None + if any(type(value) is not str for value in values): + return None + found.append(tuple(values)) + if len({len(values) for values in found}) != 1: + return None + return tuple(found) + except Exception: + return None + + def observe_cached_context( executor: object, message_type: object, message_data: object ) -> None: @@ -101,19 +177,8 @@ def observe_cached_context( nodes = message_data.get("nodes") if type(nodes) not in (list, tuple) or len(nodes) > 500: return - from comfy_execution import caching - from execution import CacheEntry - - cache = executor.caches.outputs - supported = tuple( - cls - for name in ("HierarchicalCache", "LRUCache", "RAMPressureCache") - if isinstance(cls := getattr(caching, name, None), type) - ) - if type(cache) not in supported: - return - local = getattr(cache, "get_local", None) - if not callable(local): + local = _local_cache_reader(executor) + if local is None: return # Only source IDs retained by the bounded submitted graph are queried. for source in cap.sources: @@ -124,11 +189,8 @@ def observe_cached_context( if source.node_type == "PromptManagerPair" else 0 if source.node_type == "PromptManagerText" else 1 ) - entry = local(source.node_id) - if type(entry) is not CacheEntry: - continue - outputs = entry.outputs - if type(outputs) not in (list, tuple) or not slot < len(outputs) <= 4: + outputs = local(source.node_id) + if outputs is None or not slot < len(outputs): continue values = outputs[slot] if type(values) not in (list, tuple) or len(values) > 64: diff --git a/utils/exit_signal.py b/utils/exit_signal.py new file mode 100644 index 00000000..fa7d82b6 --- /dev/null +++ b/utils/exit_signal.py @@ -0,0 +1,110 @@ +"""The process exit signal: running work stops at its next safe point (#376). + +ComfyUI catches Ctrl+C in ``main.py`` and returns without closing its event +loop. The interpreter then runs ``threading._shutdown``, which first calls the +``threading._register_atexit`` callbacks, newest first. One of them, +``concurrent.futures``'s ``_python_exit``, joins every executor thread, and +only after that do the ``atexit`` callbacks run (the package teardown). A call +still running on the default executor (a scan batch, a model download, a +thumbnail batch) used to keep the process alive until it returned. + +``install()`` registers a callback in that same list after ``concurrent.futures`` +did, so it runs before the join. It sets the signal that ``requested()`` reads +(``Job.cancelled`` and ``map_parallel`` read it) and runs ``on_exit``. + +``threading._register_atexit`` is CPython-internal. It has the same contract +on 3.12, 3.13 and 3.14. Without it, or when it refuses the registration, exit +behaves as before: the call in flight runs to its end. +""" + +import logging +import threading + +logger = logging.getLogger("prompt_manager.exit") + +_requested = threading.Event() +_install_lock = threading.Lock() +_installed = False +_on_exit = [] + + +class ExitRequested(Exception): + """The interpreter is exiting: work not yet started is not started.""" + + +def requested(): + """True once the interpreter has started exiting.""" + return _requested.is_set() + + +def request(): + """Set the signal (the exit hook; tests).""" + _requested.set() + + +def checkpoint(): + """Raise ExitRequested once the interpreter is exiting.""" + if _requested.is_set(): + raise ExitRequested("ComfyUI is exiting") + + +def checked(func): + """``func`` behind a checkpoint: a call that starts after the signal + raises ExitRequested instead of running.""" + + def run(*args, **kwargs): + checkpoint() + return func(*args, **kwargs) + + return run + + +def _hook(): + """The threading atexit callback. It never raises: an exception here + would end ``threading._shutdown`` before it shuts the executors.""" + _requested.set() + for action in tuple(_on_exit): + try: + action() + except (Exception, KeyboardInterrupt): + logger.error("Stopping work at exit failed", exc_info=True) + + +def install(on_exit=None): + """Register the exit hook once per process; True when it is registered. + + ``on_exit`` (optional) runs inside the hook, after the signal is set. + False (logged at DEBUG) when this Python has no ``_register_atexit`` or + refuses it; exit then waits for running work as before. + """ + global _installed + with _install_lock: + if on_exit is not None and on_exit not in _on_exit: + _on_exit.append(on_exit) + if _installed: + return True + # Imported first so its pool join is registered before our hook: + # the callbacks run newest first, so ours runs before that join. + import concurrent.futures.thread # noqa: F401 + + register = getattr(threading, "_register_atexit", None) + if not callable(register): + logger.debug("threading._register_atexit is missing; exit waits for jobs") + return False + try: + register(_hook) + except RuntimeError as error: # already shutting down + logger.debug(f"Exit hook not registered: {error}") + return False + _installed = True + return True + + +def reset(): + """Clear the signal and forget the registration (tests only; a hook + already registered stays in ``threading`` and sets the signal at exit).""" + global _installed + with _install_lock: + _requested.clear() + _installed = False + _on_exit.clear() diff --git a/utils/image_monitor.py b/utils/image_monitor.py index 6e13dede..7c70d876 100644 --- a/utils/image_monitor.py +++ b/utils/image_monitor.py @@ -35,6 +35,7 @@ from watchdog.events import FileSystemEventHandler from .hashing import generate_prompt_hash from .metadata_extractor import ComfyUIMetadataExtractor +from .parallel import note_refusal, refused_at_shutdown from .logging_config import get_logger from .prompt_graph import resolve_text, run_prompt_nodes from .prompt_pair_resolver import resolve_prompt_pair @@ -194,8 +195,9 @@ class ImageGenerationHandler(FileSystemEventHandler): and once its size has stopped changing. Returns: - False when the path is already pending, the pending set is full or - the handler is closed + False when the path is already pending, the pending set is full, + the handler is closed or the interpreter is exiting (its worker + cannot start) """ with self._pending_lock: if self._closed: @@ -211,7 +213,15 @@ class ImageGenerationHandler(FileSystemEventHandler): return False due = time.monotonic() + self.processing_delay self._pending[image_path] = (prompt_snapshot, due) - self._ensure_worker() + try: + self._ensure_worker() + except RuntimeError as error: + # No worker will ever take it: never leave it pending (#375). + self._pending.pop(image_path, None) + if not refused_at_shutdown(error): + raise + note_refusal(self.logger, "The image worker") + return False self._work.put(image_path) return True diff --git a/utils/inspection_response.py b/utils/inspection_response.py index 048103ac..ba6a43f0 100644 --- a/utils/inspection_response.py +++ b/utils/inspection_response.py @@ -300,6 +300,49 @@ def _blank(value: str | bytes | None) -> bool: return value is None or not value.strip() +def inherited_category( + positive_category: str | bytes | None, + recipe_category: str | bytes | None, + has_recipe: bool, +) -> tuple: + """``(value, scope)`` of the category an image's link supplies (#332 MC4). + + The one rule for the inspector chip, the Images category filter and + History (D39): a linked recipe's own category when it is not blank, + ``(recipe category, "pair")``; otherwise the positive's value unchanged, + blank or not, ``(positive category, "positive")``. + """ + if has_recipe and not _blank(recipe_category): + return recipe_category, "pair" + return positive_category, "positive" + + +class _OwnCategoryUnread: + """The own-category read failed: hide the category (#332 DC5).""" + + def __repr__(self) -> str: + return "OWN_CATEGORY_UNREAD" + + +OWN_CATEGORY_UNREAD = _OwnCategoryUnread() + + +def apply_own_category(library: dict, own) -> dict: + """*library* with an image's own category applied (#332 ยง5). + + ``own`` is the own category (a string, shown with scope ``"image"`` + whatever the link status), None (the inherited values stay) or + OWN_CATEGORY_UNREAD (category and scope hidden rather than possibly + wrong). Returns a new dict. + """ + result = dict(library) + if own is OWN_CATEGORY_UNREAD: + result.update(category=None, category_scope=None) + elif isinstance(own, str): + result.update(category=own, category_scope="image") + return result + + def library_metadata( positive_category: str | bytes | None, positive_tags: list, @@ -312,13 +355,14 @@ def library_metadata( the linked positive prompt's, the values Set category writes and the Images filters match. Tag sets are never merged. ``recipe`` is (category, tags). """ - category, category_scope = positive_category, "positive" + category, category_scope = inherited_category( + positive_category, + recipe[0] if recipe is not None else None, + recipe is not None, + ) tags, tags_scope = positive_tags, "positive" - if recipe is not None: - if not _blank(recipe[0]): - category, category_scope = recipe[0], "pair" - if recipe[1]: - tags, tags_scope = recipe[1], "pair" + if recipe is not None and recipe[1]: + tags, tags_scope = recipe[1], "pair" return { "category": category, "category_scope": category_scope, @@ -345,9 +389,13 @@ def _library_projection(library: dict) -> dict: value = library.get(key) if value is None or isinstance(value, str): result[key] = value - for key in ("category_scope", "tags_scope"): + # An image's own category (#332) has the scope "image"; tags never do. + for key, scopes in ( + ("category_scope", ("pair", "positive", "image", None)), + ("tags_scope", ("pair", "positive", None)), + ): scope = library.get(key) - if scope in ("pair", "positive", None): + if scope in scopes: result[key] = scope tags = library.get("tags") result["tags"] = ( diff --git a/utils/logging_config.py b/utils/logging_config.py index 97a37639..08bb0a26 100644 --- a/utils/logging_config.py +++ b/utils/logging_config.py @@ -39,6 +39,35 @@ class RotatedChanged(Exception): self.name = name +# Clear all logs (#389): why it changed nothing, as the API and the pages say it. +CLEAR_CHANGED = ( + "The log files changed since you confirmed; nothing was cleared." + " Read the list again and clear again." +) +CLEAR_FOREIGN_WRITER = ( + "Another program is writing to prompt_manager.log, such as a second ComfyUI" + " sharing this log folder; nothing was cleared. Stop it and clear again." +) +CLEAR_HARD_LINK = ( + "prompt_manager.log has another name (a hard link); emptying it would empty" + " that file too, so nothing was cleared. Remove the other name, then clear" + " again." +) +# A confirmed row's identity: "::", a string because +# mtime_ns and many inodes are larger than a browser's exact integers. +# mtime_ns is negative for a time before 1970 (a restored archive, say). +_IDENTITY = re.compile(r"([0-9]{1,20}):(-?[0-9]{1,20}):([0-9]{1,40})") +MAX_CONFIRMED_ROWS = 10000 + + +class ClearRefused(Exception): + """Clear all logs changed nothing; ``code`` names why, the message says it.""" + + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + def _name_key(name: str) -> str: """How a name compares: case-folded where the filesystem folds it (Windows).""" return os.path.normcase(name) @@ -87,35 +116,37 @@ def scan_log_folder(log_dir) -> List[Dict[str, Any]]: be read is left out. Never logs: the cleanup calls it while it holds the file handler's lock. """ - rows = [] try: names = [entry.name for entry in os.scandir(log_dir)] except OSError: - return rows - for name in names: - path = os.path.join(log_dir, name) - try: - st = os.lstat(path) - kind = _kind(name, path, st) - except OSError: - continue - try: - modified = datetime.fromtimestamp(st.st_mtime).isoformat() - except (OSError, OverflowError, ValueError): # a time before 1970 on Windows - modified = "" - rows.append( - { - "name": name, - "kind": kind, - "size": st.st_size, - "mtime_ns": st.st_mtime_ns, - "ino": st.st_ino, - "modified": modified, - "_dir": stat.S_ISDIR(st.st_mode) and kind != "link", - "_regular": stat.S_ISREG(st.st_mode) and kind != "link", - } - ) - return rows + return [] + rows = (scan_log_entry(log_dir, name) for name in names) + return [row for row in rows if row is not None] + + +def scan_log_entry(log_dir, name) -> Optional[Dict[str, Any]]: + """One entry's scan row (``lstat``), or None when it cannot be read.""" + path = os.path.join(log_dir, name) + try: + st = os.lstat(path) + kind = _kind(name, path, st) + except OSError: + return None + try: + modified = datetime.fromtimestamp(st.st_mtime).isoformat() + except (OSError, OverflowError, ValueError): # a time before 1970 on Windows + modified = "" + return { + "name": name, + "kind": kind, + "size": st.st_size, + "mtime_ns": st.st_mtime_ns, + "ino": st.st_ino, + "modified": modified, + "_dir": stat.S_ISDIR(st.st_mode) and kind != "link", + "_regular": stat.S_ISREG(st.st_mode) and kind != "link", + "_nlink": st.st_nlink, + } def public_row(row: Dict[str, Any]) -> Dict[str, Any]: @@ -127,6 +158,37 @@ def rotated_sort_key(row: Dict[str, Any]) -> int: return _rotated_number(row["name"]) or 0 +def file_identity(row: Dict[str, Any]) -> str: + """A listed file's identity as the client sends it back: size, mtime_ns, ino.""" + return f"{row['size']}:{row['mtime_ns']}:{row['ino']}" + + +def parse_confirmed(expected) -> List[Dict[str, Any]]: + """The rows a Clear all logs confirmation showed, as review rows. + + ``expected`` is a list of ``{name, identity}`` objects from + ``/logs/files``; anything else raises ValueError. + """ + if not isinstance(expected, list) or len(expected) > MAX_CONFIRMED_ROWS: + raise ValueError("expected must be a list of listed log files") + rows, seen = [], set() + for item in expected: + if not isinstance(item, dict): + raise ValueError("each expected file must be an object") + name, identity = item.get("name"), item.get("identity") + if not isinstance(name, str) or not isinstance(identity, str): + raise ValueError("each expected file needs a name and an identity") + match = _IDENTITY.fullmatch(identity) + if match is None: + raise ValueError("an expected file's identity is not size:mtime_ns:ino") + size, mtime_ns, ino = (int(part) for part in match.groups()) + if _name_key(name) in seen: + raise ValueError("an expected file is listed twice") + seen.add(_name_key(name)) + rows.append({"name": name, "size": size, "mtime_ns": mtime_ns, "ino": ino}) + return rows + + def log_file_entry(log_dir, name) -> Path: """The path of a downloadable log file named ``name`` (blocking). @@ -157,6 +219,11 @@ def _stream_is(stream, ino: int) -> bool: return not (ino and open_ino and open_ino != ino) +def _stream_ino_is(opened: os.stat_result, ino: int) -> bool: + """Whether a stream's ``fstat`` is the file with inode ``ino`` (0: unknown).""" + return not (ino and opened.st_ino and opened.st_ino != ino) + + def _error_text(error: BaseException) -> str: """An OSError's reason and file name, without the folder it is in.""" if isinstance(error, OSError): @@ -432,6 +499,7 @@ class PromptManagerLogger: "modified": row["modified"], "is_main": row["kind"] == "active", "kind": row["kind"], + "identity": file_identity(row), } ) @@ -530,8 +598,14 @@ class PromptManagerLogger: finally: handler.release() if not handled: - flags = os.O_WRONLY | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0) - os.close(os.open(str(log_file), flags)) + flags = os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(str(log_file), flags) + try: + if os.fstat(fd).st_nlink > 1: # emptying it would empty the other + raise ClearRefused("logs-hard-link", CLEAR_HARD_LINK) + os.ftruncate(fd, 0) + finally: + os.close(fd) def delete_rotated(self, expected: List[Dict[str, Any]]) -> Dict[str, Any]: """Delete exactly the reviewed rotated copies, or none of them (blocking). @@ -582,47 +656,126 @@ class PromptManagerLogger: ) return {"removed": removed, "failed": failed} - def truncate_logs(self) -> Dict[str, Any]: - """Empty the active log and delete its rotated copies (Rule R). + def _active_streams(self, active: Optional[Dict[str, Any]]): + """``fstat`` and offset of every handler stream writing the active name.""" + target = os.path.normcase( + os.path.realpath(os.path.join(self.log_dir, ACTIVE_LOG_NAME)) + ) + for handler in self._file_handlers(): + stream = getattr(handler, "stream", None) + base = getattr(handler, "baseFilename", None) + if stream is None or base is None: + continue + if os.path.normcase(os.path.realpath(base)) != target: + continue + try: + stream.flush() + fd = stream.fileno() + yield os.fstat(fd), os.lseek(fd, 0, os.SEEK_CUR) + except (OSError, ValueError, AttributeError): + continue - Every other file in the log folder, and every link, is kept. Also - clears the memory buffer. Runs under ``_files_lock`` and the file - handlers' locks, so no rollover recreates a just-deleted copy, and - logs only once they are released. + def _check_clear_is_safe(self, rows, active) -> None: + """Refuse a clear that would empty or delete what nobody confirmed. + + Called under the files lock and every handler lock, so nothing of this + process writes or rolls over meanwhile; what changes is another + program's doing. Our handler's stream is flushed: a file larger than + the stream's own offset was appended to by someone else, and a stream + whose file is now a rotated copy was rolled over by someone else. + Either is a second writer (another ComfyUI sharing the folder). This + is best effort, not a lock: writes before our own last write, or any + write while this process has file logging off, go unseen. An + active log with more than one name (a hard link) is never emptied: the + other name would be emptied too. + """ + rotated = {row["ino"] for row in rows if row["kind"] == "rotated"} + rotated.discard(0) + nlink = active["_nlink"] if active is not None else 1 + for opened, offset in self._active_streams(active): + if (opened.st_ino and opened.st_ino in rotated) or opened.st_size > offset: + raise ClearRefused("logs-foreign-writer", CLEAR_FOREIGN_WRITER) + if active is not None and _stream_ino_is(opened, active["ino"]): + nlink = max(nlink, opened.st_nlink) + if nlink > 1: + raise ClearRefused("logs-hard-link", CLEAR_HARD_LINK) + + @staticmethod + def _confirmed_targets(confirmed, current, active) -> List[Dict[str, Any]]: + """The rotated copies a confirmation listed, each as it was listed. + + Refuses (``logs-changed``) when any listed copy is missing or differs + in kind, size, ``mtime_ns`` or inode, or when the listed active log is + now another file (it was rolled over). A copy nobody listed is kept. + """ + targets = [] + for reviewed in confirmed: + key = _name_key(reviewed["name"]) + row = current.get(key) + if key == _name_key(ACTIVE_LOG_NAME): + ino = reviewed["ino"] + if active is None or (ino and active["ino"] and active["ino"] != ino): + raise ClearRefused("logs-changed", CLEAR_CHANGED) + continue + if _rotated_number(reviewed["name"]) is None: + continue # never deleted, listed or not + if not _matches_review(row, reviewed): + raise ClearRefused("logs-changed", CLEAR_CHANGED) + targets.append(row) + return targets + + def truncate_logs(self, expected=None) -> Dict[str, Any]: + """Empty the active log and delete its rotated copies (Rule R, #389). + + ``expected`` is what the confirmation showed: ``{name, identity}`` + rows of ``/logs/files`` (``parse_confirmed``; ValueError when + malformed). Then only the listed rotated copies are deleted, and only + when each still has the listed size, ``mtime_ns`` and inode and the + active log is still the listed file; otherwise ClearRefused + (``logs-changed``) with nothing changed. Without it (the legacy admin + page, whose confirmation lists nothing) every rotated copy present now + is deleted. + + Either way, a second writer of the active log (``logs-foreign-writer``) + or an active log with a hard link (``logs-hard-link``) refuses the + whole clear first. Each copy's identity is checked again just before + its unlink; one that changed (another process rolling over meanwhile) + stops the clear there, keeping it, the rest and the active log, which + is emptied last. Every other file in the log folder, and every link, + is kept. Also clears the memory buffer. Runs under ``_files_lock`` and + the file handlers' locks and logs only once they are released. Returns: Dictionary with: - truncated: List of successfully truncated files - errors: List of error messages for failed operations """ + confirmed = None if expected is None else parse_confirmed(expected) results = {"truncated": [], "errors": []} + refused = None with self._files_lock: held = self._acquire_file_handlers() try: rows = scan_log_folder(self.log_dir) - active = [row for row in rows if row["kind"] == "active"] - rotated = sorted( - (row for row in rows if row["kind"] == "rotated"), - key=rotated_sort_key, - ) - for row in active + rotated: - path = os.path.join(self.log_dir, row["name"]) - try: - if row["kind"] == "active": - self._truncate_main_log(Path(path), row["ino"]) - else: - os.unlink(path) - results["truncated"].append(row["name"]) - except OSError as error: - results["errors"].append( - f"Failed to truncate {row['name']}: {_error_text(error)}" - ) - with self._buffer_lock: - self._log_buffer.clear() + current = {_name_key(row["name"]): row for row in rows} + active = next((row for row in rows if row["kind"] == "active"), None) + try: + if confirmed is None: + targets = [row for row in rows if row["kind"] == "rotated"] + else: + targets = self._confirmed_targets(confirmed, current, active) + self._check_clear_is_safe(rows, active) + except ClearRefused as error: + refused = error + else: + self._clear(sorted(targets, key=rotated_sort_key), active, results) finally: self._release_handlers(held) + if refused is not None: + self.logger.warning("Clear all logs refused (%s)", refused.code) + raise refused if results["truncated"]: self.logger.info("Truncated log files: %s", ", ".join(results["truncated"])) for error_msg in results["errors"]: @@ -630,6 +783,39 @@ class PromptManagerLogger: return results + def _clear(self, targets, active, results) -> None: + """Delete ``targets`` (each re-checked) then empty ``active`` (locks held).""" + for reviewed in targets: + name = reviewed["name"] + if not _matches_review(scan_log_entry(self.log_dir, name), reviewed): + results["errors"].append( + f"{name} changed while clearing (another program may be writing" + " to this log folder); it, the copies after it and the active" + " log were kept" + ) + return + try: + os.unlink(os.path.join(self.log_dir, name)) + results["truncated"].append(name) + except OSError as error: + results["errors"].append( + f"Failed to truncate {name}: {_error_text(error)}" + ) + if active is not None: + try: + self._truncate_main_log( + Path(self.log_dir, active["name"]), active["ino"] + ) + results["truncated"].append(active["name"]) + except ClearRefused as error: + results["errors"].append(str(error)) + except OSError as error: + results["errors"].append( + f"Failed to truncate {active['name']}: {_error_text(error)}" + ) + with self._buffer_lock: + self._log_buffer.clear() + def update_config(self, new_config: Dict[str, Any]): """Update logging configuration. diff --git a/utils/parallel.py b/utils/parallel.py index 8b183dd8..b790794b 100644 --- a/utils/parallel.py +++ b/utils/parallel.py @@ -5,22 +5,92 @@ a CUDA-initialised, multi-threaded process is unsafe and the spawn start method would re-import ComfyUI's main module in every child. Pillow releases the GIL while decoding, resizing and encoding, so threads still scale close to linearly for both jobs. + +Interpreter exit (#375): ``threading._shutdown`` shuts every thread pool +(``submit`` then raises "cannot schedule new futures after interpreter +shutdown") before it joins the non-daemon threads and long before the package +teardown runs, while our daemon threads keep working. Measured on 3.12 and +3.14, ``Thread.start`` still succeeds in that window; it is refused ("can't +create new thread at interpreter shutdown") only once the interpreter is +marked finalizing, after the ``atexit`` callbacks. ``refused_at_shutdown`` +recognises both; ``note_refusal`` logs one DEBUG line per place, with no +traceback. """ +import logging +import threading from concurrent.futures import ThreadPoolExecutor +from .exit_signal import checked + +SHUTDOWN_REFUSAL = "interpreter shutdown" +_logger = logging.getLogger("prompt_manager.parallel") +_noted = set() +_noted_lock = threading.Lock() + + +def refused_at_shutdown(error): + """Whether ``error`` is a thread or pool start refused at interpreter exit.""" + return isinstance(error, RuntimeError) and SHUTDOWN_REFUSAL in str(error) + + +def note_refusal(logger, what): + """Log once per ``what``, at DEBUG and without a traceback, that ``what`` + was refused because the interpreter is exiting.""" + key = (logger.name, what) + with _noted_lock: + if key in _noted: + return + _noted.add(key) + logger.debug(f"{what}: refused, the interpreter is exiting") + def map_parallel(func, items, workers): """Apply ``func`` to every item on up to ``workers`` threads, keeping order. ``workers`` of 1 (or fewer) runs on the calling thread with no pool. An exception in ``func`` propagates to the caller, so callers that must - survive bad input wrap ``func`` themselves. + survive bad input wrap ``func`` themselves. A pool refused at interpreter + exit runs the items it did not start on the calling thread (#375). An item + that has not started when the interpreter starts exiting raises + ExitRequested instead (#376). """ + func = checked(func) items = list(items) workers = max(1, int(workers or 1)) if workers == 1 or len(items) <= 1: return [func(item) for item in items] max_workers = min(workers, len(items)) with ThreadPoolExecutor(max_workers, thread_name_prefix="pm-worker") as pool: - return list(pool.map(func, items)) + futures = [] + try: + for item in items: + futures.append(pool.submit(func, item)) + except RuntimeError as error: + if not refused_at_shutdown(error): + raise + note_refusal(_logger, "A worker pool") + # Drop what was queued but not started; it runs here instead. + # (A refused Thread.start leaves its item queued without a future + # here: it is cancelled too, unless a busy pool thread took it in + # between, the one case where an item can run twice.) + pool.shutdown(wait=True, cancel_futures=True) + else: + return [future.result() for future in futures] + return _run_rest(func, items, futures) + + +def _run_rest(func, items, futures): + """The pool's finished results, and every item it did not run, run here. + + After ``shutdown(cancel_futures=True)`` each submitted future either ran + (its result or exception is kept) or was cancelled before it started. + """ + results = [] + for index, item in enumerate(items): + future = futures[index] if index < len(futures) else None + if future is not None and not future.cancelled(): + results.append(future.result()) + else: + results.append(func(item)) + return results diff --git a/utils/prompt_inspector.py b/utils/prompt_inspector.py index 0307b5ca..1daaef6c 100644 --- a/utils/prompt_inspector.py +++ b/utils/prompt_inspector.py @@ -10,6 +10,7 @@ import threading import zlib from dataclasses import dataclass, field as dataclass_field from pathlib import Path +from collections.abc import Mapping from typing import Any from PIL import Image @@ -461,6 +462,14 @@ def read_inspection_metadata(path: Path) -> dict[str, str]: return read_inspection_file(path).metadata +# A side whose branch recorded run history named (#386): the save node that +# wrote the file picked the sampler; the text is still that branch's graph walk. +RECORDED_RUN_SOURCE = "recorded-run" +RECORDED_TEXT_REASON = "PromptManager recorded this text when the run executed" +# Outcomes the file alone leaves open; only these ask recorded history. +_UNDECIDED = frozenset({"ambiguous", "unresolved"}) + + def _side(status: str, text: str | None = None, source: str | None = None) -> dict: return {"status": status, "text": text, "source": source} @@ -529,15 +538,60 @@ def _parsed_workflow(value: Any) -> dict | None: return value if isinstance(value, dict) else None +def _recorded_execution(resolved: dict, texts: Mapping) -> dict | None: + """Evidence from the texts recorded at one sampler of the file's run (#386). + + Only for an outcome that names one sampler (a single stage stopped at a + run-time value, such as a language model's output feeding PromptManager) + and only when that sampler's positive text was recorded and is not empty. + """ + sampler = resolved.get("sampler_id") + roles = texts.get(sampler) if isinstance(sampler, str) else None + if not isinstance(roles, Mapping): + return None + positive = roles.get("positive") + if not isinstance(positive, str) or not positive.strip(): + return None + negative = roles.get("negative") + prompts = { + "positive": _side("resolved", positive, "execution"), + "negative": ( + _side("resolved", negative, "execution") + if isinstance(negative, str) + else _side("unresolved") + ), + } + resolution = { + "state": "resolved", + "sampler_id": sampler, + "reason": RECORDED_TEXT_REASON, + "reason_code": None, + "node": None, + } + return {"prompts": prompts, "resolution": resolution} + + def inspect_prompt_evidence( metadata: dict, saved: dict | None = None, *, profile: ResolverProfile = INSPECTION_PROFILE, file_name: str | None = None, + recorded_saver_id: str | None = None, + recorded_texts: Mapping[str, Mapping[str, str | None]] | None = None, ) -> dict: """Keep uncertainty visible; a saved association never replaces file evidence. + ``recorded_saver_id`` is the save node recorded run history says wrote the + file (#386). It is asked only when the file alone leaves the prompt open + (ambiguous or unresolved), and the anchored result is kept only when it + names the positive (resolved, or a guider with no negative); its + graph-inferred sides are then labelled ``recorded-run``. ``recorded_texts`` + (sampler id to the effective texts PromptManager recorded at execution in + that run) is used last: when the prompt is still open at one named sampler + whose positive text was recorded, the recorded texts are shown as + ``execution``. + Inspection routes resolve with ``INSPECTION_PROFILE``: bounded work, not graph size. Ingestion (scan, live monitor, admin import) calls the resolver directly with its default profile. ``resolution`` carries the closed @@ -599,6 +653,24 @@ def inspect_prompt_evidence( # Only to recognize overwritten PromptManager text (R1); never a role. canonical["workflow"] = workflow resolved = resolve_prompt_pair(canonical, profile=profile, file_name=file_name) + recorded = False + if resolved["state"] in _UNDECIDED and recorded_saver_id is not None: + anchored = resolve_prompt_pair( + canonical, + profile=profile, + file_name=file_name, + recorded_saver_id=recorded_saver_id, + ) + # Only a result whose samplers history's saver chose is about this file. + if anchored.get("recorded_anchor") is True: + if anchored["state"] not in _UNDECIDED and isinstance( + anchored["positive"], dict + ): + resolved, recorded = anchored, True + elif recorded_texts: + executed = _recorded_execution(anchored, recorded_texts) + if executed is not None: + return executed state = resolved["state"] result = {} for role in ("positive", "negative"): @@ -608,9 +680,10 @@ def inspect_prompt_evidence( and isinstance(side, dict) and (role == "negative" or side["text"].strip()) ): - result[role] = _side( - "resolved", side["text"], side["provenance"]["text_source"] - ) + source = side["provenance"]["text_source"] + if recorded and source == "graph-inferred": + source = RECORDED_RUN_SOURCE + result[role] = _side("resolved", side["text"], source) elif state == "resolved" and isinstance(side, dict): # A resolved pair withholds only an empty positive. result[role] = {**_side("unresolved"), "message": _EMPTY_POSITIVE} diff --git a/utils/prompt_pair_resolver.py b/utils/prompt_pair_resolver.py index 15200481..ae871f19 100644 --- a/utils/prompt_pair_resolver.py +++ b/utils/prompt_pair_resolver.py @@ -1601,6 +1601,11 @@ class _GraphResolver: several, an unreadable walk or no candidate sampler is ``None``: the file is then judged unanchored, never guessed (#381). """ + match = self.saver_match(file_name) + return self.saver_walk(match) if match is not None else None + + def saver_match(self, file_name: Any) -> Optional[str]: + """The one node whose ``filename_prefix`` names ``file_name``, or None.""" stem = _file_stem(file_name) if stem is None: return None @@ -1611,15 +1616,31 @@ class _GraphResolver: for node_id, node in self.graph.items() if self.writes(node, stem) ] - if len(matches) != 1: - return None - _, inputs = self.node(matches[0]) + except (_Unresolved, _Diverging): + return None + return matches[0] if len(matches) == 1 else None + + def recorded_saver_stages(self, node_id: str) -> Optional[list]: + """The candidate samplers of the save node recorded history names (#386). + + The node is walked exactly like a file-name match. A node the graph + lacks, a failing walk or no candidate sampler is ``None``: unanchored. + """ + if not isinstance(self.graph.get(node_id), dict): + return None + self.budget.reset(self.per_candidate_visits) + return self.saver_walk(node_id) + + def saver_walk(self, saver_id: str) -> Optional[list]: + """The listed samplers in one saver's linked inputs' ancestry, or None.""" + try: + _, inputs = self.node(saver_id) values = [ value for name, value in inputs.items() if name != "filename_prefix" and _link(value) ] - found = self.stages_from(matches[0], values) + found = self.stages_from(saver_id, values) except (_Unresolved, _Diverging): return None candidates = [ @@ -2256,6 +2277,7 @@ def resolve_prompt_pair( max_depth: Optional[int] = None, max_text_length: Optional[int] = None, file_name: Optional[str] = None, + recorded_saver_id: Optional[str] = None, ) -> Dict[str, Any]: """Resolve parsed metadata; a present API graph is authoritative over text. @@ -2268,9 +2290,14 @@ def resolve_prompt_pair( ``profile`` selects the bounds; explicit ``max_*`` keywords override it. ``file_name`` (inspection only, unanchored) anchors on the one saver that wrote that file (``_GraphResolver.saver_stages``) when exactly one matches. + ``recorded_saver_id`` (inspection only, unanchored) is the save node that + recorded history says wrote the file (#386); it is tried before the file + name, and a node that yields no candidate sampler anchors nothing. """ profile = _profile_with(profile, max_visits, max_depth, max_text_length) - result = _resolve(metadata, output_node_id, adapters, profile, file_name) + result = _resolve( + metadata, output_node_id, adapters, profile, file_name, recorded_saver_id + ) return _legacy_result(result) if profile.legacy else result @@ -2324,6 +2351,7 @@ def _resolve( adapters: Optional[Mapping[Tuple[str, int], OutputAdapter]], profile: ResolverProfile, file_name: Optional[str] = None, + recorded_saver_id: Optional[str] = None, ) -> Dict[str, Any]: if not isinstance(metadata, dict) or not metadata: return _result("absent", reason="No parsed prompt metadata") @@ -2379,11 +2407,12 @@ def _resolve( "unresolved", reason=str(error), reason_code=error.code, node=error.node ) else: - anchored = ( - resolver.saver_stages(file_name) - if semantics is not None and file_name is not None - else None - ) + anchored, by_record = None, False + if semantics is not None and recorded_saver_id is not None: + anchored = _recorded_anchor(resolver, str(recorded_saver_id), file_name) + by_record = anchored is not None + elif semantics is not None and file_name is not None: + anchored = resolver.saver_stages(file_name) sampler_ids = anchored or [ node_id for node_id, node in normalized.items() @@ -2393,6 +2422,34 @@ def _resolve( ] if semantics is not None and anchored is None: sampler_ids = resolver.final_stages(sampler_ids) + if semantics is not None and recorded_saver_id is not None: + # The caller can tell whether history's saver chose the samplers. + return { + **_judge(resolver, sampler_ids, normalized, profile), + "recorded_anchor": by_record, + } + return _judge(resolver, sampler_ids, normalized, profile) + + +def _recorded_anchor( + resolver: "_GraphResolver", saver_id: str, file_name: Optional[str] +) -> Optional[list]: + """The samplers of the saver history recorded, or None (#386). + + A file name that names a different saver contradicts history: no anchor. + """ + named = resolver.saver_match(file_name) if file_name is not None else None + if named is not None and named != saver_id: + return None + return resolver.recorded_saver_stages(saver_id) + + +def _judge( + resolver: "_GraphResolver", + sampler_ids: list, + normalized: dict, + profile: ResolverProfile, +) -> Dict[str, Any]: if not sampler_ids: return _no_sampler(normalized, profile) if len(sampler_ids) > 1: diff --git a/utils/review_capture.py b/utils/review_capture.py index 1859dee7..8fadab72 100644 --- a/utils/review_capture.py +++ b/utils/review_capture.py @@ -40,6 +40,19 @@ while it is pending and undecided (never beside a decision, see profile the item recorded (``captured_review(..., item_profile(evidence))``). The inspection profile only ever produces review choices; ingestion's own links stay on the default profile, which never loads the table (D1). + +The saver anchor (#390): the inspection profile resolves a file with its +name, as the workspace inspector does since #383, so the one saver whose +``filename_prefix`` names the file anchors its branch. When that anchor alone +resolves a file the default profile left ambiguous and nothing links +(``saver_anchored``), the scan queues it with the anchor's pair as its one +candidate (evidence ``anchor: "saver"``), never linking it: a file never +falls through both the link and the queue (maintainer ruling). No match, +several matches or an unfinished walk anchor nothing and the file is queued +as before. Apply re-resolves with the file's name, so only the anchored +branch can be applied. ``REVIEW_RULE_VERSION`` is folded into inspection +item versions, so inspection items recorded before the anchor are recorded +again by the next scan. """ from __future__ import annotations @@ -104,10 +117,16 @@ MAX_CANDIDATE_DEPTH = 8 # The resolver profile an item's evidence was built with (``profile``). DEFAULT = "default" INSPECTION = "inspection" +# The ``anchor`` an item recorded from the saver anchor names (#390). +SAVER_ANCHOR = "saver" +ANCHOR_REASON = "The saver that wrote this file names its branch" # Inspection item versions: above every plausible ``RESOLVER_VERSION`` and, # with 40 bits of the folded digest, below 2**53 (exact as a JSON number). INSPECTION_VERSION_BASE = 1 << 40 _INSPECTION_VERSION_HEX = 10 +# How the scan and Apply use the inspection profile, folded into inspection +# item versions. 1: they resolve with the file's name (the saver anchor, #390). +REVIEW_RULE_VERSION = 1 # Above this many distinct node classes the whole table is the fingerprint. @@ -154,38 +173,92 @@ def inspection_review_version(metadata: Any = None) -> int | None: """The ``resolver_version`` of an item recorded from the inspection profile. Folds the profile with its budgets, ``RESOLVER_VERSION``, - ``INSPECTION_RESOLVER_VERSION`` and ``semantics_version(metadata)``, so - a change of either resolver, of the profile's bounds or of the table - about the graph's classes moves it. None when the table cannot be loaded. + ``INSPECTION_RESOLVER_VERSION``, ``REVIEW_RULE_VERSION`` and + ``semantics_version(metadata)``, so a change of either resolver, of how + review uses the profile, of the profile's bounds or of the table about + the graph's classes moves it. None when the table cannot be loaded. """ table = semantics_version(metadata) if table is None: return None profile = repr(astuple(INSPECTION_PROFILE)) seed = f"{profile}:{RESOLVER_VERSION}:{INSPECTION_RESOLVER_VERSION}:{table}" + if REVIEW_RULE_VERSION: # 0 is the seed items had before the rule existed + seed += f":review-rule-{REVIEW_RULE_VERSION}" folded = hashlib.sha256(seed.encode("utf-8")).hexdigest() return INSPECTION_VERSION_BASE + int(folded[:_INSPECTION_VERSION_HEX], 16) -def resolve_inspection(metadata: dict) -> dict: - """``metadata`` resolved with the inspection profile (bounded work).""" - return resolve_prompt_pair(metadata, profile=INSPECTION_PROFILE) +def resolve_inspection(metadata: dict, file_name: str | None = None) -> dict: + """``metadata`` resolved with the inspection profile (bounded work). + + ``file_name`` (the file's last path part) anchors the file on the one + saver that wrote it, as in the workspace inspector (#390). + """ + return resolve_prompt_pair( + metadata, profile=INSPECTION_PROFILE, file_name=file_name + ) -def inspection_choice(metadata: dict) -> dict | None: - """The inspection profile's ambiguous result when it offers a choice. +def inspection_choice(metadata: dict, file_name: str | None = None) -> dict | None: + """The inspection profile's result when the file is a review case. The scan asks only for a file the default profile found ambiguous - without a choice that nothing links. + without a choice that nothing links. The result is ambiguous with a + choice, or ``resolved`` when ``file_name`` anchors the file on its saver + and only that anchor resolves it (``saver_anchored``): the item then + offers the anchor's pair. None otherwise. """ if inspection_review_version(metadata) is None: return None - resolution = resolve_inspection(metadata) + resolution = resolve_inspection(metadata, file_name) + if resolution.get("state") == "resolved": + return _anchor_of(metadata, file_name, resolution) if resolution.get("state") != "ambiguous" or not reviewable(resolution): return None return resolution +def _anchor_of(metadata: Any, file_name: str | None, named: dict) -> dict | None: + """``named`` tagged ``anchor: "saver"`` when only the saver anchor + resolves the file, to a pair with a positive text to choose; else None. + + ``named`` is ``metadata`` resolved with ``file_name``; the profile runs + once more without the name only when it is resolved and choosable. + """ + if not file_name or named.get("state") != "resolved": + return None + try: + offered = choosable(_projection(named)) + except (KeyError, TypeError): + return None + if not offered or resolve_inspection(metadata).get("state") == "resolved": + return None + return {**named, "anchor": SAVER_ANCHOR} + + +def saver_anchored(metadata: Any, file_name: str | None) -> bool: + """True when the saver that wrote ``file_name`` resolves its branch (#390). + + The inspection profile resolves ``metadata`` with the file's name, and + without it the file does not resolve: the anchor alone decides it. A file + the default profile left ambiguous is then queued with the anchor's pair + as its one candidate. No name, an unusable table, no matching saver, + several, or an unfinished walk anchor nothing (False). The profile + without the name runs only when the named run resolved. + """ + return _anchored(metadata, file_name) is not None + + +def _anchored(metadata: Any, file_name: str | None) -> dict | None: + """The anchored result ``saver_anchored`` describes, else None.""" + if not file_name or not isinstance(metadata, dict): + return None + if inspection_review_version(metadata) is None: + return None + return _anchor_of(metadata, file_name, resolve_inspection(metadata, file_name)) + + def item_profile(evidence: Any) -> str: """The profile an item's stored evidence was built with.""" if isinstance(evidence, dict) and evidence.get("profile") == INSPECTION: @@ -246,7 +319,9 @@ def record_ambiguous_file( """ versions = item_versions(scanned[0] if scanned is not None else None) spelled_path, queued = _queued_spelling(db, root, relative, versions) - if queued is not None: + if queued is not None and not ( + inspection and _default_item_to_anchor(db, queued, scanned, relative) + ): return _already_queued(db, *queued, versions) epoch = db.get_execution_epoch() try: @@ -272,24 +347,24 @@ def record_ambiguous_file( if resolution is None or resolution.get("state") != "ambiguous": return "skipped" profile, version = DEFAULT, RESOLVER_VERSION - if not reviewable(resolution): - version = inspection_review_version(captured[0]) if inspection else None - if version is None: + named = _captured_inspection(captured[0], relative, scanned) if inspection else None + anchored = named is not None and named[0].get("anchor") == SAVER_ANCHOR + if anchored: + # Only the saver anchor resolves it (#390): offer that one pair. + profile, version = INSPECTION, named[1] + resolution = { + "state": "ambiguous", + "reason": ANCHOR_REASON, + "candidates": [named[0]], + } + elif not reviewable(resolution): + if named is None: return "untraced" - profile = INSPECTION - if ( - scanned is not None - and scanned[1] is not None - and _same_records(captured[0], scanned[0]) - ): - resolution = scanned[1] # the same records: no second run - else: - # Heavier, so only for a file ambiguous with nothing to choose. - resolution = resolve_inspection(captured[0]) + profile, (resolution, version) = INSPECTION, named if resolution.get("state") != "ambiguous" or not reviewable(resolution): return "untraced" evidence, count, limited = review_evidence( - resolution, profile, version, captured[0] + resolution, profile, version, captured[0], anchored=anchored ) try: inserted = db.record_review_item( @@ -313,6 +388,60 @@ def record_ambiguous_file( return "limited" if limited else "recorded" +def _default_item_to_anchor(db, queued: tuple, scanned, relative: str) -> bool: + """True when the bytes' only pending item is a default item recorded + before #390 and the saver anchor alone resolves the scanned records. + + The file is then captured again so the anchored item supersedes that + pending, undecided item. Decided bytes and bytes already queued under + the current inspection version are never re-read. + """ + if scanned is None or scanned[0] is None: + return False + version = inspection_review_version(scanned[0]) + if version is None or db.review_pending_at(*queued, version): + return False + if not db.review_pending_at(*queued, RESOLVER_VERSION): + return False + return _anchored(scanned[0], _file_name(relative)) is not None + + +def _captured_inspection(metadata: dict, relative: str, scanned) -> tuple | None: + """``(resolution, item_version)`` of the captured records under the + inspection profile, anchored on the file's name; None when the table is + unusable. + + A result tagged ``anchor: "saver"`` means the saver anchor alone + resolves the file (#390); any other resolved result is returned as + ``unresolved`` (no review case, as before). The scan's result is reused + when the capture holds the same records (``inspection_choice`` applies + the same rule). + """ + version = inspection_review_version(metadata) + if version is None: + return None + if ( + scanned is not None + and scanned[1] is not None + and _same_records(metadata, scanned[0]) + ): + return scanned[1], version # the same records: no second run + name = _file_name(relative) + resolution = resolve_inspection(metadata, name) + if resolution.get("state") == "resolved": + # Resolved without the anchor too: no review case, as before. + resolution = _anchor_of(metadata, name, resolution) or { + **resolution, + "state": "unresolved", + } + return resolution, version + + +def _file_name(relative: str) -> str: + """The last part of a ``/``-separated relative path (the saved name).""" + return relative.rsplit("/", 1)[-1] + + def _already_queued(db, locator: str, size: int, mtime_ns: int, versions) -> str: """``duplicate`` while an existing item is pending, else ``decided``.""" for version in versions: @@ -354,14 +483,19 @@ def _queued_spelling(db, root, relative: str, versions) -> tuple[str, tuple | No return spelled, None -def captured_review(store, capture, profile: str = DEFAULT) -> tuple[dict, dict] | None: +def captured_review( + store, capture, profile: str = DEFAULT, file_name: str | None = None +) -> tuple[dict, dict] | None: """``(metadata, resolution)`` of the capture's own bytes, else None. ``metadata`` is what the scan hands the ingester: prompt and workflow JSON-decoded and ``file_info`` (size, dimensions, format) from the capture. ``resolution`` comes from ``profile`` (``DEFAULT`` or - ``INSPECTION``). None when the capture is gone or carries no prompt - metadata. + ``INSPECTION``; the latter anchored on ``file_name``, the file's own + name, as the item was recorded). With ``file_name`` (Apply), a default + result its saver anchor alone resolves (``saver_anchored``) is that + anchored result, so only the anchored branch is still offered. None + when the capture is gone or carries no prompt metadata. """ if store.acquire(capture.token) is None: return None @@ -389,8 +523,15 @@ def captured_review(store, capture, profile: str = DEFAULT) -> tuple[dict, dict] info["format"] = read.format canonical["file_info"] = info if profile == INSPECTION: - return canonical, resolve_inspection(canonical) - return canonical, resolve_prompt_pair(canonical) + named = resolve_inspection(canonical, file_name) + return canonical, _anchor_of(canonical, file_name, named) or named + resolution = resolve_prompt_pair(canonical) + if file_name is not None and resolution.get("state") == "ambiguous": + anchored = _anchored(canonical, file_name) + if anchored is not None: + # Its saver names its branch (#390): only that one is offered. + return canonical, anchored + return canonical, resolution def reviewable(resolution: Any) -> bool: @@ -416,12 +557,14 @@ def review_evidence( profile: str = DEFAULT, version: int | None = None, metadata: Any = None, + anchored: bool = False, ) -> tuple[dict, int, bool]: """``(evidence, candidate_count, limited)`` for one ambiguous result. Inspection evidence also names its ``profile`` and ``semantics``, the table fingerprint of ``metadata``'s graph it was resolved with; - ``version`` is the item's version. + ``version`` is the item's version. ``anchored`` evidence (the saver + anchor's one pair, #390) also names ``anchor: "saver"``. """ base = { "resolver_version": RESOLVER_VERSION if version is None else version, @@ -430,6 +573,8 @@ def review_evidence( if profile == INSPECTION: base["profile"] = INSPECTION base["semantics"] = semantics_version(metadata) + if anchored: + base["anchor"] = SAVER_ANCHOR candidates: list[dict] = [] seen: set[str] = set() total = 0 diff --git a/utils/run_keys.py b/utils/run_keys.py index 05d74a85..f53df44d 100644 --- a/utils/run_keys.py +++ b/utils/run_keys.py @@ -51,6 +51,53 @@ def _links(inputs: Any) -> List[str]: return [value[0] for value in inputs.values() if is_link(value)] +def upstream_closure( + graph: Dict[str, Any], starts: List[str], limit: Optional[int] = None +) -> Optional[Dict[str, Any]]: + """``{node_id: [class_type, inputs]}`` of ``starts`` and everything upstream. + + None when a node or an upstream link's node is missing, or the closure + exceeds ``limit`` nodes. + """ + closure: Dict[str, Any] = {} + stack = list(starts) + while stack: + current = stack.pop() + if current in closure: + continue + node = graph.get(current) + if not isinstance(node, dict): + return None + if limit is not None and len(closure) >= limit: + return None + inputs = node.get("inputs") + closure[current] = [node.get("class_type"), inputs] + stack.extend(_links(inputs)) + return closure + + +def upstream_nodes(graph: Any, starts: List[str]) -> List[str]: + """Ids of ``starts`` and every node upstream of them that exists in ``graph``. + + Used to tell which nodes a job's outputs to execute need ("run selected + output" jobs leave the rest of the graph out). Missing nodes are skipped. + """ + if not isinstance(graph, dict): + return [] + found: Dict[str, None] = {} + stack = [str(node) for node in starts] + while stack: + current = stack.pop() + if current in found: + continue + node = graph.get(current) + if not isinstance(node, dict): + continue + found[current] = None + stack.extend(_links(node.get("inputs"))) + return list(found) + + def prompt_node_key(graph: Any, node_id: Any) -> Optional[str]: """Digest of ``node_id`` and its whole upstream closure, or None. @@ -62,20 +109,9 @@ def prompt_node_key(graph: Any, node_id: Any) -> Optional[str]: if not isinstance(graph, dict) or node_id is None: return None start = str(node_id) - closure: Dict[str, Any] = {} - stack = [start] - while stack: - current = stack.pop() - if current in closure: - continue - node = graph.get(current) - if not isinstance(node, dict): - return None - if len(closure) >= MAX_VISITS: - return None - inputs = node.get("inputs") - closure[current] = [node.get("class_type"), inputs] - stack.extend(_links(inputs)) + closure = upstream_closure(graph, [start], MAX_VISITS) + if closure is None: + return None try: encoded = json.dumps( [RUN_KEY_VERSION, start, closure], diff --git a/utils/usage_tracking.py b/utils/usage_tracking.py index 0365e252..6fb6d39c 100644 --- a/utils/usage_tracking.py +++ b/utils/usage_tracking.py @@ -16,7 +16,9 @@ it never counts. Servers without a ``prompt_queue`` fall back to counting at submit time. Text from known pure string nodes is resolved at queue time too. Anything else -(PromptSearchList batches, unknown nodes) is counted by the node as it executes. +(PromptSearchList batches, unknown nodes) is counted by the node as it executes, +and when ComfyUI serves such a node from its cache, the job replays what that +execution counted (``cached_runs``, #347). """ import threading @@ -167,13 +169,22 @@ def queued_pair_runs(graph: Any) -> List[Tuple[str, str, str]]: def find_pair_id(db: Any, positive: str, negative: str) -> Optional[int]: """Find a saved pair without creating or overwriting its metadata.""" - positive_row = db.get_prompt_by_hash(generate_prompt_hash(positive)) - negative_row = ( - db.get_prompt_by_hash(generate_prompt_hash(negative)) - if negative.strip() - else None + return find_pair_id_by_hashes( + db, + generate_prompt_hash(positive), + generate_prompt_hash(negative) if negative.strip() else None, ) - if not positive_row or (negative.strip() and not negative_row): + + +def find_pair_id_by_hashes( + db: Any, positive_hash: str, negative_hash: Optional[str] +) -> Optional[int]: + """The saved pair of two prompt hashes; ``None`` negative is the empty one.""" + positive_row = db.get_prompt_by_hash(positive_hash) + negative_row = ( + db.get_prompt_by_hash(negative_hash) if negative_hash is not None else None + ) + if not positive_row or (negative_hash is not None and not negative_row): return None row = ( db.model.get_connection() diff --git a/web/admin.html b/web/admin.html index f80b3565..e55fbda6 100644 --- a/web/admin.html +++ b/web/admin.html @@ -164,26 +164,31 @@ class="px-2.5 py-1 bg-pm-surface border border-pm hover:bg-pm-hover text-pm text-xs font-medium rounded-pm-sm transition-colors"> Export - - - - - + - + Loading... @@ -367,61 +374,13 @@