Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b5b643ba7a | ||
|
|
07474afd46 | ||
|
|
9ec07abe47 |
@@ -3,10 +3,17 @@
|
||||
"strict": false,
|
||||
"contexts": [
|
||||
"py3.12 on ubuntu-latest",
|
||||
"py3.13 on ubuntu-latest",
|
||||
"py3.14 on ubuntu-latest",
|
||||
"py3.12 on windows-latest",
|
||||
"py3.13 on windows-latest",
|
||||
"py3.14 on windows-latest",
|
||||
"py3.12 on macos-latest",
|
||||
"py3.13 on macos-latest",
|
||||
"py3.14 on macos-latest",
|
||||
"coverage gate (85%)",
|
||||
"black and flake8",
|
||||
"frontend modules",
|
||||
"all tests",
|
||||
"viewerjs",
|
||||
"real ComfyUI compatibility",
|
||||
"bandit (high severity)"
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
name: Windows and macOS
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context
|
||||
# here is a required check. A red run is fixed at its cause before the workspace cutover (#334).
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
schedule:
|
||||
- cron: "17 3 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
# Every run gets its own group (run_id), so no main commit's run is ever cancelled or dropped as pending.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.run_id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PYTHONFAULTHANDLER: "1"
|
||||
|
||||
# Time limit rule as in test.yml: ceil(1.5 x the slowest of three recent green runs on main) + 3
|
||||
# minutes, never under 5. Measured 2026-10-26 in test.yml on runs 40257217681 (3be0ada),
|
||||
# 40240706568 (fb9134f), 40228167328 (f9835aa), before these legs moved here in #420:
|
||||
# slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 27
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [windows-latest, macos-latest]
|
||||
python-version: ["3.12", "3.13", "3.14"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run tests
|
||||
run: python -m tests.run_suite -v
|
||||
@@ -28,18 +28,10 @@ env:
|
||||
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
# 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
|
||||
# Windows and macOS moved to portability.yml (#420), which keeps this 27-minute limit; this
|
||||
# Linux-only matrix keeps it too until it is re-measured.
|
||||
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
|
||||
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
|
||||
# perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s,
|
||||
# frontend modules 23s / 19s / 29s -> 5 min (the floor)
|
||||
# all tests: a result check without setup, 5 min (the floor)
|
||||
#
|
||||
# Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend
|
||||
# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on
|
||||
# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set.
|
||||
# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
@@ -48,8 +40,8 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest]
|
||||
python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.12"]') || fromJSON('["3.12", "3.13", "3.14"]') }}
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
python-version: ["3.12", "3.13", "3.14"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
@@ -71,9 +63,6 @@ jobs:
|
||||
|
||||
comfyui:
|
||||
name: real ComfyUI compatibility
|
||||
# Merge queue, main and manual runs only. A job skipped by its condition reports Success, so this
|
||||
# required context never blocks a pull request; "all tests" fails if it is skipped anywhere else.
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
# Chromium and its system libraries come from the pinned Playwright image (tag = @playwright/test in
|
||||
# tools/vendor/package-lock.json; tests/vendor/browser-image.test.mjs checks it). The runner install
|
||||
@@ -232,10 +221,7 @@ jobs:
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
# The exact black and flake8 pins from requirements-dev.txt, so a new release cannot turn CI red.
|
||||
- run: pip install $(grep -E '^(black|flake8)==' requirements-dev.txt)
|
||||
- run: pip install black flake8
|
||||
- run: black --check .
|
||||
- run: flake8 .
|
||||
|
||||
@@ -249,8 +235,6 @@ jobs:
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
cache-dependency-path: tools/vendor/package-lock.json
|
||||
|
||||
- name: Check shipped extension and page modules parse
|
||||
run: |
|
||||
@@ -264,32 +248,3 @@ jobs:
|
||||
|
||||
- name: Run JS tests
|
||||
run: node --test tests/js/*.test.js
|
||||
|
||||
# The required context "all tests". A dropped matrix leg never reports a check at all, so the eight
|
||||
# legs that pull requests skip cannot be required by name; this gate requires them instead. Every
|
||||
# job must succeed; real ComfyUI must be skipped on a pull request and succeed on every other event.
|
||||
all-tests:
|
||||
name: all tests
|
||||
needs: [test, comfyui, coverage, lint, js-test]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Require every test job for this event
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
TEST_RESULT: ${{ needs.test.result }}
|
||||
COMFYUI_RESULT: ${{ needs.comfyui.result }}
|
||||
COVERAGE_RESULT: ${{ needs.coverage.result }}
|
||||
LINT_RESULT: ${{ needs.lint.result }}
|
||||
JS_RESULT: ${{ needs.js-test.result }}
|
||||
run: |
|
||||
queue_only=success
|
||||
if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi
|
||||
echo "event: $EVENT_NAME (real ComfyUI must be $queue_only)"
|
||||
echo "test matrix: $TEST_RESULT"
|
||||
echo "comfyui: $COMFYUI_RESULT"
|
||||
echo "coverage: $COVERAGE_RESULT"
|
||||
echo "lint: $LINT_RESULT"
|
||||
echo "js-test: $JS_RESULT"
|
||||
test "$TEST_RESULT" = success && test "$COVERAGE_RESULT" = success && test "$LINT_RESULT" = success && test "$JS_RESULT" = success && test "$COMFYUI_RESULT" = "$queue_only"
|
||||
|
||||
@@ -27,15 +27,6 @@ concurrency:
|
||||
# coverage shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox 7m57s/5m21s (tests
|
||||
# 7m08s/4m34s), webkit 8m16s/5m03s (tests 7m26s/3m58s); a cache miss adds up to 5m to npm ci, which
|
||||
# the 20-minute limits absorb.
|
||||
# Hangs (#425): a job killed at its limit keeps no log and skips its upload steps, so each browser
|
||||
# test step has its own timeout-minutes (18) below the job's 20. At the step limit, SIGTERM makes
|
||||
# tools/vendor/running-tests-reporter.mjs print the tests still running and hand over to Playwright's
|
||||
# interrupt, which writes the blob report; the always() uploads then run.
|
||||
# Measured on main (runs 40725289094, 40719375442, 40700439399): setup before the test step <= 52s,
|
||||
# uploads <= 35s; test steps up to 14m42s (webkit shard 1/2) and 13m05s for coverage (shard 1/3:
|
||||
# 12m53s, node units and the viewer suite before 11.8m of workflows), so coverage is the tight one
|
||||
# in its 18 minutes. This holds only when the npm cache hits: a cold cache adds up to 5m to npm ci,
|
||||
# and then the job limit can still end the job before the step limit does.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
@@ -88,7 +79,6 @@ jobs:
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
@@ -97,7 +87,6 @@ jobs:
|
||||
path: coverage/js/raw/
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
# Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
|
||||
- name: Upload Chromium and touch traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
@@ -176,8 +165,6 @@ jobs:
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
|
||||
# Merge queue, main and manual runs only (#420); pull requests run the Chromium/touch coverage shards.
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -222,9 +209,7 @@ jobs:
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
|
||||
# Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
@@ -234,9 +219,8 @@ jobs:
|
||||
retention-days: 14
|
||||
|
||||
# Branch protection requires the exact context "viewerjs". This job keeps that
|
||||
# name and fails unless every coverage shard and the coverage merge succeeded and
|
||||
# the browser shards succeeded, or, on a pull request only, were skipped (#420).
|
||||
# Any failure, cancellation or other skip fails it.
|
||||
# name and fails unless every frontend job above succeeded (every shard, the
|
||||
# coverage merge and every browser shard), including when one was cancelled or skipped.
|
||||
viewerjs:
|
||||
needs: [frontend-coverage, frontend-coverage-merge, browser-workflows]
|
||||
if: always()
|
||||
@@ -245,15 +229,11 @@ jobs:
|
||||
steps:
|
||||
- name: Require every frontend job to succeed
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
|
||||
MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }}
|
||||
WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
|
||||
run: |
|
||||
queue_only=success
|
||||
if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi
|
||||
echo "event: $EVENT_NAME (browser-workflows must be $queue_only)"
|
||||
echo "frontend-coverage: $COVERAGE_RESULT"
|
||||
echo "frontend-coverage-merge: $MERGE_RESULT"
|
||||
echo "browser-workflows: $WORKFLOWS_RESULT"
|
||||
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = "$queue_only"
|
||||
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
|
||||
@@ -13,11 +13,9 @@ repos:
|
||||
language: system
|
||||
types: [python]
|
||||
|
||||
# Fails at the severity CI enforces (code-quality.yml: -lll, high only), never stricter.
|
||||
# pre-commit appends the staged files, so no shell wrapper can drop the first one.
|
||||
- id: bandit
|
||||
name: bandit
|
||||
entry: bandit -lll -q
|
||||
entry: bash -c 'bandit "$@" -ll -q 2>&1; true'
|
||||
language: system
|
||||
types: [python]
|
||||
exclude: ^(tests/|reference/)
|
||||
|
||||
@@ -56,7 +56,7 @@ Both nodes include the complete PromptManager feature set:
|
||||
- **📊 Analytics**: Track prompt usage patterns and effectiveness over time
|
||||
- **🔬 Workflow Analysis**: Extract and analyze ComfyUI workflow data from PNG images
|
||||
- **📋 Metadata Viewer**: Standalone tool for analyzing ComfyUI-generated images
|
||||
- **🛠️ System Management**: Built-in diagnostics, backup/restore, and maintenance tools in System
|
||||
- **🛠️ System Management**: Built-in diagnostics, backup/restore, and maintenance tools
|
||||
- **🏷️ AI AutoTag**: Automatically tag your image collection using WD14 or JoyCaption vision models
|
||||
- **🔗 LoRA Manager Integration**: Import LoRA metadata and preview images from [ComfyUI-Lora-Manager](https://github.com/willmiao/ComfyUI-Lora-Manager)
|
||||
|
||||
@@ -206,11 +206,14 @@ match Classic and Focused.
|
||||
- **Responsive Design**: Works on desktop, tablet, and mobile devices
|
||||
- **Real-time Search**: Instant search results as you type
|
||||
- **Bulk Operations**: Edit multiple prompts simultaneously with batch tools
|
||||
- **Settings and System tools live in System**: the dashboard's **Settings** opens System › Overview, **Maintenance** opens System › Maintenance, **Backup DB** and **Restore DB** open System › Backup & export, and **Diagnostics** and **View Logs** open System › Diagnostics & logs, each in a new tab. System reviews every destructive operation before it runs it as an Activity job.
|
||||
- **Page options**: the list header's **Page options** keeps the prompt list's Infinite Scrolling default
|
||||
- **Dashboard tools**:
|
||||
- **Settings Panel**: Configure behavior and display options
|
||||
- **System Management**: Complete toolset for database operations:
|
||||
- **🔍 Metadata Viewer**: Launch standalone PNG analysis tool
|
||||
- **📸 Scan Images**: Bulk scan output directories for ComfyUI images (with a quick backup first)
|
||||
- **📸 Scan Images**: Bulk scan output directories for ComfyUI images
|
||||
- **💾 Backup/Restore**: Database backup and recovery operations
|
||||
- **🔧 Maintenance**: Database optimization and cleanup tools including orphaned prompt removal
|
||||
- **📋 Logs**: Comprehensive logging system with download capability
|
||||
- **🔍 Diagnostics**: System health checks and troubleshooting
|
||||
- **Export Tools**: Download prompts and metadata in various formats
|
||||
- **Progress Tracking**: Real-time progress indicators for long operations
|
||||
- **Error Handling**: Graceful error display with recovery suggestions
|
||||
@@ -312,12 +315,6 @@ Both nodes support the same metadata fields:
|
||||
|
||||
Both nodes will automatically save prompts to the database and link any generated images to the prompt.
|
||||
|
||||
- **Save to library** (`save_to_library`, on by default; "Save to library" in the Focused, Split and Pair editors):
|
||||
turn it off while iterating and PromptManager ignores that node's runs. The prompt is not saved, the run is not
|
||||
counted and its images are not linked (a scan skips them too), while the node's output is unchanged. Workflows
|
||||
saved before this switch existed load with it on. If any PromptManager node feeding a sampler's positive input
|
||||
has it off, none of that run's images are linked, because the image cannot show which sampler produced it.
|
||||
|
||||
### 🖼️ Using the Output-Wide Gallery
|
||||
|
||||
The comprehensive gallery system provides complete access to all your ComfyUI output:
|
||||
@@ -336,11 +333,14 @@ The comprehensive gallery system provides complete access to all your ComfyUI ou
|
||||
|
||||
#### **⚙️ Gallery Settings**
|
||||
|
||||
The gallery's **Settings** opens System › Appearance in a new tab: image quality, details on cards, collapsed viewer details, video autoplay/mute/loop, metadata loading and file paths are set there and apply to this page too.
|
||||
Configure your viewing experience in the settings panel:
|
||||
|
||||
The gallery's **Page options** keep the page's own settings: the thumbnail check at startup, lazy loading, items per page, grid columns, infinite scrolling, debug mode and API timeout. Reset puts back only these.
|
||||
|
||||
Thumbnail generation, the server thumbnail and browser cache clears, refreshing the file listing and finding duplicate images are in System › Sources & scans and System › Maintenance (Page options links both); each runs as an Activity job you can follow after closing the page.
|
||||
- **Display Settings**: Default view, items per page, grid columns, media info, infinite scrolling
|
||||
- **Performance Settings**: Image quality, lazy loading
|
||||
- **Video Settings**: Autoplay, mute by default, loop videos
|
||||
- **Thumbnail Management**: Generate optimized thumbnails for faster loading (runs on the configured worker threads; a run that stops at the per-run cap tells you how many remain)
|
||||
- **Find Duplicate Images**: Hashes every file on the worker threads with a progress bar; close the window and it keeps running, results wait in a pill
|
||||
- **Rescan Output Folder**: Drops the server-side listing cache and rewalks the folders
|
||||
|
||||
#### **🖼️ Image Viewing Experience**
|
||||
|
||||
@@ -398,7 +398,7 @@ Import existing ComfyUI images into your database:
|
||||
- Any errors or issues encountered
|
||||
5. **Access imported data** through the normal gallery interface
|
||||
|
||||
The scan reads PNG text chunks and A1111-style parameters, and, when `ffprobe` is installed, the prompt ComfyUI's video savers embed in MP4/MOV/WebM files. Files are read on the number of **worker threads** set in System › Sources & scans (default: half your cores), which makes a large library several times faster.
|
||||
The scan reads PNG text chunks and A1111-style parameters, and, when `ffprobe` is installed, the prompt ComfyUI's video savers embed in MP4/MOV/WebM files. Files are read on the number of **worker threads** set in Settings → Performance (default: half your cores), which makes a large library several times faster.
|
||||
|
||||
### 🏷️ AI AutoTag
|
||||
|
||||
@@ -414,7 +414,7 @@ Automatically tag your entire image collection using AI vision models:
|
||||
|
||||
1. **AutoTag (Batch Mode)**: Tag your entire collection automatically
|
||||
- Click **"🏷️ AutoTag"** in the admin dashboard
|
||||
- Choose your model type (FP16 or GGUF). A model that is not downloaded yet shows **Manage in System**: download (or unload) it in System › Auto-tagging, then reopen the dialog
|
||||
- Choose your model type (FP16 or GGUF)
|
||||
- Select how to handle already-tagged images:
|
||||
- **Skip images with existing tags**: Preserve your meticulous manual tagging work
|
||||
- **Re-tag all images**: Overwrite existing tags with fresh AI analysis
|
||||
@@ -445,7 +445,7 @@ Adjust the system prompt to match your tagging style:
|
||||
#### **Requirements**
|
||||
|
||||
- Sufficient VRAM for the chosen model (FP16 requires more, GGUF is lighter)
|
||||
- Models are downloaded in System › Auto-tagging
|
||||
- Models are downloaded automatically on first use
|
||||
- Feature requests and improvements are welcome!
|
||||
|
||||
### 📁 Folder Filter
|
||||
@@ -470,18 +470,18 @@ If you use [ComfyUI-Lora-Manager](https://github.com/willmiao/ComfyUI-Lora-Manag
|
||||
|
||||
#### **Enabling the Integration**
|
||||
|
||||
1. Open System › Integrations (the admin dashboard's **Settings** opens System)
|
||||
2. PromptManager auto-detects whether LoRA Manager is installed
|
||||
1. Open the admin dashboard and click **Settings**
|
||||
2. Scroll to the **Integrations** section — PromptManager will auto-detect if LoRA Manager is installed
|
||||
|
||||

|
||||
|
||||
3. Toggle the **LoraManager** switch to enable
|
||||
4. Optionally enable **Auto-inject trigger words** to automatically append LoRA trigger words when `<lora:name:weight>` is detected in your prompts
|
||||
5. Click **Save preferences** (requires a ComfyUI restart to take effect)
|
||||
5. Click **Save Settings** (requires a ComfyUI restart to take effect)
|
||||
|
||||
#### **Importing LoRA Data**
|
||||
|
||||
Click **Review import** in System › Integrations; Start runs the import as an Activity job:
|
||||
Click **Import LoRA Data** in the settings panel to start the import. A progress popup shows real-time status as each LoRA is processed:
|
||||
|
||||
- Reads LoRA Manager's own settings the way LoRA Manager does: from the folder in `LORA_MANAGER_SETTINGS_DIR`, from the plugin's `settings.json` in portable mode, or from your user config folder, and only the keys that say where its files are
|
||||
- Finds each LoRA's metadata in both storage modes: beside the model (the default) or in the central sidecar folder, and imports only LoRAs whose model file still exists in a LoRA folder (the review says how many metadata files it skipped)
|
||||
@@ -502,7 +502,7 @@ A CivitAI API key is **required if your library contains NSFW LoRAs** — CivitA
|
||||
|
||||
To add your key:
|
||||
1. Get your API key from [civitai.com/user/account](https://civitai.com/user/account)
|
||||
2. Paste it into the **CivitAI API key** field in System › Integrations
|
||||
2. Paste it into the **CivitAI API Key** field in the Integrations settings
|
||||
3. Save and re-import to download any previously skipped images
|
||||
|
||||
The key is sent only over HTTPS to CivitAI's own hosts. CivitAI serves images through a redirect to its storage host; PromptManager follows such a redirect only over HTTPS to a host on CivitAI's own `civitai.com`, `civitai.red` or `civitai.green` domains, at most three hops, and never sends the key with it. When downloads fail, the import's status line says how many; when none could be downloaded, it starts with a warning. The server log has one warning per failed image with the reason (HTTP status, refused redirect, size cap, unreadable image, timeout or network error).
|
||||
@@ -517,20 +517,23 @@ The comprehensive web interface provides:
|
||||
- **Filter Options**: Filter by category, tags, rating, and date
|
||||
- **Bulk Operations**: Select multiple prompts for batch editing
|
||||
- **Export Tools**: Download your prompt collection
|
||||
- **Settings**: System (new tab) for settings, maintenance, backups and diagnostics; **Page options** for the page's own display settings
|
||||
- **Settings Panel**: Configure auto-save and display options
|
||||
- **Diagnostics**: System health checks and troubleshooting
|
||||
|
||||
### 🔧 Database Maintenance
|
||||
|
||||
Keep your database optimized in System › Maintenance (the admin dashboard's **Maintenance** opens it in a new tab):
|
||||
Keep your database optimized with built-in maintenance tools:
|
||||
|
||||
- **Duplicate prompt text**: merge duplicate prompts while preserving images
|
||||
- **Optimize database**: reclaim unused SQLite space
|
||||
- **Missing-file links**: remove image records for missing files
|
||||
- **Unused prompt components**: remove prompts nothing uses
|
||||
- **Run health checks**: verify database integrity and relationships (the Health report)
|
||||
- **Database size and prompt counts**: System › Diagnostics & logs (Database) and the Overview's Statistics
|
||||
|
||||
Each operation opens a review that shows what it will change and the safety backup it takes, then runs as an Activity job.
|
||||
1. **Open the admin dashboard** and click **"🔧 Maintenance"**
|
||||
2. **Select maintenance operations**:
|
||||
- **Remove Duplicates**: Find and merge duplicate prompts while preserving images
|
||||
- **Vacuum Database**: Optimize database file size and performance
|
||||
- **Clean Orphaned Images**: Remove image records for missing files
|
||||
- **🚮 Prune Orphaned Prompts**: Remove prompts that have no linked images
|
||||
- **Check Consistency**: Verify database integrity and relationships
|
||||
- **Database Statistics**: Generate detailed database reports
|
||||
3. **Run selected operations** and monitor progress in real-time
|
||||
4. **Review results** showing detailed information about each operation
|
||||
|
||||
### Database Location
|
||||
|
||||
@@ -832,8 +835,8 @@ Before upgrading or changing this setting:
|
||||
1. Identify the database your current installation actually uses and confirm its
|
||||
library contents. Check both the configured path and the prior fallback
|
||||
`<extension>/prompts.db`; keep track of which existing file you intend to use.
|
||||
2. Preserve a consistent backup of the active database with
|
||||
System › Backup & export's Back up or SQLite's online backup API. Do not copy only
|
||||
2. Preserve a consistent backup of the active database with the dashboard's
|
||||
Backup/Restore backup action or SQLite's online backup API. Do not copy only
|
||||
the main file of a live SQLite database: committed data may still be in its
|
||||
`-wal` file. For a filesystem copy, stop ComfyUI cleanly first and preserve any
|
||||
remaining `-wal` file together with the main database.
|
||||
|
||||
+38
-511
@@ -181,12 +181,10 @@ check, so run it before pushing a new or deleted frontend file instead of
|
||||
waiting for the `frontend coverage` jobs.
|
||||
|
||||
The `viewerjs` check runs on every PR, including backend-only changes. It is an
|
||||
aggregate job. It requires the three `frontend coverage (shard N/3)` jobs and
|
||||
`frontend coverage (merge)` to succeed on every event. The four
|
||||
`page workflows (firefox|webkit, shard N/2)` jobs run only in the merge queue,
|
||||
on `main` and on `workflow_dispatch` (see "Pull requests and the merge queue").
|
||||
`viewerjs` requires them to be skipped on a pull request and to succeed
|
||||
everywhere else. Any other failed, cancelled or skipped job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
|
||||
aggregate job that passes only when every frontend job succeeds: the three
|
||||
`frontend coverage (shard N/3)` jobs, `frontend coverage (merge)`, and the four
|
||||
`page workflows (firefox|webkit, shard N/2)` jobs. A failed, cancelled or skipped
|
||||
job fails it. Every browser job runs with `PM_TEST_WORKERS: 3` (the public
|
||||
runners have 4 vCPUs). Each coverage shard runs `test:coverage:shard -- N/3`
|
||||
(shard 1 also runs the node units and the viewer suite, shard 3 the vendor pins
|
||||
and tooling tests) and uploads its raw V8 files as `coverage-raw-N` (kept 3
|
||||
@@ -262,31 +260,6 @@ watchdog dump, is a failure like any other: read the dump, find the blocking
|
||||
call, fix it. Never raise a limit or rerun to get green; a limit changes only
|
||||
with new measurements written next to it.
|
||||
|
||||
Browser jobs (#425): GitHub keeps no log of a job killed at its limit and skips
|
||||
its upload steps, so the test step in each browser job in `viewerjs-check.yml`
|
||||
has its own `timeout-minutes` (18, below the job's 20). At the step limit
|
||||
GitHub sends SIGTERM, which would end Playwright without any report; under `CI`
|
||||
`tools/vendor/running-tests-reporter.mjs` catches the first one, prints
|
||||
`Still running when the run ended (SIGTERM): N` and one line per unfinished
|
||||
test (path relative to the suite's `rootDir`), then sends Playwright its own
|
||||
SIGINT, so the workers stop, the run ends interrupted and the blob report is
|
||||
written to `tools/vendor/test-results/blob/`. In the coverage job npm signals
|
||||
only `frontend-coverage.mjs`, which forwards SIGTERM and SIGINT to the child it
|
||||
is running (started in its own process group, so a group-wide signal still
|
||||
arrives once). Delivery is proven locally in the pinned image for a SIGTERM to
|
||||
the whole process group (`timeout -s TERM`). A SIGTERM to npm alone
|
||||
(`timeout --foreground`) reaches nothing: npm forwards it to the `sh -c` that
|
||||
runs the script, and that shell exits without passing it on. Delivery in CI,
|
||||
where the step runs through `docker exec`, is not yet observed; the next real
|
||||
step timeout will show it. That directory and the traces
|
||||
upload with `if: always()`; `npx playwright merge-reports` opens the blob.
|
||||
Each test keeps the 30 s default and there is no `globalTimeout`. Two limits
|
||||
of this: it assumes the npm cache hits (a cold cache adds up to 5 minutes to
|
||||
`npm ci`, and the job limit can then still win), and coverage shard 1 (12m53s
|
||||
measured) is the tight one in its 18-minute step. Local runs keep the `list`
|
||||
reporter only; reproduce with `CI=1 timeout -s TERM 40 npm run --prefix
|
||||
tools/vendor test:workflows -- --project=firefox`.
|
||||
|
||||
### After a push
|
||||
|
||||
Follow the run for the commit you pushed, never an older one:
|
||||
@@ -298,13 +271,6 @@ gh run view <run id> --json jobs # shows a job that alr
|
||||
gh run view --job <job id> --log-failed # that job's failed log, at once
|
||||
```
|
||||
|
||||
Python 3.13/3.14, real ComfyUI and Firefox/WebKit failures first show up in the
|
||||
merge queue, on the merge-group commit rather than the
|
||||
pushed one. Find that run with `gh run list --event merge_group --workflow test.yml`
|
||||
(or `viewerjs-check.yml`) and match its head branch
|
||||
(`gh-readonly-queue/main/pr-<N>-...`). Windows and macOS failures show up only
|
||||
after the merge, in `gh run list --workflow portability.yml`.
|
||||
|
||||
Start fixing as soon as a failed log shows the cause, but push the fix only after
|
||||
the run for that commit has finished: pull-request runs use `cancel-in-progress`,
|
||||
so a push cancels the running jobs and can hide a second failure.
|
||||
@@ -347,26 +313,6 @@ from the tests' package unload (`cleanup_package`), and it removes its own
|
||||
ComfyUI-Manager restart through `os.execv`, a closed Windows console, or
|
||||
ComfyUI Desktop stopping its server. SQLite's WAL journal covers those.
|
||||
|
||||
Jobs running at exit (#376): before any `atexit` callback, Python joins every
|
||||
executor thread (`concurrent.futures` registers that join with
|
||||
`threading._register_atexit`). The package registers its own callback there
|
||||
(`utils/exit_signal.install`, after `concurrent.futures`, so it runs first):
|
||||
it sets the exit signal and kills the running encoder processes
|
||||
(`shutdown_encoders`, the ffmpeg that never sees Ctrl+C on Windows). The call in
|
||||
flight then stops at its next safe point: `Job.cancelled` is True, so every
|
||||
check a job already makes between files stops it, `map_parallel` starts no
|
||||
further item (`ExitRequested`), the scan walk stops between folders, LoRA
|
||||
import staging stops before its next sidecar (also for the legacy scan) and a
|
||||
tagging model download stops before its next chunk where huggingface_hub takes
|
||||
`tqdm_class` (1.x removes its temp file; 0.x `hf_hub_download` does not take it
|
||||
and finishes the file). Xet transfers ignore that checkpoint, so the hook also
|
||||
calls hub's own `abort_xet_session` when huggingface_hub is loaded. The
|
||||
trigger-word cache load (small JSON sidecars) has no checkpoint. No check sits
|
||||
inside a library write. That hook is
|
||||
CPython-internal; without it (or when it is refused) `install` returns False and
|
||||
exit waits for the call in flight, as before. `tests/test_exit_cancel.py` runs a
|
||||
child process with three such calls running and requires its exit within 15 s.
|
||||
|
||||
Stop contract: `PromptTracker.shutdown(timeout)`,
|
||||
`ImageGenerationHandler.shutdown(timeout)` (restartable),
|
||||
`ImageGenerationHandler.close(timeout)` (final: `schedule()` refuses and a
|
||||
@@ -416,10 +362,6 @@ A spec performance budget gets two tests:
|
||||
`test_filter_budget` (Images filters, spec D12: p95 < 150 ms at 50,000 files
|
||||
and 20,000 links).
|
||||
|
||||
A hot SQLite read that returns many rows aggregates them into one step
|
||||
(`json_group_array`) instead of stepping once per row, so it does not wait for
|
||||
the GIL once per row while ComfyUI generates (#354, `get_image_link_matches`).
|
||||
|
||||
The `perf budget` job in `.github/workflows/test.yml` is the reference
|
||||
measurement: `ubuntu-latest`, one Python, no coverage, only the wall-clock tests.
|
||||
It runs on every push to `main` and on `workflow_dispatch`, never on pull
|
||||
@@ -577,95 +519,29 @@ Before changing paths, SQLite, streams, restore or cleanup:
|
||||
open with `FirstWriterHold`) and give peers that must all succeed the
|
||||
ordinary busy timeout; cross-thread handoffs wait on events with generous
|
||||
ceilings, never a short wall clock (#242).
|
||||
- Run native Windows CI early for these changes. Pull requests and the merge
|
||||
queue run only Linux, so start Windows and macOS on the branch with
|
||||
`gh workflow run portability.yml --ref <branch>`.
|
||||
Include a Windows impact and evidence note in the PR. All six Windows/macOS
|
||||
combinations in that run, every Linux leg and every applicable
|
||||
- Run native Windows CI early for these changes and include a Windows impact and
|
||||
evidence note in the PR. All twelve OS/Python combinations and every applicable
|
||||
exact-head check must pass before merge. Portable `ntpath`/`PureWindowsPath`
|
||||
controls do not establish native filesystem, locking or cleanup success.
|
||||
|
||||
## Pull requests and the merge queue
|
||||
|
||||
Each PR runs the full matrix once, in the merge queue (#420). The event decides
|
||||
what runs:
|
||||
|
||||
| Job | `pull_request` | `merge_group`, push to `main`, `workflow_dispatch` |
|
||||
| --- | --- | --- |
|
||||
| `py3.12 on ubuntu-latest` | runs | runs |
|
||||
| `py3.13 on ubuntu-latest`, `py3.14 on ubuntu-latest` | not created | run |
|
||||
| `real ComfyUI compatibility` | skipped | runs |
|
||||
| `coverage gate (85%)`, `black and flake8`, `frontend modules`, `bandit (high severity)` | run | run |
|
||||
| `frontend coverage (shard N/3)`, `frontend coverage (merge)` (vendor pins and tooling tests on shard 3) | run | run |
|
||||
| `page workflows (firefox\|webkit, shard N/2)` | skipped | run |
|
||||
| `all tests`, `viewerjs` (gates) | run | run |
|
||||
| `perf budget` | skipped | push and dispatch only (not in the queue) |
|
||||
| `py3.12-3.14 on windows-latest/macos-latest` (`portability.yml`) | not run | push to `main`, nightly, `workflow_dispatch` only; never required |
|
||||
|
||||
GitHub has one required set for pull requests and the queue. A job skipped by
|
||||
its `if:` reports Success. A dropped matrix leg never reports, so it would stay
|
||||
pending. That is why `all tests` (in `test.yml`) and `viewerjs` carry the
|
||||
guarantee:
|
||||
- On a pull request they require the subset to succeed and the queue-only jobs
|
||||
to be skipped.
|
||||
- On every other event they require every job to succeed.
|
||||
|
||||
So a merge group cannot turn green unless all three Linux Python legs, real
|
||||
ComfyUI and the four browser shards passed.
|
||||
|
||||
Windows and macOS (Python 3.12-3.14) are out of the blocking path for now
|
||||
(maintainer, 2026-10-28). They run in `portability.yml` on every push to `main`,
|
||||
nightly and on demand, and no context there is required. A red portability run
|
||||
is still a failure: fix it at its cause, not by rerunning, and have every one
|
||||
green before the workspace cutover (#334). `tests/vendor/ci-workflow.test.mjs` pins the
|
||||
event split and runs both gate scripts for each event.
|
||||
|
||||
## Required merge checks
|
||||
|
||||
`.github/branch-protection.json` records the required checks:
|
||||
`py3.12 on ubuntu-latest`, `coverage gate (85%)`, `black and flake8`,
|
||||
`frontend modules`, `all tests`, `viewerjs`, `real ComfyUI compatibility` and
|
||||
`bandit (high severity)`. Each one reports on a pull request.
|
||||
The same set lives in two places, and both must change together: the `main`
|
||||
ruleset (25960526, which also holds the merge queue) and the classic branch
|
||||
protection on `main`, which the manifest mirrors. GitHub requires the union of
|
||||
the two, so a context left in either one blocks every PR that does not report it.
|
||||
`.github/branch-protection.json` records the required checks: all twelve Python/OS
|
||||
combinations, Python coverage, formatting, frontend modules, `viewerjs`, `real ComfyUI compatibility`,
|
||||
and `bandit (high severity)`.
|
||||
The branch must be current with main, and administrators follow the same checks.
|
||||
It does not require an additional human approval.
|
||||
|
||||
A repository administrator applies the configuration after the new workflow is
|
||||
passing, then verifies it:
|
||||
|
||||
```sh
|
||||
gh api repos/ComfyAssets/ComfyUI_PromptManager/rulesets/25960526 \
|
||||
| jq '.rules[] | select(.type == "required_status_checks") | [.parameters.required_status_checks[].context]'
|
||||
```
|
||||
|
||||
A PUT replaces the whole `rules` array, so edit the current ruleset in place.
|
||||
Keep the `deletion`, `non_fast_forward` and `merge_queue` rules, and give every
|
||||
context `integration_id` 15368 (GitHub Actions), so no other integration can
|
||||
post a required check. Save the current ruleset first; it is the rollback:
|
||||
|
||||
```sh
|
||||
gh api repos/ComfyAssets/ComfyUI_PromptManager/rulesets/25960526 > ruleset-before.json
|
||||
jq '{name, target, enforcement, conditions, bypass_actors, rules: [.rules[]
|
||||
| if .type == "required_status_checks"
|
||||
then .parameters.required_status_checks = [$ARGS.positional[] | {context: ., integration_id: 15368}]
|
||||
else . end]}' ruleset-before.json --args "py3.12 on ubuntu-latest" "..." > ruleset-new.json
|
||||
gh api --method PUT repos/ComfyAssets/ComfyUI_PromptManager/rulesets/25960526 --input ruleset-new.json
|
||||
```
|
||||
|
||||
Afterwards, read it back. Check that the `merge_queue` rule is still there and
|
||||
that every context keeps its `integration_id`. Then apply the manifest to the
|
||||
classic protection and read both back:
|
||||
|
||||
```sh
|
||||
gh api --method PUT repos/ComfyAssets/ComfyUI_PromptManager/branches/main/protection \
|
||||
--input .github/branch-protection.json
|
||||
gh api repos/ComfyAssets/ComfyUI_PromptManager/branches/main/protection/required_status_checks
|
||||
gh api repos/ComfyAssets/ComfyUI_PromptManager/rules/branches/main
|
||||
```
|
||||
Update the manifest whenever
|
||||
required job names change. Keep the frontend check
|
||||
|
||||
Update the manifest whenever required job names change. Keep the frontend check
|
||||
free of path filters so a required check is never silently skipped.
|
||||
|
||||
## Unified workspace
|
||||
@@ -918,25 +794,6 @@ unattributed stage, when it is a plain first stage: not a detailer, no prompt ov
|
||||
set, and no listed sampler in its own latent ancestry (else it stays unattributed) (`tests/test_resolver_saver_anchor.py`, shapes in
|
||||
`tests/saver_anchor_graphs.py` from the maintainer's `gpu_unlock_bench`, dataset and
|
||||
ANIMA runs; browser parity through the `anchor-*` cases, which carry `__file_name`).
|
||||
Recorded run history (#386): both inspect routes (Images and a history occurrence)
|
||||
ask `database.run_attribution.recorded_output` inside the file's stat window. It
|
||||
answers only for a file that exactly one recorded run reported and that was written
|
||||
during that run (the #342 rules): the one output node that run reported it from (none
|
||||
when several did) and, per sampler anchor, the effective texts PromptManager recorded
|
||||
at execution (a role only inferred from the graph is none). Only when the file alone
|
||||
leaves the prompt open (`ambiguous` or `unresolved`) the inspector resolves again
|
||||
anchored on that node (`recorded_saver_id`, walked like a file-name match; a file
|
||||
name naming a different saver contradicts history and anchors nothing), and keeps
|
||||
that result only when history's saver chose its samplers (`recorded_anchor`) and it
|
||||
names the positive (resolved, or a guider with no negative); its `graph-inferred`
|
||||
sides read `recorded-run` ("Recorded run" badge, ranked below every "Workflow
|
||||
inferred" source). Still open at the one sampler that saver leads to, with that
|
||||
sampler's positive text recorded (a language model's output feeding PromptManager),
|
||||
the recorded texts show as `execution`. Outputs and texts are read in one
|
||||
transaction. A result the file resolves alone never changes; a lookup failure is
|
||||
logged and judges the file alone; local Inspect and ingestion are unchanged
|
||||
(`tests/test_recorded_saver_anchor.py`, `tests/test_run_attribution.py`,
|
||||
`tests/test_api_history_outputs.py`).
|
||||
|
||||
`tools/inspection_corpus.py --root DIR --out JSONL [--workers N] [--baseline JSONL]
|
||||
[--summary TXT] [--code TREE]` measures a media folder read-only against the #240
|
||||
@@ -1369,7 +1226,7 @@ How the earlier workspace Images filters moved (old → new). The legacy `/galle
|
||||
| Folder | "Refine this view" → Folder (D14); shown in the scope chip |
|
||||
| Media (All media / Images / Videos) | "Refine this view" → Media (D14) |
|
||||
| Sort: Newest first / Oldest first / Filename A–Z / Filename Z–A | toolbar sort: Recently added / Oldest first / Name A–Z / Name Z–A, plus Highest rated (D13, D14) |
|
||||
| (none) | toolbar category (the effective category, D141) and "Refine this view" → Tags, through the linked positive (D13, D14) |
|
||||
| (none) | toolbar category and "Refine this view" → Tags, through the linked positive (D13, D14) |
|
||||
| Refresh | Import → "Scan an output folder" → Rescan (D15) |
|
||||
| Grid / List / Masonry, Comfortable / Large / Compact | the same choices in the concept filter row |
|
||||
| Card image → side panel; preview icon → viewer | image → viewer; Info → side panel (D19) |
|
||||
@@ -1452,8 +1309,7 @@ Recorded deviations on this page (spec §8):
|
||||
`overscroll-behavior: contain`.
|
||||
- **D35**: counts pluralize ("1 positive prompt", "1 image", "1 more image loaded").
|
||||
- **D36**: the category list is the Library's, sorted alphabetically (case-insensitive)
|
||||
by the page, whatever order the server sends (amended by D141: Images adds its
|
||||
image categories).
|
||||
by the page, whatever order the server sends.
|
||||
- **D37**: production loads 40 images per page (the concept 12), so the continuation
|
||||
and toast use 40; after Escape production returns focus to the opening image.
|
||||
- **D44** (#249 PR E): Filters reads "Filters · N" while N of Source, Folder, Media and
|
||||
@@ -1539,46 +1395,15 @@ Recorded deviations on this page (spec §8):
|
||||
- **D39** (ruled 2026-10-21: fallback): a recipe-linked image shows its recipe's own
|
||||
category when set, otherwise the linked positive prompt's, and its recipe's own tags
|
||||
when it has any, otherwise the positive's; each field falls back on its own and the
|
||||
two tag sets are never merged. The Images Tags filter matches the positive's tags.
|
||||
The category filter matches the same inherited category as the panel (D142, #332), and
|
||||
an image's own category overrides both (D141, D146). The inspection response names each
|
||||
two tag sets are never merged. Set category (#259) writes the positive's category and
|
||||
the Images category and Tags filters match the positive's, so the panel now agrees
|
||||
with them whenever the recipe adds nothing. The inspection response names each
|
||||
field's source (`library.category_scope`, `library.tags_scope`: `pair`, `positive` or
|
||||
null); the category chip's visually hidden prefix (D21) says "Recipe category: " or
|
||||
"Positive prompt category: " from it, and tag chips are Tags filter buttons only when
|
||||
they are the positive's (#260 D2). History's media inspector uses the same rule. The
|
||||
concept shows the prompt's category and tags for every image and the concept
|
||||
fixture's recipes have neither, so the inspector comparison needs no mapping for it.
|
||||
- **D141** (#332, amends D13 and D36): the Images category filter matches each image's
|
||||
effective category: its own category (Images Set category, PR 3) when it has one,
|
||||
else its inherited one (D142), else none. An image whose own category differs never
|
||||
matches its prompt's category, and an unlinked or ambiguous image with an own
|
||||
category is listed under it (the members-only gate now applies to the Tags filter
|
||||
only). "Select all N matching" and saved views use the same value. The caption's
|
||||
positive-prompt count still counts the linked positives of the images shown. With a
|
||||
category set the listing reads the own categories once per source (a primary-key
|
||||
range); a failed read is the existing 500 `listing-failed`, never an unfiltered page.
|
||||
The link read then keeps only links whose raw recipe or positive category is the
|
||||
value (a superset of those that inherit it; Python decides), and reads the links of
|
||||
files whose own category is the value by path, so their rating and positive count.
|
||||
`test_filter_budget` seeds 2,000 own categories on its 50,000-file fixture.
|
||||
The option list adds the image categories of listed files
|
||||
(`GET /prompt_manager/images/categories`, PR 3).
|
||||
- **D142** (#332 MC4, amends D39's filter note): the inherited category is one rule,
|
||||
`inherited_category` in `utils/inspection_response.py`: a linked recipe's own
|
||||
category when it is not blank, else the linked positive's. Before #332 the filter
|
||||
matched the positive's even when the chip showed the recipe's. The link reads select
|
||||
the raw recipe and positive categories and never decide inheritance in SQL
|
||||
(`tests/test_api_workspace_image_categories.py` checks the filter and the chip agree
|
||||
on blank values stored as spaces, tabs, U+00A0 and BLOB). The concept has no recipe
|
||||
categories, so this is behavior only.
|
||||
- **D145** (#332, DC4): History's media inspector shows an image's own category with
|
||||
the chip rule of the side panel and viewer. Only the inspection request reads it, once,
|
||||
inside the History snapshot: the file is the symlink-resolved output folder plus the
|
||||
receipt's subfolder and filename, keyed with `image_category_key` over the configured
|
||||
output sources, as Images keys it. A temp output or a file no source holds has none;
|
||||
a failed read hides the chip (DC5). The outputs list and media requests read nothing
|
||||
new. The chip's hidden prefix reads "Image category: " for an own category; the
|
||||
visible origin text and its comparison are D146 (PR 3).
|
||||
- **D42**: the concept never shows a toast and a page alert together. Production keeps
|
||||
the concept's fixed toast (centred, 22px from the bottom, 12px at 700px and below)
|
||||
and, when an Import → Rescan toast leaves the Images discovery alert
|
||||
@@ -2614,14 +2439,6 @@ production hub `#/overview` is unchanged (D28); its System card opens
|
||||
the pressed section button and disposes the previous section's scope, so its reads,
|
||||
listeners and dialogs never outlive it.
|
||||
|
||||
System is the home for settings and maintenance (#330 System S13). The legacy pages keep
|
||||
their prompt list, tags, Scan Images, auto-tag batch and review and gallery browsing; their
|
||||
Settings, Diagnostics, View Logs, Maintenance, Backup DB, Restore DB, the gallery's
|
||||
thumbnail, cache, rescan and duplicate tools and the auto-tag model downloads open the
|
||||
System section that owns them in a new tab (`legacy-system-links.spec.mjs`). Each page
|
||||
keeps only its own settings in Page options. What left the legacy pages is listed in
|
||||
"Legacy System UI removed by S13".
|
||||
|
||||
Drafts (`system-state.mjs`) live for the page session (Q10). Save writes the dirty
|
||||
browser fields, then the host fields (`POST /prompt_manager/settings` with
|
||||
`expected_revision`, #220), then the runtime fields. A failed domain keeps its drafts and
|
||||
@@ -3383,16 +3200,12 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati
|
||||
again"; an Unload that found nothing loaded only reads the status again. At phone width
|
||||
the concept's 120px select cap cuts every engine label, so `#tag-model` alone keeps its
|
||||
natural width there and its row wraps the select under the text (`workspace.css`
|
||||
`/* D132 */`). The concept's cap cuts Existing tags too ("Replace after review"); by the
|
||||
maintainer's decision (#398) that parity is not kept: `#tag-existing` never shrinks for
|
||||
its row text and at phone width keeps its natural width beside the text (its row does
|
||||
not wrap; the text column narrows). 1600 is as the concept draws it. The comparison
|
||||
expects `selectsFit` true for every select and measures each row's `flexWrap` and
|
||||
whether its control sits under the text. Pinned by `auto-tagging matches the concept`
|
||||
(both themes and sizes), `auto-tagging at 390 wraps only the engine row`, `auto-tagging
|
||||
selects fit every option at 390/320 wide` (every option of both selects, both themes, no
|
||||
page scroll) and the mutation check (a wrapped row, a squeezed engine and Existing tags
|
||||
select) (system-tagging-concept.spec), and the model, Download, Unload and Preview
|
||||
`/* D132 */`); Existing tags stays as the concept draws it, cut at 390 as there. The
|
||||
comparison maps `selectsFit` only for the engine select and measures each row's
|
||||
`flexWrap` and whether its control sits under the text. Pinned by `auto-tagging matches
|
||||
the concept` (both themes and sizes), `auto-tagging at 390 wraps only the engine row`,
|
||||
`auto-tagging engine select fits at 320 wide` and the mutation check (it includes a
|
||||
wrapped row) (system-tagging-concept.spec), and the model, Download, Unload and Preview
|
||||
tests of workspace-system-tagging.spec.
|
||||
- **D133** (Auto-tagging, S11): a "Suggestions per image" row (`#tag-suggestions`,
|
||||
number 1–100, host key `autotag_suggestion_limit`) follows the two confidence rows:
|
||||
@@ -4166,30 +3979,7 @@ modals use the same routes and keep working.
|
||||
other: each file's identity is checked just before its unlink.
|
||||
- **Clear all logs** (`POST /logs/truncate`, also the admin page's Clear) empties the
|
||||
active log through its handler, deletes the rotated copies (Rule R) and clears the
|
||||
buffer; other files and links are kept. Response keys are unchanged. It deletes or
|
||||
empties no more than its confirmation showed (#389):
|
||||
- `/logs/files` rows carry `identity`, the string `"<size>:<mtime_ns>:<ino>"` (a
|
||||
string: `mtime_ns` and many inodes are beyond a browser's exact integers). The
|
||||
workspace sends `{"expected": [{name, identity}]}` with the active and rotated rows
|
||||
its dialog was built from. Under the files lock and every handler lock each listed
|
||||
rotated copy must still match (`_matches_review`) and the listed active log must be
|
||||
the same inode, or nothing changes: 409 `logs-changed`. Only listed copies are
|
||||
deleted; one that appeared after the confirmation is kept. A malformed list is 400.
|
||||
- Without a body (the legacy admin page, whose confirm lists nothing; also the
|
||||
workspace when the file list could not be read and the dialog said "deletes its
|
||||
rotated copies") every rotated copy present at the locked rescan is deleted.
|
||||
- Either way, before anything changes (best effort, not a cross-process lock: a
|
||||
write before our own last write, or any while file logging is off here, is not
|
||||
seen): a second writer of the active log (our
|
||||
handler's flushed stream is behind the file's size, or its file is now a rotated
|
||||
copy: another ComfyUI sharing the folder) is 409 `logs-foreign-writer`; an active
|
||||
log with more than one name (`st_nlink > 1` by `lstat` or the handler's `fstat`,
|
||||
NTFS included) is 409 `logs-hard-link`, never truncated (replacing it instead would
|
||||
rename a file our handler holds open, which Windows refuses). Each copy is
|
||||
`lstat`-checked again just before its unlink; one that changed stops the clear there
|
||||
and keeps it, the copies after it and the active log, which is emptied last.
|
||||
- The browser fixture grows the active log through the handler's own stream, since a
|
||||
second handle appending to it is exactly what Clear refuses.
|
||||
buffer; other files and links are kept. Response keys are unchanged.
|
||||
- **Runtime settings** (`GET/POST /logs/config`): the level (DEBUG…CRITICAL) and console
|
||||
logging apply until the server restarts; the retention values are shown read-only in
|
||||
the workspace and the API keeps accepting them.
|
||||
@@ -4236,9 +4026,6 @@ auto-tag / review", which the Images selection bar owns, not a System section. A
|
||||
is not `covered` names none (`—`). Test is the exact title of a test that runs, then its
|
||||
file. Anchors are line numbers on `main` at this PR.
|
||||
|
||||
S13 (#330) removed the legacy copy of the `covered` rows the legacy pages linked to System;
|
||||
"Legacy System UI removed by S13" below lists every removed element against its row.
|
||||
|
||||
`tests/vendor/legacy-coverage.test.mjs` fails on a row with any of these problems:
|
||||
- no owner, or a status outside the three;
|
||||
- a malformed Native control cell;
|
||||
@@ -4312,62 +4099,6 @@ list stay unscoped: any row's control proves them.
|
||||
| API timeout `apiTimeout` | `gallery-settings.js:37` | Legacy gallery (S12-10; read by no request) | kept in legacy | — | defaults are a complete, sane settings object (gallery-settings.test.js) |
|
||||
| config.json-only keys | `py/config.py:212–234,722–739`; `py/api/diagnostics_report.py:185` | S12 report (read-only) | covered | `diagnostics` · button · "Download diagnostic report" | test_report_lists_config_only_keys (test_api_diagnostics_report.py) |
|
||||
|
||||
### Legacy System UI removed by S13
|
||||
|
||||
System S13 (#330) points the legacy pages' System buttons at native System (a link that
|
||||
opens `workspace#/system/<id>` in a new tab) and removes the legacy copy of a capability
|
||||
only when its row above is `covered`. One row per removed capability: Capability is the
|
||||
exact Capability cell of its coverage row, Page is `admin` or `gallery`, Removed ids and
|
||||
Removed methods list what went (`—` for none), and Native test copies that row's Test
|
||||
cell. An id that stays on a System link is not listed, only its handler. The gallery's
|
||||
Default View Mode left Page options too: System › Appearance's Default layout is the one
|
||||
default-layout control. Nothing writes the legacy `defaultViewMode` any more; a stored
|
||||
value still picks the view the legacy gallery opens in (grid otherwise), and Page options
|
||||
Save and Reset keep it.
|
||||
|
||||
`tests/vendor/legacy-coverage.test.mjs` fails when a row cites a capability that is not
|
||||
`covered` or copies another test, when an id or method is listed twice for one page, or
|
||||
when a listed id is still on its page (`id="…"` in the markup or the script, a
|
||||
`getElementById`/`querySelector('#…')` lookup, an entry of a modal-id list) or a listed
|
||||
method is still defined or called (`.name(`). A deletion listed nowhere is caught by the
|
||||
kept-flow tests (`kept admin flows still work` in `legacy-system-links.spec.mjs` and the
|
||||
legacy page workflows), not by this check.
|
||||
|
||||
| Capability | Page | Removed ids | Removed methods | Native test |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Diagnostics run | admin | `#diagnosticsModal`, `#runDiagnosticsBtn`, `#diagnosticsContent` | `showDiagnosticsModal()`, `closeDiagnostics()`, `runDiagnostics()`, `renderDiagnostics()` | Run checks opens the shared Health report (workspace-system-diagnostics.spec.mjs) |
|
||||
| Synthetic image-link test | admin | `#testImageLinkBtn` | `testImageLink()` | Create test link confirms first and names the prompt (workspace-system-diagnostics.spec.mjs) |
|
||||
| View logs with level and limit | admin | `#logsModal`, `#logLevel`, `#logLimit`, `#refreshLogsBtn`, `#logsContainer` | `showLogsModal()`, `closeLogs()`, `refreshLogs()`, `displayLogs()` | level filter is minimum-level over all six choices (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log statistics | admin | `#logStatsTotal`, `#logStatsErrors`, `#logStatsWarnings`, `#logStatsLevel`, `#logStatsSize` | `loadLogStats()`, `formatBytes()` | log statistics show entries, errors, warnings, files and size (workspace-system-diagnostics.spec.mjs) |
|
||||
| Log files list and download | admin | `#downloadLogsBtn`, `#logFilesList` | `loadLogFiles()`, `downloadLogs()`, `downloadLogFile()` | each listed file downloads by its returned name and a link has no Download (workspace-system-diagnostics.spec.mjs) |
|
||||
| Clear/truncate logs | admin | `#clearLogsBtn` | `clearLogs()` | Clear all logs states both counts, confirms first and reads everything again (workspace-system-diagnostics.spec.mjs) |
|
||||
| Set log level (incl. CRITICAL), console logging | admin | `#updateLogConfigBtn`, `#setLogLevel`, `#consoleLogging` | `loadLogConfig()`, `updateLogConfig()` | runtime level (including CRITICAL) and console logging save and say they apply until the server restarts (workspace-system-diagnostics.spec.mjs) |
|
||||
| Image scan directories add/remove | admin | `#scanPathsList`, `#addScanPathBtn` | `renderScanPaths()`, `addScanPath()`, `removeScanPath()`, `_collectScanPaths()` | Add source validates on the server, then marks the host draft dirty; Save persists (workspace-system-sources.spec.mjs) |
|
||||
| Currently monitoring list + Refresh | admin | `#refreshMonitoringStatus`, `#monitoringStatus` | `updateMonitoringStatus()` | watching only with a live observer; restart pending otherwise (workspace-system-sources.spec.mjs) |
|
||||
| Search results auto-hide `result_timeout` | admin | `#resultTimeout` | — | host fields round-trip through /settings with revision (workspace-system.spec.mjs) |
|
||||
| Web UI display mode | admin | `#webuiDisplayMode` | — | host fields round-trip through /settings with revision (workspace-system.spec.mjs) |
|
||||
| Worker threads | admin | `#workerThreads`, `#workerThreadsValue` | `renderWorkerThreads()` | worker range max is cpu_count (workspace-system-sources.spec.mjs) |
|
||||
| LoraManager badge, enable, path, trigger words, CivitAI key, clear key | admin | `#loraDetectionBadge`, `#loraEnabled`, `#loraSettings`, `#loraManagerPath`, `#loraTriggerWords`, `#civitaiApiKey`, `#clearCivitaiKey` | `detectLoraManager()`, `_bindLoraEvents()`, `saveLoraSettings()` | LoRA rows show detected / enabled / cache-loaded separately (workspace-system-integrations.spec.mjs) |
|
||||
| LoRA data import with progress | admin | `#loraImportModal`, `#loraImportBtn`, `#loraImportStatus`, `#loraImportPercent`, `#loraImportBar`, `#loraImportProcessed`, `#loraImportImported` | `runLoraImport()` | Review import shows replacement, blocked items and network effects with the backup box; Start opens the job (D107) (workspace-system-integrations.spec.mjs) |
|
||||
| Maintenance cleanup_duplicates, vacuum, cleanup_orphaned_images, prune_orphaned_prompts | admin | `#maintenanceModal`, `#cleanup_duplicates`, `#vacuum`, `#cleanup_orphaned_images`, `#prune_orphaned_prompts`, `#runMaintenanceBtn`, `#maintenanceResults` | `showMaintenanceModal()`, `closeMaintenance()`, `runMaintenance()`, `renderMaintenanceResults()` | Apply reviewed operation closes, toasts, opens the job in Activity (workspace-system-maintenance.spec.mjs) |
|
||||
| Maintenance check_hash_duplicates, check_consistency | admin | `#check_hash_duplicates`, `#check_consistency` | — | Run health checks opens Health report and updates metric 4 (workspace-system.spec.mjs) |
|
||||
| Maintenance statistics (database info) | admin | `#statistics` | — | meta-grid shows real database and source status, the pinned Viewer.js version and the host renderers (workspace-system-diagnostics.spec.mjs) |
|
||||
| Maintenance select all / clear all / run | admin | `#selectAllMaintenanceBtn`, `#clearAllMaintenanceBtn` | `selectAllMaintenance()`, `clearAllMaintenance()` | plan dialog shows the server's scope, protected, backup policy and media lines with the backup box (D100); Cancel writes nothing (workspace-system-maintenance.spec.mjs) |
|
||||
| Database backup | admin | — | `backupDatabase()` | Back up downloads through the api client and revokes the object URL (D103) (workspace-system-backups.spec.mjs) |
|
||||
| Restore from file | admin | `#restoreModal`, `#restoreFileInput`, `#confirmRestore`, `#cancelRestore` | `showRestoreModal()`, `confirmRestore()` | Review restore stages the chosen file and shows the real summary with the backup box (D105) (workspace-system-backups.spec.mjs) |
|
||||
| Gallery thumbnail generation | gallery | `#generateThumbnailsBtn`, `#thumbnailStatus`, `#thumbnailProgress`, `#progressText`, `#progressPercent`, `#cancelThumbnailsBtn`, `#progressBar`, `#progressDetails`, `#progressETA` | `generateThumbnails()`, `generateThumbnailsWithProgress()` | Start closes the dialog, toasts, and opens the job in Activity (workspace-system-sources.spec.mjs) |
|
||||
| Default view mode, image quality | gallery | `#defaultViewModeSelect`, `#imageQualitySelect` | — | defaults apply to a new visit but not to the current view or a saved view (workspace-system.spec.mjs) |
|
||||
| Image info on cards; sidebar collapsed | gallery | `#showImageInfoToggle`, `#sidebarCollapsedByDefaultToggle` | — | added rows reach their consumers (workspace-system.spec.mjs) |
|
||||
| Video autoplay, mute, loop | gallery | `#videoAutoplayToggle`, `#videoMuteToggle`, `#videoLoopToggle` | — | video defaults reach the viewer (workspace-system.spec.mjs) |
|
||||
| Auto-load metadata, cache metadata, show file paths | gallery | `#autoLoadMetadataToggle`, `#cacheMetadataToggle`, `#showFilePathsToggle` | — | metadata off shows an explicit load and loads once on request (workspace-system.spec.mjs) |
|
||||
| Browser cache clear | gallery | `#clearCacheBtn` | `clearCache()` | Metadata cache reviews and clears only owned browser entries when there are some (D102) (workspace-system-maintenance.spec.mjs) |
|
||||
| Clear server thumbnails | gallery | — | — | Thumbnail cache opens the S5 dialog (workspace-system-maintenance.spec.mjs) |
|
||||
| Refresh file listing | gallery | `#rescanFolderBtn` | `rescanFolder()` | Refresh files calls /gallery/rescan and toasts its counts (workspace-system-sources.spec.mjs) |
|
||||
| Duplicate image scan, review, delete | gallery | `#scanDuplicatesBtn`, `#dupScanPill`, `#dupScanPillText`, `#duplicatesModal`, `#closeDuplicatesBtn`, `#duplicatesScanStatus`, `#dupProgressText`, `#dupProgressBar`, `#duplicatesContent`, `#duplicatesList`, `#duplicatesFooter`, `#duplicatesCount`, `#selectAllBtn`, `#removeDuplicatesBtn` | `scanDuplicates()`, `attachDuplicateStream()`, `resumeDuplicateScan()`, `applyDuplicateEvent()`, `renderDuplicateProgress()`, `setDuplicateButton()`, `showDuplicatePill()`, `hideDuplicatePill()`, `openDuplicatesModal()`, `showDuplicatesModal()`, `hideDuplicatesModal()`, `displayDuplicates()`, `selectAllDuplicates()`, `removeDuplicates()` | Find duplicates starts a job; its view links Review groups; Keep radios build a duplicate_files plan (D101) (workspace-system-maintenance.spec.mjs) |
|
||||
| Auto-tag model download | admin | `#autoTagDownloadModal`, `#downloadModelName`, `#downloadStatus`, `#downloadProgressPercent`, `#downloadProgressBar`, `#cancelDownloadBtn` | `downloadModel()`, `cancelDownload()` | Download starts the job and Activity shows it without Cancel (workspace-system-tagging.spec.mjs) |
|
||||
| Auto-tag model download | gallery | `#autoTagDownloadModal`, `#downloadModelName`, `#downloadStatus`, `#downloadProgressPercent`, `#downloadProgressBar`, `#cancelDownloadBtn` | `downloadModel()`, `cancelDownload()` | Download starts the job and Activity shows it without Cancel (workspace-system-tagging.spec.mjs) |
|
||||
| Unload model | gallery | — | `unloadModel()` | Unload is enabled only while a model is loaded, and a held gate's refusal names the operation (workspace-system-tagging.spec.mjs) |
|
||||
|
||||
### Browser cache ownership (#222)
|
||||
|
||||
The legacy gallery's Clear cache used to delete every localStorage key starting `pm_` or
|
||||
@@ -4385,15 +4116,17 @@ page writes a key that is neither a preference nor inventoried, writes one it ca
|
||||
CacheStorage or IndexedDB; `the storage scan reports every way of writing browser
|
||||
storage` pins those patterns. It skips `web/lib` (third-party builds) and the pinned
|
||||
Toolkit build (`web/js/shared/toolkit`, never hand-edited, verified by `npm run check`;
|
||||
its storage belongs to the Toolkit repository and a new pin is reviewed on its own). Since
|
||||
System S13 (#330) the legacy gallery no longer offers Clear cache: System › Maintenance's
|
||||
Metadata cache and Thumbnail cache reviews own both parts (removal table above), and
|
||||
`cache-ownership.js` serves the native review.
|
||||
its storage belongs to the Toolkit repository and a new pin is reviewed on its own). The
|
||||
gallery's server thumbnail clear stays its own request and runs first: refused or
|
||||
unreachable, it changes nothing and says so (the old code toasted success when the
|
||||
request failed); refused browser storage is reported, never hidden
|
||||
(`tests/js/legacy-refusals.test.js`; the browser case `the gallery cache clear keeps
|
||||
preferences and other tools caches` in `workflows.spec.mjs`).
|
||||
|
||||
The duplicates dialog's View Full named its source (`?root=<index>`, with the path
|
||||
URL-encoded) when the gallery's output folder is a listed source
|
||||
(`test_view_full_names_the_gallery_folder_source`); S13 removed the legacy dialog, and the
|
||||
route keeps that behaviour for System › Maintenance's duplicate review.
|
||||
The legacy duplicates dialog's View Full now names its source (`?root=<index>`, with the
|
||||
path URL-encoded) when the gallery's output folder is a listed source; before, the serve
|
||||
route opened the first source holding a file at that path
|
||||
(`test_view_full_names_the_gallery_folder_source`).
|
||||
|
||||
### Library location contract (#249 System S6b)
|
||||
|
||||
@@ -4685,18 +4418,8 @@ Filters (`mode=workspace`; anything invalid is 400 "Invalid gallery filters"):
|
||||
rating 5 to 1, then linked but unrated, then unlinked and ambiguous; ties break
|
||||
newest first. A signed cursor covers every filter and the sort, so reusing it
|
||||
with a change is 400.
|
||||
- One link read (`get_image_link_matches`) serves the filters, the rating sort
|
||||
and the count. SQLite returns its rows as one JSON value in one step (#354):
|
||||
Python's `sqlite3` releases and retakes the GIL around every step, and while
|
||||
ComfyUI generates each retake can wait on the busy thread, so a read that
|
||||
stepped once per linked row took seconds. The result keeps only the listed
|
||||
paths; peak memory grows with the database's matching linked rows (about
|
||||
530 bytes each, freed on return). When the JSON step fails (a SQLite without
|
||||
JSON functions, a BLOB value, a JSON text over SQLite's length limit) the read
|
||||
streams the rows as before and logs one warning per process; a missing JSON
|
||||
function stops the attempt for the rest of the process. An interrupted read
|
||||
(Move library) raises as before. `tests/test_database_link_matches.py` proves both reads
|
||||
equal on every filter and on odd values.
|
||||
- One link read (`get_image_link_matches`, streamed and bounded to the listed
|
||||
paths) serves the filters, the rating sort and the count.
|
||||
|
||||
`positive_count` is the number of distinct linked positives over the full matching
|
||||
set, not the page:
|
||||
@@ -4747,70 +4470,6 @@ concept fixture stars the concept's favorites (each other prompt's first image).
|
||||
restore. Old backups without the table restore, and backup plus restore keeps
|
||||
WAL-resident favorites and closes every handle (`tests/test_database_backup.py`).
|
||||
|
||||
Image categories (#332, store and routes): an image file may carry its own
|
||||
category; without one it shows the category its link supplies.
|
||||
|
||||
- The table is `image_categories(root_key, relative_path, category, updated_at)`
|
||||
with primary key `(root_key, relative_path)` and the index
|
||||
`idx_image_categories_category`. `migrate_image_categories` creates both empty
|
||||
inside a savepoint, with no backfill. It reads only `sqlite_master` and its own
|
||||
table's PRAGMAs, and writes only its two objects. `tests/test_image_categories.py`
|
||||
checks this with an SQLite authorizer on a 23,000-link library, and checks that
|
||||
`prompts` and `prompt_pairs` `(id, category, updated_at)` are byte-for-byte equal
|
||||
afterwards.
|
||||
- The **file key** is `image_category_key(sources, absolute path)` in
|
||||
`py/api/images.py`: the deepest output source that contains the path (compared
|
||||
lexically, never resolving the path, so a symlinked subfolder keeps its listed
|
||||
spelling) and the path below it, keyed with the favorites helpers. A source
|
||||
matches by its real path or its configured spelling (a detected output folder
|
||||
can be a symlink). Without nested sources it equals a favorite's key. Every write and read uses it. A path
|
||||
that no source contains has no key, so it has no own category.
|
||||
- `image_categories_schema_error` checks the structure at startup (a value saved
|
||||
under older rules never stops the library opening; a table without its index,
|
||||
which this code never leaves, is repaired by the install). Restore also checks the
|
||||
values: text columns, keys within the favorites bound, `category` 1 to 100
|
||||
characters with no character below U+0020 and no leading or trailing space, and
|
||||
`updated_at` of at most 64 bytes. Backups made before #332 restore without the
|
||||
table; a migrated restore stage must have it (`require_library_state`).
|
||||
- `PUT /prompt_manager/images/category` `{items, category}` answers
|
||||
`{success, category, updated, unchanged, not_found}`.
|
||||
`DELETE /prompt_manager/images/category` `{items}` answers `{success, cleared}`.
|
||||
`GET /prompt_manager/images/categories` answers `{success, categories}`: the
|
||||
sorted distinct own values whose file is in some source's listing.
|
||||
- `items` are 1 to 500 Images targets `{root_index, root_scope, path}`, checked by
|
||||
`py/api/media_target.py` (shared with Collections, #331). Refusals:
|
||||
- 400: a malformed body, a field other than `items` and `category`, a
|
||||
`root_index` outside the output sources (a LoRA folder), or a category that is
|
||||
blank after stripping, over 100 characters, or holds a control character;
|
||||
- 403: traversal, a drive, a backslash, a non-media name ("Only media files can
|
||||
have a category") or a key over the stored bound;
|
||||
- 409 `source-changed`: a stale `root_scope`;
|
||||
- 413: a body over 1 MiB.
|
||||
Any invalid item fails the whole request before a write. Targets that fold to one
|
||||
key are written and counted once. Instead of `items` a request may send `filter`,
|
||||
the Images listing's own query ("Select all N matching", #392, keys as in
|
||||
`parse_listing_filter`), resolved now: over 500 matches is a 400
|
||||
`too-many-images` ("Select at most 500 images."), a renumbered source a 409
|
||||
`source-changed`.
|
||||
- PUT writes only files the source's listing holds and counts the rest in
|
||||
`not_found`; when none is listed it answers 404 `files-not-found` and writes
|
||||
nothing. A file that already has the value counts as `unchanged` and keeps its
|
||||
`updated_at`. DELETE accepts any well-formed target, so a missing file's
|
||||
category can still be removed. An unexpected failure is a logged 500 ("The
|
||||
category could not be changed." / "Image categories could not be read.") without
|
||||
the exception text. The routes never read or write `prompts`, `prompt_pairs` or
|
||||
`generated_images`; `POST /prompt_manager/bulk/category` is unchanged.
|
||||
- Duplicate cleanup (System › Maintenance): the kept copy's own category wins. A
|
||||
kept copy without one receives the removed copy's (`COPY_SQL`, part of the
|
||||
cleanup's query-plan check). The removed copy's row is never deleted, so a
|
||||
give-back (the kept copy changed) and a failed delete lose nothing. The job log
|
||||
says "Kept N image categories on the kept copies".
|
||||
`tests/test_duplicate_image_categories.py` covers nested sources in both orders.
|
||||
- A moved, renamed or deleted file's row stays and is hidden; a file that later
|
||||
appears at the same path shows it. The real-host check is the last test in
|
||||
`tests/comfyui/image-attribution.spec.mjs`: PUT, list, DELETE on a generated
|
||||
output.
|
||||
|
||||
Image annotations schema (#249, `database/annotation_schema.py`): the group
|
||||
`media_annotation_metadata` (one row: schema version, `image_index_revision`),
|
||||
`media_annotation_files` (one private locator per physical file,
|
||||
@@ -4933,23 +4592,7 @@ files, oldest forgotten first, keyed by 16-byte digests: 8.6 MiB measured at
|
||||
the cap). A file whose inspection raises is logged, counted in
|
||||
`review_errors` and reported `unresolved`; the scan goes on. An older version's
|
||||
pending, undecided item of bytes now untraced leaves the queue
|
||||
(`settle_stale_review_items`), so an item never stays stuck at 409.
|
||||
Saver anchor (#390, `tests/test_review_saver_anchor.py`): the scan and Apply
|
||||
resolve with the file's name as the workspace inspector does, so the one saver
|
||||
whose `filename_prefix` names the file anchors its branch. A file is never both
|
||||
unlinked and unqueued (maintainer ruling): when only that anchor resolves it
|
||||
(the named inspection run resolves, the unnamed one does not) and neither the
|
||||
default profile nor the parser floor links it, the scan queues it with the
|
||||
anchor's pair as its one candidate (evidence `anchor: "saver"`; the page
|
||||
preselects it), never linking it. Such an item counts as a choice for the
|
||||
untraced settle. Files already linked or resolved by the default path are
|
||||
unchanged. No match, several matches or an unfinished walk anchor nothing, and
|
||||
the file is queued as before. `REVIEW_RULE_VERSION` is folded into inspection
|
||||
item versions, so older inspection items are recorded again once. Apply accepts
|
||||
the anchored pair and refuses a choice the anchor contradicts. Measured on the
|
||||
maintainer's 40,890 outputs: the queue is unchanged (249 files); the 242
|
||||
`gpu_unlock_bench` files the inspector resolves are linked by the parser
|
||||
floor. The 16-candidate limited
|
||||
(`settle_stale_review_items`), so an item never stays stuck at 409. The 16-candidate limited
|
||||
rule and the 8 MiB bound apply unchanged. The queue heals itself: after each batch, the scan removes the pending
|
||||
items of every listed file (read or skipped as known) that a saved row now
|
||||
links, and the monitor does the same for each ambiguous output it links
|
||||
@@ -5274,75 +4917,6 @@ same as a file the scan could not attribute, so the file stays on disk and in th
|
||||
gallery as unlinked, and its favorite and tags (kept by file) are untouched. Every
|
||||
run prints this. Queue-linked batch images can't be proven after the fact and read
|
||||
`unlink`, so review before applying. Stop ComfyUI first.
|
||||
|
||||
### Foreign workflows (#247): the shared per-file decision
|
||||
|
||||
The output scan decides each file through `utils/file_prompt.py`
|
||||
(`decide_file_prompt`, the ladder), which the monitor will share for outputs of
|
||||
workflows without a PromptManager node. It does no file or database I/O.
|
||||
|
||||
- **Save to library (#90).** Before anything else, a file whose PromptManager
|
||||
node had Save to library off is `not_saved`, and one whose switch is wired
|
||||
from a node the file cannot show is `not_saved_wired`: nothing is written, no
|
||||
recorded-run fallback, no Needs review item. A resolved pair whose negative's
|
||||
node is off stores the positive alone (source `opted-out-negative`). The
|
||||
#90 graph walk follows only prompt-like inputs, so the ladder also walks
|
||||
every input upstream of each sampler or guider positive (not negative, model,
|
||||
clip, latent or image inputs; every PromptManager node of a graph without a
|
||||
sampler) and checks the text later steps name: a resolved or inspected
|
||||
positive whose path passes such a node (`side_opted_out`), or a legacy answer
|
||||
that is such a node's text (resolved, else literal). A switch that is off
|
||||
makes the file `not_saved`; one wired from a node the graph cannot read makes
|
||||
it `not_saved_wired`, as #90's direct check does.
|
||||
- **Order.** The default profile decides first, exactly as before (resolved
|
||||
pair, A1111, sampler-less). Only a graph it leaves ambiguous or unresolved
|
||||
reaches the inspector's resolve (`inspection_resolution` in
|
||||
`utils/prompt_inspector.py`, the inspection profile, then the recorded saver
|
||||
anchor of #386; `name_anchor=False`: the file-name anchor never decides a
|
||||
link; the scan's legacy floor may still link the file, as before #247, and a
|
||||
file nothing links is queued with the anchor's choice per #390), and only that positive's `storage_text` is stored (never
|
||||
prepend or append text, never `recorded_texts`). Then the #170 browser parser
|
||||
(`graph_prompt_text`), the legacy floor, then the recorded run's stored
|
||||
prompt. The legacy answer is refused when a PromptManager-family node has a
|
||||
prepend or append part and the answer is not that node's own `text` (the
|
||||
parser joins the parts, so the stored text cannot be told from it). History is asked once per file and only at that step; the scan's
|
||||
batch lookup stays lazy and makes one `recorded_runs` pass for the stored
|
||||
prompt and the saver (`_BatchHistory`).
|
||||
- **Policies.** `SCAN` keeps the legacy floor for every undecided file.
|
||||
`MONITOR` waits for history inside the output folder (`history`) and stores
|
||||
nothing for a file the inspector calls ambiguous; an untraceable or
|
||||
malformed one still gets the legacy floor.
|
||||
- **Junk.** `junk_text` refuses `blank`, `number` (the #252 `_numeric` rule),
|
||||
`lora-only` (only `<lora:...>` tags and separators) and `model-file` (one
|
||||
token ending in a model extension) at whichever step found the text and
|
||||
nothing later is tried; the file reports `found`, except at the undecided
|
||||
graph step, which keeps its own status (an ambiguous file is still queued).
|
||||
- **One prompt per file.** `database/scanned_prompts.py` stores a positive in
|
||||
one `BEGIN IMMEDIATE` transaction: a file already linked to another prompt
|
||||
(either spelling, `file_link_rows`/`link_conflict`, the pair ingester's rule)
|
||||
is `conflict` and nothing is written; a failed link rolls the new prompt back
|
||||
(`found`); two writers of one text store one prompt. A recorded run's stored
|
||||
prompt links the same way (`link_stored_prompt_or_raise`). A rescan therefore
|
||||
never adds a second link to a file an older scan linked to the legacy answer;
|
||||
the row reads "Kept: already linked to a different prompt". Link rows are
|
||||
matched under the path and its `realpath` spelling (`file_link_rows`), so a
|
||||
file the scan stored through the resolved folder is found when it arrives
|
||||
through a symlinked or junction folder. The reverse order (a row stored under
|
||||
the watcher's raw spelling, then a scan whose root resolves differently) is
|
||||
matched only when that raw spelling equals the realpath. Every scan write
|
||||
runs through `survive_library_switch`, so a library move reruns it.
|
||||
- **Marks.** Every prompt ingestion creates, pair components included, carries
|
||||
category `scanned`, tag `auto-scanned` and the note `Auto-scanned from
|
||||
<file>` (`SCAN_MARKS`); a reused prompt keeps its fields. Run counts are
|
||||
never touched.
|
||||
|
||||
Tests: `tests/test_file_prompt.py` (one test per ladder step and junk class),
|
||||
`tests/test_scanned_prompts.py` (store, conflict, rollback, concurrency with
|
||||
`FirstWriterHold`, pair marks), `tests/test_pair_scan.py` `TestScanDecision`,
|
||||
`tests/test_run_attribution.py` (dataset branches linked by the recorded
|
||||
saver, a legacy-linked file left as it is, `_BatchHistory.recorded`) and
|
||||
`tests/test_recorded_saver_anchor.py` `TestInspectionResolution`.
|
||||
|
||||
## Linked prompt text (#343)
|
||||
|
||||
A link drives `text` on every PromptManager node, and `negative_text` on the Pair,
|
||||
@@ -5409,7 +4983,7 @@ which used to save `null` under a link.
|
||||
Primitive or through `PreviewAny`, for PromptManager, PromptManagerText, Focused, Split
|
||||
and the Pair (positive, negative, both, neither linked, and a Primitive negative). A
|
||||
wildcard Primitive on PromptManager (queued as written) and Focused (one choice) saves
|
||||
and counts what was queued. The linked text is saved, counted once per job, recorded as
|
||||
and counts what was queued. The linked text is saved, counted once, recorded as
|
||||
the run's raw positive/negative, and after a scan the run's image links to it. Text
|
||||
only known at run time (`PreviewAny`) cannot be read from the image; the case only
|
||||
requires that it is never linked to another prompt (its attribution from the recorded
|
||||
@@ -5423,39 +4997,6 @@ PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:comfyui
|
||||
PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:executions -- execution-text-link
|
||||
```
|
||||
|
||||
## Cached runtime-linked runs (#347)
|
||||
|
||||
Text only known at run time (a `PreviewAny` or LLM link, a list node) is counted by the
|
||||
node when it executes; the queue hook cannot read it. When ComfyUI serves such a node from
|
||||
its output cache, `utils/cached_runs.py` counts the job instead: every execution records
|
||||
a receipt (digests of its STRING outputs and hashes of the text it stores, per run key and
|
||||
queue item), and on `execution_cached` the receipt whose digests equal the cache entry's
|
||||
text is replayed once. The role (positive, negative, which Pair outputs are used) and what
|
||||
the hook already saw are recomputed from the job's own graph, and the node and replay
|
||||
decide through the same `own_counts`.
|
||||
|
||||
- Fail closed: no receipt in this process (after a restart, an external cache provider),
|
||||
changed cached text, more than 64 list values, a node the job's outputs do not need, or
|
||||
a host without the queue/executor interfaces: nothing is counted, as before #347.
|
||||
- A cached node counts at job start like the hook: a job failing later still counts it.
|
||||
A node that executes after its replay in the same job (RAM-pressure freed the entry)
|
||||
skips its own count but still consumes the pending first-use catch-up.
|
||||
- No backfill: earlier cached jobs keep their counts. The history page's "tracked runs"
|
||||
can stay below the Library count for these prompts (#403).
|
||||
- Tests: `tests/test_cached_runs.py` (whole jobs with the real nodes, a real database,
|
||||
fake host caches); `tests/comfyui/execution-text-link.spec.mjs` "#347" cases on the
|
||||
managed host: PreviewAny into each node kind, 3 jobs changing only the sampler seed,
|
||||
count 3 in both renderers; in one renderer the Classic, LRU, RAM-pressure and no-cache
|
||||
modes, A, B, A from the owned `PMCycledText` under LRU, typed text, a downstream
|
||||
failure, a "run selected output" job that does not need the node, and a rewire to the
|
||||
negative input. The RAM-pressure eviction path cannot be forced on a real host and is
|
||||
covered by the Python tests.
|
||||
|
||||
```sh
|
||||
python3 -m unittest discover -s tests -p "test_cached_runs.py"
|
||||
PM_COMFY_ROOT=... PM_TEST_PYTHON=... npm run --prefix tools/vendor test:executions -- execution-text-link --grep "#347"
|
||||
```
|
||||
|
||||
## Durable execution ledger
|
||||
|
||||
`npm run --prefix tools/vendor test:executions` runs 52 real execution cases
|
||||
@@ -5875,20 +5416,6 @@ rotates the epoch, so a pre-restore `expected_epoch` is refused with `storage-ch
|
||||
Owner rows and their timestamps never change on a tag command; a category apply only
|
||||
touches rows carrying the exact source.
|
||||
|
||||
Image categories (#332 §7, MC9, MC10, DC7): each `GET categories` row also carries
|
||||
`image_count`, the stored `image_categories` rows holding that exact value (files
|
||||
missing on disk included). A value prompts and images share appears once, ordered by
|
||||
its oldest positive; a value only images hold appears once after every prompt-backed
|
||||
value (`first_pid` 9223372036854775807 in the keyset cursor), ordered by value, with
|
||||
`prompt_count` 0. `/library/options` keeps its own query and never lists image-only
|
||||
values. Category preview and apply report `impact.images` (rows holding the source),
|
||||
the target exists when an image row holds it, an image-only value can be renamed, the
|
||||
apply renames the image rows in the same transaction (file keys and rowids kept, so a
|
||||
merge cannot collide), and the impact token hashes the sorted image rowids, so an image
|
||||
row gaining or losing the value between preview and apply is 409
|
||||
`vocabulary-conflict` with nothing written. Every image-category statement reads
|
||||
`idx_image_categories_category` (`test_category_reads_use_the_image_category_index`).
|
||||
|
||||
Two legacy routes change additively: `GET /tags` now lists unused registry tags, and
|
||||
`/tags/suggest` without context appends unused names at count 0. Unused tags become
|
||||
deletable only through the native Delete tag (vocabulary UI PR), not on the legacy
|
||||
|
||||
-58
@@ -195,26 +195,6 @@ except Exception as e:
|
||||
f"Failed to register usage hook: {e}"
|
||||
)
|
||||
|
||||
# A cached node counts what its last execution of the same text counted (#347).
|
||||
try:
|
||||
from .py.config import server_instance
|
||||
from .utils.cached_runs import register_cached_runs
|
||||
|
||||
register_cached_runs(server_instance, _db_factory)
|
||||
except Exception as e:
|
||||
try:
|
||||
from .utils.logging_config import get_logger
|
||||
|
||||
get_logger("prompt_manager.init").error(
|
||||
f"Failed to register cached-run counting: {e}"
|
||||
)
|
||||
except Exception:
|
||||
import logging
|
||||
|
||||
logging.getLogger("prompt_manager.init").error(
|
||||
f"Failed to register cached-run counting: {e}"
|
||||
)
|
||||
|
||||
|
||||
def _history_db_factory() -> "PromptDatabase":
|
||||
"""Initialize history on its worker even if monitoring initialization failed."""
|
||||
@@ -375,44 +355,6 @@ async def _stop_background_work_on_cleanup(app):
|
||||
_stop_background_work()
|
||||
|
||||
|
||||
def _kill_encoders_at_exit():
|
||||
"""Run by the exit hook before Python joins the executor threads (#376):
|
||||
a JPEG XL or ffmpeg encoder (no console on Windows, so Ctrl+C never
|
||||
reaches it) is killed instead of being waited for."""
|
||||
from .py.api.thumbnail_encode import shutdown_encoders
|
||||
|
||||
shutdown_encoders()
|
||||
|
||||
|
||||
def _abort_downloads_at_exit():
|
||||
"""Run by the exit hook (#376): huggingface_hub's Xet transfers ignore
|
||||
the chunk checkpoint, so they are aborted the way hub aborts on Ctrl+C."""
|
||||
import sys
|
||||
|
||||
if "huggingface_hub" not in sys.modules:
|
||||
return # no download ever ran: import nothing at exit
|
||||
from .py.autotag import abort_hub_downloads
|
||||
|
||||
abort_hub_downloads()
|
||||
|
||||
|
||||
try:
|
||||
# Before the executor join: running jobs stop at their next file, item or
|
||||
# chunk (#376). The teardown below still runs from atexit afterwards.
|
||||
from .utils import exit_signal as _exit_signal
|
||||
|
||||
_exit_signal.install(_kill_encoders_at_exit)
|
||||
_exit_signal.install(_abort_downloads_at_exit)
|
||||
except Exception as e:
|
||||
try:
|
||||
from .utils.logging_config import get_logger
|
||||
|
||||
get_logger("prompt_manager.init").error(
|
||||
f"Failed to register the exit signal: {e}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
atexit.register(_stop_background_work)
|
||||
from .py.config import server_instance as _teardown_server
|
||||
|
||||
@@ -11,8 +11,6 @@ IMMEDIATE`` transaction, so no row ever names a deleted file:
|
||||
- its favorite;
|
||||
- its image tags (``media_image_tags`` of its byte record), joined onto the
|
||||
kept copy's byte record as their union, in order;
|
||||
- its own image category (#332), copied to the kept copy when that copy has
|
||||
none; the removed copy's row is kept, so a give-back loses nothing;
|
||||
- its pending Needs review items, settled: only an item still ``pending`` at
|
||||
revision 0 with no decision row is removed (``review._SETTLE_ITEM``), so a
|
||||
decided item is never touched.
|
||||
@@ -24,8 +22,6 @@ import os
|
||||
|
||||
from .annotation_schema import BYTES_TABLE, FILES_TABLE
|
||||
from .annotations import _BUMP_INDEX
|
||||
from .image_categories import COPY_SQL as COPY_CATEGORY_SQL
|
||||
from .image_categories import copy_image_category
|
||||
from .models import normalize_image_path, utc_now_iso
|
||||
from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM
|
||||
from .review_changes import notify_review_change
|
||||
@@ -80,7 +76,6 @@ FILE_LINK_SQL = (
|
||||
COPY_BYTE_SQL,
|
||||
COPY_TAG_COUNT_SQL,
|
||||
PENDING_COUNT_SQL,
|
||||
COPY_CATEGORY_SQL,
|
||||
)
|
||||
|
||||
|
||||
@@ -142,10 +137,8 @@ class DuplicateCopyOperations:
|
||||
removed key, kept root key, kept key)`` per listed root holding the
|
||||
file), ``removed_locator``,
|
||||
``kept_locator``, ``sha256`` and the kept copy's ``kept_size`` and
|
||||
``kept_mtime_ns``; ``categories``: ``(removed file key, kept file
|
||||
key)`` pairs of the image categories (#332). Returns ``{moved,
|
||||
merged, favorites, tags, review, categories}`` (``categories``: the
|
||||
image categories copied to the kept copy).
|
||||
``kept_mtime_ns``. Returns ``{moved, merged, favorites, tags,
|
||||
review}``.
|
||||
"""
|
||||
counts = dict.fromkeys(("moved", "merged", "favorites", "tags", "review"), 0)
|
||||
kept_keys = [normalize_image_path(path) for path in move["kept"]]
|
||||
@@ -181,10 +174,6 @@ class DuplicateCopyOperations:
|
||||
for favorite in move.get("favorites", ()):
|
||||
counts["favorites"] += _move_favorite(conn, *favorite)
|
||||
counts["tags"] = self._move_copy_tags(conn, move)
|
||||
counts["categories"] = sum(
|
||||
copy_image_category(conn, removed, kept)
|
||||
for removed, kept in move.get("categories", ())
|
||||
)
|
||||
for (item_id,) in conn.execute(
|
||||
_PENDING_OF_LOCATOR, (move["removed_locator"],)
|
||||
).fetchall():
|
||||
|
||||
@@ -1,249 +0,0 @@
|
||||
"""Image categories (#332): an image file's own category.
|
||||
|
||||
An image shows its own category when it has one, otherwise the category its
|
||||
link supplies (its recipe's, else its positive prompt's); this table holds
|
||||
only the own values. A row is keyed by the file key: the deepest configured
|
||||
output source that contains the file (its real, case-normalized path) and the
|
||||
path below it, case-folded where the OS folds case
|
||||
(``py/api/images.py`` ``image_category_key``). Without nested sources the key
|
||||
equals a favorite's key. Nothing follows a moved file: its row stays and is
|
||||
hidden until a file appears at the same path again.
|
||||
|
||||
Writes validate strictly (the route strips the value and applies
|
||||
``validate_category``). Startup checks the structure only, so a value saved
|
||||
under older rules never stops the library opening; restore also checks the
|
||||
values. Optional in old backups, required once a database is current.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Iterable
|
||||
|
||||
TABLE = "image_categories"
|
||||
INDEX = "idx_image_categories_category"
|
||||
MAX_KEY_LENGTH = 4096
|
||||
MAX_CATEGORY_LENGTH = 100
|
||||
_DDL = (
|
||||
f"CREATE TABLE {TABLE} (root_key TEXT NOT NULL, relative_path TEXT NOT NULL, "
|
||||
"category TEXT NOT NULL, updated_at TEXT NOT NULL, "
|
||||
"PRIMARY KEY (root_key, relative_path))"
|
||||
)
|
||||
_INDEX_DDL = f"CREATE INDEX {INDEX} ON {TABLE}(category)"
|
||||
_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
|
||||
_COLUMNS = (
|
||||
("root_key", "TEXT", 1, 1),
|
||||
("relative_path", "TEXT", 1, 2),
|
||||
("category", "TEXT", 1, 0),
|
||||
("updated_at", "TEXT", 1, 0),
|
||||
)
|
||||
# A category may not hold a character below U+0020 (validate_category).
|
||||
_CONTROL_GLOB = "*[" + "".join(chr(code) for code in range(1, 32)) + "]*"
|
||||
|
||||
CATEGORY_OF_SQL = (
|
||||
f"SELECT category FROM {TABLE} WHERE root_key = ? AND relative_path = ?"
|
||||
)
|
||||
CATEGORIES_OF_ROOT_SQL = (
|
||||
f"SELECT relative_path, category FROM {TABLE} WHERE root_key = ?"
|
||||
)
|
||||
UPSERT_SQL = (
|
||||
f"INSERT INTO {TABLE} (root_key, relative_path, category, updated_at) "
|
||||
"VALUES (?, ?, ?, ?) ON CONFLICT(root_key, relative_path) "
|
||||
"DO UPDATE SET category = excluded.category, updated_at = excluded.updated_at"
|
||||
)
|
||||
DELETE_SQL = f"DELETE FROM {TABLE} WHERE root_key = ? AND relative_path = ?"
|
||||
# The duplicate cleanup's copy (#332 MC11): the removed copy's row is copied
|
||||
# to the kept copy only when the kept copy has none; it is never deleted.
|
||||
COPY_SQL = (
|
||||
f"INSERT OR IGNORE INTO {TABLE} (root_key, relative_path, category, updated_at) "
|
||||
f"SELECT ?, ?, category, updated_at FROM {TABLE} "
|
||||
"WHERE root_key = ? AND relative_path = ?"
|
||||
)
|
||||
STATEMENTS = (CATEGORY_OF_SQL, CATEGORIES_OF_ROOT_SQL, UPSERT_SQL, DELETE_SQL, COPY_SQL)
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def _structure_error(
|
||||
conn: sqlite3.Connection, allow_missing: bool, allow_missing_index: bool = False
|
||||
) -> str | None:
|
||||
objects = conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? "
|
||||
"OR (type IN ('index','trigger') AND tbl_name=?)",
|
||||
(TABLE, TABLE),
|
||||
).fetchall()
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing image categories schema"
|
||||
tables = [sql for kind, _name, sql in objects if kind == "table"]
|
||||
others = {(kind, name) for kind, name, _sql in objects if kind != "table"}
|
||||
expected = {("index", _AUTOINDEX), ("index", INDEX)}
|
||||
# Before its install, a table without the category index is repaired by
|
||||
# CREATE INDEX IF NOT EXISTS rather than refused.
|
||||
if allow_missing_index and others == {("index", _AUTOINDEX)}:
|
||||
expected = others
|
||||
if len(tables) != 1 or others != expected:
|
||||
return "Image categories name is reserved by an unsupported schema object"
|
||||
if _compact(tables[0]) != _compact(_DDL):
|
||||
return "Image categories has an invalid table definition"
|
||||
index_sql = next((sql for kind, name, sql in objects if name == INDEX), None)
|
||||
if ("index", INDEX) in expected and _compact(index_sql) != _compact(_INDEX_DDL):
|
||||
return "Image categories has an invalid index"
|
||||
autoindex_sql = next(sql for kind, name, sql in objects if name == _AUTOINDEX)
|
||||
if autoindex_sql is not None:
|
||||
return "Image categories has an invalid index"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({TABLE})")
|
||||
)
|
||||
if columns != _COLUMNS:
|
||||
return "Image categories has invalid columns"
|
||||
return None
|
||||
|
||||
|
||||
def _values_error(conn: sqlite3.Connection) -> str | None:
|
||||
# Bytes bound characters from above (UTF-8 is at most 4 bytes each).
|
||||
limit = MAX_KEY_LENGTH * 4
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(root_key) != 'text' " # nosec B608
|
||||
"OR typeof(relative_path) != 'text' OR typeof(category) != 'text' "
|
||||
"OR typeof(updated_at) != 'text' "
|
||||
f"OR length(CAST(root_key AS BLOB)) > {limit} "
|
||||
f"OR length(CAST(relative_path AS BLOB)) > {limit} "
|
||||
"OR length(CAST(updated_at AS BLOB)) > 64 "
|
||||
"OR instr(CAST(category AS BLOB), x'00') > 0 "
|
||||
f"OR length(category) < 1 OR length(category) > {MAX_CATEGORY_LENGTH} "
|
||||
"OR category != TRIM(category) OR category GLOB ? LIMIT 1",
|
||||
(_CONTROL_GLOB,),
|
||||
).fetchone():
|
||||
return "Image categories contains invalid values"
|
||||
for root, path in conn.execute(
|
||||
f"SELECT CAST(root_key AS BLOB), CAST(relative_path AS BLOB) " # nosec B608
|
||||
f"FROM {TABLE} WHERE length(CAST(root_key AS BLOB)) > {MAX_KEY_LENGTH} "
|
||||
f"OR length(CAST(relative_path AS BLOB)) > {MAX_KEY_LENGTH}"
|
||||
):
|
||||
for raw in (root, path):
|
||||
if len(bytes(raw).decode("utf-8", "replace")) > MAX_KEY_LENGTH:
|
||||
return "Image categories contains invalid values"
|
||||
return None
|
||||
|
||||
|
||||
def image_categories_schema_error(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
allow_missing: bool = True,
|
||||
check_values: bool = True,
|
||||
) -> str | None:
|
||||
"""Reject an unsupported image categories schema (and, optionally, values).
|
||||
|
||||
Reads only. A missing table is acceptable for backups that predate it
|
||||
unless ``allow_missing`` is false. ``check_values`` also refuses a stored
|
||||
value the write path could never have written.
|
||||
"""
|
||||
error = _structure_error(conn, allow_missing)
|
||||
if error is not None or not check_values:
|
||||
return error
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone()
|
||||
return _values_error(conn) if exists else None
|
||||
|
||||
|
||||
def migrate_image_categories(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table and its index atomically; structure is checked
|
||||
before and after. Reads only ``sqlite_master`` and this table's PRAGMAs
|
||||
and writes only its own two objects (no backfill)."""
|
||||
conn.execute("SAVEPOINT image_categories_schema")
|
||||
try:
|
||||
error = _structure_error(conn, True, allow_missing_index=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
conn.execute(
|
||||
_INDEX_DDL.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS", 1)
|
||||
)
|
||||
error = image_categories_schema_error(
|
||||
conn, allow_missing=False, check_values=False
|
||||
)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE image_categories_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO image_categories_schema")
|
||||
conn.execute("RELEASE image_categories_schema")
|
||||
raise
|
||||
|
||||
|
||||
def copy_image_category(conn: sqlite3.Connection, removed_key, kept_key) -> int:
|
||||
"""Give the kept copy the removed copy's category when it has none.
|
||||
|
||||
Runs inside the caller's transaction; never deletes the removed copy's
|
||||
row, so a give-back or a failed delete loses nothing. Returns 1 when a
|
||||
row was inserted, else 0.
|
||||
"""
|
||||
if tuple(removed_key) == tuple(kept_key):
|
||||
return 0
|
||||
return max(conn.execute(COPY_SQL, (*kept_key, *removed_key)).rowcount, 0)
|
||||
|
||||
|
||||
class ImageCategoryOperations:
|
||||
"""PromptDatabase mixin: set, clear and read images' own categories.
|
||||
|
||||
Never reads or writes ``prompts``, ``prompt_pairs`` or
|
||||
``generated_images``.
|
||||
"""
|
||||
|
||||
def image_category(self, root_key: str, relative_key: str) -> str | None:
|
||||
"""One file's own category, or None."""
|
||||
row = (
|
||||
self.model.get_connection()
|
||||
.execute(CATEGORY_OF_SQL, (root_key, relative_key))
|
||||
.fetchone()
|
||||
)
|
||||
return row[0] if row else None
|
||||
|
||||
def own_categories_of(self, root_keys: Iterable[str]) -> dict:
|
||||
"""``{(root key, relative key): category}`` under the given sources.
|
||||
|
||||
One primary-key range read per source.
|
||||
"""
|
||||
found = {}
|
||||
conn = self.model.get_connection()
|
||||
for root_key in dict.fromkeys(root_keys):
|
||||
for relative, category in conn.execute(CATEGORIES_OF_ROOT_SQL, (root_key,)):
|
||||
found[(root_key, relative)] = category
|
||||
return found
|
||||
|
||||
def set_image_categories(self, file_keys, category: str) -> dict:
|
||||
"""Set *category* on every file key, in one transaction.
|
||||
|
||||
``file_keys``: ``(root key, relative key)`` pairs from any number of
|
||||
sources. Returns ``{updated, unchanged}``; a file that already had
|
||||
exactly this value is ``unchanged`` and keeps its ``updated_at``.
|
||||
"""
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
keys = list(dict.fromkeys(tuple(key) for key in file_keys))
|
||||
counts = {"updated": 0, "unchanged": 0}
|
||||
now = utc_now_iso()
|
||||
with self._pair_transaction() as conn:
|
||||
for key in keys:
|
||||
row = conn.execute(CATEGORY_OF_SQL, key).fetchone()
|
||||
if row is not None and row[0] == category:
|
||||
counts["unchanged"] += 1
|
||||
continue
|
||||
conn.execute(UPSERT_SQL, (*key, category, now))
|
||||
counts["updated"] += 1
|
||||
return counts
|
||||
|
||||
def clear_image_categories(self, file_keys) -> int:
|
||||
"""Remove the own category of every file key (one transaction).
|
||||
|
||||
Returns the number of rows removed.
|
||||
"""
|
||||
keys = list(dict.fromkeys(tuple(key) for key in file_keys))
|
||||
removed = 0
|
||||
with self._pair_transaction() as conn:
|
||||
for key in keys:
|
||||
removed += conn.execute(DELETE_SQL, key).rowcount
|
||||
return removed
|
||||
+2
-11
@@ -34,7 +34,6 @@ from .review_schema import (
|
||||
review_without_annotations_error,
|
||||
)
|
||||
from .saved_views import migrate_saved_views, saved_views_schema_error
|
||||
from .image_categories import image_categories_schema_error, migrate_image_categories
|
||||
from .workflow_documents import (
|
||||
DOCUMENT_ID_COLUMNS,
|
||||
migrate_workflow_documents,
|
||||
@@ -351,8 +350,6 @@ class PromptModel:
|
||||
migrate_annotation_state(conn)
|
||||
migrate_review_state(conn)
|
||||
migrate_autotag_state(conn)
|
||||
# Image categories (#332): independent of every table above.
|
||||
migrate_image_categories(conn)
|
||||
# Last: the older generated_images rebuilds run only before file_path.
|
||||
migrate_workflow_documents(conn)
|
||||
|
||||
@@ -1158,9 +1155,8 @@ class PromptModel:
|
||||
Args:
|
||||
path: File to inspect
|
||||
require_library_state: Also require the tables every current
|
||||
database has (Library state, image favorites, image
|
||||
categories, saved views, image annotations and the review
|
||||
queue), as a migrated
|
||||
database has (Library state, image favorites, saved views,
|
||||
image annotations and the review queue), as a migrated
|
||||
restore stage must; old backups may lack them.
|
||||
documents: Read every workflow document and refuse a damaged
|
||||
one. A restore checks its upload without it and repairs
|
||||
@@ -1224,11 +1220,6 @@ class PromptModel:
|
||||
)
|
||||
if favorites_error:
|
||||
return False, favorites_error
|
||||
categories_error = image_categories_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if categories_error:
|
||||
return False, categories_error
|
||||
views_error = saved_views_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
|
||||
+41
-186
@@ -7,12 +7,7 @@ import json
|
||||
import math
|
||||
import datetime
|
||||
import os
|
||||
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
|
||||
|
||||
try:
|
||||
from ..utils.inspection_response import inherited_category
|
||||
except ImportError:
|
||||
from utils.inspection_response import inherited_category
|
||||
from typing import Optional, List, Dict, Any, Set, Tuple
|
||||
|
||||
from .models import PromptModel, normalize_image_path, utc_now_iso
|
||||
from .pairs import PairDependencyError, PairOperations # noqa: F401 (re-exported)
|
||||
@@ -32,7 +27,6 @@ from .health import HealthOperations, iter_consistency_issues
|
||||
from .duplicate_copies import DuplicateCopyOperations
|
||||
from .workflow_storage import WorkflowStorageOperations
|
||||
from .category_import import CategoryImportOperations, EmptyReplacement # noqa: F401
|
||||
from .image_categories import ImageCategoryOperations
|
||||
from .workflow_documents import (
|
||||
DocumentReader,
|
||||
documents_of_rows,
|
||||
@@ -243,17 +237,12 @@ LEFT JOIN prompt_library_state pls ON pls.prompt_id = pp.positive_prompt_id
|
||||
WHERE gi.image_path IN ({marks}) OR gi.file_path IN ({marks}){fallback}"""
|
||||
# One image row describes a card only when no other row claims the same file
|
||||
# (as in get_image_link_summaries, several rows are ambiguous). The pair scope
|
||||
# and the linked positive follow _LINK_SUMMARY_SQL; rating, tags and the
|
||||
# searchable text and titles come from that positive (spec D12/D13). The raw
|
||||
# recipe and positive categories are read for the inherited category, which
|
||||
# Python decides (``inherited_category``, #332 MC4); SQL never filters by
|
||||
# category. Every
|
||||
# and the linked positive follow _LINK_SUMMARY_SQL; category, rating, tags and
|
||||
# the searchable text and titles come from that positive (spec D12/D13). Every
|
||||
# writer since the 3.2.4 migration stores file_path, so it is the path key.
|
||||
# The prompt-level conditions are uncorrelated IN sets SQLite builds once, and
|
||||
# the CASE evaluates the per-row ambiguity probe only for admitted rows.
|
||||
_LINK_FILTER_SQL = """SELECT gi.file_path AS file_path, p.rating AS rating,
|
||||
{hit} AS matched, p.id AS positive, pp.id IS NOT NULL AS has_pair,
|
||||
pp.category AS pair_category, p.category AS category
|
||||
_LINK_FILTER_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id
|
||||
FROM generated_images gi
|
||||
LEFT JOIN prompt_pairs pp ON pp.id = (
|
||||
SELECT CASE WHEN COUNT(DISTINCT ip.pair_id) = 1 THEN MIN(ip.pair_id) END
|
||||
@@ -262,31 +251,10 @@ JOIN prompts p ON p.id = COALESCE(pp.positive_prompt_id, gi.prompt_id)
|
||||
WHERE gi.file_path IS NOT NULL AND CASE WHEN {admit} THEN NOT EXISTS (
|
||||
SELECT 1 FROM generated_images other
|
||||
WHERE other.file_path = gi.file_path AND other.id <> gi.id) ELSE 0 END"""
|
||||
# _LINK_FILTER_SQL's rows as one JSON array, built by SQLite in one step (#354).
|
||||
# Python's sqlite3 releases and retakes the GIL around every step, so a read
|
||||
# that steps once per linked row waits on a busy ComfyUI thread thousands of
|
||||
# times. A BLOB value (no writer stores one) makes the value NULL (or
|
||||
# json_array refuses it), and the read streams instead. The raw categories are
|
||||
# put in only for a category read (#332), so plain reads stay as small as #354
|
||||
# measured them.
|
||||
_LINK_MATCHES_JSON_SQL = """SELECT CASE
|
||||
WHEN max({blob}) THEN NULL
|
||||
ELSE json_group_array(json_array({columns})) END
|
||||
FROM ({select})"""
|
||||
_LINK_PLAIN_COLUMNS = ("file_path", "rating", "matched", "positive")
|
||||
_LINK_CATEGORY_COLUMNS = _LINK_PLAIN_COLUMNS + ("has_pair", "pair_category", "category")
|
||||
# The columns a BLOB can reach: matched, positive and has_pair are computed.
|
||||
_LINK_STORED_COLUMNS = ("file_path", "rating", "pair_category", "category")
|
||||
# Set when this SQLite has no JSON functions: every later read streams.
|
||||
_one_step_unsupported = False
|
||||
# One warning per process when a read falls back to streaming.
|
||||
_one_step_warned = False
|
||||
# The linked positive of each given path (one unambiguous link, pair scope as
|
||||
# in _LINK_FILTER_SQL) with the raw categories of its inherited category; {hit}
|
||||
# as in link_filter_query. Used for the few files whose current image tags meet
|
||||
# a Tags filter (#249 B3).
|
||||
_LINK_FACTS_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id,
|
||||
pp.id IS NOT NULL, pp.category, p.category
|
||||
# in _LINK_FILTER_SQL) with its category; {hit} as in link_filter_query. Used
|
||||
# for the few files whose current image tags meet a Tags filter (#249 B3).
|
||||
_LINK_FACTS_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id, p.category
|
||||
FROM generated_images gi
|
||||
LEFT JOIN prompt_pairs pp ON pp.id = (
|
||||
SELECT CASE WHEN COUNT(DISTINCT ip.pair_id) = 1 THEN MIN(ip.pair_id) END
|
||||
@@ -315,24 +283,8 @@ _LINK_FILTER_TAG = (
|
||||
)
|
||||
|
||||
|
||||
# One unambiguously linked image's facts for the Images filters, a plain tuple
|
||||
# (hit, rating, positive id, inherited category): built per linked row, so no
|
||||
# NamedTuple. The category is the inherited one (#332 MC4: the recipe's when
|
||||
# set, else the positive's), never the image's own category.
|
||||
LinkMatch = Tuple[bool, Optional[int], int, Any]
|
||||
|
||||
|
||||
def _link_match(rating, matched, positive, has_pair, pair_category, category):
|
||||
inherited, _scope = inherited_category(category, pair_category, bool(has_pair))
|
||||
return (bool(matched), rating, positive, inherited)
|
||||
|
||||
|
||||
def link_filter_query(
|
||||
q: str,
|
||||
tags: List[str],
|
||||
*,
|
||||
every_link: bool = False,
|
||||
category: Optional[str] = None,
|
||||
q: str, category: Optional[str], tags: List[str], *, every_link: bool = False
|
||||
) -> Tuple[str, Dict[str, Any]]:
|
||||
"""The workspace filter SELECT and its named parameters (all bound).
|
||||
|
||||
@@ -346,9 +298,7 @@ def link_filter_query(
|
||||
params["pattern"] = f"%{escape_like(q)}%"
|
||||
admitted = []
|
||||
if category is not None:
|
||||
# Only narrows: an inherited category is one of the two raw values;
|
||||
# which one is decided in Python (inherited_category, #332 MC4).
|
||||
admitted.append("(p.category = :category OR pp.category = :category)")
|
||||
admitted.append("p.category = :category")
|
||||
params["category"] = category
|
||||
for index, tag in enumerate(tags):
|
||||
admitted.append(_LINK_FILTER_TAG.format(index=index))
|
||||
@@ -359,50 +309,6 @@ def link_filter_query(
|
||||
return sql, params
|
||||
|
||||
|
||||
def link_matches_json_query(sql: str, categories: bool = False) -> str:
|
||||
"""*sql* from link_filter_query, wrapped to return its rows as one JSON value.
|
||||
|
||||
With *categories* each row also carries the raw values the inherited
|
||||
category is decided from; otherwise only path, rating, hit and positive.
|
||||
"""
|
||||
columns = _LINK_CATEGORY_COLUMNS if categories else _LINK_PLAIN_COLUMNS
|
||||
blob = " OR ".join(
|
||||
f"typeof({name}) = 'blob'" for name in _LINK_STORED_COLUMNS if name in columns
|
||||
)
|
||||
return _LINK_MATCHES_JSON_SQL.format(
|
||||
blob=blob, columns=", ".join(columns), select=sql
|
||||
)
|
||||
|
||||
|
||||
def _link_matches_from_rows(
|
||||
rows: Iterable[Sequence[Any]], categories: bool, wanted: Optional[Set[str]]
|
||||
) -> Dict[str, LinkMatch]:
|
||||
"""LinkMatch per wanted path from link rows (one-step or streamed).
|
||||
|
||||
Without *categories* the inherited category is None, as the caller then
|
||||
never compares it.
|
||||
"""
|
||||
matches: Dict[str, LinkMatch] = {}
|
||||
for path, rating, matched, positive, *raw in rows:
|
||||
if wanted is None or path in wanted:
|
||||
if categories:
|
||||
matches[path] = _link_match(rating, matched, positive, *raw[:3])
|
||||
else:
|
||||
matches[path] = (bool(matched), rating, positive, None)
|
||||
return matches
|
||||
|
||||
|
||||
def _streamed_link_matches(
|
||||
conn: sqlite3.Connection,
|
||||
sql: str,
|
||||
params: Dict[str, Any],
|
||||
wanted: Optional[Set[str]],
|
||||
categories: bool = False,
|
||||
) -> Dict[str, LinkMatch]:
|
||||
"""The link read stepped once per row (the fallback of the one-step read)."""
|
||||
return _link_matches_from_rows(conn.execute(sql, params), categories, wanted)
|
||||
|
||||
|
||||
_ASCII_FOLD = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
|
||||
|
||||
|
||||
@@ -462,7 +368,6 @@ class PromptDatabase(
|
||||
DuplicateCopyOperations,
|
||||
WorkflowStorageOperations,
|
||||
CategoryImportOperations,
|
||||
ImageCategoryOperations,
|
||||
):
|
||||
"""Database operations class for managing prompts."""
|
||||
|
||||
@@ -2636,94 +2541,38 @@ class PromptDatabase(
|
||||
def get_image_link_matches(
|
||||
self,
|
||||
q: str,
|
||||
category: Optional[str],
|
||||
tags: List[str],
|
||||
*,
|
||||
every_link: bool = False,
|
||||
wanted: Optional[Set[str]] = None,
|
||||
category: Optional[str] = None,
|
||||
) -> Dict[str, LinkMatch]:
|
||||
"""Map normalized paths of unambiguously linked images to a LinkMatch
|
||||
(hit, rating, positive id, inherited category).
|
||||
) -> Dict[str, Tuple[bool, Optional[int], int]]:
|
||||
"""Map normalized paths of unambiguously linked images to
|
||||
(hit, rating, positive id).
|
||||
|
||||
Rows are limited to images whose linked positive has every tag in
|
||||
*tags* (exact names). The category filter is applied by the caller to
|
||||
the effective category (#332). ``hit`` says whether *q* occurs in
|
||||
Rows are limited to images whose linked positive has *category* and
|
||||
every tag in *tags* (exact names). ``hit`` says whether *q* occurs in
|
||||
the positive's text, Library title or a tag name; a pair's own title
|
||||
names its negative variant and is not searched.
|
||||
Without tags only hits are returned, unless *every_link*
|
||||
Without category or tags only hits are returned, unless *every_link*
|
||||
asks for every linked image (the rating sort and the positive count).
|
||||
The positive id is the card's linked positive. With *category* only
|
||||
links whose raw recipe or positive category is that value are read (a
|
||||
superset of the links that inherit it; the caller compares the
|
||||
inherited value) and the inherited category is decided per row;
|
||||
otherwise it is None, which keeps the plain reads cheap.
|
||||
|
||||
One SELECT, returned as one JSON value (#354): SQLite steps once, so
|
||||
the read takes the GIL back a constant two or three times instead of
|
||||
once per linked row while ComfyUI generates. Peak memory therefore
|
||||
grows with the database's matching linked rows (about 530 bytes each,
|
||||
more with the raw categories of a category read, freed on return); the
|
||||
result is bounded by *wanted* (the listing's normalized paths) when
|
||||
given. Where the JSON step fails (no JSON functions, a BLOB value, a
|
||||
JSON text over SQLite's length limit) the rows are streamed as before.
|
||||
The positive id is the card's linked positive. One SELECT whose rows
|
||||
are streamed; with *wanted* (the listing's normalized paths) only those
|
||||
paths are kept, so memory is bounded by the listing, not the database.
|
||||
"""
|
||||
sql, params = link_filter_query(
|
||||
q, tags, every_link=every_link, category=category
|
||||
)
|
||||
categories = category is not None
|
||||
sql, params = link_filter_query(q, category, tags, every_link=every_link)
|
||||
matches: Dict[str, Tuple[bool, Optional[int], int]] = {}
|
||||
with self.model.get_connection() as conn:
|
||||
rows = None
|
||||
if not _one_step_unsupported:
|
||||
rows = self._one_step_link_rows(conn, sql, params, categories)
|
||||
if rows is None:
|
||||
return _streamed_link_matches(conn, sql, params, wanted, categories)
|
||||
return _link_matches_from_rows(rows, categories, wanted)
|
||||
|
||||
def _one_step_link_rows(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
sql: str,
|
||||
params: Dict[str, Any],
|
||||
categories: bool = False,
|
||||
) -> Optional[List[List[Any]]]:
|
||||
"""*sql*'s rows from one JSON value, or None when this read must stream.
|
||||
|
||||
An interrupted read (Move library retiring the connection) is raised
|
||||
as before, never retried row by row.
|
||||
"""
|
||||
global _one_step_unsupported, _one_step_warned
|
||||
error: Optional[str] = None
|
||||
try:
|
||||
cursor = conn.execute(link_matches_json_query(sql, categories), params)
|
||||
try:
|
||||
((document,),) = cursor.fetchall()
|
||||
finally:
|
||||
cursor.close() # resets the statement: its snapshot ends now
|
||||
except (sqlite3.OperationalError, sqlite3.DataError) as failure:
|
||||
# DataError: the JSON text is longer than SQLITE_LIMIT_LENGTH.
|
||||
error = str(failure)
|
||||
if error == "interrupted":
|
||||
raise
|
||||
if error.startswith("no such function"):
|
||||
_one_step_unsupported = True
|
||||
else:
|
||||
if document is not None:
|
||||
return json.loads(document)
|
||||
error = "a BLOB value"
|
||||
if not _one_step_warned:
|
||||
_one_step_warned = True
|
||||
self.logger.warning(
|
||||
"Reading image links row by row (slower while ComfyUI "
|
||||
"generates); the one-step read failed: %s",
|
||||
error,
|
||||
)
|
||||
return None
|
||||
for path, rating, matched, positive in conn.execute(sql, params):
|
||||
if wanted is None or path in wanted:
|
||||
matches[path] = (bool(matched), rating, positive)
|
||||
return matches
|
||||
|
||||
def get_image_link_facts(
|
||||
self, paths: List[str], q: str = ""
|
||||
) -> Dict[str, Tuple[bool, Optional[int], int, Any, Tuple[str, ...]]]:
|
||||
) -> Dict[str, Tuple[bool, Optional[int], int, Optional[str], Tuple[str, ...]]]:
|
||||
"""Map each normalized path with one unambiguous link to its positive's
|
||||
(hit, rating, positive id, inherited category, tag names).
|
||||
(hit, rating, positive id, category, tag names).
|
||||
|
||||
The link is resolved as in get_image_link_matches, and ``hit`` says
|
||||
whether *q* occurs in the positive's text, Library title or a tag
|
||||
@@ -2734,7 +2583,7 @@ class PromptDatabase(
|
||||
if q:
|
||||
hit = _LINK_FILTER_HIT
|
||||
params["pattern"] = f"%{escape_like(q)}%"
|
||||
rows: Dict[str, LinkMatch] = {}
|
||||
rows: Dict[str, Tuple[bool, Optional[int], int, Optional[str]]] = {}
|
||||
names: Dict[int, List[str]] = {}
|
||||
unique = list(dict.fromkeys(paths))
|
||||
with self.model.get_connection() as conn:
|
||||
@@ -2743,8 +2592,10 @@ class PromptDatabase(
|
||||
bound = {f"path{i}": path for i, path in enumerate(chunk)}
|
||||
marks = ",".join(f":{key}" for key in bound)
|
||||
sql = _LINK_FACTS_SQL.format(hit=hit, marks=marks)
|
||||
for path, *row in conn.execute(sql, {**params, **bound}):
|
||||
rows[path] = _link_match(*row)
|
||||
for path, rating, matched, positive, category in conn.execute(
|
||||
sql, {**params, **bound}
|
||||
):
|
||||
rows[path] = (bool(matched), rating, positive, category)
|
||||
positives = sorted({row[2] for row in rows.values()})
|
||||
for start in range(0, len(positives), _FACTS_CHUNK):
|
||||
chunk = positives[start : start + _FACTS_CHUNK]
|
||||
@@ -2757,16 +2608,20 @@ class PromptDatabase(
|
||||
path: (*row, tuple(names.get(row[2], ()))) for path, row in rows.items()
|
||||
}
|
||||
|
||||
def get_image_paths_matching(self, q: str, tags: List[str]) -> Set[str]:
|
||||
"""Normalized paths whose linked positive matches *q* and *tags*.
|
||||
def get_image_paths_matching(
|
||||
self, q: str, category: Optional[str], tags: List[str]
|
||||
) -> Set[str]:
|
||||
"""Normalized paths whose linked positive matches *q*, *category* and *tags*.
|
||||
|
||||
See get_image_link_matches; an empty *q* matches every linked image
|
||||
the tags admit.
|
||||
the category and tags admit.
|
||||
"""
|
||||
return {
|
||||
path
|
||||
for path, match in self.get_image_link_matches(q, tags).items()
|
||||
if match[0] or not q
|
||||
for path, (hit, _rating, _positive) in self.get_image_link_matches(
|
||||
q, category, tags
|
||||
).items()
|
||||
if hit or not q
|
||||
}
|
||||
|
||||
def get_prompt_id_for_image(self, image_path: str) -> Optional[int]:
|
||||
|
||||
+15
-111
@@ -1,14 +1,7 @@
|
||||
"""Atomic metadata enrichment for a single known generation context.
|
||||
|
||||
Also the helpers both ingestion store paths share (#247): the file's link
|
||||
rows and the one-prompt-per-file rule (``file_link_rows``,
|
||||
``link_conflict``) and the marks a prompt created by ingestion carries
|
||||
(``Marks``, ``mark_created``).
|
||||
"""
|
||||
"""Atomic metadata enrichment for a single known generation context."""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Dict
|
||||
|
||||
from .models import normalize_image_path
|
||||
from .operations import _coerce_file_info
|
||||
@@ -44,85 +37,16 @@ def blank_positive(resolution: Dict[str, Any]) -> bool:
|
||||
return not (isinstance(text, str) and text.strip())
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Marks:
|
||||
"""What a prompt ingestion creates carries, so it can be found and removed.
|
||||
|
||||
``note`` is formatted with ``name``, the file's name.
|
||||
"""
|
||||
|
||||
category: str
|
||||
tags: Tuple[str, ...]
|
||||
note: str
|
||||
|
||||
|
||||
SCAN_MARKS = Marks("scanned", ("auto-scanned",), "Auto-scanned from {name}")
|
||||
RUN_MARKS = Marks("scanned", ("auto-scanned",), "Saved from a workflow run: {name}")
|
||||
|
||||
|
||||
def mark_created(conn: Any, db: Any, prompt_id: int, marks: Marks, name: str) -> None:
|
||||
"""Apply ``marks`` to a prompt row the caller just created on ``conn``."""
|
||||
conn.execute(
|
||||
"UPDATE prompts SET category = ?, notes = ? WHERE id = ?",
|
||||
(marks.category, marks.note.format(name=name), prompt_id),
|
||||
)
|
||||
db._sync_prompt_tags(conn, prompt_id, list(marks.tags))
|
||||
|
||||
|
||||
def file_link_rows(conn: Any, image_path: str) -> List[Any]:
|
||||
"""The file's link rows, each with its prompt's hash.
|
||||
|
||||
Matched under the path and its resolved spelling (an output folder
|
||||
reached through a symlink or junction), each normalized and raw;
|
||||
deduplicated by row id.
|
||||
"""
|
||||
rows: Dict[Any, Any] = {}
|
||||
for spelling in dict.fromkeys((image_path, _resolved(image_path))):
|
||||
for row in conn.execute(
|
||||
"SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
|
||||
"JOIN prompts p ON p.id=gi.prompt_id "
|
||||
"WHERE gi.file_path=? OR gi.image_path=?",
|
||||
(normalize_image_path(spelling), spelling),
|
||||
).fetchall():
|
||||
rows.setdefault(row["id"], row)
|
||||
return list(rows.values())
|
||||
|
||||
|
||||
def _resolved(path: str) -> str:
|
||||
try:
|
||||
return os.path.realpath(path)
|
||||
except (OSError, ValueError):
|
||||
return path
|
||||
|
||||
|
||||
def link_conflict(rows: List[Any], prompt_hash: str) -> bool:
|
||||
"""One prompt per file: more than one row, or a row of another prompt."""
|
||||
return len(rows) > 1 or bool(rows and rows[0]["positive_hash"] != prompt_hash)
|
||||
|
||||
|
||||
def ingest_resolved_pair(
|
||||
db: Any,
|
||||
image_path: str,
|
||||
metadata: Dict[str, Any],
|
||||
resolution: Dict[str, Any],
|
||||
*,
|
||||
marks: Optional[Marks] = None,
|
||||
db: Any, image_path: str, metadata: Dict[str, Any], resolution: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
"""Store a resolved pair and link the file to it in one transaction.
|
||||
|
||||
With ``marks`` (ingestion from files), each pair component this call
|
||||
creates carries them; without (the PromptManager node path) components
|
||||
are created bare, as always.
|
||||
"""
|
||||
state = resolution.get("state", "unresolved")
|
||||
if state == "resolved" and blank_positive(resolution):
|
||||
state = "unresolved"
|
||||
if state != "resolved":
|
||||
return {"status": state, "pairs_added": 0, "links_added": 0}
|
||||
with db._pair_transaction() as conn:
|
||||
return _ingest_resolved_pair_in(
|
||||
conn, db, image_path, metadata, resolution, marks=marks
|
||||
)
|
||||
return _ingest_resolved_pair_in(conn, db, image_path, metadata, resolution)
|
||||
|
||||
|
||||
def _conflict() -> Dict[str, Any]:
|
||||
@@ -137,7 +61,6 @@ def _ingest_resolved_pair_in(
|
||||
resolution: Dict[str, Any],
|
||||
*,
|
||||
provenance: Dict[str, Any] | None = None,
|
||||
marks: Optional[Marks] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""``ingest_resolved_pair`` inside the caller's open ``_pair_transaction``.
|
||||
|
||||
@@ -146,8 +69,7 @@ def _ingest_resolved_pair_in(
|
||||
negative is a ``conflict`` and nothing is written. The caller owns the
|
||||
transaction, so its own later failure rolls the link back too.
|
||||
``provenance`` replaces the link's default resolver provenance (Needs
|
||||
review records the user's selection there). ``marks`` as in
|
||||
``ingest_resolved_pair``.
|
||||
review records the user's selection there).
|
||||
"""
|
||||
positive, negative = resolution["positive"], resolution["negative"]
|
||||
positive_text = positive.get("storage_text", positive["text"])
|
||||
@@ -160,8 +82,15 @@ def _ingest_resolved_pair_in(
|
||||
"negative": negative,
|
||||
}
|
||||
encoded = dumps_strict(provenance, sort_keys=True, separators=(",", ":"))
|
||||
images = file_link_rows(conn, image_path)
|
||||
if link_conflict(images, generate_prompt_hash(positive_text)):
|
||||
images = conn.execute(
|
||||
"SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
|
||||
"JOIN prompts p ON p.id=gi.prompt_id "
|
||||
"WHERE gi.file_path=? OR gi.image_path=?",
|
||||
(normalized, image_path),
|
||||
).fetchall()
|
||||
if len(images) > 1 or (
|
||||
images and images[0]["positive_hash"] != generate_prompt_hash(positive_text)
|
||||
):
|
||||
return _conflict()
|
||||
existing = images[0] if images else None
|
||||
if existing:
|
||||
@@ -186,12 +115,7 @@ def _ingest_resolved_pair_in(
|
||||
generate_prompt_hash(negative_text) if negative_text.strip() else None,
|
||||
),
|
||||
).fetchone()
|
||||
if marks is None:
|
||||
pair_id = db._save_pair(conn, positive_text, negative_text)
|
||||
else:
|
||||
pair_id = _save_marked_pair(
|
||||
conn, db, positive_text, negative_text, marks, os.path.basename(image_path)
|
||||
)
|
||||
pair_id = db._save_pair(conn, positive_text, negative_text)
|
||||
pair = conn.execute("SELECT * FROM prompt_pairs WHERE id=?", (pair_id,)).fetchone()
|
||||
pairs_added = int(prior_pair is None)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
@@ -273,26 +197,6 @@ def _ingest_resolved_pair_in(
|
||||
}
|
||||
|
||||
|
||||
def _save_marked_pair(
|
||||
conn: Any,
|
||||
db: Any,
|
||||
positive_text: str,
|
||||
negative_text: str,
|
||||
marks: Marks,
|
||||
name: str,
|
||||
) -> int:
|
||||
"""``_save_pair`` that marks each component it creates."""
|
||||
texts = [positive_text] + ([negative_text] if negative_text.strip() else [])
|
||||
ids = []
|
||||
for text in texts:
|
||||
prompt_id, created = db._pair_component_outcome(conn, text)
|
||||
if created:
|
||||
mark_created(conn, db, prompt_id, marks, name)
|
||||
ids.append(prompt_id)
|
||||
negative_id = ids[1] if len(ids) > 1 else None
|
||||
return db._save_pair_components(conn, ids[0], negative_id)[0]
|
||||
|
||||
|
||||
def _document_form(conn: Any, values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""``values`` with the workflow and prompt graph texts stored once.
|
||||
|
||||
|
||||
@@ -229,15 +229,13 @@ def evidence_offers_choice(evidence: Any) -> bool:
|
||||
"""
|
||||
if not isinstance(evidence, dict):
|
||||
return False
|
||||
# The saver anchor's one pair (#390) is a suggestion to confirm.
|
||||
needed = 1 if evidence.get("anchor") == "saver" else MIN_CHOOSABLE
|
||||
if evidence.get("limited"):
|
||||
total = evidence.get("candidate_total")
|
||||
return type(total) is int and total >= needed
|
||||
return type(total) is int and total >= MIN_CHOOSABLE
|
||||
candidates = evidence.get("candidates")
|
||||
if not isinstance(candidates, list):
|
||||
return False
|
||||
return sum(1 for c in candidates if choosable(c)) >= needed
|
||||
return sum(1 for c in candidates if choosable(c)) >= MIN_CHOOSABLE
|
||||
|
||||
|
||||
def evidence_digest(text: str) -> str:
|
||||
|
||||
+28
-153
@@ -15,10 +15,6 @@ only when all of this holds:
|
||||
|
||||
Anything else returns None: the caller leaves the file unlinked. Read-only.
|
||||
|
||||
``recorded_output`` applies the first two rules and returns, for the
|
||||
inspector (#386), the one output node (save node) that run reported the file
|
||||
from and the texts PromptManager recorded at execution per sampler.
|
||||
|
||||
Lookups are batched (``stored_run_prompts``): one parameterized query per
|
||||
chunk of at most ``LOOKUP_CHUNK`` distinct file names, matched in memory. The
|
||||
history schema is left exactly as it is (no extra index), so older builds can
|
||||
@@ -30,7 +26,6 @@ import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import weakref
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
try:
|
||||
@@ -50,8 +45,7 @@ LOOKUP_CHUNK = 500
|
||||
|
||||
_RUNS_FOR_NAMES = (
|
||||
"SELECT DISTINCT o.filename, o.subfolder, o.root_key, o.path_style, o.run_id, "
|
||||
"r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at), "
|
||||
"o.output_node_id "
|
||||
"r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at) "
|
||||
"FROM execution_run_outputs o JOIN execution_runs r ON r.run_id = o.run_id "
|
||||
"WHERE o.folder_type = 'output' AND o.filename IN ({marks})"
|
||||
)
|
||||
@@ -144,10 +138,9 @@ def stored_run_prompt(
|
||||
|
||||
|
||||
def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]:
|
||||
"""``{(filename, subfolder, root_key, style): {run_id: (start, end, nodes)}}``.
|
||||
"""``{(filename, subfolder, root_key, style): {run_id: (start, end)}}``.
|
||||
|
||||
``nodes`` is the set of output node ids that reported the location in that
|
||||
run. One query per ``LOOKUP_CHUNK`` names; None without history tables.
|
||||
One query per ``LOOKUP_CHUNK`` names; None without history tables.
|
||||
"""
|
||||
found: Dict[tuple, dict] = {}
|
||||
conn = db.model.get_connection()
|
||||
@@ -158,51 +151,13 @@ def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]:
|
||||
rows = conn.execute(query, chunk).fetchall() # nosec B608
|
||||
except sqlite3.Error:
|
||||
return None # no history tables (a library without history)
|
||||
for name, subfolder, root_key, style, run_id, began, ended, node in rows:
|
||||
for name, subfolder, root_key, style, run_id, began, ended in rows:
|
||||
runs = found.setdefault((name, subfolder, root_key, style), {})
|
||||
runs.setdefault(run_id, (began, ended, set()))[2].add(node)
|
||||
runs[run_id] = (began, ended)
|
||||
return found
|
||||
|
||||
|
||||
def recorded_runs(
|
||||
db: Any,
|
||||
image_paths: Iterable[str],
|
||||
output_root: Optional[str],
|
||||
path_style: Optional[str],
|
||||
) -> Dict[str, tuple]:
|
||||
"""``{path: (run_id, output node ids)}`` for each file history attributes.
|
||||
|
||||
A file is attributed when exactly one recorded run reported its location
|
||||
and it was last written while that run executed.
|
||||
"""
|
||||
paths = list(dict.fromkeys(path for path in image_paths if path))
|
||||
if not paths or not output_root:
|
||||
return {}
|
||||
style = path_style or ("windows" if os.name == "nt" else "posix")
|
||||
located = {}
|
||||
for path in paths:
|
||||
location = _location(path, output_root, style)
|
||||
if location is not None:
|
||||
located[path] = location
|
||||
if not located:
|
||||
return {}
|
||||
names = sorted({filename for _key, _sub, filename in located.values()})
|
||||
runs = _runs_by_location(db, names)
|
||||
if not runs:
|
||||
return {}
|
||||
attributed = {}
|
||||
for path, (root_key, subfolder, filename) in located.items():
|
||||
reported = runs.get((filename, subfolder, root_key, style), {})
|
||||
if len(reported) != 1:
|
||||
continue
|
||||
((run_id, (began, ended, nodes)),) = reported.items()
|
||||
if _written_during(path, began, ended):
|
||||
attributed[path] = (run_id, nodes)
|
||||
return attributed
|
||||
|
||||
|
||||
def named_prompt(db: Any, run_id: str) -> Optional[int]:
|
||||
"""The one stored prompt every positive context of ``run_id`` names, else None."""
|
||||
def _named_prompt(db: Any, run_id: str) -> Optional[int]:
|
||||
contexts = db.get_execution_contexts(run_id).get("contexts") or []
|
||||
named = {
|
||||
context.get("positive_prompt_id")
|
||||
@@ -228,115 +183,35 @@ def stored_run_prompts(
|
||||
"""
|
||||
paths = list(dict.fromkeys(path for path in image_paths if path))
|
||||
result: Dict[str, Optional[int]] = dict.fromkeys(paths)
|
||||
if not paths or not output_root:
|
||||
return result
|
||||
style = path_style or ("windows" if os.name == "nt" else "posix")
|
||||
located = {}
|
||||
for path in paths:
|
||||
location = _location(path, output_root, style)
|
||||
if location is not None:
|
||||
located[path] = location
|
||||
if not located:
|
||||
return result
|
||||
names = sorted({filename for _key, _sub, filename in located.values()})
|
||||
runs = _runs_by_location(db, names)
|
||||
if not runs:
|
||||
return result
|
||||
prompts: Dict[str, Optional[int]] = {}
|
||||
for path, (run_id, _nodes) in recorded_runs(
|
||||
db, paths, output_root, path_style
|
||||
).items():
|
||||
for path, (root_key, subfolder, filename) in located.items():
|
||||
reported = runs.get((filename, subfolder, root_key, style), {})
|
||||
if len(reported) != 1:
|
||||
continue
|
||||
((run_id, (began, ended)),) = reported.items()
|
||||
if not _written_during(path, began, ended):
|
||||
continue
|
||||
if run_id not in prompts:
|
||||
prompts[run_id] = named_prompt(db, run_id)
|
||||
prompts[run_id] = _named_prompt(db, run_id)
|
||||
result[path] = prompts[run_id]
|
||||
return result
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RecordedOutput:
|
||||
"""What recorded history knows about the run that wrote one file (#386).
|
||||
|
||||
``saver_id`` is the one output node the run reported the file from (None
|
||||
when several did). ``texts`` maps each sampler anchor of the run to the
|
||||
effective texts PromptManager recorded at execution for its roles
|
||||
(``{"positive": text or None, "negative": text or None}``); a role whose
|
||||
text was only inferred from the graph is None.
|
||||
"""
|
||||
|
||||
saver_id: Optional[str]
|
||||
texts: Dict[str, Dict[str, Optional[str]]]
|
||||
|
||||
|
||||
def _recorded_texts(db: Any, run_id: str) -> Dict[str, Dict[str, Optional[str]]]:
|
||||
texts = {}
|
||||
for context in db.get_execution_contexts(run_id).get("contexts") or []:
|
||||
evidence = context.get("evidence") or {}
|
||||
roles = {}
|
||||
for role in ("positive", "negative"):
|
||||
values = context.get(role) or {}
|
||||
proof = (evidence.get(role) or {}).get("effective") or {}
|
||||
value = values.get("effective")
|
||||
recorded = proof.get("provenance") == "recorded"
|
||||
roles[role] = value if recorded and isinstance(value, str) else None
|
||||
texts[context["anchor_id"]] = roles
|
||||
return texts
|
||||
|
||||
|
||||
def recorded_output(
|
||||
db: Any,
|
||||
image_path: str,
|
||||
output_root: Optional[str],
|
||||
path_style: Optional[str] = None,
|
||||
) -> Optional[RecordedOutput]:
|
||||
"""The recorded run facts for ``image_path``, else None (#386).
|
||||
|
||||
The file must be attributed to one run (``recorded_runs``: one run
|
||||
reported its location and it was written during that run). Read-only; it
|
||||
never links anything.
|
||||
"""
|
||||
if not image_path:
|
||||
return None
|
||||
conn = db.model.get_connection()
|
||||
own_transaction = not conn.in_transaction
|
||||
if own_transaction: # outputs and texts from one state of history
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
found = recorded_runs(db, [image_path], output_root, path_style)
|
||||
if image_path not in found:
|
||||
return None
|
||||
run_id, nodes = found[image_path]
|
||||
saver = next(iter(nodes)) if len(nodes) == 1 else None
|
||||
return RecordedOutput(saver, _recorded_texts(db, run_id))
|
||||
finally:
|
||||
if own_transaction:
|
||||
conn.rollback()
|
||||
|
||||
|
||||
_LOGGER = logging.getLogger("prompt_manager.run_attribution")
|
||||
|
||||
|
||||
def recorded_output_lookup(
|
||||
db: Any, *, close: bool = False, reference: Optional[dict] = None
|
||||
) -> Callable[[Any], Any]:
|
||||
"""``lookup(path) -> RecordedOutput or None`` for the inspect routes (#386).
|
||||
|
||||
Uses ComfyUI's output folder. Any failure (no history, a locked library)
|
||||
is logged and names no node, so the file is judged from its own metadata.
|
||||
``close`` closes the calling thread's connection afterwards, for routes
|
||||
that close theirs after reading. ``reference`` (a history occurrence's
|
||||
output reference) spells the file as history recorded it: the route's
|
||||
target is case-normalized on Windows, and history matches names exactly.
|
||||
"""
|
||||
|
||||
def lookup(path: Any) -> Optional[RecordedOutput]:
|
||||
root = host_output_directory()
|
||||
if reference is not None:
|
||||
if reference.get("folder_type") != "output" or not root:
|
||||
return None
|
||||
path = os.path.join(
|
||||
root, reference.get("subfolder") or "", reference.get("filename") or ""
|
||||
)
|
||||
try:
|
||||
return recorded_output(db, str(path), root)
|
||||
except (sqlite3.Error, OSError, ValueError):
|
||||
_LOGGER.warning("Recorded run lookup failed", exc_info=True)
|
||||
return None
|
||||
except Exception:
|
||||
_LOGGER.exception("Recorded run lookup failed unexpectedly")
|
||||
return None
|
||||
finally:
|
||||
if close:
|
||||
db.model.close()
|
||||
|
||||
return lookup
|
||||
|
||||
|
||||
_SEALED_LISTENERS: List[weakref.WeakMethod] = []
|
||||
_SEALED_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
"""Link an output file to the prompt of one text, creating it when new (#247).
|
||||
|
||||
The store step of ingestion from file metadata when the decision is a
|
||||
positive text (``utils.file_prompt``). The prompt insert and the link run in
|
||||
one ``BEGIN IMMEDIATE`` transaction: two writers never store one text twice,
|
||||
a failed link leaves no new prompt behind, and a file stays linked to one
|
||||
prompt (a file already linked to another prompt is a ``conflict``).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from .pair_ingestion import Marks, file_link_rows, link_conflict, mark_created
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
_LOGGER = logging.getLogger("prompt_manager.database.scanned_prompts")
|
||||
|
||||
|
||||
def store_scanned_prompt_or_raise(
|
||||
db: Any, path: str, canonical: Dict[str, Any], text: str, marks: Marks
|
||||
) -> Tuple[str, Optional[int]]:
|
||||
"""``(status, prompt id)`` after linking ``path`` to ``text``'s prompt.
|
||||
|
||||
``added`` (prompt created, carrying ``marks``), ``linked`` (stored text
|
||||
reused), ``unchanged`` (already linked to it) or ``conflict`` (linked to
|
||||
another prompt; nothing written, id None). Database errors are raised
|
||||
and roll the whole store back, so a caller can run it again.
|
||||
"""
|
||||
with db._pair_transaction() as conn:
|
||||
rows = file_link_rows(conn, path)
|
||||
if link_conflict(rows, generate_prompt_hash(text)):
|
||||
return "conflict", None
|
||||
if rows:
|
||||
return "unchanged", rows[0]["prompt_id"]
|
||||
prompt_id, created = db._pair_component_outcome(conn, text)
|
||||
if created:
|
||||
mark_created(conn, db, prompt_id, marks, os.path.basename(path))
|
||||
db._insert_image_link(conn, prompt_id, path, canonical)
|
||||
return ("added" if created else "linked"), prompt_id
|
||||
|
||||
|
||||
def store_scanned_prompt(
|
||||
db: Any, path: str, canonical: Dict[str, Any], text: str, marks: Marks
|
||||
) -> Tuple[str, Optional[int]]:
|
||||
"""The scan's form: an error is logged and reported as ``("found", None)``.
|
||||
|
||||
A write that failed because Move library switched runs again on the new
|
||||
library (``survive_library_switch``); the rerun re-checks the link rows.
|
||||
"""
|
||||
try:
|
||||
return db.survive_library_switch(
|
||||
lambda: store_scanned_prompt_or_raise(db, path, canonical, text, marks)
|
||||
)
|
||||
except Exception as error:
|
||||
_LOGGER.error(
|
||||
f"Failed to store the prompt of {os.path.basename(path)}: {error}"
|
||||
)
|
||||
return "found", None
|
||||
|
||||
|
||||
def link_stored_prompt_or_raise(
|
||||
db: Any, path: str, canonical: Dict[str, Any], prompt_id: int
|
||||
) -> Optional[str]:
|
||||
"""Link ``path`` to the stored prompt ``prompt_id`` (recorded history, #342).
|
||||
|
||||
``linked``, ``unchanged`` (already linked to it), ``conflict`` (linked to
|
||||
another prompt; nothing written) or None when the prompt is gone. One
|
||||
``BEGIN IMMEDIATE`` transaction; database errors are raised.
|
||||
"""
|
||||
with db._pair_transaction() as conn:
|
||||
rows = file_link_rows(conn, path)
|
||||
if len(rows) > 1 or (rows and rows[0]["prompt_id"] != prompt_id):
|
||||
return "conflict"
|
||||
if rows:
|
||||
return "unchanged"
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE id = ?", (prompt_id,)
|
||||
).fetchone()
|
||||
if exists is None:
|
||||
return None
|
||||
db._insert_image_link(conn, prompt_id, path, canonical)
|
||||
return "linked"
|
||||
+15
-55
@@ -13,9 +13,7 @@ same connection before commit. Owner rows and their timestamps never change.
|
||||
|
||||
Category consolidation is previewed in one ``library_snapshot`` and applied in
|
||||
one ``_pair_transaction`` that recomputes the owner projection and requires the
|
||||
preview's ``impact_token``; only rows carrying the exact source change. Image
|
||||
rows holding the source (``image_categories``, #332) are owners too: they are
|
||||
counted, hashed into the token and renamed with the prompts and pairs.
|
||||
preview's ``impact_token``; only rows carrying the exact source change.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
@@ -99,30 +97,16 @@ def _validate_tag_id(tag_id: object) -> None:
|
||||
raise ValueError("Invalid tag ID")
|
||||
|
||||
|
||||
# Category values in Tags & categories (#332 §7): the prompt-backed values
|
||||
# (nonblank values on eligible parents or on active pairs, grouped exactly, so
|
||||
# whitespace and case variants stay distinct; /library/options lists the same
|
||||
# prompt-backed set with its own query) plus the values only image rows hold.
|
||||
# A prompt-backed value is ordered by its oldest owning positive; an image-only
|
||||
# value sorts after all of them (first_pid is SQLite's INTEGER maximum, above
|
||||
# any prompt id), then by value. ``image_count`` counts stored rows, including
|
||||
# files missing on disk. No FULL OUTER JOIN, so it runs on SQLite 3.31+.
|
||||
_IMAGE_ONLY_FIRST_PID = 2**63 - 1
|
||||
_CATEGORY_ROWS = (
|
||||
# The /library/options universe: nonblank values on eligible parents or on
|
||||
# active pairs, grouped exactly (whitespace and case variants stay distinct).
|
||||
_CATEGORY_GROUPS = (
|
||||
"WITH owners(value,pid) AS ("
|
||||
"SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL "
|
||||
"AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!=''),"
|
||||
"image_values(value,image_count) AS ("
|
||||
"SELECT category,COUNT(*) FROM image_categories GROUP BY category),"
|
||||
"grouped(value,first_pid,prompt_count) AS ("
|
||||
"SELECT value,MIN(pid),COUNT(DISTINCT pid) FROM owners GROUP BY value "
|
||||
"UNION ALL SELECT value," + str(_IMAGE_ONLY_FIRST_PID) + ",0 FROM image_values "
|
||||
"WHERE value NOT IN (SELECT value FROM owners)),"
|
||||
"counted AS (SELECT g.value,g.first_pid,g.prompt_count,"
|
||||
"COALESCE(i.image_count,0) AS image_count FROM grouped g "
|
||||
"LEFT JOIN image_values i ON i.value=g.value) "
|
||||
"grouped AS (SELECT value,MIN(pid) AS first_pid,"
|
||||
"COUNT(DISTINCT pid) AS prompt_count FROM owners GROUP BY value) "
|
||||
)
|
||||
|
||||
|
||||
@@ -133,23 +117,15 @@ _CATEGORY_COMPONENTS = (
|
||||
"WHERE p.category=? ORDER BY p.id"
|
||||
)
|
||||
_CATEGORY_PAIRS = "SELECT id,archived FROM prompt_pairs WHERE category=? ORDER BY id"
|
||||
# Image rows holding one exact value (#332 DC7), an index search.
|
||||
_IMAGE_CATEGORY_ROWIDS = (
|
||||
"SELECT rowid FROM image_categories WHERE category=? ORDER BY rowid"
|
||||
)
|
||||
_CATEGORY_EXISTS = (
|
||||
"SELECT EXISTS(SELECT 1 FROM prompts WHERE category=:v)"
|
||||
" OR EXISTS(SELECT 1 FROM prompt_pairs WHERE category=:v)"
|
||||
" OR EXISTS(SELECT 1 FROM image_categories WHERE category=:v)"
|
||||
)
|
||||
|
||||
# Pairs after components and image rows last, so a failing update rolls every
|
||||
# earlier one back. An image row is keyed by its file, never by its value, so
|
||||
# merging into a value other files already hold cannot conflict.
|
||||
# Pairs after components, so a failing pair update rolls both back.
|
||||
_CATEGORY_UPDATES = (
|
||||
"UPDATE prompts SET category=?,updated_at=? WHERE category=?",
|
||||
"UPDATE prompt_pairs SET category=?,updated_at=? WHERE category=?",
|
||||
"UPDATE image_categories SET category=?,updated_at=? WHERE category=?",
|
||||
)
|
||||
|
||||
|
||||
@@ -160,15 +136,8 @@ def category_impact_token(
|
||||
target_exists: bool,
|
||||
components: list,
|
||||
pairs: list,
|
||||
*,
|
||||
images: list = (),
|
||||
) -> str:
|
||||
"""Hash a category projection like ``library_edit_token`` (no secret).
|
||||
|
||||
``images`` are the rowids of the image rows holding ``source`` (#332 DC7).
|
||||
With none the preimage has no ``images`` key, so a prompt-only token keeps
|
||||
the value it had before image categories existed.
|
||||
"""
|
||||
"""Hash a category projection like ``library_edit_token`` (no secret)."""
|
||||
preimage = {
|
||||
"epoch": epoch,
|
||||
"source": source,
|
||||
@@ -177,8 +146,6 @@ def category_impact_token(
|
||||
"components": sorted([int(i), bool(flag)] for i, flag in components),
|
||||
"pairs": sorted([int(i), bool(flag)] for i, flag in pairs),
|
||||
}
|
||||
if images:
|
||||
preimage["images"] = sorted(int(i) for i in images)
|
||||
return canonical_token(_CATEGORY_DOMAIN, preimage)
|
||||
|
||||
|
||||
@@ -358,8 +325,7 @@ class VocabularyOperations:
|
||||
pairs = [
|
||||
[row[0], bool(row[1])] for row in conn.execute(_CATEGORY_PAIRS, (source,))
|
||||
]
|
||||
images = [row[0] for row in conn.execute(_IMAGE_CATEGORY_ROWIDS, (source,))]
|
||||
if not components and not pairs and not images:
|
||||
if not components and not pairs:
|
||||
raise LibraryCommandError("vocabulary-conflict", epoch)
|
||||
target_exists = bool(
|
||||
conn.execute(_CATEGORY_EXISTS, {"v": target}).fetchone()[0]
|
||||
@@ -373,10 +339,9 @@ class VocabularyOperations:
|
||||
"other_prompts": len(components) - eligible,
|
||||
"negative_variants": len(pairs) - archived,
|
||||
"archived_negative_variants": archived,
|
||||
"images": len(images),
|
||||
},
|
||||
"impact_token": category_impact_token(
|
||||
epoch, source, target, target_exists, components, pairs, images=images
|
||||
epoch, source, target, target_exists, components, pairs
|
||||
),
|
||||
}
|
||||
|
||||
@@ -400,8 +365,7 @@ class VocabularyOperations:
|
||||
def apply_vocabulary_category(
|
||||
self, *, expected_epoch: str, source: str, target: str, impact_token: str
|
||||
) -> dict:
|
||||
"""Set the exact ``source`` to ``target`` on every component, pair
|
||||
and image row.
|
||||
"""Set the exact ``source`` to ``target`` on every component and pair.
|
||||
|
||||
The recomputed projection must still match the preview's token; a
|
||||
mismatch is ``vocabulary-conflict`` with nothing written.
|
||||
@@ -458,15 +422,15 @@ class VocabularyOperations:
|
||||
_limit(limit)
|
||||
key = _category_after(after)
|
||||
total = conn.execute(
|
||||
_CATEGORY_ROWS + "SELECT COUNT(*) FROM counted"
|
||||
_CATEGORY_GROUPS + "SELECT COUNT(*) FROM grouped"
|
||||
).fetchone()[0]
|
||||
where, params = "", []
|
||||
if key is not None:
|
||||
where = " WHERE first_pid>? OR (first_pid=? AND value>?)"
|
||||
params = [key[0], key[0], key[1]]
|
||||
rows = conn.execute(
|
||||
_CATEGORY_ROWS
|
||||
+ "SELECT value,first_pid,prompt_count,image_count FROM counted"
|
||||
_CATEGORY_GROUPS
|
||||
+ "SELECT value,first_pid,prompt_count FROM grouped"
|
||||
+ where
|
||||
+ " ORDER BY first_pid,value LIMIT ?",
|
||||
[*params, limit + 1],
|
||||
@@ -476,11 +440,7 @@ class VocabularyOperations:
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"categories": [
|
||||
{
|
||||
"value": row["value"],
|
||||
"prompt_count": row["prompt_count"],
|
||||
"image_count": row["image_count"],
|
||||
}
|
||||
{"value": row["value"], "prompt_count": row["prompt_count"]}
|
||||
for row in rows
|
||||
],
|
||||
"total": total,
|
||||
|
||||
+6
-14
@@ -26,13 +26,13 @@ except ImportError:
|
||||
InputTypeDict = dict
|
||||
|
||||
try:
|
||||
from .prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
|
||||
from .prompt_manager_base import PromptManagerBase
|
||||
except ImportError:
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
|
||||
from prompt_manager_base import PromptManagerBase
|
||||
|
||||
|
||||
class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
@@ -115,7 +115,6 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
)
|
||||
},
|
||||
),
|
||||
"save_to_library": SAVE_TO_LIBRARY_INPUT,
|
||||
},
|
||||
"hidden": {
|
||||
# The queued graph and this node's id: used to tell the positive
|
||||
@@ -153,7 +152,6 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
append_text: str = "",
|
||||
prompt: Optional[Dict[str, Any]] = None,
|
||||
unique_id: Optional[str] = None,
|
||||
save_to_library: bool = True,
|
||||
) -> Tuple[Any]:
|
||||
"""
|
||||
Encode the text prompt and save it to the database.
|
||||
@@ -166,7 +164,6 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
search_text: Text to search for in past prompts
|
||||
prepend_text: Text to prepend to the main prompt
|
||||
append_text: Text to append to the main prompt
|
||||
save_to_library: Off: nothing is saved, counted or tracked (#90)
|
||||
|
||||
Returns:
|
||||
Tuple containing the conditioning for the diffusion model and the final
|
||||
@@ -208,9 +205,7 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
# Save prompt to database and set execution context for gallery tracking
|
||||
prompt_id = None
|
||||
extended_tags = []
|
||||
if not save_to_library:
|
||||
self._skip_not_saved(prompt, unique_id)
|
||||
elif storage_text and storage_text.strip():
|
||||
if storage_text and storage_text.strip():
|
||||
self.logger.debug(f"Processing prompt text: {storage_text[:100]}...")
|
||||
|
||||
# Add prepend/append info to tags if they exist
|
||||
@@ -254,11 +249,6 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
tokens = clip.tokenize(encoding_text)
|
||||
conditioning = clip.encode_from_tokens_scheduled(tokens)
|
||||
|
||||
self.logger.info(f"CLIP encoding completed, text: {repr(encoding_text)[:80]}")
|
||||
result = (conditioning, encoding_text)
|
||||
if not save_to_library:
|
||||
return result # #90: no metadata registration, no history text
|
||||
|
||||
# Register with ComfyUI integration for standard metadata compatibility
|
||||
node_id = f"promptmanager_{int(time.time() * 1000)}"
|
||||
self.comfyui_integration.register_prompt(
|
||||
@@ -272,10 +262,12 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
"append_text": append_text.strip() if append_text else None,
|
||||
},
|
||||
)
|
||||
|
||||
self.logger.info(f"CLIP encoding completed, text: {repr(encoding_text)[:80]}")
|
||||
result = (conditioning, encoding_text)
|
||||
record_runtime_context(
|
||||
type(self).__name__, text, prepend_text, append_text, encoding_text
|
||||
)
|
||||
self._record_cached_item(unique_id, (encoding_text,), storage_text)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
|
||||
+5
-96
@@ -23,16 +23,9 @@ try:
|
||||
from .utils.image_monitor import get_image_monitor
|
||||
from .utils.prompt_tracker import get_prompt_tracker
|
||||
from .utils.hashing import generate_prompt_hash
|
||||
from .utils.prompt_graph import resolve_text, run_prompt_nodes, save_setting
|
||||
from .utils.prompt_graph import resolve_text, run_prompt_nodes
|
||||
from .utils.run_keys import executing_run_id, prompt_node_key
|
||||
from .utils.usage_tracking import PENDING_FIRST_USE
|
||||
from .utils.cached_runs import (
|
||||
CACHED_RUNS,
|
||||
Role,
|
||||
Seen,
|
||||
own_counts,
|
||||
stored_facts,
|
||||
)
|
||||
except ImportError:
|
||||
import sys
|
||||
|
||||
@@ -42,32 +35,9 @@ except ImportError:
|
||||
from utils.image_monitor import get_image_monitor
|
||||
from utils.prompt_tracker import get_prompt_tracker
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.prompt_graph import resolve_text, run_prompt_nodes, save_setting
|
||||
from utils.prompt_graph import resolve_text, run_prompt_nodes
|
||||
from utils.run_keys import executing_run_id, prompt_node_key
|
||||
from utils.usage_tracking import PENDING_FIRST_USE
|
||||
from utils.cached_runs import (
|
||||
CACHED_RUNS,
|
||||
Role,
|
||||
Seen,
|
||||
own_counts,
|
||||
stored_facts,
|
||||
)
|
||||
|
||||
|
||||
# The optional input every PromptManager node declares last (#90): declared
|
||||
# last so no widget value of a workflow saved before it shifts.
|
||||
SAVE_TO_LIBRARY_INPUT = (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"label_on": "save",
|
||||
"label_off": "skip",
|
||||
"tooltip": (
|
||||
"Save this node's prompt to the library, count its runs and link its "
|
||||
"images. Off: PromptManager ignores the run; the output is unchanged."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class PromptManagerBase:
|
||||
@@ -101,10 +71,8 @@ class PromptManagerBase:
|
||||
"""Save the prompt to the SQLite database.
|
||||
|
||||
Runs are normally counted by the queue hook (utils/usage_tracking.py), so a
|
||||
save only counts one when asked to (text the hook could not see, unless
|
||||
this job already counted the node from ComfyUI's cache) or when the hook
|
||||
queued this prompt before it existed. The two are independent: the
|
||||
pending catch-up is written even when ``count_run`` is False.
|
||||
save only counts one when asked to (batch items whose text the hook could not
|
||||
see) or when the hook queued this prompt before it existed.
|
||||
|
||||
Args:
|
||||
text: The prompt text
|
||||
@@ -172,9 +140,7 @@ class PromptManagerBase:
|
||||
resolved = resolve_text(prompt_graph, node_id)
|
||||
return {
|
||||
"is_run_prompt": node_id in run_prompt_nodes(prompt_graph),
|
||||
# The hook skips a switch it cannot read (#90): the node counts
|
||||
"seen_at_queue": resolved is not None
|
||||
and save_setting(prompt_graph, node_id) is True
|
||||
and generate_prompt_hash(resolved) == generate_prompt_hash(text),
|
||||
"known": True,
|
||||
}
|
||||
@@ -201,19 +167,8 @@ class PromptManagerBase:
|
||||
batch_item = (
|
||||
role["known"] and role["is_run_prompt"] and not role["seen_at_queue"]
|
||||
)
|
||||
counts = own_counts(
|
||||
type(self).__name__,
|
||||
stored_facts(text),
|
||||
Role(
|
||||
frozenset({"text"} if role["known"] and role["is_run_prompt"] else ())
|
||||
),
|
||||
Seen(text=role["seen_at_queue"]),
|
||||
)
|
||||
prompt_id = self._save_prompt_to_database(
|
||||
text=text,
|
||||
category=category,
|
||||
tags=tags,
|
||||
count_run=bool(counts) and not self._replayed(unique_id),
|
||||
text=text, category=category, tags=tags, count_run=batch_item
|
||||
)
|
||||
if role["is_run_prompt"] and not prompt_id:
|
||||
# The save failed: an older entry of this key must not stand in (#342)
|
||||
@@ -227,34 +182,6 @@ class PromptManagerBase:
|
||||
)
|
||||
return prompt_id
|
||||
|
||||
def _cached_runs(self) -> Any:
|
||||
return getattr(self, "cached_runs", CACHED_RUNS)
|
||||
|
||||
def _replayed(self, unique_id: Any) -> bool:
|
||||
"""This job already counted the node from ComfyUI's cache (#347).
|
||||
|
||||
A RAM-pressure cache can free the entry after ``execution_cached``, so
|
||||
the node executes in a job that replayed it: its own count is skipped.
|
||||
"""
|
||||
try:
|
||||
return unique_id is not None and self._cached_runs().replayed(unique_id)
|
||||
except Exception as error:
|
||||
self.logger.debug(f"Could not check the cached-run replay: {error}")
|
||||
return False
|
||||
|
||||
def _record_cached_item(
|
||||
self, unique_id: Any, outputs: tuple, text: Any, negative_text: Any = None
|
||||
) -> None:
|
||||
"""Remember this execution's outputs and stored text for cached jobs (#347)."""
|
||||
if unique_id is None:
|
||||
return
|
||||
try:
|
||||
self._cached_runs().record_item(
|
||||
unique_id, outputs, stored_facts(text, negative_text)
|
||||
)
|
||||
except Exception as error:
|
||||
self.logger.debug(f"Could not record the cached-run receipt: {error}")
|
||||
|
||||
@staticmethod
|
||||
def _run_identity(prompt_graph: Any, unique_id: Any) -> Dict[str, Any]:
|
||||
"""The tracker's proof that an image belongs to this execution (#342)."""
|
||||
@@ -263,24 +190,6 @@ class PromptManagerBase:
|
||||
"run_id": executing_run_id(),
|
||||
}
|
||||
|
||||
def _skip_not_saved(self, prompt_graph: Any = None, unique_id: Any = None) -> None:
|
||||
"""``save_to_library`` is off: nothing is saved, counted or tracked (#90).
|
||||
|
||||
A literal off makes the node no run prompt (``utils.prompt_graph``), so
|
||||
no image looks up its run key. A switch wired from a node the graph
|
||||
cannot read keeps the node a run prompt: its run key is marked like an
|
||||
empty editor's, so an earlier run's entry of that key never links this
|
||||
run's images.
|
||||
"""
|
||||
message = (
|
||||
"Save to library is off for this node: its prompt is not saved, the "
|
||||
"run is not counted and its images are not linked"
|
||||
)
|
||||
if isinstance(prompt_graph, dict) and unique_id is not None:
|
||||
self._mark_untracked(prompt_graph, unique_id, message)
|
||||
else:
|
||||
self.logger.info(message)
|
||||
|
||||
def _skip_empty_prompt(self, prompt_graph: Any, unique_id: Any) -> None:
|
||||
"""The ``text`` editor is empty: save and track nothing (#342).
|
||||
|
||||
|
||||
+26
-63
@@ -12,21 +12,9 @@ else:
|
||||
|
||||
try:
|
||||
from .prompt_manager import IO, InputTypeDict, PromptManagerFocused
|
||||
from .utils.cached_runs import (
|
||||
PAIR,
|
||||
Role,
|
||||
Seen,
|
||||
own_counts,
|
||||
pair_components,
|
||||
stored_facts,
|
||||
)
|
||||
from .utils.hashing import generate_prompt_hash
|
||||
from .utils.pair_execution import record_pair_execution
|
||||
from .utils.prompt_graph import (
|
||||
intrinsic_pair_nodes,
|
||||
run_prompt_components,
|
||||
save_setting,
|
||||
)
|
||||
from .utils.prompt_graph import intrinsic_pair_nodes, run_prompt_components
|
||||
from .utils.prompt_pair_resolver import resolve_text_input
|
||||
from .utils.usage_tracking import (
|
||||
PENDING_FIRST_USE,
|
||||
@@ -35,21 +23,9 @@ try:
|
||||
)
|
||||
except ImportError:
|
||||
from prompt_manager import IO, InputTypeDict, PromptManagerFocused
|
||||
from utils.cached_runs import (
|
||||
PAIR,
|
||||
Role,
|
||||
Seen,
|
||||
own_counts,
|
||||
pair_components,
|
||||
stored_facts,
|
||||
)
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.pair_execution import record_pair_execution
|
||||
from utils.prompt_graph import (
|
||||
intrinsic_pair_nodes,
|
||||
run_prompt_components,
|
||||
save_setting,
|
||||
)
|
||||
from utils.prompt_graph import intrinsic_pair_nodes, run_prompt_components
|
||||
from utils.prompt_pair_resolver import resolve_text_input
|
||||
from utils.usage_tracking import (
|
||||
PENDING_FIRST_USE,
|
||||
@@ -98,13 +74,8 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
prompt: Optional[Dict[str, Any]] = None,
|
||||
unique_id: Optional[str] = None,
|
||||
extra_pnginfo: Optional[Dict[str, Any]] = None,
|
||||
save_to_library: bool = True,
|
||||
) -> Tuple[Any, Any, str, str]:
|
||||
"""Encode both sides once; database and tracking failures never block CLIP.
|
||||
|
||||
With ``save_to_library`` off nothing is saved, counted, tracked or
|
||||
recorded (#90); the four outputs are the same.
|
||||
"""
|
||||
"""Encode both sides once; database and tracking failures never block CLIP."""
|
||||
if clip is None:
|
||||
raise RuntimeError("ERROR: clip input is invalid: None")
|
||||
if not isinstance(text, str) or not isinstance(negative_text, str):
|
||||
@@ -122,10 +93,6 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
negative_conditioning = clip.encode_from_tokens_scheduled(
|
||||
clip.tokenize(negative_text)
|
||||
)
|
||||
result = positive_conditioning, negative_conditioning, positive, negative_text
|
||||
if not save_to_library:
|
||||
self._skip_not_saved(prompt, unique_id)
|
||||
return result
|
||||
record_pair_execution(
|
||||
extra_pnginfo,
|
||||
unique_id,
|
||||
@@ -186,6 +153,7 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
)
|
||||
except Exception as error:
|
||||
self.logger.warning(f"Could not register paired metadata: {error}")
|
||||
result = positive_conditioning, negative_conditioning, positive, negative_text
|
||||
record_runtime_context(
|
||||
type(self).__name__,
|
||||
text,
|
||||
@@ -197,9 +165,6 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
record_runtime_context(
|
||||
type(self).__name__, negative_text, "", "", negative_text, role="negative"
|
||||
)
|
||||
self._record_cached_item(
|
||||
unique_id, (positive, negative_text), text, negative_text
|
||||
)
|
||||
return result
|
||||
|
||||
def _save_and_track_pair(
|
||||
@@ -235,9 +200,6 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
resolved_negative = (
|
||||
resolve_text_input(prompt, node_id, "negative_text") if known else None
|
||||
)
|
||||
if known and save_setting(prompt, node_id) is not True:
|
||||
# The hook skips a switch it cannot read (#90): the node counts
|
||||
resolved_positive = resolved_negative = None
|
||||
positive_seen = resolved_positive is not None and generate_prompt_hash(
|
||||
resolved_positive
|
||||
) == generate_prompt_hash(text)
|
||||
@@ -247,35 +209,36 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
and generate_prompt_hash(resolved_negative)
|
||||
== generate_prompt_hash(negative_text)
|
||||
)
|
||||
negative_seen = resolved_negative is not None and generate_prompt_hash(
|
||||
resolved_negative
|
||||
) == generate_prompt_hash(negative_text)
|
||||
positive_pending = getattr(self, "pending_first_use", PENDING_FIRST_USE)
|
||||
pair_pending = getattr(self, "pending_pair_first_use", PENDING_PAIR_FIRST_USE)
|
||||
facts = stored_facts(text, negative_text)
|
||||
role = Role(frozenset(used_fields), pair_used)
|
||||
# Own counts are skipped when this job already counted the node from
|
||||
# ComfyUI's cache (#347); the pending first-use catch-up never is.
|
||||
counts = (
|
||||
[]
|
||||
if self._replayed(unique_id)
|
||||
else own_counts(
|
||||
PAIR, facts, role, Seen(positive_seen, negative_seen, pair_seen)
|
||||
)
|
||||
)
|
||||
component_ids = {"text": prompt_id, "negative_text": pair["negative_prompt_id"]}
|
||||
for field, component_hash in pair_components(facts, role):
|
||||
component_id = component_ids[field]
|
||||
if not component_id:
|
||||
counted = set()
|
||||
for field, component_text, resolved_text, component_id in (
|
||||
("text", text, resolved_positive, prompt_id),
|
||||
(
|
||||
"negative_text",
|
||||
negative_text,
|
||||
resolved_negative,
|
||||
pair["negative_prompt_id"],
|
||||
),
|
||||
):
|
||||
component_hash = generate_prompt_hash(component_text)
|
||||
if (
|
||||
field not in used_fields
|
||||
or not component_id
|
||||
or component_hash in counted
|
||||
):
|
||||
continue
|
||||
runs = positive_pending.consume(component_hash) + int(
|
||||
("prompt", component_hash) in counts
|
||||
counted.add(component_hash)
|
||||
seen_at_queue = (
|
||||
resolved_text is not None
|
||||
and generate_prompt_hash(resolved_text) == component_hash
|
||||
)
|
||||
runs = positive_pending.consume(component_hash) + int(not seen_at_queue)
|
||||
if runs:
|
||||
self.db.record_prompt_use(component_id, times=runs)
|
||||
if pair_used:
|
||||
runs = pair_pending.consume(pair_run_key(text, negative_text)) + int(
|
||||
("pair", facts.positive_hash, facts.negative_hash) in counts
|
||||
not pair_seen
|
||||
)
|
||||
if runs:
|
||||
self.db.record_prompt_pair_use(pair_id, times=runs)
|
||||
|
||||
+6
-14
@@ -26,13 +26,13 @@ except ImportError:
|
||||
InputTypeDict = dict
|
||||
|
||||
try:
|
||||
from .prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
|
||||
from .prompt_manager_base import PromptManagerBase
|
||||
except ImportError:
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
|
||||
from prompt_manager_base import PromptManagerBase
|
||||
|
||||
|
||||
class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
@@ -115,7 +115,6 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
),
|
||||
},
|
||||
),
|
||||
"save_to_library": SAVE_TO_LIBRARY_INPUT,
|
||||
},
|
||||
"hidden": {
|
||||
# The queued graph and this node's id: used to tell the positive
|
||||
@@ -150,7 +149,6 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
append_text: str = "",
|
||||
prompt: Optional[Dict[str, Any]] = None,
|
||||
unique_id: Optional[str] = None,
|
||||
save_to_library: bool = True,
|
||||
) -> Tuple[str]:
|
||||
"""
|
||||
Process the text prompt and save it to the database.
|
||||
@@ -162,7 +160,6 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
search_text: Text to search for in past prompts
|
||||
prepend_text: Text to prepend to the main prompt
|
||||
append_text: Text to append to the main prompt
|
||||
save_to_library: Off: nothing is saved, counted or tracked (#90)
|
||||
|
||||
Returns:
|
||||
Tuple containing the final processed text string
|
||||
@@ -187,9 +184,7 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
# Save prompt to database and set execution context for gallery tracking
|
||||
prompt_id = None
|
||||
extended_tags = []
|
||||
if not save_to_library:
|
||||
self._skip_not_saved(prompt, unique_id)
|
||||
elif storage_text and storage_text.strip():
|
||||
if storage_text and storage_text.strip():
|
||||
self.logger.debug(f"Processing prompt text: {storage_text[:100]}...")
|
||||
|
||||
# Add prepend/append info to tags if they exist
|
||||
@@ -225,11 +220,6 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
else:
|
||||
self._skip_empty_prompt(prompt, unique_id)
|
||||
|
||||
self.logger.debug(f"Text processing completed: {final_text[:100]}...")
|
||||
result = (final_text,)
|
||||
if not save_to_library:
|
||||
return result # #90: no metadata registration, no history text
|
||||
|
||||
# Register with ComfyUI integration for standard metadata compatibility
|
||||
node_id = f"promptmanagertext_{int(time.time() * 1000)}"
|
||||
self.comfyui_integration.register_prompt(
|
||||
@@ -243,10 +233,12 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
"append_text": append_text.strip() if append_text else None,
|
||||
},
|
||||
)
|
||||
|
||||
self.logger.debug(f"Text processing completed: {final_text[:100]}...")
|
||||
result = (final_text,)
|
||||
record_runtime_context(
|
||||
type(self).__name__, text, prepend_text, append_text, final_text
|
||||
)
|
||||
self._record_cached_item(unique_id, (final_text,), storage_text)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
|
||||
+82
-6
@@ -72,11 +72,9 @@ from .diagnostics_report import DiagnosticsReportMixin
|
||||
from .autotag_routes import AutotagRoutesMixin
|
||||
from .lora_integration import LoraIntegrationMixin
|
||||
from .lora_import_routes import LoraImportRoutesMixin
|
||||
from .image_categories import ImageCategoryRoutesMixin
|
||||
|
||||
try:
|
||||
from ...database.operations import PromptDatabase
|
||||
from ...utils.file_prompt import parse_comfyui_prompt
|
||||
from ...utils.logging_config import get_logger
|
||||
from ...utils.video_metadata import is_video_path, read_video_metadata
|
||||
except ImportError:
|
||||
@@ -86,7 +84,6 @@ except ImportError:
|
||||
0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
)
|
||||
from database.operations import PromptDatabase
|
||||
from utils.file_prompt import parse_comfyui_prompt
|
||||
from utils.logging_config import get_logger
|
||||
from utils.video_metadata import is_video_path, read_video_metadata
|
||||
|
||||
@@ -339,7 +336,6 @@ class PromptManagerAPI(
|
||||
AutotagRoutesMixin,
|
||||
LoraIntegrationMixin,
|
||||
LoraImportRoutesMixin,
|
||||
ImageCategoryRoutesMixin,
|
||||
):
|
||||
"""REST API handler for PromptManager operations and web interface.
|
||||
|
||||
@@ -704,7 +700,6 @@ class PromptManagerAPI(
|
||||
self._register_autotag_routes(routes)
|
||||
self._register_lora_routes(routes)
|
||||
self._register_lora_import_routes(routes)
|
||||
self._register_image_category_routes(routes)
|
||||
|
||||
# Register gzip compression middleware (once)
|
||||
global _gzip_registered
|
||||
@@ -1030,4 +1025,85 @@ class PromptManagerAPI(
|
||||
|
||||
def _parse_comfyui_prompt(self, metadata):
|
||||
"""Parse ComfyUI and A1111 prompt data from metadata."""
|
||||
return parse_comfyui_prompt(metadata)
|
||||
result = {
|
||||
"prompt": None,
|
||||
"workflow": None,
|
||||
"parameters": {},
|
||||
"positive_prompt": None,
|
||||
"negative_prompt": None,
|
||||
}
|
||||
|
||||
# Check for A1111/Forge style parameters first.
|
||||
# The chunk is one block: a possibly multi-line positive prompt, then a
|
||||
# "Negative prompt:" section, then a trailing "Steps: ..." settings line.
|
||||
# Taking lines[0] as the positive prompt truncates any multi-line prompt
|
||||
# to one line, so split on the settings line instead.
|
||||
if "parameters" in metadata:
|
||||
import re as _re
|
||||
|
||||
params = metadata["parameters"]
|
||||
body = params
|
||||
settings = ""
|
||||
_sm = _re.search(r"\nSteps:\s", body)
|
||||
if _sm:
|
||||
settings = body[_sm.start() + 1 :].strip()
|
||||
body = body[: _sm.start()]
|
||||
negative = None
|
||||
_nm = _re.search(r"(?im)^\s*negative prompt:\s*", body)
|
||||
if _nm:
|
||||
negative = body[_nm.end() :].strip()
|
||||
body = body[: _nm.start()]
|
||||
elif body.lower().lstrip().startswith("negative prompt:"):
|
||||
negative = body.lower().lstrip()[len("negative prompt:") :].strip()
|
||||
body = ""
|
||||
result["positive_prompt"] = body.strip() or None
|
||||
if negative:
|
||||
result["negative_prompt"] = negative
|
||||
# Store raw parameters too
|
||||
result["parameters"]["parameters"] = params
|
||||
if settings:
|
||||
for _f in ("Steps", "Sampler", "Schedule type", "CFG scale", "Seed"):
|
||||
_fm = _re.search(
|
||||
r"(?i)(?:^|,\s*)" + _re.escape(_f) + r":\s*([^,]+)", settings
|
||||
)
|
||||
if _fm:
|
||||
result["parameters"][_f.lower().replace(" ", "_")] = _fm.group(
|
||||
1
|
||||
).strip()
|
||||
|
||||
# If no A1111 format found, proceed with ComfyUI parsing
|
||||
if result["positive_prompt"] is None:
|
||||
# Check for direct prompt field
|
||||
if "prompt" in metadata:
|
||||
try:
|
||||
prompt_data = json.loads(metadata["prompt"])
|
||||
result["prompt"] = prompt_data
|
||||
except json.JSONDecodeError:
|
||||
result["prompt"] = metadata["prompt"]
|
||||
|
||||
# Check for workflow
|
||||
if "workflow" in metadata:
|
||||
try:
|
||||
workflow_data = json.loads(metadata["workflow"])
|
||||
result["workflow"] = workflow_data
|
||||
except json.JSONDecodeError:
|
||||
result["workflow"] = metadata["workflow"]
|
||||
|
||||
# Check for other common ComfyUI fields
|
||||
common_fields = [
|
||||
"positive",
|
||||
"negative",
|
||||
"steps",
|
||||
"cfg",
|
||||
"sampler",
|
||||
"scheduler",
|
||||
"seed",
|
||||
]
|
||||
for field in common_fields:
|
||||
if field in metadata:
|
||||
try:
|
||||
result["parameters"][field] = json.loads(metadata[field])
|
||||
except json.JSONDecodeError:
|
||||
result["parameters"][field] = metadata[field]
|
||||
|
||||
return result
|
||||
|
||||
+223
-174
@@ -35,56 +35,43 @@ except ImportError:
|
||||
from utils.parallel import map_parallel
|
||||
|
||||
try:
|
||||
from ...utils import exit_signal
|
||||
except ImportError:
|
||||
from utils import exit_signal
|
||||
|
||||
try:
|
||||
from ...database.pair_ingestion import SCAN_MARKS, ingest_resolved_pair
|
||||
from ...database.pair_ingestion import blank_positive, ingest_resolved_pair
|
||||
from ...database.run_attribution import (
|
||||
RecordedOutput,
|
||||
host_output_directory,
|
||||
named_prompt,
|
||||
recorded_runs,
|
||||
stored_run_prompt,
|
||||
stored_run_prompts,
|
||||
)
|
||||
from ...database.scanned_prompts import (
|
||||
link_stored_prompt_or_raise,
|
||||
store_scanned_prompt,
|
||||
)
|
||||
from ...utils.file_prompt import (
|
||||
NEVER,
|
||||
NOT_SAVED,
|
||||
NOT_SAVED_WIRED,
|
||||
PAIR,
|
||||
Policy,
|
||||
decide_file_prompt,
|
||||
from ...utils.generation_params import (
|
||||
PLACEHOLDERS,
|
||||
GenerationParamsError,
|
||||
extract_generation_params,
|
||||
parse_page_json,
|
||||
)
|
||||
from ...utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt
|
||||
from ...utils.review_capture import reviewable
|
||||
except ImportError:
|
||||
from database.pair_ingestion import SCAN_MARKS, ingest_resolved_pair
|
||||
from database.pair_ingestion import blank_positive, ingest_resolved_pair
|
||||
from database.run_attribution import (
|
||||
RecordedOutput,
|
||||
host_output_directory,
|
||||
named_prompt,
|
||||
recorded_runs,
|
||||
stored_run_prompt,
|
||||
stored_run_prompts,
|
||||
)
|
||||
from database.scanned_prompts import (
|
||||
link_stored_prompt_or_raise,
|
||||
store_scanned_prompt,
|
||||
)
|
||||
from utils.file_prompt import (
|
||||
NEVER,
|
||||
NOT_SAVED,
|
||||
NOT_SAVED_WIRED,
|
||||
PAIR,
|
||||
Policy,
|
||||
decide_file_prompt,
|
||||
from utils.generation_params import (
|
||||
PLACEHOLDERS,
|
||||
GenerationParamsError,
|
||||
extract_generation_params,
|
||||
parse_page_json,
|
||||
)
|
||||
from utils.prompt_pair_resolver import resolve_prompt_pair, sampler_less_prompt
|
||||
from utils.review_capture import reviewable
|
||||
|
||||
|
||||
try:
|
||||
from ...database.library import display_title
|
||||
from ...utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from database.library import display_title
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff")
|
||||
VIDEO_EXTENSIONS = (".mp4", ".avi", ".mov", ".mkv", ".webm", ".m4v", ".wmv")
|
||||
@@ -131,7 +118,6 @@ def _collect_media_files(output_dirs, extensions):
|
||||
if not os.path.isdir(root):
|
||||
continue
|
||||
for current, dirnames, filenames in os.walk(root, followlinks=False):
|
||||
exit_signal.checkpoint() # #376: stop between directories at exit
|
||||
depth = 0 if current == root else len(Path(current).relative_to(root).parts)
|
||||
dirnames[:] = sorted(
|
||||
d
|
||||
@@ -264,15 +250,10 @@ SCAN_DETAILS = {
|
||||
"skipped": "Skipped · already linked",
|
||||
"ambiguous": "Needs review: more than one prompt could match",
|
||||
"unresolved": "Needs review: the prompt could not be traced in the workflow",
|
||||
"conflict": "Kept: already linked to a different prompt",
|
||||
"conflict": "Needs review: already linked to a different prompt",
|
||||
"failed": "Error: the prompt could not be stored",
|
||||
"error": "Error: the file could not be processed (see the server log)",
|
||||
"found": "Metadata found, but no prompt text to store",
|
||||
"not_saved": "Skipped · its PromptManager node does not save to the library",
|
||||
"not_saved_wired": (
|
||||
"Skipped · its Save to library switch is wired from another node, so only "
|
||||
"the run itself can link it"
|
||||
),
|
||||
None: "No prompt metadata in the file",
|
||||
}
|
||||
|
||||
@@ -331,6 +312,22 @@ def _file_size(path):
|
||||
return None
|
||||
|
||||
|
||||
def _with_file_size(canonical, media_file):
|
||||
"""Record the file's size in ``file_info`` unless the metadata carries one.
|
||||
|
||||
Stored as ``generated_images.file_size``; the "new" scan scope skips a known
|
||||
file only while its size is unchanged.
|
||||
"""
|
||||
info = canonical.get("file_info")
|
||||
info = dict(info) if isinstance(info, dict) else {}
|
||||
if info.get("size") is None:
|
||||
size = _file_size(media_file)
|
||||
if size is not None:
|
||||
info["size"] = size
|
||||
if info:
|
||||
canonical["file_info"] = info
|
||||
|
||||
|
||||
def _scan_roots(output_dirs):
|
||||
"""(absolute root, public label, listing index), the most specific first.
|
||||
|
||||
@@ -586,56 +583,18 @@ class _BatchHistory:
|
||||
"""A scan batch's recorded-history lookup, made on the first file needing it.
|
||||
|
||||
Most files link through their metadata and never ask; the first one that
|
||||
does triggers one ``recorded_runs`` pass for the whole batch, from which
|
||||
both the stored prompt (``get``) and the recorded saver (``recorded``)
|
||||
of every file are read. A failure is logged and names nothing.
|
||||
does triggers one lookup for the whole batch (``_recorded_runs_for``).
|
||||
"""
|
||||
|
||||
def __init__(self, api, media_files):
|
||||
self._api = api
|
||||
self._files = [str(path) for path in media_files]
|
||||
self._runs = None
|
||||
self._prompts = {}
|
||||
|
||||
def _found(self):
|
||||
if self._runs is None:
|
||||
try:
|
||||
self._runs = recorded_runs(
|
||||
self._api.db, self._files, host_output_directory(), None
|
||||
)
|
||||
except Exception:
|
||||
self._api.logger.warning(
|
||||
"History attribution failed for a scan batch", exc_info=True
|
||||
)
|
||||
self._runs = {}
|
||||
return self._runs
|
||||
self._files = media_files
|
||||
self._found = None
|
||||
|
||||
def get(self, path):
|
||||
"""The stored prompt id history attributes to ``path``, else None."""
|
||||
found = self._found().get(path)
|
||||
if found is None:
|
||||
return None
|
||||
run_id = found[0]
|
||||
if run_id not in self._prompts:
|
||||
try:
|
||||
self._prompts[run_id] = named_prompt(self._api.db, run_id)
|
||||
except Exception:
|
||||
self._api.logger.warning(
|
||||
"History attribution failed for a scan batch", exc_info=True
|
||||
)
|
||||
self._prompts[run_id] = None
|
||||
return self._prompts[run_id]
|
||||
|
||||
def recorded(self, path):
|
||||
"""``RecordedOutput(saver, {})`` for an attributed file, else None.
|
||||
|
||||
``saver`` is set only when the run reported the file from one node.
|
||||
"""
|
||||
found = self._found().get(path)
|
||||
if found is None:
|
||||
return None
|
||||
nodes = found[1]
|
||||
return RecordedOutput(next(iter(nodes)) if len(nodes) == 1 else None, {})
|
||||
if self._found is None:
|
||||
self._found = self._api._recorded_runs_for(self._files)
|
||||
return self._found.get(path)
|
||||
|
||||
|
||||
class AdminRoutesMixin:
|
||||
@@ -1671,14 +1630,7 @@ class AdminRoutesMixin:
|
||||
if self._review_file_linked(media_file, roots):
|
||||
outcome = "unchanged" # already linked: not a review case
|
||||
else:
|
||||
# The records name the item versions that count; the
|
||||
# saver anchor (#390) is checked on the captured bytes.
|
||||
self._record_scanned_review(
|
||||
media_file,
|
||||
roots,
|
||||
review,
|
||||
scanned=(records.get("canonical"), None),
|
||||
)
|
||||
self._record_scanned_review(media_file, roots, review)
|
||||
elif untraced and outcome == "unresolved":
|
||||
# Untraced by the default profile: the inspection profile may
|
||||
# still offer a choice (queued for review, never linked).
|
||||
@@ -1700,6 +1652,24 @@ class AdminRoutesMixin:
|
||||
note(media_file, status)
|
||||
return counts
|
||||
|
||||
@staticmethod
|
||||
def _graph_prompt_text(metadata):
|
||||
"""The positive prompt the browser parser traces from the graphs, or None.
|
||||
|
||||
Records parse as the legacy pages parsed them (NaN reads as null).
|
||||
"""
|
||||
records = {}
|
||||
for key in ("prompt", "workflow"):
|
||||
value = metadata.get(key)
|
||||
records[key] = parse_page_json(value) if isinstance(value, str) else value
|
||||
try:
|
||||
positive = extract_generation_params(records)["positivePrompt"]
|
||||
except GenerationParamsError:
|
||||
return None
|
||||
if not isinstance(positive, str) or positive == PLACEHOLDERS["positivePrompt"]:
|
||||
return None
|
||||
return positive.strip() or None
|
||||
|
||||
def _ingest_scanned_file(
|
||||
self,
|
||||
media_file,
|
||||
@@ -1710,81 +1680,122 @@ class AdminRoutesMixin:
|
||||
records=None,
|
||||
history=None,
|
||||
):
|
||||
"""Store the prompt the shared per-file decision names (#247).
|
||||
"""Enrich exact pairs; retain legacy positive-only import for absent pairs.
|
||||
|
||||
``utils.file_prompt.decide_file_prompt`` (scan policy) decides; this
|
||||
stores the result. Returns the file's scan status, or None without
|
||||
prompt metadata. With ``detailed``, a stored or linked file returns a
|
||||
dict instead: its ``status``, the prompt's display ``title`` and, for
|
||||
a resolved pair, the pair counts. When the resolver finds the file
|
||||
ambiguous (whether or not a later step then links it) and
|
||||
``ambiguous`` is a list, ``media_file`` is appended to it; only a file
|
||||
that then returns ``"ambiguous"`` (nothing linked it) is a Needs
|
||||
review case. An ambiguous result with nothing to choose
|
||||
(``reviewable`` is False: fewer than two candidates carry a prompt)
|
||||
goes to ``untraced`` instead and returns ``"unresolved"`` when nothing
|
||||
links it. The parsed records of either ambiguous file go to
|
||||
``records["canonical"]`` when ``records`` is a dict. ``history`` is
|
||||
the batch's lazy recorded-history lookup.
|
||||
Returns the file's scan status, or None without prompt metadata. With
|
||||
``detailed``, a stored or linked file returns a dict instead: its
|
||||
``status``, the prompt's display ``title`` and, for a resolved pair,
|
||||
the pair counts. When the resolver finds the file ambiguous (whether
|
||||
or not the browser parser then links it) and ``ambiguous`` is a list,
|
||||
``media_file`` is appended to it; only a file that then returns
|
||||
``"ambiguous"`` (nothing linked it) is a Needs review case. An
|
||||
ambiguous result with nothing to choose (``reviewable`` is False: fewer
|
||||
than two candidates carry a prompt) that nothing links returns
|
||||
``"unresolved"`` instead, and ``media_file`` goes to ``untraced``;
|
||||
its parsed records go to ``records["canonical"]`` when ``records`` is
|
||||
a dict.
|
||||
"""
|
||||
path = str(media_file)
|
||||
if history is None: # one file: history is still asked once
|
||||
history = _BatchHistory(self, [media_file])
|
||||
decision = decide_file_prompt(
|
||||
metadata,
|
||||
file_name=Path(path).name,
|
||||
file_size=functools.partial(_file_size, media_file),
|
||||
in_output=False, # only the monitor policy reads it
|
||||
lookup=functools.partial(history.recorded, path),
|
||||
policy=Policy.SCAN,
|
||||
)
|
||||
if decision.kind in (NOT_SAVED, NOT_SAVED_WIRED):
|
||||
# Save to library off (#90): no write, recorded-run fallback or
|
||||
# Needs review item
|
||||
return decision.kind
|
||||
self._note_review(media_file, decision, ambiguous, untraced, records)
|
||||
if decision.kind == NEVER:
|
||||
if decision.try_recorded_run:
|
||||
linked = self._link_recorded_run(
|
||||
media_file, decision.canonical, detailed, history
|
||||
)
|
||||
if linked is not None:
|
||||
return linked
|
||||
return decision.unlinked_status
|
||||
if decision.kind == PAIR:
|
||||
# A rerun after a library move re-checks the link rows.
|
||||
result = self.db.survive_library_switch(
|
||||
lambda: ingest_resolved_pair(
|
||||
self.db,
|
||||
path,
|
||||
decision.canonical,
|
||||
decision.resolution,
|
||||
marks=SCAN_MARKS,
|
||||
)
|
||||
)
|
||||
if not detailed:
|
||||
return result["status"]
|
||||
title_text = decision.resolution["positive"]["text"]
|
||||
return {**result, "title": self._scan_title(media_file, title_text)}
|
||||
status, _prompt_id = store_scanned_prompt(
|
||||
self.db, path, decision.canonical, decision.text, SCAN_MARKS
|
||||
)
|
||||
if not detailed or status not in SCAN_TITLED_DETAILS:
|
||||
return status
|
||||
return {"status": status, "title": self._scan_title(media_file, decision.text)}
|
||||
|
||||
@staticmethod
|
||||
def _note_review(media_file, decision, ambiguous, untraced, records):
|
||||
"""Hand an ambiguous file to the batch's Needs review lists."""
|
||||
if decision.review == "untraced":
|
||||
if not metadata:
|
||||
# No prompt metadata (--disable-metadata, bare savers): only the
|
||||
# recorded run can say which prompt made it (#342)
|
||||
return self._link_recorded_run(media_file, {}, detailed, history)
|
||||
canonical = dict(metadata)
|
||||
for key in ("prompt", "workflow"):
|
||||
if isinstance(canonical.get(key), str):
|
||||
try:
|
||||
canonical[key] = json.loads(canonical[key])
|
||||
except (ValueError, TypeError):
|
||||
# Preserve malformed metadata for conservative resolver rejection.
|
||||
pass
|
||||
if not canonical.get("prompt") and not canonical.get("parameters"):
|
||||
return self._link_recorded_run(media_file, canonical, detailed, history)
|
||||
_with_file_size(canonical, media_file)
|
||||
resolution = resolve_prompt_pair(canonical)
|
||||
if blank_positive(resolution):
|
||||
# Only prepend/append text: no stored prompt can own it (#342)
|
||||
resolution = {**resolution, "state": "unresolved"}
|
||||
# Ambiguous without a choice: unresolved unless something links it.
|
||||
unchoosable = resolution["state"] == "ambiguous" and not reviewable(resolution)
|
||||
if unchoosable:
|
||||
if untraced is not None:
|
||||
untraced.append(media_file)
|
||||
if records is not None:
|
||||
records["canonical"] = decision.canonical
|
||||
elif decision.review == "ambiguous" and ambiguous is not None:
|
||||
records["canonical"] = canonical
|
||||
elif resolution["state"] == "ambiguous" and ambiguous is not None:
|
||||
ambiguous.append(media_file)
|
||||
if records is not None:
|
||||
records["canonical"] = decision.canonical
|
||||
if resolution["state"] == "resolved":
|
||||
result = ingest_resolved_pair(
|
||||
self.db, str(media_file), canonical, resolution
|
||||
)
|
||||
if not detailed:
|
||||
return result["status"]
|
||||
return {
|
||||
**result,
|
||||
"title": self._scan_title(media_file, resolution["positive"]["text"]),
|
||||
}
|
||||
if canonical.get("prompt") and resolution["state"] in (
|
||||
"ambiguous",
|
||||
"unresolved",
|
||||
):
|
||||
# The resolver could not decide: link the prompt the inspector and the
|
||||
# legacy pages show for this file (the browser parser, in Python).
|
||||
# Graphs without a recognized sampler keep the legacy extractor below.
|
||||
prompt_text = self._graph_prompt_text(metadata)
|
||||
if not prompt_text:
|
||||
linked = self._link_recorded_run(
|
||||
media_file, canonical, detailed, history
|
||||
)
|
||||
if linked is not None:
|
||||
return linked
|
||||
return "unresolved" if unchoosable else resolution["state"]
|
||||
elif resolution["state"] == "ambiguous":
|
||||
return "unresolved" if unchoosable else "ambiguous"
|
||||
elif resolution["state"] == "unresolved" and not isinstance(
|
||||
canonical.get("parameters"), str
|
||||
):
|
||||
return resolution["state"]
|
||||
elif canonical.get("prompt"):
|
||||
# No node takes a positive or negative input (a guider or wrapper
|
||||
# graph): the one wired text, never a guess (#252).
|
||||
found = sampler_less_prompt(canonical["prompt"])
|
||||
prompt_text = found["text"]
|
||||
if prompt_text is None and isinstance(canonical.get("parameters"), str):
|
||||
# A metadata saver also wrote A1111 text: its positive prompt.
|
||||
parsed = self._parse_comfyui_prompt(metadata)["positive_prompt"]
|
||||
prompt_text = parsed.strip() if parsed and parsed.strip() else None
|
||||
if prompt_text is None:
|
||||
linked = self._link_recorded_run(
|
||||
media_file, canonical, detailed, history
|
||||
)
|
||||
if linked is not None:
|
||||
return linked
|
||||
return "unresolved" if found["state"] == "ambiguous" else "found"
|
||||
else:
|
||||
prompt_text = self._parse_comfyui_prompt(metadata)["positive_prompt"]
|
||||
if not (prompt_text and prompt_text.strip()):
|
||||
return "found"
|
||||
prompt_text = prompt_text.strip()
|
||||
|
||||
status = self._store_scanned_prompt(media_file, canonical, prompt_text)
|
||||
if not detailed or status not in SCAN_TITLED_DETAILS:
|
||||
return status
|
||||
return {"status": status, "title": self._scan_title(media_file, prompt_text)}
|
||||
|
||||
def _recorded_runs_for(self, media_files):
|
||||
"""``{path: prompt id or None}`` from recorded history for a scan batch.
|
||||
|
||||
One output lookup per chunk of names (``stored_run_prompts``), so a
|
||||
batch never queries history once per file. An empty dict on failure.
|
||||
"""
|
||||
try:
|
||||
return stored_run_prompts(
|
||||
self.db, [str(path) for path in media_files], host_output_directory()
|
||||
)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"History attribution failed for a scan batch", exc_info=True
|
||||
)
|
||||
return {}
|
||||
|
||||
def _link_recorded_run(self, media_file, canonical, detailed=False, history=None):
|
||||
"""Link a file recorded history attributes to one stored prompt (#342).
|
||||
@@ -1793,25 +1804,31 @@ class AdminRoutesMixin:
|
||||
``{"status", "title"}``), or None when history names no single stored
|
||||
prompt for the file (``database.run_attribution``). Never creates a
|
||||
prompt. Works without any prompt metadata in the file. ``history`` is
|
||||
the batch's lookup (``_BatchHistory``). A file already linked to
|
||||
another prompt is a ``conflict``: one prompt per file.
|
||||
the batch's precomputed lookup; without it the file is looked up alone.
|
||||
"""
|
||||
if history is None:
|
||||
history = _BatchHistory(self, [media_file])
|
||||
prompt_id = history.get(str(media_file))
|
||||
if history is not None:
|
||||
prompt_id = history.get(str(media_file))
|
||||
else:
|
||||
try:
|
||||
prompt_id = stored_run_prompt(
|
||||
self.db, str(media_file), host_output_directory()
|
||||
)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
f"History attribution failed for {media_file.name}",
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
if prompt_id is None:
|
||||
return None
|
||||
try:
|
||||
status = self.db.survive_library_switch(
|
||||
lambda: link_stored_prompt_or_raise(
|
||||
self.db, str(media_file), canonical, prompt_id
|
||||
)
|
||||
image_id = self.db.link_image_to_prompt(
|
||||
prompt_id, str(media_file), metadata=canonical
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to link {media_file.name} from history: {e}")
|
||||
return "found"
|
||||
if status is None:
|
||||
return None
|
||||
status = "linked" if image_id else "unchanged"
|
||||
if not detailed or status not in SCAN_TITLED_DETAILS:
|
||||
return status
|
||||
prompt = self.db.get_prompt_by_id(prompt_id) or {}
|
||||
@@ -1831,3 +1848,35 @@ class AdminRoutesMixin:
|
||||
except Exception:
|
||||
self.logger.warning("Scan row title could not be read; using its text")
|
||||
return display_title((), text)
|
||||
|
||||
def _store_scanned_prompt(self, media_file, canonical, prompt_text):
|
||||
"""Link ``media_file`` to ``prompt_text``'s prompt, saving it when new."""
|
||||
prompt_hash = generate_prompt_hash(prompt_text)
|
||||
existing = self.db.get_prompt_by_hash(prompt_hash)
|
||||
if existing:
|
||||
try:
|
||||
image_id = self.db.link_image_to_prompt(
|
||||
existing["id"], str(media_file), metadata=canonical
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(
|
||||
f"Failed to link {media_file.name} to existing prompt: {e}"
|
||||
)
|
||||
return "found"
|
||||
return "linked" if image_id else "unchanged"
|
||||
|
||||
prompt_id = self.db.save_prompt(
|
||||
prompt_text,
|
||||
"scanned",
|
||||
["auto-scanned"],
|
||||
None,
|
||||
f"Auto-scanned from {media_file.name}",
|
||||
prompt_hash,
|
||||
)
|
||||
if not prompt_id:
|
||||
return "found"
|
||||
try:
|
||||
self.db.link_image_to_prompt(prompt_id, str(media_file), metadata=canonical)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to link {media_file.name} to new prompt: {e}")
|
||||
return "added"
|
||||
|
||||
+36
-8
@@ -27,12 +27,7 @@ from .annotation_suggest import (
|
||||
validate_options,
|
||||
)
|
||||
from .autotag_routes import model_busy_response
|
||||
|
||||
# The target check is shared with the collection routes (#331); both names
|
||||
# stay importable from here.
|
||||
from .media_target import MAX_TARGET_PATH # noqa: F401
|
||||
from .media_target import media_target as _target
|
||||
from .images import _unavailable_link, _verified_title
|
||||
from .images import _forbidden, _has_traversal, _unavailable_link, _verified_title
|
||||
from .prompts import bad_request
|
||||
from .. import model_gate
|
||||
|
||||
@@ -45,7 +40,7 @@ try:
|
||||
)
|
||||
from ...database.execution_history import subscribe_history_restore
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...utils.media_capture import CaptureError, CaptureStore
|
||||
from ...utils.media_capture import CAPTURE_EXTENSIONS, CaptureError, CaptureStore
|
||||
except ImportError:
|
||||
from database.annotations import (
|
||||
MAX_IMAGE_TAGS,
|
||||
@@ -55,13 +50,15 @@ except ImportError:
|
||||
)
|
||||
from database.execution_history import subscribe_history_restore
|
||||
from database.library_commands import LibraryCommandError
|
||||
from utils.media_capture import CaptureError, CaptureStore
|
||||
from utils.media_capture import CAPTURE_EXTENSIONS, CaptureError, CaptureStore
|
||||
|
||||
# Request bodies of the annotation routes are small (a target plus a tag set).
|
||||
MAX_BODY_BYTES = 64 * 1024
|
||||
MAX_TARGET_PATH = 1024
|
||||
CAPTURE_URL = "/prompt_manager/image-annotations/capture/"
|
||||
_PREVIEW_CHUNK = 64 * 1024
|
||||
|
||||
_TARGET_FIELDS = frozenset({"root_index", "root_scope", "path"})
|
||||
_OBSERVE_FIELDS = frozenset({"target"})
|
||||
_SAVE_FIELDS = frozenset(
|
||||
{
|
||||
@@ -191,6 +188,37 @@ def _capture_error(exc: CaptureError) -> web.Response:
|
||||
return _coded(exc.code, _MESSAGES.get(exc.code) or str(exc))
|
||||
|
||||
|
||||
def _target(data: object):
|
||||
"""``((index, scope, relative), None)`` for a target, else ``(None, error)``.
|
||||
|
||||
Only the listing identity is accepted: an int root index, the root's
|
||||
64-hex scope and a bounded root-relative POSIX media path, never an
|
||||
absolute, drive-qualified, backslash or ``..`` path. Nothing here touches
|
||||
the filesystem or the database.
|
||||
"""
|
||||
if not isinstance(data, dict) or set(data) != _TARGET_FIELDS:
|
||||
return None, bad_request("target must name root_index, root_scope and path")
|
||||
index, scope, relative = data["root_index"], data["root_scope"], data["path"]
|
||||
if type(index) is not int or index < 0:
|
||||
return None, bad_request("Invalid gallery root")
|
||||
if not isinstance(scope, str) or _HEX64.fullmatch(scope) is None:
|
||||
return None, bad_request("Invalid gallery root scope")
|
||||
if not isinstance(relative, str) or not relative or len(relative) > MAX_TARGET_PATH:
|
||||
return None, bad_request("path must be a relative media path")
|
||||
parts = relative.split("/")
|
||||
if (
|
||||
"\0" in relative
|
||||
or "\\" in relative
|
||||
or relative.startswith("/")
|
||||
or _has_traversal(relative)
|
||||
or any(part in ("", ".", "..") or ":" in part for part in parts)
|
||||
):
|
||||
return None, _forbidden()
|
||||
if os.path.splitext(parts[-1])[1].lower() not in CAPTURE_EXTENSIONS:
|
||||
return None, _forbidden("Only media files can be tagged")
|
||||
return (index, scope, relative), None
|
||||
|
||||
|
||||
def _expires_at(seconds: float | None) -> str:
|
||||
moment = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
|
||||
seconds=max(0.0, seconds or 0.0)
|
||||
|
||||
@@ -589,7 +589,6 @@ class DuplicateFilesMixin:
|
||||
spellings = await self._run_in_executor(self._source_spellings)
|
||||
removed, failure, keepers = 0, None, {}
|
||||
totals = dict.fromkeys(("moved", "merged", "favorites", "tags", "review"), 0)
|
||||
totals["categories"] = 0 # image categories kept on the kept copies (#332)
|
||||
for done, target in enumerate(targets, start=1):
|
||||
if failure is not None:
|
||||
self._record_target(job, "kept", target, NOT_ATTEMPTED)
|
||||
@@ -636,9 +635,6 @@ class DuplicateFilesMixin:
|
||||
job.log_line(
|
||||
f"Added {_plural(totals['tags'], 'image tag')} to the kept copies"
|
||||
)
|
||||
if totals.get("categories"):
|
||||
kept = _plural(totals["categories"], "image category", "image categories")
|
||||
job.log_line(f"Kept {kept} on the kept copies")
|
||||
if totals["review"]:
|
||||
job.log_line(
|
||||
f"Settled {_plural(totals['review'], 'Needs review item')} "
|
||||
|
||||
@@ -18,7 +18,7 @@ Blocking: each lookup stats the file's folders once (cached).
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from .images import _favorite_relative_key, _favorite_root_key, image_category_key
|
||||
from .images import _favorite_relative_key, _favorite_root_key
|
||||
|
||||
|
||||
def _identity(path):
|
||||
@@ -78,15 +78,6 @@ class Spellings:
|
||||
)
|
||||
return spelled
|
||||
|
||||
@property
|
||||
def listed_roots(self):
|
||||
"""The listed roots (the output sources), in order."""
|
||||
return [root for root, _identity in self._listed]
|
||||
|
||||
def category_key(self, path):
|
||||
"""*path*'s image category file key (#332), or None outside every source."""
|
||||
return image_category_key(self.listed_roots, path)
|
||||
|
||||
def favorites(self, path):
|
||||
"""``{listed root: path below it (POSIX)}`` for each listed root holding it."""
|
||||
path = os.path.abspath(str(path))
|
||||
@@ -134,23 +125,9 @@ def copy_move(target, kept_stat, spellings):
|
||||
"targets": targets,
|
||||
"kept_path": target["keeper_abs"],
|
||||
"favorites": favorites,
|
||||
"categories": _category_pairs(target, spellings),
|
||||
"removed_locator": target["locator"],
|
||||
"kept_locator": target["keeper_locator"],
|
||||
"sha256": target["hash"],
|
||||
"kept_size": kept_stat.st_size,
|
||||
"kept_mtime_ns": kept_stat.st_mtime_ns,
|
||||
}
|
||||
|
||||
|
||||
def _category_pairs(target, spellings):
|
||||
"""``[(removed file key, kept file key)]`` for the image category (#332).
|
||||
|
||||
From the two absolute paths, not the favorites tuples: those follow the
|
||||
listed root, which differs from the file key when sources nest.
|
||||
"""
|
||||
removed = spellings.category_key(target["abs"])
|
||||
kept = spellings.category_key(target["keeper_abs"])
|
||||
if removed is None or kept is None:
|
||||
return []
|
||||
return [(removed, kept)]
|
||||
|
||||
+5
-58
@@ -5,7 +5,6 @@ import base64
|
||||
import binascii
|
||||
from functools import wraps
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Callable
|
||||
@@ -28,12 +27,10 @@ from .history_current import (
|
||||
snapshot_current_output_scope,
|
||||
)
|
||||
from .inspection_service import CurrentFileChanged, inspect_current_file
|
||||
from .images import image_category_key
|
||||
|
||||
try:
|
||||
from ...database.execution_history import HistoryEpochChanged
|
||||
from ...database.history_gallery import read_history_gallery
|
||||
from ...database.run_attribution import recorded_output_lookup
|
||||
from ...database.history_reads import (
|
||||
HistoryCaptureChanged,
|
||||
OutputPosition,
|
||||
@@ -41,15 +38,10 @@ try:
|
||||
WindowPosition,
|
||||
)
|
||||
from ...utils.history_read_fence import HistoryReadFence
|
||||
from ...utils.inspection_response import (
|
||||
OWN_CATEGORY_UNREAD,
|
||||
apply_own_category,
|
||||
encode_history_inspection_response,
|
||||
)
|
||||
from ...utils.inspection_response import encode_history_inspection_response
|
||||
except ImportError:
|
||||
from database.execution_history import HistoryEpochChanged
|
||||
from database.history_gallery import read_history_gallery
|
||||
from database.run_attribution import recorded_output_lookup
|
||||
from database.history_reads import (
|
||||
HistoryCaptureChanged,
|
||||
OutputPosition,
|
||||
@@ -57,11 +49,7 @@ except ImportError:
|
||||
WindowPosition,
|
||||
)
|
||||
from utils.history_read_fence import HistoryReadFence
|
||||
from utils.inspection_response import (
|
||||
OWN_CATEGORY_UNREAD,
|
||||
apply_own_category,
|
||||
encode_history_inspection_response,
|
||||
)
|
||||
from utils.inspection_response import encode_history_inspection_response
|
||||
|
||||
|
||||
_RUN_MAX_BYTES = 8 * 1024 * 1024
|
||||
@@ -687,32 +675,6 @@ class HistoryRoutesMixin:
|
||||
) -> web.Response:
|
||||
return await self._history_current(request, "inspection")
|
||||
|
||||
def _history_own_category(self, reference: dict, scope):
|
||||
"""The recorded output's own category (#332), inside the snapshot.
|
||||
|
||||
The path is the symlink-resolved output folder plus the receipt's
|
||||
subfolder and filename, keyed by ``image_category_key`` over the
|
||||
configured output sources as Images keys it. A temp output or a path
|
||||
no source holds has none. A failed read is OWN_CATEGORY_UNREAD.
|
||||
"""
|
||||
if (
|
||||
reference.get("folder_type") != "output"
|
||||
or scope.canonical_output is None
|
||||
or candidate_locator(reference, scope) is None
|
||||
):
|
||||
return None
|
||||
path = os.path.join(
|
||||
str(scope.canonical_output), reference["subfolder"], reference["filename"]
|
||||
)
|
||||
try:
|
||||
key = image_category_key(self._get_all_output_dirs(), path)
|
||||
return None if key is None else self.db.image_category(*key)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"History image category could not be read", exc_info=True
|
||||
)
|
||||
return OWN_CATEGORY_UNREAD
|
||||
|
||||
async def _history_current(
|
||||
self, request: web.Request, kind: str
|
||||
) -> web.StreamResponse:
|
||||
@@ -737,7 +699,6 @@ class HistoryRoutesMixin:
|
||||
if scope.current_scope != current_scope:
|
||||
raise _current_stale()
|
||||
result = db.get_history_occurrence(run_id, node, ordinal)
|
||||
own = None
|
||||
if result is not None:
|
||||
receipt = result["receipt"]
|
||||
path = candidate_locator(receipt["reference"], scope)
|
||||
@@ -745,10 +706,6 @@ class HistoryRoutesMixin:
|
||||
db.model.get_connection(),
|
||||
(path,) if path is not None else (),
|
||||
)
|
||||
if kind == "inspection":
|
||||
own = self._history_own_category(
|
||||
receipt["reference"], scope
|
||||
)
|
||||
finally:
|
||||
db.model.close()
|
||||
if result is None:
|
||||
@@ -767,19 +724,9 @@ class HistoryRoutesMixin:
|
||||
resolved.canonical_target,
|
||||
source_label=resolved.public["source_label"],
|
||||
saved=saved.get("saved"),
|
||||
library=apply_own_category(
|
||||
saved.get(
|
||||
"library",
|
||||
{
|
||||
"status": (
|
||||
"unavailable" if saved_unavailable else "none"
|
||||
)
|
||||
},
|
||||
),
|
||||
own,
|
||||
),
|
||||
recorded=recorded_output_lookup(
|
||||
db, close=True, reference=receipt["reference"]
|
||||
library=saved.get(
|
||||
"library",
|
||||
{"status": "unavailable" if saved_unavailable else "none"},
|
||||
),
|
||||
)
|
||||
_reauthorize_current_target(resolved, scope)
|
||||
|
||||
@@ -1,374 +0,0 @@
|
||||
"""Image category routes (#332): an image file's own category.
|
||||
|
||||
``PUT /prompt_manager/images/category`` sets the own category of up to 500
|
||||
listed files; ``DELETE`` removes it (listed or not, so a missing file's
|
||||
category can still be removed); ``GET /prompt_manager/images/categories``
|
||||
lists the own-category values of files the sources list. Targets are the
|
||||
Images listing identity (``{root_index, root_scope, path}``, checked by
|
||||
``media_target``) of an output source; LoRA folders are not sources here.
|
||||
Instead of ``items`` a request may name ``filter``, the Images listing's own
|
||||
query ("Select all N matching", #392), resolved now up to 500 images.
|
||||
Each file is stored under its file key (``images.image_category_key``).
|
||||
These routes never read or write prompts, recipes or image links.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .image_tag_jobs import MAX_JOB_TARGETS, _parse_targets
|
||||
from .images import (
|
||||
_favorite_relative_key,
|
||||
_favorite_root_key,
|
||||
_forbidden,
|
||||
_json_body_or_error,
|
||||
image_category_key,
|
||||
)
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
from ...database.image_categories import MAX_CATEGORY_LENGTH, MAX_KEY_LENGTH
|
||||
from ...database.models import normalize_image_path
|
||||
from ...utils.validators import validate_category
|
||||
except ImportError:
|
||||
from database.image_categories import MAX_CATEGORY_LENGTH, MAX_KEY_LENGTH
|
||||
from database.models import normalize_image_path
|
||||
from utils.validators import validate_category
|
||||
|
||||
CATEGORY_MEDIA_MESSAGE = "Only media files can have a category"
|
||||
_PUT_FIELDS = frozenset({"category"})
|
||||
_DELETE_FIELDS = frozenset()
|
||||
TOO_MANY = f"Select at most {MAX_JOB_TARGETS} images."
|
||||
SOURCE_CHANGED_MESSAGE = "The sources changed. Refresh the images and try again."
|
||||
WRITE_FAILED = "The category could not be changed."
|
||||
READ_FAILED = "Image categories could not be read."
|
||||
|
||||
|
||||
def _coded(status: int, code: str, message: str) -> web.Response:
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": message},
|
||||
status=status,
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
|
||||
def _category_value(raw) -> tuple[str | None, web.Response | None]:
|
||||
"""The stripped category, or a 400 (an empty value is a 400: use DELETE)."""
|
||||
if not isinstance(raw, str):
|
||||
return None, bad_request("category must be a string")
|
||||
value = raw.strip()
|
||||
if not value:
|
||||
return None, bad_request("category must not be empty")
|
||||
if len(value) > MAX_CATEGORY_LENGTH:
|
||||
return None, bad_request(
|
||||
f"Category cannot exceed {MAX_CATEGORY_LENGTH} characters"
|
||||
)
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
validate_category(value)
|
||||
except UnicodeEncodeError:
|
||||
return None, bad_request("category must be valid text")
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
return value, None
|
||||
|
||||
|
||||
class _KeyRefused(Exception):
|
||||
"""A target has no storable file key (outside every source, or too long)."""
|
||||
|
||||
def __init__(self, message: str = "Access denied"):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
|
||||
|
||||
def _file_key(roots, root, relative) -> tuple[str, str]:
|
||||
"""The target's file key (blocking: resolves the sources)."""
|
||||
key = image_category_key(roots, Path(root) / relative)
|
||||
if key is None:
|
||||
raise _KeyRefused()
|
||||
if max(len(key[0]), len(key[1])) > MAX_KEY_LENGTH:
|
||||
raise _KeyRefused("Path is too long")
|
||||
return key
|
||||
|
||||
|
||||
def _nested_sources(root: Path, sources) -> list[tuple[str, str]]:
|
||||
"""``[(relative key prefix, source root key)]`` of sources inside *root*.
|
||||
|
||||
Deepest first, so the first prefix that matches is the file's source.
|
||||
"""
|
||||
base = os.path.join(os.path.normcase(os.path.realpath(str(root))), "")
|
||||
nested = []
|
||||
for source in sources:
|
||||
real = os.path.realpath(str(source))
|
||||
if not os.path.normcase(real).startswith(base):
|
||||
continue
|
||||
relative = os.path.relpath(real, os.path.realpath(str(root)))
|
||||
nested.append(
|
||||
(
|
||||
_favorite_relative_key(relative.replace(os.sep, "/")),
|
||||
_favorite_root_key(source),
|
||||
)
|
||||
)
|
||||
nested.sort(key=lambda item: item[0].count("/"), reverse=True)
|
||||
return nested
|
||||
|
||||
|
||||
def _source_prefix(relative_key: str, nested) -> str | None:
|
||||
"""The prefix of the deepest nested source holding *relative_key*."""
|
||||
for prefix, _source_key in nested:
|
||||
if relative_key.startswith(prefix + "/"):
|
||||
return prefix
|
||||
return None
|
||||
|
||||
|
||||
class ImageCategoryRoutesMixin:
|
||||
"""Routes for images' own categories (#332)."""
|
||||
|
||||
def _register_image_category_routes(self, routes):
|
||||
# Fixed paths beside DELETE /prompt_manager/images/{image_id}, as the
|
||||
# favorite routes are. Registered after it, they still win: aiohttp
|
||||
# (3.10+) looks fixed paths up before dynamic resources
|
||||
# (test_delete_category_never_reaches_delete_image).
|
||||
@routes.put("/prompt_manager/images/category")
|
||||
async def set_image_category_route(request):
|
||||
return await self.set_image_category(request)
|
||||
|
||||
@routes.delete("/prompt_manager/images/category")
|
||||
async def clear_image_category_route(request):
|
||||
return await self.clear_image_category(request)
|
||||
|
||||
@routes.get("/prompt_manager/images/categories")
|
||||
async def list_image_categories_route(request):
|
||||
return await self.list_image_categories(request)
|
||||
|
||||
# -- reading ---------------------------------------------------------------
|
||||
|
||||
def _own_category_map(self, sources, listings) -> dict:
|
||||
"""``{normalized listed path: own category}`` for the listed files (blocking).
|
||||
|
||||
One primary-key range read per source; each stored row is matched to
|
||||
the listing of every selected root that holds its source, so the work
|
||||
follows the stored rows, not the files. A file under a nested source
|
||||
is keyed by the deepest one (``image_category_key``). A row whose file
|
||||
is gone matches nothing. A failed read raises.
|
||||
"""
|
||||
sources = list(sources)
|
||||
rows = self.db.own_categories_of(_favorite_root_key(s) for s in sources)
|
||||
if not rows:
|
||||
return {}
|
||||
by_source: dict[str, dict[str, str]] = {}
|
||||
for (root_key, relative), category in rows.items():
|
||||
by_source.setdefault(root_key, {})[relative] = category
|
||||
found = {}
|
||||
for _index, root, files in listings:
|
||||
listing = self._favorite_listing(root, files)
|
||||
root_key = _favorite_root_key(root)
|
||||
# normalize_image_path(root / listed) without a call per file: the
|
||||
# listed spellings are clean relative POSIX paths. Windows abspath
|
||||
# also drops a component's trailing dots and spaces, so it keeps
|
||||
# the call there.
|
||||
base = os.path.join(normalize_image_path(str(root)), "")
|
||||
|
||||
def normalized(listed, base=base, root=root):
|
||||
if os.name == "nt":
|
||||
return normalize_image_path(str(root / listed))
|
||||
return base + os.path.normcase(listed.replace("/", os.sep))
|
||||
|
||||
nested = [
|
||||
item for item in _nested_sources(root, sources) if item[1] != root_key
|
||||
]
|
||||
for relative, category in by_source.get(root_key, {}).items():
|
||||
listed = listing.get(relative)
|
||||
if listed is not None and _source_prefix(relative, nested) is None:
|
||||
found[normalized(listed)] = category
|
||||
for prefix, source_key in nested:
|
||||
for relative, category in by_source.get(source_key, {}).items():
|
||||
full = f"{prefix}/{relative}"
|
||||
listed = listing.get(full)
|
||||
if listed is not None and _source_prefix(full, nested) == prefix:
|
||||
found[normalized(listed)] = category
|
||||
return found
|
||||
|
||||
async def _source_listings(self, roots) -> list[tuple]:
|
||||
return [
|
||||
(index, root, await self._get_gallery_files(root))
|
||||
for index, root in enumerate(roots)
|
||||
]
|
||||
|
||||
async def list_image_categories(self, request):
|
||||
"""GET: the own-category values of listed files, sorted."""
|
||||
try:
|
||||
roots = list(self._get_all_output_dirs())
|
||||
listings = await self._source_listings(roots)
|
||||
found = await self._run_in_executor(self._own_category_map, roots, listings)
|
||||
except Exception:
|
||||
self.logger.exception("Image categories read failed")
|
||||
return _coded(500, "categories-failed", READ_FAILED)
|
||||
return web.json_response(
|
||||
{"success": True, "categories": sorted(set(found.values()))},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
# -- writing ---------------------------------------------------------------
|
||||
|
||||
async def _filter_targets(self, raw):
|
||||
"""``(targets, None)`` for an Images listing filter, else ``(None, error)``.
|
||||
|
||||
"Select all N matching" (#392) sends the listing's own query; it is
|
||||
resolved now, up to 500 images (a 400 ``too-many-images`` above).
|
||||
"""
|
||||
from .images import ListingSourceChanged, parse_listing_filter
|
||||
|
||||
try:
|
||||
selection = await self.resolve_listing_filter(
|
||||
parse_listing_filter(raw), MAX_JOB_TARGETS
|
||||
)
|
||||
except ListingSourceChanged:
|
||||
return None, _coded(409, "source-changed", SOURCE_CHANGED_MESSAGE)
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
if selection.truncated:
|
||||
return None, _coded(400, "too-many-images", TOO_MANY)
|
||||
return tuple(selection.targets), None
|
||||
|
||||
async def _category_request(self, request, fields: frozenset):
|
||||
"""``(data, targets, roots, None)`` or ``(None, None, None, error)``.
|
||||
|
||||
The images are ``items`` (1 to 500 targets) or ``filter`` (all images
|
||||
an Images listing filter admits), besides *fields*.
|
||||
"""
|
||||
data, error = await _json_body_or_error(request)
|
||||
if error is not None:
|
||||
return None, None, None, error
|
||||
names = set(data) - fields
|
||||
if set(data) & fields != fields or names not in ({"items"}, {"filter"}):
|
||||
return (
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
bad_request(
|
||||
"Request must contain "
|
||||
+ ", ".join(sorted(fields))
|
||||
+ (" and " if fields else "")
|
||||
+ "either items or filter"
|
||||
),
|
||||
)
|
||||
if "filter" in data:
|
||||
targets, error = await self._filter_targets(data["filter"])
|
||||
if error is not None:
|
||||
return None, None, None, error
|
||||
return data, targets, list(self._get_all_output_dirs()), None
|
||||
items = data["items"]
|
||||
if not isinstance(items, list) or not 1 <= len(items) <= MAX_JOB_TARGETS:
|
||||
return (
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
bad_request(f"items must list 1 to {MAX_JOB_TARGETS} images"),
|
||||
)
|
||||
targets, error = _parse_targets(items, media_message=CATEGORY_MEDIA_MESSAGE)
|
||||
if error is not None:
|
||||
return None, None, None, error
|
||||
try:
|
||||
for _index, _scope, relative in targets:
|
||||
relative.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
return None, None, None, bad_request("path must be valid text")
|
||||
roots = list(self._get_all_output_dirs())
|
||||
for index, scope in dict.fromkeys((t[0], t[1]) for t in targets):
|
||||
if index >= len(roots):
|
||||
return None, None, None, bad_request("Invalid gallery root")
|
||||
error = self._gallery_scope_error(
|
||||
{"root_scope": scope, "root": str(index)}, roots
|
||||
)
|
||||
if error is not None:
|
||||
if error.status == 409:
|
||||
error = _coded(409, "source-changed", SOURCE_CHANGED_MESSAGE)
|
||||
return None, None, None, error
|
||||
return data, targets, roots, None
|
||||
|
||||
def _listed_file_keys(self, roots, targets, files) -> tuple[list, list]:
|
||||
"""``(listed keys, unlisted keys)`` of the targets (blocking).
|
||||
|
||||
A target counts as listed when its source's listing holds it; its key
|
||||
then follows the listed spelling. Targets folding to one key count
|
||||
once, and a key that is listed through any target is listed.
|
||||
"""
|
||||
listings, listed, missing = {}, {}, {}
|
||||
for index, _scope, relative in targets:
|
||||
root = roots[index]
|
||||
if index not in listings:
|
||||
listings[index] = self._favorite_listing(root, files[index])
|
||||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||||
key = _file_key(roots, root, spelled or relative)
|
||||
(missing if spelled is None else listed)[key] = True
|
||||
return list(listed), [key for key in missing if key not in listed]
|
||||
|
||||
async def set_image_category(self, request):
|
||||
"""PUT ``{items, category}``: set the own category of listed files."""
|
||||
data, targets, roots, error = await self._category_request(request, _PUT_FIELDS)
|
||||
if error is not None:
|
||||
return error
|
||||
category, error = _category_value(data["category"])
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
files = {}
|
||||
for index in dict.fromkeys(t[0] for t in targets):
|
||||
files[index] = await self._get_gallery_files(roots[index])
|
||||
listed_keys, missing_keys = await self._run_in_executor(
|
||||
self._listed_file_keys, roots, targets, files
|
||||
)
|
||||
except _KeyRefused as refused:
|
||||
return _forbidden(refused.message)
|
||||
except Exception:
|
||||
self.logger.exception("Image category update failed")
|
||||
return _coded(500, "category-failed", WRITE_FAILED)
|
||||
try:
|
||||
if not listed_keys:
|
||||
return _coded(
|
||||
404,
|
||||
"files-not-found",
|
||||
"None of the selected images could be found.",
|
||||
)
|
||||
counts = await self._run_in_executor(
|
||||
self.db.set_image_categories, listed_keys, category
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Image category update failed")
|
||||
return _coded(500, "category-failed", WRITE_FAILED)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"category": category,
|
||||
"updated": counts["updated"],
|
||||
"unchanged": counts["unchanged"],
|
||||
"not_found": len(missing_keys),
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
async def clear_image_category(self, request):
|
||||
"""DELETE ``{items}``: remove the own category, listed or not."""
|
||||
_data, targets, roots, error = await self._category_request(
|
||||
request, _DELETE_FIELDS
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
keys = await self._run_in_executor(
|
||||
lambda: list(
|
||||
dict.fromkeys(_file_key(roots, roots[i], r) for i, _s, r in targets)
|
||||
)
|
||||
)
|
||||
cleared = await self._run_in_executor(self.db.clear_image_categories, keys)
|
||||
except _KeyRefused as refused:
|
||||
return _forbidden(refused.message)
|
||||
except Exception:
|
||||
self.logger.exception("Image category removal failed")
|
||||
return _coded(500, "category-failed", WRITE_FAILED)
|
||||
return web.json_response(
|
||||
{"success": True, "cleared": cleared},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
@@ -193,17 +193,13 @@ class _Spec:
|
||||
return scope
|
||||
|
||||
|
||||
def _parse_targets(raw, **target_options):
|
||||
"""Unique validated targets in request order, or ``(None, error)``.
|
||||
|
||||
``target_options`` reach the target check (the image category routes
|
||||
pass their own ``media_message``, #332).
|
||||
"""
|
||||
def _parse_targets(raw):
|
||||
"""Unique validated targets in request order, or ``(None, error)``."""
|
||||
if not isinstance(raw, list) or not 1 <= len(raw) <= MAX_JOB_TARGETS:
|
||||
return None, bad_request(f"targets must list 1 to {MAX_JOB_TARGETS} images")
|
||||
seen, targets = set(), []
|
||||
for item in raw:
|
||||
target, error = _target(item, **target_options)
|
||||
target, error = _target(item)
|
||||
if error is not None:
|
||||
return None, error
|
||||
if target not in seen:
|
||||
|
||||
+25
-147
@@ -61,13 +61,10 @@ try:
|
||||
from ...database.image_favorites import MAX_KEY_LENGTH
|
||||
from ...database.library import display_title
|
||||
from ...database.models import normalize_image_path
|
||||
from ...database.run_attribution import recorded_output_lookup
|
||||
from ...utils.card_seed import card_seed, header_params, parsed_positive
|
||||
from ...utils.parallel import map_parallel
|
||||
from ...utils.json_strict import dumps_strict
|
||||
from ...utils.inspection_response import (
|
||||
OWN_CATEGORY_UNREAD,
|
||||
apply_own_category,
|
||||
encode_inspection_response,
|
||||
library_metadata,
|
||||
)
|
||||
@@ -84,13 +81,10 @@ except ImportError:
|
||||
from database.image_favorites import MAX_KEY_LENGTH
|
||||
from database.library import display_title
|
||||
from database.models import normalize_image_path
|
||||
from database.run_attribution import recorded_output_lookup
|
||||
from utils.card_seed import card_seed, header_params, parsed_positive
|
||||
from utils.parallel import map_parallel
|
||||
from utils.json_strict import dumps_strict
|
||||
from utils.inspection_response import (
|
||||
OWN_CATEGORY_UNREAD,
|
||||
apply_own_category,
|
||||
encode_inspection_response,
|
||||
library_metadata,
|
||||
)
|
||||
@@ -569,8 +563,7 @@ class _ImageTagMatch(NamedTuple):
|
||||
|
||||
tags: bool # every filter tag is in its image tags or linked positive tags
|
||||
query: bool # ``q`` is a case-folded substring of one image tag name
|
||||
# (hit, rating, positive id, inherited category) of its unambiguous link
|
||||
link: tuple | None
|
||||
link: tuple | None # (hit, rating, positive id) of its unambiguous link
|
||||
|
||||
|
||||
def _current_fingerprint(path: Path) -> tuple[int, int] | None:
|
||||
@@ -807,53 +800,6 @@ def _favorite_root_key(root) -> str:
|
||||
return _favorite_relative_key(os.path.realpath(str(root)))
|
||||
|
||||
|
||||
def _source_depth(source: str) -> int:
|
||||
return len([part for part in re.split(r"[\\/]+", source) if part])
|
||||
|
||||
|
||||
def deepest_source(sources, absolute_path):
|
||||
"""``(source, POSIX path below it)`` of the deepest source holding the path.
|
||||
|
||||
A source matches by its real path or by its spelling as configured (a
|
||||
detected output folder can be a symlink), and the deepest real path wins.
|
||||
The path is compared lexically after ``normpath`` and never resolved, so a
|
||||
symlinked subfolder keeps its listed spelling. Containment folds case
|
||||
where the OS does. *source* is the real path. None when no source
|
||||
contains the path.
|
||||
"""
|
||||
path = os.path.normpath(str(absolute_path))
|
||||
folded = os.path.normcase(path)
|
||||
best = None
|
||||
for source in sources:
|
||||
real = os.path.normpath(os.path.realpath(str(source)))
|
||||
for spelling in dict.fromkeys((real, os.path.normpath(str(source)))):
|
||||
prefix = os.path.join(os.path.normcase(spelling), "")
|
||||
if not folded.startswith(prefix):
|
||||
continue
|
||||
if best is None or _source_depth(real) > _source_depth(best[0]):
|
||||
best = (real, spelling)
|
||||
break
|
||||
if best is None:
|
||||
return None
|
||||
relative = os.path.relpath(path, best[1])
|
||||
return best[0], relative.replace(os.sep, "/")
|
||||
|
||||
|
||||
def image_category_key(sources, absolute_path):
|
||||
"""The file key of an image's own category (#332), or None.
|
||||
|
||||
``(root key, relative key)``: the deepest output source that contains the
|
||||
file and the path below it, keyed with the favorites helpers. Without
|
||||
nested sources it equals the file's favorite key. A path no source
|
||||
contains has no key (and so no own category).
|
||||
"""
|
||||
found = deepest_source(sources, absolute_path)
|
||||
if found is None:
|
||||
return None
|
||||
source, relative = found
|
||||
return _favorite_root_key(source), _favorite_relative_key(relative)
|
||||
|
||||
|
||||
def _name_suffix(name):
|
||||
"""``Path(name).suffix`` as Python 3.10-3.13 compute it.
|
||||
|
||||
@@ -1225,27 +1171,20 @@ def _admitted_entries(
|
||||
links: dict | None,
|
||||
favorites: frozenset | None = None,
|
||||
image_tags: dict | None = None,
|
||||
own_categories: dict | None = None,
|
||||
):
|
||||
"""Yield ``(position, entry, link match)`` for each entry the filters admit.
|
||||
|
||||
*links* maps normalized paths to (hit, rating, positive id, inherited
|
||||
category) for every unambiguously linked image (see
|
||||
PromptDatabase.get_image_link_matches); a tag filter admits only those,
|
||||
and ``q`` admits a filename match or a linked-positive hit. A category
|
||||
admits a file whose effective category equals it (#332 MC5): its own
|
||||
category (*own_categories*, ``{normalized path: category}``) when it has one,
|
||||
else the inherited one, so an unlinked file with its own category is
|
||||
admitted. *favorites*, when given, admits only those listed
|
||||
*links* maps normalized paths to (hit, rating, positive id) for every
|
||||
unambiguously linked image (see PromptDatabase.get_image_link_matches); a
|
||||
category or tag filter admits only those, and ``q`` admits a filename match
|
||||
or a linked-positive hit. *favorites*, when given, admits only those listed
|
||||
paths (``favorites=1``). *image_tags* maps normalized paths to their
|
||||
_ImageTagMatch: a tag filter also admits a file whose current image tags
|
||||
(with its linked positive's tags) carry every tag, and ``q`` also admits an
|
||||
image tag name hit.
|
||||
"""
|
||||
query, media, folder = filters["query"], filters["media"], filters["folder"]
|
||||
category = filters["category"]
|
||||
members_only = bool(filters["tags"])
|
||||
own = own_categories or {}
|
||||
members_only = filters["category"] is not None or bool(filters["tags"])
|
||||
native_folder = folder.replace("/", os.sep)
|
||||
nested = native_folder + os.sep
|
||||
exact = filters.get("path", "")
|
||||
@@ -1264,12 +1203,6 @@ def _admitted_entries(
|
||||
image = None if image_tags is None else image_tags.get(entry.normalized)
|
||||
if members_only and match is None and not (image and image.tags):
|
||||
continue
|
||||
if category is not None:
|
||||
effective = own.get(entry.normalized) if own else None
|
||||
if effective is None and match is not None:
|
||||
effective = match[3]
|
||||
if effective != category:
|
||||
continue
|
||||
if (
|
||||
query
|
||||
and query not in entry.folded
|
||||
@@ -1288,7 +1221,6 @@ def _filtered_page(
|
||||
links: dict | None,
|
||||
favorites: frozenset | None = None,
|
||||
image_tags: dict | None = None,
|
||||
own_categories: dict | None = None,
|
||||
) -> tuple[int, list, int | None]:
|
||||
"""Count the *entries* the filters admit and take *size* from *start* on.
|
||||
|
||||
@@ -1298,7 +1230,7 @@ def _filtered_page(
|
||||
"""
|
||||
total, page, positives = 0, [], set()
|
||||
for position, entry, match in _admitted_entries(
|
||||
entries, filters, links, favorites, image_tags, own_categories
|
||||
entries, filters, links, favorites, image_tags
|
||||
):
|
||||
total += 1
|
||||
if match is not None:
|
||||
@@ -2238,10 +2170,9 @@ class ImageRoutesMixin:
|
||||
stat'ed (never every listed file), and a candidate's tags count only
|
||||
while a byte record is verified at its present ``(size, mtime_ns)``
|
||||
(spec amendment S1). Under a Tags filter the candidates' linked
|
||||
positives are read too, so their tags combine; the category is checked
|
||||
once, on the effective category, by _admitted_entries (#332).
|
||||
positives are read too, so their tags and category combine.
|
||||
"""
|
||||
tags, query = filters["tags"], filters["query"]
|
||||
tags, query, category = filters["tags"], filters["query"], filters["category"]
|
||||
wanted = set(tags)
|
||||
located = self._workspace_locators(listings)
|
||||
# The tag index answers which files carry a relevant name (#379); no
|
||||
@@ -2271,9 +2202,11 @@ class ImageRoutesMixin:
|
||||
link = facts.get(entry.normalized)
|
||||
carried = set(names).union(link[4] if link else ())
|
||||
matches[entry.normalized] = _ImageTagMatch(
|
||||
bool(wanted) and wanted <= carried,
|
||||
bool(wanted)
|
||||
and wanted <= carried
|
||||
and (category is None or (link is not None and link[3] == category)),
|
||||
bool(query) and any(query in name.casefold() for name in names),
|
||||
tuple(link[:4]) if link else None,
|
||||
link[:3] if link else None,
|
||||
)
|
||||
return matches
|
||||
|
||||
@@ -2282,14 +2215,10 @@ class ImageRoutesMixin:
|
||||
) -> tuple:
|
||||
"""The ordered entries and what admits them (blocking; run in executor).
|
||||
|
||||
Returns ``(entries, key, links, image_tags, favorites, own)``: the
|
||||
listing in the filters' order, its keyset key, and the link, image-tag,
|
||||
favorite and own-category maps _admitted_entries reads. One SELECT
|
||||
(get_image_link_matches) reads every linked positive when *read_links*;
|
||||
under a category it keeps only links whose raw recipe or positive
|
||||
category is that value (the inherited one is decided in Python), and
|
||||
the links of files whose own category is that value are read by path,
|
||||
so their rating and positive still count (#332). A failed read raises when the
|
||||
Returns ``(entries, key, links, image_tags, favorites)``: the listing in
|
||||
the filters' order, its keyset key, and the link, image-tag and favorite
|
||||
maps _admitted_entries reads. One SELECT (get_image_link_matches) reads
|
||||
every linked positive when *read_links*. A failed read raises when the
|
||||
filters need it (category, tags, the rating sort) or *strict*; else the
|
||||
listing stays usable without links (a search then matches filenames
|
||||
only). A failed image-tag read raises under a Tags filter; a search then
|
||||
@@ -2301,10 +2230,10 @@ class ImageRoutesMixin:
|
||||
try:
|
||||
links = self.db.get_image_link_matches(
|
||||
filters["text"],
|
||||
category,
|
||||
tags,
|
||||
every_link=True,
|
||||
wanted=self._workspace_wanted(listings),
|
||||
category=category,
|
||||
)
|
||||
except Exception:
|
||||
if strict or category is not None or tags or order == "rating":
|
||||
@@ -2318,13 +2247,6 @@ class ImageRoutesMixin:
|
||||
),
|
||||
exc_info=True,
|
||||
)
|
||||
own = None
|
||||
if category is not None:
|
||||
# A failed read raises: an unfiltered page must never pass for the
|
||||
# category (DC6).
|
||||
own = self._own_category_map(self._get_all_output_dirs(), listings)
|
||||
if links is not None:
|
||||
links = self._own_category_links(links, own, filters)
|
||||
image_tags = None
|
||||
if filters["query"] or tags:
|
||||
try:
|
||||
@@ -2371,30 +2293,7 @@ class ImageRoutesMixin:
|
||||
favorites = (
|
||||
self._workspace_favorite_paths(listings) if filters["favorites"] else None
|
||||
)
|
||||
return entries, key, links, image_tags, favorites, own
|
||||
|
||||
def _own_category_links(self, links: dict, own: dict, filters: dict) -> dict:
|
||||
"""*links* plus the links of files whose own category is the filter's.
|
||||
|
||||
Their inherited category differs, so the category's link read left
|
||||
them out; read by path (get_image_link_facts), kept only when their
|
||||
positive has every filter tag, as the link read would.
|
||||
"""
|
||||
category, wanted = filters["category"], set(filters["tags"])
|
||||
missing = [
|
||||
path
|
||||
for path, value in own.items()
|
||||
if value == category and path not in links
|
||||
]
|
||||
if not missing:
|
||||
return links
|
||||
facts = self.db.get_image_link_facts(missing, filters["text"])
|
||||
extra = {
|
||||
path: tuple(fact[:4])
|
||||
for path, fact in facts.items()
|
||||
if wanted <= set(fact[4])
|
||||
}
|
||||
return {**links, **extra} if extra else links
|
||||
return entries, key, links, image_tags, favorites
|
||||
|
||||
def _select_workspace_page(
|
||||
self,
|
||||
@@ -2423,7 +2322,7 @@ class ImageRoutesMixin:
|
||||
or bool(filters["tags"])
|
||||
or order == "rating"
|
||||
)
|
||||
entries, key, links, image_tags, favorites, own = self._workspace_admission(
|
||||
entries, key, links, image_tags, favorites = self._workspace_admission(
|
||||
listings, filters, read_links=needed or after is None
|
||||
)
|
||||
start = 0
|
||||
@@ -2434,7 +2333,7 @@ class ImageRoutesMixin:
|
||||
else:
|
||||
start = bisect_right(entries, after, key=key)
|
||||
total, page, positive_count = _filtered_page(
|
||||
entries, start, limit + 1, filters, links, favorites, image_tags, own
|
||||
entries, start, limit + 1, filters, links, favorites, image_tags
|
||||
)
|
||||
if not needed and after is not None:
|
||||
positive_count = _COUNT_OMITTED
|
||||
@@ -2456,12 +2355,12 @@ class ImageRoutesMixin:
|
||||
|
||||
The links are read strictly: the positives must be the listing's own.
|
||||
"""
|
||||
entries, _key, links, image_tags, favorites, own = self._workspace_admission(
|
||||
entries, _key, links, image_tags, favorites = self._workspace_admission(
|
||||
listings, filters, read_links=True, strict=True
|
||||
)
|
||||
targets, positives, unlinked, scopes = [], {}, 0, {}
|
||||
for _position, entry, match in _admitted_entries(
|
||||
entries, filters, links, favorites, image_tags, own
|
||||
entries, filters, links, favorites, image_tags
|
||||
):
|
||||
if len(targets) == cap:
|
||||
return ListingSelection(targets, list(positives), unlinked, True)
|
||||
@@ -3780,25 +3679,17 @@ class ImageRoutesMixin:
|
||||
except Exception:
|
||||
# The link stays known; only its title is withheld.
|
||||
self.logger.warning("Inspection link title could not be read")
|
||||
# An output file's own category (#332); LoRA roots have none.
|
||||
own = (
|
||||
self._read_own_category(outputs, path)
|
||||
if index < len(outputs)
|
||||
else None
|
||||
)
|
||||
try:
|
||||
observed = inspect_current_file(
|
||||
path,
|
||||
source_label=_public_path(roots[index]),
|
||||
saved=saved,
|
||||
library=apply_own_category(
|
||||
self._inspection_library(saved, unavailable=unavailable),
|
||||
own,
|
||||
library=self._inspection_library(
|
||||
saved, unavailable=unavailable
|
||||
),
|
||||
image_tags=self._inspection_image_tags(
|
||||
roots[index], Path(relative).parts
|
||||
),
|
||||
recorded=recorded_output_lookup(self.db),
|
||||
)
|
||||
except CurrentFileChanged:
|
||||
return None
|
||||
@@ -3861,19 +3752,6 @@ class ImageRoutesMixin:
|
||||
|
||||
return read
|
||||
|
||||
def _read_own_category(self, sources, path):
|
||||
"""One file's own category, None, or OWN_CATEGORY_UNREAD (blocking).
|
||||
|
||||
Keyed by ``image_category_key`` over the output sources, as Images
|
||||
writes it. A failed read is logged and hides the category (DC5).
|
||||
"""
|
||||
try:
|
||||
key = image_category_key(sources, path)
|
||||
return None if key is None else self.db.image_category(*key)
|
||||
except Exception:
|
||||
self.logger.warning("Image category could not be read", exc_info=True)
|
||||
return OWN_CATEGORY_UNREAD
|
||||
|
||||
def _inspection_library(self, saved: dict | None, *, unavailable: bool) -> dict:
|
||||
"""The inspector's Library facts; the route sets the title afterwards."""
|
||||
result = {
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
from typing import Callable
|
||||
|
||||
try:
|
||||
from ...utils.prompt_inspector import (
|
||||
@@ -100,23 +100,18 @@ def inspect_current_file(
|
||||
saved: dict | None,
|
||||
library: dict,
|
||||
image_tags: Callable[[int, int], dict] | None = None,
|
||||
recorded: Callable[[Path], Any] | None = None,
|
||||
) -> dict:
|
||||
"""Current-file evidence read between two equal stats of *path*.
|
||||
|
||||
*image_tags*, when given, is called once with that stable
|
||||
``(st_size, st_mtime_ns)`` and its facet is returned as ``image_tags``
|
||||
(spec amendment S1: tags of a byte record verified at exactly that stat).
|
||||
*recorded*, when given, is asked within the same stat window for what
|
||||
recorded run history knows about *path* (``RecordedOutput`` or None: the
|
||||
save node that wrote it and the texts PromptManager recorded, #386).
|
||||
"""
|
||||
before = _current_file_facts(path)
|
||||
try:
|
||||
inspection = read_inspection_file(path)
|
||||
except MetadataReadError:
|
||||
inspection = None
|
||||
run = recorded(path) if recorded is not None and inspection is not None else None
|
||||
after = _current_file_facts(path)
|
||||
fields = ("st_dev", "st_ino", "st_mode", "st_size", "st_mtime_ns", "st_ctime_ns")
|
||||
if any(getattr(before, key) != getattr(after, key) for key in fields):
|
||||
@@ -134,15 +129,9 @@ def inspect_current_file(
|
||||
profile = (
|
||||
PREVIEW_INSPECTION_PROFILE if _in_host_temp(path) else INSPECTION_PROFILE
|
||||
)
|
||||
# The file name anchors on the saver that wrote it (#381); recorded run
|
||||
# history names that saver when the file alone cannot (#386).
|
||||
# The file name anchors on the saver that wrote it (#381).
|
||||
evidence = inspect_prompt_evidence(
|
||||
inspection.metadata,
|
||||
role_saved,
|
||||
profile=profile,
|
||||
file_name=path.name,
|
||||
recorded_saver_id=run.saver_id if run is not None else None,
|
||||
recorded_texts=run.texts if run is not None else None,
|
||||
inspection.metadata, role_saved, profile=profile, file_name=path.name
|
||||
)
|
||||
return {
|
||||
"file": {
|
||||
|
||||
+1
-16
@@ -47,11 +47,6 @@ import time
|
||||
import uuid
|
||||
from pathlib import PurePosixPath, PureWindowsPath
|
||||
|
||||
try:
|
||||
from ...utils import exit_signal
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils import exit_signal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# Start and finish order: wall-clock times can tie (coarse Windows clocks).
|
||||
_sequence = itertools.count()
|
||||
@@ -200,21 +195,11 @@ class Job:
|
||||
self.counters = dict.fromkeys(groups, 0)
|
||||
self.outcomes = {group: [] for group in groups}
|
||||
self.log = []
|
||||
self._cancel_requested = False
|
||||
self.cancelled = False
|
||||
self.last_event = None
|
||||
self.task = None
|
||||
self._subscribers = set()
|
||||
|
||||
@property
|
||||
def cancelled(self):
|
||||
"""Cancel was requested, or the interpreter is exiting (#376): every
|
||||
check a job already makes between files also stops it at exit."""
|
||||
return self._cancel_requested or exit_signal.requested()
|
||||
|
||||
@cancelled.setter
|
||||
def cancelled(self, value):
|
||||
self._cancel_requested = bool(value)
|
||||
|
||||
@property
|
||||
def record_changing(self):
|
||||
return self.admission == "records"
|
||||
|
||||
@@ -11,11 +11,6 @@ import stat
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from ...utils.logging_config import ClearRefused, parse_confirmed
|
||||
except ImportError: # loaded as the top-level ``py`` package (tests)
|
||||
from utils.logging_config import ClearRefused, parse_confirmed
|
||||
|
||||
# Keys PromptManagerLogger.update_config() understands. Anything else is
|
||||
# rejected so a client cannot inject arbitrary entries into the logging config.
|
||||
LOG_CONFIG_KEYS = frozenset(
|
||||
@@ -268,43 +263,10 @@ class LoggingRoutesMixin:
|
||||
)
|
||||
|
||||
async def truncate_logs(self, request):
|
||||
"""Clear all logs: what ``{"expected": [...]}`` lists, or all (#389).
|
||||
|
||||
``expected`` holds the ``{name, identity}`` rows of ``/logs/files`` the
|
||||
confirmation showed. Without a body (the legacy admin page, whose
|
||||
confirmation lists nothing) every rotated copy present now is deleted.
|
||||
A malformed list is 400; a clear that would empty or delete what was
|
||||
not confirmed changes nothing and is 409 with the reason's ``code``.
|
||||
"""
|
||||
"""Truncate all log files."""
|
||||
try:
|
||||
args = ()
|
||||
if request.can_read_body:
|
||||
data, error_response = await self._read_json_body(request)
|
||||
if error_response is not None:
|
||||
return error_response
|
||||
if "expected" in data:
|
||||
try:
|
||||
parse_confirmed(data["expected"])
|
||||
except ValueError:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "expected must list name and identity"
|
||||
" rows from /logs/files",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
args = (data["expected"],)
|
||||
logger_manager = self._get_logger_manager()
|
||||
try:
|
||||
results = await self._run_in_executor(
|
||||
logger_manager.truncate_logs, *args
|
||||
)
|
||||
except ClearRefused as refused:
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(refused), "code": refused.code},
|
||||
status=409,
|
||||
)
|
||||
results = await self._run_in_executor(logger_manager.truncate_logs)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
"""The listing identity of one media file, as a request names it.
|
||||
|
||||
Shared by the image-annotation routes and the collection routes (#331): a
|
||||
target is ``{root_index, root_scope, path}``, checked here without touching
|
||||
the filesystem or the database. The scope is compared with the current roots
|
||||
by the caller.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from .images import _forbidden, _has_traversal
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
from ...utils.media_capture import CAPTURE_EXTENSIONS
|
||||
except ImportError:
|
||||
from utils.media_capture import CAPTURE_EXTENSIONS
|
||||
|
||||
MAX_TARGET_PATH = 1024
|
||||
TARGET_FIELDS = frozenset({"root_index", "root_scope", "path"})
|
||||
_HEX64 = re.compile(r"[0-9a-f]{64}")
|
||||
TAGGING_MEDIA_MESSAGE = "Only media files can be tagged"
|
||||
|
||||
|
||||
def media_target(data: object, *, media_message: str = TAGGING_MEDIA_MESSAGE):
|
||||
"""``((index, scope, relative), None)`` for a target, else ``(None, error)``.
|
||||
|
||||
Only the listing identity is accepted: an int root index, the root's
|
||||
64-hex scope and a bounded root-relative POSIX media path, never an
|
||||
absolute, drive-qualified, backslash or ``..`` path (a 400 for a malformed
|
||||
shape, a 403 for an unsafe or non-media path, carrying *media_message*).
|
||||
"""
|
||||
if not isinstance(data, dict) or set(data) != TARGET_FIELDS:
|
||||
return None, bad_request("target must name root_index, root_scope and path")
|
||||
index, scope, relative = data["root_index"], data["root_scope"], data["path"]
|
||||
if type(index) is not int or index < 0:
|
||||
return None, bad_request("Invalid gallery root")
|
||||
if not isinstance(scope, str) or _HEX64.fullmatch(scope) is None:
|
||||
return None, bad_request("Invalid gallery root scope")
|
||||
if not isinstance(relative, str) or not relative or len(relative) > MAX_TARGET_PATH:
|
||||
return None, bad_request("path must be a relative media path")
|
||||
parts = relative.split("/")
|
||||
if (
|
||||
"\0" in relative
|
||||
or "\\" in relative
|
||||
or relative.startswith("/")
|
||||
or _has_traversal(relative)
|
||||
or any(part in ("", ".", "..") or ":" in part for part in parts)
|
||||
):
|
||||
return None, _forbidden()
|
||||
if os.path.splitext(parts[-1])[1].lower() not in CAPTURE_EXTENSIONS:
|
||||
return None, _forbidden(media_message)
|
||||
return (index, scope, relative), None
|
||||
+1
-11
@@ -677,14 +677,7 @@ def _verify_and_read(store, capture, item, root, need_metadata: bool):
|
||||
facts = store.verify(root, relative)
|
||||
captured = None
|
||||
if need_metadata:
|
||||
# The file's own name, so an inspection item resolves anchored on its
|
||||
# saver as it was recorded (#390).
|
||||
captured = captured_review(
|
||||
store,
|
||||
capture,
|
||||
item_profile(item["evidence"]),
|
||||
relative.rsplit("/", 1)[-1],
|
||||
)
|
||||
captured = captured_review(store, capture, item_profile(item["evidence"]))
|
||||
return relative, facts, captured
|
||||
|
||||
|
||||
@@ -730,9 +723,6 @@ def _candidate_still_offered(resolution: dict, choice: str) -> bool:
|
||||
that rewords the top-level reason or reorders candidates leaves a
|
||||
pending item applicable.
|
||||
"""
|
||||
if isinstance(resolution, dict) and resolution.get("anchor") == "saver":
|
||||
# Only the saver anchor resolves the bytes (#390): its pair alone.
|
||||
resolution = {"state": "ambiguous", "candidates": [resolution]}
|
||||
if not isinstance(resolution, dict) or resolution.get("state") != "ambiguous":
|
||||
return False
|
||||
evidence, _count, limited = review_evidence(resolution)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
The rule (maintainer test of #284/#288): a file is queued only when its
|
||||
final outcome is ambiguous and unlinked. The strict resolver must find it
|
||||
``ambiguous``, the scan's legacy fallback (the browser parser,
|
||||
``graph_prompt_text``) must not link it, and no saved row may link it
|
||||
``_graph_prompt_text``) must not link it, and no saved row may link it
|
||||
already. A file the fallback links, an exact pair the ingester stored and an
|
||||
already linked file are never queued. Neither is an ambiguous file with
|
||||
nothing to choose: fewer than two of its candidates carry a prompt
|
||||
@@ -88,8 +88,7 @@ class ReviewRecordingMixin:
|
||||
def _record_scanned_review(self, media_file, roots, review=None, scanned=None):
|
||||
"""Record one ambiguous, unlinked scanned file (logged and counted).
|
||||
|
||||
``scanned`` is the scan's ``(metadata, inspection_resolution)``; the
|
||||
resolution is None when the scan did not run the inspection profile.
|
||||
``scanned`` is the scan's ``(metadata, inspection_resolution)``.
|
||||
"""
|
||||
try:
|
||||
outcome = self._record_review_sync(
|
||||
@@ -200,7 +199,7 @@ class ReviewRecordingMixin:
|
||||
key = _memory_key(locator)
|
||||
if state is not None and memory.get(key) == state:
|
||||
return None
|
||||
resolution = review_capture.inspection_choice(canonical, media_file.name)
|
||||
resolution = review_capture.inspection_choice(canonical)
|
||||
if resolution is not None:
|
||||
memory.pop(key, None)
|
||||
return canonical, resolution
|
||||
|
||||
@@ -92,11 +92,6 @@ try:
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.parallel import map_parallel, note_refusal, refused_at_shutdown
|
||||
|
||||
try:
|
||||
from ...utils.exit_signal import ExitRequested
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.exit_signal import ExitRequested
|
||||
|
||||
# The placeholder frame a video without a preview is answered with (JPEG
|
||||
# bytes, never written to disk); written previews follow the settings.
|
||||
PLACEHOLDER_SIZE = (thumbnail_settings.DEFAULT_EDGE, thumbnail_settings.DEFAULT_EDGE)
|
||||
@@ -465,8 +460,6 @@ class AutoThumbnailer:
|
||||
def _run(self):
|
||||
try:
|
||||
self._drain()
|
||||
except ExitRequested:
|
||||
pass # #376: the interpreter is exiting; queued files are dropped
|
||||
except Exception:
|
||||
self._logger.warning("The automatic preview worker stopped", exc_info=True)
|
||||
finally:
|
||||
|
||||
@@ -7,7 +7,6 @@ Refactored from standalone_tagger.py for integration with PromptManager API.
|
||||
|
||||
import errno
|
||||
import gc
|
||||
import inspect
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -33,54 +32,6 @@ try:
|
||||
except ImportError:
|
||||
from py.tagging_defaults import DEFAULT_TAG_TEMPLATE
|
||||
|
||||
try:
|
||||
from ..utils import exit_signal
|
||||
except ImportError:
|
||||
from utils import exit_signal
|
||||
|
||||
|
||||
def abort_hub_downloads() -> None:
|
||||
"""Abort huggingface_hub's Xet transfers (run by the exit hook, #376).
|
||||
|
||||
Xet downloads report progress from native code, which ignores an
|
||||
exception raised in ``tqdm.update``; huggingface_hub aborts them itself
|
||||
on Ctrl+C with ``abort_xet_session``. Nothing happens when
|
||||
huggingface_hub was never imported or has no Xet support.
|
||||
"""
|
||||
if "huggingface_hub" not in sys.modules:
|
||||
return
|
||||
try:
|
||||
from huggingface_hub.utils import _xet
|
||||
except ImportError:
|
||||
return
|
||||
abort = getattr(_xet, "abort_xet_session", None)
|
||||
if callable(abort):
|
||||
abort()
|
||||
|
||||
|
||||
def _stop_at_exit(download) -> Dict[str, Any]:
|
||||
"""Keyword arguments that stop a huggingface_hub download at its next
|
||||
chunk once the interpreter is exiting (#376), or none when this
|
||||
huggingface_hub does not take ``tqdm_class``.
|
||||
|
||||
huggingface_hub 1.x removes its partial temp file on any failure; older
|
||||
versions keep their resumable ``.incomplete`` file. The model's own path
|
||||
is only written by the final rename either way.
|
||||
"""
|
||||
try:
|
||||
if "tqdm_class" not in inspect.signature(download).parameters:
|
||||
return {}
|
||||
from huggingface_hub.utils import tqdm as hub_tqdm
|
||||
except (ImportError, TypeError, ValueError):
|
||||
return {}
|
||||
|
||||
class StopAtExit(hub_tqdm):
|
||||
def update(self, n=1):
|
||||
exit_signal.checkpoint()
|
||||
return super().update(n)
|
||||
|
||||
return {"tqdm_class": StopAtExit}
|
||||
|
||||
|
||||
def _autotag_config():
|
||||
"""The saved defaults; imported late, as config needs ComfyUI's server."""
|
||||
@@ -446,7 +397,6 @@ class AutoTagService:
|
||||
filename=filename,
|
||||
local_dir=str(model_dir),
|
||||
local_dir_use_symlinks=False,
|
||||
**_stop_at_exit(hf_hub_download),
|
||||
)
|
||||
|
||||
if progress_callback:
|
||||
@@ -591,13 +541,6 @@ class AutoTagService:
|
||||
else:
|
||||
return self._download_hf_model(progress_callback)
|
||||
except Exception as e:
|
||||
if exit_signal.requested():
|
||||
# ExitRequested between chunks, or the Xet session aborted
|
||||
# by the exit hook (abort_hub_downloads).
|
||||
self.logger.info(
|
||||
f"Model download stopped: ComfyUI is exiting ({model_type})"
|
||||
)
|
||||
return False
|
||||
self.logger.error(f"Download failed: {e}")
|
||||
if progress_callback:
|
||||
progress_callback(f"Error: {str(e)}", 0)
|
||||
@@ -627,7 +570,6 @@ class AutoTagService:
|
||||
filename=config["filename"],
|
||||
local_dir=str(gguf_dir),
|
||||
local_dir_use_symlinks=False,
|
||||
**_stop_at_exit(hf_hub_download),
|
||||
)
|
||||
self.logger.info("GGUF model downloaded")
|
||||
|
||||
@@ -645,7 +587,6 @@ class AutoTagService:
|
||||
filename=config["mmproj_filename"],
|
||||
local_dir=str(gguf_dir),
|
||||
local_dir_use_symlinks=False,
|
||||
**_stop_at_exit(hf_hub_download),
|
||||
)
|
||||
self.logger.info("mmproj downloaded")
|
||||
|
||||
@@ -672,7 +613,6 @@ class AutoTagService:
|
||||
repo_id=config["repo"],
|
||||
local_dir=str(model_dir),
|
||||
local_dir_use_symlinks=False,
|
||||
**_stop_at_exit(snapshot_download),
|
||||
)
|
||||
self.logger.info("HF model downloaded")
|
||||
|
||||
|
||||
+2
-5
@@ -44,10 +44,8 @@ from .lora_manager_layout import (
|
||||
)
|
||||
|
||||
try:
|
||||
from ..utils import exit_signal
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError: # pragma: no cover - flat test imports
|
||||
from utils import exit_signal
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
LORA_CATEGORY = "lora-manager"
|
||||
@@ -191,15 +189,14 @@ def stage(
|
||||
downloads its CivitAI example images. A sidecar that cannot be read, or
|
||||
whose record fails to build, is skipped by its name, as are the
|
||||
centralized sidecars discovery could not parse. ``cancelled()`` is asked
|
||||
before each sidecar, and so is the exit signal (#376: the legacy scan's
|
||||
``cancelled`` does not read it); either raises ImportCancelled. ``on_record(index,
|
||||
before each sidecar (raises ImportCancelled); ``on_record(index,
|
||||
record_or_None, staged)`` follows each one.
|
||||
"""
|
||||
sidecars = discovery.sidecars
|
||||
staged = StagedImport(total=len(sidecars))
|
||||
staged.skipped.extend(discovery.unreadable)
|
||||
for index, sidecar in enumerate(sidecars):
|
||||
if cancelled() or exit_signal.requested():
|
||||
if cancelled():
|
||||
raise ImportCancelled()
|
||||
record = None
|
||||
path = sidecar.metadata_path
|
||||
|
||||
@@ -8,8 +8,8 @@ coverage>=7.16.2
|
||||
numpy>=2.2.6 # WD14 engine tests
|
||||
|
||||
# Code quality
|
||||
black==26.10.1
|
||||
flake8==7.4.1
|
||||
black>=26.10.1
|
||||
flake8>=7.4.1
|
||||
|
||||
# Security scanning
|
||||
bandit[toml]>=1.9.4
|
||||
|
||||
@@ -19,7 +19,7 @@ test.beforeEach(async ({page, browserName}) => {
|
||||
await import('/extensions/promptmanager/prompt_manager.js');
|
||||
class Node {
|
||||
constructor(){
|
||||
this.widgets=[...['text','category','tags','search_text'].map(name=>({name,value:'',inputEl:document.createElement('textarea')})),{name:'save_to_library',type:'toggle',value:true}];
|
||||
this.widgets=['text','category','tags','search_text'].map(name=>({name,value:'',inputEl:document.createElement('textarea')}));
|
||||
this.widgets_values=[]; this.inputs=[];
|
||||
}
|
||||
addDOMWidget(name,type,element){document.body.append(element);return {element};}
|
||||
@@ -36,14 +36,6 @@ test.afterEach(async ({page,browserName}) => {
|
||||
expect(page.classicErrors).toEqual([]);
|
||||
});
|
||||
|
||||
test('Classic loads an older workflow with Save to library on and keeps a saved off (#90)',async({page})=>{
|
||||
// A workflow saved before the switch held the search UI's slot ('') at the switch's index
|
||||
const loaded=await page.evaluate(()=>{const widget=node.widgets.find(w=>w.name==='save_to_library');widget.value='';node.configure({});return widget.value;});
|
||||
expect(loaded).toBe(true);
|
||||
const off=await page.evaluate(()=>{const widget=node.widgets.find(w=>w.name==='save_to_library');widget.value=false;node.configure({});return [widget.value,node.serialize().widgets_values[4]];});
|
||||
expect(off).toEqual([false,false]);
|
||||
});
|
||||
|
||||
test('Classic keeps immediate selection, search query and metadata serialization',async({page})=>{
|
||||
await page.route('**/prompt_manager/search?**', route=>route.fulfill({json:{success:true,results:[{text:'A new portrait',category:'Portrait',tags:['film grain'],rating:5}]}}));
|
||||
await page.evaluate(()=>{node.widgets.find(w=>w.name==='search_text').value='tag:"film grain" portrait';});
|
||||
|
||||
@@ -26,7 +26,7 @@ test.beforeEach(async({page,browserName})=>{
|
||||
window.fixtureApp={registerExtension(extension){this.extension=extension;},ui:{settings:new EventTarget()},graph:{setDirtyCanvas(){}},canvas:{}};
|
||||
await import('/extensions/promptmanager/prompt_manager.js');
|
||||
class Node {
|
||||
constructor(){this.widgets=[...['text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')})),{name:'save_to_library',type:'toggle',value:true,options:{}}];this.inputs=[...['clip','prepend_text','append_text'].map(name=>({name,type:name==='clip'?'CLIP':'STRING',link:null})),...['text','category','tags','search_text'].map(name=>({name,type:'STRING',link:null,widget:{name}})),{name:'save_to_library',type:'BOOLEAN',link:null,widget:{name:'save_to_library'}}];this.properties={};this.size=[550,700];}
|
||||
constructor(){this.widgets=['text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')}));this.inputs=[...['clip','prepend_text','append_text'].map(name=>({name,type:name==='clip'?'CLIP':'STRING',link:null})),...['text','category','tags','search_text'].map(name=>({name,type:'STRING',link:null,widget:{name}}))];this.properties={};this.size=[550,700];}
|
||||
findInputByType(type){const index=this.inputs.findIndex(input=>input.type===type&&input.link==null);return index<0?undefined:{index,slot:this.inputs[index]};}
|
||||
addWidget(type,name,value,callback,options){const widget={type,name,value,callback,options};this.widgets.push(widget);return widget;}
|
||||
addDOMWidget(name,type,element){const host=document.createElement('div');host.style.height='650px';host.className='fixture-dom-widget';host.append(element);document.body.append(host);return {element};}
|
||||
@@ -41,32 +41,6 @@ test.beforeEach(async({page,browserName})=>{
|
||||
});
|
||||
test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.focusedErrors).toEqual([]);});
|
||||
|
||||
test(`${variant} Save to library switch saves its value and loads older workflows on (#90)`, async ({page}) => {
|
||||
const toggle=page.getByRole('checkbox',{name:'Save to library',exact:true});
|
||||
await expect(toggle).toBeChecked();
|
||||
expect(await page.evaluate(()=>node.widgets.map(widget=>widget.name))).toEqual(['text','category','tags','search_text','prepend_text','append_text','save_to_library']);
|
||||
// The native switch is drawn in the toolbar only: the host keeps it hidden and zero-height, like the text fallbacks
|
||||
expect(await page.evaluate(()=>{const widget=node.widgets.find(item=>item.name==='save_to_library');return {type:widget.type,hidden:widget.hidden,size:widget.computeSize()};})).toEqual({type:'hidden',hidden:true,size:[0,-4]});
|
||||
expect(await page.evaluate(()=>node.widgets.find(item=>item.name==='text').computeSize())).toEqual([0,-4]);
|
||||
await toggle.uncheck();
|
||||
await expect(page.locator('.pm-focused > .pm-editor-status[role=status]')).toHaveText('Save to library is off: runs of this node are not added to the library.');
|
||||
const saved=await page.evaluate(()=>node.serialize());
|
||||
expect(saved.widgets_values[6]).toBe(false);
|
||||
await page.evaluate(saved=>node.configure(saved),saved);
|
||||
await expect(toggle).not.toBeChecked();
|
||||
await page.getByLabel('Main prompt',{exact:true}).fill('Typed while off');
|
||||
await expect(page.locator('.pm-focused > .pm-editor-status[role=status]')).toHaveText('Save to library is off: runs of this node are not added to the library.');
|
||||
// A workflow saved before the switch held the editor's own slot ('') at its index
|
||||
await page.evaluate(()=>node.configure({widgets_values:['Old text','','','','','',''],properties:{}}));
|
||||
await expect(toggle).toBeChecked();
|
||||
expect(await page.evaluate(()=>node.widgets.find(widget=>widget.name==='save_to_library').value)).toBe(true);
|
||||
await toggle.check();
|
||||
await page.getByLabel('Main prompt',{exact:true}).fill('Typed while on');
|
||||
await expect(page.locator('.pm-focused > .pm-editor-status[role=status]')).toHaveText('Main prompt is saved when this node runs.');
|
||||
await page.evaluate(()=>{node.inputs.find(input=>input.name==='save_to_library').link=9;node.onConnectionsChange?.();});
|
||||
await expect(toggle).toBeDisabled();
|
||||
});
|
||||
|
||||
test(`${variant} library autocompletes filters and searches without changing editor metadata`, async ({page}) => {
|
||||
const requests=[];
|
||||
await page.route('**/prompt_manager/categories',route=>route.fulfill({json:{success:true,categories:['Portrait']}}));
|
||||
@@ -283,7 +257,7 @@ test(`${variant} restores workflow values and disclosures and protects connected
|
||||
test(`${variant} offers text as a link socket, keeps typed text while linked and through a reload (#343)`,async({page})=>{
|
||||
const sockets=()=>page.evaluate(()=>Object.fromEntries(node.inputs.map(input=>[input.name,input.widget?.name??null])));
|
||||
// The hidden text widget no longer owns the input, so the host draws it as a socket; the other widget inputs are unchanged.
|
||||
expect(await sockets()).toEqual({clip:null,prepend_text:null,append_text:null,text:null,category:'category',tags:'tags',search_text:'search_text',save_to_library:'save_to_library'});
|
||||
expect(await sockets()).toEqual({clip:null,prepend_text:null,append_text:null,text:null,category:'category',tags:'tags',search_text:'search_text'});
|
||||
expect(await page.evaluate(()=>node.findInputByType('STRING').slot.name)).toBe('text');
|
||||
expect(await page.evaluate(()=>node.findInputByType('CLIP').slot.name)).toBe('clip');
|
||||
const main=page.getByRole('textbox',{name:'Main prompt',exact:true});
|
||||
|
||||
@@ -286,8 +286,7 @@ const continuationText=page=>page.locator('.infinite-continuation > p').first().
|
||||
async function searchOf(page,term) {
|
||||
const before=await countText(page);
|
||||
await page.locator('#search').fill(term);
|
||||
// Production's caption reads "Loading images…" until the search's listing answers (#391); the concept's is instant.
|
||||
await expect.poll(async()=>{const text=await countText(page);return text!==before&&text!=='Loading images…';}).toBe(true);
|
||||
await expect.poll(()=>countText(page)).not.toBe(before);
|
||||
await expect.poll(()=>page.locator(CARDS).count()).toBeLessThan(FIXTURE_IMAGES);
|
||||
await settle(page);
|
||||
const result={count:await countText(page),continuation:await continuationText(page),cards:await page.locator(`${CARDS} .image-open`).evaluateAll(nodes=>nodes.map(node=>node.getAttribute('aria-label')))};
|
||||
|
||||
@@ -228,8 +228,7 @@ async function productionSendOf(page,context) {
|
||||
async function tagOf(page) {
|
||||
const before=await page.locator('#result-count').textContent();
|
||||
await panelOf(page).locator('.detail-inner > .tags').last().getByRole('button',{name:'cinematic',exact:true}).click();
|
||||
// Production's caption reads "Loading images…" until the tag's listing answers (#391); the concept's search is instant.
|
||||
await expect.poll(async()=>{const text=await page.locator('#result-count').textContent();return text!==before&&text!=='Loading images…';}).toBe(true);
|
||||
await expect.poll(()=>page.locator('#result-count').textContent()).not.toBe(before);
|
||||
await settle(page);
|
||||
return {count:await page.locator('#result-count').textContent(),
|
||||
cards:await page.locator(`${CARDS} .image-open`).evaluateAll(nodes=>nodes.map(node=>node.getAttribute('aria-label')))};
|
||||
@@ -329,25 +328,6 @@ test('a recipe-linked image without recipe metadata shows its positive prompt me
|
||||
await expect(panelOf(page).locator('.detail-footer button')).toHaveText(['Send to ComfyUI','Prompt history']);
|
||||
});
|
||||
|
||||
// #391: a tag chip replaces the collection; until its listing answers the caption says it is loading, never that the
|
||||
// filter matched nothing ("0 of 0 images" was painted before the response and read by a fast reader).
|
||||
test('a tag chip collection reads as loading, not empty, until its listing answers (#391)',async({page})=>{
|
||||
await page.setViewportSize({width:1600,height:1100});
|
||||
await openProductionPanel(page,'dark');
|
||||
let release,arrived;
|
||||
const held=new Promise(resolve=>{release=resolve;}),reached=new Promise(resolve=>{arrived=resolve;});
|
||||
await page.route(url=>url.pathname==='/prompt_manager/images/output'&&url.searchParams.get('tags')==='cinematic',
|
||||
async route=>{arrived();await held;await route.continue();},{times:1});
|
||||
const count=page.locator('#result-count');
|
||||
await panelOf(page).locator('.detail-inner > .tags').last().getByRole('button',{name:'cinematic',exact:true}).click();
|
||||
await reached;
|
||||
await expect(count).toHaveText('Loading images…');
|
||||
await expect(page.locator(CARDS)).toHaveCount(0);
|
||||
release();
|
||||
await expect(count).toHaveText(countCopy('8 of 8 images · 2 positive prompts'));
|
||||
await expect(page.locator(CARDS)).toHaveCount(8);
|
||||
});
|
||||
|
||||
// Task C3 (D88, #249 PR C): the concept's chip row is the image's own tags (`i.tags`, app.js a402). With those names
|
||||
// saved as the file's current image tags, production draws the row from them (Tags filter buttons in the panel, plain
|
||||
// in the viewer, the linked positive's equal tags not repeated) and still equals the concept, in both sizes.
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
// #328: the legacy admin and gallery auto-tag batch and review keep working beside System › Auto-tagging. Since #330
|
||||
// the pages download and unload models only through System (legacy-system-links.spec.mjs); a batch never downloads
|
||||
// one. The fixture's fake download (`/__test__/tagging/download`) is held so nothing is ever fetched.
|
||||
// #328: the legacy admin and gallery auto-tag tools keep working beside System › Auto-tagging. The model download is
|
||||
// the `model_download` Activity job; the pages' stream attaches to it. No model is ever downloaded: the fixture's
|
||||
// fake download waits until the test releases it (`/__test__/tagging/download`).
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const TITLE = 'Tagging model download · WD14 ViT';
|
||||
|
||||
async function post(request, url, data) {
|
||||
await expect(await request.post(url, data === undefined ? {} : { data })).toBeOK();
|
||||
}
|
||||
@@ -31,6 +33,61 @@ test.afterEach(async ({ page, request, browserName }) => {
|
||||
await post(request, '/__test__/jobs/restart');
|
||||
});
|
||||
|
||||
async function startDownload(page, request) {
|
||||
await page.locator('#autoTagBtn').click();
|
||||
const status = page.locator('#wd14VitModelStatus');
|
||||
await status.getByRole('button', { name: 'Download', exact: true }).click();
|
||||
const modal = page.locator('#autoTagDownloadModal');
|
||||
await expect(modal).toBeVisible();
|
||||
await expect(page.locator('#downloadStatus')).toHaveText('Downloading…');
|
||||
await expect(page.locator('#downloadProgressBar')).toHaveClass(/animate-pulse/);
|
||||
await expect(modal).toContainText('The download continues in Activity.');
|
||||
await expect(page.locator('#cancelDownloadBtn')).toHaveText('Close');
|
||||
const running = (await downloads(request)).filter(job => job.state === 'running');
|
||||
expect(running.map(job => [job.title, job.cancellable])).toEqual([[TITLE, false]]);
|
||||
return { modal, status };
|
||||
}
|
||||
|
||||
test('admin download modal closes on complete and Activity lists the job without Cancel', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
const { modal, status } = await startDownload(page, request);
|
||||
|
||||
await page.goto('/prompt_manager/workspace#/activity');
|
||||
const item = page.locator('#workspace-main section.surface > .activity-item')
|
||||
.filter({ has: page.getByRole('link', { name: new RegExp(`${TITLE}$`) }) });
|
||||
await expect(item.locator('.chip')).toHaveText('Running');
|
||||
await expect(item.locator('progress')).not.toHaveAttribute('value');
|
||||
await expect(item.getByRole('button', { name: 'Cancel job', exact: true })).toHaveCount(0);
|
||||
|
||||
// Back on the admin page a second download of the same model attaches to the running job.
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await startDownload(page, request);
|
||||
await post(request, '/__test__/tagging/download', { mode: 'succeed' });
|
||||
await expect(modal).toBeHidden();
|
||||
await expect(status).toContainText('Downloaded');
|
||||
expect((await downloads(request)).map(job => [job.title, job.state])).toEqual([[TITLE, 'finished']]);
|
||||
});
|
||||
|
||||
test('gallery download modal closes on complete', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
const { modal, status } = await startDownload(page, request);
|
||||
await post(request, '/__test__/tagging/download', { mode: 'succeed' });
|
||||
await expect(modal).toBeHidden();
|
||||
await expect(page.locator('#autoTagModal')).toBeVisible();
|
||||
await expect(status).toContainText('Downloaded');
|
||||
expect((await downloads(request)).map(job => job.state)).toEqual(['finished']);
|
||||
});
|
||||
|
||||
test('a failed download closes the modal and says so', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
const { modal, status } = await startDownload(page, request);
|
||||
await post(request, '/__test__/tagging/download', { mode: 'fail' });
|
||||
await expect(modal).toBeHidden();
|
||||
await expect(page.getByText('Download failed: Download failed')).toBeVisible();
|
||||
await expect(status.getByRole('button', { name: 'Download', exact: true })).toBeVisible();
|
||||
expect((await downloads(request)).map(job => job.state)).toEqual(['failed']);
|
||||
});
|
||||
|
||||
// (wd14-vit is not downloaded in these tests: beforeEach marks it so the download tests can offer it.)
|
||||
// The gallery's batch reader keyed on `data.status` values the server never sends, so its modal never moved or
|
||||
// closed (#328 Problem 5). It now reads `type` frames as the admin page does.
|
||||
@@ -114,6 +171,25 @@ test('admin review opens the first image', async ({ page }) => {
|
||||
await expect(page.getByText('Failed to start review mode')).toHaveCount(0);
|
||||
});
|
||||
|
||||
test('gallery unload works and is refused while the gate is held', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await runGalleryBatch(page); // keeps the model in memory
|
||||
await page.locator('#autoTagBtn').click();
|
||||
const unload = page.locator('#unloadModelBtn');
|
||||
await expect(unload).toBeVisible();
|
||||
|
||||
// A held download owns the model gate: Unload is refused with the server's error.
|
||||
const held = await request.post('/prompt_manager/jobs/model-download', { data: { model_type: 'gguf' } });
|
||||
expect(held.status()).toBe(202);
|
||||
await unload.click();
|
||||
await expect(page.getByText(/^Failed to unload model: A tagging model is busy/)).toBeVisible();
|
||||
|
||||
await post(request, '/__test__/tagging/download', { mode: 'succeed' });
|
||||
await expect.poll(async () => (await downloads(request)).map(job => job.state)).toEqual(['finished']);
|
||||
await unload.click();
|
||||
await expect(page.getByText('WD14-SWINV2 model unloaded successfully')).toBeVisible();
|
||||
});
|
||||
|
||||
/** The admin review lists every row, videos too: Skip any image that is not one until the confirmation shows. */
|
||||
async function untilAutoTagged(page, confirm) {
|
||||
// Each image's skip check answers before its confirmation or its error shows.
|
||||
|
||||
@@ -1,286 +0,0 @@
|
||||
// System S13 (#330): the legacy admin and gallery pages open native System for their System tools. Each replaced
|
||||
// control is a plain link (`workspace#/system/<id>`, a new tab), and the legacy flows S13 keeps still work.
|
||||
import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs';
|
||||
import { test, expect } from './fixtures.mjs';
|
||||
|
||||
const PAGES = Object.freeze({ admin: '/prompt_manager/admin', gallery: '/prompt_manager/gallery' });
|
||||
const LABELS = Object.freeze({
|
||||
overview: 'Overview',
|
||||
appearance: 'Appearance',
|
||||
sources: 'Sources & scans',
|
||||
integrations: 'Integrations',
|
||||
tagging: 'Auto-tagging',
|
||||
maintenance: 'Maintenance',
|
||||
backups: 'Backup & export',
|
||||
diagnostics: 'Diagnostics & logs',
|
||||
});
|
||||
|
||||
async function openPageOptions(page) {
|
||||
await page.locator('#pageOptionsBtn').click();
|
||||
await expect(page.locator('#settingsModal')).toBeVisible();
|
||||
}
|
||||
|
||||
/** The auto-tag modal with its model rows read from /autotag/models. */
|
||||
async function openAutoTag(page) {
|
||||
const read = page.waitForResponse(item => item.url().endsWith('/prompt_manager/autotag/models'));
|
||||
await page.locator('#autoTagBtn').click();
|
||||
await read;
|
||||
await expect(page.locator('#autoTagModal')).toBeVisible();
|
||||
}
|
||||
|
||||
const notDownloaded = (page, request) => post(request, '/__test__/tagging/models', { unavailable: ['gguf'] });
|
||||
const modelLoaded = (page, request) => post(request, '/__test__/tagging/loaded', { model: 'wd14-vit' });
|
||||
|
||||
// One row per System link of the design's §2 map: the page, the control's label, how to find it, the section it opens
|
||||
// and what a test must arrange before the control shows.
|
||||
const LINKS = Object.freeze([
|
||||
{ page: 'admin', label: 'Settings', locate: page => page.locator('#settingsBtn'), section: 'overview' },
|
||||
{ page: 'admin', label: 'Maintenance', locate: page => page.locator('#maintenanceBtn'), section: 'maintenance' },
|
||||
{ page: 'admin', label: 'Backup DB', locate: page => page.locator('#backupBtn'), section: 'backups' },
|
||||
{ page: 'admin', label: 'Restore DB', locate: page => page.locator('#restoreBtn'), section: 'backups' },
|
||||
{ page: 'gallery', label: 'Settings', locate: page => page.locator('#settingsBtn'), section: 'appearance' },
|
||||
{ page: 'admin', label: 'Manage in System (model row)', arrange: notDownloaded, reveal: openAutoTag,
|
||||
locate: page => page.locator('#ggufModelStatus a'), section: 'tagging' },
|
||||
{ page: 'gallery', label: 'Manage in System (model row)', arrange: notDownloaded, reveal: openAutoTag,
|
||||
locate: page => page.locator('#ggufModelStatus a'), section: 'tagging' },
|
||||
{ page: 'gallery', label: 'Manage in System (Unload Model)', arrange: modelLoaded, reveal: openAutoTag,
|
||||
locate: page => page.locator('#unloadModelBtn'), section: 'tagging' },
|
||||
{ page: 'gallery', label: 'Page options Maintenance', reveal: openPageOptions,
|
||||
locate: page => page.locator('#pageOptionsMaintenanceLink'), section: 'maintenance' },
|
||||
{ page: 'gallery', label: 'Page options Sources & scans', reveal: openPageOptions,
|
||||
locate: page => page.locator('#pageOptionsSourcesLink'), section: 'sources' },
|
||||
{ page: 'admin', label: 'Diagnostics', locate: page => page.locator('#diagnosticsBtn'), section: 'diagnostics' },
|
||||
{ page: 'admin', label: 'View Logs', locate: page => page.locator('#logsBtn'), section: 'diagnostics' },
|
||||
]);
|
||||
|
||||
async function post(request, url, data) {
|
||||
await expect(await request.post(url, data === undefined ? {} : { data })).toBeOK();
|
||||
}
|
||||
|
||||
test.beforeEach(async ({ page, request, browserName }) => {
|
||||
await startCoverage(page, browserName);
|
||||
await post(request, '/__test__/reset');
|
||||
await post(request, '/__test__/system/reset');
|
||||
await post(request, '/__test__/tagging/reset');
|
||||
// The 3.2.0 update notice would cover the admin page's toolbar.
|
||||
await page.addInitScript(() => { localStorage.setItem('pm_update_notice_dismissed', '3.2.0'); });
|
||||
page.legacyErrors = [];
|
||||
page.on('pageerror', error => page.legacyErrors.push(error.message));
|
||||
});
|
||||
|
||||
test.afterEach(async ({ page, request, browserName }) => {
|
||||
await saveCoverage(page, browserName);
|
||||
await post(request, '/__test__/tagging/reset');
|
||||
await post(request, '/__test__/system/reset');
|
||||
expect(page.legacyErrors).toEqual([]);
|
||||
});
|
||||
|
||||
/** Opens the legacy page with the link showing; returns the link. */
|
||||
async function openLink(page, request, link) {
|
||||
if (link.arrange) await link.arrange(page, request);
|
||||
await page.goto(PAGES[link.page]);
|
||||
if (link.reveal) await link.reveal(page);
|
||||
const control = link.locate(page);
|
||||
await expect(control).toBeVisible();
|
||||
return control;
|
||||
}
|
||||
|
||||
// Each row opens its section in a new tab and sends no request from the legacy page (the design's `a System link sends
|
||||
// no request from the legacy page`, checked per row so one slow page never times out the whole table).
|
||||
for (const link of LINKS) {
|
||||
test(`legacy ${link.page} ${link.label} opens System ${link.section}`, async ({ page, context, request }) => {
|
||||
// The gallery's thumbnail startup check (K9) reads its status on a timer of its own; it is off here.
|
||||
await page.addInitScript(() => {
|
||||
localStorage.setItem('gallerySettings', JSON.stringify({ checkThumbnailsAtStartup: false }));
|
||||
});
|
||||
const control = await openLink(page, request, link);
|
||||
const legacyUrl = page.url();
|
||||
await expect(control).toHaveAttribute('href', `workspace#/system/${link.section}`);
|
||||
await expect(control).toHaveAttribute('target', '_blank');
|
||||
await expect(control).toHaveAttribute('rel', 'noopener');
|
||||
// The page's own start-up reads (list, film strips, settings) finish first; then the click must add none.
|
||||
// Lazy thumbnails may still load whenever the layout changes; they are not the click's.
|
||||
await page.waitForLoadState('networkidle');
|
||||
const sent = [];
|
||||
const record = item => { if (item.resourceType() !== 'image') sent.push(`${item.method()} ${item.url()}`); };
|
||||
page.on('request', record);
|
||||
const opened = context.waitForEvent('page');
|
||||
await control.click();
|
||||
const system = await opened;
|
||||
await expect(system).toHaveURL(new RegExp(`/prompt_manager/workspace#/system/${link.section}$`));
|
||||
page.off('request', record);
|
||||
expect(sent, 'a System link sends no request from the legacy page').toEqual([]);
|
||||
// System's page heading is "System"; the open section is the pressed button of its section switcher.
|
||||
const main = system.locator('#workspace-main');
|
||||
await expect(main.getByRole('heading', { level: 1, name: 'System', exact: true })).toBeVisible();
|
||||
await expect(main.getByRole('group', { name: 'System sections', exact: true })
|
||||
.getByRole('button', { name: LABELS[link.section], exact: true })).toHaveAttribute('aria-pressed', 'true');
|
||||
expect(page.url()).toBe(legacyUrl);
|
||||
await system.close();
|
||||
});
|
||||
}
|
||||
|
||||
test('kept admin flows still work', async ({ page, context }) => {
|
||||
await page.goto(PAGES.admin);
|
||||
const texts = page.locator('#resultsList .prompt-text');
|
||||
await expect(texts.first()).toBeVisible();
|
||||
|
||||
// K1: sorting reorders the list.
|
||||
for (const [value, order] of [['text_asc', 1], ['text_desc', -1]]) {
|
||||
const listed = page.waitForResponse(item => /\/prompt_manager\/(recent|search)\?/.test(item.url()) && item.url().includes(`sort=${value}`));
|
||||
await page.locator('#sortBy').selectOption(value);
|
||||
await listed;
|
||||
await expect.poll(async () => {
|
||||
const shown = await texts.allTextContents();
|
||||
// The server sorts case-insensitively (SQLite NOCASE), not by locale.
|
||||
const key = text => text.toLowerCase();
|
||||
const sorted = [...shown].sort((a, b) => order * (key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0));
|
||||
return shown.length > 1 && shown.join('\n') === sorted.join('\n');
|
||||
}).toBe(true);
|
||||
}
|
||||
// K3: Add Prompt saves a prompt the list then shows.
|
||||
await page.locator('#addPromptBtn').click();
|
||||
await page.locator('#addPromptText').fill('Kept flow lighthouse');
|
||||
const saved = page.waitForResponse(item => item.request().method() === 'POST' && item.url().endsWith('/prompt_manager/save'));
|
||||
await page.locator('#saveNewPromptBtn').click();
|
||||
expect((await saved).ok()).toBe(true);
|
||||
await expect(page.locator('#addPromptModal')).toBeHidden();
|
||||
await expect(texts.filter({ hasText: 'Kept flow lighthouse' })).toBeVisible();
|
||||
|
||||
// K1: a search narrows the list, and a selection enables the bulk buttons.
|
||||
const all = await texts.allTextContents();
|
||||
const word = all[0].trim().split(/\s+/)[0];
|
||||
const searched = page.waitForResponse(item => item.url().includes('/prompt_manager/search?') && item.url().includes('text='));
|
||||
await page.locator('#searchText').fill(word);
|
||||
await page.locator('#searchBtn').click();
|
||||
await searched;
|
||||
await expect.poll(async () => {
|
||||
const shown = await texts.allTextContents();
|
||||
return shown.length > 0 && shown.length < all.length && shown.every(text => text.toLowerCase().includes(word.toLowerCase()));
|
||||
}).toBe(true);
|
||||
await expect(page.locator('#bulkDeleteBtn')).toBeDisabled();
|
||||
await page.locator('#resultsList .prompt-checkbox').first().check();
|
||||
for (const id of ['#bulkDeleteBtn', '#bulkTagBtn', '#bulkCategoryBtn']) await expect(page.locator(id)).toBeEnabled();
|
||||
|
||||
// K4: Stats reads the statistics again; Export downloads the prompts.
|
||||
const stats = page.waitForResponse(item => item.url().endsWith('/prompt_manager/stats') && item.ok());
|
||||
await page.locator('#statsBtn').click();
|
||||
await stats;
|
||||
await expect(page.getByText('Stats refreshed successfully!')).toBeVisible();
|
||||
const download = page.waitForEvent('download');
|
||||
await page.locator('#exportBtn').click();
|
||||
expect((await download).suggestedFilename()).toMatch(/^prompts-\d{4}-\d{2}-\d{2}\.json$/);
|
||||
|
||||
// K5: Scan Images opens its dialog (with the quick backup) without starting a scan.
|
||||
await page.locator('#scanBtn').click();
|
||||
await expect(page.locator('#scanModal')).toBeVisible();
|
||||
await expect(page.locator('#quickBackupBtn')).toBeVisible();
|
||||
await page.locator('#cancelScan').click();
|
||||
await expect(page.locator('#scanModal')).toBeHidden();
|
||||
|
||||
// K7: Metadata and Gallery open their pages in a new tab.
|
||||
for (const [id, path] of [['#metadataBtn', '/prompt_manager/gallery.html'], ['#galleryBtn', '/prompt_manager/gallery']]) {
|
||||
const opened = context.waitForEvent('page');
|
||||
await page.locator(id).click();
|
||||
const tab = await opened;
|
||||
await tab.waitForLoadState('domcontentloaded');
|
||||
expect(new URL(tab.url()).pathname).toBe(path);
|
||||
await tab.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('Page options keeps the page-only settings admin', async ({ page, request }) => {
|
||||
const before = (await (await request.get('/prompt_manager/settings')).json()).settings;
|
||||
await page.goto(PAGES.admin);
|
||||
await expect(page.locator('#resultsList .prompt-text').first()).toBeVisible();
|
||||
await page.locator('#pageOptionsBtn').click();
|
||||
const dialog = page.locator('#settingsModal');
|
||||
await expect(dialog).toBeVisible();
|
||||
await expect(dialog.locator('h3')).toHaveText('Page options');
|
||||
// Exactly the page-only setting (G5): the list's Infinite Scrolling default.
|
||||
expect(await dialog.locator('input, select, textarea').evaluateAll(fields => fields.map(field => field.id)))
|
||||
.toEqual(['infiniteScrollDefault']);
|
||||
await expect(page.locator('#infiniteScrollDefault')).not.toBeChecked();
|
||||
await page.locator('#infiniteScrollDefault').check();
|
||||
const saved = page.waitForRequest(item => item.method() === 'POST' && item.url().endsWith('/prompt_manager/settings'));
|
||||
await page.locator('#saveSettings').click();
|
||||
expect((await saved).postDataJSON()).toEqual({ infinite_scroll: true });
|
||||
await expect(dialog).toBeHidden();
|
||||
// The list header follows the saved default at once; nothing else changed.
|
||||
await expect(page.locator('#infiniteScrollSession')).toBeChecked();
|
||||
const after = (await (await request.get('/prompt_manager/settings')).json()).settings;
|
||||
expect(after).toEqual({ ...before, infinite_scroll: true });
|
||||
await page.reload();
|
||||
// The dialog shows the saved default once the page has read it (the header follows it first).
|
||||
await expect(page.locator('#infiniteScrollSession')).toBeChecked();
|
||||
await page.locator('#pageOptionsBtn').click();
|
||||
await expect(page.locator('#infiniteScrollDefault')).toBeChecked();
|
||||
});
|
||||
|
||||
test('Page options keeps the page-only settings gallery', async ({ page }) => {
|
||||
// Native Appearance keys and the startup prompt's own keys are in the shared object before the page opens.
|
||||
const stored = { imageQuality: 'high', showImageInfo: false, videoMute: false,
|
||||
autoLoadMetadata: false, workspaceLayout: 'masonry', thumbnailPromptDismissed: 'sig', thumbnailsGenerated: true,
|
||||
checkThumbnailsAtStartup: false, defaultViewMode: 'list' };
|
||||
await page.addInitScript(value => {
|
||||
if (!sessionStorage.getItem('seeded')) {
|
||||
sessionStorage.setItem('seeded', '1');
|
||||
localStorage.setItem('gallerySettings', value);
|
||||
}
|
||||
}, JSON.stringify(stored));
|
||||
const listed = page.waitForResponse(item => item.url().includes('/prompt_manager/images/output?'));
|
||||
await page.goto(PAGES.gallery);
|
||||
await listed;
|
||||
// With no Page options control, a stored default view still opens the page in it.
|
||||
await expect(page.locator('#listViewBtn')).toHaveClass(/bg-pm-accent/);
|
||||
await openPageOptions(page);
|
||||
const dialog = page.locator('#settingsModal');
|
||||
await expect(dialog.locator('h3')).toHaveText('Page options');
|
||||
// Exactly the page-only settings (G5); the default view is System › Appearance's Default layout.
|
||||
expect(await dialog.locator('input, select, textarea').evaluateAll(fields => fields.map(field => field.id))).toEqual([
|
||||
'checkThumbnailsAtStartup', 'lazyLoadingToggle', 'defaultLimitSelect', 'gridColumnsSelect',
|
||||
'infiniteScrollToggle', 'debugModeToggle', 'apiTimeoutInput']);
|
||||
await expect(dialog.getByText('Default View Mode')).toHaveCount(0);
|
||||
await page.locator('#defaultLimitSelect').selectOption('50');
|
||||
await page.locator('#debugModeToggle').check();
|
||||
await page.locator('#saveSettingsBtn').click();
|
||||
await expect(dialog).toBeHidden();
|
||||
const read = () => page.evaluate(() => JSON.parse(localStorage.getItem('gallerySettings')));
|
||||
let saved = await read();
|
||||
expect(saved).toMatchObject({ ...stored, defaultLimit: 50, debugMode: true });
|
||||
|
||||
// Reset puts back only the page options' defaults; the Appearance, startup-prompt and default-view keys stay.
|
||||
await openPageOptions(page);
|
||||
page.once('dialog', dialog => dialog.accept());
|
||||
await page.locator('#resetSettingsBtn').click();
|
||||
await expect(page.getByText('Page options reset to defaults')).toBeVisible();
|
||||
await expect(page.locator('#defaultLimitSelect')).toHaveValue('100');
|
||||
saved = await read();
|
||||
expect(saved).toMatchObject({ defaultViewMode: 'list', imageQuality: 'high', showImageInfo: false, videoMute: false,
|
||||
autoLoadMetadata: false, workspaceLayout: 'masonry', thumbnailPromptDismissed: 'sig', thumbnailsGenerated: true,
|
||||
defaultLimit: 100, debugMode: false, checkThumbnailsAtStartup: true, lazyLoading: true, gridColumns: 8,
|
||||
infiniteScroll: false, apiTimeout: 30 });
|
||||
});
|
||||
|
||||
test('the legacy auto-tag modal stays open after Manage in System', async ({ page, context, request }) => {
|
||||
for (const legacy of ['admin', 'gallery']) {
|
||||
await post(request, '/__test__/tagging/models', { unavailable: ['gguf'] });
|
||||
await page.goto(PAGES[legacy]);
|
||||
await openAutoTag(page);
|
||||
const link = page.locator('#ggufModelStatus').getByRole('link', { name: 'Manage in System', exact: true });
|
||||
await expect(link).toHaveAttribute('title', 'Opens System › Auto-tagging in a new tab');
|
||||
// A downloaded model keeps its status text.
|
||||
await expect(page.locator('#wd14VitModelStatus')).toHaveText('\u2713 Downloaded');
|
||||
const opened = context.waitForEvent('page');
|
||||
await link.click();
|
||||
const system = await opened;
|
||||
await expect(system).toHaveURL(/#\/system\/tagging$/);
|
||||
await expect(page.locator('#autoTagModal')).toBeVisible();
|
||||
await system.close();
|
||||
// Downloaded in System meanwhile: reopening the modal reads the models again (CUJ 4).
|
||||
await post(request, '/__test__/tagging/models', { unavailable: [] });
|
||||
await page.goto(PAGES[legacy]);
|
||||
await openAutoTag(page);
|
||||
await expect(page.locator('#ggufModelStatus')).toHaveText('\u2713 Downloaded');
|
||||
}
|
||||
});
|
||||
@@ -17,7 +17,7 @@ test.beforeEach(async({page,browserName})=>{
|
||||
window.fixtureApp={registerExtension(extension){this.extension=extension;},ui:{settings},graph:{_nodes:[],setDirtyCanvas(){}},canvas:{},graphToPrompt:async()=>({})};
|
||||
await import('/extensions/promptmanager/prompt_manager.js');
|
||||
class Node {
|
||||
constructor(){this.id=1;this.type='PromptManagerPair';this.widgets=[...['text','negative_text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')})),{name:'save_to_library',type:'toggle',value:true,options:{}}];this.inputs=[...['clip','prepend_text','append_text'].map(name=>({name,type:name==='clip'?'CLIP':'STRING',link:null})),...['text','negative_text','category','tags','search_text'].map(name=>({name,type:'STRING',link:null,widget:{name}})),{name:'save_to_library',type:'BOOLEAN',link:null,widget:{name:'save_to_library'}}];this.properties={};this.size=[1100,700];}
|
||||
constructor(){this.id=1;this.type='PromptManagerPair';this.widgets=['text','negative_text','category','tags','search_text'].map(name=>({name,type:'text',value:'',options:{},inputEl:document.createElement('textarea')}));this.inputs=[...['clip','prepend_text','append_text'].map(name=>({name,type:name==='clip'?'CLIP':'STRING',link:null})),...['text','negative_text','category','tags','search_text'].map(name=>({name,type:'STRING',link:null,widget:{name}}))];this.properties={};this.size=[1100,700];}
|
||||
findInputByType(type){const index=this.inputs.findIndex(input=>input.type===type&&input.link==null);return index<0?undefined:{index,slot:this.inputs[index]};}
|
||||
addWidget(type,name,value,callback,options){const widget={type,name,value,callback,options};this.widgets.push(widget);return widget;}
|
||||
addDOMWidget(name,type,element){const host=document.createElement('div');host.style.cssText='height:700px;width:1100px;max-width:100%';host.className='fixture-dom-widget';host.append(element);document.body.append(host);return {element};}
|
||||
@@ -34,18 +34,6 @@ test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);
|
||||
async function selectPair(page){await page.getByRole('button',{name:'Recent',exact:true}).click();await page.locator('.pm-editor-result').click();}
|
||||
async function draft(page,positive='Draft positive',negative='Draft negative'){await page.getByRole('textbox',{name:'Positive prompt',exact:true}).fill(positive);await page.getByRole('textbox',{name:'Negative prompt',exact:true}).fill(negative);}
|
||||
|
||||
test('paired editor Save to library switch keeps the widget order and loads older workflows on (#90)',async({page})=>{
|
||||
await expect(page.locator('.pm-pair')).toBeVisible();
|
||||
const toggle=page.getByRole('checkbox',{name:'Save to library',exact:true});
|
||||
await expect(toggle).toBeChecked();
|
||||
expect(await page.evaluate(()=>node.widgets.map(widget=>widget.name))).toEqual(['text','negative_text','category','tags','search_text','prepend_text','append_text','save_to_library']);
|
||||
await toggle.uncheck();
|
||||
expect(await page.evaluate(()=>node.serialize().widgets_values[7])).toBe(false);
|
||||
await expect(page.locator('.pm-pair > .pm-editor-status[role=status]').last()).toHaveText('Save to library is off: runs of this node are not added to the library.');
|
||||
await page.evaluate(()=>node.configure({widgets_values:['P','N','','','','','',''],properties:{}}));
|
||||
await expect(toggle).toBeChecked();
|
||||
});
|
||||
|
||||
test('paired editor previews both texts before explicitly replacing both',async({page})=>{
|
||||
await expect(page.locator('.pm-pair')).toBeVisible();
|
||||
await draft(page);
|
||||
|
||||
@@ -268,7 +268,6 @@ def create_app(root: Path) -> web.Application:
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_categories",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -127,9 +127,9 @@ function mapTaggingConcept(reference, { template, phone }) {
|
||||
actions: { ...reference.actions, buttons: [preview, ['Download again', ''], ['Unload', '']] },
|
||||
// D134: the prompt-tag batch, reviewed first.
|
||||
operations: [{ tree: ['div', 'div.row-actions'], title: 'Tag output images', description: BATCH, actions: [['Review', '']] }],
|
||||
// D132 (#398): every select shows its whole chosen option at every width (at 390 the concept's 120px cap cuts
|
||||
// them; production lets each grow past it).
|
||||
selectsFit: reference.selectsFit.map(() => true),
|
||||
// D132: the engine select shows its whole chosen option at every width (at 390 the concept's 120px cap cuts it;
|
||||
// `#tag-model` alone may grow there). Existing tags keeps the concept's own fit, cut or not.
|
||||
selectsFit: reference.selectsFit.map((fit, index) => (index === 0 ? true : fit)),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -168,28 +168,16 @@ for (const theme of THEMES) for (const [width, height] of SIZES) {
|
||||
});
|
||||
}
|
||||
|
||||
// D132 (#398) at phone width and below the concept's smallest size: every option of every select, once chosen, shows
|
||||
// whole, with no page scroll.
|
||||
for (const theme of THEMES) for (const width of [390, 320]) {
|
||||
test(`auto-tagging selects fit every option at ${width} wide ${theme}`, async ({ page }, testInfo) => {
|
||||
await page.setViewportSize({ width, height: 844 });
|
||||
await productionTagging(page, theme);
|
||||
await page.locator('#workspace-main #tag-existing').selectOption('replace_after_review');
|
||||
await page.locator('#workspace-main section.surface').screenshot({ path: testInfo.outputPath(`production-tagging-selects-${theme}-${width}.png`) });
|
||||
const selects = page.locator('#workspace-main section.surface > .setting-row > select');
|
||||
await expect(selects).toHaveCount(2);
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
const select = selects.nth(index);
|
||||
const values = await select.evaluate(node => [...node.options].map(option => option.value));
|
||||
for (const value of values) {
|
||||
await select.selectOption(value);
|
||||
const measured = await page.evaluate(measureTagging, '#workspace-main');
|
||||
expect(measured.selectsFit, `${await select.getAttribute('id')}=${value}`).toEqual([true, true]);
|
||||
expect(measured.pageScroll, value).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
// D132 below the concept's smallest size: on a 320px phone the engine select still shows its whole option.
|
||||
for (const theme of THEMES) test(`auto-tagging engine select fits at 320 wide ${theme}`, async ({ page }) => {
|
||||
await page.setViewportSize({ width: 320, height: 700 });
|
||||
const production = await productionTagging(page, theme);
|
||||
for (const model of ['hf', 'gguf', 'wd14-swinv2']) {
|
||||
await page.locator('#workspace-main #tag-model').selectOption(model);
|
||||
expect((await page.evaluate(measureTagging, '#workspace-main')).selectsFit[0], model).toBe(true);
|
||||
}
|
||||
expect(production.pageScroll).toBe(false);
|
||||
});
|
||||
|
||||
test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
@@ -234,9 +222,6 @@ test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
const undoSqueeze = await mutate(page, { css: '#workspace-main #tag-model { max-width: 60px !important; min-width: 0 !important; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.0']));
|
||||
await undoSqueeze();
|
||||
const undoSqueezeExisting = await mutate(page, { css: '#workspace-main #tag-existing { max-width: 60px !important; min-width: 0 !important; }' });
|
||||
expect(await paths()).toEqual(expect.arrayContaining(['selectsFit.1']));
|
||||
await undoSqueezeExisting();
|
||||
// A row that wraps where the concept's does not.
|
||||
const undoWrap = await mutate(page, { css: '#workspace-main .setting-row:has(> #tag-existing) { flex-wrap: wrap; }' });
|
||||
expect(await paths()).toEqual(['rows.1.flexWrap']);
|
||||
@@ -248,14 +233,14 @@ test('auto-tagging comparison detects mutations', async ({ page, request }) => {
|
||||
expect(diffInventory(expected, await measure())).toEqual([]);
|
||||
});
|
||||
|
||||
// D132 at phone width: only the engine row wraps; Existing tags grows beside its text instead (#398).
|
||||
// D132 at phone width, unmapped: only the engine row wraps; Existing tags keeps the concept's cut select.
|
||||
test('auto-tagging at 390 wraps only the engine row', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
const concept = await conceptTagging(page, 'dark');
|
||||
const production = await productionTagging(page, 'dark');
|
||||
expect(diffInventory(mapTaggingConcept(concept, await savedTemplate(request, page)), production)).toEqual([]);
|
||||
expect(production.rows.map(row => row.flexWrap)).toEqual(['wrap', 'nowrap', 'nowrap', 'nowrap', 'nowrap']);
|
||||
expect(production.selectsFit).toEqual([true, true]);
|
||||
expect(production.selectsFit[1]).toBe(concept.selectsFit[1]);
|
||||
});
|
||||
|
||||
/** Runs in the page: the open review dialog: its title, body and the concept's tokens. */
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
// A restore the open workspace page has not noticed yet (#413).
|
||||
//
|
||||
// Specs that restore through the API while a page is open check what the page does with a restore it did not see:
|
||||
// its next write is refused as "Prompt storage was restored since this dialog opened". But the page's Activity store
|
||||
// reads the job list every 15 s (activity-store.mjs POLL_MS), and a read the server answers while the restore job runs
|
||||
// makes the page follow it: when it finishes, the page is shown again from the restored data (S9, index.mjs
|
||||
// onRestoreFinished), which closes any open dialog. On a slow runner a restore can take longer than the time left to
|
||||
// the next read (CI run 40646112668: the restore took 16 s), so whether the page saw it depended on timing.
|
||||
//
|
||||
// `armUnseenRestores(page)`, before the page loads, passes every job-list read of the page through one route, so it
|
||||
// knows which reads the server has not answered yet. `unseenRestore(page, restoring)` then closes the gate, waits
|
||||
// for those reads' answers (sent before the restore began, they cannot list it), runs `restoring()`, and only after
|
||||
// it has answered lets the held reads go on: they read the restore as already ended, as a tab that read its job list
|
||||
// just before and just after the restore does. A restore that is never seen running never reloads the page.
|
||||
const JOBS = '**/prompt_manager/jobs';
|
||||
const gates = new WeakMap();
|
||||
|
||||
/** Route `page`'s job-list reads through the restore gate. Call before the page's first navigation. */
|
||||
export async function armUnseenRestores(page) {
|
||||
const gate = { closed: null, inFlight: 0, idle: [] };
|
||||
gates.set(page, gate);
|
||||
await page.route(JOBS, async route => {
|
||||
if (gate.closed) {
|
||||
await gate.closed;
|
||||
return route.fallback();
|
||||
}
|
||||
gate.inFlight += 1;
|
||||
try {
|
||||
const response = await route.fetch();
|
||||
await route.fulfill({ response }).catch(() => {});
|
||||
} catch {
|
||||
await route.abort().catch(() => {});
|
||||
} finally {
|
||||
gate.inFlight -= 1;
|
||||
if (gate.inFlight === 0) for (const resolve of gate.idle.splice(0)) resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Run `restoring()` (an API restore) unseen by `page`'s Activity store; resolves with what it resolves with. */
|
||||
export async function unseenRestore(page, restoring) {
|
||||
const gate = gates.get(page);
|
||||
if (!gate) throw new Error('armUnseenRestores(page) must run before the page loads');
|
||||
let open;
|
||||
gate.closed = new Promise(resolve => { open = resolve; });
|
||||
try {
|
||||
// Reads already on their way were sent before the restore: their answers cannot list it.
|
||||
if (gate.inFlight > 0) await new Promise(resolve => gate.idle.push(resolve));
|
||||
return await restoring();
|
||||
} finally {
|
||||
gate.closed = null;
|
||||
open();
|
||||
}
|
||||
}
|
||||
@@ -104,6 +104,33 @@ for (const leave of ['beforeunload', 'pagehide']) {
|
||||
});
|
||||
}
|
||||
|
||||
// #222: the gallery's Clear cache clears the server's thumbnails by its own request and keeps this browser's
|
||||
// preferences and other tools' storage; nothing of Prompt Manager's is cached in a browser, and the toast says so.
|
||||
test('the gallery cache clear keeps preferences and other tools caches', async ({ page }) => {
|
||||
await page.addInitScript(() => {
|
||||
if (sessionStorage.getItem('seeded')) return;
|
||||
sessionStorage.setItem('seeded', '1');
|
||||
localStorage.setItem('pm_workspace_theme', 'light');
|
||||
localStorage.setItem('other-tool-setting', 'keep');
|
||||
localStorage.setItem('pm_other_tool', 'keep');
|
||||
});
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
await page.evaluate(async () => { await (await caches.open('other-tool-gallery-thumbnails')).put('/x', new Response('x')); });
|
||||
const clears = [];
|
||||
page.on('request', req => { if (req.url().includes('/clear-thumbnails')) clears.push(req.method()); });
|
||||
page.once('dialog', dialog => dialog.accept());
|
||||
await page.locator('#settingsBtn').click();
|
||||
await page.locator('#clearCacheBtn').click();
|
||||
await expect(page.getByText("Thumbnails cleared; original images untouched. Nothing of Prompt Manager's is cached in this browser."))
|
||||
.toBeVisible();
|
||||
expect(clears).toEqual(['POST']);
|
||||
expect(await page.evaluate(async () => ({
|
||||
theme: localStorage.getItem('pm_workspace_theme'), other: localStorage.getItem('other-tool-setting'),
|
||||
prefixed: localStorage.getItem('pm_other_tool'), notice: localStorage.getItem('pm_update_notice_dismissed'),
|
||||
caches: await caches.keys(),
|
||||
}))).toEqual({ theme: 'light', other: 'keep', prefixed: 'keep', notice: '3.2.0', caches: ['other-tool-gallery-thumbnails'] });
|
||||
});
|
||||
|
||||
// #379: the reviewed tags are that image's own tags; no prompt changes.
|
||||
test('actual gallery review generates and persists tags for the selected root\'s image', async ({ page, request }) => {
|
||||
@@ -199,7 +226,7 @@ test('gallery infinite scroll appends a real backend page while retaining existi
|
||||
await expect(await request.post('/__test__/reset', { data: { extra_images: 108 } })).toBeOK();
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(100);
|
||||
await page.locator('#pageOptionsBtn').click();
|
||||
await page.locator('#settingsBtn').click();
|
||||
await page.locator('#infiniteScrollToggle').check();
|
||||
await page.locator('#saveSettingsBtn').click();
|
||||
await expect(page.locator('#settingsModal')).toBeHidden();
|
||||
@@ -219,6 +246,37 @@ async function promptSnapshot(request) {
|
||||
})).sort((a, b) => a.id - b.id);
|
||||
}
|
||||
|
||||
async function downloadBackup(page) {
|
||||
const downloading = page.waitForEvent('download');
|
||||
await page.locator('#backupBtn').click();
|
||||
const download = await downloading;
|
||||
expect(download.suggestedFilename()).toMatch(/^prompts_backup_.*\.db$/);
|
||||
expect(await download.failure()).toBeNull();
|
||||
const chunks = [];
|
||||
for await (const chunk of await download.createReadStream()) chunks.push(chunk);
|
||||
const buffer = Buffer.concat(chunks);
|
||||
expect(buffer.subarray(0, 16).toString()).toBe('SQLite format 3\0');
|
||||
return { name: download.suggestedFilename(), mimeType: 'application/octet-stream', buffer };
|
||||
}
|
||||
|
||||
async function chooseRestore(page, file) {
|
||||
await page.locator('#restoreBtn').click();
|
||||
await expect(page.locator('#restoreModal')).toBeVisible();
|
||||
await page.locator('#restoreFileInput').setInputFiles(file);
|
||||
}
|
||||
|
||||
// Confirms the chosen restore and waits for the server's answer, not for a timer: a restore is disk-bound and can take
|
||||
// longer than an expect timeout on a slow runner (#359). The modal closes only after that answer.
|
||||
async function confirmRestoreAndWait(page) {
|
||||
const restored = page.waitForResponse(response => response.url().endsWith('/prompt_manager/restore'));
|
||||
await page.locator('#confirmRestore').click();
|
||||
const response = await restored;
|
||||
const body = await response.text();
|
||||
expect(response.status(), body).toBe(200);
|
||||
expect(JSON.parse(body)).toMatchObject({ success: true, prompt_count: 2, backup_created: expect.any(String) });
|
||||
await expect(page.locator('#restoreModal')).toBeHidden();
|
||||
}
|
||||
|
||||
function confirmNextDialog(page, accept, message) {
|
||||
page.once('dialog', async dialog => {
|
||||
expect(dialog.type()).toBe('confirm');
|
||||
@@ -234,6 +292,88 @@ async function deletePromptButton(page, text) {
|
||||
return page.locator(`[data-action="delete"][data-prompt-id="${id}"]`);
|
||||
}
|
||||
|
||||
test('downloaded backup restores deleted prompts, tags, and image links through the actual upload form', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
await expect(page.locator('.prompt-text')).toHaveCount(2);
|
||||
const before = await promptSnapshot(request);
|
||||
expect(before.map(prompt => prompt.tags)).toEqual([['cats'], ['dogs']]);
|
||||
expect(before.every(prompt => prompt.images.length === 1)).toBe(true);
|
||||
const backup = await downloadBackup(page);
|
||||
const button = await deletePromptButton(page, 'Cat portrait');
|
||||
confirmNextDialog(page, true, 'delete this prompt');
|
||||
await button.click();
|
||||
await expect(page.locator('.prompt-text')).toHaveCount(1);
|
||||
await expect(await request.post('/prompt_manager/save', { data: { text: 'Added after backup', tags: ['temporary'] } })).toBeOK();
|
||||
expect(await promptSnapshot(request)).not.toEqual(before);
|
||||
await chooseRestore(page, backup);
|
||||
confirmNextDialog(page, true, backup.name);
|
||||
await confirmRestoreAndWait(page);
|
||||
await expect(page.locator('.prompt-text')).toHaveCount(2);
|
||||
await page.reload();
|
||||
await expect(page.locator('.prompt-text').filter({ hasText: 'Cat portrait' })).toBeVisible();
|
||||
await expect(page.locator('.prompt-text').filter({ hasText: 'Added after backup' })).toHaveCount(0);
|
||||
expect(await promptSnapshot(request)).toEqual(before);
|
||||
await page.goto('/prompt_manager/admin#/tags');
|
||||
await page.locator('[data-tag-name="cats"]').click();
|
||||
await expect(page.locator('#tagPromptsGrid')).toContainText('Cat portrait');
|
||||
await page.locator('#tagPromptsGrid img').first().click();
|
||||
await expect(page.locator('.viewer-canvas img')).toBeVisible();
|
||||
await expect.poll(() => page.locator('.viewer-canvas img').evaluate(image => image.complete && image.naturalWidth > 0)).toBe(true);
|
||||
});
|
||||
|
||||
test('restore validation and cancellation preserve live data and allow a valid retry', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
const before = await promptSnapshot(request);
|
||||
const backup = await downloadBackup(page);
|
||||
let uploads = 0;
|
||||
page.on('request', request => { if (request.url().endsWith('/prompt_manager/restore')) uploads++; });
|
||||
await page.locator('#restoreBtn').click();
|
||||
await page.locator('#confirmRestore').click();
|
||||
await expect(page.getByText('Please select a database file to restore', { exact: true })).toBeVisible();
|
||||
await page.locator('#restoreFileInput').setInputFiles({ ...backup, name: 'backup.txt' });
|
||||
await page.locator('#confirmRestore').click();
|
||||
await expect(page.getByText('Please select a valid .db file', { exact: true })).toBeVisible();
|
||||
expect(uploads).toBe(0);
|
||||
await page.locator('#restoreFileInput').setInputFiles(backup);
|
||||
confirmNextDialog(page, false, 'replace your current database');
|
||||
await page.locator('#confirmRestore').click();
|
||||
await expect(page.locator('#restoreModal')).toBeVisible();
|
||||
expect(uploads).toBe(0);
|
||||
await page.locator('#restoreFileInput').setInputFiles({ name: 'corrupt.db', mimeType: 'application/octet-stream', buffer: Buffer.from('not a SQLite database') });
|
||||
confirmNextDialog(page, true, 'corrupt.db');
|
||||
const rejected = page.waitForResponse(response => response.url().endsWith('/prompt_manager/restore'));
|
||||
await page.locator('#confirmRestore').click();
|
||||
expect((await rejected).status()).toBe(400);
|
||||
await expect(page.getByText(/Restore failed:/)).toBeVisible();
|
||||
await expect(page.locator('#restoreModal')).toBeVisible();
|
||||
await expect(page.locator('#confirmRestore')).toBeEnabled();
|
||||
expect(await promptSnapshot(request)).toEqual(before);
|
||||
await page.locator('#restoreFileInput').setInputFiles(backup);
|
||||
confirmNextDialog(page, true, backup.name);
|
||||
await confirmRestoreAndWait(page);
|
||||
expect(await promptSnapshot(request)).toEqual(before);
|
||||
});
|
||||
|
||||
test('backup and restore recover from interrupted network requests without losing data', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
const before = await promptSnapshot(request);
|
||||
await page.route('**/prompt_manager/backup', route => route.abort('failed'), { times: 1 });
|
||||
await page.locator('#backupBtn').click();
|
||||
await expect(page.getByText(/Backup failed:/)).toBeVisible();
|
||||
const backup = await downloadBackup(page);
|
||||
await chooseRestore(page, backup);
|
||||
await page.route('**/prompt_manager/restore', route => route.abort('failed'), { times: 1 });
|
||||
confirmNextDialog(page, true, backup.name);
|
||||
await page.locator('#confirmRestore').click();
|
||||
await expect(page.getByText(/Restore failed:/)).toBeVisible();
|
||||
await expect(page.locator('#confirmRestore')).toBeEnabled();
|
||||
await expect(page.locator('#restoreModal')).toBeVisible();
|
||||
expect(await promptSnapshot(request)).toEqual(before);
|
||||
confirmNextDialog(page, true, backup.name);
|
||||
await confirmRestoreAndWait(page);
|
||||
expect(await promptSnapshot(request)).toEqual(before);
|
||||
});
|
||||
|
||||
test('prompt deletion requires confirmation and can retry after a network failure', async ({ page, request }) => {
|
||||
await page.goto('/prompt_manager/admin');
|
||||
const before = await promptSnapshot(request);
|
||||
@@ -261,3 +401,98 @@ test('prompt deletion requires confirmation and can retry after a network failur
|
||||
await expect(await request.get(url)).toBeOK();
|
||||
}
|
||||
});
|
||||
|
||||
// The duplicate scan reads the output root only, but the fixture's input root holds a different
|
||||
// shared/same.png. A bare /images/serve/<path> URL tries every gallery root in turn, so once the
|
||||
// output copy is deleted it answers 200 with the input copy: check each file in its own root.
|
||||
const serveIn = (root, path) => `/prompt_manager/images/serve/${path.split(/[\\/]/).map(encodeURIComponent).join('/')}?root=${root}`;
|
||||
const rootOf = url => new URL(url, 'http://gallery.test').searchParams.get('root');
|
||||
|
||||
// Which copy the scan keeps depends on file times: the fixture stamps shared/same.png with the
|
||||
// wall clock, and the kernel can stamp its bulk copies a fraction of a millisecond earlier.
|
||||
// listOrder rewrites the finished scan's groups so a test can pick the order it needs.
|
||||
async function openDuplicateScan(page, request, listOrder = null) {
|
||||
await expect(await request.post('/__test__/reset', { data: { extra_images: 2 } })).toBeOK();
|
||||
if (listOrder) {
|
||||
await page.route('**/prompt_manager/scan_duplicates/stream', async route => {
|
||||
const response = await route.fetch();
|
||||
const body = (await response.text()).replace(/^data: (.*)$/gm, (line, json) => {
|
||||
const event = JSON.parse(json);
|
||||
if (event.type !== 'complete') return line;
|
||||
for (const group of event.duplicates) group.images = listOrder(group.images);
|
||||
return `data: ${JSON.stringify(event)}`;
|
||||
});
|
||||
await route.fulfill({ response, body });
|
||||
});
|
||||
}
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(4);
|
||||
const originals = await page.locator('#galleryGrid img[data-original]').evaluateAll(images => images.map(image => image.dataset.original));
|
||||
const output = rootOf(originals.find(url => url.includes('/bulk/')));
|
||||
const input = rootOf(originals.find(url => url.includes('/shared/same.png') && rootOf(url) !== output));
|
||||
expect(output).not.toBeNull();
|
||||
expect(input).not.toBeNull();
|
||||
await page.locator('#settingsBtn').click();
|
||||
await page.locator('#scanDuplicatesBtn').click();
|
||||
await expect(page.locator('#duplicatesCount')).toHaveText('1');
|
||||
const candidates = page.locator('.duplicate-checkbox');
|
||||
await expect(candidates).toHaveCount(2);
|
||||
return { originals, output, input, candidates };
|
||||
}
|
||||
|
||||
async function expectOnlySelectedGone(page, request, { originals, output, input }, selected, retained) {
|
||||
expect((await request.get(serveIn(output, selected))).status()).toBe(404);
|
||||
await expect(await request.get(serveIn(output, retained))).toBeOK();
|
||||
await expect(await request.get(serveIn(input, 'shared/same.png'))).toBeOK();
|
||||
await page.reload();
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(3);
|
||||
const remaining = await page.locator('#galleryGrid img[data-original]').evaluateAll(images => images.map(image => image.dataset.original));
|
||||
expect(remaining).toHaveLength(originals.length - 1);
|
||||
for (const url of remaining) {
|
||||
expect(originals).toContain(url);
|
||||
await expect(await request.get(url)).toBeOK();
|
||||
}
|
||||
}
|
||||
|
||||
test('duplicate cleanup cancels safely and deletes only the selected file after a retry', async ({ page, request }) => {
|
||||
const scan = await openDuplicateScan(page, request);
|
||||
const { output, candidates } = scan;
|
||||
const selected = await candidates.first().getAttribute('data-image-path');
|
||||
const retained = await candidates.last().getAttribute('data-image-path');
|
||||
await candidates.first().check();
|
||||
let deletes = 0;
|
||||
page.on('request', request => { if (request.url().endsWith('/delete_duplicate_images')) deletes++; });
|
||||
confirmNextDialog(page, false, 'permanently delete 1 duplicate image files');
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
expect(deletes).toBe(0);
|
||||
await expect(await request.get(serveIn(output, selected))).toBeOK();
|
||||
await page.route('**/prompt_manager/delete_duplicate_images', route => route.abort('failed'), { times: 1 });
|
||||
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
await expect(page.getByText('Failed to delete duplicate images', { exact: true })).toBeVisible();
|
||||
await expect(page.locator('#removeDuplicatesBtn')).toBeEnabled();
|
||||
await expect(await request.get(serveIn(output, selected))).toBeOK();
|
||||
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
|
||||
const deleted = page.waitForResponse(response => response.url().endsWith('/delete_duplicate_images'));
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
expect((await (await deleted).json())).toMatchObject({ success: true, deleted_count: 1, failed_count: 0 });
|
||||
await expect(page.locator('#duplicatesModal')).toBeHidden();
|
||||
await expectOnlySelectedGone(page, request, scan, selected, retained);
|
||||
});
|
||||
|
||||
test('deleting a duplicate whose path another gallery root also holds removes only the scanned copy', async ({ page, request }) => {
|
||||
// The order CI saw: a bulk copy stamped first, so shared/same.png is the first Delete candidate.
|
||||
const ciOrder = ['bulk/0000.png', 'shared/same.png', 'bulk/0001.png'];
|
||||
const rank = image => ciOrder.indexOf(image.path.replace(/\\/g, '/'));
|
||||
const scan = await openDuplicateScan(page, request, images => [...images].sort((a, b) => rank(a) - rank(b)));
|
||||
const selected = await scan.candidates.first().getAttribute('data-image-path');
|
||||
const retained = await scan.candidates.last().getAttribute('data-image-path');
|
||||
expect(selected.replace(/\\/g, '/')).toBe('shared/same.png');
|
||||
await scan.candidates.first().check();
|
||||
confirmNextDialog(page, true, 'permanently delete 1 duplicate image files');
|
||||
const deleted = page.waitForResponse(response => response.url().endsWith('/delete_duplicate_images'));
|
||||
await page.locator('#removeDuplicatesBtn').click();
|
||||
expect((await (await deleted).json())).toMatchObject({ success: true, deleted_count: 1, failed_count: 0 });
|
||||
await expect(page.locator('#duplicatesModal')).toBeHidden();
|
||||
await expectOnlySelectedGone(page, request, scan, selected, retained);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
|
||||
import {readFile,writeFile} from 'node:fs/promises';
|
||||
import {createHash} from 'node:crypto';
|
||||
|
||||
@@ -17,13 +16,13 @@ const marker=(page,id,anchor='10')=>recorded(page).locator(`button[data-run-id="
|
||||
const exact=async locator=>locator.evaluate(node=>node.textContent);
|
||||
|
||||
test.beforeEach(async({page,request,browserName})=>{
|
||||
await startCoverage(page,browserName);await armUnseenRestores(page);page.historyErrors=[];page.on('pageerror',error=>page.historyErrors.push(error.message));
|
||||
await startCoverage(page,browserName);page.historyErrors=[];page.on('pageerror',error=>page.historyErrors.push(error.message));
|
||||
const reset=await request.post('/__test__/reset',{data:{chronological_history:true}});await expect(reset).toBeOK();page.fixture=await reset.json();
|
||||
});
|
||||
test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.historyErrors).toEqual([]);});
|
||||
|
||||
const command=async(request,data)=>{const response=await request.post('/__test__/history/commands',{data});await expect(response).toBeOK();return response.json();};
|
||||
const restoreSnapshot=async(page,request,bytes)=>{const response=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'history.db',mimeType:'application/octet-stream',buffer:bytes}}}));await expect(response).toBeOK();expect((await response.json()).success).toBe(true);};
|
||||
const restoreSnapshot=async(request,bytes)=>{const response=await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'history.db',mimeType:'application/octet-stream',buffer:bytes}}});await expect(response).toBeOK();expect((await response.json()).success).toBe(true);};
|
||||
const historyRead=async(request,id)=>{const response=await request.get(`/prompt_manager/library/${id}/history`);await expect(response).toBeOK();return response.json();};
|
||||
const frozenContexts=contexts=>contexts.map(({navigation,positive_prompt_id,negative_prompt_id,pair_id,...frozen})=>frozen);
|
||||
const immutableReceipts=items=>items.map(({current_candidate,...receipt})=>receipt);
|
||||
@@ -63,7 +62,7 @@ for(const boundary of ['window','detail','candidate','inspection','media']) test
|
||||
await command(request,{action:'arm',boundary});const pending=request.get(target);
|
||||
try {
|
||||
await expect.poll(async()=> (await(await request.get('/__test__/history/checkpoint')).json()).entered).toBe(true);
|
||||
await restoreSnapshot(page,request,bytes);
|
||||
await restoreSnapshot(request,bytes);
|
||||
} finally {await command(request,{action:'release'});}
|
||||
const retired=await pending;expect(retired.status()).toBe(409);expect((await retired.json()).code).toBe('history-changed');
|
||||
for(const url of [`/prompt_manager/library/${f.detail_id}/history?limit=1&cursor=${encodeURIComponent(firstWindow.next_cursor)}`,`${base}?epoch=${epoch}`,`${base}/outputs?epoch=${epoch}`,`${base}/outputs?epoch=${epoch}&cursor=${encodeURIComponent(outputs.next_cursor)}`,media.url,media.thumbnail_url,media.inspection_url]) {
|
||||
@@ -131,7 +130,7 @@ test('archive delete merge and prune preserve frozen evidence without repairing
|
||||
|
||||
test('actual restore with reused numeric owner rejects stale history and offers explicit saved recovery',async({page,request})=>{
|
||||
const f=page.fixture;await open(page,f.detail_id);const original=await exact(saved(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'));
|
||||
const source=await request.get('/__test__/history/reused-snapshot');await expect(source).toBeOK();await restoreSnapshot(page,request,await source.body());
|
||||
const source=await request.get('/__test__/history/reused-snapshot');await expect(source).toBeOK();await restoreSnapshot(request,await source.body());
|
||||
await marker(page,f.runs.shared).click();await expect(dialog(page)).toContainText('Recorded evidence changed');await page.keyboard.press('Escape');
|
||||
await expect(page.locator('.library-parent-status')).toContainText('changed');expect(await exact(saved(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'))).toBe(original);
|
||||
await expect(recorded(page).getByRole('button',{name:'Reload recorded history',exact:true})).toBeVisible();
|
||||
@@ -143,12 +142,12 @@ for(const outcome of ['success','error'])test(`delayed actual old browser ${outc
|
||||
const f=page.fixture;await open(page,f.detail_id);const snapshot=await request.get('/__test__/history/snapshot');const bytes=await snapshot.body();
|
||||
let entered,release;const started=new Promise(resolve=>{entered=resolve;}),held=new Promise(resolve=>{release=resolve;});
|
||||
await page.route(new RegExp(`/history/runs/${f.runs.shared}\\?`),async route=>{
|
||||
if(outcome==='error')await restoreSnapshot(page,request,bytes);
|
||||
if(outcome==='error')await restoreSnapshot(request,bytes);
|
||||
const response=await route.fetch();expect(response.status()).toBe(outcome==='success'?200:409);entered();await held;
|
||||
await route.fulfill({response}).catch(()=>{});
|
||||
});
|
||||
try {
|
||||
await marker(page,f.runs.shared).click();await started;if(outcome==='success')await restoreSnapshot(page,request,bytes);
|
||||
await marker(page,f.runs.shared).click();await started;if(outcome==='success')await restoreSnapshot(request,bytes);
|
||||
await page.evaluate(()=>{location.hash='#/overview';});await expect(page.locator('#workspace-main h1')).toHaveText('Overview');release();
|
||||
await page.unroute(new RegExp(`/history/runs/${f.runs.shared}\\?`));await open(page,f.detail_id);
|
||||
await expect(recorded(page).locator('button[data-run-id]')).toHaveCount(0);
|
||||
@@ -259,7 +258,7 @@ test('inspection first discovers a real restore and immediately retires every su
|
||||
if(changed)window.recordedRecoveryStates.push({receipts:run.querySelectorAll('[data-receipt]').length,active:[...run.querySelectorAll('[data-send-role],[data-copy-role],[data-view-receipt]')].filter(node=>!node.disabled&&node.checkVisibility()).length});
|
||||
});observer.observe(run,{subtree:true,childList:true,attributes:true,characterData:true});
|
||||
});
|
||||
await restoreSnapshot(page,request,await snapshot.body());
|
||||
await restoreSnapshot(request,await snapshot.body());
|
||||
} finally {await command(request,{action:'release'});}
|
||||
const response=await inspectionRead;expect(response.status()).toBe(409);expect((await response.json()).code).toBe('history-changed');
|
||||
await expect(dialog(page).locator('.recorded-run-status')).toContainText('Recorded evidence changed');
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs';
|
||||
|
||||
@@ -14,7 +13,7 @@ const RATINGS=[['5','5 / 5'],['4','4 / 5'],['3','3 / 5'],['2','2 / 5'],['1','1 /
|
||||
const editorDialog=page=>page.locator('dialog.library-editor');
|
||||
|
||||
test.beforeEach(async({page,request,browserName})=>{
|
||||
await startCoverage(page,browserName);await armUnseenRestores(page);
|
||||
await startCoverage(page,browserName);
|
||||
const reset=await request.post('/__test__/library-editor/reset',{data:{}});await expect(reset).toBeOK();page.fixture=await reset.json();
|
||||
page.editorErrors=[];page.on('pageerror',error=>page.editorErrors.push(error.message));
|
||||
page.writes=[];page.on('request',item=>{if(item.method()!=='GET')page.writes.push({method:item.method(),url:new URL(item.url()).pathname,body:item.postDataJSON()});});
|
||||
@@ -261,8 +260,7 @@ test('recovery states render inline and never replay a write',async({page,reques
|
||||
page.writes.length=0;
|
||||
dialog=await openEditor(page,{mode:'new'});await ready(dialog);
|
||||
const snapshot=await request.get('/__test__/library-editor/snapshot');await expect(snapshot).toBeOK();
|
||||
const buffer=await snapshot.body();
|
||||
const restore=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer}}}));
|
||||
const restore=await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:await snapshot.body()}}});
|
||||
await expect(restore).toBeOK();page.writes.length=0;
|
||||
await field(dialog,'Title').fill('After restore');await field(dialog,'Positive prompt').fill('Restored storage prompt');await button(dialog,'Save').click();
|
||||
await expect(note(dialog)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was saved.');
|
||||
@@ -590,7 +588,7 @@ test('saved refresh failure retries only reads',async({page})=>{
|
||||
|
||||
test('restored storage after a confirmed save or before an archive offers an explicit Library reload',async({page,request})=>{
|
||||
const f=page.fixture;const snapshot=await(await request.get('/__test__/library-editor/snapshot')).body();
|
||||
const restore=()=>unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}}));
|
||||
const restore=()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}});
|
||||
await openHistory(page,f.positive_id);
|
||||
await page.route(`**/prompt_manager/library/${f.positive_id}`,async route=>{
|
||||
if(route.request().method()!=='PATCH')return route.continue();
|
||||
@@ -617,7 +615,7 @@ test('restored storage after a confirmed save or before an archive offers an exp
|
||||
const DISCARDS='Leaving this dialog discards your entries.';
|
||||
test('restored storage before an Edit or Add negative save offers only the Library reload exit',async({page,request})=>{
|
||||
const f=page.fixture;const snapshot=await(await request.get('/__test__/library-editor/snapshot')).body();
|
||||
const restore=()=>unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}}));
|
||||
const restore=()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}});
|
||||
await openHistory(page,f.positive_id);
|
||||
await page.getByRole('button',{name:'Edit metadata',exact:true}).click();let dialog=editorDialog(page);await ready(dialog);
|
||||
await expect(await restore()).toBeOK();
|
||||
|
||||
@@ -47,17 +47,6 @@ async function serverJson(request, url) {
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* The restore job's own end (#361): its progress stream closes when the job takes its final state, however long the
|
||||
* disk-bound publish takes (bounded by the test, not by a 5 s expect), and the job finished. The toast the page shows
|
||||
* for it is then the page's reaction alone, checked with the default timeout.
|
||||
*/
|
||||
async function restoreFinished(request, id) {
|
||||
await expect(await request.get(`/prompt_manager/jobs/${id}/events`)).toBeOK();
|
||||
const { job } = await serverJson(request, `/prompt_manager/jobs/${id}`);
|
||||
expect([job.kind, job.state]).toEqual(['restore', 'finished']);
|
||||
}
|
||||
|
||||
async function openBackups(page) {
|
||||
// A fresh page each time: the drafts live for the page session (Q10), and a hash-only goto would not reload.
|
||||
if (page.url().endsWith('#/system/backups')) await page.reload();
|
||||
@@ -227,7 +216,6 @@ test('Restore starts the job, opens Activity; views reload after it finishes', a
|
||||
const { job } = await (await applied).json();
|
||||
expect([job.kind, job.title, job.cancellable]).toEqual(['restore', 'Database restore', false]);
|
||||
await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`);
|
||||
await restoreFinished(request, job.id);
|
||||
await expect(toast(page)).toHaveText(/^Database restore finished · Restored \d+ prompts?, /);
|
||||
// The library is the backup's again, and the page says so without a reload: the sidebar counts were read again.
|
||||
expect(await libraryTotal(request)).toBe(restoredTotal);
|
||||
@@ -256,8 +244,7 @@ test('a finished restore shows the open page again from the restored data', asyn
|
||||
const card = page.locator('#workspace-main article.prompt-card').filter({ hasText: 'S9 prompt the restore removes' });
|
||||
await expect(card).toHaveCount(1);
|
||||
await expect(await request.post('/__test__/release')).toBeOK();
|
||||
const { job } = await (await applied).json();
|
||||
await restoreFinished(request, job.id);
|
||||
await applied;
|
||||
await expect(toast(page)).toHaveText(/^Database restore finished · /);
|
||||
// The shelf was read again from the restored library, on the same route.
|
||||
await expect(card).toHaveCount(0);
|
||||
@@ -395,15 +382,14 @@ test('a failed settings read leaves the policy disabled with the reason', async
|
||||
await expect(row.locator('input')).toBeDisabled();
|
||||
});
|
||||
|
||||
test('the legacy Backup and Restore routes keep working and the admin page links to System', async ({ page, request }) => {
|
||||
test('the legacy admin Backup and Restore keep working', async ({ page, request }) => {
|
||||
const bytes = await backupBytes(request);
|
||||
const restored = await request.post('/prompt_manager/restore', { multipart: { database_file: {
|
||||
name: 'prompts_backup.db', mimeType: 'application/octet-stream', buffer: bytes } } });
|
||||
await expect(restored).toBeOK();
|
||||
expect((await restored.json()).success).toBe(true);
|
||||
// #330: the admin page's Backup DB and Restore DB open System › Backup & export.
|
||||
await page.goto('/prompt_manager/admin');
|
||||
for (const id of ['#backupBtn', '#restoreBtn']) await expect(page.locator(id)).toHaveAttribute('href', 'workspace#/system/backups');
|
||||
await expect(page.locator('#restoreFileInput')).toHaveCount(1);
|
||||
});
|
||||
|
||||
/** Start a fake running job (a metadata scan another tab started), which an exclusive restore must wait for. */
|
||||
@@ -436,7 +422,6 @@ test('a refused restore reopens its review: Restore retries it, Discard releases
|
||||
await dialog(page).getByRole('button', { name: 'Restore', exact: true }).click();
|
||||
const { job } = await (await applied).json();
|
||||
await expect(page).toHaveURL(`${workspace}#/activity/${job.id}`);
|
||||
await restoreFinished(request, job.id);
|
||||
await expect(toast(page)).toHaveText(/^Database restore finished · /);
|
||||
expect(await stagesOf(request)).toBe(0);
|
||||
});
|
||||
|
||||
@@ -516,7 +516,7 @@ test('a rollover after the review deletes nothing and Activity offers Review aga
|
||||
test('Clear all logs states both counts, confirms first and reads everything again', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
const truncates = [];
|
||||
page.on('request', item => { if (item.url().endsWith('/prompt_manager/logs/truncate')) truncates.push(item.postDataJSON()); });
|
||||
page.on('request', item => { if (item.url().endsWith('/prompt_manager/logs/truncate')) truncates.push(item.url()); });
|
||||
await openDiagnostics(page);
|
||||
await expect(lines(page)).toContainText('Database backup verified');
|
||||
const details = await openFiles(page);
|
||||
@@ -535,10 +535,6 @@ test('Clear all logs states both counts, confirms first and reads everything aga
|
||||
await dialog(page).getByRole('button', { name: 'Clear', exact: true }).click();
|
||||
await expect(toast(page)).toHaveText('Logs cleared · 3 rotated copies deleted.');
|
||||
expect(truncates).toHaveLength(1);
|
||||
// #389: Clear sends back exactly the files the dialog was built from, each with its identity string.
|
||||
expect(truncates[0].expected.map(row => row.name).sort()).toEqual(['prompt_manager.log', 'prompt_manager.log.1',
|
||||
'prompt_manager.log.2', 'prompt_manager.log.3']);
|
||||
for (const row of truncates[0].expected) expect(row.identity).toMatch(/^\d+:\d+:\d+$/);
|
||||
// Read again: the viewer, the file line and the list.
|
||||
await expect(lines(page)).not.toContainText('Database backup verified');
|
||||
await expect(details.locator(':scope > p.small').first()).toHaveText(/rotated copies: 0$/);
|
||||
@@ -568,23 +564,6 @@ test('Clear all logs reads the files again before it states what it deletes', as
|
||||
expect(await names(request)).toEqual(['prompt_manager.log', 'prompt_manager.log.1', 'prompt_manager.log.2', 'prompt_manager.log.3']);
|
||||
});
|
||||
|
||||
test('a rollover while Clear all logs is open deletes nothing and says why (#389)', async ({ page, request }) => {
|
||||
await seedLogs(request, { rotated: [1024, 1024], active: 100 });
|
||||
await openDiagnostics(page);
|
||||
await openFiles(page);
|
||||
await operation(page, 'Clear all logs').getByRole('button', { name: 'Clear all logs…', exact: true }).click();
|
||||
await expect(dialog(page).locator('#modal-body > p')).toHaveText(/^Empties prompt_manager\.log, deletes 2 rotated copies /);
|
||||
// The log rolls over after the confirmation was built: the copies it listed are not the ones there now.
|
||||
await expect(await request.post('/__test__/logs/rotate')).toBeOK();
|
||||
const rolled = await names(request);
|
||||
const answer = page.waitForResponse(response => response.url().endsWith('/prompt_manager/logs/truncate'));
|
||||
await dialog(page).getByRole('button', { name: 'Clear', exact: true }).click();
|
||||
expect((await answer).status()).toBe(409);
|
||||
await expect(toast(page)).toHaveText('The log files changed since you confirmed; nothing was cleared. '
|
||||
+ 'Read the list again and clear again.');
|
||||
expect(await names(request)).toEqual(rolled);
|
||||
});
|
||||
|
||||
test('each listed file downloads by its returned name and a link has no Download', async ({ page, request }) => {
|
||||
await seedLogs(request, LOG_FOLDER);
|
||||
await openDiagnostics(page);
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
|
||||
import {writeFile} from 'node:fs/promises';
|
||||
import {test,expect} from './fixtures.mjs';
|
||||
import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
|
||||
import {captureCanonicalState} from './history-canonical-comparison.mjs';
|
||||
import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs';
|
||||
|
||||
@@ -41,7 +40,7 @@ async function expectLibrary(page,{tags='',category='',count}) {
|
||||
|
||||
let manifest;
|
||||
test.beforeEach(async({page,request,browserName})=>{
|
||||
await startCoverage(page,browserName);await armUnseenRestores(page);manifest=await seed(request);
|
||||
await startCoverage(page,browserName);manifest=await seed(request);
|
||||
page.vocabularyErrors=[];page.on('pageerror',error=>page.vocabularyErrors.push(error.message));
|
||||
});
|
||||
test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.vocabularyErrors).toEqual([]);});
|
||||
@@ -265,8 +264,7 @@ const toast=page=>page.locator('#workspace-toast');
|
||||
const actionNames=scope=>scope.locator('.dialog-actions button').evaluateAll(nodes=>nodes.map(node=>node.textContent.trim()));
|
||||
const snapshot=async request=>{const response=await request.get('/__test__/vocabulary/snapshot');await expect(response).toBeOK();return response.json();};
|
||||
const backup=async request=>{const response=await request.get('/__test__/vocabulary/backup');await expect(response).toBeOK();return response.body();};
|
||||
// A restore the open page has not seen, so its dialog stays to report it (#413).
|
||||
const restore=async(page,request,buffer)=>expect(await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'vocabulary.db',mimeType:'application/octet-stream',buffer}}}))).toBeOK();
|
||||
const restore=async(request,buffer)=>expect(await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'vocabulary.db',mimeType:'application/octet-stream',buffer}}})).toBeOK();
|
||||
const tagNames=rows=>rows.tags.map(([,name])=>name);
|
||||
function trackWrites(page) {
|
||||
const writes=[];page.on('request',item=>{if(item.method()!=='GET')writes.push({method:item.method(),url:new URL(item.url()).pathname,body:item.postDataJSON()});});return writes;
|
||||
@@ -484,7 +482,7 @@ test('conflict and restore states keep the dialog and write nothing',async({page
|
||||
|
||||
const before=await snapshot(request);
|
||||
box=await openDialog(page,main(page).getByRole('button',{name:'+ Create tag',exact:true}),'Create tag');
|
||||
await restore(page,request,saved);
|
||||
await restore(request,saved);
|
||||
await field(box,'Tag name').fill('sunrise');await button(box,'Apply').click();
|
||||
await expect(note(box)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was changed.');
|
||||
await expect(button(box,'Apply')).toBeDisabled();
|
||||
@@ -624,7 +622,7 @@ test('delete confirmation read failure, missing tag and restore keep Delete disa
|
||||
await expect(button(box,'Delete')).toBeDisabled();await expect(tagRow(page,'calm').locator('td:nth-child(2)')).toHaveText('2');
|
||||
await button(box,'Cancel').click();
|
||||
|
||||
box=await openRename(page,'mist');await restore(page,request,saved);
|
||||
box=await openRename(page,'mist');await restore(request,saved);
|
||||
await button(box,'Delete tag…').click();
|
||||
await expect(note(box)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was changed.');
|
||||
await expect(button(box,'Delete')).toBeDisabled();expect(await actionNames(box)).toEqual(['Reload','Cancel','Delete']);
|
||||
|
||||
@@ -181,7 +181,6 @@ _TABLES = (
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_categories",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -123,34 +123,6 @@ def _touch(path: Path, size: int, moment: datetime.datetime) -> None:
|
||||
os.utime(path, (stamp, stamp))
|
||||
|
||||
|
||||
def _append_active(manager, log_dir: Path, size: int) -> None:
|
||||
"""Grow the active log as this process's handler does (#389).
|
||||
|
||||
A second handle appending to the file the handler writes is what Clear
|
||||
all logs refuses as another program writing to it, so the bytes go
|
||||
through the handler's own stream; with file logging off, to the file.
|
||||
"""
|
||||
|
||||
def same(path) -> bool:
|
||||
return os.path.normcase(os.path.realpath(path)) == os.path.normcase(
|
||||
os.path.realpath(log_dir / ACTIVE)
|
||||
)
|
||||
|
||||
for handler in manager._file_handlers():
|
||||
stream = getattr(handler, "stream", None)
|
||||
if stream is None or not same(handler.baseFilename):
|
||||
continue
|
||||
handler.acquire()
|
||||
try:
|
||||
stream.write("x" * size)
|
||||
stream.flush()
|
||||
finally:
|
||||
handler.release()
|
||||
return
|
||||
with open(log_dir / ACTIVE, "ab") as handle:
|
||||
handle.write(b"x" * size)
|
||||
|
||||
|
||||
def _bounded(value, name, top=MAX_SEED_BYTES):
|
||||
if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= top:
|
||||
raise web.HTTPBadRequest(text=f"{name} must be an integer 0..{top}")
|
||||
@@ -197,7 +169,8 @@ def register_diagnostics_commands(app, routes, api, root: Path):
|
||||
for index, (time_text, level, message) in enumerate(CONCEPT_RECORDS):
|
||||
record = _record(time_text, level, message, index)
|
||||
manager.add_to_buffer(record, record.getMessage())
|
||||
_append_active(manager, log_dir, CONCEPT_ACTIVE)
|
||||
with open(log_dir / ACTIVE, "ab") as handle:
|
||||
handle.write(b"x" * CONCEPT_ACTIVE)
|
||||
for number, size in enumerate(CONCEPT_ROTATED, start=1):
|
||||
moment = ROTATED_TIME - datetime.timedelta(hours=number)
|
||||
_touch(log_dir / f"{ACTIVE}.{number}", size, moment)
|
||||
@@ -221,7 +194,8 @@ def register_diagnostics_commands(app, routes, api, root: Path):
|
||||
manager.add_to_buffer(record, record.getMessage())
|
||||
active = _bounded(data.get("active", 0), "active")
|
||||
if active:
|
||||
_append_active(manager, log_dir, active)
|
||||
with open(log_dir / ACTIVE, "ab") as handle:
|
||||
handle.write(b"x" * active)
|
||||
rotated = data.get("rotated", [])
|
||||
if not isinstance(rotated, list) or len(rotated) > 20:
|
||||
raise web.HTTPBadRequest(text="rotated must be a list of sizes")
|
||||
|
||||
@@ -33,7 +33,6 @@ _TABLES = (
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_categories",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -85,7 +85,6 @@ _TABLES = (
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_categories",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -35,8 +35,7 @@ read. These per-server hooks only change what a System test needs on top:
|
||||
previews for Convert outdated;
|
||||
- ``POST /__test__/system/jxl-encoder`` ``{available}`` (#317) makes the
|
||||
server read as having (``cjxl``) or lacking a JPEG XL encoder without
|
||||
probing, whatever the host has installed (#371: ``false`` stands in for the
|
||||
detector itself); reset restores the thumbnail settings and the real detection.
|
||||
probing; reset restores the thumbnail settings and the real detection.
|
||||
|
||||
- ``POST /__test__/system/duplicate-prompts`` (S7) adds two prompts whose
|
||||
text differs only in case and spacing (a duplicate prompt text group);
|
||||
@@ -119,11 +118,6 @@ def register_system_commands(app, routes, api, root: Path):
|
||||
}
|
||||
from py.api import thumbnail_encode
|
||||
|
||||
real_detect = thumbnail_encode._detect
|
||||
|
||||
def _no_encoder():
|
||||
return None
|
||||
|
||||
monitor = getattr(image_monitor, "_monitor_instance", None)
|
||||
failures: dict[str, int] = {}
|
||||
|
||||
@@ -153,7 +147,6 @@ def register_system_commands(app, routes, api, root: Path):
|
||||
api._health_report = None
|
||||
for (owner, name), value in host_values.items():
|
||||
setattr(owner, name, value)
|
||||
thumbnail_encode._detect = real_detect
|
||||
thumbnail_encode.reset_encoders()
|
||||
image_monitor._monitor_instance = monitor
|
||||
api.__dict__.pop("_library_resolver", None)
|
||||
@@ -305,12 +298,7 @@ def register_system_commands(app, routes, api, root: Path):
|
||||
available = data.get("available")
|
||||
if not isinstance(available, bool):
|
||||
raise web.HTTPBadRequest()
|
||||
# Test-only: the fixture, not the host, decides (#371). Without an encoder the
|
||||
# detector itself stands in, so a re-probe (a "none found" is probed again on
|
||||
# every settings read) cannot find the host's real encoder; with one the cached
|
||||
# answer stands, and a failed write still probes the real detector as before.
|
||||
# Reset restores the real detector.
|
||||
thumbnail_encode._detect = real_detect if available else _no_encoder
|
||||
# Test-only: stand in the cached detection (no probe).
|
||||
with thumbnail_encode._detect_lock:
|
||||
thumbnail_encode._detected = "cjxl" if available else None
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@@ -51,7 +51,6 @@ _TABLES = (
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_categories",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
@@ -248,22 +247,6 @@ def register_tagging_commands(
|
||||
model.unavailable = set(data["unavailable"])
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/tagging/loaded")
|
||||
async def loaded(request: web.Request) -> web.Response:
|
||||
"""Mark one fake model loaded (``{"model": "<key>"}``), as a batch that
|
||||
kept it in memory leaves it; ``/autotag/models`` then reports
|
||||
``model_loaded``. Reset unloads it."""
|
||||
data = await request.json() if request.can_read_body else None
|
||||
if (
|
||||
request.query
|
||||
or not isinstance(data, dict)
|
||||
or set(data) != {"model"}
|
||||
or data["model"] not in MODELS
|
||||
):
|
||||
raise web.HTTPBadRequest()
|
||||
model.loaded = data["model"]
|
||||
return web.json_response({"success": True})
|
||||
|
||||
@routes.post("/__test__/tagging/download")
|
||||
async def download_mode(request: web.Request) -> web.Response:
|
||||
"""Switch the model's download to a fake (never a real download).
|
||||
|
||||
@@ -119,7 +119,6 @@ def seed_workspace_visual(api: PromptManagerAPI, root: Path) -> None:
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_categories",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -86,7 +86,6 @@ _TABLES = (
|
||||
"media_image_tags",
|
||||
"media_annotation_bytes",
|
||||
"media_annotation_files",
|
||||
"image_categories",
|
||||
"image_prompt_pairs",
|
||||
"prompt_pair_tags",
|
||||
"prompt_pairs",
|
||||
|
||||
@@ -11,13 +11,13 @@ import { installHostHooks } from './host-fixture.mjs';
|
||||
|
||||
// The worker's owned host must be up before the shared host hooks open the page.
|
||||
test.beforeEach(async ({ execution }) => { expect(execution.databasePath).toBeTruthy(); });
|
||||
installHostHooks(test, ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerPair', 'PMHistoryInputs', 'PMCycledText']);
|
||||
installHostHooks(test, ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerPair', 'PMHistoryInputs']);
|
||||
|
||||
// `primitive`: a core string node the queue-time resolver follows. `legacy`: the frontend's virtual Primitive node,
|
||||
// whose value is queued as literal text. `preview`: PrimitiveString -> PreviewAny, the shape of the maintainer's
|
||||
// TextGenerate -> PreviewAny chain, whose text is only known when the node runs.
|
||||
function build(page, { kind, positive, negative, prefix, saveToLibrary }) {
|
||||
return page.evaluate(async ({ kind, positive, negative, prefix, saveToLibrary }) => {
|
||||
function build(page, { kind, positive, negative, prefix }) {
|
||||
return page.evaluate(async ({ kind, positive, negative, prefix }) => {
|
||||
const { app } = await import('/scripts/app.js');
|
||||
app.graph.clear();
|
||||
const add = (type, x, y) => { const node = LiteGraph.createNode(type); app.graph.add(node); node.pos = [x, y]; return node; };
|
||||
@@ -28,7 +28,6 @@ function build(page, { kind, positive, negative, prefix, saveToLibrary }) {
|
||||
const set = (name, value) => { const widget = prompt.widgets.find(item => item.name === name); widget.value = value; if (widget.inputEl) widget.inputEl.value = value; };
|
||||
set('text', positive.typed);
|
||||
if (negative) set('negative_text', negative.typed);
|
||||
if (saveToLibrary !== undefined) prompt.widgets.find(item => item.name === 'save_to_library').value = saveToLibrary;
|
||||
const drive = (name, side, y) => {
|
||||
if (!side.via) return;
|
||||
if (side.via === 'legacy') {
|
||||
@@ -38,7 +37,6 @@ function build(page, { kind, positive, negative, prefix, saveToLibrary }) {
|
||||
primitive.widgets[0].value = side.text;
|
||||
return;
|
||||
}
|
||||
if (side.via === 'cycled') { connect(add('PMCycledText', 0, y), 0, prompt, name); return; }
|
||||
const text = add('PrimitiveString', 0, y);
|
||||
text.widgets.find(widget => widget.name === 'value').value = side.text;
|
||||
let source = text;
|
||||
@@ -74,12 +72,12 @@ function build(page, { kind, positive, negative, prefix, saveToLibrary }) {
|
||||
await prompt._pmTextSerializerReady;
|
||||
const { output, workflow } = await app.graphToPrompt();
|
||||
return { output, workflow, promptNode: String(prompt.id) };
|
||||
}, { kind, positive, negative, prefix, saveToLibrary });
|
||||
}, { kind, positive, negative, prefix });
|
||||
}
|
||||
|
||||
async function run(execution, page, testCase, built) {
|
||||
async function run(execution, page, testCase) {
|
||||
const prefix = `textlink/${randomUUID()}`;
|
||||
const { output, workflow, promptNode } = built ?? await build(page, { ...testCase, prefix });
|
||||
const { output, workflow, promptNode } = await build(page, { ...testCase, prefix });
|
||||
const id = randomUUID();
|
||||
const response = await execution.request.post('/prompt', { data: { prompt: output, prompt_id: id, extra_data: { extra_pnginfo: { workflow } } } });
|
||||
expect(response.ok(), await response.text()).toBe(true);
|
||||
@@ -195,205 +193,3 @@ for (const [label, linked, negativeVia] of [['positive', ['text'], 'primitive'],
|
||||
await expectImagesLinked(execution, evidence, sides.text.via === 'preview');
|
||||
});
|
||||
}
|
||||
|
||||
// #90: with Save to library off the run still produces its image, but PromptManager saves, counts and links nothing,
|
||||
// and the run's history records no prompt text for it.
|
||||
for (const kind of ['PromptManager', 'PromptManagerText', 'PromptManagerFocused', 'PromptManagerSplit', 'PromptManagerPair']) {
|
||||
test(`${kind} with Save to library off saves, counts and links nothing (#90)`, async ({ execution, page }) => {
|
||||
test.setTimeout(90000);
|
||||
const text = unique(`Not saved ${kind}`);
|
||||
const negative = kind === 'PromptManagerPair' ? { typed: unique('Not saved negative') } : undefined;
|
||||
const prefix = `notsaved/${randomUUID()}`;
|
||||
const { output, workflow, promptNode } = await build(page, { kind, positive: { typed: text }, negative, prefix, saveToLibrary: false });
|
||||
expect(output[promptNode].inputs.save_to_library).toBe(false);
|
||||
const id = randomUUID();
|
||||
const response = await execution.request.post('/prompt', { data: { prompt: output, prompt_id: id, extra_data: { extra_pnginfo: { workflow } } } });
|
||||
expect(response.ok(), await response.text()).toBe(true);
|
||||
let item;
|
||||
await expect.poll(async () => {
|
||||
item = (await (await execution.request.get(`/history/${id}`)).json())[id];
|
||||
return item?.status?.status_str ?? null;
|
||||
}, { timeout: 20000 }).toBe('success');
|
||||
let result;
|
||||
await expect.poll(async () => {
|
||||
result = await readExecutionRuns(execution.databasePath, id);
|
||||
const run = result.rows[0];
|
||||
return Boolean(run && result.captures[run.run_id]);
|
||||
}, { timeout: 10000, message: 'the run reaches SQLite' }).toBe(true);
|
||||
const run = result.rows[0];
|
||||
for (const context of result.captures[run.run_id].contexts || []) {
|
||||
expect(context.positive_prompt_id ?? null).toBeNull();
|
||||
expect(context.pair_id ?? null).toBeNull();
|
||||
}
|
||||
expect((result.usage[run.run_id] || []).every(entry => entry.run_count === 0)).toBe(true);
|
||||
const files = Object.values(item.outputs).flatMap(value => value.images || []).map(image => image.filename);
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
await scan(execution);
|
||||
expect(Object.values(await linkedPrompts(execution, files))).toEqual(files.map(() => null));
|
||||
for (const query of [text, negative?.typed].filter(Boolean)) {
|
||||
const found = await (await execution.request.get(`/prompt_manager/search?text=${encodeURIComponent(query)}`)).json();
|
||||
expect(found.results ?? [], query).toEqual([]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// #347: a cached PromptManager node counts every job, as an executing one does. Jobs after the first change only
|
||||
// the sampler seed, so ComfyUI serves the PromptManager node (and its PreviewAny source) from its output cache.
|
||||
const COUNTS = `import json, sqlite3, sys, pathlib
|
||||
db = sqlite3.connect(pathlib.Path(sys.argv[1]).resolve().as_uri() + "?mode=ro", uri=True)
|
||||
one = lambda sql, key: (db.execute(sql, (int(key),)).fetchone() or [None])[0] if key != "-" else None
|
||||
print(json.dumps({"positive": one("SELECT run_count FROM prompts WHERE id=?", sys.argv[2]),
|
||||
"negative": one("SELECT run_count FROM prompts WHERE id=?", sys.argv[3]),
|
||||
"pair": one("SELECT run_count FROM prompt_pairs WHERE id=?", sys.argv[4])}))`;
|
||||
async function runCounts(execution, context) {
|
||||
const key = value => value == null ? '-' : String(value);
|
||||
const { stdout } = await promisify(execFile)(process.env.PM_TEST_PYTHON || 'python',
|
||||
['-c', COUNTS, execution.databasePath, key(context.positive_prompt_id), key(context.negative_prompt_id), key(context.pair_id)], { timeout: 5000 });
|
||||
return JSON.parse(stdout);
|
||||
}
|
||||
|
||||
async function setCache(execution, mode) {
|
||||
expect((await execution.request.post('/pm_execution_fixture/cache', { data: { mode } })).ok()).toBe(true);
|
||||
}
|
||||
|
||||
// Queue an already built workflow again with another sampler seed; returns the job's history item and cached nodes.
|
||||
async function queueAgain(execution, first, { seed, edit, targets, status = 'success' } = {}) {
|
||||
const output = structuredClone(first.output);
|
||||
const sampler = Object.values(output).find(node => node.class_type === 'KSampler');
|
||||
sampler.inputs.seed = seed;
|
||||
edit?.(output);
|
||||
const id = randomUUID();
|
||||
const data = { prompt: output, prompt_id: id, extra_data: { extra_pnginfo: { workflow: first.workflow } } };
|
||||
if (targets) data.partial_execution_targets = targets;
|
||||
const response = await execution.request.post('/prompt', { data });
|
||||
expect(response.ok(), await response.text()).toBe(true);
|
||||
let item;
|
||||
await expect.poll(async () => {
|
||||
item = (await (await execution.request.get(`/history/${id}`)).json())[id];
|
||||
return item?.status?.status_str ?? null;
|
||||
}, { timeout: 20000 }).toBe(status);
|
||||
const cached = item.status.messages.find(([event]) => event === 'execution_cached')?.[1]?.nodes ?? [];
|
||||
return { id, item, cached };
|
||||
}
|
||||
|
||||
// The recorded context of one job, once its capture reached SQLite.
|
||||
async function capturedContext(execution, id) {
|
||||
let result;
|
||||
await expect.poll(async () => {
|
||||
result = await readExecutionRuns(execution.databasePath, id);
|
||||
const row = result.rows[0];
|
||||
return row ? result.captures[row.run_id].state : null;
|
||||
}, { timeout: 10000 }).toMatch(/^(complete|partial)$/);
|
||||
return result.captures[result.rows[0].run_id].contexts[0];
|
||||
}
|
||||
|
||||
async function firstRun(execution, page, testCase) {
|
||||
const prefix = `textlink/${randomUUID()}`;
|
||||
const built = await build(page, { ...testCase, prefix });
|
||||
const evidence = await run(execution, page, testCase, built);
|
||||
return { ...evidence, workflow: built.workflow };
|
||||
}
|
||||
|
||||
for (const kind of ['PromptManager', 'PromptManagerFocused', 'PromptManagerSplit', 'PromptManagerText', 'PromptManagerPair']) {
|
||||
test(`${kind} counts each cached re-run of text linked from PreviewAny (#347)`, async ({ execution, page }) => {
|
||||
test.setTimeout(90000);
|
||||
await setCache(execution, 'RAM_PRESSURE');
|
||||
const pair = kind === 'PromptManagerPair';
|
||||
const testCase = { kind, positive: { typed: 'Typed under the link', text: unique(`Cached ${kind}`), via: 'preview' },
|
||||
negative: pair ? { typed: unique('Typed negative') } : undefined };
|
||||
const first = await firstRun(execution, page, testCase);
|
||||
for (const seed of [2, 3]) {
|
||||
const again = await queueAgain(execution, first, { seed });
|
||||
expect(again.cached, 'ComfyUI served the PromptManager node from its cache').toContain(first.promptNode);
|
||||
}
|
||||
const counts = await runCounts(execution, first.context);
|
||||
expect(counts.positive).toBe(3);
|
||||
if (pair) expect(counts).toMatchObject({ negative: 0, pair: 3 });
|
||||
});
|
||||
}
|
||||
|
||||
test.describe('cached runs in one renderer (#347)', () => {
|
||||
// Cache behavior does not depend on the renderer: one project runs these.
|
||||
test.beforeEach(({}, testInfo) => { test.skip(testInfo.project.name !== 'classic-start', 'covered in classic-start'); });
|
||||
const linked = () => ({ kind: 'PromptManager', positive: { typed: 'Typed under the link', text: unique('Cached'), via: 'preview' } });
|
||||
|
||||
for (const mode of ['CLASSIC', 'LRU', 'RAM_PRESSURE', 'NONE']) {
|
||||
test(`counts 3 jobs under the ${mode} cache`, async ({ execution, page }) => {
|
||||
test.setTimeout(90000);
|
||||
await setCache(execution, mode);
|
||||
try {
|
||||
const first = await firstRun(execution, page, linked());
|
||||
for (const seed of [2, 3]) {
|
||||
const again = await queueAgain(execution, first, { seed });
|
||||
if (mode === 'NONE') expect(again.cached).not.toContain(first.promptNode);
|
||||
else expect(again.cached).toContain(first.promptNode);
|
||||
}
|
||||
expect((await runCounts(execution, first.context)).positive).toBe(3);
|
||||
} finally { await setCache(execution, 'RAM_PRESSURE'); }
|
||||
});
|
||||
}
|
||||
|
||||
test('alternating upstream values A, B, A under LRU each count their own prompt', async ({ execution, page }) => {
|
||||
test.setTimeout(90000);
|
||||
await setCache(execution, 'LRU');
|
||||
try {
|
||||
const [a, b] = [unique('Cycled A'), unique('Cycled B')];
|
||||
const cycle = async value => expect((await execution.request.post('/pm_execution_fixture/cycled', { data: { value } })).ok()).toBe(true);
|
||||
await cycle(a);
|
||||
const first = await firstRun(execution, page, { kind: 'PromptManager', positive: { typed: 'Typed under the link', text: a, via: 'cycled' } });
|
||||
expect(first.context.positive.raw).toBe(a);
|
||||
await cycle(b);
|
||||
const second = await queueAgain(execution, first, { seed: 2 });
|
||||
expect(second.cached).not.toContain(first.promptNode);
|
||||
await cycle(a);
|
||||
const third = await queueAgain(execution, first, { seed: 3 });
|
||||
expect(third.cached, 'LRU kept A').toContain(first.promptNode);
|
||||
expect((await runCounts(execution, first.context)).positive).toBe(2);
|
||||
const bContext = await capturedContext(execution, second.id);
|
||||
expect(bContext.positive.raw).toBe(b);
|
||||
expect((await runCounts(execution, bContext)).positive).toBe(1);
|
||||
} finally { await setCache(execution, 'RAM_PRESSURE'); }
|
||||
});
|
||||
|
||||
test('typed text served from the cache still counts once per job', async ({ execution, page }) => {
|
||||
test.setTimeout(90000);
|
||||
await setCache(execution, 'RAM_PRESSURE');
|
||||
const first = await firstRun(execution, page, { kind: 'PromptManager', positive: { typed: unique('Typed cached') } });
|
||||
for (const seed of [2, 3]) expect((await queueAgain(execution, first, { seed })).cached).toContain(first.promptNode);
|
||||
expect((await runCounts(execution, first.context)).positive).toBe(3);
|
||||
});
|
||||
|
||||
test('a job failing downstream of a cached node counts it once; a job that does not need it, never', async ({ execution, page }) => {
|
||||
test.setTimeout(90000);
|
||||
await setCache(execution, 'RAM_PRESSURE');
|
||||
const first = await firstRun(execution, page, linked());
|
||||
const failing = await queueAgain(execution, first, { seed: 2, status: 'error', edit: output => {
|
||||
output.pm_failure = { class_type: 'PMHistoryOutput', inputs: { value: [Object.keys(output).find(id => output[id].class_type === 'KSampler'), 0], mode: 'failure' } };
|
||||
} });
|
||||
expect(failing.cached).toContain(first.promptNode);
|
||||
expect((await runCounts(execution, first.context)).positive).toBe(2);
|
||||
// "Run selected output": only an unrelated output node; ComfyUI still reports the cached PromptManager node
|
||||
const unrelated = await queueAgain(execution, first, { seed: 2, targets: ['pm_unrelated'], edit: output => {
|
||||
output.pm_unrelated = { class_type: 'PMHistoryCached', inputs: { value: unique('unrelated') } };
|
||||
} });
|
||||
expect(unrelated.cached).toContain(first.promptNode);
|
||||
expect((await runCounts(execution, first.context)).positive).toBe(2);
|
||||
});
|
||||
|
||||
test('a cached node rewired to the negative input does not count the positive prompt', async ({ execution, page }) => {
|
||||
test.setTimeout(90000);
|
||||
await setCache(execution, 'RAM_PRESSURE');
|
||||
const first = await firstRun(execution, page, linked());
|
||||
const rewired = await queueAgain(execution, first, { seed: 2, edit: output => {
|
||||
const sampler = Object.values(output).find(node => node.class_type === 'KSampler');
|
||||
[sampler.inputs.positive, sampler.inputs.negative] = [sampler.inputs.negative, sampler.inputs.positive];
|
||||
} });
|
||||
expect(rewired.cached).toContain(first.promptNode);
|
||||
expect((await runCounts(execution, first.context)).positive).toBe(1);
|
||||
// The history capture of a cached job is unchanged: effective text from the cache, no stored-prompt hint
|
||||
const cachedContext = await capturedContext(execution, (await queueAgain(execution, first, { seed: 3 })).id);
|
||||
expect(cachedContext.positive.effective).toBe(first.context.positive.effective);
|
||||
expect(cachedContext.positive_prompt_id).toBeNull();
|
||||
expect((await runCounts(execution, first.context)).positive).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -201,39 +201,6 @@ NODE_CLASS_MAPPINGS.update(
|
||||
{"PMHistoryImages": PMHistoryImages, "PMHistoryImageOutput": PMHistoryImageOutput}
|
||||
)
|
||||
|
||||
# #347: text whose value (and cache signature) follows a value the test sets, so
|
||||
# one queued graph can serve A, then B, then A again (each cached under LRU).
|
||||
_CYCLED = {"value": "cycled"}
|
||||
|
||||
|
||||
class PMCycledText:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls) -> dict:
|
||||
return {"required": {}}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "execute"
|
||||
CATEGORY = "testing/execution-history"
|
||||
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, **kwargs: object) -> str:
|
||||
return _CYCLED["value"]
|
||||
|
||||
def execute(self) -> tuple:
|
||||
return (_CYCLED["value"],)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS["PMCycledText"] = PMCycledText
|
||||
|
||||
|
||||
@PromptServer.instance.routes.post("/pm_execution_fixture/cycled")
|
||||
async def cycled_text(request: web.Request) -> web.Response:
|
||||
value = (await request.json())["value"]
|
||||
if type(value) is not str:
|
||||
raise web.HTTPBadRequest()
|
||||
_CYCLED["value"] = value
|
||||
return web.json_response({"value": value})
|
||||
|
||||
|
||||
@PromptServer.instance.routes.post("/pm_execution_fixture/metadata")
|
||||
async def metadata(request: web.Request) -> web.Response:
|
||||
|
||||
@@ -140,26 +140,3 @@ test('an output saved without metadata links once its recorded run names a store
|
||||
expect((await request.post('/pm_execution_fixture/metadata', { data: { disabled: false } })).ok()).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
// #332 on the real host: the migration ran at startup and the image category routes
|
||||
// set, list and remove an own category on a generated output.
|
||||
test('an output gets its own image category and loses it again', async ({ execution }, testInfo) => {
|
||||
const request = execution.request;
|
||||
const filename = await saved(request, graph({ text: `category host ${randomUUID()}`, prefix: 'pm332/category' }));
|
||||
let image;
|
||||
await expect.poll(async () => {
|
||||
const listing = await (await request.get('/prompt_manager/images/output?mode=workspace&q=pm332&rescan=1')).json();
|
||||
image = (listing.images || []).find(entry => entry.filename === filename || entry.path?.endsWith(`/${filename}`));
|
||||
return Boolean(image);
|
||||
}, { timeout: 20000, message: `${filename} is listed by Images` }).toBe(true);
|
||||
const items = [{ root_index: image.root_index, root_scope: image.root_scope, path: image.path }];
|
||||
const value = `PM332 host ${testInfo.project.name}`;
|
||||
const set = await request.put('/prompt_manager/images/category', { data: { items, category: value } });
|
||||
expect(set.status()).toBe(200);
|
||||
expect(await set.json()).toEqual({ success: true, category: value, updated: 1, unchanged: 0, not_found: 0 });
|
||||
expect((await (await request.get('/prompt_manager/images/categories')).json()).categories).toContain(value);
|
||||
const cleared = await request.delete('/prompt_manager/images/category', { data: { items } });
|
||||
expect(cleared.status()).toBe(200);
|
||||
expect(await cleared.json()).toEqual({ success: true, cleared: 1 });
|
||||
expect((await (await request.get('/prompt_manager/images/categories')).json()).categories).not.toContain(value);
|
||||
});
|
||||
|
||||
@@ -370,7 +370,6 @@ test('workflows saved before #343 load unchanged and their linked text keeps wor
|
||||
const nodes = Object.fromEntries(app.graph._nodes.map(node => [node.id, node]));
|
||||
const reloaded = app.graph.serialize();
|
||||
return {
|
||||
saves: Object.fromEntries(Object.entries(output).filter(([, node]) => node.class_type.startsWith('PromptManager')).map(([id, node]) => [id, node.inputs.save_to_library])),
|
||||
values: Object.fromEntries(reloaded.nodes.map(node => [node.id, node.widgets_values])),
|
||||
links: Object.fromEntries(Object.entries(output).map(([id, node]) => [id, { text: node.inputs.text, negative_text: node.inputs.negative_text }])),
|
||||
widgetBound: Object.fromEntries(Object.values(nodes).filter(node => node.type.startsWith('PromptManager')).map(node => [node.id, node.inputs.filter(input => ['text', 'negative_text'].includes(input.name) && input.widget).map(input => input.name)])),
|
||||
@@ -378,16 +377,8 @@ test('workflows saved before #343 load unchanged and their linked text keeps wor
|
||||
}, workflow);
|
||||
for (const node of workflow.nodes) {
|
||||
// Every saved widget value comes back as it was saved, except the typed text main lost under a link (null).
|
||||
// #90: the last saved value is the editor's own slot (''), where the Save to library switch is now: it loads on.
|
||||
if (node.widgets_values?.[0] === null) continue;
|
||||
if (!node.type.startsWith('PromptManager')) { expect(result.values[node.id], `${node.type} ${node.id}`).toEqual(node.widgets_values); continue; }
|
||||
const saved = node.widgets_values.length - 1;
|
||||
expect(result.values[node.id].slice(0, saved), `${node.type} ${node.id}`).toEqual(node.widgets_values.slice(0, saved));
|
||||
expect(result.values[node.id][saved], `${node.type} ${node.id} save_to_library`).toBe(true);
|
||||
if (node.widgets_values?.[0] !== null) expect(result.values[node.id], `${node.type} ${node.id}`).toEqual(node.widgets_values);
|
||||
}
|
||||
// Every PromptManager node of the old workflow queues with Save to library on.
|
||||
for (const save of Object.values(result.saves)) expect(save).toBe(true);
|
||||
expect(Object.keys(result.saves).length).toBeGreaterThan(0);
|
||||
expect(result.links).toMatchObject({
|
||||
1: { text: 'Saved PromptManager text' }, 2: { text: 'Saved PromptManagerText text' }, 3: { text: 'Saved PromptManagerFocused text' },
|
||||
4: { text: 'Saved PromptManagerSplit text' }, 5: { text: 'Saved PromptManagerPair text', negative_text: 'Saved pair negative' },
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
"""Hang guards for waits on disk-bound work (tests only).
|
||||
|
||||
A test that waits for a job or handoff whose work is file and SQLite I/O
|
||||
(a safety backup with fsync, a restore, hashing and unlinking files) waits
|
||||
on that state; the ceiling below only ends a real hang. Hosted Windows
|
||||
runners stall disk I/O for tens of seconds at a time: a restore took over
|
||||
10 s (merge queue, #382), and on PR #405 every test setup in a 50 s window
|
||||
took up to 9 s against 0.1-0.6 s around it while the test bodies stayed
|
||||
fast, so a one-file duplicate cleanup missed a 10 s bound (#409).
|
||||
|
||||
The ceiling is per wait and stays under the suite's 120 s per-test hang
|
||||
watchdog. A test with several such waits in a row can reach the watchdog
|
||||
first; either way a real hang fails, the watchdog with every thread's stack.
|
||||
"""
|
||||
|
||||
DISK_BOUND_SECONDS = 90
|
||||
Vendored
-16
@@ -113,19 +113,3 @@ by their own suites. The expectations start as the observed baseline. A change
|
||||
that alters an outcome edits the vector in the same commit.
|
||||
`tests/js/legacy-metadata-characterization.test.js` pins what
|
||||
the legacy viewers show on the same files.
|
||||
|
||||
# Legacy contract snapshot
|
||||
|
||||
`legacy_contract_snapshot.json` records what the legacy admin and gallery pages
|
||||
talk to while System S13 (#330) moves their tools to native System: every
|
||||
registered route as `"<METHOD> <path>"` (`routes`) and the key paths of
|
||||
`GET /prompt_manager/settings` to depth 2 (`settings_keys`; list items give the
|
||||
union of their keys as `name[].key`). `tests/test_legacy_contract_snapshot.py`
|
||||
compares a temp-database API against it. Record it again only for an intended
|
||||
change, and say why in the PR body:
|
||||
|
||||
```sh
|
||||
PM_RECORD_LEGACY_SNAPSHOT=1 python3 -m unittest tests.test_legacy_contract_snapshot
|
||||
```
|
||||
|
||||
Cutover (#334) removes the test and this fixture.
|
||||
|
||||
-203
@@ -1,203 +0,0 @@
|
||||
{
|
||||
"routes": [
|
||||
"DELETE /prompt_manager/delete/{prompt_id}",
|
||||
"DELETE /prompt_manager/image-annotations/capture/{token}",
|
||||
"DELETE /prompt_manager/images/category",
|
||||
"DELETE /prompt_manager/images/favorite",
|
||||
"DELETE /prompt_manager/images/{image_id}",
|
||||
"DELETE /prompt_manager/pairs/{pair_id}",
|
||||
"DELETE /prompt_manager/prompts/{prompt_id}/tags",
|
||||
"DELETE /prompt_manager/restore/preview/{token}",
|
||||
"DELETE /prompt_manager/saved-views/{view_id}",
|
||||
"DELETE /prompt_manager/tags/{tag_name}",
|
||||
"GET /prompt_manager/admin",
|
||||
"GET /prompt_manager/autotag/models",
|
||||
"GET /prompt_manager/backup",
|
||||
"GET /prompt_manager/backup/status",
|
||||
"GET /prompt_manager/categories",
|
||||
"GET /prompt_manager/diagnostics",
|
||||
"GET /prompt_manager/export",
|
||||
"GET /prompt_manager/gallery",
|
||||
"GET /prompt_manager/gallery.html",
|
||||
"GET /prompt_manager/gallery/subfolders",
|
||||
"GET /prompt_manager/history/runs/{run_id}",
|
||||
"GET /prompt_manager/history/runs/{run_id}/output/current/inspection",
|
||||
"GET /prompt_manager/history/runs/{run_id}/output/current/media",
|
||||
"GET /prompt_manager/history/runs/{run_id}/output/current/thumbnail",
|
||||
"GET /prompt_manager/history/runs/{run_id}/outputs",
|
||||
"GET /prompt_manager/image-annotations/capture/{token}",
|
||||
"GET /prompt_manager/images/all",
|
||||
"GET /prompt_manager/images/categories",
|
||||
"GET /prompt_manager/images/inspect/{image_path}",
|
||||
"GET /prompt_manager/images/output",
|
||||
"GET /prompt_manager/images/prompt/{image_path}",
|
||||
"GET /prompt_manager/images/recent",
|
||||
"GET /prompt_manager/images/search",
|
||||
"GET /prompt_manager/images/serve/{filepath}",
|
||||
"GET /prompt_manager/images/thumbnails/plan",
|
||||
"GET /prompt_manager/images/thumbnails/status",
|
||||
"GET /prompt_manager/images/{image_id}/file",
|
||||
"GET /prompt_manager/jobs",
|
||||
"GET /prompt_manager/jobs/{job_id}",
|
||||
"GET /prompt_manager/jobs/{job_id}/events",
|
||||
"GET /prompt_manager/jobs/{job_id}/report",
|
||||
"GET /prompt_manager/js/{filepath}",
|
||||
"GET /prompt_manager/lib/{filepath}",
|
||||
"GET /prompt_manager/library",
|
||||
"GET /prompt_manager/library/export",
|
||||
"GET /prompt_manager/library/options",
|
||||
"GET /prompt_manager/library/{positive_id}",
|
||||
"GET /prompt_manager/library/{positive_id}/history",
|
||||
"GET /prompt_manager/library/{positive_id}/history/pairs/{pair_id}/newest",
|
||||
"GET /prompt_manager/library/{positive_id}/images",
|
||||
"GET /prompt_manager/logs",
|
||||
"GET /prompt_manager/logs/config",
|
||||
"GET /prompt_manager/logs/download/{filename}",
|
||||
"GET /prompt_manager/logs/files",
|
||||
"GET /prompt_manager/logs/stats",
|
||||
"GET /prompt_manager/lora/detect",
|
||||
"GET /prompt_manager/lora/status",
|
||||
"GET /prompt_manager/lora/trigger-words",
|
||||
"GET /prompt_manager/maintenance/duplicates/{job_id}",
|
||||
"GET /prompt_manager/pairs",
|
||||
"GET /prompt_manager/pairs/export",
|
||||
"GET /prompt_manager/pairs/{pair_id}",
|
||||
"GET /prompt_manager/prompts/{prompt_id}/images",
|
||||
"GET /prompt_manager/recent",
|
||||
"GET /prompt_manager/review/events",
|
||||
"GET /prompt_manager/review/items",
|
||||
"GET /prompt_manager/review/items/{item_id}",
|
||||
"GET /prompt_manager/saved-views",
|
||||
"GET /prompt_manager/scan/status",
|
||||
"GET /prompt_manager/scan_duplicates",
|
||||
"GET /prompt_manager/scan_duplicates/status",
|
||||
"GET /prompt_manager/scan_output_dir",
|
||||
"GET /prompt_manager/search",
|
||||
"GET /prompt_manager/settings",
|
||||
"GET /prompt_manager/stats",
|
||||
"GET /prompt_manager/subfolders",
|
||||
"GET /prompt_manager/system/diagnostics",
|
||||
"GET /prompt_manager/system/health",
|
||||
"GET /prompt_manager/system/library",
|
||||
"GET /prompt_manager/system/overview",
|
||||
"GET /prompt_manager/system/statistics",
|
||||
"GET /prompt_manager/tags",
|
||||
"GET /prompt_manager/tags/filter",
|
||||
"GET /prompt_manager/tags/stats",
|
||||
"GET /prompt_manager/tags/suggest",
|
||||
"GET /prompt_manager/tags/{tag_name}/prompts",
|
||||
"GET /prompt_manager/test",
|
||||
"GET /prompt_manager/vocabulary/categories",
|
||||
"GET /prompt_manager/vocabulary/tags",
|
||||
"GET /prompt_manager/vocabulary/tags/{tag_id}/impact",
|
||||
"GET /prompt_manager/web",
|
||||
"GET /prompt_manager/workspace",
|
||||
"PATCH /prompt_manager/library/{positive_id}",
|
||||
"PATCH /prompt_manager/pairs/{pair_id}",
|
||||
"PATCH /prompt_manager/saved-views/{view_id}",
|
||||
"PATCH /prompt_manager/vocabulary/tags/{tag_id}",
|
||||
"POST /prompt_manager/autotag/apply",
|
||||
"POST /prompt_manager/autotag/batch/preview",
|
||||
"POST /prompt_manager/autotag/download/{model_type}",
|
||||
"POST /prompt_manager/autotag/single",
|
||||
"POST /prompt_manager/autotag/start",
|
||||
"POST /prompt_manager/autotag/unload",
|
||||
"POST /prompt_manager/bulk/category",
|
||||
"POST /prompt_manager/bulk/delete",
|
||||
"POST /prompt_manager/bulk/tags",
|
||||
"POST /prompt_manager/cleanup",
|
||||
"POST /prompt_manager/delete_duplicate_images",
|
||||
"POST /prompt_manager/diagnostics/test-link",
|
||||
"POST /prompt_manager/gallery/rescan",
|
||||
"POST /prompt_manager/image-annotations/observe",
|
||||
"POST /prompt_manager/image-annotations/suggest",
|
||||
"POST /prompt_manager/images/clear-thumbnails",
|
||||
"POST /prompt_manager/images/generate-thumbnails",
|
||||
"POST /prompt_manager/images/generate-thumbnails/progress",
|
||||
"POST /prompt_manager/images/link",
|
||||
"POST /prompt_manager/images/reveal",
|
||||
"POST /prompt_manager/jobs/autotag",
|
||||
"POST /prompt_manager/jobs/duplicates",
|
||||
"POST /prompt_manager/jobs/image-tags",
|
||||
"POST /prompt_manager/jobs/model-download",
|
||||
"POST /prompt_manager/jobs/scan",
|
||||
"POST /prompt_manager/jobs/thumbnails",
|
||||
"POST /prompt_manager/jobs/{job_id}/cancel",
|
||||
"POST /prompt_manager/library",
|
||||
"POST /prompt_manager/library/{positive_id}/variants",
|
||||
"POST /prompt_manager/logs/config",
|
||||
"POST /prompt_manager/logs/truncate",
|
||||
"POST /prompt_manager/lora/enable",
|
||||
"POST /prompt_manager/lora/import/apply",
|
||||
"POST /prompt_manager/lora/import/preview",
|
||||
"POST /prompt_manager/lora/refresh-cache",
|
||||
"POST /prompt_manager/lora/scan",
|
||||
"POST /prompt_manager/maintenance",
|
||||
"POST /prompt_manager/maintenance/apply",
|
||||
"POST /prompt_manager/maintenance/preview",
|
||||
"POST /prompt_manager/pairs",
|
||||
"POST /prompt_manager/prompts/tags",
|
||||
"POST /prompt_manager/prompts/{prompt_id}/tags",
|
||||
"POST /prompt_manager/restore",
|
||||
"POST /prompt_manager/restore/apply",
|
||||
"POST /prompt_manager/restore/preview",
|
||||
"POST /prompt_manager/review/items/{item_id}/decision",
|
||||
"POST /prompt_manager/save",
|
||||
"POST /prompt_manager/saved-views",
|
||||
"POST /prompt_manager/scan",
|
||||
"POST /prompt_manager/scan/cancel",
|
||||
"POST /prompt_manager/scan_duplicates/stream",
|
||||
"POST /prompt_manager/settings",
|
||||
"POST /prompt_manager/settings/validate-source",
|
||||
"POST /prompt_manager/system/health",
|
||||
"POST /prompt_manager/system/library/keep",
|
||||
"POST /prompt_manager/system/library/move",
|
||||
"POST /prompt_manager/tags/merge",
|
||||
"POST /prompt_manager/vocabulary/categories/apply",
|
||||
"POST /prompt_manager/vocabulary/categories/preview",
|
||||
"POST /prompt_manager/vocabulary/tags",
|
||||
"POST /prompt_manager/vocabulary/tags/{tag_id}/delete",
|
||||
"PUT /prompt_manager/image-annotations/tags",
|
||||
"PUT /prompt_manager/images/category",
|
||||
"PUT /prompt_manager/images/favorite",
|
||||
"PUT /prompt_manager/prompts/{prompt_id}",
|
||||
"PUT /prompt_manager/prompts/{prompt_id}/rating",
|
||||
"PUT /prompt_manager/tags/{tag_name}"
|
||||
],
|
||||
"settings_keys": [
|
||||
"monitor",
|
||||
"monitor.observer_alive",
|
||||
"monitor.present",
|
||||
"monitor.scheduled",
|
||||
"revision",
|
||||
"settings",
|
||||
"settings.autotag_character_threshold",
|
||||
"settings.autotag_existing",
|
||||
"settings.autotag_general_threshold",
|
||||
"settings.autotag_model_type",
|
||||
"settings.autotag_suggestion_limit",
|
||||
"settings.autotag_template",
|
||||
"settings.cpu_count",
|
||||
"settings.gallery_root_path",
|
||||
"settings.gallery_root_paths",
|
||||
"settings.infinite_scroll",
|
||||
"settings.monitored_directories",
|
||||
"settings.monitoring_enabled",
|
||||
"settings.result_timeout",
|
||||
"settings.safety_backup",
|
||||
"settings.thumbnail_edge",
|
||||
"settings.thumbnail_format",
|
||||
"settings.thumbnail_quality",
|
||||
"settings.webui_display_mode",
|
||||
"settings.worker_threads",
|
||||
"sources",
|
||||
"sources.auto_detected",
|
||||
"sources.configured",
|
||||
"sources.effective",
|
||||
"success",
|
||||
"thumbnail_encoders",
|
||||
"thumbnail_encoders.jpeg",
|
||||
"thumbnail_encoders.jxl",
|
||||
"thumbnail_encoders.webp"
|
||||
]
|
||||
}
|
||||
@@ -76,101 +76,35 @@ test("the workspace keys have their own defaults and validation", () => {
|
||||
assert.equal(normalizeGallerySettings({ defaultViewMode: "masonry" }).defaultViewMode, "grid");
|
||||
});
|
||||
|
||||
// #330 S13: the legacy gallery's Page options edit only the page-only keys (PAGE_OPTION_KEYS); System › Appearance owns the rest, so a Save
|
||||
// or a Reset from the legacy page keeps every other stored key.
|
||||
function legacyGallery(stored, controls, extra = {}) {
|
||||
test("a legacy gallery Save keeps the workspace keys and every other stored key", () => {
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const vm = require("node:vm");
|
||||
const settings = require("../../web/js/gallery-settings.js");
|
||||
const storage = new Map([["gallerySettings", JSON.stringify(stored)]]);
|
||||
const storage = new Map([["gallerySettings", JSON.stringify({ workspaceLayout: "masonry", workspaceDensity: "compact", someOtherTool: 1 })]]);
|
||||
const controls = {
|
||||
lazyLoadingToggle: { checked: true }, imageQualitySelect: { value: "high" }, defaultViewModeSelect: { value: "list" },
|
||||
defaultLimitSelect: { value: "200" }, gridColumnsSelect: { value: "6" }, showImageInfoToggle: { checked: false },
|
||||
autoLoadMetadataToggle: { checked: true }, cacheMetadataToggle: { checked: true }, showFilePathsToggle: { checked: false },
|
||||
sidebarCollapsedByDefaultToggle: { checked: true }, infiniteScrollToggle: { checked: false }, debugModeToggle: { checked: false },
|
||||
apiTimeoutInput: { value: "30" }, checkThumbnailsAtStartup: { checked: true }, videoAutoplayToggle: { checked: true },
|
||||
videoMuteToggle: { checked: false }, videoLoopToggle: { checked: true },
|
||||
};
|
||||
const context = vm.createContext({
|
||||
document: { addEventListener() {}, getElementById: id => controls[id] },
|
||||
window: {}, console: { debug() {}, warn() {}, error() {} }, GallerySettings: settings,
|
||||
localStorage: { getItem: key => storage.get(key) ?? null, setItem: (key, value) => storage.set(key, value),
|
||||
removeItem: key => storage.delete(key) },
|
||||
...extra,
|
||||
localStorage: { getItem: key => storage.get(key) ?? null, setItem: (key, value) => storage.set(key, value) },
|
||||
});
|
||||
const file = path.join(__dirname, "../../web/js/gallery.js");
|
||||
const Manager = vm.runInContext(`${fs.readFileSync(file, "utf8")}\nGalleryManager;`, context, { filename: file });
|
||||
const manager = Object.create(Manager.prototype);
|
||||
const calls = [];
|
||||
Object.assign(manager, { showNotification: (message, type) => calls.push([type, message]), hideSettings() {},
|
||||
applySettings: applied => calls.push(["apply", applied.lazyLoading]) });
|
||||
return { manager, storage, calls, saved: () => JSON.parse(storage.get("gallerySettings")) };
|
||||
}
|
||||
|
||||
const PAGE_OPTION_CONTROLS = () => ({
|
||||
checkThumbnailsAtStartup: { checked: false }, lazyLoadingToggle: { checked: false },
|
||||
defaultLimitSelect: { value: "200" },
|
||||
gridColumnsSelect: { value: "6" }, infiniteScrollToggle: { checked: true }, debugModeToggle: { checked: true },
|
||||
apiTimeoutInput: { value: "45" },
|
||||
});
|
||||
|
||||
test("a legacy gallery Save keeps the workspace keys and every other stored key", () => {
|
||||
const { manager, saved } = legacyGallery({ workspaceLayout: "masonry", workspaceDensity: "compact", someOtherTool: 1,
|
||||
imageQuality: "high", showImageInfo: false, videoMute: false, thumbnailPromptDismissed: "sig", defaultViewMode: "list" },
|
||||
PAGE_OPTION_CONTROLS());
|
||||
Object.assign(manager, { showNotification() {}, hideSettings() {}, applySettings() {} });
|
||||
manager.saveSettings();
|
||||
const result = saved();
|
||||
assert.equal(result.workspaceLayout, "masonry");
|
||||
assert.equal(result.workspaceDensity, "compact");
|
||||
assert.equal(result.someOtherTool, 1);
|
||||
assert.equal(result.imageQuality, "high");
|
||||
assert.equal(result.showImageInfo, false);
|
||||
assert.equal(result.videoMute, false);
|
||||
assert.equal(result.thumbnailPromptDismissed, "sig");
|
||||
// Page options no longer edit the default view (S13 G5), so a stored one stays.
|
||||
assert.equal(result.defaultViewMode, "list");
|
||||
// The page-only keys come from the dialog.
|
||||
assert.equal(result.lazyLoading, false);
|
||||
assert.equal(result.debugMode, true);
|
||||
assert.equal(result.checkThumbnailsAtStartup, false);
|
||||
assert.equal(result.defaultLimit, 200);
|
||||
assert.equal(result.gridColumns, 6);
|
||||
assert.equal(result.infiniteScroll, true);
|
||||
assert.equal(result.apiTimeout, 45);
|
||||
});
|
||||
|
||||
test("a legacy gallery Reset restores only the page options and keeps every other stored key", () => {
|
||||
const controls = PAGE_OPTION_CONTROLS();
|
||||
const { manager, saved, calls } = legacyGallery({ workspaceLayout: "masonry", imageQuality: "high", videoMute: false,
|
||||
thumbnailPromptDismissed: "sig", thumbnailsGenerated: true, lazyLoading: false, debugMode: true, defaultLimit: 200,
|
||||
gridColumns: 6, infiniteScroll: true, apiTimeout: 45, checkThumbnailsAtStartup: false, defaultViewMode: "list" }, controls,
|
||||
{ confirm: message => (calls.push(["confirm", message]), true) });
|
||||
manager.resetSettings();
|
||||
const result = saved();
|
||||
const { DEFAULT_SETTINGS } = require("../../web/js/gallery-settings.js");
|
||||
for (const key of ["checkThumbnailsAtStartup", "lazyLoading", "defaultLimit", "gridColumns",
|
||||
"infiniteScroll", "debugMode", "apiTimeout"]) {
|
||||
assert.equal(result[key], DEFAULT_SETTINGS[key], key);
|
||||
}
|
||||
// The default view is not a page option (S13 G5): Reset keeps a stored one.
|
||||
assert.equal(result.defaultViewMode, "list");
|
||||
assert.equal(result.workspaceLayout, "masonry");
|
||||
assert.equal(result.imageQuality, "high");
|
||||
assert.equal(result.videoMute, false);
|
||||
assert.equal(result.thumbnailPromptDismissed, "sig");
|
||||
assert.equal(result.thumbnailsGenerated, true);
|
||||
// The dialog shows the defaults again and the page applies them.
|
||||
assert.equal(controls.lazyLoadingToggle.checked, DEFAULT_SETTINGS.lazyLoading);
|
||||
assert.equal(controls.apiTimeoutInput.value, DEFAULT_SETTINGS.apiTimeout);
|
||||
assert.deepEqual(calls, [["confirm", "Reset these page options to their defaults?"], ["apply", DEFAULT_SETTINGS.lazyLoading],
|
||||
["success", "Page options reset to defaults"]]);
|
||||
});
|
||||
|
||||
test("Page options no longer hold the default view; System › Appearance's Default layout is the one control", () => {
|
||||
const { manager } = legacyGallery({}, {});
|
||||
assert.deepEqual([...manager.constructor.PAGE_OPTION_KEYS], ["checkThumbnailsAtStartup", "lazyLoading", "defaultLimit",
|
||||
"gridColumns", "infiniteScroll", "debugMode", "apiTimeout"]);
|
||||
});
|
||||
|
||||
test("a viewer mute or loop click remembers only that key and never reads the Page options dialog", () => {
|
||||
const { manager, saved, calls } = legacyGallery({ defaultLimit: 200, lazyLoading: false, videoMute: true }, {});
|
||||
manager.storeSetting("videoMute", false);
|
||||
const result = saved();
|
||||
assert.equal(result.videoMute, false);
|
||||
assert.equal(result.defaultLimit, 200);
|
||||
assert.equal(result.lazyLoading, false);
|
||||
assert.deepEqual(calls, []);
|
||||
const saved = JSON.parse(storage.get("gallerySettings"));
|
||||
assert.equal(saved.workspaceLayout, "masonry");
|
||||
assert.equal(saved.workspaceDensity, "compact");
|
||||
assert.equal(saved.someOtherTool, 1);
|
||||
assert.equal(saved.defaultViewMode, "list");
|
||||
assert.equal(saved.showImageInfo, false);
|
||||
assert.equal(saved.videoMute, false);
|
||||
});
|
||||
|
||||
@@ -112,12 +112,11 @@ test('saving gallery settings retains the remembered thumbnail dismissal', () =>
|
||||
});
|
||||
|
||||
// #249 System S5: the stream's work is now an Activity job that outlives the page's connection, so the gallery's
|
||||
// Cancel asks the server to stop the job. Since #330 the startup dialog is the gallery's only thumbnail stream (System
|
||||
// › Sources & scans generates the rest). The stream's first frame, `job {id, started}`, names the job; the gallery
|
||||
// Cancel asks the server to stop the job. The stream's first frame, `job {id, started}`, names the job; the gallery
|
||||
// cancels by that id, only a job this stream started, and a Cancel before the id waits for it (review I1).
|
||||
const JOB = 'b'.repeat(32);
|
||||
|
||||
function streamFixture({ postFails = false } = {}) {
|
||||
function streamFixture({ modal = false, postFails = false } = {}) {
|
||||
const posts = [];
|
||||
const warnings = [];
|
||||
const gets = [];
|
||||
@@ -147,20 +146,18 @@ function streamFixture({ postFails = false } = {}) {
|
||||
return { success: true };
|
||||
} } });
|
||||
const emit = (type, data) => (listeners[type] || []).forEach(handler => handler({ data: JSON.stringify(data) }));
|
||||
// The dialog's Cancel runs the handler and from then on reads as cancelled, as the startup dialog does.
|
||||
let cancelled = false;
|
||||
const ui = { addStatusMessage() {}, progressText: element(), progressPercent: element(), progressBar: element(),
|
||||
progressDetails: element(), progressETA: element(), isCancelled: () => cancelled,
|
||||
onCancel: handler => { ui.cancel = () => { cancelled = true; handler(); }; } };
|
||||
const result = manager.generateThumbnailsWithProgressInModal(ui);
|
||||
progressDetails: element(), progressETA: element(), isCancelled: () => false,
|
||||
onCancel: handler => { ui.cancel = handler; } };
|
||||
const result = modal ? manager.generateThumbnailsWithProgressInModal(ui)
|
||||
: manager.generateThumbnailsWithProgress({ quality: 'medium' });
|
||||
const outcome = result.then(value => ({ value }), error => ({ error: error.message }));
|
||||
const cancel = () => ui.cancel();
|
||||
const cancel = () => (modal ? ui.cancel() : elements.cancelThumbnailsBtn.click());
|
||||
const settle = () => new Promise(resolve => setImmediate(resolve));
|
||||
return { emit, source, posts, gets, warnings, outcome, cancel, settle };
|
||||
}
|
||||
|
||||
const cancelUrl = id => `/prompt_manager/jobs/${id}/cancel`;
|
||||
const CANCELLED = 'Thumbnail generation cancelled; previews already written are kept';
|
||||
|
||||
test('Cancel before the job id arrives cancels that job once it does', async () => {
|
||||
const f = streamFixture();
|
||||
@@ -168,8 +165,8 @@ test('Cancel before the job id arrives cancels that job once it does', async ()
|
||||
await f.settle();
|
||||
assert.deepEqual(f.posts, []);
|
||||
assert.equal(f.source.closed, 0); // kept open until the stream names its job
|
||||
assert.deepEqual(await f.outcome, { error: CANCELLED });
|
||||
// The job frame is the stream's first (the dialog closes the stream on any later frame once cancelled).
|
||||
assert.deepEqual(await f.outcome, { error: 'Thumbnail generation cancelled by user' });
|
||||
f.emit('status', { phase: 'scanning', message: 'Scanning…' });
|
||||
f.emit('job', { id: JOB, started: true });
|
||||
await f.settle();
|
||||
assert.equal(JSON.stringify(f.posts), JSON.stringify([[cancelUrl(JOB), {}]]));
|
||||
@@ -188,7 +185,7 @@ test('Cancel after the id cancels it at once, and a double Cancel posts once', a
|
||||
await f.settle();
|
||||
assert.equal(JSON.stringify(f.posts), JSON.stringify([[cancelUrl(JOB), {}]]));
|
||||
assert.equal(f.source.closed, 1);
|
||||
assert.deepEqual(await f.outcome, { error: CANCELLED });
|
||||
assert.deepEqual(await f.outcome, { error: 'Thumbnail generation cancelled by user' });
|
||||
assert.deepEqual(f.gets, []);
|
||||
});
|
||||
|
||||
@@ -219,11 +216,11 @@ test('a cancel post that fails is reported, not thrown', async () => {
|
||||
await f.settle();
|
||||
assert.equal(f.posts.length, 1);
|
||||
assert.equal(f.warnings.length, 1);
|
||||
assert.deepEqual(await f.outcome, { error: CANCELLED });
|
||||
assert.deepEqual(await f.outcome, { error: 'Thumbnail generation cancelled by user' });
|
||||
});
|
||||
|
||||
test('the startup dialog cancels its own job by id too', async () => {
|
||||
const f = streamFixture();
|
||||
const f = streamFixture({ modal: true });
|
||||
f.cancel();
|
||||
await f.settle();
|
||||
assert.deepEqual(f.posts, []);
|
||||
|
||||
@@ -65,12 +65,109 @@ function load(file, className, fetchUrl, extra = {}, dom = elements()) {
|
||||
|
||||
const BUSY = 'Scan for metadata already running';
|
||||
|
||||
test('gallery cache clear reports a refused server clear and never claims success', async () => {
|
||||
const calls = [];
|
||||
const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async url => {
|
||||
calls.push(url);
|
||||
return jsonResponse(409, { success: false, error: BUSY });
|
||||
});
|
||||
page.getSettings = () => ({ thumbnailsGenerated: true });
|
||||
dom.getElementById('thumbnailStatus').textContent = 'Generated';
|
||||
|
||||
await page.clearCache();
|
||||
|
||||
assert.deepEqual(calls, ['/prompt_manager/images/clear-thumbnails']);
|
||||
assert.deepEqual(notifications, [['error', `Thumbnails were not cleared: ${BUSY}`]]);
|
||||
assert.equal(dom.nodes.thumbnailStatus.textContent, 'Generated');
|
||||
assert.equal(storage.get('gallerySettings'), undefined); // thumbnailsGenerated is not reset
|
||||
});
|
||||
|
||||
test('gallery cache clear says what it cleared and keeps preferences and other data (#222)', async () => {
|
||||
const calls = [];
|
||||
const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async url => {
|
||||
calls.push(url);
|
||||
return jsonResponse(200, { success: true, cleared_count: 3 });
|
||||
});
|
||||
page.getSettings = () => ({ thumbnailsGenerated: true });
|
||||
|
||||
await page.clearCache();
|
||||
|
||||
// The server's thumbnails are cleared by their own explicit call; nothing Prompt Manager owns is cached in this
|
||||
// browser, so no browser entry is removed and the toast says exactly that.
|
||||
assert.deepEqual(calls, ['/prompt_manager/images/clear-thumbnails']);
|
||||
assert.deepEqual(notifications, [['success',
|
||||
"Thumbnails cleared; original images untouched. Nothing of Prompt Manager's is cached in this browser."]]);
|
||||
assert.equal(dom.nodes.thumbnailStatus.textContent, 'Not generated');
|
||||
assert.equal(JSON.parse(storage.get('gallerySettings')).thumbnailsGenerated, false);
|
||||
assert.equal(storage.get('userData'), 'keep');
|
||||
assert.equal(storage.get('pm_cached'), '1');
|
||||
assert.equal(storage.get('pm_workspace_theme'), 'light');
|
||||
assert.equal(storage.get('pm_update_notice_dismissed'), '3.3.0');
|
||||
});
|
||||
|
||||
test("gallery cache clear never deletes another tool's gallery or thumbnail caches (#222)", async () => {
|
||||
const names = new Set(['comfy-gallery', 'thumbnail-cache']);
|
||||
const caches = { keys: async () => [...names], delete: async name => names.delete(name) };
|
||||
const { page, notifications } = load('gallery.js', 'GalleryManager',
|
||||
async () => jsonResponse(200, { success: true }), { caches, window: { caches } });
|
||||
page.getSettings = () => ({});
|
||||
|
||||
await page.clearCache();
|
||||
|
||||
assert.deepEqual([...names], ['comfy-gallery', 'thumbnail-cache']);
|
||||
assert.equal(notifications.at(-1)[0], 'success');
|
||||
});
|
||||
|
||||
test('gallery cache clear reports a lost server clear instead of success', async () => {
|
||||
const { page, dom, notifications, storage } = load('gallery.js', 'GalleryManager', async () => {
|
||||
throw new TypeError('Failed to fetch');
|
||||
});
|
||||
page.getSettings = () => ({ thumbnailsGenerated: true });
|
||||
dom.getElementById('thumbnailStatus').textContent = 'Generated';
|
||||
|
||||
await page.clearCache();
|
||||
|
||||
assert.deepEqual(notifications, [['error', 'Thumbnails were not cleared: the server could not be reached.']]);
|
||||
assert.equal(dom.nodes.thumbnailStatus.textContent, 'Generated');
|
||||
assert.equal(storage.get('gallerySettings'), undefined);
|
||||
});
|
||||
|
||||
test('gallery cache clear says so when browser storage is denied', async () => {
|
||||
const denied = { get length() { throw new Error('SecurityError'); }, key() { throw new Error('SecurityError'); },
|
||||
getItem() { throw new Error('SecurityError'); }, setItem() { throw new Error('SecurityError'); },
|
||||
removeItem() { throw new Error('SecurityError'); } };
|
||||
const { page, notifications } = load('gallery.js', 'GalleryManager',
|
||||
async () => jsonResponse(200, { success: true }), { localStorage: denied });
|
||||
page.getSettings = () => ({});
|
||||
|
||||
await page.clearCache();
|
||||
|
||||
assert.deepEqual(notifications, [['warning',
|
||||
'Thumbnails cleared; original images untouched. This browser refused access to its storage, so nothing there was cleared.']]);
|
||||
});
|
||||
|
||||
test('admin LoRA import shows a refusal frame and closes its modal', async () => {
|
||||
const { page, dom, timers, notifications } = load('admin.js', 'PromptAdmin',
|
||||
async () => sseResponse([{ type: 'error', status: BUSY, progress: 0, message: BUSY }]));
|
||||
page.saveLoraSettings = async () => true;
|
||||
page.loadStatistics = () => assert.fail('a refused import must not reload statistics');
|
||||
|
||||
await page.runLoraImport();
|
||||
|
||||
assert.equal(dom.nodes.loraImportStatus.textContent, `Error: ${BUSY}`);
|
||||
assert.deepEqual(notifications.filter(([type]) => type === 'error'), [['error', `LoRA import failed: ${BUSY}`]]);
|
||||
assert.equal(timers.length, 1);
|
||||
timers[0]();
|
||||
assert.deepEqual(notifications.at(-1), ['hide', 'loraImportModal']);
|
||||
});
|
||||
|
||||
test('admin settings save shows why config.json refused it', async () => {
|
||||
const message = 'config.json cannot be read, so nothing was saved. Fix or remove the file, then save again.';
|
||||
const { page, notifications } = load('admin.js', 'PromptAdmin',
|
||||
async () => jsonResponse(409, { success: false, persisted: false, code: 'config-unreadable', error: message }));
|
||||
page.settings = {};
|
||||
page.setInfiniteScroll = () => assert.fail('nothing changes after a refusal');
|
||||
page._collectScanPaths = () => [];
|
||||
page.saveLoraSettings = async () => assert.fail('nothing else is saved after a refusal');
|
||||
|
||||
await page.saveSettings();
|
||||
|
||||
@@ -81,30 +178,27 @@ test('admin settings save keeps its plain message when the reply has no reason',
|
||||
const { page, notifications } = load('admin.js', 'PromptAdmin', async () => ({ ok: false, status: 500,
|
||||
json: async () => { throw new SyntaxError('not json'); } }));
|
||||
page.settings = {};
|
||||
page._collectScanPaths = () => [];
|
||||
|
||||
await page.saveSettings();
|
||||
|
||||
assert.deepEqual(notifications, [['error', 'Failed to save settings']]);
|
||||
});
|
||||
|
||||
// #330 S13-9: the admin page's reduced Page options save sends only the list's scroll default, so a stale legacy form
|
||||
// can no longer overwrite scan folders or other settings edited in System.
|
||||
test('admin Page options save posts only infinite_scroll', async () => {
|
||||
const sent = [];
|
||||
const { page, dom, notifications } = load('admin.js', 'PromptAdmin', async (url, init) => {
|
||||
sent.push([url, init.method, JSON.parse(init.body)]);
|
||||
return jsonResponse(200, { success: true, restart_required: true });
|
||||
});
|
||||
page.settings = { infiniteScroll: false };
|
||||
const scrolled = [];
|
||||
page.setInfiniteScroll = (on, options) => scrolled.push([on, { ...options }]);
|
||||
dom.getElementById('infiniteScrollDefault').checked = true;
|
||||
test('admin maintenance names the job that refused it', async () => {
|
||||
const { page, dom, notifications } = load('admin.js', 'PromptAdmin',
|
||||
async () => jsonResponse(409, { success: false, error: BUSY }), {
|
||||
escapeHtml: text => String(text),
|
||||
document: { readyState: 'loading', addEventListener() {}, getElementById: id => dom.getElementById(id),
|
||||
querySelectorAll: selector => (selector === '.maintenance-option:checked' ? [{ id: 'vacuum' }] : []) },
|
||||
});
|
||||
|
||||
await page.saveSettings();
|
||||
await page.runMaintenance();
|
||||
|
||||
assert.deepEqual(sent, [['/prompt_manager/settings', 'POST', { infinite_scroll: true }]]);
|
||||
assert.deepEqual(scrolled, [[true, { session: false }]]);
|
||||
assert.equal(page.settings.infiniteScroll, true);
|
||||
assert.deepEqual(notifications, [['success', 'Page options saved'], ['hide', 'settingsModal']]);
|
||||
const results = dom.nodes.maintenanceResults.innerHTML;
|
||||
assert.match(results, /Maintenance Failed/);
|
||||
assert.match(results, new RegExp(BUSY));
|
||||
assert.doesNotMatch(results, /HTTP 409/);
|
||||
assert.deepEqual(notifications, [['error', '❌ Maintenance failed']]);
|
||||
assert.equal(dom.nodes.runMaintenanceBtn.disabled, false);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
// #90: the node's save_to_library switch (web/js/save-to-library.js).
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const source = import('../../web/js/save-to-library.js');
|
||||
|
||||
function classicNode(saved) {
|
||||
const names = ['text', 'category', 'tags', 'search_text', 'prepend_text', 'append_text', 'save_to_library', 'prompt_manager_search_ui'];
|
||||
const widgets = names.map(name => ({name, value: name === 'save_to_library' ? true : ''}));
|
||||
const node = {widgets, widgets_values: saved ? [...saved] : undefined};
|
||||
// What a host configure does: each saved value goes to the widget at its index.
|
||||
(saved || []).forEach((value, index) => { if (widgets[index]) widgets[index].value = value; });
|
||||
return node;
|
||||
}
|
||||
|
||||
test('an older workflow stored the editor slot where the switch is now: it loads on', async () => {
|
||||
const {normalizeSaveWidget, savesToLibrary} = await source;
|
||||
const node = classicNode(['a cat', 'c', 't', '', '', '', '']);
|
||||
assert.equal(node.widgets[6].value, '', 'the host put the old DOM slot value into the switch');
|
||||
assert.equal(normalizeSaveWidget(node), true);
|
||||
assert.equal(node.widgets[6].value, true);
|
||||
assert.equal(node.widgets_values[6], true);
|
||||
assert.equal(savesToLibrary(node), true);
|
||||
assert.deepEqual(node.widgets_values.slice(0, 6), ['a cat', 'c', 't', '', '', ''], 'no earlier value moved');
|
||||
});
|
||||
|
||||
test('null, undefined and missing slots load on; saved booleans are kept', async () => {
|
||||
const {normalizeSaveWidget, savesToLibrary} = await source;
|
||||
for (const value of [null, undefined, 0, 'false']) {
|
||||
const node = classicNode(['a', '', '', '', '', '', value]);
|
||||
normalizeSaveWidget(node);
|
||||
assert.equal(savesToLibrary(node), true, String(value));
|
||||
}
|
||||
const short = classicNode(['a', '', '', '', '', '']);
|
||||
assert.equal(normalizeSaveWidget(short), false);
|
||||
assert.equal(savesToLibrary(short), true);
|
||||
const off = classicNode(['a', '', '', '', '', '', false]);
|
||||
assert.equal(normalizeSaveWidget(off), false);
|
||||
assert.equal(savesToLibrary(off), false);
|
||||
});
|
||||
|
||||
test('a node without the widget (an older backend) saves', async () => {
|
||||
const {normalizeSaveWidget, savesToLibrary} = await source;
|
||||
assert.equal(savesToLibrary({widgets: [{name: 'text', value: ''}]}), true);
|
||||
assert.equal(normalizeSaveWidget({widgets: []}), false);
|
||||
assert.equal(savesToLibrary(undefined), true);
|
||||
});
|
||||
|
||||
test('the switch moves after the editor fallbacks so every frontend has one order', async () => {
|
||||
const {placeSaveWidgetLast} = await source;
|
||||
const widgets = ['text', 'category', 'tags', 'search_text', 'save_to_library', 'prepend_text', 'append_text'].map(name => ({name}));
|
||||
const node = {widgets};
|
||||
assert.equal(placeSaveWidgetLast(node), true);
|
||||
assert.deepEqual(node.widgets.map(widget => widget.name), ['text', 'category', 'tags', 'search_text', 'prepend_text', 'append_text', 'save_to_library']);
|
||||
assert.equal(placeSaveWidgetLast(node), false, 'already last');
|
||||
assert.equal(placeSaveWidgetLast({widgets: [{name: 'text'}]}), false);
|
||||
});
|
||||
|
||||
test('setting the switch updates the widget, its saved value and notifies the host', async () => {
|
||||
const {setSaveToLibrary, savesToLibrary} = await source;
|
||||
const calls = [];
|
||||
const node = classicNode(['a', '', '', '', '', '', true]);
|
||||
node.widgets[6].callback = value => calls.push(value);
|
||||
setSaveToLibrary(node, false);
|
||||
assert.equal(savesToLibrary(node), false);
|
||||
assert.equal(node.widgets_values[6], false);
|
||||
assert.deepEqual(calls, [false]);
|
||||
setSaveToLibrary({widgets: []}, false);
|
||||
});
|
||||
@@ -185,26 +185,6 @@ test('legacy fill applies to local records', async () => {
|
||||
assert.notEqual(zeroed.prompts.positive.text, '');
|
||||
});
|
||||
|
||||
test('a branch named by recorded run history shows the Recorded run badge', async () => {
|
||||
// #386: the server labels sides whose save node recorded history named; Execution recorded stays stronger.
|
||||
const base = await localRecord(new File([fs.readFileSync(new URL('f09.png', PARITY))], 'f09.png'));
|
||||
const recorded = {...base, prompts:{positive:{...base.prompts.positive, source:'recorded-run'}, negative:{...base.prompts.negative, source:'recorded-run'}}};
|
||||
let view = renderFull(recorded);
|
||||
try {
|
||||
assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Recorded run']);
|
||||
assert.ok(view.texts(n => n.tagName === 'p').includes('Prompt evidence: recorded run'));
|
||||
} finally { view.done(); }
|
||||
view = renderFull({...recorded, prompts:{...recorded.prompts, negative:{...recorded.prompts.negative, source:'execution'}}});
|
||||
try {
|
||||
assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Execution recorded']);
|
||||
} finally { view.done(); }
|
||||
// A recorded branch whose negative left out a run-time part is still inferred from the workflow.
|
||||
view = renderFull({...recorded, prompts:{...recorded.prompts, negative:{...recorded.prompts.negative, source:'runtime-parts-omitted'}}});
|
||||
try {
|
||||
assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Workflow inferred']);
|
||||
} finally { view.done(); }
|
||||
});
|
||||
|
||||
// #249 PR C Task C3 (D88): one chip row, the file's current image tags first, then D21's linked tags not already shown.
|
||||
function tagged(imageTags, library = {status:'linked', tags:['Minimal', 'warm_light', 'editorial'], tags_scope:'positive'}) {
|
||||
return {...record(), imageTags, library};
|
||||
@@ -383,32 +363,3 @@ test('the D117 wrap rule targets only the settings summary button', async () =>
|
||||
assert.deepEqual(rules.map(line => line.trim()), ['.ca-tools #viewer-details .settings-summary { white-space: normal; /* D117 */ }']);
|
||||
assert.doesNotMatch(css, /#viewer-details > div > button/);
|
||||
});
|
||||
|
||||
test('category chip names its origin and shows an own category for any link status (#332)', async () => {
|
||||
const base = await localRecord(localPng('a.png'));
|
||||
const chipTexts = record => {
|
||||
const {f, done} = renderFull(record);
|
||||
try {
|
||||
return f.walk(f.container, n => /(^| )chip( |$)/.test(n.className ?? '')).map(n => n.textContent);
|
||||
} finally { done(); }
|
||||
};
|
||||
const cases = [
|
||||
[{status:'none', category:'Scans', category_scope:'image'}, 'Image category: Scans'],
|
||||
[{status:'ambiguous', category:'Scans', category_scope:'image'}, 'Image category: Scans'],
|
||||
[{status:'linked', category:'Portraits', category_scope:'image'}, 'Image category: Portraits'],
|
||||
[{status:'linked', category:'Landscapes', category_scope:'positive'}, 'Positive prompt category: Landscapes'],
|
||||
[{status:'linked', category:'Recipes', category_scope:'pair'}, 'Recipe category: Recipes'],
|
||||
];
|
||||
for (const [library, text] of cases) {
|
||||
const chips = chipTexts({...base, library:{...base.library, tags:[], tags_scope:null, ...library}});
|
||||
assert.ok(chips.includes(text), `${JSON.stringify(library)} -> ${chips}`);
|
||||
}
|
||||
for (const library of [
|
||||
{status:'none', category:'Inherited', category_scope:'positive'},
|
||||
{status:'linked', category:null, category_scope:null},
|
||||
{status:'none', category:null, category_scope:null},
|
||||
]) {
|
||||
const chips = chipTexts({...base, library:{...base.library, tags:[], tags_scope:null, ...library}});
|
||||
assert.equal(chips.some(chip => /category: /.test(chip)), false, JSON.stringify(library));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -234,23 +234,6 @@ test('the count line reads N of M images and K positive prompts only when known'
|
||||
assert.equal(countLine({loaded:1200,total:3400,positiveCount:0}),'1,200 of 3,400 images · 0 positive prompts');
|
||||
});
|
||||
|
||||
test('the count line reads as loading, never as an empty result, until a first page answers (#391)', async () => {
|
||||
const {countLine,createImageStream}=await source;
|
||||
assert.equal(countLine({loaded:0,total:0,positiveCount:null,loading:true,error:null}),'Loading images…');
|
||||
// A failed first page and a settled empty one say what the server said; a later page keeps the loaded count.
|
||||
assert.equal(countLine({loaded:0,total:0,positiveCount:null,loading:false,error:new Error('x')}),'0 of 0 images');
|
||||
assert.equal(countLine({loaded:0,total:0,positiveCount:0,loading:false,error:null}),'0 of 0 images · 0 positive prompts');
|
||||
assert.equal(countLine({loaded:40,total:102,positiveCount:2,loading:true,error:null}),'40 of 102 images · 2 positive prompts');
|
||||
// While a reset's first page is in flight the line reads as loading (the cleared state before it is replaced in
|
||||
// the same task, so it is never painted).
|
||||
let answer;const lines=[];
|
||||
const stream=createImageStream({api:{get:()=>new Promise(resolve=>{answer=resolve;})},onChange:state=>lines.push(countLine(state))});
|
||||
const loading=stream.reset({tags:['cinematic']});
|
||||
assert.equal(lines.at(-1),'Loading images…');
|
||||
answer({images:[{path:'a.png',root_index:0}],total:1,has_more:false,next_cursor:null,positive_count:1});await loading;
|
||||
assert.equal(lines.at(-1),'1 of 1 image · 1 positive prompt');
|
||||
});
|
||||
|
||||
test('the caption date names the newest result day under Newest first only', async () => {
|
||||
const {captionDate}=await source;
|
||||
const now=new Date(2026,9,17,15,0);
|
||||
|
||||
@@ -278,7 +278,6 @@ test('record rejects root path media and invalid versioned fields with retryable
|
||||
data => { data.resolution.sampler_id = {}; },
|
||||
data => { data.library.tags_scope = 'recipe'; },
|
||||
data => { data.library.category_scope = 'recipe'; },
|
||||
data => { data.library.tags_scope = 'image'; },
|
||||
data => { delete data.library.category_scope; },
|
||||
data => { delete data.library.tags_scope; },
|
||||
]) {
|
||||
@@ -943,14 +942,3 @@ test("names the server quotes (parentheses, '!*') still match their original and
|
||||
assert.equal(record.download.url, entry.url);
|
||||
assert.deepEqual(record.preview, { url: entry.thumbnail_url });
|
||||
});
|
||||
|
||||
test('record accepts an image\'s own category on any link status (#332)', () => {
|
||||
const toRecord = contract('toInspectionRecord');
|
||||
for (const status of ['linked', 'none', 'ambiguous', 'unavailable']) {
|
||||
const data = envelope();
|
||||
Object.assign(data.library, { status, category: 'Scans', category_scope: 'image' });
|
||||
const record = toRecord(data, image(), extractGenerationFacts);
|
||||
assert.equal(record.library.category, 'Scans');
|
||||
assert.equal(record.library.category_scope, 'image');
|
||||
}
|
||||
});
|
||||
|
||||
@@ -175,28 +175,6 @@ test('Clear all logs states both counts and its toast says what was done', async
|
||||
assert.equal(clearToast(undefined), 'Logs cleared · 0 rotated copies deleted.');
|
||||
});
|
||||
|
||||
test('Clear all logs sends back exactly the files its confirmation showed (#389)', async () => {
|
||||
const { clearBody, clearFailureText, COPY } = await diagnostics;
|
||||
const files = [
|
||||
{ filename: 'prompt_manager.log', kind: 'active', identity: '10:1761000000123456789:9007199254740993' },
|
||||
{ filename: 'prompt_manager.log.1', kind: 'rotated', identity: '20:1761000000123456789:42' },
|
||||
{ filename: 'prompt_manager.log.4', kind: 'link', identity: '5:1:7' },
|
||||
{ filename: 'prompt_manager.log.bak', kind: 'other', identity: '1:1:8' },
|
||||
{ filename: 'prompt_manager.log.2', kind: 'rotated' },
|
||||
];
|
||||
// Identities stay strings: mtime_ns and large inodes are not exact JS numbers.
|
||||
assert.deepEqual(clearBody(files), { expected: [
|
||||
{ name: 'prompt_manager.log', identity: '10:1761000000123456789:9007199254740993' },
|
||||
{ name: 'prompt_manager.log.1', identity: '20:1761000000123456789:42' },
|
||||
] });
|
||||
assert.deepEqual(clearBody([]), { expected: [] });
|
||||
assert.deepEqual(clearBody(null), {});
|
||||
const refused = Object.assign(new Error('The log files changed since you confirmed; nothing was cleared.'), { status: 409 });
|
||||
assert.equal(clearFailureText(refused), refused.message);
|
||||
assert.equal(clearFailureText(Object.assign(new Error('boom'), { status: 500 })), COPY.clearFailed);
|
||||
assert.equal(clearFailureText(undefined), COPY.clearFailed);
|
||||
});
|
||||
|
||||
test('the test-link dialog names the prompt and how Missing-file links removes the record', async () => {
|
||||
const { testLinkSentence, promptName } = await diagnostics;
|
||||
assert.equal(testLinkSentence({ id: 812, text: 'Misty harbor' }), 'Writes a placeholder image record '
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
"""A temp SQLite database in a folder no other test process shares (#346).
|
||||
|
||||
``PromptManagerAPI`` sweeps its database's folder at startup for the stages,
|
||||
uploads and safety-backup copies an interrupted restore or backup left. A
|
||||
fixture database directly in the system temp folder made every other test
|
||||
process that started an API (another worktree's suite, a pre-push hook) delete
|
||||
this process's in-flight restore files, and left safety backups behind there.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
|
||||
|
||||
def private_temp_db(case):
|
||||
"""Return a closed ``NamedTemporaryFile`` handle for an empty ``.db`` file.
|
||||
|
||||
The file lives in a new folder that ``case`` removes at cleanup, after its
|
||||
own teardown has closed and unlinked the database (cleanups run after
|
||||
``tearDown``), together with anything a restore or backup left next to it.
|
||||
"""
|
||||
folder = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
|
||||
case.addCleanup(folder.cleanup)
|
||||
handle = tempfile.NamedTemporaryFile(delete=False, suffix=".db", dir=folder.name)
|
||||
handle.close()
|
||||
return handle
|
||||
+3
-2
@@ -8,6 +8,7 @@ without requiring a full ComfyUI server.
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
@@ -17,7 +18,6 @@ from aiohttp.test_utils import AioHTTPTestCase
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from py.api import PromptManagerAPI
|
||||
from tests.private_temp import private_temp_db
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
@@ -25,7 +25,8 @@ class APITestCase(AioHTTPTestCase):
|
||||
"""Base class that stands up a test aiohttp app with PromptManager routes."""
|
||||
|
||||
async def get_application(self):
|
||||
self._temp_db = private_temp_db(self)
|
||||
self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db")
|
||||
self._temp_db.close()
|
||||
|
||||
app = web.Application()
|
||||
routes = web.RouteTableDef()
|
||||
|
||||
+9
-15
@@ -2205,30 +2205,24 @@ class TestOutputScanInternals(AdminAPITestCase):
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "unchanged")
|
||||
|
||||
def test_ingest_link_failures_are_tolerated(self):
|
||||
"""A failed link rolls the new prompt back: found, nothing left behind."""
|
||||
f = Path("gen.png")
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a dog"))}
|
||||
self.api.db._insert_image_link = _raise
|
||||
with self.assertLogs("prompt_manager.database.scanned_prompts", "ERROR"):
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
self.assertEqual(self.api.db.get_recent_prompts()["prompts"], [])
|
||||
self.api.db.link_image_to_prompt = _raise
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "added")
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
|
||||
def test_ingest_save_failure_counts_as_found(self):
|
||||
f = Path("gen.png")
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a bird"))}
|
||||
with patch(
|
||||
"py.api.admin.store_scanned_prompt", return_value=("found", None)
|
||||
) as store:
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
self.assertEqual(store.call_args.args[3], "a bird")
|
||||
self.api.db.save_prompt = lambda *a, **k: None
|
||||
self.assertEqual(self.api._ingest_scanned_file(f, meta), "found")
|
||||
|
||||
def test_batch_counts_and_per_file_errors(self):
|
||||
meta = {"prompt": json.dumps(_wired_prompt("a fish"))}
|
||||
with patch("py.api.admin.store_scanned_prompt", side_effect=_raise):
|
||||
counts = self.api._ingest_scan_batch_sync(
|
||||
[(Path("a.png"), meta), (Path("b.png"), {})]
|
||||
)
|
||||
self.api.db.get_prompt_by_hash = _raise
|
||||
counts = self.api._ingest_scan_batch_sync(
|
||||
[(Path("a.png"), meta), (Path("b.png"), {})]
|
||||
)
|
||||
self.assertEqual(
|
||||
counts,
|
||||
{
|
||||
|
||||
@@ -35,7 +35,6 @@ from aiohttp.test_utils import AioHTTPTestCase
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from py.api import PromptManagerAPI
|
||||
from tests.private_temp import private_temp_db
|
||||
from py.api.autotag_routes import MODEL_BUSY_ERROR
|
||||
from py.api.jobs import JobBusy
|
||||
from utils.hashing import generate_prompt_hash
|
||||
@@ -144,7 +143,8 @@ class AutotagAPITestCase(AioHTTPTestCase):
|
||||
fake engine."""
|
||||
|
||||
async def get_application(self):
|
||||
self._temp_db = private_temp_db(self)
|
||||
self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db")
|
||||
self._temp_db.close()
|
||||
self.tmp_root = Path(tempfile.mkdtemp())
|
||||
self.output_dir = self.tmp_root / "output"
|
||||
self.output_dir.mkdir()
|
||||
|
||||
@@ -264,41 +264,6 @@ class TestRestoreRoute(BackupRestoreTestCase):
|
||||
texts = sorted(p["text"] for p in self.api.db.search_prompts())
|
||||
self.assertEqual(texts, ["restored a", "restored b"])
|
||||
|
||||
async def test_restore_round_trip_keeps_tags_and_image_links(self):
|
||||
"""A downloaded backup brings back every prompt with its tags and images.
|
||||
|
||||
The legacy admin page's Restore form left for System (#330); this keeps
|
||||
its browser test's data check at the route.
|
||||
"""
|
||||
db = self.api.db
|
||||
originals = {}
|
||||
for text, tags, image in (
|
||||
("round trip cat", ["cats", "pets"], "cat.png"),
|
||||
("round trip dog", ["dogs"], "dog.png"),
|
||||
):
|
||||
prompt_id = db.save_prompt(
|
||||
text=text, tags=tags, prompt_hash=generate_prompt_hash(text)
|
||||
)
|
||||
path = self._write(image, b"\x89PNG\r\n\x1a\n")
|
||||
self.assertTrue(db.link_image_to_prompt(str(prompt_id), path))
|
||||
originals[text] = (prompt_id, sorted(tags), [path])
|
||||
|
||||
resp = await self.client.get("/prompt_manager/backup")
|
||||
self.assertEqual(resp.status, 200)
|
||||
backup = self._write("downloaded.db", await resp.read())
|
||||
self.assertTrue(db.delete_prompt(originals["round trip dog"][0]))
|
||||
|
||||
resp = await self._restore(backup)
|
||||
self.assertEqual(resp.status, 200, await resp.text())
|
||||
|
||||
restored = {p["text"]: p for p in self.api.db.search_prompts()}
|
||||
self.assertEqual(sorted(restored), sorted(originals))
|
||||
for text, (prompt_id, tags, paths) in originals.items():
|
||||
self.assertEqual(restored[text]["id"], prompt_id)
|
||||
self.assertEqual(sorted(restored[text]["tags"]), tags)
|
||||
images = self.api.db.get_prompt_images(str(prompt_id))
|
||||
self.assertEqual([image["image_path"] for image in images], paths)
|
||||
|
||||
async def test_restore_leaves_no_temp_files_behind(self):
|
||||
upload = self._make_db_file(["restored"])
|
||||
created = []
|
||||
|
||||
@@ -18,19 +18,15 @@ from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.disk_waits import DISK_BOUND_SECONDS
|
||||
from tests.open_handles import open_handles_under
|
||||
from tests.test_api_maintenance import ApplyTestCase, _digest
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from disk_waits import DISK_BOUND_SECONDS
|
||||
from open_handles import open_handles_under
|
||||
from test_api_maintenance import ApplyTestCase, _digest
|
||||
|
||||
from py.api import duplicate_files # noqa: E402
|
||||
from py.config import GalleryConfig, PromptManagerConfig # noqa: E402
|
||||
|
||||
# Gates and server-state polls. Waits on a find or cleanup job's task use
|
||||
# DISK_BOUND_SECONDS: those jobs hash, back up and unlink files (#409).
|
||||
WAIT = 10
|
||||
FIND = "/prompt_manager/jobs/duplicates"
|
||||
GROUPS = "/prompt_manager/maintenance/duplicates/"
|
||||
@@ -87,7 +83,7 @@ class DuplicateFilesTestCase(ApplyTestCase):
|
||||
self.assertEqual(response.status, 202, text)
|
||||
self.assertNotIn(self.tmpdir, text)
|
||||
job = self.api._jobs.get(json.loads(text)["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
return job
|
||||
|
||||
async def groups(self, job_id, status=200, **query):
|
||||
@@ -131,7 +127,7 @@ class DuplicateFilesTestCase(ApplyTestCase):
|
||||
plan = await self.review(job.id, [self.keep_of(group, path)])
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
return plan, cleanup
|
||||
|
||||
async def until(self, predicate):
|
||||
@@ -182,7 +178,7 @@ class TestFindJob(DuplicateFilesTestCase):
|
||||
gate.set()
|
||||
body = await (await legacy).text()
|
||||
job = self.api._jobs.get(summary["id"])
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
events = _sse_payloads(body)
|
||||
done = events[-1]
|
||||
self.assertEqual(done["type"], "complete")
|
||||
@@ -245,7 +241,7 @@ class TestFindJob(DuplicateFilesTestCase):
|
||||
self.assertTrue((await cancel.json())["cancellation_requested"])
|
||||
gate.set()
|
||||
job = self.api._jobs.get(job_id)
|
||||
await asyncio.wait_for(asyncio.shield(job.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(job.task), WAIT)
|
||||
self.assertEqual(job.state, "cancelled")
|
||||
last = _sse_payloads(await (await stream).text())[-1]
|
||||
self.assertEqual(last, {"type": "error", "message": "Duplicate scan cancelled"})
|
||||
@@ -345,8 +341,6 @@ class TestReview(DuplicateFilesTestCase):
|
||||
self.assertEqual(plan["protected"], {"count": 0, "reasons": []})
|
||||
|
||||
async def test_cannot_delete_keeper_or_outside_group(self):
|
||||
# About which files may go, not the backup: skip its fsync-heavy copy.
|
||||
PromptManagerConfig.SAFETY_BACKUP = False
|
||||
self.trio()
|
||||
self.write("e.png", b"other pair")
|
||||
self.write("f.png", b"other pair")
|
||||
@@ -378,7 +372,7 @@ class TestReview(DuplicateFilesTestCase):
|
||||
self.assertEqual([row["label"] for row in plan["details"]], ["b.png"])
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
self.assertEqual(cleanup.counters["removed"], 1)
|
||||
self.assertTrue((self.output_dir / "a.png").is_file())
|
||||
self.assertTrue((self.output_dir / "sub" / "c.png").is_file())
|
||||
@@ -502,7 +496,7 @@ class TestCleanupJob(DuplicateFilesTestCase):
|
||||
plan = await self.review(job.id, keep)
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
self.assertTrue(kept.is_file())
|
||||
self.assertFalse(gone.exists())
|
||||
self.assertFalse(video_thumb.exists())
|
||||
@@ -514,7 +508,7 @@ class TestCleanupJob(DuplicateFilesTestCase):
|
||||
self.write("b.png", b"new bytes") # changed since the review
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
self.assertEqual(cleanup.counters, {"removed": 1, "kept": 2, "errors": 0})
|
||||
self.assertEqual((self.output_dir / "b.png").read_bytes(), b"new bytes")
|
||||
self.assertFalse((self.output_dir / "sub" / "c.png").exists())
|
||||
@@ -528,7 +522,7 @@ class TestCleanupJob(DuplicateFilesTestCase):
|
||||
self.write("a.png", b"edited keeper")
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
self.assertEqual(cleanup.counters["removed"], 0)
|
||||
self.assertTrue((self.output_dir / "b.png").is_file())
|
||||
self.assertTrue((self.output_dir / "sub" / "c.png").is_file())
|
||||
@@ -555,7 +549,7 @@ class TestCleanupJob(DuplicateFilesTestCase):
|
||||
with patch.object(duplicate_files, "_unlink", side_effect=unlink):
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
self.assertEqual(cleanup.state, "failed")
|
||||
self.assertEqual(cleanup.counters, {"removed": 1, "kept": 1, "errors": 1})
|
||||
self.assertEqual(
|
||||
@@ -590,7 +584,7 @@ class TestCleanupJob(DuplicateFilesTestCase):
|
||||
):
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
self.assertEqual(cleanup.counters, {"removed": 0, "kept": 3, "errors": 1})
|
||||
self.assertEqual(
|
||||
cleanup.outcomes["errors"][0]["detail"], duplicate_files.DELETE_FAILED
|
||||
@@ -625,7 +619,7 @@ class TestCleanupJob(DuplicateFilesTestCase):
|
||||
with context:
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
finally:
|
||||
if handle is not None: # pragma: no cover - Windows
|
||||
handle.close()
|
||||
@@ -648,7 +642,7 @@ class TestCleanupJob(DuplicateFilesTestCase):
|
||||
with patch.object(duplicate_files, "_unlink", side_effect=unlink):
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
self.assertEqual(cleanup.counters["removed"], 2)
|
||||
self.assertEqual(open_at_unlink, [])
|
||||
|
||||
@@ -709,7 +703,7 @@ class TestCleanupJob(DuplicateFilesTestCase):
|
||||
with patch.object(duplicate_files, "_unlink", side_effect=unlink):
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
# Its records moved to the identical kept copy before the unlink failed.
|
||||
[image] = self.api.db.get_prompt_images(prompt)
|
||||
self.assertTrue(image["image_path"].endswith("a.png"))
|
||||
@@ -725,7 +719,7 @@ class TestCleanupJob(DuplicateFilesTestCase):
|
||||
self.assertEqual((busy["code"], busy["running"]["id"]), ("busy", find["id"]))
|
||||
gate.set()
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(self.api._jobs.get(find["id"]).task), DISK_BOUND_SECONDS
|
||||
asyncio.shield(self.api._jobs.get(find["id"]).task), WAIT
|
||||
)
|
||||
# The token survived the refusal; the cleanup holds off a new find.
|
||||
release = asyncio.Event()
|
||||
@@ -743,7 +737,7 @@ class TestCleanupJob(DuplicateFilesTestCase):
|
||||
self.assertEqual(legacy.status, 409)
|
||||
release.set()
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
|
||||
async def test_cleanup_refused_up_front_when_the_backup_does_not_fit(self):
|
||||
from py.api import maintenance
|
||||
@@ -971,7 +965,7 @@ class TestReviewHelpers(DuplicateFilesTestCase):
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.second_source())]
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
self.assertEqual(cleanup.state, "failed")
|
||||
self.assertEqual(cleanup.last_event["code"], "sources-changed")
|
||||
self.assertEqual(self.backups(), [])
|
||||
@@ -991,7 +985,7 @@ class TestReviewHelpers(DuplicateFilesTestCase):
|
||||
):
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
self.assertEqual(cleanup.state, "failed")
|
||||
self.assertEqual(
|
||||
cleanup.outcomes["errors"][-1]["detail"], duplicate_files.RECORDS_FAILED
|
||||
|
||||
@@ -19,17 +19,17 @@ from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.disk_waits import DISK_BOUND_SECONDS
|
||||
from tests.open_handles import open_handles_under
|
||||
from tests import test_api_duplicate_files as dup_tests
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
import test_api_duplicate_files as dup_tests
|
||||
from disk_waits import DISK_BOUND_SECONDS
|
||||
from open_handles import open_handles_under
|
||||
|
||||
from py.api import duplicate_files # noqa: E402
|
||||
from py.config import GalleryConfig # noqa: E402
|
||||
|
||||
WAIT = 10
|
||||
|
||||
|
||||
def _locator(path):
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
@@ -49,7 +49,7 @@ class SafetyTestCase(dup_tests.DuplicateFilesTestCase):
|
||||
async def apply_plan(self, plan):
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), DISK_BOUND_SECONDS)
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
return cleanup
|
||||
|
||||
|
||||
|
||||
@@ -1169,47 +1169,3 @@ class HistoryHostCleanupTests(unittest.IsolatedAsyncioTestCase):
|
||||
fence.validate(epoch, generation)
|
||||
with self.assertRaises(sqlite3.Error):
|
||||
api._history_fence()
|
||||
|
||||
|
||||
class SharedTempFolderRestoreTests(ImageAPITestCase):
|
||||
"""#346: a restore survives another suite starting in the shared temp folder.
|
||||
|
||||
Every ``PromptManagerAPI`` sweeps its database's folder for stages and
|
||||
uploads an interrupted restore left. A fixture database directly in the
|
||||
shared temp folder let any other test process (another worktree's suite,
|
||||
a pre-push hook) delete this restore's files mid-request. The shared
|
||||
folder here is a scratch stand-in for the system temp folder, so the test
|
||||
never sweeps the real one.
|
||||
"""
|
||||
|
||||
async def get_application(self):
|
||||
shared = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
|
||||
# Cleanups run last-in first-out: this one after the fixture's own.
|
||||
self.addCleanup(shared.cleanup)
|
||||
self.shared_tmp = shared.name
|
||||
with patch.object(tempfile, "tempdir", shared.name):
|
||||
return await super().get_application()
|
||||
|
||||
async def test_restore_survives_another_suite_sweeping_the_shared_folder(self):
|
||||
from database import restore_stage
|
||||
from database.models import PromptModel
|
||||
|
||||
self.api.db.save_prompt("kept by the upload")
|
||||
source = self.comfy_dir / "upload.db"
|
||||
self.assertTrue(self.api.db.model.backup_database(str(source)))
|
||||
stage = PromptModel.stage_restore
|
||||
swept = []
|
||||
|
||||
def stage_then_other_suite_starts(model, src_path):
|
||||
staged = stage(model, src_path)
|
||||
# What another suite's PromptManagerAPI does at startup.
|
||||
other = os.path.join(self.shared_tmp, "other-suite.db")
|
||||
swept.extend(restore_stage.sweep_restore_temps(other))
|
||||
return staged
|
||||
|
||||
form = FormData()
|
||||
form.add_field("database_file", source.read_bytes(), filename="upload.db")
|
||||
with patch.object(PromptModel, "stage_restore", stage_then_other_suite_starts):
|
||||
response = await self.client.post("/prompt_manager/restore", data=form)
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
self.assertEqual(swept, [], "another suite's sweep reached this restore")
|
||||
|
||||
@@ -21,8 +21,6 @@ from tests.test_api_images import ImageAPITestCase, make_png
|
||||
from tests.test_api_prompt_inspector import graph, png
|
||||
from tests.test_execution_context_database import bundle as context_bundle
|
||||
from tests.test_execution_history_database import JOB, SESSION
|
||||
from tests.saver_anchor_graphs import BRANCH_TEXTS, shared_prefix
|
||||
from tests.test_recorded_saver_anchor import LLM_TEXT, text_generate_run
|
||||
from utils.execution_contexts import ContextBundle
|
||||
from utils.execution_history_events import make_finish, make_start
|
||||
from utils.execution_outputs import OutputBundle, OutputReference, snapshot_output_roots
|
||||
@@ -1507,88 +1505,3 @@ class HistoryOutputAPITests(ImageAPITestCase):
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(body["prompts"]["positive"]["text"], "Readable portrait")
|
||||
|
||||
async def recorded_shared_prefix_run(self):
|
||||
"""#386: the dataset shape, branch 3's saver wrote the file in a run."""
|
||||
path = png(self.output_dir / "IMG_GEN_00371_.png", prompt=shared_prefix())
|
||||
# make_start/make_finish observe the run at 0 ms: the file was written then.
|
||||
os.utime(path, ns=(1_000_000, 1_000_000))
|
||||
start = self.capture(
|
||||
[self.ref(node="33", filename=path.name), self.ref("43", 1, "other.png")]
|
||||
)
|
||||
return start
|
||||
|
||||
async def test_both_inspect_routes_name_the_branch_recorded_history_saved(self):
|
||||
start = await self.recorded_shared_prefix_run()
|
||||
_, body = await self.outputs(start)
|
||||
descriptor = next(
|
||||
item["current_candidate"]["media"]
|
||||
for item in body["items"]
|
||||
if item["occurrence"]["output_node_id"] == "33"
|
||||
)
|
||||
response = await self.current(descriptor["inspection_url"])
|
||||
recorded = await response.json()
|
||||
self.assertEqual(response.status, 200, recorded)
|
||||
response = await self.client.get(
|
||||
"/prompt_manager/images/inspect/IMG_GEN_00371_.png", params={"root": "0"}
|
||||
)
|
||||
images = await response.json()
|
||||
self.assertEqual(response.status, 200, images)
|
||||
for result in (recorded, images):
|
||||
self.assertEqual(result["resolution"]["state"], "resolved")
|
||||
self.assertEqual(
|
||||
result["prompts"]["positive"],
|
||||
{
|
||||
"status": "resolved",
|
||||
"text": BRANCH_TEXTS[2],
|
||||
"source": "recorded-run",
|
||||
},
|
||||
)
|
||||
|
||||
async def test_a_file_rewritten_after_its_run_keeps_several_prompts(self):
|
||||
await self.recorded_shared_prefix_run()
|
||||
os.utime(self.output_dir / "IMG_GEN_00371_.png") # written now: not that run
|
||||
response = await self.client.get(
|
||||
"/prompt_manager/images/inspect/IMG_GEN_00371_.png", params={"root": "0"}
|
||||
)
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(body["resolution"]["state"], "ambiguous")
|
||||
self.assertEqual(body["prompts"]["positive"]["status"], "ambiguous")
|
||||
|
||||
async def test_a_failing_history_lookup_judges_the_file_alone(self):
|
||||
await self.recorded_shared_prefix_run()
|
||||
with (
|
||||
patch(
|
||||
"database.run_attribution.recorded_output",
|
||||
side_effect=sqlite3.OperationalError("locked"),
|
||||
),
|
||||
self.assertLogs("prompt_manager.run_attribution", "WARNING"),
|
||||
):
|
||||
response = await self.client.get(
|
||||
"/prompt_manager/images/inspect/IMG_GEN_00371_.png",
|
||||
params={"root": "0"},
|
||||
)
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(body["resolution"]["state"], "ambiguous")
|
||||
|
||||
async def test_a_language_model_text_shows_what_promptmanager_recorded(self):
|
||||
path = png(self.output_dir / "moss_00026_.png", prompt=text_generate_run())
|
||||
os.utime(path, ns=(1_000_000, 1_000_000))
|
||||
start = self.capture([self.ref(node="9", filename=path.name)])
|
||||
self.api.db.write_execution_contexts(
|
||||
start.run_id,
|
||||
start.storage_epoch,
|
||||
context_bundle("", LLM_TEXT, anchor="7"),
|
||||
)
|
||||
response = await self.client.get(
|
||||
"/prompt_manager/images/inspect/moss_00026_.png", params={"root": "0"}
|
||||
)
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(
|
||||
body["prompts"]["positive"],
|
||||
{"status": "resolved", "text": LLM_TEXT, "source": "execution"},
|
||||
)
|
||||
self.assertEqual(body["prompts"]["negative"]["text"], "")
|
||||
|
||||
@@ -1,375 +0,0 @@
|
||||
"""Image category routes (#332): PUT/DELETE /images/category, GET /images/categories."""
|
||||
|
||||
import json
|
||||
import ntpath
|
||||
import sqlite3
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from test_api_images import ImageAPITestCase, make_png
|
||||
from open_handles import assert_closed
|
||||
from test_image_categories import category_checksum
|
||||
except ImportError:
|
||||
from tests.test_api_images import ImageAPITestCase, make_png
|
||||
from tests.open_handles import assert_closed
|
||||
from tests.test_image_categories import category_checksum
|
||||
|
||||
from database.image_categories import MAX_KEY_LENGTH, TABLE
|
||||
from py.api.images import _favorite_root_key, image_category_key
|
||||
|
||||
CATEGORY = "/prompt_manager/images/category"
|
||||
CATEGORIES = "/prompt_manager/images/categories"
|
||||
|
||||
|
||||
class ImageCategoryTestCase(ImageAPITestCase):
|
||||
async def tearDownAsync(self):
|
||||
self.api.db.close_all()
|
||||
assert_closed(self, self._temp_db.name, self._output_tmp.name)
|
||||
await super().tearDownAsync()
|
||||
|
||||
def target(self, path, index=0):
|
||||
roots = self.api._get_all_output_dirs()
|
||||
return {
|
||||
"root_index": index,
|
||||
"root_scope": self.api._gallery_root_scope(roots[index], index),
|
||||
"path": path,
|
||||
}
|
||||
|
||||
async def send(self, method, body, status=200):
|
||||
response = await self.client.request(method, CATEGORY, json=body)
|
||||
text = await response.text()
|
||||
self.assertEqual(response.status, status, text)
|
||||
return json.loads(text)
|
||||
|
||||
async def put(self, paths, category, status=200):
|
||||
items = [p if isinstance(p, dict) else self.target(p) for p in paths]
|
||||
return await self.send("PUT", {"items": items, "category": category}, status)
|
||||
|
||||
async def delete(self, paths, status=200):
|
||||
items = [p if isinstance(p, dict) else self.target(p) for p in paths]
|
||||
return await self.send("DELETE", {"items": items}, status)
|
||||
|
||||
async def listed(self, status=200):
|
||||
response = await self.client.get(CATEGORIES)
|
||||
text = await response.text()
|
||||
self.assertEqual(response.status, status, text)
|
||||
return json.loads(text)
|
||||
|
||||
def rows(self):
|
||||
return [
|
||||
tuple(row)
|
||||
for row in self.api.db.model.get_connection().execute(
|
||||
f"SELECT root_key, relative_path, category FROM {TABLE} "
|
||||
"ORDER BY root_key, relative_path"
|
||||
)
|
||||
]
|
||||
|
||||
def key(self, relative, root=None):
|
||||
roots = self.api._get_all_output_dirs()
|
||||
return image_category_key(roots, (root or self.output_dir) / relative)
|
||||
|
||||
|
||||
class TestSetAndClear(ImageCategoryTestCase):
|
||||
async def test_put_sets_listed_and_counts_missing(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
make_png(self.output_dir / "sub" / "b.png")
|
||||
data = await self.put(["a.png", "sub/b.png", "missing.png"], " Portraits ")
|
||||
self.assertEqual(
|
||||
data,
|
||||
{
|
||||
"success": True,
|
||||
"category": "Portraits",
|
||||
"updated": 2,
|
||||
"unchanged": 0,
|
||||
"not_found": 1,
|
||||
},
|
||||
)
|
||||
root_key = _favorite_root_key(self.output_dir)
|
||||
self.assertEqual(
|
||||
self.rows(),
|
||||
[(root_key, "a.png", "Portraits"), (root_key, "sub/b.png", "Portraits")],
|
||||
)
|
||||
data = await self.put(["a.png", "sub/b.png"], "Portraits")
|
||||
self.assertEqual((data["updated"], data["unchanged"]), (0, 2))
|
||||
data = await self.put(["a.png"], "Scans")
|
||||
self.assertEqual((data["updated"], data["unchanged"]), (1, 0))
|
||||
self.assertEqual(self.api.db.image_category(*self.key("a.png")), "Scans")
|
||||
|
||||
async def test_put_with_no_listed_file_is_404_and_writes_nothing(self):
|
||||
data = await self.put(["missing.png", "gone/x.png"], "Scans", status=404)
|
||||
self.assertEqual(data["code"], "files-not-found")
|
||||
self.assertEqual(self.rows(), [])
|
||||
|
||||
async def test_duplicate_targets_are_written_and_counted_once(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
item = self.target("a.png")
|
||||
data = await self.put([item, dict(item)], "Scans")
|
||||
self.assertEqual((data["updated"], data["not_found"]), (1, 0))
|
||||
|
||||
async def test_delete_clears_listed_and_missing_files(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
await self.put(["a.png"], "Scans")
|
||||
self.api.db.set_image_categories([self.key("gone.png")], "Scans")
|
||||
data = await self.delete(["a.png", "gone.png", "never.png"])
|
||||
self.assertEqual(data, {"success": True, "cleared": 2})
|
||||
self.assertEqual(self.rows(), [])
|
||||
self.assertEqual((await self.delete(["a.png"]))["cleared"], 0)
|
||||
|
||||
async def test_put_never_writes_prompts(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
make_png(self.output_dir / "b.png")
|
||||
positive = self._save_prompt("castle at dusk", category="Landscapes")
|
||||
self.api.db.link_image_to_prompt(positive, str(self.output_dir / "a.png"))
|
||||
negative = self._save_prompt("blurry", category="Negatives")
|
||||
self.api.db.save_prompt_pair("castle at dusk", "blurry", category="Recipes")
|
||||
conn = self.api.db.model.get_connection()
|
||||
before = category_checksum(conn)
|
||||
links = conn.execute("SELECT * FROM generated_images").fetchall()
|
||||
await self.put(["a.png", "b.png"], "Portraits")
|
||||
await self.listed()
|
||||
await self.delete(["a.png"])
|
||||
self.assertEqual(category_checksum(conn), before)
|
||||
self.assertEqual(
|
||||
conn.execute("SELECT * FROM generated_images").fetchall(), links
|
||||
)
|
||||
self.assertIsNotNone(negative)
|
||||
|
||||
async def test_two_sources_in_one_request(self):
|
||||
second = self.comfy_dir / "renders"
|
||||
make_png(self.output_dir / "a.png")
|
||||
make_png(second / "a.png")
|
||||
self.api._get_all_output_dirs = lambda: [self.output_dir, second]
|
||||
data = await self.put([self.target("a.png", 0), self.target("a.png", 1)], "X")
|
||||
self.assertEqual(data["updated"], 2)
|
||||
self.assertEqual(
|
||||
{row[0] for row in self.rows()},
|
||||
{_favorite_root_key(self.output_dir), _favorite_root_key(second)},
|
||||
)
|
||||
self.assertEqual((await self.delete([self.target("a.png", 1)]))["cleared"], 1)
|
||||
|
||||
async def test_nested_sources_key_by_the_deepest(self):
|
||||
inner = self.output_dir / "nested"
|
||||
make_png(inner / "a.png")
|
||||
self.api._get_all_output_dirs = lambda: [self.output_dir, inner]
|
||||
# Named through the outer source, stored under the inner one.
|
||||
data = await self.put([self.target("nested/a.png", 0)], "Scans")
|
||||
self.assertEqual(data["updated"], 1)
|
||||
self.assertEqual(self.rows(), [(_favorite_root_key(inner), "a.png", "Scans")])
|
||||
# The same file through the inner source is the same key.
|
||||
data = await self.put([self.target("a.png", 1)], "Scans")
|
||||
self.assertEqual(data["unchanged"], 1)
|
||||
self.assertEqual((await self.listed())["categories"], ["Scans"])
|
||||
self.assertEqual((await self.delete([self.target("a.png", 1)]))["cleared"], 1)
|
||||
|
||||
async def test_video_files_can_have_a_category(self):
|
||||
(self.output_dir / "clip.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42")
|
||||
data = await self.put(["clip.mp4"], "Clips")
|
||||
self.assertEqual(data["updated"], 1)
|
||||
|
||||
async def test_windows_case_folds_the_key(self):
|
||||
make_png(self.output_dir / "Sub" / "A.png")
|
||||
with patch("py.api.images._path_flavor", ntpath):
|
||||
data = await self.put(["Sub/A.png"], "Scans")
|
||||
self.assertEqual(data["updated"], 1)
|
||||
data = await self.put(["sub/a.PNG"], "Scans")
|
||||
self.assertEqual(data["unchanged"], 1)
|
||||
self.assertEqual(self.rows()[0][1], "sub/a.png")
|
||||
self.assertEqual((await self.delete(["SUB/A.png"]))["cleared"], 1)
|
||||
|
||||
async def test_a_key_beyond_the_stored_bound_is_403(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
too_long = "/" + "r" * MAX_KEY_LENGTH
|
||||
with patch("py.api.images._favorite_root_key", return_value=too_long):
|
||||
data = await self.put(["a.png"], "Scans", status=403)
|
||||
self.assertNotIn(too_long, json.dumps(data))
|
||||
await self.delete(["a.png"], status=403)
|
||||
self.assertEqual(self.rows(), [])
|
||||
|
||||
async def test_an_unexpected_failure_is_a_logged_500(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
|
||||
def broken(*_args, **_kwargs):
|
||||
raise sqlite3.OperationalError("secret detail")
|
||||
|
||||
self.api.db.set_image_categories = broken
|
||||
self.api.db.clear_image_categories = broken
|
||||
with self.assertLogs("prompt_manager", level="ERROR"):
|
||||
data = await self.put(["a.png"], "Scans", status=500)
|
||||
self.assertEqual(data["error"], "The category could not be changed.")
|
||||
self.assertNotIn("secret", json.dumps(data))
|
||||
with self.assertLogs("prompt_manager", level="ERROR"):
|
||||
data = await self.delete(["a.png"], status=500)
|
||||
self.assertNotIn("secret", json.dumps(data))
|
||||
|
||||
async def test_a_symlinked_output_folder_keys_like_its_favorites(self):
|
||||
"""A detected output folder can be a symlink: keys use its real path."""
|
||||
real = self.comfy_dir / "real-output"
|
||||
make_png(real / "a.png")
|
||||
link = self.comfy_dir / "linked-output"
|
||||
self._make_symlink(real, link, target_is_directory=True)
|
||||
self.api._get_all_output_dirs = lambda: [link]
|
||||
data = await self.put(["a.png"], "Scans")
|
||||
self.assertEqual(data["updated"], 1)
|
||||
self.assertEqual(self.rows(), [(_favorite_root_key(link), "a.png", "Scans")])
|
||||
self.assertEqual((await self.listed())["categories"], ["Scans"])
|
||||
self.assertEqual((await self.delete(["a.png"]))["cleared"], 1)
|
||||
|
||||
async def test_lone_surrogates_are_400_not_500(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
await self.send(
|
||||
"PUT", {"items": [self.target("a.png")], "category": "x\ud800"}, 400
|
||||
)
|
||||
await self.send("DELETE", {"items": [self.target("a\ud800.png")]}, 400)
|
||||
self.assertEqual(self.rows(), [])
|
||||
|
||||
async def test_delete_category_never_reaches_delete_image(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
calls = []
|
||||
|
||||
async def recorded(request):
|
||||
calls.append(request.match_info)
|
||||
raise AssertionError("delete_image reached")
|
||||
|
||||
self.api.delete_image = recorded
|
||||
await self.put(["a.png"], "Scans")
|
||||
data = await self.delete(["a.png"])
|
||||
self.assertEqual(data["cleared"], 1)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
|
||||
class TestRefusals(ImageCategoryTestCase):
|
||||
async def test_malformed_bodies_are_400(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
item = self.target("a.png")
|
||||
cases = [
|
||||
{"items": [item]},
|
||||
{"category": "x"},
|
||||
{"items": [item], "category": "x", "extra": 1},
|
||||
{"items": item, "category": "x"},
|
||||
{"items": [], "category": "x"},
|
||||
{"items": [item] * 501, "category": "x"},
|
||||
{"items": [item], "category": 3},
|
||||
{"items": [item], "category": None},
|
||||
{"items": [item], "category": " \t "},
|
||||
{"items": [item], "category": "x" * 101},
|
||||
{"items": [item], "category": "a\x07b"},
|
||||
{"items": [{**item, "root_index": "0"}], "category": "x"},
|
||||
{"items": [{**item, "root_scope": "nothex"}], "category": "x"},
|
||||
{"items": [{"root_index": 0, "path": "a.png"}], "category": "x"},
|
||||
{"items": [{**item, "root_index": 1}], "category": "x"},
|
||||
]
|
||||
for body in cases:
|
||||
with self.subTest(body=str(body)[:80]):
|
||||
data = await self.send("PUT", body, status=400)
|
||||
self.assertFalse(data["success"])
|
||||
await self.send("DELETE", {"items": [item], "category": "x"}, status=400)
|
||||
await self.send("DELETE", {"items": []}, status=400)
|
||||
response = await self.client.request(
|
||||
"PUT", CATEGORY, data="not json", headers={"Content-Type": "text/plain"}
|
||||
)
|
||||
self.assertEqual(response.status, 400)
|
||||
self.assertEqual(self.rows(), [])
|
||||
|
||||
async def test_unsafe_paths_are_403(self):
|
||||
(self.output_dir / "notes.txt").write_text("not media")
|
||||
for path in ("../outside.png", "/a.png", "C:/a.png", "a\\b.png", "a/./b.png"):
|
||||
with self.subTest(path=path):
|
||||
await self.put([self.target(path)], "x", status=403)
|
||||
data = await self.put([self.target("notes.txt")], "x", status=403)
|
||||
self.assertEqual(data["error"], "Only media files can have a category")
|
||||
await self.delete([self.target("notes.txt")], status=403)
|
||||
|
||||
async def test_a_stale_scope_is_409(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
item = {**self.target("a.png"), "root_scope": "0" * 64}
|
||||
data = await self.put([item], "x", status=409)
|
||||
self.assertEqual(data["code"], "source-changed")
|
||||
data = await self.delete([item], status=409)
|
||||
self.assertEqual(data["code"], "source-changed")
|
||||
|
||||
async def test_an_oversized_body_is_413(self):
|
||||
response = await self.client.request(
|
||||
"PUT",
|
||||
CATEGORY,
|
||||
data=b'{"items": [], "category": "' + b"x" * (2 * 1024 * 1024) + b'"}',
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
self.assertEqual(response.status, 413)
|
||||
|
||||
|
||||
class TestCategoryList(ImageCategoryTestCase):
|
||||
async def test_lists_only_values_whose_file_is_listed(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
make_png(self.output_dir / "c.png")
|
||||
await self.put(["a.png"], "B")
|
||||
await self.put(["c.png"], "A")
|
||||
self.api.db.set_image_categories([self.key("gone.png")], "Z")
|
||||
self.assertEqual(
|
||||
await self.listed(), {"success": True, "categories": ["A", "B"]}
|
||||
)
|
||||
|
||||
async def test_empty_library_lists_nothing(self):
|
||||
self.assertEqual((await self.listed())["categories"], [])
|
||||
|
||||
async def test_a_failed_read_is_500(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
|
||||
def broken(*_args, **_kwargs):
|
||||
raise sqlite3.OperationalError("secret detail")
|
||||
|
||||
self.api.db.own_categories_of = broken
|
||||
with self.assertLogs("prompt_manager", level="ERROR"):
|
||||
data = await self.listed(status=500)
|
||||
self.assertEqual(data["error"], "Image categories could not be read.")
|
||||
self.assertNotIn("secret", json.dumps(data))
|
||||
|
||||
|
||||
class TestAllMatchingFilter(ImageCategoryTestCase):
|
||||
"""#392 "Select all N matching": the listing filter, resolved now."""
|
||||
|
||||
async def test_filter_sets_and_clears_every_match(self):
|
||||
for name in ("bulk/a.png", "bulk/b.png", "other/c.png"):
|
||||
make_png(self.output_dir / name)
|
||||
data = await self.send(
|
||||
"PUT", {"filter": {"subfolder": "bulk"}, "category": "Scans"}
|
||||
)
|
||||
self.assertEqual((data["updated"], data["not_found"]), (2, 0))
|
||||
self.assertEqual(
|
||||
sorted(row[1] for row in self.rows()), ["bulk/a.png", "bulk/b.png"]
|
||||
)
|
||||
data = await self.send("DELETE", {"filter": {"subfolder": "bulk"}})
|
||||
self.assertEqual(data["cleared"], 2)
|
||||
data = await self.send(
|
||||
"PUT", {"filter": {"subfolder": "none"}, "category": "Scans"}, 404
|
||||
)
|
||||
self.assertEqual(data["code"], "files-not-found")
|
||||
self.assertEqual(
|
||||
(await self.send("DELETE", {"filter": {"q": "zz"}}))["cleared"], 0
|
||||
)
|
||||
|
||||
async def test_filter_refusals(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
item = self.target("a.png")
|
||||
for body in (
|
||||
{"filter": {}, "items": [item], "category": "x"},
|
||||
{"filter": {"unknown": "1"}, "category": "x"},
|
||||
{"filter": "q=a", "category": "x"},
|
||||
{"filter": {"root": "0", "root_scope": "nothex"}, "category": "x"},
|
||||
):
|
||||
with self.subTest(body=body):
|
||||
await self.send("PUT", body, 400)
|
||||
data = await self.send(
|
||||
"PUT",
|
||||
{"filter": {"root": "0", "root_scope": "0" * 64}, "category": "x"},
|
||||
409,
|
||||
)
|
||||
self.assertEqual(data["code"], "source-changed")
|
||||
self.assertEqual(self.rows(), [])
|
||||
|
||||
async def test_more_than_500_matching_is_refused(self):
|
||||
for number in range(501):
|
||||
make_png(self.output_dir / f"many/{number:03}.png", size=(1, 1))
|
||||
data = await self.send("PUT", {"filter": {}, "category": "x"}, 400)
|
||||
self.assertEqual(data["code"], "too-many-images")
|
||||
self.assertEqual(data["error"], "Select at most 500 images.")
|
||||
self.assertEqual(self.rows(), [])
|
||||
@@ -42,7 +42,6 @@ from py import lora_utils # noqa: E402
|
||||
from py import lora_manager_layout as layout_module # noqa: E402
|
||||
from py.api.thumbnail_settings import ThumbnailSettings # noqa: E402
|
||||
from py.api import PromptManagerAPI # noqa: E402
|
||||
from tests.private_temp import private_temp_db # noqa: E402
|
||||
from py.api.prompts import MAX_PAGE_LIMIT # noqa: E402
|
||||
from py.config import GalleryConfig # noqa: E402
|
||||
from utils.hashing import generate_prompt_hash # noqa: E402
|
||||
@@ -61,7 +60,8 @@ class ImageAPITestCase(AioHTTPTestCase):
|
||||
"""App with PromptManager routes, a temp DB and a temp output directory."""
|
||||
|
||||
async def get_application(self):
|
||||
self._temp_db = private_temp_db(self)
|
||||
self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db")
|
||||
self._temp_db.close()
|
||||
|
||||
self._output_tmp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
|
||||
# A fake ComfyUI tree: <tmp>/output is the gallery root and <tmp> is the
|
||||
|
||||
@@ -415,74 +415,5 @@ class TestApply(LogCleanupTestCase):
|
||||
self.assertEqual(job.status_line, "The log cleanup failed.")
|
||||
|
||||
|
||||
TRUNCATE = "/prompt_manager/logs/truncate"
|
||||
FILES = "/prompt_manager/logs/files"
|
||||
|
||||
|
||||
class TestClearAllLogsConfirmed(LogCleanupTestCase):
|
||||
"""#389: POST /logs/truncate deletes no more than the confirmation listed."""
|
||||
|
||||
async def confirmed(self):
|
||||
response = await self.client.request("GET", FILES)
|
||||
data = await response.json()
|
||||
return [
|
||||
{"name": row["filename"], "identity": row["identity"]}
|
||||
for row in data["files"]
|
||||
if row["kind"] in ("active", "rotated")
|
||||
]
|
||||
|
||||
def handler(self):
|
||||
return next(
|
||||
h
|
||||
for h in self.manager.logger.handlers
|
||||
if isinstance(h, logging.FileHandler)
|
||||
)
|
||||
|
||||
async def test_a_rollover_after_the_confirmation_is_refused(self):
|
||||
self.write("prompt_manager.log.1", b"a" * 100)
|
||||
self.write("prompt_manager.log.2", b"b" * 200)
|
||||
expected = await self.confirmed()
|
||||
self.app_logger.info("written after the confirmation")
|
||||
self.handler().doRollover()
|
||||
before = self.snapshot()
|
||||
body = await self.post(TRUNCATE, {"expected": expected}, 409)
|
||||
self.assertEqual(body["success"], False)
|
||||
self.assertEqual(body["code"], "logs-changed")
|
||||
self.assertIn("nothing was cleared", body["error"])
|
||||
self.assertEqual(self.snapshot(), before)
|
||||
|
||||
async def test_the_confirmed_files_are_cleared(self):
|
||||
self.write("prompt_manager.log.1", b"a" * 100)
|
||||
expected = await self.confirmed()
|
||||
body = await self.post(TRUNCATE, {"expected": expected}, 200)
|
||||
self.assertEqual(
|
||||
sorted(body["results"]["truncated"]),
|
||||
["prompt_manager.log", "prompt_manager.log.1"],
|
||||
)
|
||||
self.assertEqual(self.names(), ["prompt_manager.log"])
|
||||
|
||||
async def test_the_legacy_call_without_a_body_still_clears(self):
|
||||
self.write("prompt_manager.log.1", b"a" * 100)
|
||||
response = await self.client.request("POST", TRUNCATE)
|
||||
self.assertEqual(response.status, 200, await response.text())
|
||||
self.assertEqual(self.names(), ["prompt_manager.log"])
|
||||
|
||||
async def test_a_foreign_writer_is_refused_with_a_clear_message(self):
|
||||
self.write("prompt_manager.log.1", b"a" * 100)
|
||||
with open(self.path("prompt_manager.log"), "ab") as other:
|
||||
other.write(b"line from another ComfyUI\n")
|
||||
body = await self.post(TRUNCATE, {"expected": await self.confirmed()}, 409)
|
||||
self.assertEqual(body["code"], "logs-foreign-writer")
|
||||
self.assertIn("Another program is writing", body["error"])
|
||||
self.assertIn("prompt_manager.log.1", self.names())
|
||||
|
||||
async def test_a_malformed_confirmation_is_400(self):
|
||||
self.write("prompt_manager.log.1", b"a" * 100)
|
||||
for expected in ("all", [{"name": "prompt_manager.log.1", "identity": 5}]):
|
||||
with self.subTest(expected=expected):
|
||||
await self.post(TRUNCATE, {"expected": expected}, 400)
|
||||
self.assertIn("prompt_manager.log.1", self.names())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -36,7 +36,6 @@ from aiohttp.test_utils import AioHTTPTestCase
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from py.api import PromptManagerAPI
|
||||
from tests.private_temp import private_temp_db
|
||||
from py import config_store, lora_import, lora_utils
|
||||
from py.config import IntegrationConfig, PromptManagerConfig
|
||||
from py.api import settings_routes as settings_module
|
||||
@@ -58,7 +57,8 @@ class LoraAPITestCase(AioHTTPTestCase):
|
||||
"""Stands up the PromptManager routes with a temp DB and neutral LoRA config."""
|
||||
|
||||
async def get_application(self):
|
||||
self._temp_db = private_temp_db(self)
|
||||
self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db")
|
||||
self._temp_db.close()
|
||||
self.tmp_root = Path(tempfile.mkdtemp())
|
||||
self.custom_nodes = self.tmp_root / "custom_nodes"
|
||||
self.custom_nodes.mkdir()
|
||||
|
||||
@@ -5,6 +5,7 @@ Route tests for the prompt API (py/api/prompts.py).
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -28,7 +29,6 @@ except ImportError: # discovered with tests/ as the top-level directory
|
||||
|
||||
from database.operations import PromptDatabase # noqa: E402
|
||||
from py.api import PromptManagerAPI # noqa: E402
|
||||
from tests.private_temp import private_temp_db # noqa: E402
|
||||
import py.api.prompts as prompts_module # noqa: E402
|
||||
from py.api.prompts import ( # noqa: E402
|
||||
BULK_CATEGORY_MAX_PROMPTS,
|
||||
@@ -43,7 +43,8 @@ class PromptAPITestCase(AioHTTPTestCase):
|
||||
"""App with PromptManager routes and a temporary SQLite database."""
|
||||
|
||||
async def get_application(self):
|
||||
self._temp_db = private_temp_db(self)
|
||||
self._temp_db = tempfile.NamedTemporaryFile(delete=False, suffix=".db")
|
||||
self._temp_db.close()
|
||||
|
||||
app = web.Application()
|
||||
routes = web.RouteTableDef()
|
||||
|
||||
@@ -1040,8 +1040,8 @@ class TestApplyRefusals(ReviewRoutesTestCase):
|
||||
_path, item, detail, observed, cat = await self._opened()
|
||||
real = review_capture_module.resolve_prompt_pair
|
||||
|
||||
def reworded(metadata, **kwargs):
|
||||
resolution = real(metadata, **kwargs)
|
||||
def reworded(metadata):
|
||||
resolution = real(metadata)
|
||||
return {
|
||||
**resolution,
|
||||
"reason": "a reworded reason from a newer resolver",
|
||||
@@ -1065,8 +1065,8 @@ class TestApplyRefusals(ReviewRoutesTestCase):
|
||||
_path, item, detail, observed, cat = await self._opened()
|
||||
real = review_capture_module.resolve_prompt_pair
|
||||
|
||||
def without_cat(metadata, **kwargs):
|
||||
resolution = real(metadata, **kwargs)
|
||||
def without_cat(metadata):
|
||||
resolution = real(metadata)
|
||||
kept = [
|
||||
c
|
||||
for c in resolution["candidates"]
|
||||
|
||||
@@ -62,16 +62,12 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
},
|
||||
)
|
||||
self.assertEqual(
|
||||
categories["categories"],
|
||||
[{"value": "Animal", "prompt_count": 1, "image_count": 0}],
|
||||
categories["categories"], [{"value": "Animal", "prompt_count": 1}]
|
||||
)
|
||||
_, rest = await self.get(
|
||||
"categories", limit=1, cursor=categories["next_cursor"]
|
||||
)
|
||||
self.assertEqual(
|
||||
rest["categories"],
|
||||
[{"value": "Pet", "prompt_count": 1, "image_count": 0}],
|
||||
)
|
||||
self.assertEqual(rest["categories"], [{"value": "Pet", "prompt_count": 1}])
|
||||
self.assertFalse(rest["has_more"])
|
||||
|
||||
async def test_vocabulary_cursor_is_signed_epoch_scoped(self):
|
||||
@@ -492,7 +488,6 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
"other_prompts": 0,
|
||||
"negative_variants": 1,
|
||||
"archived_negative_variants": 0,
|
||||
"images": 0,
|
||||
}
|
||||
self.assertEqual(
|
||||
preview,
|
||||
@@ -533,73 +528,6 @@ class TestVocabularyAPI(ImageAPITestCase):
|
||||
self.assertEqual(response.status, 409)
|
||||
self.assertEqual(replay["code"], "vocabulary-conflict")
|
||||
|
||||
async def test_category_rows_report_images_and_image_only_values(self):
|
||||
self._save_prompt("Owner", category="Prompt value")
|
||||
_, options_before = await self.get_options()
|
||||
self.api.db.set_image_categories(
|
||||
[("root", "a.png"), ("root", "b.png")], "Prompt value"
|
||||
)
|
||||
self.api.db.set_image_categories([("root", "c.png")], "Image value")
|
||||
response, first = await self.get("categories", limit=1)
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(
|
||||
first["categories"],
|
||||
[{"value": "Prompt value", "prompt_count": 1, "image_count": 2}],
|
||||
)
|
||||
self.assertEqual(first["total"], 2)
|
||||
_, rest = await self.get("categories", limit=1, cursor=first["next_cursor"])
|
||||
self.assertEqual(
|
||||
rest["categories"],
|
||||
[{"value": "Image value", "prompt_count": 0, "image_count": 1}],
|
||||
)
|
||||
self.assertFalse(rest["has_more"])
|
||||
# The Prompt library's category list does not gain image values.
|
||||
_, options_after = await self.get_options()
|
||||
self.assertEqual(options_after, options_before)
|
||||
self.assertEqual(options_after["categories"], ["Prompt value"])
|
||||
|
||||
async def get_options(self):
|
||||
response = await self.client.get("/prompt_manager/library/options")
|
||||
return response, await response.json()
|
||||
|
||||
async def test_category_routes_rename_image_only_value(self):
|
||||
epoch = self.api.db.get_execution_epoch()
|
||||
self._save_prompt("Bystander", category="Kept")
|
||||
self.api.db.set_image_categories(
|
||||
[("root", "a.png"), ("second", "b.png")], "Image value"
|
||||
)
|
||||
prompts_before = self.category_rows()
|
||||
body = {"expected_epoch": epoch, "source": "Image value", "target": "Moved"}
|
||||
response, preview = await self.send("POST", "categories/preview", body)
|
||||
self.assertEqual(response.status, 200, preview)
|
||||
impact = {
|
||||
"positive_prompts": 0,
|
||||
"other_prompts": 0,
|
||||
"negative_variants": 0,
|
||||
"archived_negative_variants": 0,
|
||||
"images": 2,
|
||||
}
|
||||
self.assertEqual(preview["impact"], impact)
|
||||
stale = preview["impact_token"]
|
||||
self.api.db.set_image_categories([("root", "late.png")], "Image value")
|
||||
response, refused = await self.send(
|
||||
"POST", "categories/apply", {**body, "impact_token": stale}
|
||||
)
|
||||
self.assertEqual(response.status, 409)
|
||||
self.assertEqual(refused["code"], "vocabulary-conflict")
|
||||
self.assertEqual(self.api.db.image_category("root", "a.png"), "Image value")
|
||||
_, preview = await self.send("POST", "categories/preview", body)
|
||||
response, applied = await self.send(
|
||||
"POST",
|
||||
"categories/apply",
|
||||
{**body, "impact_token": preview["impact_token"]},
|
||||
)
|
||||
self.assertEqual(response.status, 200, applied)
|
||||
self.assertEqual(applied["impact"], {**impact, "images": 3})
|
||||
for key in (("root", "a.png"), ("second", "b.png"), ("root", "late.png")):
|
||||
self.assertEqual(self.api.db.image_category(*key), "Moved")
|
||||
self.assertEqual(self.category_rows(), prompts_before)
|
||||
|
||||
async def test_preview_validation(self):
|
||||
epoch = self.api.db.get_execution_epoch()
|
||||
self._save_prompt("Owner", category="Scenery")
|
||||
|
||||
@@ -1,262 +0,0 @@
|
||||
"""Images filters by the effective category (#332 MC4, MC5, DC6).
|
||||
|
||||
An image's effective category is its own category when it has one, else the
|
||||
category its link supplies (the recipe's when set, else the positive's), else
|
||||
none. The category filter, "select all N matching" and saved views match it.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
import test_api_workspace_images as workspace
|
||||
from test_image_categories import category_checksum
|
||||
except ImportError:
|
||||
from tests import test_api_workspace_images as workspace
|
||||
from tests.test_image_categories import category_checksum
|
||||
|
||||
from py.api.images import image_category_key, parse_listing_filter
|
||||
|
||||
|
||||
class WorkspaceCategoryCase(workspace.ImageAPITestCase):
|
||||
"""Listing helpers plus own-category writes keyed as the routes key them."""
|
||||
|
||||
media = workspace.TestWorkspaceImages.media
|
||||
get = workspace.TestWorkspaceImages.get
|
||||
tag = workspace.TestWorkspaceImageTags.tag
|
||||
names = workspace.TestWorkspaceImageTags.names
|
||||
|
||||
async def tearDownAsync(self):
|
||||
self.api.db.close_all()
|
||||
workspace.assert_closed(self, self._temp_db.name, self._output_tmp.name)
|
||||
await super().tearDownAsync()
|
||||
|
||||
def own(self, path, value, sources=None):
|
||||
sources = sources or self.api._get_all_output_dirs()
|
||||
self.api.db.set_image_categories([image_category_key(sources, path)], value)
|
||||
|
||||
async def listed(self, **params):
|
||||
response, body = await self.get(sort="name", **params)
|
||||
self.assertEqual(response.status, 200, body)
|
||||
return self.names(body), body
|
||||
|
||||
|
||||
class TestEffectiveCategoryFilter(WorkspaceCategoryCase):
|
||||
def batch(self):
|
||||
"""Four images of "castle at dusk" (Landscapes); one is overridden."""
|
||||
castle = self._save_prompt("castle at dusk", category="Landscapes", rating=4)
|
||||
files = [self.media(f"castle_{n}.png", mtime=100 + n) for n in range(4)]
|
||||
for path in files:
|
||||
self.api.db.link_image_to_prompt(castle, str(path))
|
||||
self.own(files[0], "Portraits")
|
||||
return castle, files
|
||||
|
||||
async def test_category_filter_prefers_own_category(self):
|
||||
castle, _files = self.batch()
|
||||
conn = self.api.db.model.get_connection()
|
||||
before = category_checksum(conn)
|
||||
names, body = await self.listed(category="Portraits")
|
||||
self.assertEqual(names, ["castle_0.png"])
|
||||
# The caption still counts the prompt behind the image shown.
|
||||
self.assertEqual(body["positive_count"], 1)
|
||||
names, body = await self.listed(category="Landscapes")
|
||||
self.assertEqual(names, ["castle_1.png", "castle_2.png", "castle_3.png"])
|
||||
self.assertEqual(body["total"], 3)
|
||||
names, _ = await self.listed()
|
||||
self.assertEqual(len(names), 4)
|
||||
self.assertEqual(category_checksum(conn), before)
|
||||
self.assertEqual(self.api.db.get_prompt_by_id(castle)["category"], "Landscapes")
|
||||
|
||||
async def test_category_filter_lists_unlinked_with_own(self):
|
||||
scans = [self.media(f"scan_{n}.png", mtime=100 + n) for n in range(2)]
|
||||
self.media("plain.png", mtime=50)
|
||||
for path in scans:
|
||||
self.own(path, "Scans")
|
||||
names, body = await self.listed(category="Scans")
|
||||
self.assertEqual(names, ["scan_0.png", "scan_1.png"])
|
||||
self.assertEqual(body["positive_count"], 0)
|
||||
|
||||
async def test_recipe_category_is_inherited(self):
|
||||
"""MC4: a recipe's own category, not the positive's, is inherited."""
|
||||
db = self.api.db
|
||||
self._save_prompt("Pair positive", category="Night")
|
||||
pair = db.save_prompt_pair("Pair positive", "Noise", category="Recipes")
|
||||
blank = db.save_prompt_pair("Pair positive", "Other noise", category=" ")
|
||||
recipe = self.media("recipe.png", mtime=103)
|
||||
blank_recipe = self.media("blank_recipe.png", mtime=102)
|
||||
positive = db.get_prompt_pair(pair)["positive_prompt_id"]
|
||||
for path, pair_id in ((recipe, pair), (blank_recipe, blank)):
|
||||
image = db.link_image_to_prompt(positive, str(path))
|
||||
db.link_image_to_pair(pair_id, image, {"source": "fixture"})
|
||||
self.assertEqual((await self.listed(category="Recipes"))[0], ["recipe.png"])
|
||||
# A blank recipe category falls back to the positive's.
|
||||
self.assertEqual((await self.listed(category="Night"))[0], ["blank_recipe.png"])
|
||||
# An own category still wins over the recipe's.
|
||||
self.own(recipe, "Mine")
|
||||
self.assertEqual((await self.listed(category="Recipes"))[0], [])
|
||||
self.assertEqual((await self.listed(category="Mine"))[0], ["recipe.png"])
|
||||
|
||||
async def test_ambiguous_files_inherit_nothing_but_keep_their_own(self):
|
||||
first = self._save_prompt("Owner one", category="Night")
|
||||
second = self._save_prompt("Owner two", category="Night")
|
||||
ambiguous = self.media("ambiguous.png")
|
||||
for owner in (first, second):
|
||||
self.api.db.link_image_to_prompt(owner, str(ambiguous))
|
||||
self.assertEqual((await self.listed(category="Night"))[0], [])
|
||||
self.own(ambiguous, "Night")
|
||||
self.assertEqual((await self.listed(category="Night"))[0], ["ambiguous.png"])
|
||||
|
||||
async def test_combines_with_tags_search_and_rating(self):
|
||||
stone = self._save_prompt(
|
||||
"castle at dusk", category="Landscapes", tags=["stone"], rating=4
|
||||
)
|
||||
files = [self.media(f"castle_{n}.png", mtime=100 + n) for n in range(2)]
|
||||
for path in files:
|
||||
self.api.db.link_image_to_prompt(stone, str(path))
|
||||
self.own(files[0], "Portraits")
|
||||
other = self._save_prompt("harbor at noon", category="Portraits", rating=2)
|
||||
harbor = self.media("harbor.png", mtime=90)
|
||||
self.api.db.link_image_to_prompt(other, str(harbor))
|
||||
names, _ = await self.listed(category="Portraits", tags="stone")
|
||||
self.assertEqual(names, ["castle_0.png"])
|
||||
names, _ = await self.listed(category="Portraits", q="harbor")
|
||||
self.assertEqual(names, ["harbor.png"])
|
||||
names, _ = await self.listed(category="Portraits", q="castle_0")
|
||||
self.assertEqual(names, ["castle_0.png"])
|
||||
response, body = await self.get(category="Portraits", sort="rating")
|
||||
self.assertEqual(response.status, 200, body)
|
||||
self.assertEqual(self.names(body), ["castle_0.png", "harbor.png"])
|
||||
self.assertEqual(body["positive_count"], 2)
|
||||
|
||||
async def test_image_tag_admitted_file_keeps_its_inherited_category(self):
|
||||
night = self._save_prompt("Night owl", category="Night", tags=["a"])
|
||||
tagged = self.media("tagged.png", mtime=103)
|
||||
unlinked = self.media("unlinked.png", mtime=102)
|
||||
self.api.db.link_image_to_prompt(night, str(tagged))
|
||||
self.tag(tagged, ["b"])
|
||||
self.tag(unlinked, ["a", "b"])
|
||||
names, _ = await self.listed(tags="a,b", category="Night")
|
||||
self.assertEqual(names, ["tagged.png"])
|
||||
self.own(unlinked, "Night")
|
||||
names, _ = await self.listed(tags="a,b", category="Night")
|
||||
self.assertEqual(names, ["tagged.png", "unlinked.png"])
|
||||
|
||||
async def test_cursor_pages_walk_only_the_category(self):
|
||||
files = [self.media(f"img_{n:02}.png", mtime=100 + n) for n in range(7)]
|
||||
for path in files[::2]:
|
||||
self.own(path, "Even")
|
||||
seen, cursor = [], None
|
||||
while True:
|
||||
params = {"category": "Even", "limit": 2, "sort": "name"}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
response, body = await self.get(**params)
|
||||
self.assertEqual(response.status, 200, body)
|
||||
seen += self.names(body)
|
||||
self.assertEqual(body["total"], 4)
|
||||
cursor = body["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
self.assertEqual(seen, ["img_00.png", "img_02.png", "img_04.png", "img_06.png"])
|
||||
|
||||
async def test_an_outer_source_lists_a_nested_sources_own_category(self):
|
||||
inner = self.output_dir / "nested"
|
||||
nested = self.media("nested/a.png")
|
||||
self.media("b.png")
|
||||
self.api._get_all_output_dirs = lambda: [self.output_dir, inner]
|
||||
self.own(nested, "Scans")
|
||||
names, _ = await self.listed(category="Scans", root="0")
|
||||
self.assertEqual(names, ["nested/a.png"])
|
||||
names, _ = await self.listed(category="Scans", root="1")
|
||||
self.assertEqual(names, ["a.png"])
|
||||
# The outer source's own key for that path is not the file's key.
|
||||
from py.api.images import _favorite_root_key
|
||||
|
||||
self.api.db.clear_image_categories([image_category_key([inner], nested)])
|
||||
self.api.db.set_image_categories(
|
||||
[(_favorite_root_key(self.output_dir), "nested/a.png")], "Stale"
|
||||
)
|
||||
self.assertEqual((await self.listed(category="Stale"))[0], [])
|
||||
|
||||
async def test_a_gone_files_category_is_not_listed(self):
|
||||
gone = self.media("gone.png")
|
||||
self.own(gone, "Scans")
|
||||
os.unlink(gone)
|
||||
self.api.invalidate_gallery_cache()
|
||||
self.assertEqual((await self.listed(category="Scans"))[0], [])
|
||||
|
||||
async def test_a_failed_own_category_read_is_listing_failed(self):
|
||||
self.media("a.png")
|
||||
|
||||
def broken(*_args, **_kwargs):
|
||||
raise sqlite3.OperationalError("secret detail")
|
||||
|
||||
with patch.object(self.api.db, "own_categories_of", side_effect=broken):
|
||||
with self.assertLogs("prompt_manager", level="ERROR"):
|
||||
response, body = await self.get(category="Scans")
|
||||
self.assertEqual(response.status, 500)
|
||||
self.assertNotIn("secret", str(body))
|
||||
# Without a category nothing new is read.
|
||||
response, body = await self.get()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
|
||||
async def test_select_all_matching_uses_the_effective_category(self):
|
||||
_castle, files = self.batch()
|
||||
resolved = await self.api.resolve_listing_filter(
|
||||
parse_listing_filter({"category": "Portraits"}), 100
|
||||
)
|
||||
self.assertEqual([t[2] for t in resolved.targets], ["castle_0.png"])
|
||||
resolved = await self.api.resolve_listing_filter(
|
||||
parse_listing_filter({"category": "Landscapes"}), 100
|
||||
)
|
||||
self.assertEqual(len(resolved.targets), 3)
|
||||
self.assertEqual(len(files), 4)
|
||||
|
||||
async def test_saved_view_category_means_the_effective_category(self):
|
||||
"""CUJ 8: a saved view's category is the listing's query value."""
|
||||
_castle, files = self.batch()
|
||||
self.own(files[1], "Portraits")
|
||||
names, _ = await self.listed(category="Portraits")
|
||||
self.assertEqual(names, ["castle_0.png", "castle_1.png"])
|
||||
|
||||
|
||||
class TestFilterAndChipAgreeOnBlanks(WorkspaceCategoryCase):
|
||||
"""The filter and the inspector use one inherited-category rule."""
|
||||
|
||||
async def chip(self, path):
|
||||
response = await self.client.get(
|
||||
f"/prompt_manager/images/inspect/{path}", params={"root": "0"}
|
||||
)
|
||||
body = await response.json()
|
||||
self.assertEqual(response.status, 200, body)
|
||||
return body["library"]["category"]
|
||||
|
||||
async def test_blank_recipe_values_agree(self):
|
||||
db = self.api.db
|
||||
positive = self._save_prompt("Pair positive", category="Night")
|
||||
blanks = {"spaces": " ", "tab": "\t", "nbsp": " ", "set": "Recipes"}
|
||||
for index, (label, value) in enumerate(blanks.items()):
|
||||
pair = db.save_prompt_pair("Pair positive", f"noise {label}")
|
||||
path = self.media(f"{label}.png", mtime=100 + index)
|
||||
image = db.link_image_to_prompt(positive, str(path))
|
||||
db.link_image_to_pair(pair, image, {"source": "fixture"})
|
||||
with db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE prompt_pairs SET category=? WHERE id=?", (value, pair)
|
||||
)
|
||||
blob = db.save_prompt_pair("Pair positive", "noise blob")
|
||||
path = self.media("blob.png", mtime=90)
|
||||
image = db.link_image_to_prompt(positive, str(path))
|
||||
db.link_image_to_pair(blob, image, {"source": "fixture"})
|
||||
with db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE prompt_pairs SET category=CAST(' ' AS BLOB) WHERE id=?",
|
||||
(blob,),
|
||||
)
|
||||
for name in ("spaces", "tab", "nbsp", "set", "blob"):
|
||||
with self.subTest(name=name):
|
||||
shown = await self.chip(f"{name}.png")
|
||||
for value in ("Night", "Recipes"):
|
||||
listed = f"{name}.png" in (await self.listed(category=value))[0]
|
||||
self.assertEqual(listed, shown == value, (name, value, shown))
|
||||
@@ -1347,7 +1347,7 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
_, data = await self.get(q="harbor")
|
||||
self.assertEqual(self._names(data), ["night_a.png"])
|
||||
self.assertEqual(
|
||||
self.api.db.get_image_paths_matching("zebra", []),
|
||||
self.api.db.get_image_paths_matching("zebra", None, []),
|
||||
{
|
||||
normalize_image_path(str(files[name]))
|
||||
for name in ("night_ab", "day_ab", "titled")
|
||||
@@ -1407,10 +1407,8 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
for query in statements
|
||||
if query.lstrip().upper().startswith(("SELECT", "WITH"))
|
||||
]
|
||||
# Plus the one stream of tagged image records q and tags read (B3),
|
||||
# and the category's own-category read of the one source (#332).
|
||||
self.assertEqual(len(selects), 5, selects)
|
||||
self.assertEqual(sum("image_categories" in q for q in selects), 1, selects)
|
||||
# Plus the one stream of tagged image records q and tags read (B3).
|
||||
self.assertEqual(len(selects), 4, selects)
|
||||
self.assertEqual(sum("image_favorites" in q for q in selects), 1, selects)
|
||||
self.assertEqual(sum("media_image_tags" in q for q in selects), 1, selects)
|
||||
|
||||
@@ -1530,16 +1528,15 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
elsewhere = self.api.db.get_prompt_id_for_image("/elsewhere/7.png")
|
||||
self.assertEqual(
|
||||
self.api.db.get_image_link_matches(
|
||||
"", [], every_link=True, wanted=wanted, category="Night"
|
||||
"", "Night", [], every_link=True, wanted=wanted
|
||||
),
|
||||
{
|
||||
normalize_image_path(str(files["night_ab"])): (
|
||||
True,
|
||||
5,
|
||||
self.api.db.get_prompt_id_for_image(str(files["night_ab"])),
|
||||
"Night",
|
||||
),
|
||||
"/elsewhere/7.png": (True, 5, elsewhere, "Night"),
|
||||
"/elsewhere/7.png": (True, 5, elsewhere),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1591,24 +1588,6 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
for i in range(links)
|
||||
],
|
||||
)
|
||||
# 2,000 images (one in 25 at 50,000 files) carry their own category
|
||||
# (#332), so a category filter also reads and matches those.
|
||||
from py.api.images import _favorite_root_key
|
||||
|
||||
root_key = _favorite_root_key(self.output_dir)
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
conn.executemany(
|
||||
"INSERT INTO image_categories VALUES(?,?,?,?)",
|
||||
[
|
||||
(
|
||||
root_key,
|
||||
listed[i][0].relative_to(self.output_dir).as_posix(),
|
||||
f"cat-{i % 10}",
|
||||
"2026-10-27T00:00:00+00:00",
|
||||
)
|
||||
for i in range(1, files, 25)[:2_000]
|
||||
],
|
||||
)
|
||||
return listed
|
||||
|
||||
_BUDGET_REQUESTS = (
|
||||
@@ -1636,7 +1615,7 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
import math
|
||||
|
||||
from py.api import images as images_module
|
||||
from database.operations import link_filter_query, link_matches_json_query
|
||||
from database.operations import link_filter_query
|
||||
|
||||
listed = await asyncio.get_running_loop().run_in_executor(
|
||||
None, self._budget_fixture, 5_000, 2_000
|
||||
@@ -1699,13 +1678,7 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
# q and tags also stream the tagged image records once
|
||||
# (#249 B3); none are tagged here, so nothing is stat'ed.
|
||||
streamed = "q" in params or "tags" in params
|
||||
# A category also reads the own categories, one primary
|
||||
# key range per source (#332).
|
||||
own = "category" in params
|
||||
self.assertEqual(len(found), 1 + streamed + own, found)
|
||||
self.assertEqual(
|
||||
sum("image_categories" in query for query in found), int(own)
|
||||
)
|
||||
self.assertEqual(len(found), 2 if streamed else 1, found)
|
||||
self.assertEqual(
|
||||
sum("media_image_tags" in query for query in found),
|
||||
int(streamed),
|
||||
@@ -1765,42 +1738,13 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
self.assertEqual(self.api._workspace_listed_cache, {})
|
||||
self.assertEqual(self.api._workspace_order_cache, {})
|
||||
# The per-row ambiguity probe is an index search, never a scan.
|
||||
# The read runs it wrapped in one JSON aggregate (#354): same plan.
|
||||
sql, params = link_filter_query("lantern", ["tag-1"], every_link=True)
|
||||
for query in (sql, link_matches_json_query(sql)):
|
||||
with self.subTest(wrapped=query is not sql):
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
plan = [
|
||||
row[3]
|
||||
for row in conn.execute("EXPLAIN QUERY PLAN " + query, params)
|
||||
]
|
||||
self.assertIn(
|
||||
"SEARCH other USING COVERING INDEX idx_image_file_path "
|
||||
"(file_path=?)",
|
||||
plan,
|
||||
)
|
||||
self.assertFalse(
|
||||
[step for step in plan if step.startswith("SCAN other")]
|
||||
)
|
||||
# The category's narrowing clause (#332) keeps every join an index
|
||||
# search, plain and wrapped with the raw categories.
|
||||
sql, params = link_filter_query(
|
||||
"", ["tag-1"], every_link=True, category="cat-3"
|
||||
sql, params = link_filter_query("lantern", "cat-3", ["tag-1"], every_link=True)
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
plan = [row[3] for row in conn.execute("EXPLAIN QUERY PLAN " + sql, params)]
|
||||
self.assertIn(
|
||||
"SEARCH other USING COVERING INDEX idx_image_file_path (file_path=?)", plan
|
||||
)
|
||||
for query in (sql, link_matches_json_query(sql, categories=True)):
|
||||
with self.subTest(category_wrapped=query is not sql):
|
||||
with self.api.db.model.get_connection() as conn:
|
||||
plan = [
|
||||
row[3]
|
||||
for row in conn.execute("EXPLAIN QUERY PLAN " + query, params)
|
||||
]
|
||||
self.assertIn(
|
||||
"SEARCH other USING COVERING INDEX idx_image_file_path "
|
||||
"(file_path=?)",
|
||||
plan,
|
||||
)
|
||||
scans = [step for step in plan if step.startswith("SCAN")]
|
||||
self.assertTrue(all(step.startswith("SCAN gi") for step in scans), plan)
|
||||
self.assertFalse([step for step in plan if step.startswith("SCAN other")])
|
||||
|
||||
@unittest.skipUnless(
|
||||
PERF_BUDGET and not TRACED,
|
||||
|
||||
@@ -1,863 +0,0 @@
|
||||
"""Cached re-runs of runtime-linked prompt text count once per job (#347).
|
||||
|
||||
Jobs are simulated end to end: the queue hook (submit + dequeue), the
|
||||
``execution_cached`` replay, the real node executions and ComfyUI's output
|
||||
cache (fakes of ``execution`` and ``comfy_execution.caching``, as in
|
||||
tests/test_execution_context_host.py).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import types
|
||||
import unittest
|
||||
from collections import namedtuple
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed
|
||||
|
||||
from database.operations import PromptDatabase
|
||||
from utils import cached_runs as module
|
||||
from utils.cached_runs import (
|
||||
CachedRuns,
|
||||
Facts,
|
||||
Role,
|
||||
Seen,
|
||||
graph_role,
|
||||
graph_seen,
|
||||
own_counts,
|
||||
register_cached_runs,
|
||||
stored_facts,
|
||||
)
|
||||
from utils.execution_context_host import cached_string_outputs
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.run_keys import prompt_node_key, upstream_closure, upstream_nodes
|
||||
from utils.usage_tracking import (
|
||||
PendingFirstUse,
|
||||
QueuedRuns,
|
||||
find_pair_id,
|
||||
find_pair_id_by_hashes,
|
||||
handle_queued_prompt,
|
||||
on_dequeue,
|
||||
)
|
||||
|
||||
Entry = namedtuple("CacheEntry", "ui outputs")
|
||||
LLM = "a lighthouse the LLM wrote"
|
||||
|
||||
|
||||
class _Cache:
|
||||
def __init__(self):
|
||||
self.entries = {}
|
||||
|
||||
def get_local(self, node_id):
|
||||
return self.entries.get(node_id)
|
||||
|
||||
|
||||
class HierarchicalCache(_Cache):
|
||||
pass
|
||||
|
||||
|
||||
class LRUCache(_Cache):
|
||||
pass
|
||||
|
||||
|
||||
class RAMPressureCache(LRUCache):
|
||||
pass
|
||||
|
||||
|
||||
class NullCache(_Cache):
|
||||
pass
|
||||
|
||||
|
||||
def host_modules():
|
||||
"""Fake ``execution`` and ``comfy_execution.caching`` for the cache reader."""
|
||||
execution = types.ModuleType("execution")
|
||||
execution.CacheEntry = Entry
|
||||
|
||||
class PromptExecutor:
|
||||
def add_message(self, event, data, broadcast):
|
||||
return None
|
||||
|
||||
execution.PromptExecutor = PromptExecutor
|
||||
caching = types.ModuleType("comfy_execution.caching")
|
||||
for cls in (HierarchicalCache, LRUCache, RAMPressureCache, NullCache):
|
||||
setattr(caching, cls.__name__, cls)
|
||||
package = types.ModuleType("comfy_execution")
|
||||
package.caching = caching
|
||||
return {
|
||||
"execution": execution,
|
||||
"comfy_execution": package,
|
||||
"comfy_execution.caching": caching,
|
||||
}
|
||||
|
||||
|
||||
def graph(
|
||||
kind="PromptManager",
|
||||
text=None,
|
||||
role="positive",
|
||||
sampler=True,
|
||||
negative_text="",
|
||||
seed=1,
|
||||
):
|
||||
"""PrimitiveString -> PreviewAny -> ``kind`` -> KSampler (runtime text).
|
||||
|
||||
``text`` set: typed text instead of the runtime link. ``role`` "negative":
|
||||
the node feeds the sampler's negative input. Pair: its negative editor holds
|
||||
``negative_text`` and it feeds both sampler inputs.
|
||||
"""
|
||||
g = {
|
||||
"str": {"class_type": "PrimitiveString", "inputs": {"value": "seed text"}},
|
||||
"llm": {"class_type": "PreviewAny", "inputs": {"source": ["str", 0]}},
|
||||
"ckpt": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt": "m"}},
|
||||
"pm": {
|
||||
"class_type": kind,
|
||||
"inputs": {"text": ["llm", 0] if text is None else text},
|
||||
},
|
||||
"other": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": "", "clip": ["ckpt", 1]},
|
||||
},
|
||||
"save": {"class_type": "SaveImage", "inputs": {"images": ["ks", 0]}},
|
||||
}
|
||||
if kind != "PromptManagerText":
|
||||
g["pm"]["inputs"]["clip"] = ["ckpt", 1]
|
||||
if kind == "PromptManagerPair":
|
||||
g["pm"]["inputs"]["negative_text"] = negative_text
|
||||
source = ["pm", 0]
|
||||
if kind == "PromptManagerText":
|
||||
g["enc"] = {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": ["pm", 0], "clip": ["ckpt", 1]},
|
||||
}
|
||||
source = ["enc", 0]
|
||||
positive, negative = (source, ["other", 0])
|
||||
if role == "negative":
|
||||
positive, negative = negative, positive
|
||||
if kind == "PromptManagerPair":
|
||||
positive, negative = ["pm", 0], ["pm", 1]
|
||||
if sampler:
|
||||
g["ks"] = {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"model": ["ckpt", 0],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
"seed": seed,
|
||||
},
|
||||
}
|
||||
else:
|
||||
g["save"]["inputs"]["images"] = positive
|
||||
return g
|
||||
|
||||
|
||||
class JobCase(unittest.TestCase):
|
||||
"""A database, the queue hook stashes and one ``CachedRuns`` per test."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.addCleanup(assert_closed, self, self.tmp.name)
|
||||
self.db = PromptDatabase(os.path.join(self.tmp.name, "runs.db"))
|
||||
self.addCleanup(self.db.close_all)
|
||||
patcher = mock.patch.dict(sys.modules, host_modules())
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
self.queued, self.pair_queued = QueuedRuns(), QueuedRuns()
|
||||
self.pending, self.pair_pending = PendingFirstUse(), PendingFirstUse()
|
||||
self.context = (None, None)
|
||||
self.runs = CachedRuns(context=lambda: self.context)
|
||||
self.queue = object()
|
||||
self.executor = types.SimpleNamespace(
|
||||
server=types.SimpleNamespace(prompt_queue=self.queue),
|
||||
caches=types.SimpleNamespace(outputs=HierarchicalCache()),
|
||||
)
|
||||
self.item_id = 0
|
||||
self.trackers = []
|
||||
|
||||
# -- nodes ----------------------------------------------------------------
|
||||
|
||||
def node(self, kind):
|
||||
if kind == "PromptManagerPair":
|
||||
from prompt_manager_pair import PromptManagerPair as cls
|
||||
elif kind == "PromptManagerText":
|
||||
from prompt_manager_text import PromptManagerText as cls
|
||||
else:
|
||||
import prompt_manager
|
||||
|
||||
cls = getattr(prompt_manager, kind)
|
||||
node = cls.__new__(cls)
|
||||
node.db = self.db
|
||||
node.logger = logging.getLogger("test.cached_runs")
|
||||
node.prompt_tracker = mock.Mock()
|
||||
self.trackers.append(node.prompt_tracker)
|
||||
node.comfyui_integration = mock.Mock()
|
||||
node._inject_lora_trigger_words = lambda text: text
|
||||
node.pending_first_use = self.pending
|
||||
node.pending_pair_first_use = self.pair_pending
|
||||
node.cached_runs = self.runs
|
||||
return node
|
||||
|
||||
def execute(self, g, node_id="pm", text=LLM, negative="", index=None):
|
||||
"""Run the node as ComfyUI would; returns its STRING outputs."""
|
||||
kind = g[node_id]["class_type"]
|
||||
node = self.node(kind)
|
||||
self.context = (self.context[0], index)
|
||||
clip = mock.Mock()
|
||||
# The node receives its switch as the queued graph holds it (#90)
|
||||
save = g[node_id]["inputs"].get("save_to_library", True)
|
||||
if kind == "PromptManagerText":
|
||||
return node.process_text(
|
||||
text, prompt=g, unique_id=node_id, save_to_library=save
|
||||
)
|
||||
if kind == "PromptManagerPair":
|
||||
return node.encode_prompt(
|
||||
clip,
|
||||
text,
|
||||
negative_text=negative,
|
||||
prompt=g,
|
||||
unique_id=node_id,
|
||||
save_to_library=save,
|
||||
)
|
||||
return node.encode_prompt(
|
||||
clip, text, prompt=g, unique_id=node_id, save_to_library=save
|
||||
)
|
||||
|
||||
def cache(self, node_id, *results):
|
||||
"""Store executed results (one per list index) as ComfyUI's cache entry."""
|
||||
width = len(results[0])
|
||||
outputs = [[result[slot] for result in results] for slot in range(width)]
|
||||
self.executor.caches.outputs.entries[node_id] = Entry({}, outputs)
|
||||
|
||||
# -- jobs -----------------------------------------------------------------
|
||||
|
||||
def job(self, g, cached=(), run=None, outputs=("save",), prompt_id=None):
|
||||
"""One queued job: hook, dequeue, ``execution_cached``, executions, done."""
|
||||
self.item_id += 1
|
||||
prompt_id = prompt_id or f"job-{self.item_id}"
|
||||
request = handle_queued_prompt(
|
||||
{"prompt": g, "prompt_id": prompt_id},
|
||||
self.queued,
|
||||
pair_queued=self.pair_queued,
|
||||
)
|
||||
on_dequeue(
|
||||
request["prompt_id"],
|
||||
self.queued,
|
||||
lambda: self.db,
|
||||
self.pending,
|
||||
pair_queued=self.pair_queued,
|
||||
pair_pending=self.pair_pending,
|
||||
)
|
||||
self.runs.begin_attempt(
|
||||
((self.item_id, prompt_id, g, {}, list(outputs), {}), self.item_id)
|
||||
)
|
||||
self.context = (prompt_id, None)
|
||||
self.runs.replay(self.executor, prompt_id, list(cached), lambda: self.db)
|
||||
result = run() if run else None
|
||||
self.runs.end_attempt(self.item_id)
|
||||
self.context = (None, None)
|
||||
return result
|
||||
|
||||
def first(self, g, text=LLM, negative="", node_id="pm"):
|
||||
"""A job that executes the node and fills the cache."""
|
||||
result = self.job(
|
||||
g, run=lambda: self.execute(g, node_id, text=text, negative=negative)
|
||||
)
|
||||
self.cache(node_id, result)
|
||||
return result
|
||||
|
||||
def count(self, text):
|
||||
row = self.db.get_prompt_by_hash(generate_prompt_hash(text))
|
||||
return row["run_count"] if row else None
|
||||
|
||||
def pair_count(self, positive, negative):
|
||||
pair_id = find_pair_id(self.db, positive, negative)
|
||||
return self.db.get_prompt_pair(pair_id)["run_count"] if pair_id else None
|
||||
|
||||
|
||||
class TestCachedJobsCount(JobCase):
|
||||
def test_runtime_text_counts_every_cached_job_for_each_node_kind(self):
|
||||
for kind in (
|
||||
"PromptManager",
|
||||
"PromptManagerFocused",
|
||||
"PromptManagerSplit",
|
||||
"PromptManagerText",
|
||||
):
|
||||
with self.subTest(kind=kind):
|
||||
text = f"{LLM} for {kind}"
|
||||
g = graph(kind)
|
||||
self.first(g, text=text)
|
||||
self.job(graph(kind, seed=2), cached=["pm"])
|
||||
self.job(graph(kind, seed=3), cached=["pm"])
|
||||
self.assertEqual(self.count(text), 3)
|
||||
|
||||
def test_typed_text_still_counts_once_per_job(self):
|
||||
g = graph(text="typed castle")
|
||||
self.first(g, text="typed castle")
|
||||
self.job(g, cached=["pm"])
|
||||
self.job(g, cached=["pm"])
|
||||
self.assertEqual(self.count("typed castle"), 3)
|
||||
|
||||
def test_rewired_to_negative_while_cached_counts_nothing(self):
|
||||
self.first(graph())
|
||||
self.job(graph(role="negative"), cached=["pm"])
|
||||
self.assertEqual(self.count(LLM), 1)
|
||||
# Rewired back to positive: the receipt still proves the text
|
||||
self.job(graph(), cached=["pm"])
|
||||
self.assertEqual(self.count(LLM), 2)
|
||||
|
||||
def test_no_sampler_counts_every_node_on_replay(self):
|
||||
g = graph(sampler=False)
|
||||
self.first(g)
|
||||
self.job(g, cached=["pm"])
|
||||
self.assertEqual(self.count(LLM), 2)
|
||||
|
||||
def test_empty_editor_replays_nothing(self):
|
||||
g = graph()
|
||||
self.first(g, text="")
|
||||
self.job(g, cached=["pm"])
|
||||
self.assertEqual(self.db.search_prompts(), [])
|
||||
|
||||
def test_disabled_node_then_cached_jobs_counts_0(self):
|
||||
# #90 x #347: a node with Save to library off records no receipt and is
|
||||
# no run prompt, so its cached jobs replay nothing. The prompt exists
|
||||
# (an enabled run saved it), so any replayed count would show.
|
||||
for kind in (
|
||||
"PromptManager",
|
||||
"PromptManagerFocused",
|
||||
"PromptManagerSplit",
|
||||
"PromptManagerText",
|
||||
"PromptManagerPair",
|
||||
):
|
||||
with self.subTest(kind=kind):
|
||||
text = f"{LLM} not saved by {kind}"
|
||||
negative = "blurry" if kind == "PromptManagerPair" else ""
|
||||
self.first(graph(kind, negative_text=negative), text, negative)
|
||||
before = (self.count(text), self.pair_count(text, negative))
|
||||
|
||||
def off(seed):
|
||||
g = graph(kind, negative_text=negative, seed=seed)
|
||||
g["pm"]["inputs"]["save_to_library"] = False
|
||||
return g
|
||||
|
||||
self.first(off(1), text, negative)
|
||||
self.job(off(2), cached=["pm"])
|
||||
self.job(off(3), cached=["pm"])
|
||||
after = (self.count(text), self.pair_count(text, negative))
|
||||
self.assertEqual(before[0], 1)
|
||||
self.assertEqual(after, before)
|
||||
|
||||
def test_alternating_upstream_values_each_count_their_own_prompt(self):
|
||||
g = graph()
|
||||
self.first(g, text="value A")
|
||||
self.first(g, text="value B")
|
||||
self.cache("pm", self.execute_outputs("value A"))
|
||||
self.job(g, cached=["pm"])
|
||||
self.assertEqual((self.count("value A"), self.count("value B")), (2, 1))
|
||||
|
||||
def execute_outputs(self, text):
|
||||
return (mock.ANY, text)
|
||||
|
||||
def test_list_items_replay_each_item_in_order(self):
|
||||
g = graph()
|
||||
results = []
|
||||
|
||||
def run():
|
||||
for index, text in enumerate(("one", "two", "three")):
|
||||
results.append(self.execute(g, text=text, index=index))
|
||||
|
||||
self.job(g, run=run)
|
||||
self.cache("pm", *results)
|
||||
self.job(g, cached=["pm"])
|
||||
self.assertEqual([self.count(t) for t in ("one", "two", "three")], [2, 2, 2])
|
||||
|
||||
def test_a_repeated_id_or_message_never_counts_a_node_twice(self):
|
||||
g = graph()
|
||||
self.first(g)
|
||||
self.job(g, cached=["pm", "pm"])
|
||||
self.assertEqual(self.count(LLM), 2)
|
||||
self.item_id += 1
|
||||
self.runs.begin_attempt(((1, "again", g, {}, ["save"], {}), self.item_id))
|
||||
for _ in range(2):
|
||||
self.runs.replay(self.executor, "again", ["pm"], lambda: self.db)
|
||||
self.assertEqual(self.count(LLM), 3)
|
||||
|
||||
def test_a_shorter_list_keeps_its_receipt_after_a_longer_one(self):
|
||||
g = graph()
|
||||
self.first(g, text="alpha")
|
||||
results = []
|
||||
|
||||
def run():
|
||||
for index, text in enumerate(("alpha", "beta")):
|
||||
results.append(self.execute(g, text=text, index=index))
|
||||
|
||||
self.job(g, run=run)
|
||||
self.cache("pm", (mock.ANY, "alpha"))
|
||||
self.job(g, cached=["pm"])
|
||||
self.assertEqual((self.count("alpha"), self.count("beta")), (3, 1))
|
||||
|
||||
def test_a_job_reusing_a_prompt_id_counts_and_executes_normally(self):
|
||||
g = graph()
|
||||
self.first(g)
|
||||
self.job(g, cached=["pm"], prompt_id="same")
|
||||
self.job(g, run=lambda: self.execute(g), prompt_id="same")
|
||||
self.job(g, cached=["pm"], prompt_id="same")
|
||||
self.assertEqual(self.count(LLM), 4)
|
||||
|
||||
|
||||
class TestFailClosed(JobCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.g = graph()
|
||||
self.first(self.g)
|
||||
|
||||
def assertNotCounted(self):
|
||||
self.assertEqual(self.count(LLM), 1)
|
||||
|
||||
def test_changed_cached_text_is_not_counted(self):
|
||||
self.cache("pm", (mock.ANY, LLM + " edited"))
|
||||
self.job(self.g, cached=["pm"])
|
||||
self.assertNotCounted()
|
||||
|
||||
def test_no_receipt_in_this_process_is_not_counted(self):
|
||||
self.runs = CachedRuns(context=lambda: self.context)
|
||||
self.job(self.g, cached=["pm"])
|
||||
self.assertNotCounted()
|
||||
|
||||
def test_missing_cache_entry_is_not_counted(self):
|
||||
self.executor.caches.outputs.entries.clear()
|
||||
self.job(self.g, cached=["pm"])
|
||||
self.assertNotCounted()
|
||||
|
||||
def test_unsupported_cache_is_not_counted(self):
|
||||
entries = self.executor.caches.outputs.entries
|
||||
self.executor.caches.outputs = NullCache()
|
||||
self.executor.caches.outputs.entries = entries
|
||||
self.job(self.g, cached=["pm"])
|
||||
self.assertNotCounted()
|
||||
|
||||
def test_more_list_values_than_the_bound_are_not_counted(self):
|
||||
self.cache("pm", *[(mock.ANY, LLM)] * 65)
|
||||
self.job(self.g, cached=["pm"])
|
||||
self.assertNotCounted()
|
||||
|
||||
def test_missing_list_index_is_not_counted(self):
|
||||
self.cache("pm", (mock.ANY, LLM), (mock.ANY, LLM))
|
||||
self.job(self.g, cached=["pm"])
|
||||
self.assertNotCounted()
|
||||
|
||||
def test_node_the_job_does_not_run_is_not_counted(self):
|
||||
self.job(self.g, cached=["pm"], outputs=("other",))
|
||||
self.assertNotCounted()
|
||||
|
||||
def test_message_for_another_job_is_not_counted(self):
|
||||
self.item_id += 1
|
||||
self.runs.begin_attempt(((1, "this", self.g, {}, ["save"], {}), self.item_id))
|
||||
self.runs.replay(self.executor, "another", ["pm"], lambda: self.db)
|
||||
self.assertNotCounted()
|
||||
|
||||
def test_unkeyable_node_is_not_counted(self):
|
||||
g = graph()
|
||||
g["pm"]["inputs"]["extra"] = ["missing", 0]
|
||||
self.job(g, cached=["pm"])
|
||||
self.assertIsNone(prompt_node_key(g, "pm"))
|
||||
self.assertNotCounted()
|
||||
|
||||
def test_deleted_prompt_is_skipped(self):
|
||||
self.db.delete_prompt(
|
||||
self.db.get_prompt_by_hash(generate_prompt_hash(LLM))["id"]
|
||||
)
|
||||
self.job(self.g, cached=["pm"])
|
||||
self.assertIsNone(self.count(LLM))
|
||||
|
||||
def test_database_errors_never_raise(self):
|
||||
def broken():
|
||||
raise RuntimeError("database unavailable")
|
||||
|
||||
self.item_id += 1
|
||||
self.runs.begin_attempt(((1, "j", self.g, {}, ["save"], {}), self.item_id))
|
||||
self.runs.replay(self.executor, "j", ["pm"], broken)
|
||||
self.assertNotCounted()
|
||||
|
||||
def test_malformed_messages_and_results_are_ignored(self):
|
||||
self.runs.begin_attempt(None)
|
||||
self.runs.begin_attempt((("x",), "not an int"))
|
||||
self.runs.replay(self.executor, None, ["pm"], lambda: self.db)
|
||||
self.item_id += 1
|
||||
self.runs.begin_attempt(((1, "j", self.g, {}, ["save"], {}), self.item_id))
|
||||
self.runs.replay(self.executor, "j", "pm", lambda: self.db)
|
||||
self.runs.replay(self.executor, "j", ["pm"] * 501, lambda: self.db)
|
||||
self.runs.replay(self.executor, "j", [7, None], lambda: self.db)
|
||||
self.assertNotCounted()
|
||||
|
||||
|
||||
class TestReplayThenExecution(JobCase):
|
||||
def test_execution_after_replay_in_the_same_job_is_not_counted_again(self):
|
||||
# RAM-pressure frees the entry after execution_cached: the node executes
|
||||
g = graph()
|
||||
self.first(g)
|
||||
self.job(g, cached=["pm"], run=lambda: self.execute(g))
|
||||
self.assertEqual(self.count(LLM), 2)
|
||||
# The suppressed execution still leaves a receipt: the next job counts
|
||||
self.job(g, cached=["pm"])
|
||||
self.assertEqual(self.count(LLM), 3)
|
||||
|
||||
def test_suppression_keeps_the_pending_catch_up_and_the_tracker_queue(self):
|
||||
g = graph()
|
||||
self.first(g)
|
||||
prompt_id = self.db.get_prompt_by_hash(generate_prompt_hash(LLM))["id"]
|
||||
|
||||
def run():
|
||||
# The replay counted, then the prompt was deleted and re-saved
|
||||
self.db.delete_prompt(prompt_id)
|
||||
self.pending.add(generate_prompt_hash(LLM))
|
||||
return self.execute(g)
|
||||
|
||||
self.job(g, cached=["pm"], run=run)
|
||||
self.assertEqual(self.count(LLM), 1) # the catch-up, not the own count
|
||||
push = self.trackers[-1].set_current_prompt.call_args.kwargs["push_to_queue"]
|
||||
self.assertTrue(push)
|
||||
|
||||
def test_without_host_context_the_node_counts_as_before(self):
|
||||
g = graph()
|
||||
self.runs = CachedRuns(context=lambda: (None, None))
|
||||
self.first(g)
|
||||
self.job(g, cached=["pm"])
|
||||
self.job(g, run=lambda: self.execute(g))
|
||||
self.assertEqual(self.count(LLM), 2)
|
||||
|
||||
|
||||
class TestPairReplay(JobCase):
|
||||
def test_runtime_positive_counts_positive_and_pair_every_job(self):
|
||||
g = graph("PromptManagerPair", negative_text="blurry")
|
||||
self.first(g, negative="blurry")
|
||||
self.job(graph("PromptManagerPair", negative_text="blurry", seed=2), ["pm"])
|
||||
self.job(graph("PromptManagerPair", negative_text="blurry", seed=3), ["pm"])
|
||||
self.assertEqual(self.count(LLM), 3)
|
||||
self.assertEqual(self.pair_count(LLM, "blurry"), 3)
|
||||
self.assertEqual(self.count("blurry"), 0)
|
||||
|
||||
def test_empty_negative_pair_replays_its_pair_count(self):
|
||||
g = graph("PromptManagerPair")
|
||||
self.first(g)
|
||||
self.job(g, cached=["pm"])
|
||||
self.assertEqual(self.pair_count(LLM, ""), 2)
|
||||
self.assertEqual(self.count(LLM), 2)
|
||||
|
||||
def test_pair_execution_after_replay_is_not_counted_again(self):
|
||||
g = graph("PromptManagerPair", negative_text="blurry")
|
||||
self.first(g, negative="blurry")
|
||||
self.job(g, ["pm"], run=lambda: self.execute(g, negative="blurry"))
|
||||
self.assertEqual(self.count(LLM), 2)
|
||||
self.assertEqual(self.pair_count(LLM, "blurry"), 2)
|
||||
|
||||
def test_pair_role_follows_the_current_graph(self):
|
||||
g = graph("PromptManagerPair", negative_text="blurry")
|
||||
self.first(g, negative="blurry")
|
||||
rewired = graph("PromptManagerPair", negative_text="blurry")
|
||||
rewired["ks"]["inputs"]["positive"] = ["other", 0]
|
||||
self.job(rewired, cached=["pm"])
|
||||
self.assertEqual(self.count(LLM), 1)
|
||||
self.assertEqual(self.pair_count(LLM, "blurry"), 1)
|
||||
|
||||
|
||||
class TestOwnCounts(unittest.TestCase):
|
||||
"""The node path and replay decide through the same function."""
|
||||
|
||||
def test_single_node_table(self):
|
||||
facts = Facts("p")
|
||||
for used, seen, expected in (
|
||||
({"text"}, False, [("prompt", "p")]),
|
||||
({"text"}, True, []),
|
||||
(set(), False, []),
|
||||
):
|
||||
with self.subTest(used=used, seen=seen):
|
||||
counts = own_counts(
|
||||
"PromptManager", facts, Role(frozenset(used)), Seen(text=seen)
|
||||
)
|
||||
self.assertEqual(counts, expected)
|
||||
self.assertEqual(
|
||||
own_counts("PromptManager", Facts(None), Role(frozenset({"text"})), Seen()),
|
||||
[],
|
||||
)
|
||||
|
||||
def test_pair_table(self):
|
||||
both = Role(frozenset({"text", "negative_text"}), True)
|
||||
self.assertEqual(
|
||||
own_counts("PromptManagerPair", Facts("p", "n"), both, Seen()),
|
||||
[("prompt", "p"), ("prompt", "n"), ("pair", "p", "n")],
|
||||
)
|
||||
self.assertEqual(
|
||||
own_counts("PromptManagerPair", Facts("p", "p"), both, Seen()),
|
||||
[("prompt", "p"), ("pair", "p", "p")],
|
||||
)
|
||||
self.assertEqual(
|
||||
own_counts("PromptManagerPair", Facts("p"), both, Seen(text=True)),
|
||||
[("pair", "p", None)],
|
||||
)
|
||||
self.assertEqual(
|
||||
own_counts(
|
||||
"PromptManagerPair", Facts("p", "n"), both, Seen(True, True, True)
|
||||
),
|
||||
[],
|
||||
)
|
||||
|
||||
def test_node_and_replay_agree_on_role_and_seen(self):
|
||||
for g, node_id in (
|
||||
(graph(), "pm"),
|
||||
(graph(text="typed"), "pm"),
|
||||
(graph(role="negative"), "pm"),
|
||||
(graph(sampler=False), "pm"),
|
||||
(graph("PromptManagerPair", negative_text="n"), "pm"),
|
||||
(graph("PromptManagerPair", text="typed", negative_text="n"), "pm"),
|
||||
):
|
||||
kind = g[node_id]["class_type"]
|
||||
text = "typed" if isinstance(g[node_id]["inputs"]["text"], str) else LLM
|
||||
facts = stored_facts(text, g[node_id]["inputs"].get("negative_text"))
|
||||
with self.subTest(kind=kind, inputs=g[node_id]["inputs"]):
|
||||
replay = own_counts(
|
||||
kind,
|
||||
facts,
|
||||
graph_role(g, node_id, kind),
|
||||
graph_seen(g, node_id, kind, facts),
|
||||
)
|
||||
self.assertEqual(replay, self.node_counts(g, node_id, kind, text))
|
||||
|
||||
@staticmethod
|
||||
def node_counts(g, node_id, kind, text):
|
||||
"""What the executing node counted itself, from a recording database."""
|
||||
db = mock.Mock()
|
||||
db.get_prompt_by_hash.side_effect = lambda h: {"id": h}
|
||||
db.save_prompt_pair.return_value = 1
|
||||
negative = g[node_id]["inputs"].get("negative_text", "")
|
||||
db.get_prompt_pair.return_value = {
|
||||
"positive_prompt_id": generate_prompt_hash(text),
|
||||
"negative_prompt_id": (
|
||||
generate_prompt_hash(negative) if negative.strip() else None
|
||||
),
|
||||
}
|
||||
case = JobCase()
|
||||
case.db = db
|
||||
case.pending, case.pair_pending = PendingFirstUse(), PendingFirstUse()
|
||||
case.runs = CachedRuns(context=lambda: (None, None))
|
||||
case.trackers = []
|
||||
case.context = (None, None)
|
||||
if kind == "PromptManagerPair":
|
||||
node = case.node(kind)
|
||||
node.encode_prompt(
|
||||
mock.Mock(), text, negative_text=negative, prompt=g, unique_id=node_id
|
||||
)
|
||||
else:
|
||||
node = case.node(kind)
|
||||
node._track_prompt_execution(text, text, None, None, {}, g, node_id)
|
||||
counts = [("prompt", c.args[0]) for c in db.record_prompt_use.call_args_list]
|
||||
if kind == "PromptManagerPair":
|
||||
counts += [
|
||||
("pair", generate_prompt_hash(text), db.get_prompt_pair()[k])
|
||||
for k in ["negative_prompt_id"]
|
||||
if db.record_prompt_pair_use.called
|
||||
]
|
||||
return counts
|
||||
|
||||
|
||||
class TestReceipts(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.context = ("j1", None)
|
||||
self.runs = CachedRuns(
|
||||
context=lambda: self.context, max_keys=2, max_receipts=2, max_items=3
|
||||
)
|
||||
self.g = {
|
||||
name: {"class_type": "PromptManager", "inputs": {"text": name}}
|
||||
for name in ("a", "b", "c")
|
||||
}
|
||||
|
||||
def begin(self, item_id, prompt_id="j1"):
|
||||
self.context = (prompt_id, None)
|
||||
self.runs.begin_attempt(
|
||||
((item_id, prompt_id, self.g, {}, ["a", "b", "c"], {}), item_id)
|
||||
)
|
||||
|
||||
def keys(self):
|
||||
return list(self.runs._receipts)
|
||||
|
||||
def test_least_recently_used_run_keys_are_dropped(self):
|
||||
self.begin(1)
|
||||
for name in ("a", "b", "c"):
|
||||
self.runs.record_item(name, (name,), Facts("h"))
|
||||
self.assertEqual(self.keys(), [prompt_node_key(self.g, n) for n in ("b", "c")])
|
||||
|
||||
def test_receipts_per_key_are_bounded_and_equal_outputs_replace(self):
|
||||
key = prompt_node_key(self.g, "a")
|
||||
for item_id, text in ((1, "x"), (2, "y"), (3, "z"), (4, "y")):
|
||||
self.begin(item_id)
|
||||
self.runs.record_item("a", (text,), Facts(text))
|
||||
self.runs.end_attempt(item_id)
|
||||
receipts = self.runs._receipts[key]
|
||||
self.assertEqual([r.item_id for r in receipts], [4, 3])
|
||||
|
||||
def test_list_indices_past_the_bound_are_not_recorded(self):
|
||||
self.begin(1)
|
||||
self.context = ("j1", 3)
|
||||
self.runs.record_item("a", ("x",), Facts("x"))
|
||||
self.assertEqual(self.keys(), [])
|
||||
|
||||
def test_non_string_outputs_and_unknown_nodes_are_not_recorded(self):
|
||||
self.begin(1)
|
||||
self.runs.record_item("a", (None,), Facts("x"))
|
||||
self.runs.record_item("zzz", ("x",), Facts("x"))
|
||||
self.assertEqual(self.keys(), [])
|
||||
|
||||
def test_task_done_clears_only_its_own_attempt(self):
|
||||
self.begin(1)
|
||||
self.runs.end_attempt(2)
|
||||
self.runs.record_item("a", ("x",), Facts("x"))
|
||||
self.runs.end_attempt(1)
|
||||
self.runs.record_item("b", ("x",), Facts("x"))
|
||||
self.assertEqual(self.keys(), [prompt_node_key(self.g, "a")])
|
||||
|
||||
def test_job_nodes_follow_the_outputs_to_execute(self):
|
||||
self.runs.begin_attempt(((1, "j1", self.g, {}, ["b"], {}), 1))
|
||||
self.assertEqual(list(self.runs._attempt.nodes), ["b"])
|
||||
|
||||
|
||||
class TestCacheReader(unittest.TestCase):
|
||||
def setUp(self):
|
||||
patcher = mock.patch.dict(sys.modules, host_modules())
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
self.cache = LRUCache()
|
||||
self.executor = types.SimpleNamespace(
|
||||
caches=types.SimpleNamespace(outputs=self.cache)
|
||||
)
|
||||
|
||||
def read(self, outputs, kind="PromptManager"):
|
||||
self.cache.entries["n"] = Entry({}, outputs)
|
||||
return cached_string_outputs(self.executor, "n", kind)
|
||||
|
||||
def test_reads_each_string_slot_in_order(self):
|
||||
self.assertEqual(self.read([["c"], ["a", "b"]]), (("a", "b"),))
|
||||
self.assertEqual(self.read([["t"]], "PromptManagerText"), (("t",),))
|
||||
self.assertEqual(
|
||||
self.read([[1], [2], ["p"], ["n"]], "PromptManagerPair"), (("p",), ("n",))
|
||||
)
|
||||
|
||||
def test_fails_closed(self):
|
||||
for outputs, kind in (
|
||||
([["c"], ["a", 1]], "PromptManager"),
|
||||
([["c"]], "PromptManager"),
|
||||
([["c"], ["a"] * 65], "PromptManager"),
|
||||
([[1], [2], ["p", "q"], ["n"]], "PromptManagerPair"),
|
||||
([["c"], "text"], "PromptManager"),
|
||||
([["c"], ["a"]], "CLIPTextEncode"),
|
||||
([[]] * 5, "PromptManager"),
|
||||
):
|
||||
with self.subTest(outputs=outputs, kind=kind):
|
||||
self.assertIsNone(self.read(outputs, kind))
|
||||
self.cache.entries["m"] = ({}, [["c"], ["a"]]) # not a CacheEntry
|
||||
self.assertIsNone(cached_string_outputs(self.executor, "m", "PromptManager"))
|
||||
self.executor.caches.outputs = NullCache()
|
||||
self.assertIsNone(self.read([["c"], ["a"]]))
|
||||
|
||||
|
||||
class TestGraphHelpers(unittest.TestCase):
|
||||
def test_upstream_nodes_and_closure(self):
|
||||
g = graph()
|
||||
self.assertEqual(
|
||||
set(upstream_nodes(g, ["save"])),
|
||||
{"save", "ks", "pm", "llm", "str", "ckpt", "other"},
|
||||
)
|
||||
self.assertEqual(upstream_nodes(g, ["missing"]), [])
|
||||
self.assertEqual(upstream_nodes(None, ["save"]), [])
|
||||
self.assertIsNone(upstream_closure(g, ["save"], limit=2))
|
||||
self.assertIsNone(upstream_closure({"a": {"inputs": {"x": ["b", 0]}}}, ["a"]))
|
||||
|
||||
def test_find_pair_by_hashes_matches_the_text_lookup(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = PromptDatabase(os.path.join(tmp, "p.db"))
|
||||
try:
|
||||
pair_id = db.save_prompt_pair("cat", "")
|
||||
self.assertEqual(
|
||||
find_pair_id_by_hashes(db, generate_prompt_hash("cat"), None),
|
||||
pair_id,
|
||||
)
|
||||
self.assertIsNone(
|
||||
find_pair_id_by_hashes(
|
||||
db, generate_prompt_hash("cat"), generate_prompt_hash("x")
|
||||
)
|
||||
)
|
||||
finally:
|
||||
db.close_all()
|
||||
assert_closed(self, tmp)
|
||||
|
||||
|
||||
class TestRegistration(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.modules = host_modules()
|
||||
patcher = mock.patch.dict(sys.modules, self.modules)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
flag = mock.patch.object(module, "_registered", threading.Event())
|
||||
flag.start()
|
||||
self.addCleanup(flag.stop)
|
||||
|
||||
class Queue:
|
||||
def __init__(self):
|
||||
self.done = []
|
||||
|
||||
def get(self, timeout=None):
|
||||
return self.result
|
||||
|
||||
def task_done(self, item_id, *args, **kwargs):
|
||||
self.done.append(item_id)
|
||||
|
||||
def test_registers_once_and_follows_the_job(self):
|
||||
queue = self.Queue()
|
||||
server = types.SimpleNamespace(prompt_queue=queue)
|
||||
runs = mock.Mock()
|
||||
self.assertTrue(register_cached_runs(server, lambda: "db", runs))
|
||||
self.assertFalse(register_cached_runs(server, lambda: "db", runs))
|
||||
queue.result = (("item",), 5)
|
||||
queue.get()
|
||||
runs.begin_attempt.assert_called_once_with((("item",), 5))
|
||||
executor = self.modules["execution"].PromptExecutor()
|
||||
executor.server = server
|
||||
executor.add_message("execution_start", {"prompt_id": "j"}, broadcast=False)
|
||||
executor.add_message(
|
||||
"execution_cached", {"prompt_id": "j", "nodes": ["n"]}, broadcast=False
|
||||
)
|
||||
runs.replay.assert_called_once()
|
||||
self.assertEqual(runs.replay.call_args.args[1:3], ("j", ["n"]))
|
||||
other = self.modules["execution"].PromptExecutor()
|
||||
other.server = types.SimpleNamespace(prompt_queue=object())
|
||||
other.add_message("execution_cached", {"prompt_id": "j", "nodes": []}, False)
|
||||
self.assertEqual(runs.replay.call_count, 1)
|
||||
queue.task_done(5, None, status=None)
|
||||
runs.end_attempt.assert_called_once_with(5)
|
||||
|
||||
def test_hosts_without_the_interfaces_register_nothing(self):
|
||||
class NoTaskDone:
|
||||
def get(self):
|
||||
return None
|
||||
|
||||
self.assertFalse(
|
||||
register_cached_runs(
|
||||
types.SimpleNamespace(prompt_queue=NoTaskDone()), lambda: None
|
||||
)
|
||||
)
|
||||
self.assertFalse(register_cached_runs(types.SimpleNamespace(), lambda: None))
|
||||
del self.modules["execution"].PromptExecutor.add_message
|
||||
self.assertFalse(
|
||||
register_cached_runs(
|
||||
types.SimpleNamespace(prompt_queue=self.Queue()), lambda: None
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,456 +0,0 @@
|
||||
"""The prompt-link read of the workspace Images page (#354).
|
||||
|
||||
``PromptDatabase.get_image_link_matches`` returns every row of its filter
|
||||
SELECT in one SQLite step (one JSON value) instead of one step per row, so the
|
||||
read does not have to win the GIL back once per linked image while ComfyUI
|
||||
generates. Its result must equal the streamed read it replaces, which stays as
|
||||
the fallback, on every filter and on odd values.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database import operations # noqa: E402
|
||||
from database.operations import PromptDatabase, link_filter_query # noqa: E402
|
||||
|
||||
try:
|
||||
from tests.open_handles import assert_closed # noqa: E402
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from open_handles import assert_closed # noqa: E402
|
||||
|
||||
LOGGER = "prompt_manager.database"
|
||||
MISSING_FUNCTION_SQL = (
|
||||
"SELECT pm_missing_function(json_group_array(json_array("
|
||||
"file_path, rating, matched, positive))) FROM ({select})"
|
||||
)
|
||||
QUERIES = ("", "lantern", "LANTERN title", "no-such-text")
|
||||
CATEGORIES = (None, "cat-a", "cat-b", "cat-none")
|
||||
TAG_SETS = ([], ["t1"], ["t1", "t2"], ["t-none"])
|
||||
|
||||
PROMPTS = (
|
||||
# id, text, category, rating, tags, library title
|
||||
(1, "red lantern at dusk", "cat-a", 5, ("t1", "t2"), None),
|
||||
(2, "blue harbor", "cat-a", None, ("t1",), None),
|
||||
(3, "green field", "cat-b", 3, (), "Lantern title"),
|
||||
(4, "pair positive", "cat-b", 4, ("t2",), None),
|
||||
(5, "pair negative lantern", "cat-a", 1, ("t1",), None),
|
||||
)
|
||||
QUOTED = '/out/it\'s "quoted".png'
|
||||
BACKSLASHED = "C:\\out\\back\\slash.png"
|
||||
UNICODE = "/out/ünïcødé 画像.png"
|
||||
NEWLINE = "/out/new\nline.png"
|
||||
IMAGES = (
|
||||
# prompt id, file path
|
||||
(1, "/out/a.png"),
|
||||
(2, QUOTED),
|
||||
(2, BACKSLASHED),
|
||||
(3, UNICODE),
|
||||
(1, NEWLINE),
|
||||
(1, "/out/ambiguous.png"),
|
||||
(2, "/out/ambiguous.png"),
|
||||
(5, "/out/pair.png"), # linked through pair 1: its positive is prompt 4
|
||||
(3, "/out/two-pairs.png"), # two pairs: no single pair, so prompt 3
|
||||
(1, None), # no file path: never listed
|
||||
)
|
||||
|
||||
|
||||
class LinkMatchesTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(os.path.realpath(self.temp.name))
|
||||
self.db = PromptDatabase(str(self.root / "prompts.db"))
|
||||
for name in ("_one_step_unsupported", "_one_step_warned"):
|
||||
patcher = patch.object(operations, name, False)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
self.statements = []
|
||||
original = self.db.model.get_connection
|
||||
|
||||
def traced_connection():
|
||||
connection = original()
|
||||
connection.set_trace_callback(self.statements.append)
|
||||
return connection
|
||||
|
||||
patcher = patch.object(
|
||||
self.db.model, "get_connection", side_effect=traced_connection
|
||||
)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def tearDown(self):
|
||||
self.db.close_all()
|
||||
assert_closed(self, self.root)
|
||||
self.temp.cleanup()
|
||||
|
||||
def seed(self):
|
||||
with self.db.model.get_connection() as conn:
|
||||
for prompt_id, text, category, rating, tags, title in PROMPTS:
|
||||
conn.execute(
|
||||
"INSERT INTO prompts(id,text,category,rating,hash) "
|
||||
"VALUES(?,?,?,?,?)",
|
||||
(prompt_id, text, category, rating, f"hash-{prompt_id}"),
|
||||
)
|
||||
for tag in tags:
|
||||
conn.execute("INSERT OR IGNORE INTO tags(name) VALUES(?)", (tag,))
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_tags(prompt_id,tag_id) "
|
||||
"SELECT ?, id FROM tags WHERE name=?",
|
||||
(prompt_id, tag),
|
||||
)
|
||||
if title:
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_library_state(prompt_id,title) "
|
||||
"VALUES(?,?)",
|
||||
(prompt_id, title),
|
||||
)
|
||||
ids = {}
|
||||
for prompt_id, path in IMAGES:
|
||||
cursor = conn.execute(
|
||||
"INSERT INTO generated_images"
|
||||
"(prompt_id,image_path,filename,file_path) VALUES(?,?,?,?)",
|
||||
(prompt_id, path or "/elsewhere/x.png", "x.png", path),
|
||||
)
|
||||
ids.setdefault(path, cursor.lastrowid)
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) "
|
||||
"VALUES(1,4,5),(2,3,5),(3,3,NULL)"
|
||||
)
|
||||
conn.executemany(
|
||||
"INSERT INTO image_prompt_pairs(image_id,pair_id,provenance) "
|
||||
"VALUES(?,?,'{}')",
|
||||
[
|
||||
(ids["/out/pair.png"], 1),
|
||||
(ids["/out/two-pairs.png"], 2),
|
||||
(ids["/out/two-pairs.png"], 3),
|
||||
],
|
||||
)
|
||||
|
||||
def add_image(self, prompt_id, path):
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO generated_images(prompt_id,image_path,filename,file_path)"
|
||||
" VALUES(?,?,?,?)",
|
||||
(prompt_id, "/blob/x.png", "x.png", path),
|
||||
)
|
||||
|
||||
def streamed(self, q, category, tags, every_link=False, wanted=None):
|
||||
"""Today's read, called directly (the fallback)."""
|
||||
sql, params = link_filter_query(
|
||||
q, tags, every_link=every_link, category=category
|
||||
)
|
||||
with self.db.model.get_connection() as conn:
|
||||
return operations._streamed_link_matches(
|
||||
conn, sql, params, wanted, category is not None
|
||||
)
|
||||
|
||||
def selects(self):
|
||||
found = [s for s in self.statements if s.lstrip().upper().startswith("SELECT")]
|
||||
self.statements.clear()
|
||||
return found
|
||||
|
||||
|
||||
class TestOneStepLinkRead(LinkMatchesTestCase):
|
||||
def test_plain_read_returns_every_unambiguous_link(self):
|
||||
self.seed()
|
||||
matches = self.db.get_image_link_matches("", [], every_link=True)
|
||||
self.assertEqual(
|
||||
matches,
|
||||
{
|
||||
"/out/a.png": (True, 5, 1, None),
|
||||
QUOTED: (True, None, 2, None),
|
||||
BACKSLASHED: (True, None, 2, None),
|
||||
UNICODE: (True, 3, 3, None),
|
||||
NEWLINE: (True, 5, 1, None),
|
||||
"/out/pair.png": (True, 4, 4, None),
|
||||
"/out/two-pairs.png": (True, 3, 3, None),
|
||||
},
|
||||
)
|
||||
self.assertIsNone(matches[QUOTED][1])
|
||||
self.assertIsInstance(matches["/out/a.png"][1], int)
|
||||
self.assertIsInstance(matches["/out/a.png"][2], int)
|
||||
|
||||
def test_search_hits_follow_text_title_and_tags(self):
|
||||
self.seed()
|
||||
self.assertEqual(
|
||||
self.db.get_image_link_matches("lantern", []),
|
||||
{
|
||||
"/out/a.png": (True, 5, 1, None),
|
||||
NEWLINE: (True, 5, 1, None),
|
||||
UNICODE: (True, 3, 3, None),
|
||||
"/out/two-pairs.png": (True, 3, 3, None),
|
||||
},
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.get_image_link_matches("lantern", [], category="cat-a"),
|
||||
{
|
||||
"/out/a.png": (True, 5, 1, "cat-a"),
|
||||
NEWLINE: (True, 5, 1, "cat-a"),
|
||||
QUOTED: (False, None, 2, "cat-a"),
|
||||
BACKSLASHED: (False, None, 2, "cat-a"),
|
||||
},
|
||||
)
|
||||
|
||||
def test_every_filter_equals_the_streamed_read(self):
|
||||
self.seed()
|
||||
wanted_sets = (None, {"/out/a.png", UNICODE, "/out/pair.png", "/absent.png"})
|
||||
for q, category, tags, every_link, wanted in itertools.product(
|
||||
QUERIES, CATEGORIES, TAG_SETS, (False, True), wanted_sets
|
||||
):
|
||||
with self.subTest(
|
||||
q=q, category=category, tags=tags, every=every_link, wanted=wanted
|
||||
):
|
||||
expected = self.streamed(q, category, tags, every_link, wanted)
|
||||
self.statements.clear()
|
||||
actual = self.db.get_image_link_matches(
|
||||
q,
|
||||
list(tags),
|
||||
every_link=every_link,
|
||||
wanted=wanted,
|
||||
category=category,
|
||||
)
|
||||
self.assertEqual(actual, expected)
|
||||
self.assertEqual(
|
||||
{path: tuple(map(type, row)) for path, row in actual.items()},
|
||||
{path: tuple(map(type, row)) for path, row in expected.items()},
|
||||
)
|
||||
found = self.selects()
|
||||
self.assertEqual(len(found), 1, found)
|
||||
self.assertIn("json_group_array", found[0])
|
||||
|
||||
def test_empty_database_returns_nothing_in_one_statement(self):
|
||||
self.assertEqual(self.db.get_image_link_matches("", [], every_link=True), {})
|
||||
self.assertEqual(self.db.get_image_link_matches("x", ["t"], category="c"), {})
|
||||
found = self.selects()
|
||||
self.assertEqual(len(found), 2, found)
|
||||
self.assertTrue(all("json_group_array" in query for query in found))
|
||||
|
||||
def test_wanted_limits_the_result_to_the_listing(self):
|
||||
self.seed()
|
||||
self.assertEqual(
|
||||
self.db.get_image_link_matches(
|
||||
"", [], every_link=True, wanted={"/out/pair.png", "/x.png"}
|
||||
),
|
||||
{"/out/pair.png": (True, 4, 4, None)},
|
||||
)
|
||||
self.assertEqual(
|
||||
self.db.get_image_link_matches("", [], every_link=True, wanted=set()),
|
||||
{},
|
||||
)
|
||||
|
||||
def test_category_read_carries_the_inherited_category_in_one_step(self):
|
||||
# A recipe's own category wins over its positive's (#332 MC4); the
|
||||
# read only narrows to links holding the value, the caller compares.
|
||||
self.seed()
|
||||
with self.db.model.get_connection() as conn:
|
||||
conn.execute("UPDATE prompt_pairs SET category='cat-a' WHERE id=1")
|
||||
self.statements.clear()
|
||||
for category, expected in (
|
||||
("cat-a", {"/out/pair.png": (True, 4, 4, "cat-a")}),
|
||||
("cat-b", {"/out/pair.png": (True, 4, 4, "cat-a")}),
|
||||
):
|
||||
with self.subTest(category=category):
|
||||
actual = self.db.get_image_link_matches(
|
||||
"", [], every_link=True, category=category, wanted={"/out/pair.png"}
|
||||
)
|
||||
self.assertEqual(actual, expected)
|
||||
self.assertEqual(
|
||||
actual, self.streamed("", category, [], True, {"/out/pair.png"})
|
||||
)
|
||||
found = [query for query in self.selects() if "json_group_array" in query]
|
||||
self.assertEqual(len(found), 2, found)
|
||||
self.assertTrue(all("pair_category" in query for query in found))
|
||||
|
||||
def test_plain_read_leaves_the_categories_out_of_the_json(self):
|
||||
self.seed()
|
||||
self.statements.clear()
|
||||
self.db.get_image_link_matches("", [], every_link=True)
|
||||
(query,) = self.selects()
|
||||
aggregate = query[query.index("json_array(") :].split(")", 1)[0]
|
||||
self.assertEqual(aggregate, "json_array(file_path, rating, matched, positive")
|
||||
|
||||
def test_read_releases_its_snapshot(self):
|
||||
# A statement left open after the read would pin its WAL snapshot, and
|
||||
# a TRUNCATE checkpoint from another connection would report busy.
|
||||
self.seed()
|
||||
self.assertTrue(self.db.get_image_link_matches("", [], every_link=True))
|
||||
other = sqlite3.connect(str(self.root / "prompts.db"))
|
||||
try:
|
||||
other.execute("UPDATE prompts SET rating=2 WHERE id=1")
|
||||
other.commit()
|
||||
busy, _log, _done = other.execute(
|
||||
"PRAGMA wal_checkpoint(TRUNCATE)"
|
||||
).fetchone()
|
||||
finally:
|
||||
other.close()
|
||||
self.assertEqual(busy, 0)
|
||||
self.assertEqual(os.path.getsize(self.root / "prompts.db-wal"), 0)
|
||||
|
||||
def test_wrapped_query_keeps_the_ambiguity_probe_an_index_search(self):
|
||||
self.seed()
|
||||
sql, params = link_filter_query(
|
||||
"lantern", ["t1"], every_link=True, category="cat-a"
|
||||
)
|
||||
with self.db.model.get_connection() as conn:
|
||||
plan = [
|
||||
row[3]
|
||||
for row in conn.execute(
|
||||
"EXPLAIN QUERY PLAN "
|
||||
+ operations.link_matches_json_query(sql, categories=True),
|
||||
params,
|
||||
)
|
||||
]
|
||||
self.assertIn(
|
||||
"SEARCH other USING COVERING INDEX idx_image_file_path (file_path=?)", plan
|
||||
)
|
||||
self.assertFalse([step for step in plan if step.startswith("SCAN other")])
|
||||
|
||||
|
||||
class TestStreamedFallback(LinkMatchesTestCase):
|
||||
def test_missing_json_functions_stream_and_stop_trying(self):
|
||||
self.seed()
|
||||
expected = self.streamed("", None, [], True)
|
||||
self.statements.clear()
|
||||
wrapper = operations.link_matches_json_query
|
||||
with (
|
||||
patch.object(operations, "_LINK_MATCHES_JSON_SQL", MISSING_FUNCTION_SQL),
|
||||
patch.object(
|
||||
operations, "link_matches_json_query", side_effect=wrapper
|
||||
) as attempts,
|
||||
self.assertLogs(LOGGER, "WARNING") as logged,
|
||||
):
|
||||
first = self.db.get_image_link_matches("", [], every_link=True)
|
||||
found = self.selects()
|
||||
self.assertEqual(attempts.call_count, 1)
|
||||
second = self.db.get_image_link_matches("", [], every_link=True)
|
||||
again = self.selects()
|
||||
self.assertEqual(first, expected)
|
||||
self.assertEqual(second, expected)
|
||||
self.assertEqual(len(logged.records), 1, logged.output)
|
||||
self.assertIn("no such function: pm_missing_function", logged.output[0])
|
||||
self.assertTrue(operations._one_step_unsupported)
|
||||
# SQLite refuses the wrapped statement before it runs (not traced).
|
||||
self.assertEqual(len(found), 1, found)
|
||||
self.assertNotIn("json_group_array", found[0])
|
||||
# The next read skips the attempt for the rest of the process.
|
||||
self.assertEqual(attempts.call_count, 1)
|
||||
self.assertEqual(len(again), 1, again)
|
||||
self.assertNotIn("json_group_array", again[0])
|
||||
|
||||
def test_blob_path_streams_that_read_and_keeps_trying(self):
|
||||
self.seed()
|
||||
self.add_image(2, b"/out/blob.png")
|
||||
expected = self.streamed("", None, [], True)
|
||||
self.assertEqual(expected[b"/out/blob.png"], (True, None, 2, None))
|
||||
self.statements.clear()
|
||||
with self.assertLogs(LOGGER, "WARNING") as logged:
|
||||
first = self.db.get_image_link_matches("", [], every_link=True)
|
||||
second = self.db.get_image_link_matches("", [], every_link=True)
|
||||
self.assertEqual(first, expected)
|
||||
self.assertEqual(second, expected)
|
||||
self.assertEqual(len(logged.records), 1, logged.output)
|
||||
self.assertFalse(operations._one_step_unsupported)
|
||||
found = self.selects()
|
||||
self.assertEqual(sum("json_group_array" in query for query in found), 2)
|
||||
self.assertEqual(len(found), 4, found)
|
||||
|
||||
def test_blob_that_reads_as_json_still_streams(self):
|
||||
# SQLite 3.45+ reads a BLOB that is valid JSONB as JSON (b"\x00" is
|
||||
# null) instead of refusing it; the streamed read keeps the bytes.
|
||||
self.seed()
|
||||
self.add_image(2, b"\x00")
|
||||
expected = self.streamed("", None, [], True)
|
||||
self.assertIn(b"\x00", expected)
|
||||
with self.assertLogs(LOGGER, "WARNING"):
|
||||
self.assertEqual(
|
||||
self.db.get_image_link_matches("", [], every_link=True), expected
|
||||
)
|
||||
self.assertFalse(operations._one_step_unsupported)
|
||||
|
||||
def test_blob_rating_streams(self):
|
||||
self.seed()
|
||||
with self.db.model.get_connection() as conn:
|
||||
# Only a database written around the CHECK can hold one.
|
||||
conn.execute("PRAGMA ignore_check_constraints=ON")
|
||||
conn.execute("UPDATE prompts SET rating=? WHERE id=3", (b"\x00",))
|
||||
conn.execute("PRAGMA ignore_check_constraints=OFF")
|
||||
expected = self.streamed("", None, [], True)
|
||||
self.assertEqual(expected[UNICODE], (True, b"\x00", 3, None))
|
||||
with self.assertLogs(LOGGER, "WARNING"):
|
||||
self.assertEqual(
|
||||
self.db.get_image_link_matches("", [], every_link=True), expected
|
||||
)
|
||||
|
||||
def test_blob_category_streams_a_category_read(self):
|
||||
self.seed()
|
||||
with self.db.model.get_connection() as conn:
|
||||
# Admitted by its recipe's text category; the positive's is a BLOB.
|
||||
conn.execute("UPDATE prompts SET category=? WHERE id=4", (b"cat-b",))
|
||||
conn.execute("UPDATE prompt_pairs SET category='cat-b' WHERE id=1")
|
||||
expected = self.streamed("", "cat-b", [], True)
|
||||
self.assertEqual(expected["/out/pair.png"], (True, 4, 4, "cat-b"))
|
||||
with self.assertLogs(LOGGER, "WARNING") as logged:
|
||||
actual = self.db.get_image_link_matches(
|
||||
"", [], every_link=True, category="cat-b"
|
||||
)
|
||||
self.assertEqual(actual, expected)
|
||||
self.assertIn("BLOB", logged.output[0])
|
||||
self.assertFalse(operations._one_step_unsupported)
|
||||
|
||||
def test_json_text_over_the_length_limit_streams(self):
|
||||
self.seed()
|
||||
expected = self.streamed("", None, [], True)
|
||||
conn = self.db.model.get_connection()
|
||||
limit = conn.setlimit(sqlite3.SQLITE_LIMIT_LENGTH, 200)
|
||||
try:
|
||||
with self.assertLogs(LOGGER, "WARNING") as logged:
|
||||
actual = self.db.get_image_link_matches("", [], every_link=True)
|
||||
finally:
|
||||
conn.setlimit(sqlite3.SQLITE_LIMIT_LENGTH, limit)
|
||||
self.assertEqual(actual, expected)
|
||||
self.assertIn("too big", logged.output[0])
|
||||
self.assertFalse(operations._one_step_unsupported)
|
||||
|
||||
def test_an_interrupted_read_raises_without_a_fallback(self):
|
||||
# Move library interrupts a retired connection: the read fails as
|
||||
# before, and the one warning is kept for a real fallback.
|
||||
self.seed()
|
||||
conn = self.db.model.get_connection()
|
||||
conn.set_progress_handler(lambda: 1, 1)
|
||||
self.statements.clear()
|
||||
try:
|
||||
with (
|
||||
self.assertNoLogs(LOGGER, "WARNING"),
|
||||
self.assertRaisesRegex(sqlite3.OperationalError, "^interrupted$"),
|
||||
):
|
||||
self.db.get_image_link_matches("", [], every_link=True)
|
||||
finally:
|
||||
conn.set_progress_handler(None, 1)
|
||||
self.assertFalse(operations._one_step_warned)
|
||||
self.assertFalse(operations._one_step_unsupported)
|
||||
self.assertFalse([s for s in self.statements if "json_group_array" not in s])
|
||||
|
||||
def test_the_maintenance_link_rows_reader_is_not_shadowed(self):
|
||||
from database.maintenance import MaintenanceOperations
|
||||
|
||||
self.assertIs(PromptDatabase._link_rows, MaintenanceOperations._link_rows)
|
||||
|
||||
def test_a_failed_streamed_read_still_raises(self):
|
||||
with (
|
||||
patch.object(operations, "_LINK_MATCHES_JSON_SQL", MISSING_FUNCTION_SQL),
|
||||
patch.object(operations, "_LINK_FILTER_SQL", "SELECT broken FROM nowhere"),
|
||||
self.assertLogs(LOGGER, "WARNING"),
|
||||
):
|
||||
with self.assertRaises(sqlite3.OperationalError):
|
||||
self.db.get_image_link_matches("", [], every_link=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,151 +0,0 @@
|
||||
"""Duplicate cleanup keeps images' own categories (#332 MC11).
|
||||
|
||||
The kept copy's own category wins; a kept copy without one receives the
|
||||
removed copy's. The removed copy's row is never deleted, so a give-back
|
||||
(the kept copy changed) and a failed delete lose nothing.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
try:
|
||||
from tests.test_api_duplicate_files import WAIT, DuplicateFilesTestCase
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_duplicate_files import WAIT, DuplicateFilesTestCase
|
||||
|
||||
from py.api import duplicate_files # noqa: E402
|
||||
from py.api.images import image_category_key # noqa: E402
|
||||
from py.config import GalleryConfig # noqa: E402
|
||||
|
||||
|
||||
class TestDuplicateCleanupKeepsCategories(DuplicateFilesTestCase):
|
||||
def key(self, path):
|
||||
return image_category_key(self.api._get_all_output_dirs(), path)
|
||||
|
||||
def category(self, path):
|
||||
return self.api.db.image_category(*self.key(path))
|
||||
|
||||
def set_category(self, path, value):
|
||||
self.api.db.set_image_categories([self.key(path)], value)
|
||||
|
||||
async def run_cleanup(self, keep="a.png"):
|
||||
job, group = await self.found_group()
|
||||
plan = await self.review(job.id, [self.keep_of(group, keep)])
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
return cleanup
|
||||
|
||||
async def test_kept_copy_without_a_category_receives_the_removed_one(self):
|
||||
self.trio()
|
||||
b = self.output_dir / "b.png"
|
||||
self.set_category(b, "Portraits")
|
||||
cleanup = await self.run_cleanup()
|
||||
self.assertFalse(b.exists())
|
||||
self.assertEqual(self.category(self.output_dir / "a.png"), "Portraits")
|
||||
# The removed copy's row stays (hidden while its file is gone).
|
||||
self.assertEqual(self.category(b), "Portraits")
|
||||
self.assertIn(
|
||||
"Kept 1 image category on the kept copies", "\n".join(cleanup.log)
|
||||
)
|
||||
|
||||
async def test_kept_copy_keeps_its_own_category(self):
|
||||
self.trio()
|
||||
self.set_category(self.output_dir / "a.png", "Landscapes")
|
||||
self.set_category(self.output_dir / "b.png", "Portraits")
|
||||
self.set_category(self.output_dir / "sub" / "c.png", "Scans")
|
||||
cleanup = await self.run_cleanup()
|
||||
self.assertEqual(self.category(self.output_dir / "a.png"), "Landscapes")
|
||||
self.assertNotIn("image categor", "\n".join(cleanup.log))
|
||||
|
||||
async def test_two_removed_copies_give_the_first_category(self):
|
||||
self.trio()
|
||||
self.set_category(self.output_dir / "b.png", "Portraits")
|
||||
self.set_category(self.output_dir / "sub" / "c.png", "Scans")
|
||||
cleanup = await self.run_cleanup()
|
||||
self.assertIn(self.category(self.output_dir / "a.png"), {"Portraits", "Scans"})
|
||||
self.assertIn(
|
||||
"Kept 1 image category on the kept copies", "\n".join(cleanup.log)
|
||||
)
|
||||
|
||||
async def test_give_back_loses_no_category(self):
|
||||
self.trio()
|
||||
self.set_category(self.output_dir / "b.png", "Portraits")
|
||||
with patch.object(duplicate_files, "still_kept", return_value=False):
|
||||
cleanup = await self.run_cleanup()
|
||||
self.assertEqual(cleanup.counters["removed"], 0)
|
||||
rows = {row["path"]: row["detail"] for row in cleanup.outcomes["kept"]}
|
||||
self.assertEqual(
|
||||
rows["b.png"], duplicate_files.ROW_DETAILS[duplicate_files.KEEPER_GONE]
|
||||
)
|
||||
self.assertTrue((self.output_dir / "b.png").is_file())
|
||||
self.assertEqual(self.category(self.output_dir / "b.png"), "Portraits")
|
||||
|
||||
async def test_a_failed_delete_keeps_both_categories(self):
|
||||
self.trio()
|
||||
self.set_category(self.output_dir / "b.png", "Portraits")
|
||||
real = duplicate_files._unlink
|
||||
|
||||
def unlink(path):
|
||||
if os.path.basename(path) == "b.png":
|
||||
raise PermissionError(13, "denied")
|
||||
real(path)
|
||||
|
||||
with patch.object(duplicate_files, "_unlink", side_effect=unlink):
|
||||
await self.run_cleanup()
|
||||
self.assertTrue((self.output_dir / "b.png").is_file())
|
||||
self.assertEqual(self.category(self.output_dir / "b.png"), "Portraits")
|
||||
self.assertEqual(self.category(self.output_dir / "a.png"), "Portraits")
|
||||
|
||||
async def test_nested_sources_use_the_deepest_source_keys(self):
|
||||
self.write("sub/a.png", b"same", mtime=1)
|
||||
self.write("sub/b.png", b"same", mtime=2)
|
||||
# The subfolder is listed first, so the cleanup's listed root is the
|
||||
# file key's source; the outer source still holds the files.
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [
|
||||
self._root(self.output_dir / "sub"),
|
||||
self._root(self.output_dir),
|
||||
]
|
||||
sub_key = duplicate_files.favorite_root_key(self.output_dir / "sub")
|
||||
self.api.db.set_image_categories([(sub_key, "b.png")], "Portraits")
|
||||
await self.run_cleanup()
|
||||
self.assertFalse((self.output_dir / "sub" / "b.png").exists())
|
||||
self.assertEqual(self.api.db.image_category(sub_key, "a.png"), "Portraits")
|
||||
outer_key = duplicate_files.favorite_root_key(self.output_dir)
|
||||
self.assertIsNone(self.api.db.image_category(outer_key, "sub/a.png"))
|
||||
|
||||
async def test_outer_source_listed_first_still_keys_by_the_deepest(self):
|
||||
self.write("sub/a.png", b"same", mtime=1)
|
||||
self.write("sub/b.png", b"same", mtime=2)
|
||||
GalleryConfig.MONITORING_DIRECTORIES = [
|
||||
self._root(self.output_dir),
|
||||
self._root(self.output_dir / "sub"),
|
||||
]
|
||||
sub_key = duplicate_files.favorite_root_key(self.output_dir / "sub")
|
||||
self.api.db.set_image_categories([(sub_key, "b.png")], "Portraits")
|
||||
job, group = await self.found_group()
|
||||
[kept] = [f for f in group["files"] if f["path"].endswith("a.png")][:1]
|
||||
plan = await self.review(
|
||||
job.id, [self.keep_of(group, kept["path"], kept["root_index"])]
|
||||
)
|
||||
body = await self.apply(plan["token"])
|
||||
cleanup = self.api._jobs.get(body["job"]["id"])
|
||||
await asyncio.wait_for(asyncio.shield(cleanup.task), WAIT)
|
||||
self.assertFalse((self.output_dir / "sub" / "b.png").exists())
|
||||
self.assertEqual(self.api.db.image_category(sub_key, "a.png"), "Portraits")
|
||||
|
||||
async def test_a_symlinked_output_folder_still_moves_the_category(self):
|
||||
link = self.output_dir.parent / "linked-output"
|
||||
try:
|
||||
os.symlink(self.output_dir, link, target_is_directory=True)
|
||||
except (OSError, NotImplementedError):
|
||||
self.skipTest("symlinks are not available")
|
||||
self.trio()
|
||||
with patch.object(self.api, "_get_all_output_dirs", return_value=[link]):
|
||||
key = image_category_key([link], link / "b.png")
|
||||
self.api.db.set_image_categories([key], "Portraits")
|
||||
await self.run_cleanup()
|
||||
kept = image_category_key([link], link / "a.png")
|
||||
self.assertFalse((self.output_dir / "b.png").exists())
|
||||
self.assertEqual(self.api.db.image_category(*kept), "Portraits")
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user