Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cab776af6e | ||
|
|
093a343b9c | ||
|
|
63ac68c580 | ||
|
|
c5a708a0f5 |
@@ -14,10 +14,6 @@ omit =
|
||||
*/node_modules/*
|
||||
*/static/*
|
||||
*/media/*
|
||||
*/htmlcov/*
|
||||
*/.claude/*
|
||||
restart_gallery.py
|
||||
standalone_tagger.py
|
||||
|
||||
branch = true
|
||||
parallel = true
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
[flake8]
|
||||
max-line-length = 88
|
||||
extend-ignore = E203, W503
|
||||
exclude = reference/, .git, __pycache__, web/, venv/, htmlcov/, build/, dist/, .claude/, docs/, standalone_tagger.py
|
||||
per-file-ignores =
|
||||
tests/*:E402
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"required_status_checks": {
|
||||
"strict": true,
|
||||
"contexts": [
|
||||
"py3.10 on ubuntu-latest",
|
||||
"py3.11 on ubuntu-latest",
|
||||
"py3.12 on ubuntu-latest",
|
||||
"py3.13 on ubuntu-latest",
|
||||
"py3.10 on windows-latest",
|
||||
"py3.11 on windows-latest",
|
||||
"py3.12 on windows-latest",
|
||||
"py3.13 on windows-latest",
|
||||
"py3.10 on macos-latest",
|
||||
"py3.11 on macos-latest",
|
||||
"py3.12 on macos-latest",
|
||||
"py3.13 on macos-latest",
|
||||
"coverage gate (85%)",
|
||||
"black and flake8",
|
||||
"frontend modules",
|
||||
"viewerjs",
|
||||
"real ComfyUI compatibility",
|
||||
"bandit (high severity)"
|
||||
]
|
||||
},
|
||||
"enforce_admins": true,
|
||||
"required_pull_request_reviews": null,
|
||||
"restrictions": null,
|
||||
"allow_force_pushes": false,
|
||||
"allow_deletions": false
|
||||
}
|
||||
@@ -46,15 +46,3 @@ updates:
|
||||
update-types:
|
||||
- "minor"
|
||||
- "patch"
|
||||
|
||||
# Maintainer browser test dependencies; Viewer.js itself has an asset-aware updater.
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/tools/vendor"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
allow:
|
||||
- dependency-name: "@playwright/test"
|
||||
labels:
|
||||
- "dependencies"
|
||||
commit-message:
|
||||
prefix: "deps(vendor-tools)"
|
||||
|
||||
@@ -9,15 +9,42 @@ on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
# Superseded PR pushes cancel their older run. Every main push gets its own
|
||||
# group (run_id), so no main commit's run is ever cancelled or dropped as pending.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-quality-${{ hashFiles('requirements-dev.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-quality-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Check formatting with Black
|
||||
run: black --check --diff .
|
||||
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
# Fail on syntax errors and undefined names
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# Warnings as non-blocking
|
||||
flake8 . --count --exit-zero --max-line-length=88 --statistics --exclude=reference/,.git,__pycache__,web/
|
||||
|
||||
security:
|
||||
name: bandit (high severity)
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
@@ -31,9 +58,5 @@ jobs:
|
||||
- name: Install bandit
|
||||
run: pip install bandit[toml]
|
||||
|
||||
# Medium findings are reported without failing until they are triaged.
|
||||
# High severity is a hard gate.
|
||||
- name: Security report (medium and above, informational)
|
||||
run: bandit -r . -ll -q --exclude ./tests,./reference,./venv,./tools/vendor/node_modules --exit-zero
|
||||
- name: Security scan (high severity gate)
|
||||
run: bandit -r . -lll -q --exclude ./tests,./reference,./venv,./tools/vendor/node_modules
|
||||
- name: Security scan
|
||||
run: bandit -r . -ll --exclude=./tests,./reference,./venv -q || true
|
||||
|
||||
@@ -9,24 +9,12 @@ on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
# Superseded PR pushes cancel their older run. Every main push gets its own
|
||||
# group (run_id), so no main commit's run is ever cancelled or dropped as pending.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
PYTHONFAULTHANDLER: "1"
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
||||
python-version: ["3.10", "3.11", "3.12"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
@@ -35,8 +23,14 @@ jobs:
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('requirements-dev.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-${{ matrix.python-version }}-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
@@ -44,139 +38,13 @@ jobs:
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run tests
|
||||
run: python -m unittest discover -s tests -v
|
||||
|
||||
comfyui:
|
||||
name: real ComfyUI compatibility
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
cache-dependency-path: tools/vendor/package-lock.json
|
||||
- name: Read the reviewed runtime pins
|
||||
id: runtime
|
||||
run: |
|
||||
node --input-type=module <<'JS' >> "$GITHUB_OUTPUT"
|
||||
import runtime from './tools/vendor/comfyui-runtime.json' with { type: 'json' };
|
||||
for (const [key, value] of Object.entries(runtime)) console.log(`${key}=${value}`);
|
||||
import crossPack from './tools/vendor/cross-pack-runtime.json' with { type: 'json' };
|
||||
console.log(`kikoRepository=${crossPack.repository}`);
|
||||
console.log(`kikoRevision=${crossPack.revision}`);
|
||||
JS
|
||||
- name: Check out pinned ComfyUI
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: ${{ steps.runtime.outputs.repository }}
|
||||
ref: ${{ steps.runtime.outputs.revision }}
|
||||
path: .ci-comfyui
|
||||
persist-credentials: false
|
||||
- name: Check out pinned KikoTools for compatibility
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: ${{ steps.runtime.outputs.kikoRepository }}
|
||||
ref: ${{ steps.runtime.outputs.kikoRevision }}
|
||||
path: .ci-kikotools
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ steps.runtime.outputs.pythonVersion }}
|
||||
cache: pip
|
||||
cache-dependency-path: |
|
||||
requirements-dev.txt
|
||||
.ci-comfyui/requirements.txt
|
||||
tools/vendor/comfyui-runtime.json
|
||||
- name: Install CPU runtime and test dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}" --index-url https://download.pytorch.org/whl/cpu
|
||||
pip install -r .ci-comfyui/requirements.txt -r requirements-dev.txt "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}"
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npx --prefix tools/vendor playwright install --with-deps chromium
|
||||
- name: Classic and Nodes 2.0 startup and compatibility
|
||||
env:
|
||||
PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
|
||||
PM_TEST_PYTHON: python
|
||||
run: npm run --prefix tools/vendor test:comfyui
|
||||
- name: Real cached and headless execution history
|
||||
env:
|
||||
PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
|
||||
PM_TEST_PYTHON: python
|
||||
run: npm run --prefix tools/vendor test:executions
|
||||
- name: Independent and combined pack compatibility
|
||||
env:
|
||||
PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
|
||||
PM_KIKO_ROOT: ${{ github.workspace }}/.ci-kikotools
|
||||
PM_TEST_PYTHON: python
|
||||
run: npm run --prefix tools/vendor test:compat
|
||||
- name: Upload real-host reports and startup logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: real-comfyui-reports
|
||||
path: tools/vendor/test-results/
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
coverage:
|
||||
name: coverage gate (85%)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Measure coverage
|
||||
run: |
|
||||
python -m coverage run -m unittest discover -s tests
|
||||
python -m coverage combine
|
||||
python -m coverage report --fail-under=85
|
||||
|
||||
lint:
|
||||
name: black and flake8
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install black flake8
|
||||
- run: black --check .
|
||||
- run: flake8 .
|
||||
run: python -m unittest discover tests/ -v
|
||||
|
||||
js-test:
|
||||
name: frontend modules
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
|
||||
- name: Check shipped extension and page modules parse
|
||||
run: |
|
||||
shopt -s globstar nullglob
|
||||
for f in web/comfy/**/*.js web/comfy/**/*.mjs web/js/**/*.js web/js/**/*.mjs; do
|
||||
node --input-type=module --check < "$f" || exit 1
|
||||
done
|
||||
|
||||
- name: Install pinned shared test contracts
|
||||
run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
|
||||
# Pure-JS frontend modules, tested with the runner's preinstalled Node
|
||||
- name: Run JS tests
|
||||
run: node --test tests/js/*.test.js
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
name: Viewer.js vendor and compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Superseded PR pushes cancel their older run. Every main push gets its own
|
||||
# group (run_id), so no main commit's run is ever cancelled or dropped as pending.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (chromium and touch)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
# tests/vendor/browser-image.test.mjs checks it). Browsers and fonts come from the image.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
# builds, whose entry points name host paths that do not exist in the container. The image has no
|
||||
# ensurepip and the job has no root, so a checksum-pinned pip wheel bootstraps a user-space venv.
|
||||
- name: Python environment
|
||||
env:
|
||||
PIP_WHEEL: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl
|
||||
PIP_SHA256: 71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e
|
||||
run: |
|
||||
venv="$RUNNER_TEMP/venv" wheel="$RUNNER_TEMP/pip-26.2.1-py3-none-any.whl"
|
||||
python3 -m venv --without-pip "$venv"
|
||||
curl -fsSL --retry 3 -o "$wheel" "$PIP_WHEEL"
|
||||
echo "$PIP_SHA256 $wheel" | sha256sum -c -
|
||||
"$venv/bin/python" "$wheel/pip" install --no-index --no-cache-dir "$wheel"
|
||||
"$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npm run --prefix tools/vendor check
|
||||
- run: npm run --prefix tools/vendor test
|
||||
- name: Frontend unit, viewer, workflow, and coverage checks
|
||||
run: npm run --prefix tools/vendor test:coverage
|
||||
- name: Upload frontend coverage and Chromium traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: frontend-test-reports
|
||||
path: |
|
||||
tools/vendor/test-results/
|
||||
coverage/js/
|
||||
!coverage/js/raw/**
|
||||
retention-days: 14
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }})
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
# tests/vendor/browser-image.test.mjs checks it). Browsers and fonts come from the image.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: [firefox, webkit]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
# builds, whose entry points name host paths that do not exist in the container. The image has no
|
||||
# ensurepip and the job has no root, so a checksum-pinned pip wheel bootstraps a user-space venv.
|
||||
- name: Python environment
|
||||
env:
|
||||
PIP_WHEEL: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl
|
||||
PIP_SHA256: 71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e
|
||||
run: |
|
||||
venv="$RUNNER_TEMP/venv" wheel="$RUNNER_TEMP/pip-26.2.1-py3-none-any.whl"
|
||||
python3 -m venv --without-pip "$venv"
|
||||
curl -fsSL --retry 3 -o "$wheel" "$PIP_WHEEL"
|
||||
echo "$PIP_SHA256 $wheel" | sha256sum -c -
|
||||
"$venv/bin/python" "$wheel/pip" install --no-index --no-cache-dir "$wheel"
|
||||
"$venv/bin/python" -m pip install --no-cache-dir -r requirements-dev.txt
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }}
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: browser-traces-${{ matrix.browser }}
|
||||
path: tools/vendor/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Branch protection requires the exact context "viewerjs". This job keeps that
|
||||
# name and fails unless every frontend job above succeeded, including when one
|
||||
# was cancelled or skipped.
|
||||
viewerjs:
|
||||
needs: [frontend-coverage, browser-workflows]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Require every frontend job to succeed
|
||||
env:
|
||||
COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
|
||||
WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
|
||||
run: |
|
||||
echo "frontend-coverage: $COVERAGE_RESULT"
|
||||
echo "browser-workflows: $WORKFLOWS_RESULT"
|
||||
test "$COVERAGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
@@ -1,83 +0,0 @@
|
||||
name: Update vendored Viewer.js
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '30 7 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Stable version to vendor; leave blank for the latest npm release'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: viewerjs-update
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
update:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
actions: write
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '24'
|
||||
cache: npm
|
||||
cache-dependency-path: tools/vendor/package-lock.json
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Update the dependency and regenerate assets
|
||||
env:
|
||||
VIEWER_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
if [ -n "$VIEWER_VERSION" ]; then
|
||||
npm run --prefix tools/vendor update -- --version "$VIEWER_VERSION"
|
||||
else
|
||||
npm run --prefix tools/vendor update
|
||||
fi
|
||||
- name: Detect changes
|
||||
id: changes
|
||||
run: |
|
||||
if git diff --quiet -- tools/vendor/package.json tools/vendor/package-lock.json web/lib/viewerjs; then
|
||||
echo 'changed=false' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'changed=true' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- if: steps.changes.outputs.changed == 'true'
|
||||
run: |
|
||||
npm run --prefix tools/vendor check
|
||||
npm run --prefix tools/vendor test
|
||||
node --test tests/js/*.test.js
|
||||
npx --prefix tools/vendor playwright install --with-deps chromium
|
||||
npm run --prefix tools/vendor test:browser
|
||||
- name: Open the update PR
|
||||
if: steps.changes.outputs.changed == 'true'
|
||||
env:
|
||||
BASE_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
run: |
|
||||
version=$(node -p "require('./web/lib/viewerjs/provenance.json').version")
|
||||
branch="deps/viewerjs-$version"
|
||||
existing=$(gh pr list --repo "$GITHUB_REPOSITORY" --state all --head "$branch" --json number --jq 'length')
|
||||
if [ "$existing" != '0' ]; then
|
||||
echo "A PR already exists for Viewer.js $version; leaving it for review."
|
||||
exit 0
|
||||
fi
|
||||
remote=$(git ls-remote origin "refs/heads/$branch")
|
||||
if [ -z "$remote" ]; then
|
||||
git switch -c "$branch"
|
||||
git add tools/vendor/package.json tools/vendor/package-lock.json web/lib/viewerjs
|
||||
git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' commit -m "deps: update vendored Viewer.js to $version"
|
||||
git push origin "HEAD:refs/heads/$branch"
|
||||
fi
|
||||
gh pr create --repo "$GITHUB_REPOSITORY" --base "$BASE_BRANCH" --head "$branch" --title "deps: update vendored Viewer.js to $version" --body-file tools/vendor/update-pr.md
|
||||
gh workflow run viewerjs-check.yml --repo "$GITHUB_REPOSITORY" --ref "$branch"
|
||||
@@ -9,12 +9,6 @@ AGENTS.md
|
||||
CLAUDE.md
|
||||
|
||||
# Local files
|
||||
# Local PromptManager state (config.json holds API keys and machine paths)
|
||||
config.json
|
||||
resume.sh
|
||||
*.backup_*
|
||||
promptmanager_logs/
|
||||
custom_data/
|
||||
data/
|
||||
docs/
|
||||
logs/
|
||||
@@ -200,14 +194,3 @@ cython_debug/
|
||||
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
# Maintainer-only JavaScript tooling and browser test artifacts
|
||||
node_modules/
|
||||
test-results/
|
||||
playwright-report/
|
||||
|
||||
# Combined frontend coverage reports
|
||||
coverage/
|
||||
|
||||
# Disposable upstream checkout for real-host CI
|
||||
.ci-comfyui/
|
||||
|
||||
@@ -9,7 +9,7 @@ repos:
|
||||
|
||||
- id: flake8
|
||||
name: flake8
|
||||
entry: flake8 --show-source
|
||||
entry: flake8 --select=E9,F63,F7,F82 --show-source --exclude=reference/,web/
|
||||
language: system
|
||||
types: [python]
|
||||
|
||||
|
||||
@@ -139,35 +139,6 @@ The comprehensive gallery system provides complete management of all ComfyUI out
|
||||
|
||||
_Powered by [ViewerJS](https://github.com/fengyuanchen/viewerjs) for professional image viewing capabilities._
|
||||
|
||||
Viewer.js is vendored locally. See [Viewer.js maintenance](tools/vendor/README.md)
|
||||
for reproducible updates, verification, and automated update PRs.
|
||||
|
||||
Prompt Manager appears as **Prompt Manager (Classic)** in the Add Node menu.
|
||||
It keeps the existing layout and behavior while using shared
|
||||
[ComfyAssets Toolkit](https://github.com/ComfyAssets/ComfyAssets-Toolkit) controls
|
||||
that follow ComfyUI's light/dark theme. Existing workflows keep the same internal
|
||||
node identifier and sockets. The reusable architecture is inspired by
|
||||
[Pixaroma](https://github.com/pixaroma/ComfyUI-Pixaroma); no implementation or assets
|
||||
were copied. See the [toolkit maintenance notes](tools/vendor/README.md#shared-comfyassets-browser-controls).
|
||||
|
||||
**Prompt Manager (Focused)** adds an editor-first layout with collapsible
|
||||
composition and metadata fields. Open **Browse library** to search saved prompts,
|
||||
filter by saved category and comma-separated tags, preview a selection, then choose
|
||||
**Use prompt**; **Load metadata** controls whether
|
||||
category and tags come with it. Metadata fields suggest saved categories and tags;
|
||||
choose with the mouse or arrow keys and Enter. Tag selections append `, ` for the
|
||||
next tag; category remains a single value. New values can still be typed freely.
|
||||
Prepend and append STRING sockets remain visible
|
||||
when the fields are collapsed, so LoRA trigger outputs can connect directly.
|
||||
Encoding, storage, and CONDITIONING/text outputs match Classic. Both layouts
|
||||
support ComfyUI Classic and Nodes 2.0.
|
||||
|
||||
**Prompt Manager (Split)** keeps the same editor and library visible together.
|
||||
Wide nodes place the library beside the editor; narrow nodes stack the panels.
|
||||
It shares Focused's category/tag autocomplete, search filters, preview-before-use,
|
||||
workflow persistence, and connectable prepend/append inputs. Encoding and storage
|
||||
match Classic and Focused.
|
||||
|
||||
### 🔬 ComfyUI Workflow Analysis
|
||||
|
||||
- **PNG Metadata Parsing**: Advanced PNG chunk analysis to extract embedded ComfyUI data
|
||||
@@ -193,7 +164,7 @@ match Classic and Focused.
|
||||
|
||||
- **Full-text search** across all stored prompts
|
||||
- **Category filtering** for organized prompt collections
|
||||
- **Tag-based search** with support for multiple tags, with autocomplete that narrows to tags seen together with the ones already typed
|
||||
- **Tag-based search** with support for multiple tags
|
||||
- **Rating filters** to find your best prompts
|
||||
- **Date range filtering** for temporal searches
|
||||
- **Recent prompts** quick access
|
||||
@@ -335,12 +306,9 @@ The comprehensive gallery system provides complete access to all your ComfyUI ou
|
||||
|
||||
Configure your viewing experience in the settings panel:
|
||||
|
||||
- **Display Settings**: Default view, items per page, grid columns, media info, infinite scrolling
|
||||
- **Performance Settings**: Image quality, lazy loading
|
||||
- **Performance Settings**: Grid columns, image quality, lazy loading
|
||||
- **Video Settings**: Autoplay, mute by default, loop videos
|
||||
- **Thumbnail Management**: Generate optimized thumbnails for faster loading (runs on the configured worker threads; a run that stops at the per-run cap tells you how many remain)
|
||||
- **Find Duplicate Images**: Hashes every file on the worker threads with a progress bar; close the window and it keeps running, results wait in a pill
|
||||
- **Rescan Output Folder**: Drops the server-side listing cache and rewalks the folders
|
||||
- **Thumbnail Management**: Generate optimized thumbnails for faster loading
|
||||
|
||||
#### **🖼️ Image Viewing Experience**
|
||||
|
||||
@@ -390,16 +358,14 @@ Use the standalone metadata viewer to analyze any ComfyUI-generated PNG:
|
||||
Import existing ComfyUI images into your database:
|
||||
|
||||
1. **Open admin dashboard** and click **"📸 Scan Images"**
|
||||
2. **Start scan** to analyze every configured output directory
|
||||
3. **Monitor progress** with real-time status updates. The scan runs on the server: closing the window only minimizes it to a progress pill, and reloading the page re-attaches to a running scan
|
||||
2. **Start scan** to analyze your entire output directory
|
||||
3. **Monitor progress** with real-time status updates
|
||||
4. **Review results** showing:
|
||||
- Images found and processed
|
||||
- Prompts extracted and linked
|
||||
- Any errors or issues encountered
|
||||
5. **Access imported data** through the normal gallery interface
|
||||
|
||||
The scan reads PNG text chunks and A1111-style parameters, and, when `ffprobe` is installed, the prompt ComfyUI's video savers embed in MP4/MOV/WebM files. Files are read on the number of **worker threads** set in Settings → Performance (default: half your cores), which makes a large library several times faster.
|
||||
|
||||
### 🏷️ AI AutoTag
|
||||
|
||||
Automatically tag your entire image collection using AI vision models:
|
||||
@@ -531,7 +497,7 @@ Keep your database optimized with built-in maintenance tools:
|
||||
|
||||
### Database Location
|
||||
|
||||
By default, the database is saved as `prompts.db` in the extension directory. It contains your prompt library and image links. See the [database path compatibility note](#database-path-compatibility-211) before upgrading a deployment with a custom path.
|
||||
By default, the database is saved as `example_prompts.db` in the node directory. This file contains all your prompts and linked images and can be backed up or shared.
|
||||
|
||||
### Searching Prompts
|
||||
|
||||
@@ -698,8 +664,7 @@ CREATE TABLE generated_images (
|
||||
- **`utils/diagnostics.py`** - System diagnostics and health checks
|
||||
- **`web/admin.html`** - Advanced admin dashboard with metadata panel
|
||||
- **`web/index.html`** - Simple web interface
|
||||
- **`web/comfy/prompt_manager.js`** - ComfyUI canvas extension (the only file ComfyUI loads)
|
||||
- **`web/js/admin.js`**, **`web/js/gallery.js`** - Dashboard and gallery JavaScript
|
||||
- **`web/js/prompt_manager.js`** - Dashboard JavaScript
|
||||
- **`web/js/tags-page.js`** - Tag management JavaScript
|
||||
- **`web/metadata.html`** - Standalone PNG metadata viewer
|
||||
|
||||
@@ -734,11 +699,8 @@ ComfyUI_PromptManager/
|
||||
│ ├── gallery.html # Image gallery
|
||||
│ ├── index.html # Simple web interface
|
||||
│ ├── metadata.html # Standalone metadata viewer
|
||||
│ ├── comfy/
|
||||
│ │ └── prompt_manager.js # ComfyUI canvas extension (WEB_DIRECTORY)
|
||||
│ └── js/
|
||||
│ ├── admin.js # Dashboard JavaScript
|
||||
│ ├── gallery.js # Gallery JavaScript
|
||||
│ ├── prompt_manager.js # Dashboard JavaScript
|
||||
│ └── tags-page.js # Tag management JavaScript
|
||||
├── tests/
|
||||
│ ├── __init__.py
|
||||
@@ -773,32 +735,6 @@ Relative paths are resolved against the extension directory. Absolute paths are
|
||||
}
|
||||
```
|
||||
|
||||
### Database Path Compatibility (#211)
|
||||
|
||||
PromptManager now honors the `database.default_path` setting in `config.json`
|
||||
when loaded as a ComfyUI custom node. Relative paths resolve against the extension
|
||||
directory, not ComfyUI's working directory; absolute paths are used as-is. Earlier
|
||||
versions could ignore this setting and use `<extension>/prompts.db` instead.
|
||||
Honoring a different configured path can therefore show an empty or different
|
||||
library. PromptManager does not automatically move or merge database files.
|
||||
|
||||
Before upgrading or changing this setting:
|
||||
|
||||
1. Identify the database your current installation actually uses and confirm its
|
||||
library contents. Check both the configured path and the prior fallback
|
||||
`<extension>/prompts.db`; keep track of which existing file you intend to use.
|
||||
2. Preserve a consistent backup of the active database with the dashboard's
|
||||
Backup/Restore backup action or SQLite's online backup API. Do not copy only
|
||||
the main file of a live SQLite database: committed data may still be in its
|
||||
`-wal` file. For a filesystem copy, stop ComfyUI cleanly first and preserve any
|
||||
remaining `-wal` file together with the main database.
|
||||
3. Set `database.default_path` to the existing intended database file. An absolute
|
||||
path makes the selection explicit; do not move or overwrite either database
|
||||
while choosing the path.
|
||||
4. Restart ComfyUI, then verify that the expected library appears and the active
|
||||
database path resolves to the file you selected. Keep the backup until both
|
||||
checks succeed.
|
||||
|
||||
### Gallery & Monitoring Settings
|
||||
|
||||
Configure the automatic image detection and gallery system:
|
||||
@@ -827,15 +763,11 @@ Configure the web interface behavior:
|
||||
# Web UI settings (PromptManagerConfig class)
|
||||
RESULT_TIMEOUT = 5 # Seconds to auto-hide results in ComfyUI node
|
||||
WEBUI_DISPLAY_MODE = 'newtab' # 'popup' or 'newtab' for Web UI button
|
||||
INFINITE_SCROLL = False # Dashboard list loads the next page on scroll by default
|
||||
SHOW_TEST_BUTTON = False # Show API test button in node UI
|
||||
```
|
||||
|
||||
All of these are editable in the dashboard Settings modal and saved to `config.json` (which is private to the installation and holds the CivitAI key, so it is never committed).
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
- **Worker threads** (Settings → Performance): threads used by the image rescan, thumbnail generation and duplicate detection. Default is half the detected cores, maximum is the core count. Pillow releases the GIL while decoding, so the scan and thumbnails scale almost linearly; keep a few cores free while ComfyUI is generating.
|
||||
- The database automatically creates indexes for optimal search performance
|
||||
- Regular `VACUUM` operations keep the database optimized
|
||||
- Consider backing up the database periodically
|
||||
@@ -870,7 +802,7 @@ print(f"Average rating: {info['average_rating']}")
|
||||
# Create backup
|
||||
db.model.backup_database("backup_prompts.db")
|
||||
|
||||
# The online backup includes committed data still in the WAL.
|
||||
# The database file can be copied directly for backup
|
||||
```
|
||||
|
||||
## Development
|
||||
@@ -879,14 +811,9 @@ db.model.backup_database("backup_prompts.db")
|
||||
|
||||
```bash
|
||||
cd ComfyUI_PromptManager
|
||||
pip install -r requirements-dev.txt
|
||||
python -m unittest discover -s tests # Python (what CI runs)
|
||||
node --test tests/js/*.test.js # frontend pure modules
|
||||
python -m coverage run -m unittest discover -s tests && python -m coverage combine && python -m coverage report
|
||||
python -m pytest tests/ -v
|
||||
```
|
||||
|
||||
CI runs the suite on Linux, Windows and macOS for Python 3.10 to 3.12 with an 85% coverage gate. Tests that delete a database file first check that nothing in the process still holds it open (`tests/open_handles.py`), so a connection leak fails on Linux the way it would on Windows.
|
||||
|
||||
### Code Style
|
||||
|
||||
The project follows PEP 8 guidelines with:
|
||||
@@ -947,10 +874,6 @@ MIT License - see LICENSE file for details.
|
||||
|
||||
### Recently Completed
|
||||
|
||||
- **✅ Security & Reliability Hardening (3.3.0)**: validated gallery roots, contained file serving, safe backups and restores, background scans, 85%+ test coverage on all three operating systems
|
||||
- **✅ Worker Threads**: scan, thumbnails and duplicate detection run on a configurable thread pool
|
||||
- **✅ Tag Autocomplete & Infinite Scrolling**: faster browsing in the dashboard and gallery
|
||||
- **✅ Queue-Time Run Counting (3.2.4)**: run counts and image linking follow the ComfyUI queue
|
||||
- **✅ LoRA Manager Integration**: Import LoRA metadata, trigger words, and preview images
|
||||
- **✅ Folder Filter**: Browse and filter prompts by output subdirectory
|
||||
- **✅ Multi-Directory Gallery**: Scan multiple output directories simultaneously
|
||||
@@ -970,31 +893,6 @@ MIT License - see LICENSE file for details.
|
||||
|
||||
## Changelog
|
||||
|
||||
### v3.3.1 (Proxy Routing Fix)
|
||||
|
||||
- **🌐 Proxy support**: the dashboard, gallery, tag page and metadata viewer now work when ComfyUI is reached through a proxy mount such as SwarmUI's `/ComfyBackendDirect/`. Scripts, images, API calls, progress streams and gallery folder filters stay under the mount, and the canvas node's buttons use ComfyUI's `fileURL()`. Direct ComfyUI URLs are unchanged. Thanks to @ghosttigerllc-bit ([#164](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/164))
|
||||
- **📦 Dependencies**: the runtime minimums stay at `Pillow>=8.0.0` and `watchdog>=2.1.0`, so updating Prompt Manager never forces upgrades in the shared ComfyUI environment
|
||||
|
||||
### v3.3.0 (Hardening, Background Jobs & Worker Threads)
|
||||
|
||||
- **🔒 Security**: gallery roots restricted to ComfyUI directories and validated wherever they are read; media-only, fail-closed image serving; Windows path escape in static routes closed; quote-safe HTML escaping with inline handlers moved to data attributes; the CivitAI key is never returned and only sent to HTTPS CivitAI hosts; auto-tag paths contained; side-effect routes are POST-only; backups include the WAL and restores are size-capped, integrity-checked and applied through SQLite's online backup API
|
||||
- **🧵 Worker Threads**: a Settings slider (default half the cores) used by the image rescan, thumbnail generation and duplicate detection. Measured: a 1 GB sample scanned in 2.5 s instead of 18 s at 8 threads
|
||||
- **🔁 Background Jobs**: the image rescan and the duplicate scan run on the server, stream progress, survive a closed tab, minimize to a progress pill and re-attach after a reload
|
||||
- **🎬 Video Prompts**: the scan reads the prompt ComfyUI's video savers embed in MP4/MOV/WebM containers (needs `ffprobe`)
|
||||
- **🖼️ Thumbnails**: written atomically and served under versioned URLs so a regenerated thumbnail never shows a stale cached copy; the per-run cap applies only to thumbnails that still need generating and the gallery says how many remain
|
||||
- **🏷️ Tag Autocomplete**: the dashboard tag box suggests tags, narrowed to those that co-occur with the tags already typed
|
||||
- **♾️ Infinite Scrolling**: optional in the gallery and the dashboard list (saved default plus a session toggle)
|
||||
- **⚙️ Gallery Settings** now apply at load; grid columns, media info and the Rescan button work as labelled
|
||||
- **🐛 Correctness**: run counts increment when ComfyUI dequeues a prompt; negative prompts are never traced; per-thread SQLite connections and idempotent migrations (an old rebuild could delete every image row); `truncate_logs` no longer rolls the log over; the ComfyUI canvas loads only the canvas extension; Windows console banners are ASCII-safe; subfolders use `/` on every platform
|
||||
- **🧪 Tests & CI**: 1,460 Python and 142 JavaScript tests (from 510 and 26), coverage 96% with an 85% gate, CI on Linux, Windows and macOS for Python 3.10–3.12, flake8 clean
|
||||
- **⚠️ API changes**: image paths in responses are relative to the ComfyUI tree (use `url`); `autotag/download`, `autotag/start` and the thumbnail progress stream are POST-only; `lora/status` returns `has_civitai_api_key` instead of the key; `/search` is paginated
|
||||
|
||||
### v3.2.4 (Recently Used Sorting & Queue-Time Run Counts)
|
||||
|
||||
- **🕒 Recently Used Sort**: sort the dashboard by last use, most used or rating, server-side across pages
|
||||
- **🔢 Run Counts**: `run_count` and `last_used_at` are recorded through the ComfyUI queue hook, so cached nodes still count
|
||||
- **🔗 Positive-Prompt Image Linking**: generated images link to the positive prompt traced through the queued graph
|
||||
|
||||
### v3.2.1 (LoRA Manager Integration)
|
||||
|
||||
- **🔗 LoRA Manager Integration**: Import LoRA metadata, trigger words, and CivitAI example images from [ComfyUI-Lora-Manager](https://github.com/willmiao/ComfyUI-Lora-Manager) into your prompt database
|
||||
|
||||
@@ -1,758 +0,0 @@
|
||||
# Testing and merge confidence
|
||||
|
||||
Routine changes should merge after the required checks pass and the diff is
|
||||
reviewed. Add a regression test for each bug; prove it fails before the fix.
|
||||
|
||||
## What runs
|
||||
|
||||
| Layer | Real components | Replaced boundaries |
|
||||
| --- | --- | --- |
|
||||
| Python tests | Domain logic, SQLite, API handlers, filesystem checks | ComfyUI host and ML runtimes where needed |
|
||||
| JavaScript tests | Frontend modules and controller methods | DOM/network fixtures for focused edge cases |
|
||||
| Viewer checks | Production Viewer.js and frontend methods in Chromium desktop/touch | Controlled images and page fixtures |
|
||||
| Real ComfyUI | Pinned CPU host, actual extension, node definitions, widgets, routes, SQLite and real execution lifecycle | Stock UI cases isolate host writes; the execution suite queues deterministic test-only nodes without models |
|
||||
| Page workflows | Production HTML, startup, click handlers, HTTP routes, SQLite, PNG files, thumbnails | ComfyUI host services and the tagging model |
|
||||
|
||||
Page workflows cover save/edit/reload, duplicate-edit rejection, gallery image
|
||||
identity, Tags selection and viewer cleanup, delayed responses, auto-tag review
|
||||
and persistence, and infinite scrolling without replacing existing cards.
|
||||
Data safety workflows download and upload actual SQLite backups, verify restored
|
||||
prompts/tags/image links, reject corrupt uploads without changing live data, and
|
||||
exercise cancellation and network-failure recovery for restores and deletion.
|
||||
Duplicate cleanup verifies only the selected file disappears; prompt deletion
|
||||
verifies the other prompt and original image files survive.
|
||||
They run in Chromium, Firefox, WebKit, and Chromium touch emulation. No GPU,
|
||||
model downloads, running ComfyUI instance, or personal configuration is required.
|
||||
The server binds to localhost and creates disposable directories and a database;
|
||||
its reset endpoint exists only in the test harness.
|
||||
|
||||
## Run locally
|
||||
|
||||
Use a Python environment containing `requirements-dev.txt` and Node.js 24:
|
||||
|
||||
```sh
|
||||
python -m unittest discover -s tests -q
|
||||
npm ci --prefix tools/vendor --ignore-scripts
|
||||
npx --prefix tools/vendor playwright install --with-deps chromium firefox webkit
|
||||
npm run --prefix tools/vendor test
|
||||
npm run --prefix tools/vendor test:coverage
|
||||
npm run --prefix tools/vendor test:workflows -- --project firefox --project webkit
|
||||
```
|
||||
|
||||
`PM_TEST_PYTHON` selects the Python executable; `PM_TEST_PORT` selects the local
|
||||
port (default 8765). Set these environment variables when Python is outside PATH
|
||||
or the port is occupied. Test servers are never reused. On unsupported Linux
|
||||
distributions, browser system dependencies may need manual installation. For a
|
||||
quick browser run, use `test:workflows -- --project chromium`.
|
||||
|
||||
CI runs the browser jobs inside the pinned image
|
||||
`mcr.microsoft.com/playwright:v<@playwright/test version>-noble`
|
||||
(`tests/vendor/browser-image.test.mjs` keeps the tag in step with the lockfile).
|
||||
Concept comparisons measure geometry, and geometry follows the fonts a browser
|
||||
falls back to: the concept names Inter, which neither the image nor a stock
|
||||
runner ships. On a plain `ubuntu-latest` runner the fallback is DejaVu Sans,
|
||||
whose wider metrics make the concept's own Tags table overflow a 390px viewport,
|
||||
so the 390px comparisons are only meaningful in the pinned image. To reproduce CI,
|
||||
run the suites in that image.
|
||||
|
||||
## Managed real ComfyUI
|
||||
|
||||
The shared engine comes from an exact commit of
|
||||
[ComfyAssets Toolkit](https://github.com/ComfyAssets/ComfyAssets-Toolkit), recorded
|
||||
in `tools/vendor/package.json` and its integrity-checked lockfile. The local
|
||||
adapter sets PromptManager's database/configuration and monitoring policy;
|
||||
source discovery, coverage mapping/gates and process lifecycle are shared code.
|
||||
KikoTools remains the reference and is unchanged by this integration.
|
||||
|
||||
Use a **dedicated disposable** ComfyUI checkout at the revision in
|
||||
`tools/vendor/comfyui-runtime.json`. With a separate Python environment, install
|
||||
that checkout's requirements, this repo's `requirements-dev.txt`, and the exact
|
||||
CPU torch/torchvision pins from the JSON (using PyTorch's CPU wheel index).
|
||||
Install Chromium using the Playwright command above, then run:
|
||||
|
||||
```sh
|
||||
PM_COMFY_ROOT=/path/to/disposable/ComfyUI \
|
||||
PM_TEST_PYTHON=/path/to/cpu-env/bin/python \
|
||||
npm run --prefix tools/vendor test:comfyui
|
||||
```
|
||||
|
||||
CI performs this setup automatically. `PM_COMFY_URL` can select another
|
||||
`http://127.0.0.1:PORT/` address (default port 8189). This is an owned managed
|
||||
server: a running host is never reused. Do not point it at your live checkout.
|
||||
The launcher verifies the upstream SHA and clean tracked files, snapshots only
|
||||
tracked PromptManager files, and creates temporary user/input/output/database/
|
||||
configuration/log directories. Personal untracked files are excluded. It runs
|
||||
CPU/offline with only PromptManager enabled and removes owned state on shutdown.
|
||||
The upstream checkout must also be disposable: ComfyUI may initialize local
|
||||
runtime files there. Browser host-settings writes are intercepted; actual
|
||||
PromptManager routes use the disposable host and database.
|
||||
|
||||
`real-comfyui-reports` retains startup logs, version attachments, screenshots and
|
||||
traces for 14 days. Update upstream/frontend/CPU pins together after reviewing
|
||||
and running both startup projects; never replace the SHA with a moving branch.
|
||||
Update the toolkit via an exact reviewed commit archive and commit both npm
|
||||
manifest and lockfile. No toolkit or npm installation is needed by end users.
|
||||
|
||||
## Coverage and failures
|
||||
|
||||
Python CI enforces 85% combined statement/branch coverage. Frontend coverage merges
|
||||
Node V8 unit coverage with Chromium page-workflow coverage through c8. Every
|
||||
`web/js/**/*.js` and `web/js/**/*.mjs` file appears, including untouched files at zero coverage. The
|
||||
controller tests provide V8 filenames so their executed code is counted too.
|
||||
|
||||
`tools/vendor/frontend-coverage-baseline.json` holds a floor for every file and
|
||||
metric. CI rejects regressions, missing measurements, and new files without a
|
||||
reviewed baseline. These floors reflect current coverage, with two percentage
|
||||
points of allowance for runtime/timing differences; they do not claim 80% coverage
|
||||
of the large controllers. New files need at least 80% line coverage when added via
|
||||
the baseline updater. Raise floors with `npm run --prefix tools/vendor
|
||||
coverage:update`; it reruns the suites and cannot lower an existing floor.
|
||||
Review baseline changes like code changes.
|
||||
|
||||
The `viewerjs` check runs on every PR, including backend-only changes. It is an
|
||||
aggregate job that passes only when the `frontend coverage (chromium and touch)`
|
||||
job and both `page workflows (firefox)` and `page workflows (webkit)` jobs
|
||||
succeed; the browsers run on separate runners in parallel. GitHub artifacts
|
||||
named `frontend-test-reports` contain HTML coverage, a JSON summary, and the
|
||||
Chromium/touch failure screenshots/traces; `browser-traces-firefox` and
|
||||
`browser-traces-webkit` hold the other browsers' traces. Open traces with
|
||||
Playwright's `show-trace` tool.
|
||||
|
||||
The real-host suite covers both Classic and Nodes 2.0 startup, switching both
|
||||
ways, actual database prompt selection, execution-text serialization, workflow
|
||||
reload, widget/style cleanup, node schemas and standalone-page opening. It does
|
||||
not certify every existing feature, arbitrary host versions,
|
||||
GPU/model changes, cross-pack installation or visual design changes. Those still
|
||||
need targeted validation.
|
||||
|
||||
## Windows portability preflight
|
||||
|
||||
Before changing paths, SQLite, streams, restore or cleanup:
|
||||
|
||||
- Follow the boundary's existing native canonical/identity contract;
|
||||
`Path.resolve()` alone does not fold Windows case. Keep lexical aliases
|
||||
distinct from resolved targets. Cover mixed case, drive and UNC spellings,
|
||||
plus foreign-namespace denial with an allowed native control.
|
||||
See Python's [normcase documentation](https://docs.python.org/3.13/library/os.path.html#os.path.normcase).
|
||||
- Close every owned database/model connection, worker and stream handle before
|
||||
unlink or `TemporaryDirectory.cleanup()`. Await worker/stream retirement.
|
||||
A SQLite [`with conn` manages transactions, not connection closure](https://docs.python.org/3.13/library/sqlite3.html#how-to-use-the-connection-context-manager),
|
||||
and [`addCleanup` runs after teardown](https://docs.python.org/3.13/library/unittest.html#unittest.TestCase.addCleanup);
|
||||
use explicit `finally` cleanup when teardown removes files first.
|
||||
- Windows can reject unlink of an open file; Unix can remove its directory entry
|
||||
while the handle remains usable. Preserve the shared behavior assertions and
|
||||
exercise the actual native branch, then prove handle release and successful
|
||||
cleanup. Do not hide failures with blanket skips or retries.
|
||||
See Python's [remove/unlink semantics](https://docs.python.org/3.13/library/os.html#os.remove).
|
||||
- Concurrent SQLite tests must not depend on how fast the host commits.
|
||||
Managed connections use WAL with `synchronous=NORMAL` (see
|
||||
`configure_connection`) so commits do not hold the write lock across a
|
||||
flush. Make real overlap deterministic (for example hold the first writer
|
||||
open with `FirstWriterHold`) and give peers that must all succeed the
|
||||
ordinary busy timeout; cross-thread handoffs wait on events with generous
|
||||
ceilings, never a short wall clock (#242).
|
||||
- Run native Windows CI early for these changes and include a Windows impact and
|
||||
evidence note in the PR. All twelve OS/Python combinations and every applicable
|
||||
exact-head check must pass before merge. Portable `ntpath`/`PureWindowsPath`
|
||||
controls do not establish native filesystem, locking or cleanup success.
|
||||
|
||||
## Required merge checks
|
||||
|
||||
`.github/branch-protection.json` records the required checks: all twelve Python/OS
|
||||
combinations, Python coverage, formatting, frontend modules, `viewerjs`, `real ComfyUI compatibility`,
|
||||
and `bandit (high severity)`.
|
||||
The branch must be current with main, and administrators follow the same checks.
|
||||
It does not require an additional human approval.
|
||||
|
||||
A repository administrator applies the configuration after the new workflow is
|
||||
passing, then verifies it:
|
||||
|
||||
```sh
|
||||
gh api --method PUT repos/ComfyAssets/ComfyUI_PromptManager/branches/main/protection \
|
||||
--input .github/branch-protection.json
|
||||
gh api repos/ComfyAssets/ComfyUI_PromptManager/branches/main/protection/required_status_checks
|
||||
```
|
||||
|
||||
Update the manifest whenever required job names change. Keep the frontend check
|
||||
free of path filters so a required check is never silently skipped.
|
||||
|
||||
## Unified workspace
|
||||
|
||||
The opt-in `/prompt_manager/workspace` page opens the native Images workspace.
|
||||
Overview retains real `/prompt_manager/stats` counts. Both use the shipped toolkit. Existing admin, gallery and metadata URLs remain
|
||||
available while replacement workspaces are developed. Browser smoke tests in
|
||||
`tests/browser/workspace.spec.mjs` cover navigation/history, keyboard skip links,
|
||||
mobile layout, theme persistence/fallback, loading/retry and stale responses.
|
||||
The Images cases in `tests/browser/workspace-images.spec.mjs` additionally cover
|
||||
server-side filters, cursor append/retry, Grid/List/Masonry, bounded DOM, keyboard
|
||||
navigation, restored browsing context, Viewer.js cleanup and actual video playback.
|
||||
`tests/test_api_workspace_images.py` checks signed cursors, source identities,
|
||||
legacy compatibility and header-only dimensions. Frontend stream/layout units
|
||||
cover cancellation, malformed responses and bounded visibility lookup. These run
|
||||
automatically in the existing four-browser-project workflow and frontend coverage job. `tests/comfyui/workspace.spec.mjs` verifies the page against the
|
||||
managed host with both renderer startup configurations.
|
||||
|
||||
The shared inline and collection inspector reads the current rooted file's supported
|
||||
embedded records, exact generation facts and recorded or inferred prompt evidence.
|
||||
`tests/browser/full-inspector.spec.mjs` covers original/metadata/workflow downloads,
|
||||
response limits, source changes, nested sending, matched dark/light and mobile
|
||||
geometry, and the loaded-collection viewer. The viewer never fetches another page
|
||||
or preloads all originals. Its explicit slideshow waits 3.5 seconds after image
|
||||
readiness, pauses at native video without advancing on `ended`, and resumes on a
|
||||
manually selected image only while prior intent remains. Pause/Stop, inspector
|
||||
reading/copy/download/send, hidden document, media failure, close, query/route
|
||||
disposal and pagehide cancel that intent; returning visibility or closing the sender
|
||||
does not resume it. Stop leaves native video playback alone. Video uses only the
|
||||
existing normalized `gallerySettings` autoplay/mute/loop values, read without writes;
|
||||
denied/malformed storage falls back to autoplay off, mute on and loop on. Rejected
|
||||
autoplay keeps native controls usable.
|
||||
|
||||
The inspector does not establish historical file identity, infer run membership,
|
||||
edit image tags, export all EXIF, or load/execute a workflow. Saved vocabulary is
|
||||
labelled by its recipe or positive-prompt scope. Limited raw evidence is omitted
|
||||
atomically, with JSON exports unavailable and the rooted original still downloadable.
|
||||
The inspection API uses one bounded read with per-record budgets: role records
|
||||
(`prompt`, `parameters`, the pair execution record) share 2 MiB and exceeding it
|
||||
fails the read; the `workflow` record has its own 4 MiB and past it only the workflow
|
||||
becomes record format `limited` (no raw text, graph `limited`, roles and facts
|
||||
unaffected); other text chunks are skipped unread; byte-identical duplicates are
|
||||
accepted within a bounded read budget, while differing duplicates (and any duplicate
|
||||
role record over that budget) still fail. Other read bounds are unchanged: 8 MiB
|
||||
non-PNG header, 10,000 PNG chunks and a three-second ffprobe. Resolution under
|
||||
`INSPECTION_PROFILE` bounds work, not graph size: 2,000 visits per candidate chain,
|
||||
20,000 in total, 64 link/JSON-container levels, 100,000 resolved characters, and a
|
||||
two 20,000-node ceilings: one over the top-level entries of the API prompt, and one
|
||||
over the workflow graph's UI nodes plus subgraph-definition nodes (past it the
|
||||
workflow graph and its facts become limited). The serialized
|
||||
response stays under 8 MiB by omitting, in order, the workflow graph, then the prompt
|
||||
graph, then all raw records atomically. Retry does not raise these limits. Ingestion
|
||||
(scan, live monitor, admin import) and queue-time run counting keep the default
|
||||
profile, including its 500-node gate. Only `INSPECTION_PROFILE` reads the node
|
||||
semantics table `web/js/inspection-semantics.json` (at most 1 MiB and eight nesting
|
||||
levels, loaded on first use). With it, `SamplerCustomAdvanced` follows its `guider`
|
||||
link (through Reroutes) to `CFGGuider` (positive and negative) or `BasicGuider`
|
||||
(positive only; the negative is `absent`, "Guider has no negative input"). Any
|
||||
other guider, such as `DualCFGGuider`, is `unsupported-node`. Conditioning is
|
||||
traced through the table's pass-throughs (FluxGuidance, ReferenceLatent, the
|
||||
ConditioningSet* nodes, ControlNet apply nodes, InpaintModelConditioning and
|
||||
WanImageToVideo by output slot). `ConditioningZeroOut` is an explicit empty text
|
||||
with source `zeroed-conditioning`. `easy ifElse` with a literal boolean follows that
|
||||
branch only; with a linked boolean both branches are traced, equal texts resolve and
|
||||
different texts make the sampler `ambiguous` with one candidate per choice (at most
|
||||
32, else `work-limit`). ConditioningCombine/Concat/Average resolve only when every
|
||||
part has the same text, otherwise `unresolved` with `composite` naming the node.
|
||||
Results that depend on a forced choice are never memoized. A `FaceDetailer`
|
||||
(`detailer-stage`) is a sampler stage: the samplers reached through its `image`
|
||||
ancestry are compared with it. Equal texts resolve with the detailer as the sampler;
|
||||
different texts are `ambiguous` with `multistage-differs` and one candidate per
|
||||
stage; an unreadable stage makes the chain `unresolved`. Unanchored, a sampler
|
||||
whose output only feeds detailers (or previews) is not a candidate of its own.
|
||||
A sampler's latent ancestry is walked through every link input of non-sampler
|
||||
nodes within the same visit and depth budgets: with no earlier sampler it is a
|
||||
single stage whatever the origin (VAEEncode, EmptyLatentBatch, EmptyFlux2LatentImage,
|
||||
EmptyHunyuanLatentVideo, SetLatentNoiseMask, InpaintModelConditioning or
|
||||
WanImageToVideo slot 2). An earlier sampler (img2img from a decode, hires fix) stays
|
||||
`unresolved` "Multistage sampler ancestry is ambiguous" until stage texts are
|
||||
compared; an unlisted node with sampler-shaped inputs (positive, negative, guider,
|
||||
latent_image, a pipe, video embeds, custom-sampler parts, or a `model` link together
|
||||
with conditioning, neg_conditioning, samples or latent) is `unsupported-node`
|
||||
naming it, here and in a detailer's image ancestry; a latent that is not a link or
|
||||
reaches its own sampler is `malformed`; running out of budget is `work-limit` or
|
||||
`depth-limit`, never single stage. Each sampler's walk runs once per resolution;
|
||||
selector branch runs share it and other candidates are charged its visit cost.
|
||||
Text encoders: `single` encoders (the Qwen image encoders, `TextEncodeQwenImage21`
|
||||
slot 0 `prompt` and slot 1 `negative_prompt`, `KikoTextEncode`) read one string
|
||||
input unchanged. `parts-equal` encoders (CLIPTextEncodeSDXL, Flux, SD3) resolve when
|
||||
every part is identical or exactly one part is non-empty (exact `""`, whitespace is
|
||||
text); otherwise `composite`. String sources reproduce the pinned KJNodes code from
|
||||
literal settings only: `StringConstant` is its `string`; `StringConstantMultiline`
|
||||
with a literal true `strip_newlines` is `replace("\n", "").strip()`;
|
||||
`JoinStringMulti` keeps `string_1` (even empty), appends `string_2`..`string_N`
|
||||
(`N` = `inputcount`) after the literal delimiter unless missing or exactly `""`, and
|
||||
ignores later inputs. A linked setting is `unsupported-node`; a list output
|
||||
(`return_list` true or linked) and `PromptSearchList` are `runtime-value` ("Text is
|
||||
decided at run time"), never one list item. Unverified text nodes (LoRAfromText,
|
||||
RegexReplace, XYZPrompt, ImpactWildcardProcessor) stay `unsupported-node`. Every
|
||||
intermediate text is length-checked and charged to the scanned-character budget. If
|
||||
the table cannot be loaded, graph inspection is `unresolved` ("Inspection node
|
||||
semantics are unavailable") and ingestion is unaffected, because it never imports the table.
|
||||
|
||||
`tools/inspection_corpus.py --root DIR --out JSONL [--workers N] [--baseline JSONL]
|
||||
[--summary TXT] [--code TREE]` measures a media folder read-only against the #240
|
||||
invariants H1 to H8. It runs the inspection route pipeline and the workspace JS facts
|
||||
of `--code` (default: its own tree), opens media only for reading, follows no
|
||||
symlinks, imports no database, and refuses `--out`/`--summary` inside the root or a
|
||||
code tree. Its rows hold statuses, sizes, timings and SHA-256 digests of resolved
|
||||
role texts, never file names or prompt text unless `--with-paths` is given; digests
|
||||
of short prompts can still be guessed, so the JSONL stays local and only the
|
||||
aggregate summary goes into a PR body. Run it on the base commit first and pass that
|
||||
JSONL as `--baseline` for H1 and H8 (2x the baseline p99, never below 50 ms). It
|
||||
exits 1 when an invariant fails, including the `harness` line: a tool exception or
|
||||
facts that were not computed (no Node.js) fail the run. Unreadable or unsupported
|
||||
files are inspector outcomes, not harness errors. Never point it at a live install's folders from an
|
||||
automated job. Installing the Python inspection/source-scope changes requires a
|
||||
maintainer ComfyUI restart. Automated tests use disposable hosts and never restart
|
||||
the running personal ComfyUI.
|
||||
|
||||
Run the focused page checks with:
|
||||
|
||||
```sh
|
||||
npm run --prefix tools/vendor test:workflows -- workspace.spec.mjs workspace-images.spec.mjs
|
||||
node --test tests/js/workspace-*.test.js
|
||||
```
|
||||
|
||||
Use the disposable fixture and managed-host prerequisites above. A maintainer
|
||||
live check needs a ComfyUI restart after adding the Python route, then opening
|
||||
`/prompt_manager/workspace`; the mockup preview server is a separate design tool.
|
||||
|
||||
## Prompt sending and cross-pack compatibility
|
||||
|
||||
`tests/js/prompt-bridge-contracts.test.js` runs the Toolkit's shared protocol
|
||||
contracts against the actual checked-in browser asset. Adapter and lifecycle
|
||||
units cover role isolation, connected inputs, stale previews, callback recovery,
|
||||
serialization normalization and cached-page restoration. Browser tab workflows
|
||||
exercise real BroadcastChannel delivery; `tests/comfyui/prompt-bridge.spec.mjs`
|
||||
checks the real widgets, execution serialization and renderer transitions.
|
||||
|
||||
The existing required real-ComfyUI job also runs `test:compat`: PromptManager
|
||||
alone, pinned KikoTools alone and both together, each starting in Classic and
|
||||
Nodes 2.0. It copies tracked source into disposable hosts, preserves independent
|
||||
configuration and checks edits/reload with both packs present. KikoTools remains
|
||||
on its current embedded foundation; this does not adopt the Toolkit in that pack.
|
||||
|
||||
For local runs, use a clean KikoTools checkout at the revision in
|
||||
`tools/vendor/cross-pack-runtime.json`, plus the managed-host prerequisites above:
|
||||
|
||||
```sh
|
||||
PM_COMFY_ROOT=/path/to/pinned/comfyui PM_KIKO_ROOT=/path/to/pinned/kikotools \
|
||||
npm run --prefix tools/vendor test:compat
|
||||
```
|
||||
|
||||
The compatibility runner reserves loopback ports 8193–8195; `PM_COMPAT_PORT`
|
||||
selects another consecutive unprivileged range. Never run automated tests against
|
||||
the maintainer's running ComfyUI. This JavaScript host extension needs a full
|
||||
browser refresh after deployment; its first slice adds no gallery send controls.
|
||||
|
||||
### Workspace prompt inspection and sending
|
||||
|
||||
`tests/test_api_prompt_inspector.py` covers rooted physical-file identity,
|
||||
execution/graph/parameter evidence, saved associations, ambiguity, empty negatives,
|
||||
read-only behavior, bounded PNG/video metadata and corrupt inputs. Run it with
|
||||
`python -m unittest discover -s tests -p test_api_prompt_inspector.py` in a disposable
|
||||
checkout. `tests/js/prompt-inspector.test.js` and `prompt-sender.test.js` cover
|
||||
request races, explicit destination selection, preview tickets and uncertain
|
||||
acknowledgements. Browser workflows exercise the same controls in Images and
|
||||
Viewer, including nested modal focus, mobile layouts and per-role sending.
|
||||
`tests/comfyui/prompt-sender.spec.mjs` sends through the actual workspace UI into
|
||||
real Classic and Nodes 2.0 widgets and verifies workflow serialization without
|
||||
loading a workflow or queueing generation.
|
||||
|
||||
## Durable execution ledger
|
||||
|
||||
`npm run --prefix tools/vendor test:executions` runs 46 real execution cases
|
||||
(23 each in Classic and Nodes 2.0, retaining all 42 prior execution cases),
|
||||
alongside the existing 80 stock host cases (the original 78 plus the native
|
||||
editor host case) and six pack compatibility cases. Run
|
||||
these suites serially because they share report cleanup and owned host resources. Use the same pinned disposable `PM_COMFY_ROOT`,
|
||||
`PM_TEST_PYTHON` and Playwright browser installation as `test:comfyui`:
|
||||
|
||||
```sh
|
||||
PM_COMFY_ROOT=/path/to/disposable/ComfyUI \
|
||||
PM_TEST_PYTHON=/path/to/cpu-env/bin/python \
|
||||
PLAYWRIGHT_BROWSERS_PATH=/path/to/playwright-browsers \
|
||||
npm run --prefix tools/vendor test:executions
|
||||
```
|
||||
|
||||
This suite reserves `http://127.0.0.1:8196/` and never attaches to an existing
|
||||
server. Each renderer worker launches a CPU/offline host with independent
|
||||
configuration, database, user and output directories. The shared launcher installs
|
||||
tracked snapshots of PromptManager and a tiny temporary Git-tracked fixture pack.
|
||||
Only that owned test pack provides cached text, deliberate failure, an
|
||||
interruptible queue gate, deterministic CLIP/model/latent and CPU tensor inputs,
|
||||
and control routes.
|
||||
It replaces external sampler inference and trigger injection only inside the
|
||||
disposable host; real registered PromptManager classes, queue, executor and cache
|
||||
remain active. Its cache control resets the real executor on its worker to exercise
|
||||
Classic, LRU, RAM-pressure and disabled caching; production has no test nodes or
|
||||
endpoints. Cleanup always releases the gate, stops the host and removes both
|
||||
owned directories. Logs and Playwright failures appear under
|
||||
`tools/vendor/test-results/executions/` and the existing CI artifact upload.
|
||||
|
||||
The request-only case creates no browser or WebSocket client. Other cases verify
|
||||
real `execution_cached` node lists after a failed job, both renderer switches and
|
||||
workflow reload, repeated public UUIDs, interruption, graph rejection, and queued
|
||||
removal before dequeue. Actual host history is checked alongside finite polling
|
||||
of SQLite after the background writer. The read helper uses SQLite read-only
|
||||
mode, exact UUID job/session filters and a 256-row limit; absent ledger schema is
|
||||
explicit, while missing/corrupt databases and other read errors fail the suite.
|
||||
Sensitive diagnostic and extra-data sentinels must never appear in lifecycle
|
||||
rows. Exact prompt fields are deliberately persisted only in context snapshots.
|
||||
The same read-only helper calls the internal `get_execution_contexts` and
|
||||
`get_execution_outputs` readers for requested run IDs and inspects their snapshot
|
||||
identities and current usage counts. Its `outputs` map is keyed by immutable run
|
||||
ID, alongside the existing `rows`, `captures`, `usage` and `snapshotIds` results.
|
||||
|
||||
`execution-outputs.spec.mjs` runs actual stock SaveImage and PreviewImage nodes
|
||||
from tiny CPU tensors. It compares each attempt's persisted descriptors with that
|
||||
attempt's final host history, including A,A,B batches, multiple output nodes,
|
||||
cached locators, reused public UUIDs, PNG metadata disabled, graphs without PM
|
||||
nodes and text-only outputs. The pinned SaveImage returns an IMAGE socket; linking
|
||||
it to the owned failure/gate node orders the partial-run tests explicitly. A held
|
||||
downstream gate proves completed image work does not finalize the run early.
|
||||
Other cases verify real output capture across startup in each renderer, switching
|
||||
both ways and workflow reload. PM usage counts retain their existing per-job
|
||||
behavior. References identify reported locators only: file identity is unverified,
|
||||
new generation and context membership are unknown, and temp references are
|
||||
transient. The stock suite excludes both execution files; discovery retains its
|
||||
80 cases (the original 78 plus the native editor host case) and the six pack
|
||||
cases unchanged.
|
||||
|
||||
For a meaningful historical failure, create a separate disposable detached
|
||||
checkout at `fdba6fd0126a8a40cbd6f3e50ed4c5de8991efd4` and pass its path to
|
||||
`launchExecutionHost({ repository })`. That base also predates the #211 config
|
||||
import fix: after confirming an actual headless job succeeded, use
|
||||
`readExecutionRuns` on its explicitly constructed owned path
|
||||
`host.stateDirectory/custom_nodes/promptmanager/prompts.db`. It reports
|
||||
`schemaAvailable: false` and empty rows, failing the expected succeeded-row
|
||||
assertion. This historical inspection stays outside the ordinary suite, which
|
||||
requires the configured database path. The launcher also accepts
|
||||
`PM_EXECUTION_REPOSITORY`; omit it for the implementation worktree. Stage new
|
||||
files first because snapshots use `git ls-files`. Never reset the implementation
|
||||
checkout or point this harness at a live ComfyUI checkout.
|
||||
|
||||
The package-qualified config import also corrects #211: the configured database
|
||||
path is honored when PromptManager runs as a ComfyUI custom node. Follow the
|
||||
[database path compatibility and safe backup sequence](README.md#database-path-compatibility-211)
|
||||
before restarting an existing deployment.
|
||||
|
||||
The ledger records only new dequeued jobs and observed outcomes. Existing
|
||||
counters, prompts, pairs and image links are preserved, with no reconstructed old
|
||||
runs. Schema v2 migrates automatically when the configured database opens, adding
|
||||
immutable prompt-context tables without backfilling old runs. The internal reader
|
||||
returns `not_captured` for preexisting runs; a new capture can instead be complete,
|
||||
partial, ambiguous or unavailable. Keep the existing database backup/restore
|
||||
procedure: restored history changes storage epoch, and old library navigation
|
||||
hints must never be rebound to unrelated current rows.
|
||||
|
||||
Context cases verify A,A,B as three jobs/two combinations/one positive snapshot
|
||||
when every positive field matches; all five registered PM variants; exact raw
|
||||
whitespace and Unicode; known empty negatives; Text negative roles and crossed
|
||||
pairs; fully cached repeats after trigger configuration changes; successful source
|
||||
text before downstream failure/interruption; unknown cached dynamic raw; unsupported
|
||||
transforms and duplicate list invocation ambiguity. Fresh values use
|
||||
`runtime_return`, cached effective STRING outputs use `host_cache_output`, and
|
||||
submitted literals use graph inference. SQL NULL means unknown; empty text is
|
||||
known. Text is never trimmed or normalized by capture. Cache availability and
|
||||
inferred sampler roles prove neither sampler execution nor consumption, and list
|
||||
indices do not identify generated images. External trigger configuration changes
|
||||
must not rewrite the effective text of an existing cached result. The cache matrix
|
||||
also checks that positive/pair usage increments once per dequeued job while a
|
||||
negative-only component retains its existing zero standalone-use count.
|
||||
|
||||
Fault injection for restore fencing, inherited async capability revocation, queued
|
||||
finalization, epoch and generation revocation, provider-local cache entries, eviction/reset, malformed or
|
||||
oversized payloads and missing host interfaces lives in the focused Python
|
||||
`test_execution*.py` suites. These use disposable storage and mocked host interfaces;
|
||||
they are distinct from the pinned real executor/cache checks above. Run-owned
|
||||
output descriptors do not create gallery image associations or a chronological
|
||||
timeline. Deploying the Python changes
|
||||
requires a ComfyUI restart. After restart, queue three new jobs A,A,B with identical
|
||||
positive fields and two negative recipes; existing counters should advance as usual,
|
||||
and internal context inspection should find three attempts and two combinations.
|
||||
For a live output check after that manual restart:
|
||||
|
||||
- Queue SaveImage and PreviewImage with a batch of two, then submit an unchanged
|
||||
cached repeat and a changed tensor/input under new jobs.
|
||||
- Inspect each run's internal output capture against its final ComfyUI history;
|
||||
cached references may repeat, while the attempts remain distinct.
|
||||
- Switch Classic to Nodes 2.0 and back, reload the workflow and queue again.
|
||||
- Verify prompt/pair counters and gallery links retain their existing behavior;
|
||||
output references alone do not prove file bytes or prompt membership.
|
||||
|
||||
Automated checks use only managed disposable hosts. They never restart or upgrade
|
||||
the personal ComfyUI server.
|
||||
|
||||
## Chronological Library history (#209)
|
||||
|
||||
`#/library/{positive_id}` retains saved negative selection, the full selected
|
||||
negative directly beneath its positive, acknowledged role/pair sending, and one
|
||||
pair-filtered shared gallery. Recorded executions are a separate projection.
|
||||
Selecting an execution, anchor, Raw/Effective mode, or reported occurrence does
|
||||
not select a saved variant or refetch its gallery. A,A,B means three attempts,
|
||||
two frozen contexts, and one exact raw-positive group. Different composition or
|
||||
evidence can produce different frozen contexts despite equal-looking previews.
|
||||
|
||||
The versioned public reads are:
|
||||
|
||||
| Route | Meaning |
|
||||
| --- | --- |
|
||||
| `GET /prompt_manager/library/{positive_id}/history` | Exact frozen raw-positive group; compact chronological windows |
|
||||
| `GET /prompt_manager/history/runs/{run_id}` | Frozen fields, per-field provenance, capture and outcome for a known UUID |
|
||||
| `GET /prompt_manager/history/runs/{run_id}/outputs` | Ordered host-reported occurrences, including repeated locations |
|
||||
| `GET /prompt_manager/history/runs/{run_id}/output/current/{media,thumbnail,inspection}` | Authorized current file at that reported location, using the returned epoch/scope URL |
|
||||
|
||||
Run and output reads accept the returned `epoch`; current-file routes require
|
||||
the returned epoch, current scope and occurrence coordinates. Run integers are
|
||||
decimal strings, preserving SQLite's full signed 64-bit values. Windows and
|
||||
receipt pages default to 40 with a 1–100 limit. Cursors are opaque, authenticated,
|
||||
scope-bound and capped at 4096 bytes. Run JSON has an 8 MiB UTF-8 limit and receipt
|
||||
pages a 1 MiB limit; exceeding a limit fails explicitly instead of truncating
|
||||
exact evidence. The unchanged capture limits remain 16 anchors/256 occurrences.
|
||||
|
||||
A cursor freezes its upper ordering boundary, not an application-wide snapshot:
|
||||
each page has one SQLite read transaction and a count as of that read. Later
|
||||
pages can observe completed capture, deletion or other current changes. Newer
|
||||
runs require Refresh history. Unknown positive raw text cannot enter an exact
|
||||
Library group but remains readable through its known run UUID. Unknown (`null`),
|
||||
known empty (`""`) and whitespace are distinct. Complete-zero reported outputs
|
||||
means the host reported none; missing capture means unknown, and partial or
|
||||
conflicting capture remains incomplete. Receipts prove reported locations, not
|
||||
historical bytes, newly generated files or context-output ownership. Cached
|
||||
attempts can repeat exactly the same receipts.
|
||||
|
||||
Each nonempty history page also returns an authenticated `resume_cursor`, including
|
||||
an exhausted page with no continuation. Re-entry uses `resume` instead of `cursor`
|
||||
to read from that prior upper boundary, then restores at most the prior number of
|
||||
40-run pages. It retains horizontal/vertical scroll and the visible anchor as
|
||||
in-memory visit intent. The API validates epoch and exact saved-text scope again;
|
||||
DTOs and current candidates are never saved as visit intent. Explicit Refresh
|
||||
loads the newest window/count and revalidates the selected run's exact raw-positive
|
||||
membership, context and receipt. A valid selection outside that window has an
|
||||
explicit label and **View selected run** control.
|
||||
|
||||
Every receipt can be selected, including missing/error candidates. **Retry current
|
||||
availability** rereads its existing bounded output page; a reduced response limit
|
||||
may require several reads within that one already loaded page. Earlier rows,
|
||||
context/mode, exact selection and saved gallery remain. Recovery actions use the
|
||||
shared client's strict `smaller-limit`, `reload-outputs`, `retry-read` and
|
||||
`refresh-history` vocabulary. Active inspection epoch/identity failures immediately
|
||||
reach the owning history recovery, clearing stale recorded actions before reload.
|
||||
|
||||
Restore acceptance depends on merged #223: incompatible online storage
|
||||
replacement must safely refuse without harming another owner's WAL/connection.
|
||||
History fencing does not repair unsafe replacement. For accepted replacement,
|
||||
the read transaction ends before a fresh-connection epoch check; the final
|
||||
epoch/generation validation is the publication linearization point. Browser
|
||||
generations also ignore late success/error callbacks after route/scope changes.
|
||||
Old cursors, epoch and current-scope URLs fail with bounded recovery instructions.
|
||||
Restored numeric IDs never repair an original navigation hint. Saved Library
|
||||
details are revalidated separately, and a changed owner requires explicit reload.
|
||||
|
||||
Current media/thumbnail responses use an already-opened, validated canonical
|
||||
file handle, bounded streaming and `Cache-Control: no-store`, including HEAD and
|
||||
range behavior. Inspection is a current metadata observation with pre/post file
|
||||
facts, canonical reauthorization and a fresh scope/epoch check; it is not pinned
|
||||
historical bytes or an OS filesystem snapshot. A deterministic change during the
|
||||
metadata read is rejected. A later request can legitimately observe replacement
|
||||
bytes at the same path without changing the immutable receipt.
|
||||
|
||||
The focused acceptance command retains all four projects:
|
||||
|
||||
```sh
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-history.spec.mjs workspace-visual.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
npm run --prefix tools/vendor test:executions
|
||||
npm run --prefix tools/vendor test:comfyui
|
||||
npm run --prefix tools/vendor test:compat
|
||||
```
|
||||
|
||||
Use the pinned disposable prerequisites above; managed suites run serially.
|
||||
The real execution helpers compare SQLite captures to these public routes for
|
||||
stock SaveImage/PreviewImage batches, multiple nodes, metadata disabled, cached
|
||||
headless repeats, partial failure and interruption. Additional cases verify the
|
||||
actual PM A,A,B group, unknown-positive exclusion, and exact frozen Raw/Effective
|
||||
pair sends through the acknowledged host bridge after Classic/Nodes 2.0 startup,
|
||||
both switches and workflow reload, without queuing or loading a workflow on send.
|
||||
All original 42 execution, 78 stock-host and six compatibility cases remain.
|
||||
|
||||
The browser fixture uses actual APIs/SQLite/files, fixed owned backup snapshots,
|
||||
one-shot bounded publication barriers and fixed filesystem/configuration commands.
|
||||
Reset and cleanup release barriers, restore seams and revoke commands. It never
|
||||
attaches to a personal host. The canonical HTML and its six required assets are
|
||||
served from a fixed test-only allowlist; the original concept index/app/style
|
||||
bytes are hash-pinned and unmodified. At 1600×1100 and 390×844 in both themes,
|
||||
with fixed locale/UTC/reduced motion and ready fonts, tests capture unmasked
|
||||
concept/production PNG pairs and geometry, and pin actual branch padding,
|
||||
surfaces, 2px role lines, 57px rhythm, staggering, colors, selected focus and native
|
||||
dialog behavior. Controls and actions are checked independently of screenshots.
|
||||
Keyboard focus/return, coarse targets, 200% layout, route return and persisted
|
||||
page lifecycle recovery retain their separate behavioral tests.
|
||||
|
||||
Initial production acceptance requires maintainer inspection of paired PNGs and
|
||||
geometry; never update snapshots automatically or mask content to bless a gap.
|
||||
Data-dependent text/images/counts and the clearly labeled saved/frozen/current
|
||||
distinctions are documented in the comparison helper. Broader Images card
|
||||
activation/Info/tag/favorite/select parity and shared Viewer stage/toolbar polish
|
||||
and shared role-block spacing/padding/glyph polish remain outside #209. The
|
||||
responsive production Viewer keeps its toolbar in the media panel and prompt details readable rather than reproducing the concept's
|
||||
mobile toolbar collision. These are explicit remaining visual differences, not
|
||||
a whole-application or full-gallery/Viewer pixel-parity claim.
|
||||
|
||||
Preserve raw logs, failed traces, paired PNGs, geometry, runtime versions and
|
||||
source hashes before later managed commands clear their output directories.
|
||||
Frontend baseline updates are additive through `coverage:update` only after
|
||||
every previous metric floor and each new file's 80% line requirement pass; rerun
|
||||
the coverage gate using the committed baseline. Python retains its 85% combined
|
||||
statement/branch gate. Deploying the history Python changes requires a maintainer
|
||||
ComfyUI restart; automated acceptance never restarts the personal server.
|
||||
|
||||
## Native recipe editor (#234)
|
||||
|
||||
The Library page edits recipes in one native modal (`dialog.library-editor`)
|
||||
instead of sending users to the legacy dashboard. It is frontend only: it consumes
|
||||
the Library/pair routes, epoch tokens, null-clears and conflict codes defined by
|
||||
the backend change (#233) and does not add or change a route, so this section does
|
||||
not repeat them. The editor reads fresh, `no-store` state before it opens.
|
||||
|
||||
The four dialog modes match the approved concept's shell: **New prompt**,
|
||||
**Edit recipe metadata**, **Create text revision** and **Add a negative variant**.
|
||||
Pair **Archive…** and **Restore** are one reviewed action with **Show archived**
|
||||
in the saved-variants list; archiving a pair never touches its positive prompt, its
|
||||
siblings, history or images. **Export** downloads the full v1 recipe export. A
|
||||
failed Save keeps focus inside the dialog and keeps every entry.
|
||||
|
||||
Recovery is exercised against real failure responses, not mocks of the controller:
|
||||
|
||||
- a **conflict** response keeps the draft and explains the newer saved state; a
|
||||
**storage-changed** response writes nothing, and only New prompt re-reads in
|
||||
place: every other dialog offers only **Reload Library**, because an old numeric
|
||||
owner is never resolved after a restore;
|
||||
- an **unknown outcome** (no response) keeps the draft and offers an explicit
|
||||
**Retry** that resends the identical request once, which the server
|
||||
deduplicates; an Edit metadata retry instead reloads the latest saved values for
|
||||
review and resends nothing;
|
||||
- a **confirmed receipt** whose follow-up refresh fails offers a read-only
|
||||
refresh retry and never re-sends the write.
|
||||
|
||||
The concept comparison (`library-editor-canonical-comparison.mjs`) inventories the
|
||||
concept dialog and the production dialog in the same browser: geometry, CSS tokens,
|
||||
colors and a control inventory (labels, field types, options, actions and their
|
||||
danger/primary classes). Only the documented differences (the prototype's Save
|
||||
label and demo note) are mapped. The mutation check proves the comparison fails when
|
||||
a token, label or control is changed, so it cannot pass vacuously.
|
||||
|
||||
PNG baselines are intentionally absent: `toHaveScreenshot` is not used for the
|
||||
editor. Paired concept/production captures and the artifact-only states
|
||||
(`library-editor-visual.spec.mjs`) are attached as test artifacts for maintainer
|
||||
acceptance; a baseline is added only after the maintainer accepts them.
|
||||
|
||||
Focused commands:
|
||||
|
||||
```sh
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-library-editor.spec.mjs library-editor-visual.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
npm run --prefix tools/vendor test:comfyui # library-sender.spec.mjs host regression
|
||||
```
|
||||
|
||||
`native editor preview sends exact saved roles across renderer transitions` opens
|
||||
the real workspace, creates a prompt with an explicitly empty negative, and sends the
|
||||
selected pair through the acknowledged host bridge in Classic and Nodes 2.0, across
|
||||
both switches and a workflow reload. It asserts the actual widgets and serialized
|
||||
workflow, no queue or workflow load, unchanged history and pair metadata, and that
|
||||
a connected destination rejects the preview without changing either role. It needs
|
||||
the pinned disposable ComfyUI, so it runs in CI's "real ComfyUI compatibility" job.
|
||||
|
||||
Deploying this change needs no Python change and no ComfyUI restart: the diff for
|
||||
the editor UI touches only HTML, CSS and JavaScript (plus test fixtures). A browser
|
||||
refresh is enough. New editor modules have additive coverage floors in
|
||||
`tools/vendor/frontend-coverage-baseline.json`; none were lowered.
|
||||
|
||||
## Native vocabulary backend (#236)
|
||||
|
||||
Routes under `/prompt_manager/vocabulary/`: `GET tags` and `GET categories`
|
||||
(bounded pages, signed epoch-scoped cursors), `POST tags` (create or reuse an exact
|
||||
name), `PATCH tags/{id}` (rename), `POST tags/{id}/delete`, `GET tags/{id}/impact`,
|
||||
`POST categories/preview` and `POST categories/apply`. Every command carries
|
||||
`expected_epoch`; tag commands also carry the exact `{id, name}` identity, and
|
||||
category apply carries the preview's `impact_token`.
|
||||
|
||||
Error contract: command validation failures are 400 with `code: null`; reads
|
||||
return `{success: false, error}` with no `code`, and impact on an unknown id is 404. A stale epoch is 409
|
||||
`storage-changed`; a changed or missing identity, or a changed category projection,
|
||||
is 409 `vocabulary-conflict` with a generic message. Both 409 bodies include the
|
||||
current `library_epoch`. Nothing is written on any refusal.
|
||||
|
||||
Count semantics: a count is the number of eligible Library parents carrying the value
|
||||
directly or through one active saved pair, exactly what `list_library(tags=[name])` or
|
||||
`list_library(category=value)` returns. Unused registry tags report 0 and are kept by
|
||||
`get_all_tags`, `suggest_tags`, backup and restore, duplicate cleanup, orphan pruning
|
||||
and owner deletion (`TestRetainedTags` in `tests/test_database_backup.py`). Restore
|
||||
rotates the epoch, so a pre-restore `expected_epoch` is refused with `storage-changed`.
|
||||
Owner rows and their timestamps never change on a tag command; a category apply only
|
||||
touches rows carrying the exact source.
|
||||
|
||||
Two legacy routes change additively: `GET /tags` now lists unused registry tags, and
|
||||
`/tags/suggest` without context appends unused names at count 0. Unused tags become
|
||||
deletable only through the native Delete tag (vocabulary UI PR), not on the legacy
|
||||
tags page.
|
||||
|
||||
Both token functions (`library_edit_token`, `category_impact_token`) share
|
||||
`canonical_token` in `database/library_tokens.py`; `TestSharedTokenHashing` pins their
|
||||
outputs.
|
||||
|
||||
Deploying this backend needs a ComfyUI restart (Python and route changes).
|
||||
|
||||
## Native vocabulary UI (#236)
|
||||
|
||||
`#/tags` ("Tags & categories", page title "Tags") is a native workspace route; it no
|
||||
longer hands off to the legacy tag manager. That page stays reachable from Overview's
|
||||
"Open existing tag manager" card for tag search and sort, the untagged entry and
|
||||
multi-tag AND/OR browsing; bulk prompt actions (delete, tag, category) stay on the
|
||||
legacy admin page. The screen consumes only the `/prompt_manager/vocabulary/*` routes
|
||||
above and adds no route, so PR B needs no ComfyUI restart: a browser refresh is enough.
|
||||
|
||||
Screen: a `+ Create tag` heading action, a Tag vocabulary table (columns Tag, Prompts,
|
||||
Images shown as "—" with hidden "Not available yet", and Actions with Browse and
|
||||
Rename / merge), and a Categories panel whose category rows end with the Manage
|
||||
category row (Review). Lists load 100 rows at a time, append when scrolled into view
|
||||
and keep `Load more tags` / `Load more categories` while more remain. Prompts counts
|
||||
are the Library Browse totals.
|
||||
|
||||
Dialogs (`vocabulary-dialog.mjs`): **Create tag**, **Rename or merge tag** (renaming to
|
||||
an existing name merges; it carries a **Delete tag…** confirmation that names the
|
||||
links removed) and **Manage categories** (Category select, **Preview**, then **Apply**).
|
||||
Names with commas are rejected on these native routes only.
|
||||
|
||||
Guarded states, each exercised against the real server in
|
||||
`tests/browser/workspace-vocabulary.spec.mjs`: stale epoch (`storage-changed`) and
|
||||
`vocabulary-conflict` write nothing and re-read; an unknown outcome keeps the dialog
|
||||
open, re-reads and never resends a delete; an unconfirmed Delete offers Retry that only
|
||||
re-reads the impact; a failed list read shows Retry with "Nothing was changed."; a
|
||||
failed Load more moves keyboard focus to its Retry; a write that lands after its dialog
|
||||
was force-closed or left with Back still re-reads the lists and drops the cached Library
|
||||
shelf; Manage categories reads up to 1,000 categories and says so when capped.
|
||||
**Browse** on a tag or category row opens Library with exactly that Tags or Category
|
||||
filter (the concept puts the name in the search text instead); the Prompts count equals
|
||||
the number of cards Browse shows.
|
||||
|
||||
Test files: `tests/js/workspace-vocabulary-state.test.js` (strict parsing, stale and
|
||||
abort handling, late writes after dispose, sentences), `tests/js/workspace-state.test.js`
|
||||
(route/title), `tests/browser/workspace-vocabulary.spec.mjs` (screen, dialogs, Browse,
|
||||
mobile, the concept geometry/token/control comparison and its mutation check),
|
||||
`tests/browser/vocabulary-canonical-comparison.mjs` (inventory mapping),
|
||||
`tests/browser/vocabulary-visual.spec.mjs` (paired capture artifacts) and the
|
||||
`focused node tag suggestions include a newly created tag` case in
|
||||
`tests/browser/focused-node.spec.mjs`. The fixture is
|
||||
`tests/browser/workspace_vocabulary_fixture.py`. The screen is not host-facing, so no
|
||||
`tests/comfyui` case was added.
|
||||
|
||||
```sh
|
||||
node --test tests/js/workspace-vocabulary-state.test.js tests/js/workspace-state.test.js
|
||||
PM_TEST_PORT=8766 npm run --prefix tools/vendor test:workflows -- \
|
||||
workspace-vocabulary.spec.mjs vocabulary-visual.spec.mjs focused-node.spec.mjs \
|
||||
--project chromium --project touch --project firefox --project webkit
|
||||
```
|
||||
|
||||
Recorded deviations from the concept (maintainer acceptance): no Auto-tag images
|
||||
button and an Images "—" cell until the image-annotation slice; bounded lists with
|
||||
Load more; title "Tags"; Manage categories adds a Category select and Preview before
|
||||
Apply; "Apply" instead of "Apply in prototype"; Delete tag… inside Rename or merge;
|
||||
Browse sets the exact filter; singular "1 positive prompt"; the production list appends
|
||||
fixture-only tags after the concept's sample tags. Paired concept/production captures
|
||||
for both themes and widths are attached as test artifacts; no PNG baseline is added
|
||||
until acceptance. Not compared: hover states and the dialog focus ring (the page-level
|
||||
keyboard ring and link ring offsets are asserted). No coverage floor was lowered;
|
||||
`vocabulary-state.mjs`, `vocabulary.mjs` and `vocabulary-dialog.mjs` have additive
|
||||
floors in `frontend-coverage-baseline.json`.
|
||||
@@ -3,14 +3,11 @@ ComfyUI_PromptManager: A comprehensive ComfyUI custom node that extends text enc
|
||||
with persistent prompt storage, advanced search capabilities, and an automatic image
|
||||
gallery system.
|
||||
|
||||
This module provides prompt node types:
|
||||
This module provides two main node types:
|
||||
- PromptManager: CLIP encoding node that outputs CONDITIONING
|
||||
- PromptManagerFocused: Editor-first layout using the same encoding contract
|
||||
- PromptManagerSplit: Side-by-side editor and library using the same contract
|
||||
- PromptManagerText: Text-only node that outputs STRING with prepend/append
|
||||
functionality
|
||||
- PromptManagerText: Text-only node that outputs STRING with prepend/append functionality
|
||||
|
||||
The nodes share the same database backend for persistent prompt storage and include
|
||||
Both nodes share the same database backend for persistent prompt storage and include
|
||||
an automatic image gallery system that monitors ComfyUI output directories and links
|
||||
generated images to their source prompts.
|
||||
|
||||
@@ -40,37 +37,24 @@ def get_version():
|
||||
return "unknown"
|
||||
|
||||
|
||||
from .prompt_manager import ( # noqa: E402
|
||||
PromptManager,
|
||||
PromptManagerFocused,
|
||||
PromptManagerSplit,
|
||||
)
|
||||
from .prompt_manager_pair import PromptManagerPair # noqa: E402
|
||||
from .prompt_manager_text import PromptManagerText # noqa: E402
|
||||
from .prompt_search_list import PromptSearchList # noqa: E402
|
||||
from .prompt_manager import PromptManager
|
||||
from .prompt_manager_text import PromptManagerText
|
||||
from .prompt_search_list import PromptSearchList
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"PromptManager": PromptManager,
|
||||
"PromptManagerFocused": PromptManagerFocused,
|
||||
"PromptManagerSplit": PromptManagerSplit,
|
||||
"PromptManagerPair": PromptManagerPair,
|
||||
"PromptManagerText": PromptManagerText,
|
||||
"PromptSearchList": PromptSearchList,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"PromptManager": "Prompt Manager (Classic)",
|
||||
"PromptManagerFocused": "Prompt Manager (Focused)",
|
||||
"PromptManagerSplit": "Prompt Manager (Split)",
|
||||
"PromptManagerPair": "Prompt Manager (+/-)",
|
||||
"PromptManager": "Prompt Manager",
|
||||
"PromptManagerText": "Prompt Manager Text",
|
||||
"PromptSearchList": "Prompt Search List",
|
||||
}
|
||||
|
||||
# Canvas extension only. ComfyUI loads every .js under this directory into the
|
||||
# canvas page, so the admin/gallery bundles and vendored libraries under web/
|
||||
# are served by our own routes instead (see py/api).
|
||||
WEB_DIRECTORY = "web/comfy"
|
||||
# Define path to web directory for UI components
|
||||
WEB_DIRECTORY = "web"
|
||||
|
||||
# Add API routes (same pattern as ComfyUI_Assets)
|
||||
try:
|
||||
@@ -87,9 +71,6 @@ try:
|
||||
routes = config.routes
|
||||
api = PromptManagerAPI()
|
||||
api.add_routes(routes)
|
||||
host_app = getattr(config.server_instance, "app", None)
|
||||
if host_app is not None and hasattr(host_app, "on_cleanup"):
|
||||
host_app.on_cleanup.append(api.cleanup_history_reads)
|
||||
|
||||
except Exception as e:
|
||||
# Log to internal logging system without console spam
|
||||
@@ -101,10 +82,6 @@ except Exception as e:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Shared database, set up with the monitoring system below. Stays None when that
|
||||
# fails so the queue hook can report the database as unavailable instead of crashing.
|
||||
_global_db = None
|
||||
|
||||
# Start image monitoring globally at module import time
|
||||
# This ensures images are linked regardless of which node is used in the workflow
|
||||
try:
|
||||
@@ -134,69 +111,12 @@ except Exception as e:
|
||||
except Exception:
|
||||
print(f"[ComfyUI-PromptManager] Warning: Failed to start image monitoring: {e}")
|
||||
|
||||
|
||||
def _db_factory():
|
||||
"""The shared database for the queue hook; called lazily per request."""
|
||||
if _global_db is None:
|
||||
raise RuntimeError("PromptManager database unavailable")
|
||||
return _global_db
|
||||
|
||||
|
||||
# Count prompt runs when ComfyUI dequeues workflows (it skips unchanged nodes, so
|
||||
# node execution alone misses re-runs). Separate from monitoring so either can fail.
|
||||
try:
|
||||
from .py.config import server_instance
|
||||
from .utils.usage_tracking import register_queue_hook
|
||||
|
||||
register_queue_hook(server_instance, _db_factory)
|
||||
except Exception as e:
|
||||
try:
|
||||
from .utils.logging_config import get_logger
|
||||
|
||||
get_logger("prompt_manager.init").error(f"Failed to register usage hook: {e}")
|
||||
except Exception:
|
||||
import logging
|
||||
|
||||
logging.getLogger("prompt_manager.init").error(
|
||||
f"Failed to register usage hook: {e}"
|
||||
)
|
||||
|
||||
|
||||
def _history_db_factory() -> "PromptDatabase":
|
||||
"""Initialize history on its worker even if monitoring initialization failed."""
|
||||
if _global_db is not None:
|
||||
return _global_db
|
||||
from .database.operations import PromptDatabase
|
||||
|
||||
return PromptDatabase()
|
||||
|
||||
|
||||
_history_registration = None
|
||||
try:
|
||||
from .py.config import server_instance
|
||||
from .utils.execution_history import register_history_hook
|
||||
|
||||
_history_registration = register_history_hook(server_instance, _history_db_factory)
|
||||
except Exception:
|
||||
try:
|
||||
from .utils.logging_config import get_logger
|
||||
|
||||
get_logger("prompt_manager.init").error("Failed to register execution history")
|
||||
except Exception:
|
||||
import logging
|
||||
|
||||
logging.getLogger("prompt_manager.init").error(
|
||||
"Failed to register execution history"
|
||||
)
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
|
||||
# Print startup message with loaded tools
|
||||
print()
|
||||
print(f"\033[94m[ComfyUI-PromptManager] Version:\033[0m {get_version()}")
|
||||
for node_key, display_name in NODE_DISPLAY_NAME_MAPPINGS.items():
|
||||
# ASCII only: a Windows console using a legacy code page cannot print emoji,
|
||||
# and a UnicodeEncodeError here would abort loading the whole node.
|
||||
print(f" \033[94mLoaded:\033[0m {display_name}")
|
||||
print(f"🫶 \033[94mLoaded:\033[0m {display_name}")
|
||||
print(f"\033[94mTotal: {len(NODE_CLASS_MAPPINGS)} tools loaded\033[0m")
|
||||
print()
|
||||
|
||||
@@ -1,406 +0,0 @@
|
||||
"""Owned execution-context v1 tables within execution-history schema v2.
|
||||
|
||||
Content IDs hash exact scalar values; evidence JSON contains metadata only.
|
||||
All validation scans use cursors and bounded per-row payloads.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
from .execution_history_schema import _compact, _uuid_check
|
||||
|
||||
try:
|
||||
from ..utils.execution_contexts import (
|
||||
BUNDLE_STATES,
|
||||
CAPTURE_REASONS,
|
||||
EVIDENCE_SOURCES,
|
||||
UNKNOWN_REASONS,
|
||||
MAX_CONTEXT_BYTES,
|
||||
ContextBundle,
|
||||
PromptSnapshot,
|
||||
RecipeContext,
|
||||
TextEvidence,
|
||||
validate_bundle,
|
||||
content_digest,
|
||||
text_size,
|
||||
)
|
||||
except ImportError:
|
||||
from utils.execution_contexts import (
|
||||
BUNDLE_STATES,
|
||||
CAPTURE_REASONS,
|
||||
EVIDENCE_SOURCES,
|
||||
UNKNOWN_REASONS,
|
||||
MAX_CONTEXT_BYTES,
|
||||
ContextBundle,
|
||||
PromptSnapshot,
|
||||
RecipeContext,
|
||||
TextEvidence,
|
||||
validate_bundle,
|
||||
content_digest,
|
||||
text_size,
|
||||
)
|
||||
|
||||
|
||||
FIELDS = ("raw", "effective", "prepend", "append")
|
||||
TABLE_COLUMNS = {
|
||||
"execution_context_texts": (
|
||||
("text_id", "TEXT", 1, 1),
|
||||
("text_value", "TEXT", 1, 0),
|
||||
),
|
||||
"execution_prompt_snapshots": (
|
||||
("snapshot_id", "TEXT", 1, 1),
|
||||
*((f"{field}_text_id", "TEXT", 0, 0) for field in FIELDS),
|
||||
),
|
||||
"execution_contexts": (
|
||||
("context_id", "TEXT", 1, 1),
|
||||
("positive_snapshot_id", "TEXT", 0, 0),
|
||||
("negative_snapshot_id", "TEXT", 0, 0),
|
||||
),
|
||||
"execution_run_contexts": (
|
||||
("run_id", "TEXT", 1, 1),
|
||||
("anchor_id", "TEXT", 1, 2),
|
||||
("context_id", "TEXT", 1, 0),
|
||||
("evidence_json", "TEXT", 1, 0),
|
||||
("positive_prompt_id", "INTEGER", 0, 0),
|
||||
("negative_prompt_id", "INTEGER", 0, 0),
|
||||
("pair_id", "INTEGER", 0, 0),
|
||||
("reference_epoch", "TEXT", 0, 0),
|
||||
("capture_version", "INTEGER", 1, 0),
|
||||
),
|
||||
"execution_context_capture": (
|
||||
("run_id", "TEXT", 1, 1),
|
||||
("state", "TEXT", 1, 0),
|
||||
("reason", "TEXT", 1, 0),
|
||||
("format_version", "INTEGER", 1, 0),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _digest_check(column: str, nullable: bool = False) -> str:
|
||||
check = (
|
||||
f"typeof({column}) = 'text' AND length({column}) = 64 "
|
||||
f"AND length(CAST({column} AS BLOB)) = 64 "
|
||||
f"AND {column} NOT GLOB '*[^0-9a-f]*'"
|
||||
)
|
||||
return f"{column} IS NULL OR ({check})" if nullable else check
|
||||
|
||||
|
||||
def _bounded_uuid_check(column: str) -> str:
|
||||
return f"{_uuid_check(column)} AND length(CAST({column} AS BLOB)) = 36"
|
||||
|
||||
|
||||
def _enum_check(column: str, values: frozenset[str]) -> str:
|
||||
return (
|
||||
f"typeof({column}) = 'text' AND {column} IN ("
|
||||
+ ",".join("'" + value + "'" for value in sorted(values))
|
||||
+ ")"
|
||||
)
|
||||
|
||||
|
||||
TABLE_CHECKS = {
|
||||
"execution_context_texts": (
|
||||
_digest_check("text_id"),
|
||||
"typeof(text_value) = 'text' AND length(CAST(text_value AS BLOB)) <= 65536",
|
||||
),
|
||||
"execution_prompt_snapshots": (
|
||||
_digest_check("snapshot_id"),
|
||||
*(_digest_check(f"{field}_text_id", True) for field in FIELDS),
|
||||
),
|
||||
"execution_contexts": (
|
||||
_digest_check("context_id"),
|
||||
_digest_check("positive_snapshot_id", True),
|
||||
_digest_check("negative_snapshot_id", True),
|
||||
"positive_snapshot_id IS NOT NULL OR negative_snapshot_id IS NOT NULL",
|
||||
),
|
||||
"execution_run_contexts": (
|
||||
_bounded_uuid_check("run_id"),
|
||||
"typeof(anchor_id) = 'text' AND length(anchor_id) BETWEEN 1 AND 128 "
|
||||
"AND length(CAST(anchor_id AS BLOB)) <= 512",
|
||||
_digest_check("context_id"),
|
||||
"typeof(evidence_json) = 'text' AND length(CAST(evidence_json AS "
|
||||
"BLOB)) <= 8192",
|
||||
*(
|
||||
f"{column} IS NULL OR (typeof({column}) = 'integer' AND {column} > 0)"
|
||||
for column in ("positive_prompt_id", "negative_prompt_id", "pair_id")
|
||||
),
|
||||
f"reference_epoch IS NULL OR ({_bounded_uuid_check('reference_epoch')})",
|
||||
"typeof(capture_version) = 'integer' AND capture_version = 1",
|
||||
),
|
||||
"execution_context_capture": (
|
||||
_bounded_uuid_check("run_id"),
|
||||
_enum_check("state", BUNDLE_STATES),
|
||||
_enum_check("reason", CAPTURE_REASONS),
|
||||
"length(CAST(reason AS BLOB)) <= 64",
|
||||
"typeof(format_version) = 'integer' AND format_version = 1",
|
||||
),
|
||||
}
|
||||
# (source column, destination table, destination column); actions always NO ACTION.
|
||||
TABLE_FOREIGN_KEYS = {
|
||||
"execution_context_texts": (),
|
||||
"execution_prompt_snapshots": tuple(
|
||||
(f"{field}_text_id", "execution_context_texts", "text_id") for field in FIELDS
|
||||
),
|
||||
"execution_contexts": tuple(
|
||||
(f"{role}_snapshot_id", "execution_prompt_snapshots", "snapshot_id")
|
||||
for role in ("positive", "negative")
|
||||
),
|
||||
"execution_run_contexts": (
|
||||
("run_id", "execution_runs", "run_id"),
|
||||
("context_id", "execution_contexts", "context_id"),
|
||||
),
|
||||
"execution_context_capture": (("run_id", "execution_runs", "run_id"),),
|
||||
}
|
||||
# Only SQLite-generated primary-key indexes are unique. No secondary indexes required.
|
||||
|
||||
|
||||
def install_context_schema(conn: sqlite3.Connection) -> None:
|
||||
"""Install tables inside the caller's transaction, without committing."""
|
||||
for table, columns in TABLE_COLUMNS.items():
|
||||
definitions = [
|
||||
f"{name} {kind}{' NOT NULL' if required else ''}"
|
||||
for name, kind, required, _ in columns
|
||||
]
|
||||
primary = [name for name, _, _, order in columns if order]
|
||||
definitions.append(f"PRIMARY KEY ({', '.join(primary)})")
|
||||
definitions.extend(f"CHECK({check})" for check in TABLE_CHECKS[table])
|
||||
definitions.extend(
|
||||
f"FOREIGN KEY ({column}) REFERENCES {target}({key}) "
|
||||
"ON UPDATE NO ACTION ON DELETE NO ACTION"
|
||||
for column, target, key in TABLE_FOREIGN_KEYS[table]
|
||||
)
|
||||
conn.execute(f"CREATE TABLE {table} ({', '.join(definitions)})")
|
||||
|
||||
|
||||
def _unique_object(pairs: list[tuple[str, object]]) -> dict:
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError("Duplicate evidence key")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _snapshot_values(conn: sqlite3.Connection, snapshot_id: str) -> tuple:
|
||||
row = conn.execute(
|
||||
"SELECT raw_text_id,effective_text_id,prepend_text_id,append_text_id "
|
||||
"FROM execution_prompt_snapshots WHERE snapshot_id=?",
|
||||
(snapshot_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise ValueError("Missing snapshot")
|
||||
values = []
|
||||
for text_id in row:
|
||||
if text_id is None:
|
||||
values.append(None)
|
||||
else:
|
||||
text = conn.execute(
|
||||
"SELECT text_value FROM execution_context_texts WHERE text_id=?",
|
||||
(text_id,),
|
||||
).fetchone()
|
||||
if text is None:
|
||||
raise ValueError("Missing context text")
|
||||
text_size(text[0])
|
||||
values.append(text[0])
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def _validate_evidence(
|
||||
conn: sqlite3.Connection, anchor_id: str, context_id: str, encoded: str
|
||||
) -> tuple[RecipeContext, bool]:
|
||||
text_size(encoded, 8192)
|
||||
evidence = json.loads(encoded, object_pairs_hook=_unique_object)
|
||||
if (
|
||||
type(evidence) is not dict
|
||||
or set(evidence)
|
||||
!= {"format_version", "role_provenance", "positive", "negative"}
|
||||
or type(evidence["format_version"]) is not int
|
||||
or evidence["format_version"] != 1
|
||||
or evidence["role_provenance"] != "inferred"
|
||||
):
|
||||
raise ValueError("Invalid context evidence")
|
||||
row = conn.execute(
|
||||
"SELECT positive_snapshot_id,negative_snapshot_id FROM "
|
||||
"execution_contexts WHERE context_id=?",
|
||||
(context_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise ValueError("Missing context")
|
||||
complete = True
|
||||
snapshots = []
|
||||
for role, snapshot_id in zip(("positive", "negative"), row):
|
||||
metadata = evidence[role]
|
||||
if snapshot_id is None:
|
||||
if metadata is not None:
|
||||
raise ValueError("Unexpected role evidence")
|
||||
complete = False
|
||||
snapshots.append(None)
|
||||
continue
|
||||
values = _snapshot_values(conn, snapshot_id)
|
||||
if type(metadata) is not dict or set(metadata) != set(FIELDS):
|
||||
raise ValueError("Invalid snapshot evidence")
|
||||
fields = []
|
||||
for field, value in zip(FIELDS, values):
|
||||
item = metadata[field]
|
||||
if type(item) is not dict or set(item) != {
|
||||
"provenance",
|
||||
"source",
|
||||
"reason",
|
||||
}:
|
||||
raise ValueError("Invalid field evidence")
|
||||
provenance, source, reason = (
|
||||
item["provenance"],
|
||||
item["source"],
|
||||
item["reason"],
|
||||
)
|
||||
if (
|
||||
type(provenance) is not str
|
||||
or type(source) is not str
|
||||
or source not in EVIDENCE_SOURCES.get(provenance, ())
|
||||
):
|
||||
raise ValueError("Invalid evidence source")
|
||||
if value is None:
|
||||
complete = False
|
||||
if (
|
||||
provenance != "unknown"
|
||||
or type(reason) is not str
|
||||
or reason not in UNKNOWN_REASONS
|
||||
):
|
||||
raise ValueError("Unknown evidence requires reason")
|
||||
elif provenance == "unknown" or reason is not None:
|
||||
raise ValueError("Known evidence metadata mismatch")
|
||||
fields.append(TextEvidence(value, provenance, source, reason))
|
||||
snapshots.append(PromptSnapshot(*fields))
|
||||
return RecipeContext(anchor_id, *snapshots, evidence["role_provenance"]), complete
|
||||
|
||||
|
||||
def context_schema_error(conn: sqlite3.Connection) -> str | None:
|
||||
"""Validate exact owned schema, bounded evidence, fingerprints and links."""
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type IN ('trigger','view') LIMIT 1"
|
||||
).fetchone():
|
||||
return "Unsupported execution context trigger/view"
|
||||
tables = {
|
||||
row[0]: row[1]
|
||||
for row in conn.execute("SELECT name,sql FROM sqlite_master WHERE type='table'")
|
||||
if row[0] in TABLE_COLUMNS
|
||||
}
|
||||
if tables.keys() != TABLE_COLUMNS.keys():
|
||||
return "Incomplete execution context schema"
|
||||
for table, expected in TABLE_COLUMNS.items():
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_xinfo({table})")
|
||||
)
|
||||
if columns != expected:
|
||||
return f"{table} has invalid execution context columns"
|
||||
sql = _compact(tables[table])
|
||||
checks = TABLE_CHECKS[table]
|
||||
if sql.count("check(") != len(checks) or any(
|
||||
f"check({_compact(check)})" not in sql for check in checks
|
||||
):
|
||||
return f"{table} has invalid execution context checks"
|
||||
foreign = tuple(
|
||||
sorted(
|
||||
(row[3], row[2], row[4], row[5], row[6], row[7])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({table})")
|
||||
)
|
||||
)
|
||||
expected_foreign = tuple(
|
||||
sorted(
|
||||
(*key, "NO ACTION", "NO ACTION", "NONE")
|
||||
for key in TABLE_FOREIGN_KEYS[table]
|
||||
)
|
||||
)
|
||||
if foreign != expected_foreign:
|
||||
return f"{table} has invalid execution context foreign keys"
|
||||
for index in conn.execute(f"PRAGMA index_list({table})"):
|
||||
if not index[2]:
|
||||
continue
|
||||
name = index[1].replace('"', '""')
|
||||
keys = tuple(
|
||||
row for row in conn.execute(f'PRAGMA index_xinfo("{name}")') if row[5]
|
||||
)
|
||||
primary = tuple(column[0] for column in expected if column[3])
|
||||
if (
|
||||
index[3] != "pk"
|
||||
or index[4]
|
||||
or tuple(key[2] for key in keys) != primary
|
||||
or any(key[3] or key[4] != "BINARY" for key in keys)
|
||||
):
|
||||
return f"{table} has unexpected execution context uniqueness"
|
||||
predicate = " AND ".join(f"({check})" for check in checks)
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {table} WHERE NOT ({predicate}) LIMIT 1"
|
||||
).fetchone():
|
||||
return f"{table} contains invalid execution context records"
|
||||
if conn.execute(f"PRAGMA foreign_key_check({table})").fetchone():
|
||||
return f"{table} contains orphan execution context records"
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM execution_run_contexts GROUP BY run_id HAVING "
|
||||
"COUNT(*) > 16 LIMIT 1"
|
||||
).fetchone():
|
||||
return "Too many execution contexts per run"
|
||||
try:
|
||||
for text_id, text in conn.execute(
|
||||
"SELECT text_id,text_value FROM execution_context_texts"
|
||||
):
|
||||
text_size(text)
|
||||
if content_digest("text", (text,)) != text_id:
|
||||
raise ValueError("Invalid text fingerprint")
|
||||
for (snapshot_id,) in conn.execute(
|
||||
"SELECT snapshot_id FROM execution_prompt_snapshots"
|
||||
):
|
||||
if (
|
||||
content_digest("snapshot", _snapshot_values(conn, snapshot_id))
|
||||
!= snapshot_id
|
||||
):
|
||||
raise ValueError("Invalid snapshot fingerprint")
|
||||
for context_id, positive, negative in conn.execute(
|
||||
"SELECT context_id,positive_snapshot_id,negative_snapshot_id FROM "
|
||||
"execution_contexts"
|
||||
):
|
||||
if content_digest("context", (positive, negative)) != context_id:
|
||||
raise ValueError("Invalid context fingerprint")
|
||||
payload_size = 0
|
||||
for snapshot_id in (positive, negative):
|
||||
if snapshot_id is not None:
|
||||
payload_size += sum(
|
||||
text_size(value)
|
||||
for value in _snapshot_values(conn, snapshot_id)
|
||||
if value is not None
|
||||
)
|
||||
if payload_size > MAX_CONTEXT_BYTES:
|
||||
raise ValueError("Oversized detached context")
|
||||
previous_run = None
|
||||
contexts = ()
|
||||
for run_id, anchor_id, context_id, evidence in conn.execute(
|
||||
"SELECT run_id,anchor_id,context_id,evidence_json FROM "
|
||||
"execution_run_contexts ORDER BY run_id,anchor_id"
|
||||
):
|
||||
if run_id != previous_run:
|
||||
contexts = ()
|
||||
previous_run = run_id
|
||||
context, complete = _validate_evidence(
|
||||
conn, anchor_id, context_id, evidence
|
||||
)
|
||||
status = conn.execute(
|
||||
"SELECT state,reason FROM execution_context_capture WHERE run_id=?",
|
||||
(run_id,),
|
||||
).fetchone()
|
||||
if status is None:
|
||||
raise ValueError("Context link has no capture header")
|
||||
state, reason = status
|
||||
if state == "complete" and not complete:
|
||||
raise ValueError("Incomplete evidence marked complete")
|
||||
candidate = (*contexts, context)
|
||||
validate_bundle(ContextBundle(candidate, state, reason))
|
||||
contexts = candidate
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM execution_context_capture c WHERE state='complete' "
|
||||
"AND NOT EXISTS (SELECT 1 FROM execution_run_contexts r WHERE "
|
||||
"r.run_id=c.run_id) LIMIT 1"
|
||||
).fetchone():
|
||||
raise ValueError("Complete capture has no contexts")
|
||||
except (ValueError, TypeError, RecursionError, OverflowError) as exc:
|
||||
return f"Invalid execution context content: {type(exc).__name__}"
|
||||
return None
|
||||
@@ -1,411 +0,0 @@
|
||||
"""Immutable execution capture and exact, epoch-fenced library navigation hints."""
|
||||
|
||||
from contextlib import ExitStack
|
||||
import json
|
||||
import sqlite3
|
||||
from typing import Callable, ContextManager, Iterator
|
||||
|
||||
try:
|
||||
from ..utils.execution_history_events import RunFinish, encode_command
|
||||
from ..utils.execution_contexts import (
|
||||
CAPTURE_REASONS,
|
||||
ContextBundle,
|
||||
content_digest,
|
||||
validate_bundle,
|
||||
)
|
||||
except ImportError:
|
||||
from utils.execution_history_events import RunFinish, encode_command
|
||||
from utils.execution_contexts import (
|
||||
CAPTURE_REASONS,
|
||||
ContextBundle,
|
||||
content_digest,
|
||||
validate_bundle,
|
||||
)
|
||||
|
||||
|
||||
FIELDS = ("raw", "effective", "prepend", "append")
|
||||
ROLES = ("positive", "negative")
|
||||
|
||||
|
||||
def _prepare(bundle: ContextBundle) -> list[dict]:
|
||||
validate_bundle(bundle)
|
||||
if bundle.reason not in CAPTURE_REASONS:
|
||||
raise ValueError("Invalid capture reason")
|
||||
prepared = []
|
||||
complete = bool(bundle.contexts)
|
||||
for context in bundle.contexts:
|
||||
evidence = {"format_version": 1, "role_provenance": context.role_provenance}
|
||||
snapshots = []
|
||||
for role in ROLES:
|
||||
snapshot = getattr(context, role)
|
||||
if snapshot is None:
|
||||
snapshots.append(None)
|
||||
evidence[role] = None
|
||||
complete = False
|
||||
continue
|
||||
values = tuple(getattr(snapshot, field).value for field in FIELDS)
|
||||
complete = complete and all(value is not None for value in values)
|
||||
snapshots.append((content_digest("snapshot", values), values))
|
||||
evidence[role] = {
|
||||
field: {
|
||||
"provenance": getattr(snapshot, field).provenance,
|
||||
"source": getattr(snapshot, field).source,
|
||||
"reason": getattr(snapshot, field).reason,
|
||||
}
|
||||
for field in FIELDS
|
||||
}
|
||||
if all(snapshot is None for snapshot in snapshots):
|
||||
raise ValueError("Context must contain a snapshot")
|
||||
context_id = content_digest(
|
||||
"context", tuple(s[0] if s else None for s in snapshots)
|
||||
)
|
||||
encoded = json.dumps(
|
||||
evidence,
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
)
|
||||
if len(encoded.encode("utf-8")) > 8192:
|
||||
raise ValueError("Oversized context evidence")
|
||||
prepared.append(
|
||||
{
|
||||
"anchor_id": context.anchor_id,
|
||||
"context_id": context_id,
|
||||
"snapshots": snapshots,
|
||||
"evidence_json": encoded,
|
||||
}
|
||||
)
|
||||
if bundle.state == "complete" and not complete:
|
||||
raise ValueError("Incomplete scalar capture")
|
||||
return sorted(prepared, key=lambda item: item["anchor_id"])
|
||||
|
||||
|
||||
def _existing_row(
|
||||
conn: sqlite3.Connection, table: str, columns: tuple[str, ...], values: tuple
|
||||
) -> bool:
|
||||
row = conn.execute(
|
||||
f"SELECT {','.join(columns[1:])} FROM {table} WHERE {columns[0]}=?",
|
||||
(values[0],),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
if tuple(row) != values[1:]:
|
||||
raise ValueError("Execution context digest collision")
|
||||
return True
|
||||
|
||||
|
||||
def _content_rows(item: dict) -> Iterator[tuple[str, tuple[str, ...], tuple]]:
|
||||
snapshot_ids = []
|
||||
for snapshot in item["snapshots"]:
|
||||
if snapshot is None:
|
||||
snapshot_ids.append(None)
|
||||
continue
|
||||
snapshot_id, values = snapshot
|
||||
text_ids = []
|
||||
for value in values:
|
||||
text_id = None if value is None else content_digest("text", (value,))
|
||||
if value is not None:
|
||||
yield "execution_context_texts", ("text_id", "text_value"), (
|
||||
text_id,
|
||||
value,
|
||||
)
|
||||
text_ids.append(text_id)
|
||||
yield "execution_prompt_snapshots", (
|
||||
"snapshot_id",
|
||||
*(f"{field}_text_id" for field in FIELDS),
|
||||
), (snapshot_id, *text_ids)
|
||||
snapshot_ids.append(snapshot_id)
|
||||
yield "execution_contexts", (
|
||||
"context_id",
|
||||
"positive_snapshot_id",
|
||||
"negative_snapshot_id",
|
||||
), (item["context_id"], *snapshot_ids)
|
||||
|
||||
|
||||
def _insert_exact(
|
||||
conn: sqlite3.Connection, table: str, columns: tuple[str, ...], values: tuple
|
||||
) -> None:
|
||||
if not _existing_row(conn, table, columns, values):
|
||||
names = ",".join(columns)
|
||||
placeholders = ",".join("?" for _ in columns)
|
||||
conn.execute(f"INSERT INTO {table} ({names}) VALUES ({placeholders})", values)
|
||||
|
||||
|
||||
def _role_hints(role: str) -> str:
|
||||
return (
|
||||
f"SELECT r.{role}_prompt_id, tx.text_value "
|
||||
"FROM execution_run_contexts r "
|
||||
"JOIN execution_contexts c ON c.context_id = r.context_id "
|
||||
f"JOIN execution_prompt_snapshots s ON s.snapshot_id = c.{role}_snapshot_id "
|
||||
"JOIN execution_context_texts tx ON tx.text_id = s.raw_text_id "
|
||||
f"WHERE r.{role}_prompt_id IS NOT NULL AND r.reference_epoch = ("
|
||||
"SELECT storage_epoch FROM execution_history_metadata WHERE singleton_id = 1)"
|
||||
)
|
||||
|
||||
|
||||
# Set form of the read-time prompt-hint rule in get_execution_contexts():
|
||||
# a stored role hint is live only in the current storage epoch, and only when
|
||||
# its exact raw snapshot text matches exactly one prompt, which is the hinted
|
||||
# one (_prompt_hint). Evaluated once per statement, never per prompt row.
|
||||
# Defines the CTE ``live_hint_prompts(prompt_id)``; prefix it with WITH.
|
||||
LIVE_HINT_CTES = (
|
||||
"stored_hints(prompt_id, raw) AS ("
|
||||
+ _role_hints("positive")
|
||||
+ " UNION "
|
||||
+ _role_hints("negative")
|
||||
+ "), unique_hint_texts(prompt_id, raw) AS ("
|
||||
"SELECT MIN(id), text COLLATE BINARY FROM prompts "
|
||||
"WHERE text COLLATE BINARY IN (SELECT raw FROM stored_hints) "
|
||||
"GROUP BY text COLLATE BINARY HAVING COUNT(*) = 1"
|
||||
"), live_hint_prompts(prompt_id) AS ("
|
||||
"SELECT h.prompt_id FROM stored_hints h JOIN unique_hint_texts u "
|
||||
"ON u.prompt_id = h.prompt_id AND u.raw = h.raw COLLATE BINARY)"
|
||||
)
|
||||
LIVE_HINT_PROMPT_IDS_SQL = (
|
||||
"WITH " + LIVE_HINT_CTES + " SELECT DISTINCT prompt_id FROM live_hint_prompts"
|
||||
)
|
||||
|
||||
|
||||
def _prompt_hint(conn: sqlite3.Connection, raw: str | None) -> int | None:
|
||||
if raw is None:
|
||||
return None
|
||||
rows = conn.execute(
|
||||
"SELECT id FROM prompts WHERE text=? COLLATE BINARY LIMIT 2", (raw,)
|
||||
).fetchall()
|
||||
return rows[0][0] if len(rows) == 1 else None
|
||||
|
||||
|
||||
def _hints(
|
||||
conn: sqlite3.Connection, positive: str | None, negative: str | None
|
||||
) -> tuple:
|
||||
pos_id, neg_id = _prompt_hint(conn, positive), _prompt_hint(conn, negative)
|
||||
pair_id = None
|
||||
negative_ambiguous = (
|
||||
negative is not None
|
||||
and neg_id is None
|
||||
and conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE text=? COLLATE BINARY LIMIT 1", (negative,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
if pos_id is not None and negative is not None and not negative_ambiguous:
|
||||
rows = conn.execute(
|
||||
"SELECT pp.id FROM prompt_pairs pp LEFT JOIN prompts n ON "
|
||||
"n.id=pp.negative_prompt_id "
|
||||
"WHERE pp.positive_prompt_id=? AND ((pp.negative_prompt_id IS NULL AND "
|
||||
"?='') "
|
||||
"OR (pp.negative_prompt_id=? AND n.text=? COLLATE BINARY)) LIMIT 2",
|
||||
(pos_id, negative, neg_id, negative),
|
||||
).fetchall()
|
||||
if len(rows) == 1:
|
||||
pair_id = rows[0][0]
|
||||
return pos_id, neg_id, pair_id
|
||||
|
||||
|
||||
def _values(conn: sqlite3.Connection, snapshot_id: str | None) -> dict | None:
|
||||
if snapshot_id is None:
|
||||
return None
|
||||
row = conn.execute(
|
||||
"SELECT r.text_value,e.text_value,p.text_value,a.text_value "
|
||||
"FROM execution_prompt_snapshots s "
|
||||
"LEFT JOIN execution_context_texts r ON r.text_id=s.raw_text_id "
|
||||
"LEFT JOIN execution_context_texts e ON e.text_id=s.effective_text_id "
|
||||
"LEFT JOIN execution_context_texts p ON p.text_id=s.prepend_text_id "
|
||||
"LEFT JOIN execution_context_texts a ON a.text_id=s.append_text_id "
|
||||
"WHERE s.snapshot_id=?",
|
||||
(snapshot_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise sqlite3.DatabaseError("Missing execution context snapshot")
|
||||
return dict(zip(FIELDS, row))
|
||||
|
||||
|
||||
def _write_capture(
|
||||
conn: sqlite3.Connection,
|
||||
run_id: str,
|
||||
expected_epoch: str,
|
||||
bundle: ContextBundle,
|
||||
prepared: list[dict],
|
||||
) -> bool:
|
||||
capture = conn.execute(
|
||||
"SELECT state,reason,format_version FROM execution_context_capture "
|
||||
"WHERE run_id=?",
|
||||
(run_id,),
|
||||
).fetchone()
|
||||
if capture is not None:
|
||||
for item in prepared:
|
||||
for table, columns, values in _content_rows(item):
|
||||
_existing_row(conn, table, columns, values)
|
||||
links = conn.execute(
|
||||
"SELECT anchor_id,context_id,evidence_json FROM "
|
||||
"execution_run_contexts WHERE run_id=? ORDER BY anchor_id",
|
||||
(run_id,),
|
||||
).fetchall()
|
||||
expected = [
|
||||
(item["anchor_id"], item["context_id"], item["evidence_json"])
|
||||
for item in prepared
|
||||
]
|
||||
# Compare JSON objects so serialization order never creates a conflict.
|
||||
original = [(row[0], row[1], json.loads(row[2])) for row in links]
|
||||
proposed = [
|
||||
(anchor, context, json.loads(evidence))
|
||||
for anchor, context, evidence in expected
|
||||
]
|
||||
if capture[0] != "conflicting" and (
|
||||
tuple(capture) != (bundle.state, bundle.reason, bundle.format_version)
|
||||
or original != proposed
|
||||
):
|
||||
conn.execute(
|
||||
"UPDATE execution_context_capture SET "
|
||||
"state='conflicting',reason='conflicting' WHERE run_id=?",
|
||||
(run_id,),
|
||||
)
|
||||
return True
|
||||
for item in prepared:
|
||||
for table, columns, values in _content_rows(item):
|
||||
_insert_exact(conn, table, columns, values)
|
||||
raw = [snapshot[1][0] if snapshot else None for snapshot in item["snapshots"]]
|
||||
positive_id, negative_id, pair_id = _hints(conn, *raw)
|
||||
conn.execute(
|
||||
"INSERT INTO "
|
||||
"execution_run_contexts(run_id,anchor_id,context_id,evidence_json,"
|
||||
"positive_prompt_id,negative_prompt_id,pair_id,reference_epoch,"
|
||||
"capture_version) VALUES(?,?,?,?,?,?,?,?,1)",
|
||||
(
|
||||
run_id,
|
||||
item["anchor_id"],
|
||||
item["context_id"],
|
||||
item["evidence_json"],
|
||||
positive_id,
|
||||
negative_id,
|
||||
pair_id,
|
||||
expected_epoch,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO "
|
||||
"execution_context_capture(run_id,state,reason,format_version) "
|
||||
"VALUES(?,?,?,?)",
|
||||
(run_id, bundle.state, bundle.reason, bundle.format_version),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
class ExecutionContextOperations:
|
||||
"""Compose with ExecutionHistoryOperations on PromptDatabase."""
|
||||
|
||||
def write_execution_contexts(
|
||||
self, run_id: str, expected_epoch: str, bundle: ContextBundle
|
||||
) -> bool:
|
||||
prepared = _prepare(bundle)
|
||||
with self._execution_transaction(expected_epoch) as conn:
|
||||
if (
|
||||
conn.execute(
|
||||
"SELECT 1 FROM execution_runs WHERE run_id=? AND storage_epoch=?",
|
||||
(run_id, expected_epoch),
|
||||
).fetchone()
|
||||
is None
|
||||
):
|
||||
return False
|
||||
return _write_capture(conn, run_id, expected_epoch, bundle, prepared)
|
||||
|
||||
def write_execution_contexts_for_finish(
|
||||
self,
|
||||
command: RunFinish,
|
||||
bundle: ContextBundle,
|
||||
*,
|
||||
generation_guard: Callable[[], ContextManager[bool]] | None = None,
|
||||
) -> bool:
|
||||
encode_command(command)
|
||||
if type(command) is not RunFinish:
|
||||
raise ValueError("Expected execution finish command")
|
||||
prepared = _prepare(bundle)
|
||||
# Enter the guard after BEGIN, and keep it until COMMIT/ROLLBACK exits.
|
||||
with ExitStack() as guards:
|
||||
with self._execution_transaction(command.storage_epoch) as conn:
|
||||
if generation_guard is not None and not guards.enter_context(
|
||||
generation_guard()
|
||||
):
|
||||
return False
|
||||
row = conn.execute(
|
||||
"SELECT run_id FROM execution_runs WHERE session_id=? "
|
||||
"AND queue_item_id=? AND storage_epoch=?",
|
||||
(command.session_id, command.queue_item_id, command.storage_epoch),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
return _write_capture(
|
||||
conn, row[0], command.storage_epoch, bundle, prepared
|
||||
)
|
||||
|
||||
def get_execution_contexts(self, run_id: str) -> dict:
|
||||
"""Return first-capture scalars; hints are None unless currently valid.
|
||||
|
||||
reference_epoch remains historical evidence even when hints are invalidated.
|
||||
Reads never rebind a first-capture reference or alter stored capture data.
|
||||
"""
|
||||
conn = self.model.get_connection()
|
||||
own_transaction = not conn.in_transaction
|
||||
if own_transaction:
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
capture = conn.execute(
|
||||
"SELECT state,reason,format_version FROM execution_context_capture "
|
||||
"WHERE run_id=?",
|
||||
(run_id,),
|
||||
).fetchone()
|
||||
if capture is None:
|
||||
return {
|
||||
"state": "not_captured",
|
||||
"reason": "missing",
|
||||
"format_version": None,
|
||||
"contexts": [],
|
||||
}
|
||||
epoch = self.get_execution_epoch()
|
||||
contexts = []
|
||||
rows = conn.execute(
|
||||
"SELECT r.*,c.positive_snapshot_id,c.negative_snapshot_id FROM "
|
||||
"execution_run_contexts r JOIN execution_contexts c ON "
|
||||
"c.context_id=r.context_id WHERE r.run_id=? ORDER BY r.anchor_id",
|
||||
(run_id,),
|
||||
)
|
||||
for row in rows:
|
||||
positive = _values(conn, row["positive_snapshot_id"])
|
||||
negative = _values(conn, row["negative_snapshot_id"])
|
||||
hints = (None, None, None)
|
||||
if row["reference_epoch"] == epoch:
|
||||
current = _hints(
|
||||
conn,
|
||||
positive["raw"] if positive else None,
|
||||
negative["raw"] if negative else None,
|
||||
)
|
||||
hints = tuple(
|
||||
row[key] if row[key] == value else None
|
||||
for key, value in zip(
|
||||
("positive_prompt_id", "negative_prompt_id", "pair_id"),
|
||||
current,
|
||||
)
|
||||
)
|
||||
contexts.append(
|
||||
{
|
||||
"anchor_id": row["anchor_id"],
|
||||
"context_id": row["context_id"],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
"evidence": json.loads(row["evidence_json"]),
|
||||
"positive_prompt_id": hints[0],
|
||||
"negative_prompt_id": hints[1],
|
||||
"pair_id": hints[2],
|
||||
"reference_epoch": row["reference_epoch"],
|
||||
}
|
||||
)
|
||||
return {
|
||||
"state": capture[0],
|
||||
"reason": capture[1],
|
||||
"format_version": capture[2],
|
||||
"contexts": contexts,
|
||||
}
|
||||
finally:
|
||||
if own_transaction:
|
||||
conn.rollback()
|
||||
@@ -1,198 +0,0 @@
|
||||
"""Transactional attempt writes and process-local restore invalidation."""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from typing import Callable, Iterator
|
||||
|
||||
try:
|
||||
from ..utils.execution_history_events import (
|
||||
RunFinish,
|
||||
RunStart,
|
||||
classify_evidence,
|
||||
encode_command,
|
||||
)
|
||||
from ..utils.logging_config import get_logger
|
||||
except ImportError:
|
||||
from utils.execution_history_events import (
|
||||
RunFinish,
|
||||
RunStart,
|
||||
classify_evidence,
|
||||
encode_command,
|
||||
)
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
_logger = get_logger("prompt_manager.database.execution_history")
|
||||
# Bounded wait for a history write lock. History is best effort and must
|
||||
# never stall ComfyUI's queue: a write that cannot start in time raises
|
||||
# "database is locked" and the writer records a gap instead of waiting.
|
||||
EXECUTION_BUSY_TIMEOUT_MS = 250
|
||||
_restore_lock = threading.Lock()
|
||||
_restore_observers: dict[str, dict[object, Callable[[], None]]] = {}
|
||||
|
||||
|
||||
def _database_identity(db_path: str) -> str:
|
||||
return os.path.normcase(os.path.realpath(db_path))
|
||||
|
||||
|
||||
def subscribe_history_restore(
|
||||
db_path: str, callback: Callable[[], None]
|
||||
) -> Callable[[], None]:
|
||||
"""Observe successful restores of the canonical database path."""
|
||||
identity = _database_identity(db_path)
|
||||
token = object()
|
||||
with _restore_lock:
|
||||
_restore_observers.setdefault(identity, {})[token] = callback
|
||||
|
||||
def unsubscribe() -> None:
|
||||
with _restore_lock:
|
||||
observers = _restore_observers.get(identity)
|
||||
if observers is not None:
|
||||
observers.pop(token, None)
|
||||
if not observers:
|
||||
_restore_observers.pop(identity, None)
|
||||
|
||||
return unsubscribe
|
||||
|
||||
|
||||
def notify_history_restore(db_path: str) -> None:
|
||||
"""Notify outside the registry lock; observers cannot fail a restore."""
|
||||
with _restore_lock:
|
||||
callbacks = tuple(
|
||||
_restore_observers.get(_database_identity(db_path), {}).values()
|
||||
)
|
||||
for callback in callbacks:
|
||||
try:
|
||||
callback()
|
||||
except Exception:
|
||||
_logger.warning("Execution history restore observer failed", exc_info=True)
|
||||
|
||||
|
||||
class HistoryEpochChanged(Exception):
|
||||
"""A command belongs to storage replaced by a completed restore."""
|
||||
|
||||
def __init__(self, storage_epoch: str):
|
||||
self.storage_epoch = storage_epoch
|
||||
super().__init__("Execution history storage epoch changed")
|
||||
|
||||
|
||||
class ExecutionHistoryOperations:
|
||||
"""Mixin using the calling thread's existing managed model connection."""
|
||||
|
||||
def get_execution_epoch(self) -> str:
|
||||
row = (
|
||||
self.model.get_connection()
|
||||
.execute(
|
||||
"SELECT storage_epoch FROM execution_history_metadata "
|
||||
"WHERE singleton_id=1"
|
||||
)
|
||||
.fetchone()
|
||||
)
|
||||
if row is None:
|
||||
raise sqlite3.DatabaseError("Missing execution history metadata")
|
||||
return row[0]
|
||||
|
||||
@contextmanager
|
||||
def _execution_transaction(
|
||||
self, expected_epoch: str
|
||||
) -> Iterator[sqlite3.Connection]:
|
||||
conn = self.model.get_connection()
|
||||
if conn.in_transaction:
|
||||
raise sqlite3.ProgrammingError(
|
||||
"Execution history requires its own transaction"
|
||||
)
|
||||
timeout = conn.execute("PRAGMA busy_timeout").fetchone()[0]
|
||||
conn.execute(f"PRAGMA busy_timeout={int(EXECUTION_BUSY_TIMEOUT_MS)}")
|
||||
try:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
epoch = self.get_execution_epoch()
|
||||
if epoch != expected_epoch:
|
||||
raise HistoryEpochChanged(epoch)
|
||||
yield conn
|
||||
conn.commit()
|
||||
except BaseException:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.execute(f"PRAGMA busy_timeout={timeout}")
|
||||
|
||||
def write_execution_start(self, command: RunStart) -> bool:
|
||||
encode_command(command)
|
||||
if type(command) is not RunStart:
|
||||
raise ValueError("Expected execution start command")
|
||||
with self._execution_transaction(command.storage_epoch) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO execution_runs "
|
||||
"(run_id,session_id,queue_item_id,job_id,storage_epoch,record_version,"
|
||||
"observed_dequeue_at,outcome,diagnostics_omitted) "
|
||||
"VALUES(?,?,?,?,?,1,?,'observed',0) "
|
||||
"ON CONFLICT(session_id,queue_item_id) DO NOTHING",
|
||||
(
|
||||
command.run_id,
|
||||
command.session_id,
|
||||
command.queue_item_id,
|
||||
command.job_id,
|
||||
command.storage_epoch,
|
||||
command.observed_dequeue_at,
|
||||
),
|
||||
)
|
||||
return True
|
||||
|
||||
def write_execution_finish(self, command: RunFinish) -> bool:
|
||||
encode_command(command)
|
||||
if type(command) is not RunFinish:
|
||||
raise ValueError("Expected execution finish command")
|
||||
with self._execution_transaction(command.storage_epoch) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM execution_runs WHERE session_id=? AND queue_item_id=? "
|
||||
"AND storage_epoch=?",
|
||||
(command.session_id, command.queue_item_id, command.storage_epoch),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
completion = classify_evidence(row["job_id"], command.evidence)
|
||||
if row["outcome"] == "unknown":
|
||||
return True
|
||||
if row["outcome"] != "observed":
|
||||
contradictory = completion.reason == "conflicting_terminal" or (
|
||||
completion.outcome in ("succeeded", "failed", "interrupted")
|
||||
and completion.outcome != row["outcome"]
|
||||
)
|
||||
if contradictory:
|
||||
conn.execute(
|
||||
"UPDATE execution_runs SET outcome='unknown',"
|
||||
"reason='conflicting_terminal' WHERE run_id=?",
|
||||
(row["run_id"],),
|
||||
)
|
||||
return True
|
||||
conn.execute(
|
||||
"UPDATE execution_runs SET observed_callback_at=?,"
|
||||
"executor_started_at=?,"
|
||||
"terminal_event_at=?,outcome=?,reason=?,diagnostics_omitted=?,"
|
||||
"node_id=?,"
|
||||
"node_type=?,exception_type=? WHERE run_id=?",
|
||||
(
|
||||
command.observed_callback_at,
|
||||
completion.executor_started_at,
|
||||
completion.terminal_event_at,
|
||||
completion.outcome,
|
||||
completion.reason,
|
||||
int(completion.diagnostics_omitted),
|
||||
completion.node_id,
|
||||
completion.node_type,
|
||||
completion.exception_type,
|
||||
row["run_id"],
|
||||
),
|
||||
)
|
||||
return True
|
||||
|
||||
def get_execution_run(self, run_id: str) -> dict | None:
|
||||
row = (
|
||||
self.model.get_connection()
|
||||
.execute("SELECT * FROM execution_runs WHERE run_id=?", (run_id,))
|
||||
.fetchone()
|
||||
)
|
||||
return dict(row) if row is not None else None
|
||||
@@ -1,349 +0,0 @@
|
||||
"""Versioned additive schema for observed execution attempts."""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from uuid import uuid4
|
||||
|
||||
SCHEMA_VERSION = 3
|
||||
|
||||
|
||||
def _uuid_check(column: str) -> str:
|
||||
return (
|
||||
f"typeof({column}) = 'text' AND length({column}) = 36 "
|
||||
f"AND substr({column}, 9, 1) = '-' AND substr({column}, 14, 1) = '-' "
|
||||
f"AND substr({column}, 19, 1) = '-' AND substr({column}, 24, 1) = '-' "
|
||||
f"AND length(replace({column}, '-', '')) = 32 "
|
||||
f"AND replace({column}, '-', '') NOT GLOB '*[^0-9a-f]*'"
|
||||
)
|
||||
|
||||
|
||||
def _integer_check(column: str, *, nullable: bool = False) -> str:
|
||||
expression = f"typeof({column}) = 'integer' AND {column} >= 0"
|
||||
return f"{column} IS NULL OR ({expression})" if nullable else expression
|
||||
|
||||
|
||||
_METADATA_CHECKS = (
|
||||
"singleton_id = 1",
|
||||
"schema_version = 3",
|
||||
_uuid_check("storage_epoch"),
|
||||
)
|
||||
_RUN_CHECKS = (
|
||||
*(
|
||||
_uuid_check(column)
|
||||
for column in ("run_id", "session_id", "job_id", "storage_epoch")
|
||||
),
|
||||
_integer_check("queue_item_id"),
|
||||
"record_version = 1",
|
||||
_integer_check("observed_dequeue_at"),
|
||||
*(
|
||||
_integer_check(column, nullable=True)
|
||||
for column in (
|
||||
"observed_callback_at",
|
||||
"executor_started_at",
|
||||
"terminal_event_at",
|
||||
)
|
||||
),
|
||||
"outcome IN ('observed', 'succeeded', 'failed', 'interrupted', 'unknown')",
|
||||
"(outcome = 'observed' AND reason IS NULL AND "
|
||||
"observed_callback_at IS NULL) OR (outcome != 'observed' AND "
|
||||
"reason IS NOT NULL AND observed_callback_at IS NOT NULL AND "
|
||||
"((outcome = 'succeeded' AND reason = 'terminal_success') OR "
|
||||
"(outcome = 'failed' AND reason = 'terminal_error') OR (outcome = "
|
||||
"'interrupted' AND reason = 'terminal_interrupted') OR (outcome = "
|
||||
"'unknown' AND reason IN ('missing_terminal', "
|
||||
"'conflicting_terminal', 'invalid_evidence', 'evidence_limit'))))",
|
||||
"diagnostics_omitted IN (0, 1)",
|
||||
*(
|
||||
f"{column} IS NULL OR (typeof({column}) = 'text' "
|
||||
f"AND length({column}) <= {limit})"
|
||||
for column, limit in (
|
||||
("node_id", 128),
|
||||
("node_type", 128),
|
||||
("exception_type", 256),
|
||||
)
|
||||
),
|
||||
)
|
||||
_TABLE_COLUMNS = {
|
||||
"execution_history_metadata": (
|
||||
("singleton_id", "INTEGER", 1, 1),
|
||||
("schema_version", "INTEGER", 1, 0),
|
||||
("storage_epoch", "TEXT", 1, 0),
|
||||
),
|
||||
"execution_runs": (
|
||||
("run_id", "TEXT", 1, 1),
|
||||
("session_id", "TEXT", 1, 0),
|
||||
("queue_item_id", "INTEGER", 1, 0),
|
||||
("job_id", "TEXT", 1, 0),
|
||||
("storage_epoch", "TEXT", 1, 0),
|
||||
("record_version", "INTEGER", 1, 0),
|
||||
("observed_dequeue_at", "INTEGER", 1, 0),
|
||||
("observed_callback_at", "INTEGER", 0, 0),
|
||||
("executor_started_at", "INTEGER", 0, 0),
|
||||
("terminal_event_at", "INTEGER", 0, 0),
|
||||
("outcome", "TEXT", 1, 0),
|
||||
("reason", "TEXT", 0, 0),
|
||||
("diagnostics_omitted", "INTEGER", 1, 0),
|
||||
("node_id", "TEXT", 0, 0),
|
||||
("node_type", "TEXT", 0, 0),
|
||||
("exception_type", "TEXT", 0, 0),
|
||||
),
|
||||
}
|
||||
_TABLE_CHECKS = {
|
||||
"execution_history_metadata": _METADATA_CHECKS,
|
||||
"execution_runs": _RUN_CHECKS,
|
||||
}
|
||||
# Row-only guards preserve the supported v1 DDL fingerprint while bounding reads.
|
||||
_ROW_BYTE_CHECKS = {
|
||||
"execution_history_metadata": ("length(CAST(storage_epoch AS BLOB)) = 36",),
|
||||
"execution_runs": (
|
||||
*(
|
||||
f"length(CAST({column} AS BLOB)) = 36"
|
||||
for column in (
|
||||
"run_id",
|
||||
"session_id",
|
||||
"job_id",
|
||||
"storage_epoch",
|
||||
)
|
||||
),
|
||||
*(
|
||||
f"{column} IS NULL OR length(CAST({column} AS BLOB)) <= {limit}"
|
||||
for column, limit in (
|
||||
("node_id", 512),
|
||||
("node_type", 512),
|
||||
("exception_type", 1024),
|
||||
)
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
_INDEXES = {
|
||||
"idx_execution_runs_identity": (True, ("session_id", "queue_item_id")),
|
||||
"idx_execution_runs_observed": (False, ("observed_dequeue_at", "run_id")),
|
||||
}
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return "".join(
|
||||
token if token.startswith("'") else re.sub(r"\s+", "", token).lower()
|
||||
for token in re.findall(r"'(?:''|[^'])*'|[^']+", sql)
|
||||
)
|
||||
|
||||
|
||||
def history_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> str | None:
|
||||
"""Reject partial/unsupported history schemas and invalid persisted records."""
|
||||
tables = {
|
||||
row[0]: row[1]
|
||||
for row in conn.execute("SELECT name,sql FROM sqlite_master WHERE type='table'")
|
||||
if row[0] in _TABLE_COLUMNS
|
||||
}
|
||||
from .execution_context_schema import TABLE_COLUMNS, context_schema_error
|
||||
|
||||
from .execution_output_schema import (
|
||||
TABLE_COLUMNS as OUTPUT_COLUMNS,
|
||||
output_schema_error,
|
||||
)
|
||||
|
||||
output_present = any(
|
||||
row[0] in OUTPUT_COLUMNS
|
||||
for row in conn.execute("SELECT name FROM sqlite_master")
|
||||
)
|
||||
if not tables and output_present:
|
||||
return "Execution outputs require execution history"
|
||||
context_present = any(
|
||||
row[0] in TABLE_COLUMNS
|
||||
for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
)
|
||||
if not tables and context_present:
|
||||
return "Execution contexts require execution history"
|
||||
if not tables:
|
||||
return None if allow_missing else "Missing execution history schema"
|
||||
if tables.keys() != _TABLE_COLUMNS.keys():
|
||||
return "Incomplete execution history schema"
|
||||
metadata = conn.execute(
|
||||
"SELECT CASE WHEN typeof(schema_version) = 'integer' "
|
||||
"THEN schema_version ELSE NULL END FROM execution_history_metadata LIMIT 2"
|
||||
).fetchall()
|
||||
if len(metadata) != 1 or len(metadata[0]) != 1 or metadata[0][0] not in (1, 2, 3):
|
||||
return "Unsupported execution history metadata/version"
|
||||
version = metadata[0][0]
|
||||
if version < 3 and output_present:
|
||||
return "Unexpected execution outputs for history v1/v2"
|
||||
if version == 1 and context_present:
|
||||
return "Unexpected execution contexts for history v1"
|
||||
if not allow_missing and version != SCHEMA_VERSION:
|
||||
return "Execution history migration required"
|
||||
for table, expected in _TABLE_COLUMNS.items():
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
)
|
||||
if columns != expected:
|
||||
return f"{table} has invalid execution history columns"
|
||||
sql = _compact(tables[table])
|
||||
checks = _TABLE_CHECKS[table]
|
||||
if table == "execution_history_metadata":
|
||||
checks = tuple(
|
||||
check.replace("schema_version = 3", f"schema_version = {version}")
|
||||
for check in checks
|
||||
)
|
||||
if sql.count("check(") != len(checks) or any(
|
||||
f"check({_compact(check)})" not in sql for check in checks
|
||||
):
|
||||
return f"{table} has invalid execution history checks"
|
||||
if conn.execute(f"PRAGMA foreign_key_list({table})").fetchone():
|
||||
return f"{table} has unexpected execution history foreign keys"
|
||||
for index in conn.execute(f"PRAGMA index_list({table})"):
|
||||
if not index[2]:
|
||||
continue
|
||||
if table == "execution_runs" and index[1] == "idx_execution_runs_identity":
|
||||
continue
|
||||
name = index[1].replace('"', '""')
|
||||
keys = tuple(
|
||||
row for row in conn.execute(f'PRAGMA index_xinfo("{name}")') if row[5]
|
||||
)
|
||||
primary = tuple(column[0] for column in expected if column[3])
|
||||
if (
|
||||
index[3] != "pk"
|
||||
or index[4]
|
||||
or tuple(key[2] for key in keys) != primary
|
||||
or any(key[3] or key[4] != "BINARY" for key in keys)
|
||||
):
|
||||
return f"{table} has unexpected execution history uniqueness"
|
||||
predicate = " AND ".join(
|
||||
f"({check})" for check in (*checks, *_ROW_BYTE_CHECKS[table])
|
||||
)
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {table} WHERE NOT ({predicate}) LIMIT 1"
|
||||
).fetchone():
|
||||
return f"{table} contains invalid execution history records"
|
||||
metadata = conn.execute(
|
||||
"SELECT singleton_id,schema_version,storage_epoch FROM "
|
||||
"execution_history_metadata"
|
||||
).fetchall()
|
||||
if len(metadata) != 1 or metadata[0][0] != 1 or metadata[0][1] != version:
|
||||
return "Unsupported execution history metadata/version"
|
||||
indexes = {row[1]: row for row in conn.execute("PRAGMA index_list(execution_runs)")}
|
||||
for name, (unique, columns) in _INDEXES.items():
|
||||
row = indexes.get(name)
|
||||
if row is None or bool(row[2]) != unique or row[4]:
|
||||
return f"Missing or invalid execution history index {name}"
|
||||
actual = tuple(conn.execute(f"PRAGMA index_xinfo({name})"))
|
||||
keys = tuple(item for item in actual if item[5])
|
||||
if tuple(item[2] for item in keys) != columns or any(
|
||||
item[3] or item[4] != "BINARY" for item in keys
|
||||
):
|
||||
return f"Invalid execution history index {name}"
|
||||
if version >= 2:
|
||||
error = context_schema_error(conn)
|
||||
if error:
|
||||
return error
|
||||
return output_schema_error(conn) if version == 3 else None
|
||||
|
||||
|
||||
def install_context_schema(conn: sqlite3.Connection) -> None:
|
||||
from .execution_context_schema import install_context_schema as install
|
||||
|
||||
install(conn)
|
||||
|
||||
|
||||
def install_output_schema(conn: sqlite3.Connection) -> None:
|
||||
from .execution_output_schema import install_output_schema as install
|
||||
|
||||
install(conn)
|
||||
|
||||
|
||||
def migrate_execution_history(conn: sqlite3.Connection) -> None:
|
||||
"""Install history, contexts and outputs atomically, without backfill."""
|
||||
conn.execute("SAVEPOINT execution_history_schema")
|
||||
try:
|
||||
error = history_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
present = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name IN "
|
||||
"('execution_runs','execution_history_metadata') LIMIT 1"
|
||||
).fetchone()
|
||||
if not present:
|
||||
for table, columns in _TABLE_COLUMNS.items():
|
||||
definitions = [
|
||||
f"{name} {kind}{' NOT NULL' if required else ''}"
|
||||
f"{' PRIMARY KEY' if primary else ''}"
|
||||
for name, kind, required, primary in columns
|
||||
]
|
||||
definitions.extend(f"CHECK({check})" for check in _TABLE_CHECKS[table])
|
||||
conn.execute(f"CREATE TABLE {table} ({', '.join(definitions)})")
|
||||
for name, (unique, columns) in _INDEXES.items():
|
||||
conn.execute(
|
||||
f"CREATE {'UNIQUE ' if unique else ''}INDEX {name} "
|
||||
f"ON execution_runs({', '.join(columns)})"
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO execution_history_metadata VALUES(1,?,?)",
|
||||
(SCHEMA_VERSION, str(uuid4())),
|
||||
)
|
||||
install_context_schema(conn)
|
||||
install_output_schema(conn)
|
||||
else:
|
||||
version, epoch = conn.execute(
|
||||
"SELECT schema_version,storage_epoch FROM execution_history_metadata"
|
||||
).fetchone()
|
||||
if version < SCHEMA_VERSION:
|
||||
conn.execute(
|
||||
"ALTER TABLE execution_history_metadata RENAME TO "
|
||||
"execution_history_metadata_old"
|
||||
)
|
||||
definitions = [
|
||||
f"{name} {kind}{' NOT NULL' if required else ''}"
|
||||
f"{' PRIMARY KEY' if primary else ''}"
|
||||
for name, kind, required, primary in _TABLE_COLUMNS[
|
||||
"execution_history_metadata"
|
||||
]
|
||||
]
|
||||
definitions.extend(f"CHECK({check})" for check in _METADATA_CHECKS)
|
||||
conn.execute(
|
||||
"CREATE TABLE execution_history_metadata "
|
||||
f"({', '.join(definitions)})"
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO execution_history_metadata VALUES(1,?,?)",
|
||||
(SCHEMA_VERSION, epoch),
|
||||
)
|
||||
conn.execute("DROP TABLE execution_history_metadata_old")
|
||||
if version == 1:
|
||||
install_context_schema(conn)
|
||||
install_output_schema(conn)
|
||||
# Read projections need reverse content lookup and the complete run order.
|
||||
for table, columns in (
|
||||
("execution_prompt_snapshots", "raw_text_id,snapshot_id"),
|
||||
("execution_contexts", "positive_snapshot_id,context_id"),
|
||||
("execution_run_contexts", "context_id,run_id,anchor_id"),
|
||||
("execution_runs", "observed_dequeue_at,session_id,queue_item_id,run_id"),
|
||||
):
|
||||
conn.execute(
|
||||
f"CREATE INDEX IF NOT EXISTS idx_history_read_{table} "
|
||||
f"ON {table}({columns})"
|
||||
)
|
||||
error = history_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE execution_history_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO execution_history_schema")
|
||||
conn.execute("RELEASE execution_history_schema")
|
||||
raise
|
||||
|
||||
|
||||
def rotate_history_epoch(conn: sqlite3.Connection) -> str:
|
||||
"""Publish a fresh staging epoch without rewriting restored run epochs."""
|
||||
epoch = str(uuid4())
|
||||
with conn:
|
||||
cursor = conn.execute(
|
||||
"UPDATE execution_history_metadata SET storage_epoch=? WHERE "
|
||||
"singleton_id=1",
|
||||
(epoch,),
|
||||
)
|
||||
if cursor.rowcount != 1:
|
||||
raise sqlite3.DatabaseError("Missing execution history metadata")
|
||||
return epoch
|
||||
@@ -1,218 +0,0 @@
|
||||
"""Owned schema and bounded restore validation for frozen output locators."""
|
||||
|
||||
import sqlite3
|
||||
|
||||
from .execution_history_schema import _compact, _uuid_check
|
||||
|
||||
try:
|
||||
from ..utils.execution_outputs import (
|
||||
BUNDLE_OVERHEAD,
|
||||
MAX_BUNDLE_BYTES,
|
||||
MAX_IMAGE_POSITIONS,
|
||||
MAX_REFERENCE_BYTES,
|
||||
REFERENCE_OVERHEAD,
|
||||
STATE_REASONS,
|
||||
OutputBundle,
|
||||
OutputReference,
|
||||
validate_output_bundle,
|
||||
)
|
||||
except ImportError:
|
||||
from utils.execution_outputs import (
|
||||
BUNDLE_OVERHEAD,
|
||||
MAX_BUNDLE_BYTES,
|
||||
MAX_IMAGE_POSITIONS,
|
||||
MAX_REFERENCE_BYTES,
|
||||
REFERENCE_OVERHEAD,
|
||||
STATE_REASONS,
|
||||
OutputBundle,
|
||||
OutputReference,
|
||||
validate_output_bundle,
|
||||
)
|
||||
|
||||
|
||||
TABLE_COLUMNS = {
|
||||
"execution_output_capture": (
|
||||
("run_id", "TEXT", 1, 1),
|
||||
("state", "TEXT", 1, 0),
|
||||
("reason", "TEXT", 1, 0),
|
||||
("format_version", "INTEGER", 1, 0),
|
||||
("source", "TEXT", 1, 0),
|
||||
),
|
||||
"execution_run_outputs": (
|
||||
("run_id", "TEXT", 1, 1),
|
||||
("output_node_id", "TEXT", 1, 2),
|
||||
("image_ordinal", "INTEGER", 1, 3),
|
||||
("folder_type", "TEXT", 1, 0),
|
||||
("subfolder", "TEXT", 1, 0),
|
||||
("filename", "TEXT", 1, 0),
|
||||
("path_style", "TEXT", 1, 0),
|
||||
("root_key", "TEXT", 0, 0),
|
||||
),
|
||||
}
|
||||
TABLE_FOREIGN_KEYS = {
|
||||
"execution_output_capture": (("run_id", "execution_runs", "run_id"),),
|
||||
"execution_run_outputs": (("run_id", "execution_output_capture", "run_id"),),
|
||||
}
|
||||
|
||||
|
||||
def _text_check(column: str, maximum: int, *, characters: int | None = None) -> str:
|
||||
return (
|
||||
f"typeof({column}) = 'text' AND length({column}) <= "
|
||||
f"{maximum if characters is None else characters} "
|
||||
f"AND length(CAST({column} AS BLOB)) <= {maximum} "
|
||||
f"AND instr(CAST({column} AS BLOB), x'00') = 0"
|
||||
)
|
||||
|
||||
|
||||
def _enum_check(column: str, values: tuple | frozenset) -> str:
|
||||
return (
|
||||
f"{_text_check(column, 32)} AND {column} IN ("
|
||||
+ ",".join("'" + value + "'" for value in sorted(values))
|
||||
+ ")"
|
||||
)
|
||||
|
||||
|
||||
_RUN_ID_CHECK = f"{_uuid_check('run_id')} AND {_text_check('run_id', 36)}"
|
||||
TABLE_CHECKS = {
|
||||
"execution_output_capture": (
|
||||
_RUN_ID_CHECK,
|
||||
_enum_check("state", tuple(STATE_REASONS)),
|
||||
_enum_check("reason", frozenset.union(*STATE_REASONS.values())),
|
||||
"typeof(format_version) = 'integer' AND format_version = 1",
|
||||
f"{_text_check('source', 18)} AND source = 'host_final_history'",
|
||||
),
|
||||
"execution_run_outputs": (
|
||||
_RUN_ID_CHECK,
|
||||
f"{_text_check('output_node_id', 512, characters=128)} "
|
||||
"AND length(output_node_id) >= 1",
|
||||
"typeof(image_ordinal) = 'integer' AND image_ordinal BETWEEN 0 AND 255",
|
||||
_enum_check("folder_type", ("output", "temp")),
|
||||
_text_check("subfolder", 2048),
|
||||
f"{_text_check('filename', 1024)} AND length(filename) >= 1",
|
||||
_enum_check("path_style", ("posix", "windows")),
|
||||
f"root_key IS NULL OR ({_text_check('root_key', 64)} "
|
||||
"AND length(root_key) = 64 AND root_key NOT GLOB '*[^0-9a-f]*')",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _table_sql(table: str) -> str:
|
||||
columns = TABLE_COLUMNS[table]
|
||||
definitions = [
|
||||
f"{name} {kind}{' NOT NULL' if required else ''}"
|
||||
for name, kind, required, _ in columns
|
||||
]
|
||||
primary = [name for name, _, _, order in columns if order]
|
||||
definitions.append(f"PRIMARY KEY ({', '.join(primary)})")
|
||||
definitions.extend(f"CHECK({check})" for check in TABLE_CHECKS[table])
|
||||
definitions.extend(
|
||||
f"FOREIGN KEY ({column}) REFERENCES {target}({key}) "
|
||||
"ON UPDATE NO ACTION ON DELETE NO ACTION"
|
||||
for column, target, key in TABLE_FOREIGN_KEYS[table]
|
||||
)
|
||||
return f"CREATE TABLE {table} ({', '.join(definitions)})"
|
||||
|
||||
|
||||
def install_output_schema(conn: sqlite3.Connection) -> None:
|
||||
"""Install the two owned tables within the caller's migration transaction."""
|
||||
for table in TABLE_COLUMNS:
|
||||
conn.execute(_table_sql(table))
|
||||
|
||||
|
||||
def output_schema_error(conn: sqlite3.Connection) -> str | None:
|
||||
"""Reject altered schemas and invalid rows before returning bounded DTOs."""
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type IN ('trigger','view') LIMIT 1"
|
||||
).fetchone():
|
||||
return "Unsupported execution output trigger/view"
|
||||
tables = {
|
||||
row[0]: row[1]
|
||||
for row in conn.execute("SELECT name,sql FROM sqlite_master WHERE type='table'")
|
||||
if row[0] in TABLE_COLUMNS
|
||||
}
|
||||
if tables.keys() != TABLE_COLUMNS.keys():
|
||||
return "Incomplete execution output schema"
|
||||
for table, expected in TABLE_COLUMNS.items():
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_xinfo({table})")
|
||||
)
|
||||
if columns != expected or _compact(tables[table]) != _compact(
|
||||
_table_sql(table)
|
||||
):
|
||||
return f"{table} has invalid execution output schema"
|
||||
foreign = tuple(
|
||||
sorted(
|
||||
(row[3], row[2], row[4], row[5], row[6], row[7])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({table})")
|
||||
)
|
||||
)
|
||||
if foreign != tuple(
|
||||
(*key, "NO ACTION", "NO ACTION", "NONE")
|
||||
for key in TABLE_FOREIGN_KEYS[table]
|
||||
):
|
||||
return f"{table} has invalid execution output foreign keys"
|
||||
for index in conn.execute(f"PRAGMA index_list({table})"):
|
||||
name = index[1].replace('"', '""')
|
||||
keys = tuple(
|
||||
row for row in conn.execute(f'PRAGMA index_xinfo("{name}")') if row[5]
|
||||
)
|
||||
if (
|
||||
not index[2]
|
||||
or index[3] != "pk"
|
||||
or index[4]
|
||||
or tuple(key[2] for key in keys)
|
||||
!= tuple(column[0] for column in expected if column[3])
|
||||
or any(key[3] or key[4] != "BINARY" for key in keys)
|
||||
):
|
||||
return f"{table} has unexpected execution output index/uniqueness"
|
||||
# Run every scalar guard in SQLite before any evidence text is fetched.
|
||||
predicate = " AND ".join(f"({check})" for check in TABLE_CHECKS[table])
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {table} WHERE NOT ({predicate}) LIMIT 1"
|
||||
).fetchone():
|
||||
return f"{table} contains invalid execution output records"
|
||||
if conn.execute(f"PRAGMA foreign_key_check({table})").fetchone():
|
||||
return f"{table} contains orphan execution output records"
|
||||
reference_bytes = f"{REFERENCE_OVERHEAD} + " + " + ".join(
|
||||
f"coalesce(length(CAST({column} AS BLOB)), 0)"
|
||||
for column in (
|
||||
"output_node_id",
|
||||
"folder_type",
|
||||
"subfolder",
|
||||
"filename",
|
||||
"path_style",
|
||||
"root_key",
|
||||
)
|
||||
)
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM execution_run_outputs "
|
||||
f"WHERE ({reference_bytes}) > {MAX_REFERENCE_BYTES} LIMIT 1"
|
||||
).fetchone():
|
||||
return "Oversized execution output reference"
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM execution_run_outputs GROUP BY run_id "
|
||||
f"HAVING COUNT(*) > {MAX_IMAGE_POSITIONS} OR "
|
||||
f"{BUNDLE_OVERHEAD} + SUM({reference_bytes}) > {MAX_BUNDLE_BYTES} LIMIT 1"
|
||||
).fetchone():
|
||||
return "Oversized execution output bundle"
|
||||
try:
|
||||
for run_id, state, reason, version, source in conn.execute(
|
||||
"SELECT run_id,state,reason,format_version,source "
|
||||
"FROM execution_output_capture"
|
||||
):
|
||||
references = tuple(
|
||||
OutputReference(*row)
|
||||
for row in conn.execute(
|
||||
"SELECT output_node_id,image_ordinal,folder_type,subfolder,"
|
||||
"filename,path_style,root_key FROM execution_run_outputs "
|
||||
"WHERE run_id=? ORDER BY output_node_id,image_ordinal",
|
||||
(run_id,),
|
||||
)
|
||||
)
|
||||
validate_output_bundle(
|
||||
OutputBundle(references, state, reason, version, source)
|
||||
)
|
||||
except (ValueError, TypeError, OverflowError, sqlite3.Error) as exc:
|
||||
return f"Invalid execution output content: {type(exc).__name__}"
|
||||
return None
|
||||
@@ -1,153 +0,0 @@
|
||||
"""Immutable run-owned output references, independent of mutable galleries."""
|
||||
|
||||
import sqlite3
|
||||
from contextlib import ExitStack
|
||||
from typing import Callable, ContextManager
|
||||
|
||||
try:
|
||||
from ..utils.execution_history_events import RunFinish, encode_command
|
||||
from ..utils.execution_outputs import OutputBundle, validate_output_bundle
|
||||
except ImportError:
|
||||
from utils.execution_history_events import RunFinish, encode_command
|
||||
from utils.execution_outputs import OutputBundle, validate_output_bundle
|
||||
|
||||
|
||||
_REFERENCE_FIELDS = (
|
||||
"output_node_id",
|
||||
"image_ordinal",
|
||||
"folder_type",
|
||||
"subfolder",
|
||||
"filename",
|
||||
"path_style",
|
||||
"root_key",
|
||||
)
|
||||
_REFERENCE_COLUMNS = ",".join(_REFERENCE_FIELDS)
|
||||
|
||||
|
||||
def _capture(conn: sqlite3.Connection, run_id: str) -> sqlite3.Row | None:
|
||||
capture = conn.execute(
|
||||
"SELECT state,reason,format_version,source FROM execution_output_capture "
|
||||
"WHERE run_id=?",
|
||||
(run_id,),
|
||||
).fetchone()
|
||||
# Legacy callers may disable FK enforcement; never adopt orphan evidence.
|
||||
if capture is None:
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM execution_run_outputs WHERE run_id=? LIMIT 1", (run_id,)
|
||||
).fetchone():
|
||||
raise sqlite3.DatabaseError("Output references have no capture header")
|
||||
elif (
|
||||
conn.execute(
|
||||
"SELECT 1 FROM execution_runs WHERE run_id=?", (run_id,)
|
||||
).fetchone()
|
||||
is None
|
||||
):
|
||||
raise sqlite3.DatabaseError("Output capture has no execution parent")
|
||||
return capture
|
||||
|
||||
|
||||
def _references(conn: sqlite3.Connection, run_id: str) -> list[tuple]:
|
||||
return [
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
f"SELECT {_REFERENCE_COLUMNS} FROM execution_run_outputs "
|
||||
"WHERE run_id=? ORDER BY output_node_id,image_ordinal",
|
||||
(run_id,),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
class ExecutionOutputOperations:
|
||||
"""Compose with ExecutionHistoryOperations on PromptDatabase."""
|
||||
|
||||
def write_execution_outputs_for_finish(
|
||||
self,
|
||||
command: RunFinish,
|
||||
bundle: OutputBundle,
|
||||
*,
|
||||
generation_guard: Callable[[], ContextManager[bool]] | None = None,
|
||||
) -> bool:
|
||||
encode_command(command)
|
||||
if type(command) is not RunFinish:
|
||||
raise ValueError("Expected execution finish command")
|
||||
validate_output_bundle(bundle)
|
||||
header = (bundle.state, bundle.reason, bundle.format_version, bundle.source)
|
||||
prepared = [
|
||||
tuple(getattr(reference, field) for field in _REFERENCE_FIELDS)
|
||||
for reference in bundle.references
|
||||
]
|
||||
# Enter after BEGIN and retain the guard until COMMIT/ROLLBACK finishes.
|
||||
with ExitStack() as guards:
|
||||
with self._execution_transaction(command.storage_epoch) as conn:
|
||||
if generation_guard is not None and not guards.enter_context(
|
||||
generation_guard()
|
||||
):
|
||||
return False
|
||||
parent = conn.execute(
|
||||
"SELECT run_id FROM execution_runs WHERE session_id=? "
|
||||
"AND queue_item_id=? AND storage_epoch=?",
|
||||
(command.session_id, command.queue_item_id, command.storage_epoch),
|
||||
).fetchone()
|
||||
if parent is None:
|
||||
return False
|
||||
run_id = parent[0]
|
||||
capture = _capture(conn, run_id)
|
||||
if capture is not None:
|
||||
if capture[0] != "conflicting" and (
|
||||
tuple(capture) != header
|
||||
or _references(conn, run_id) != prepared
|
||||
):
|
||||
conn.execute(
|
||||
"UPDATE execution_output_capture SET "
|
||||
"state='conflicting',reason='conflicting' WHERE run_id=?",
|
||||
(run_id,),
|
||||
)
|
||||
return True
|
||||
conn.execute(
|
||||
"INSERT INTO execution_output_capture "
|
||||
"(run_id,state,reason,format_version,source) VALUES(?,?,?,?,?)",
|
||||
(run_id, *header),
|
||||
)
|
||||
conn.executemany(
|
||||
f"INSERT INTO execution_run_outputs (run_id,{_REFERENCE_COLUMNS}) "
|
||||
"VALUES(?,?,?,?,?,?,?,?)",
|
||||
[(run_id, *reference) for reference in prepared],
|
||||
)
|
||||
return True
|
||||
|
||||
def get_execution_outputs(self, run_id: str) -> dict:
|
||||
"""Read frozen evidence in one snapshot without resolving its locators."""
|
||||
conn = self.model.get_connection()
|
||||
own_transaction = not conn.in_transaction
|
||||
if own_transaction:
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
capture = _capture(conn, run_id)
|
||||
if capture is None:
|
||||
return {
|
||||
"state": "not_captured",
|
||||
"reason": None,
|
||||
"format_version": None,
|
||||
"source": None,
|
||||
"references": [],
|
||||
}
|
||||
references = []
|
||||
for row in _references(conn, run_id):
|
||||
reference = dict(zip(_REFERENCE_FIELDS, row))
|
||||
reference.update(
|
||||
file_identity="unverified",
|
||||
newly_generated="unknown",
|
||||
context_membership="unknown",
|
||||
transient=reference["folder_type"] == "temp",
|
||||
)
|
||||
references.append(reference)
|
||||
return {
|
||||
"state": capture[0],
|
||||
"reason": capture[1],
|
||||
"format_version": capture[2],
|
||||
"source": capture[3],
|
||||
"references": references,
|
||||
}
|
||||
finally:
|
||||
if own_transaction:
|
||||
conn.rollback()
|
||||
@@ -1,127 +0,0 @@
|
||||
"""Bounded exact-path saved display within the caller's history snapshot."""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
try:
|
||||
from ..utils.inspection_response import MAX_LIBRARY_BYTES, _library_projection
|
||||
except ImportError:
|
||||
from utils.inspection_response import MAX_LIBRARY_BYTES, _library_projection
|
||||
|
||||
|
||||
def _display_bytes(value: bytes | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return value.decode("utf-8", "replace")
|
||||
|
||||
|
||||
def _tags(
|
||||
conn: sqlite3.Connection, owner: int, *, pair: bool
|
||||
) -> tuple[list[str], bool]:
|
||||
table, column = (
|
||||
("prompt_pair_tags", "pair_id") if pair else ("prompt_tags", "prompt_id")
|
||||
)
|
||||
tags = []
|
||||
used = 0
|
||||
cursor = conn.execute(
|
||||
f"SELECT substr(CAST(t.name AS BLOB),1,?) FROM {table} pt "
|
||||
f"JOIN tags t ON t.id=pt.tag_id WHERE pt.{column}=? LIMIT 129",
|
||||
(MAX_LIBRARY_BYTES + 1, owner),
|
||||
)
|
||||
for row in cursor:
|
||||
used += len(row[0])
|
||||
if used > MAX_LIBRARY_BYTES or len(tags) == 128:
|
||||
return [], True
|
||||
tags.append(_display_bytes(row[0]))
|
||||
return tags, False
|
||||
|
||||
|
||||
def _saved_projection(conn: sqlite3.Connection, row: sqlite3.Row) -> tuple[dict, dict]:
|
||||
saved = {"prompt_id": row["prompt_id"]}
|
||||
pairs = list(
|
||||
conn.execute(
|
||||
"SELECT pp.id,pp.positive_prompt_id,"
|
||||
"substr(CAST(pp.title AS BLOB),1,?) AS title,"
|
||||
"substr(CAST(pp.category AS BLOB),1,?) AS category "
|
||||
"FROM image_prompt_pairs ip JOIN prompt_pairs pp ON pp.id=ip.pair_id "
|
||||
"WHERE ip.image_id=? LIMIT 2",
|
||||
(MAX_LIBRARY_BYTES + 1, MAX_LIBRARY_BYTES + 1, row["id"]),
|
||||
)
|
||||
)
|
||||
if len(pairs) > 1:
|
||||
return saved, _library_projection({"status": "limited"})
|
||||
pair = pairs[0] if pairs else None
|
||||
title = pair["title"] if pair is not None else None
|
||||
category = pair["category"] if pair is not None else row["category"]
|
||||
tags, limited = _tags(
|
||||
conn,
|
||||
pair["id"] if pair is not None else row["prompt_id"],
|
||||
pair=pair is not None,
|
||||
)
|
||||
limited = limited or any(
|
||||
value is not None and len(value) > MAX_LIBRARY_BYTES
|
||||
for value in (title, category)
|
||||
)
|
||||
if pair is not None:
|
||||
saved["pair"] = {
|
||||
"id": pair["id"],
|
||||
"positive_prompt_id": pair["positive_prompt_id"],
|
||||
}
|
||||
library = _library_projection(
|
||||
{
|
||||
"status": "linked",
|
||||
"positive_id": row["prompt_id"],
|
||||
"pair_id": pair["id"] if pair is not None else None,
|
||||
"title": _display_bytes(title),
|
||||
"category": _display_bytes(category),
|
||||
"tags": tags,
|
||||
"metadata_scope": "pair" if pair is not None else "positive",
|
||||
}
|
||||
)
|
||||
if limited:
|
||||
library = _library_projection({"status": "limited", "tags": []})
|
||||
return saved, library
|
||||
|
||||
|
||||
def read_history_gallery(
|
||||
conn: sqlite3.Connection, paths: tuple[str, ...]
|
||||
) -> dict[str, dict]:
|
||||
"""Read at most two rows/path and two pair associations/unique row; no writes."""
|
||||
if not conn.in_transaction:
|
||||
raise sqlite3.ProgrammingError("Saved history display requires caller snapshot")
|
||||
if (
|
||||
type(paths) is not tuple
|
||||
or len(paths) > 100
|
||||
or any(
|
||||
type(path) is not str
|
||||
or not os.path.isabs(path)
|
||||
or path != os.path.normcase(os.path.normpath(path))
|
||||
or "\0" in path
|
||||
for path in paths
|
||||
)
|
||||
):
|
||||
raise ValueError("Invalid history gallery paths")
|
||||
result = {}
|
||||
for path in dict.fromkeys(paths):
|
||||
rows = list(
|
||||
conn.execute(
|
||||
"SELECT gi.id,gi.prompt_id,"
|
||||
"substr(CAST(p.category AS BLOB),1,?) AS category "
|
||||
"FROM generated_images gi JOIN prompts p ON p.id=gi.prompt_id "
|
||||
"WHERE gi.file_path=? OR gi.image_path=? LIMIT 2",
|
||||
(MAX_LIBRARY_BYTES + 1, path, path),
|
||||
)
|
||||
)
|
||||
match = "none" if not rows else "unique" if len(rows) == 1 else "ambiguous"
|
||||
saved, library = None, _library_projection(
|
||||
{"status": "none" if not rows else "ambiguous"}
|
||||
)
|
||||
if match == "unique":
|
||||
saved, library = _saved_projection(conn, rows[0])
|
||||
result[path] = {
|
||||
"gallery_match": match,
|
||||
"gallery_image_id": rows[0]["id"] if match == "unique" else None,
|
||||
"saved": saved,
|
||||
"library": library,
|
||||
}
|
||||
return result
|
||||
@@ -1,527 +0,0 @@
|
||||
"""Snapshot-owned, bounded projections of immutable execution evidence."""
|
||||
|
||||
import codecs
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Iterator
|
||||
|
||||
from .execution_contexts import content_digest
|
||||
from .execution_history import HistoryEpochChanged
|
||||
from .execution_outputs import _capture, _REFERENCE_COLUMNS
|
||||
from .library import _ELIGIBLE
|
||||
|
||||
RunKey = tuple[int, str, int, str]
|
||||
|
||||
|
||||
class HistoryCaptureChanged(ValueError):
|
||||
"""A continuation no longer describes the stored output capture header."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WindowPosition:
|
||||
raw_text_id: str
|
||||
upper: RunKey
|
||||
before: RunKey
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OutputPosition:
|
||||
capture_fingerprint: str
|
||||
after: tuple[str, int]
|
||||
|
||||
|
||||
_DIGEST = re.compile(r"[0-9a-f]{64}")
|
||||
_UUID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
|
||||
|
||||
|
||||
def _valid_digest(value: object) -> bool:
|
||||
return type(value) is str and _DIGEST.fullmatch(value) is not None
|
||||
|
||||
|
||||
def _valid_run_key(value: object) -> bool:
|
||||
return (
|
||||
type(value) is tuple
|
||||
and len(value) == 4
|
||||
and all(type(value[i]) is int and 0 <= value[i] <= 2**63 - 1 for i in (0, 2))
|
||||
and all(
|
||||
type(value[i]) is str and _UUID.fullmatch(value[i]) is not None
|
||||
for i in (1, 3)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _valid_coordinate(value: object) -> bool:
|
||||
if type(value) is not tuple or len(value) != 2:
|
||||
return False
|
||||
node, ordinal = value
|
||||
if not (
|
||||
type(node) is str
|
||||
and 1 <= len(node) <= 128
|
||||
and "\0" not in node
|
||||
and type(ordinal) is int
|
||||
and 0 <= ordinal <= 255
|
||||
):
|
||||
return False
|
||||
try:
|
||||
return len(node.encode("utf-8")) <= 512
|
||||
except UnicodeError:
|
||||
return False
|
||||
|
||||
|
||||
def _validate_window_position(position: WindowPosition | None) -> None:
|
||||
if position is not None and not (
|
||||
type(position) is WindowPosition
|
||||
and _valid_digest(position.raw_text_id)
|
||||
and _valid_run_key(position.upper)
|
||||
and _valid_run_key(position.before)
|
||||
):
|
||||
raise ValueError("Invalid history window position")
|
||||
|
||||
|
||||
def _validate_output_position(position: OutputPosition | None) -> None:
|
||||
if position is not None and not (
|
||||
type(position) is OutputPosition
|
||||
and _valid_digest(position.capture_fingerprint)
|
||||
and _valid_coordinate(position.after)
|
||||
):
|
||||
raise ValueError("Invalid history output position")
|
||||
|
||||
|
||||
_snapshots: ContextVar[tuple] = ContextVar("history_read_snapshots", default=())
|
||||
_KEY = "(r.observed_dequeue_at,r.session_id,r.queue_item_id,r.run_id)"
|
||||
_ORDER = (
|
||||
"r.observed_dequeue_at DESC,r.session_id COLLATE BINARY DESC,"
|
||||
"r.queue_item_id DESC,r.run_id COLLATE BINARY DESC"
|
||||
)
|
||||
_MATCHING = (
|
||||
"SELECT DISTINCT rc.run_id FROM execution_prompt_snapshots s "
|
||||
"JOIN execution_contexts c ON c.positive_snapshot_id=s.snapshot_id "
|
||||
"JOIN execution_run_contexts rc ON rc.context_id=c.context_id "
|
||||
"WHERE s.raw_text_id=?"
|
||||
)
|
||||
|
||||
|
||||
def _limit(limit: int) -> None:
|
||||
if type(limit) is not int or not 1 <= limit <= 100:
|
||||
raise ValueError("History limit must be an integer from 1 to 100")
|
||||
|
||||
|
||||
def _key(row: dict) -> RunKey:
|
||||
return tuple(
|
||||
row[name]
|
||||
for name in ("observed_dequeue_at", "session_id", "queue_item_id", "run_id")
|
||||
)
|
||||
|
||||
|
||||
def _context_capture(conn: sqlite3.Connection, run_id: str) -> dict:
|
||||
row = conn.execute(
|
||||
"SELECT state,reason,format_version FROM execution_context_capture "
|
||||
"WHERE run_id=?",
|
||||
(run_id,),
|
||||
).fetchone()
|
||||
return (
|
||||
dict(row)
|
||||
if row is not None
|
||||
else {"state": "not_captured", "reason": "missing", "format_version": None}
|
||||
)
|
||||
|
||||
|
||||
def _output_header(conn: sqlite3.Connection, run_id: str) -> tuple[dict, int | None]:
|
||||
row = _capture(conn, run_id)
|
||||
if row is None:
|
||||
return (
|
||||
dict(state="not_captured", reason=None, format_version=None, source=None),
|
||||
None,
|
||||
)
|
||||
count = conn.execute(
|
||||
"SELECT COUNT(*) FROM execution_run_outputs WHERE run_id=?", (run_id,)
|
||||
).fetchone()[0]
|
||||
return dict(row), count
|
||||
|
||||
|
||||
def _list_status(capture: dict) -> str:
|
||||
if capture["state"] == "not_captured":
|
||||
return "not_captured"
|
||||
return "complete" if capture["state"] == "complete" else "incomplete"
|
||||
|
||||
|
||||
def _output_summary(conn: sqlite3.Connection, run_id: str) -> dict:
|
||||
capture, count = _output_header(conn, run_id)
|
||||
return dict(
|
||||
capture, retained_occurrence_count=count, list_status=_list_status(capture)
|
||||
)
|
||||
|
||||
|
||||
def _fingerprint(capture: dict, count: int | None) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
[
|
||||
1,
|
||||
*(capture[k] for k in ("state", "reason", "format_version", "source")),
|
||||
count,
|
||||
],
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _receipt(run_id: str, row: sqlite3.Row) -> dict:
|
||||
return {
|
||||
"occurrence": {
|
||||
"run_id": run_id,
|
||||
"output_node_id": row["output_node_id"],
|
||||
"image_ordinal": row["image_ordinal"],
|
||||
},
|
||||
"reference": {
|
||||
key: row[key]
|
||||
for key in (
|
||||
"folder_type",
|
||||
"subfolder",
|
||||
"filename",
|
||||
"path_style",
|
||||
"root_key",
|
||||
)
|
||||
},
|
||||
"file_identity": "unverified",
|
||||
"newly_generated": "unknown",
|
||||
"context_membership": "unknown",
|
||||
"transient": row["folder_type"] == "temp",
|
||||
}
|
||||
|
||||
|
||||
class HistoryReadOperations:
|
||||
"""Read evidence only inside an explicitly owned, epoch-first snapshot."""
|
||||
|
||||
@contextmanager
|
||||
def recorded_history_snapshot(
|
||||
self, expected_epoch: str | None = None
|
||||
) -> Iterator[str]:
|
||||
conn = self.model.get_connection()
|
||||
if conn.in_transaction:
|
||||
raise sqlite3.ProgrammingError(
|
||||
"History reads require their own transaction"
|
||||
)
|
||||
conn.execute("BEGIN")
|
||||
token = _snapshots.set((*_snapshots.get(), conn))
|
||||
try:
|
||||
epoch = self.get_execution_epoch()
|
||||
if expected_epoch is not None and epoch != expected_epoch:
|
||||
raise HistoryEpochChanged(epoch)
|
||||
yield epoch
|
||||
finally:
|
||||
try:
|
||||
conn.rollback()
|
||||
finally:
|
||||
_snapshots.reset(token)
|
||||
|
||||
def _history_connection(self) -> sqlite3.Connection:
|
||||
conn = self.model.get_connection()
|
||||
if not conn.in_transaction or conn not in _snapshots.get():
|
||||
raise sqlite3.ProgrammingError(
|
||||
"History read requires recorded_history_snapshot"
|
||||
)
|
||||
return conn
|
||||
|
||||
def list_history_window(
|
||||
self,
|
||||
positive_id: int,
|
||||
*,
|
||||
limit: int = 40,
|
||||
after: WindowPosition | None = None,
|
||||
restart: bool = False,
|
||||
) -> dict | None:
|
||||
conn = self._history_connection()
|
||||
_limit(limit)
|
||||
_validate_window_position(after)
|
||||
if type(restart) is not bool or (
|
||||
restart and (after is None or after.before != after.upper)
|
||||
):
|
||||
raise ValueError("Invalid history restart position")
|
||||
saved = conn.execute(
|
||||
"SELECT CASE WHEN typeof(p.text)='text' AND "
|
||||
"length(CAST(p.text AS BLOB))<=65536 THEN p.text ELSE NULL END AS text "
|
||||
f"FROM prompts p WHERE p.id=? AND ({_ELIGIBLE})",
|
||||
(positive_id,),
|
||||
).fetchone()
|
||||
if saved is None:
|
||||
return None
|
||||
text = saved["text"]
|
||||
raw_id = None if text is None else content_digest("text", (text,))
|
||||
if after is not None and after.raw_text_id != raw_id:
|
||||
raise ValueError("History position text scope changed")
|
||||
group = {
|
||||
"positive_id": positive_id,
|
||||
"raw_text_id": raw_id,
|
||||
"relation": "exact_frozen_raw_text",
|
||||
"scope_status": "uncapturable_text" if text is None else "no_matches",
|
||||
}
|
||||
result = dict(
|
||||
group=group,
|
||||
recorded_run_count=0,
|
||||
items=[],
|
||||
context_summaries=[],
|
||||
has_more=False,
|
||||
next_position=None,
|
||||
resume_position=None,
|
||||
)
|
||||
if (
|
||||
raw_id is None
|
||||
or conn.execute(
|
||||
"SELECT 1 FROM execution_context_texts WHERE text_id=? "
|
||||
"AND text_value=? COLLATE BINARY",
|
||||
(raw_id, text),
|
||||
).fetchone()
|
||||
is None
|
||||
):
|
||||
return result
|
||||
where = f"r.run_id IN ({_MATCHING})"
|
||||
params = [raw_id]
|
||||
if after is not None:
|
||||
where += f" AND {_KEY}<=(?,?,?,?)"
|
||||
params.extend(after.upper)
|
||||
count = conn.execute(
|
||||
f"SELECT COUNT(*) FROM execution_runs r WHERE {where}", params
|
||||
).fetchone()[0]
|
||||
result["recorded_run_count"] = count
|
||||
group["scope_status"] = "matched" if count else "no_matches"
|
||||
if after is not None and not restart:
|
||||
where += f" AND {_KEY}<(?,?,?,?)"
|
||||
params.extend(after.before)
|
||||
rows = [
|
||||
dict(row)
|
||||
for row in conn.execute(
|
||||
f"SELECT r.* FROM execution_runs r WHERE {where} "
|
||||
f"ORDER BY {_ORDER} LIMIT ?",
|
||||
(*params, limit + 1),
|
||||
)
|
||||
]
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
if not rows:
|
||||
return result
|
||||
upper = after.upper if after is not None else _key(rows[0])
|
||||
result["resume_position"] = WindowPosition(raw_id, upper, upper)
|
||||
if more:
|
||||
result["next_position"] = WindowPosition(raw_id, upper, _key(rows[-1]))
|
||||
result["has_more"] = more
|
||||
rows.reverse()
|
||||
by_run = {row["run_id"]: row for row in rows}
|
||||
for row in rows:
|
||||
row["diagnostics_available"] = any(
|
||||
row[field] is not None
|
||||
for field in ("node_id", "node_type", "exception_type")
|
||||
)
|
||||
for field in ("node_id", "node_type", "exception_type"):
|
||||
del row[field]
|
||||
row["diagnostics_omitted"] = bool(row["diagnostics_omitted"])
|
||||
row["context_capture"] = _context_capture(conn, row["run_id"])
|
||||
row["output_summary"] = _output_summary(conn, row["run_id"])
|
||||
row["matching_anchors"] = []
|
||||
placeholders = ",".join("?" for _ in rows)
|
||||
anchors = conn.execute(
|
||||
"SELECT rc.run_id,rc.anchor_id,rc.context_id,rc.evidence_json,"
|
||||
"c.positive_snapshot_id,c.negative_snapshot_id,"
|
||||
"substr(CAST(t.text_value AS BLOB),1,512) AS prefix,"
|
||||
"length(CAST(t.text_value AS BLOB)) AS byte_length "
|
||||
"FROM execution_run_contexts rc JOIN execution_contexts c "
|
||||
"ON c.context_id=rc.context_id JOIN execution_prompt_snapshots p "
|
||||
"ON p.snapshot_id=c.positive_snapshot_id "
|
||||
"LEFT JOIN execution_prompt_snapshots n "
|
||||
"ON n.snapshot_id=c.negative_snapshot_id "
|
||||
"LEFT JOIN execution_context_texts t ON t.text_id=n.raw_text_id "
|
||||
f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id=? "
|
||||
"ORDER BY rc.anchor_id COLLATE BINARY,rc.run_id COLLATE BINARY",
|
||||
(*by_run, raw_id),
|
||||
)
|
||||
summaries = {}
|
||||
evidence_by_context = {}
|
||||
for anchor in anchors:
|
||||
reference = {
|
||||
key: anchor[key]
|
||||
for key in (
|
||||
"anchor_id",
|
||||
"context_id",
|
||||
"positive_snapshot_id",
|
||||
"negative_snapshot_id",
|
||||
)
|
||||
}
|
||||
by_run[anchor["run_id"]]["matching_anchors"].append(reference)
|
||||
context_id = anchor["context_id"]
|
||||
if context_id not in summaries:
|
||||
prefix = b"" if anchor["byte_length"] == 0 else anchor["prefix"]
|
||||
preview = (
|
||||
None
|
||||
if prefix is None
|
||||
else codecs.getincrementaldecoder("utf-8")().decode(
|
||||
prefix, final=False
|
||||
)[:100]
|
||||
)
|
||||
evidence_by_context[context_id] = []
|
||||
summaries[context_id] = {
|
||||
key: anchor[key]
|
||||
for key in (
|
||||
"context_id",
|
||||
"positive_snapshot_id",
|
||||
"negative_snapshot_id",
|
||||
)
|
||||
}
|
||||
summaries[context_id]["negative_raw"] = {
|
||||
"state": (
|
||||
"unknown"
|
||||
if preview is None
|
||||
else "explicit_empty" if not preview else "known"
|
||||
),
|
||||
"preview": preview,
|
||||
"truncated": (
|
||||
False
|
||||
if preview is None
|
||||
else len(preview.encode("utf-8")) < anchor["byte_length"]
|
||||
),
|
||||
"evidence": evidence_by_context[context_id],
|
||||
}
|
||||
negative = json.loads(anchor["evidence_json"])["negative"]
|
||||
evidence = (
|
||||
negative["raw"]
|
||||
if negative is not None
|
||||
else dict(provenance="unknown", source="unknown", reason="missing")
|
||||
)
|
||||
evidence_by_context[context_id].append(
|
||||
{
|
||||
"run_id": anchor["run_id"],
|
||||
"anchor_id": anchor["anchor_id"],
|
||||
**evidence,
|
||||
}
|
||||
)
|
||||
# Context order follows its first chronological matching run, then anchor.
|
||||
ordered = {}
|
||||
for row in rows:
|
||||
for anchor in row["matching_anchors"]:
|
||||
ordered.setdefault(
|
||||
anchor["context_id"], summaries[anchor["context_id"]]
|
||||
)
|
||||
result["items"] = rows
|
||||
result["context_summaries"] = list(ordered.values())
|
||||
return result
|
||||
|
||||
def get_history_run(self, run_id: str) -> dict | None:
|
||||
conn = self._history_connection()
|
||||
run = self.get_execution_run(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
capture = self.get_execution_contexts(run_id)
|
||||
snapshots = {
|
||||
row["anchor_id"]: row
|
||||
for row in conn.execute(
|
||||
"SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id,"
|
||||
"p.raw_text_id AS positive_raw_text_id "
|
||||
"FROM execution_run_contexts rc JOIN execution_contexts c "
|
||||
"ON c.context_id=rc.context_id LEFT JOIN execution_prompt_snapshots p "
|
||||
"ON p.snapshot_id=c.positive_snapshot_id WHERE rc.run_id=?",
|
||||
(run_id,),
|
||||
)
|
||||
}
|
||||
contexts = capture.pop("contexts")
|
||||
for context in contexts:
|
||||
row = snapshots[context["anchor_id"]]
|
||||
context.update(
|
||||
{
|
||||
key: row[key]
|
||||
for key in (
|
||||
"positive_snapshot_id",
|
||||
"negative_snapshot_id",
|
||||
"positive_raw_text_id",
|
||||
)
|
||||
}
|
||||
)
|
||||
pair = context["pair_id"]
|
||||
context["navigation"] = {
|
||||
"positive_available": context["positive_prompt_id"] is not None
|
||||
and conn.execute(
|
||||
f"SELECT 1 FROM prompts p WHERE p.id=? AND ({_ELIGIBLE})",
|
||||
(context["positive_prompt_id"],),
|
||||
).fetchone()
|
||||
is not None,
|
||||
"negative_available": context["negative_prompt_id"] is not None,
|
||||
"pair_available": pair is not None
|
||||
and conn.execute(
|
||||
"SELECT 1 FROM prompt_pairs WHERE id=? AND archived=0", (pair,)
|
||||
).fetchone()
|
||||
is not None,
|
||||
}
|
||||
return dict(
|
||||
run=run,
|
||||
context_capture=capture,
|
||||
contexts=contexts,
|
||||
output_summary=_output_summary(conn, run_id),
|
||||
)
|
||||
|
||||
def list_history_outputs(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
limit: int = 40,
|
||||
after: OutputPosition | None = None,
|
||||
) -> dict | None:
|
||||
conn = self._history_connection()
|
||||
_limit(limit)
|
||||
_validate_output_position(after)
|
||||
# Validate header/orphans even when the requested parent does not exist.
|
||||
capture, count = _output_header(conn, run_id)
|
||||
if self.get_execution_run(run_id) is None:
|
||||
return None
|
||||
fingerprint = _fingerprint(capture, count)
|
||||
if after is not None and after.capture_fingerprint != fingerprint:
|
||||
raise HistoryCaptureChanged("History output capture changed")
|
||||
where = "run_id=?"
|
||||
params = [run_id]
|
||||
if after is not None:
|
||||
where += " AND (output_node_id COLLATE BINARY,image_ordinal)>(?,?)"
|
||||
params.extend(after.after)
|
||||
rows = list(
|
||||
conn.execute(
|
||||
f"SELECT {_REFERENCE_COLUMNS} FROM execution_run_outputs WHERE {where} "
|
||||
"ORDER BY output_node_id COLLATE BINARY,image_ordinal LIMIT ?",
|
||||
(*params, limit + 1),
|
||||
)
|
||||
)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
position = (
|
||||
OutputPosition(
|
||||
fingerprint, (rows[-1]["output_node_id"], rows[-1]["image_ordinal"])
|
||||
)
|
||||
if more
|
||||
else None
|
||||
)
|
||||
return dict(
|
||||
capture=capture,
|
||||
retained_occurrence_count=count,
|
||||
list_status=_list_status(capture),
|
||||
capture_fingerprint=fingerprint,
|
||||
items=[_receipt(run_id, row) for row in rows],
|
||||
has_more=more,
|
||||
next_position=position,
|
||||
)
|
||||
|
||||
def get_history_occurrence(
|
||||
self, run_id: str, node: str, ordinal: int
|
||||
) -> dict | None:
|
||||
conn = self._history_connection()
|
||||
capture, _ = _output_header(conn, run_id)
|
||||
row = conn.execute(
|
||||
f"SELECT {_REFERENCE_COLUMNS} FROM execution_run_outputs "
|
||||
"WHERE run_id=? AND output_node_id=? COLLATE BINARY AND image_ordinal=?",
|
||||
(run_id, node, ordinal),
|
||||
).fetchone()
|
||||
return (
|
||||
dict(capture=capture, receipt=_receipt(run_id, row))
|
||||
if row is not None
|
||||
else None
|
||||
)
|
||||
@@ -1,547 +0,0 @@
|
||||
"""Read-only positive-parent views over existing prompt and pair tables."""
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from .library_tokens import library_edit_token
|
||||
|
||||
_ELIGIBLE = """(EXISTS(SELECT 1 FROM prompt_pairs ep WHERE ep.positive_prompt_id=p.id)
|
||||
OR EXISTS(SELECT 1 FROM generated_images eg WHERE eg.prompt_id=p.id)
|
||||
OR NOT EXISTS(SELECT 1 FROM prompt_pairs en WHERE en.negative_prompt_id=p.id)
|
||||
OR EXISTS(SELECT 1 FROM prompt_library_state es WHERE es.prompt_id=p.id))"""
|
||||
_FROM = "prompts p LEFT JOIN prompt_library_state ls ON ls.prompt_id=p.id"
|
||||
# The characters JavaScript's String.prototype.trim() removes, so a title the
|
||||
# display treats as blank also falls back to the prompt text for name sorting.
|
||||
_BLANK = (
|
||||
"char(9,10,11,12,13,32,160,5760,8192,8193,8194,8195,8196,8197,8198,8199,"
|
||||
"8200,8201,8202,8232,8233,8239,8287,12288,65279)"
|
||||
)
|
||||
_NAME = (
|
||||
f"LOWER(CASE WHEN TRIM(COALESCE(ls.title,''),{_BLANK})!='' "
|
||||
"THEN ls.title ELSE p.text END)"
|
||||
)
|
||||
_ORDERS = {
|
||||
"newest": ("COALESCE(julianday(p.created_at),0)", "DESC"),
|
||||
"oldest": ("COALESCE(julianday(p.created_at),0)", "ASC"),
|
||||
"name": (_NAME, "ASC"),
|
||||
"rating": ("COALESCE(p.rating,0)", "DESC"),
|
||||
}
|
||||
# Pair archive scopes for variant and image reads, as conditions on alias pp.
|
||||
ARCHIVE_SCOPES = {
|
||||
"active": " AND pp.archived=0",
|
||||
"archived": " AND pp.archived=1",
|
||||
"all": "",
|
||||
}
|
||||
|
||||
|
||||
def _escape(value: str) -> str:
|
||||
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
|
||||
def _limit(value: int) -> None:
|
||||
if type(value) is not int or not 1 <= value <= 200:
|
||||
raise ValueError("limit must be between 1 and 200")
|
||||
|
||||
|
||||
def _archive(value: str) -> str:
|
||||
"""Return the pair-archive condition for one scope (alias ``pp``)."""
|
||||
if not isinstance(value, str) or value not in ARCHIVE_SCOPES:
|
||||
raise ValueError("archive must be one of active, archived or all")
|
||||
return ARCHIVE_SCOPES[value]
|
||||
|
||||
|
||||
def _filter(text: str, category: str | None, tags: list[str]) -> tuple[str, list]:
|
||||
if not isinstance(text, str) or len(text) > 500:
|
||||
raise ValueError("Search text must be a string of at most 500 characters")
|
||||
if category is not None and (not isinstance(category, str) or len(category) > 200):
|
||||
raise ValueError("Invalid library category")
|
||||
if (
|
||||
not isinstance(tags, list)
|
||||
or len(tags) > 30
|
||||
or any(not isinstance(tag, str) or not tag or len(tag) > 200 for tag in tags)
|
||||
):
|
||||
raise ValueError("Invalid library tags")
|
||||
parent, pair, parent_params, pair_params = [], ["pp.archived=0"], [], []
|
||||
if text:
|
||||
parent.append("(p.text LIKE ? ESCAPE '\\' OR ls.title LIKE ? ESCAPE '\\')")
|
||||
parent_params.extend([f"%{_escape(text)}%"] * 2)
|
||||
pair.append(
|
||||
"(p.text LIKE ? ESCAPE '\\' OR ls.title LIKE ? ESCAPE '\\' "
|
||||
"OR n.text LIKE ? ESCAPE '\\' OR pp.title LIKE ? ESCAPE '\\')"
|
||||
)
|
||||
pair_params.extend([f"%{_escape(text)}%"] * 4)
|
||||
if category is not None:
|
||||
parent.append("p.category=?")
|
||||
pair.append("pp.category=?")
|
||||
parent_params.append(category)
|
||||
pair_params.append(category)
|
||||
for tag in tags:
|
||||
parent.append(
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name=?)"
|
||||
)
|
||||
pair.append(
|
||||
"EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.pair_id=pp.id AND t.name=?)"
|
||||
)
|
||||
parent_params.append(tag)
|
||||
pair_params.append(tag)
|
||||
condition = (
|
||||
"("
|
||||
+ (" AND ".join(parent) or "1=1")
|
||||
+ (
|
||||
") OR EXISTS(SELECT 1 FROM prompt_pairs pp "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
||||
"WHERE pp.positive_prompt_id=p.id AND "
|
||||
)
|
||||
+ " AND ".join(pair)
|
||||
+ ")"
|
||||
)
|
||||
return f"({_ELIGIBLE}) AND ({condition})", [*parent_params, *pair_params]
|
||||
|
||||
|
||||
class LibraryOperations:
|
||||
"""Group reusable negative variants without changing legacy prompt queries.
|
||||
|
||||
Each public read runs inside ``library_snapshot``; the ``_``-prefixed
|
||||
helpers take that snapshot's connection and epoch so routes can share it.
|
||||
"""
|
||||
|
||||
@contextmanager
|
||||
def library_snapshot(
|
||||
self, *, include_ceiling: bool = False
|
||||
) -> Iterator[tuple[sqlite3.Connection, str, int | None]]:
|
||||
"""Hold one read transaction, reading the epoch (and ceiling) first."""
|
||||
conn = self.model.get_connection()
|
||||
if conn.in_transaction:
|
||||
raise sqlite3.ProgrammingError("Library reads require their own snapshot")
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT storage_epoch,"
|
||||
+ (
|
||||
"(SELECT COALESCE(MAX(id),0) FROM prompts)"
|
||||
if include_ceiling
|
||||
else "NULL"
|
||||
)
|
||||
+ " FROM execution_history_metadata WHERE singleton_id=1"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise sqlite3.DatabaseError("Missing execution history metadata")
|
||||
yield conn, row[0], row[1]
|
||||
finally:
|
||||
if conn.in_transaction:
|
||||
conn.rollback()
|
||||
|
||||
def _library_parents(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
epoch: str,
|
||||
where: str,
|
||||
params: list,
|
||||
suffix: str = "",
|
||||
sort_expression: str | None = None,
|
||||
) -> list[dict]:
|
||||
rows = conn.execute(
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,p.created_at,"
|
||||
+ (f"{sort_expression} AS _sort," if sort_expression else "")
|
||||
+ "(SELECT COUNT(*) FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
|
||||
"AND v.archived=0) AS variant_count,"
|
||||
"(SELECT COUNT(*) FROM generated_images g "
|
||||
"WHERE g.prompt_id=p.id) AS image_count "
|
||||
"FROM " + _FROM + " WHERE " + where + suffix,
|
||||
params,
|
||||
).fetchall()
|
||||
result = [{**dict(row), "tags": []} for row in rows]
|
||||
if result:
|
||||
by_id = {row["id"]: row for row in result}
|
||||
marks = ",".join("?" for _ in result)
|
||||
for row in conn.execute(
|
||||
"SELECT pt.prompt_id,t.name FROM prompt_tags pt "
|
||||
"JOIN tags t ON t.id=pt.tag_id "
|
||||
f"WHERE pt.prompt_id IN ({marks}) ORDER BY t.name",
|
||||
list(by_id),
|
||||
):
|
||||
by_id[row["prompt_id"]]["tags"].append(row["name"])
|
||||
for row in result:
|
||||
row["edit_token"] = library_edit_token(epoch, row)
|
||||
return result
|
||||
|
||||
def _list_library(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
epoch: str,
|
||||
ceiling: int,
|
||||
*,
|
||||
text: str = "",
|
||||
category: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
sort: str = "newest",
|
||||
limit: int = 40,
|
||||
after: dict | None = None,
|
||||
) -> dict:
|
||||
_limit(limit)
|
||||
if sort not in _ORDERS:
|
||||
raise ValueError("Unknown library sort order")
|
||||
where, params = _filter(text, category, [] if tags is None else tags)
|
||||
if after is not None:
|
||||
if (
|
||||
not isinstance(after, dict)
|
||||
or set(after) != {"ceiling", "value", "id"}
|
||||
or type(after["ceiling"]) is not int
|
||||
or type(after["id"]) is not int
|
||||
):
|
||||
raise ValueError("Invalid library cursor")
|
||||
ceiling = after["ceiling"]
|
||||
where += " AND p.id<=?"
|
||||
params.append(ceiling)
|
||||
total = conn.execute(
|
||||
"SELECT COUNT(*) FROM " + _FROM + " WHERE " + where, params
|
||||
).fetchone()[0]
|
||||
expression, direction = _ORDERS[sort]
|
||||
if after is not None:
|
||||
operator = ">" if direction == "ASC" else "<"
|
||||
where += (
|
||||
f" AND ({expression}{operator}? OR "
|
||||
f"({expression}=? AND p.id{operator}?))"
|
||||
)
|
||||
params.extend([after["value"], after["value"], after["id"]])
|
||||
rows = self._library_parents(
|
||||
conn,
|
||||
epoch,
|
||||
where,
|
||||
[*params, limit + 1],
|
||||
f" ORDER BY {expression} {direction},p.id {direction} LIMIT ?",
|
||||
sort_expression=expression,
|
||||
)
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
next_key = None
|
||||
if has_more:
|
||||
next_key = {
|
||||
"ceiling": ceiling,
|
||||
"value": rows[-1]["_sort"],
|
||||
"id": rows[-1]["id"],
|
||||
}
|
||||
for row in rows:
|
||||
row.pop("_sort")
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"has_more": has_more,
|
||||
"next_key": next_key,
|
||||
}
|
||||
|
||||
def list_library(
|
||||
self,
|
||||
*,
|
||||
text: str = "",
|
||||
category: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
sort: str = "newest",
|
||||
limit: int = 40,
|
||||
after: dict | None = None,
|
||||
) -> dict:
|
||||
with self.library_snapshot(include_ceiling=True) as (conn, epoch, ceiling):
|
||||
return self._list_library(
|
||||
conn,
|
||||
epoch,
|
||||
ceiling,
|
||||
text=text,
|
||||
category=category,
|
||||
tags=tags,
|
||||
sort=sort,
|
||||
limit=limit,
|
||||
after=after,
|
||||
)
|
||||
|
||||
def _is_library_positive(self, conn: sqlite3.Connection, positive_id: int) -> bool:
|
||||
return (
|
||||
conn.execute(
|
||||
"SELECT 1 FROM prompts p WHERE p.id=? AND " + _ELIGIBLE, (positive_id,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
|
||||
def _get_library_positive(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
epoch: str,
|
||||
positive_id: int,
|
||||
*,
|
||||
variant_limit: int = 40,
|
||||
variant_after: int | None = None,
|
||||
archive: str = "active",
|
||||
) -> dict | None:
|
||||
_limit(variant_limit)
|
||||
scope = _archive(archive)
|
||||
if variant_after is not None and (
|
||||
type(variant_after) is not int or variant_after <= 0
|
||||
):
|
||||
raise ValueError("Invalid variant cursor")
|
||||
rows = self._library_parents(
|
||||
conn, epoch, _ELIGIBLE + " AND p.id=?", [positive_id]
|
||||
)
|
||||
if not rows:
|
||||
return None
|
||||
positive = rows[0]
|
||||
positive["archived_variant_count"] = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompt_pairs "
|
||||
"WHERE positive_prompt_id=? AND archived=1",
|
||||
(positive_id,),
|
||||
).fetchone()[0]
|
||||
variant_total = {
|
||||
"active": positive["variant_count"],
|
||||
"archived": positive["archived_variant_count"],
|
||||
"all": positive["variant_count"] + positive["archived_variant_count"],
|
||||
}[archive]
|
||||
where = "pp.positive_prompt_id=?" + scope
|
||||
params = [positive_id]
|
||||
if variant_after is not None:
|
||||
where += " AND pp.id<?"
|
||||
params.append(variant_after)
|
||||
variants = [
|
||||
dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT pp.id,COALESCE(n.text,'') AS negative_text,"
|
||||
"pp.category,pp.title,pp.archived,"
|
||||
"pp.negative_prompt_id IS NULL AS explicit_empty,"
|
||||
"(SELECT COUNT(DISTINCT image_id) FROM image_prompt_pairs ip "
|
||||
"WHERE ip.pair_id=pp.id) AS image_count "
|
||||
"FROM prompt_pairs pp "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
||||
"WHERE " + where + " ORDER BY pp.id DESC LIMIT ?",
|
||||
[*params, variant_limit + 1],
|
||||
)
|
||||
]
|
||||
variants_has_more = len(variants) > variant_limit
|
||||
variants = variants[:variant_limit]
|
||||
by_id = {row["id"]: row for row in variants}
|
||||
for row in variants:
|
||||
row["tags"] = []
|
||||
row["archived"] = bool(row["archived"])
|
||||
row["explicit_empty"] = bool(row["explicit_empty"])
|
||||
if by_id:
|
||||
marks = ",".join("?" for _ in by_id)
|
||||
for row in conn.execute(
|
||||
"SELECT pt.pair_id,t.name FROM prompt_pair_tags pt "
|
||||
"JOIN tags t ON t.id=pt.tag_id "
|
||||
f"WHERE pt.pair_id IN ({marks}) ORDER BY t.name",
|
||||
list(by_id),
|
||||
):
|
||||
by_id[row["pair_id"]]["tags"].append(row["name"])
|
||||
unassigned = conn.execute(
|
||||
"SELECT COUNT(*) FROM generated_images g WHERE g.prompt_id=? "
|
||||
"AND NOT EXISTS(SELECT 1 FROM image_prompt_pairs ip "
|
||||
"WHERE ip.image_id=g.id)",
|
||||
(positive_id,),
|
||||
).fetchone()[0]
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"positive": positive,
|
||||
"variants": variants,
|
||||
"unassigned_image_count": unassigned,
|
||||
"variant_total": variant_total,
|
||||
"variants_has_more": variants_has_more,
|
||||
"variants_next_key": variants[-1]["id"] if variants_has_more else None,
|
||||
}
|
||||
|
||||
def get_library_positive(
|
||||
self,
|
||||
positive_id: int,
|
||||
*,
|
||||
variant_limit: int = 40,
|
||||
variant_after: int | None = None,
|
||||
archive: str = "active",
|
||||
) -> dict | None:
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
return self._get_library_positive(
|
||||
conn,
|
||||
epoch,
|
||||
positive_id,
|
||||
variant_limit=variant_limit,
|
||||
variant_after=variant_after,
|
||||
archive=archive,
|
||||
)
|
||||
|
||||
def _get_library_options(self, conn: sqlite3.Connection, epoch: str) -> dict:
|
||||
rows = conn.execute(
|
||||
"SELECT category FROM (SELECT p.category AS category "
|
||||
"FROM prompts p WHERE "
|
||||
+ _ELIGIBLE
|
||||
+ (" UNION SELECT category FROM prompt_pairs " "WHERE archived=0) ")
|
||||
+ "WHERE category IS NOT NULL AND TRIM(category)!='' "
|
||||
"ORDER BY category COLLATE NOCASE,category"
|
||||
).fetchall()
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"categories": [row["category"] for row in rows],
|
||||
}
|
||||
|
||||
def get_library_options(self) -> dict:
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
return self._get_library_options(conn, epoch)
|
||||
|
||||
def get_library_previews(self, positive_ids: list[int]) -> dict[int, list[dict]]:
|
||||
"""Read at most two links per parent in one bounded page query."""
|
||||
if (
|
||||
not isinstance(positive_ids, list)
|
||||
or len(positive_ids) > 200
|
||||
or any(type(value) is not int or value <= 0 for value in positive_ids)
|
||||
):
|
||||
raise ValueError("Invalid preview parent IDs")
|
||||
if not positive_ids:
|
||||
return {}
|
||||
result = {value: [] for value in positive_ids}
|
||||
marks = ",".join("?" for _ in positive_ids)
|
||||
rows = self.model.get_connection().execute(
|
||||
"WITH ranked AS (SELECT id,prompt_id,image_path,ROW_NUMBER() OVER "
|
||||
"(PARTITION BY prompt_id ORDER BY id DESC) AS position "
|
||||
f"FROM generated_images WHERE prompt_id IN ({marks})) "
|
||||
"SELECT id,prompt_id,image_path FROM ranked WHERE position<=2 "
|
||||
"ORDER BY prompt_id,id DESC",
|
||||
positive_ids,
|
||||
)
|
||||
for row in rows:
|
||||
result[row["prompt_id"]].append(dict(row))
|
||||
return result
|
||||
|
||||
def _list_library_images(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
epoch: str,
|
||||
positive_id: int,
|
||||
*,
|
||||
pair_id: int | None = None,
|
||||
limit: int = 40,
|
||||
after: int | None = None,
|
||||
archive: str = "active",
|
||||
) -> dict:
|
||||
_limit(limit)
|
||||
scope = _archive(archive)
|
||||
if after is not None and (type(after) is not int or after <= 0):
|
||||
raise ValueError("Invalid image cursor")
|
||||
if (
|
||||
pair_id is not None
|
||||
and not conn.execute(
|
||||
"SELECT 1 FROM prompt_pairs pp WHERE pp.id=? "
|
||||
"AND pp.positive_prompt_id=?" + scope,
|
||||
(pair_id, positive_id),
|
||||
).fetchone()
|
||||
):
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM prompt_pairs WHERE id=? AND positive_prompt_id=?",
|
||||
(pair_id, positive_id),
|
||||
).fetchone():
|
||||
raise ValueError("Variant is not in the requested archive scope")
|
||||
raise ValueError("Variant does not belong to this positive prompt")
|
||||
where, params = "g.prompt_id=?", [positive_id]
|
||||
if pair_id is not None:
|
||||
where += (
|
||||
" AND EXISTS(SELECT 1 FROM image_prompt_pairs ip "
|
||||
"WHERE ip.image_id=g.id AND ip.pair_id=?)"
|
||||
)
|
||||
params.append(pair_id)
|
||||
total = conn.execute(
|
||||
"SELECT COUNT(*) FROM generated_images g WHERE " + where, params
|
||||
).fetchone()[0]
|
||||
if after is not None:
|
||||
where += " AND g.id<?"
|
||||
params.append(after)
|
||||
rows = conn.execute(
|
||||
"SELECT g.id,g.image_path FROM generated_images g WHERE "
|
||||
+ where
|
||||
+ " ORDER BY g.id DESC LIMIT ?",
|
||||
[*params, limit + 1],
|
||||
).fetchall()
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"images": [dict(row) for row in rows[:limit]],
|
||||
"total": total,
|
||||
"has_more": len(rows) > limit,
|
||||
"next_key": rows[limit - 1]["id"] if len(rows) > limit else None,
|
||||
}
|
||||
|
||||
def list_library_images(
|
||||
self,
|
||||
positive_id: int,
|
||||
*,
|
||||
pair_id: int | None = None,
|
||||
limit: int = 40,
|
||||
after: int | None = None,
|
||||
archive: str = "active",
|
||||
) -> dict:
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
return self._list_library_images(
|
||||
conn,
|
||||
epoch,
|
||||
positive_id,
|
||||
pair_id=pair_id,
|
||||
limit=limit,
|
||||
after=after,
|
||||
archive=archive,
|
||||
)
|
||||
|
||||
def export_library(self) -> dict:
|
||||
"""Read every portable owner and pair from one snapshot, writing nothing.
|
||||
|
||||
Local IDs order the rows internally and never appear in the document.
|
||||
"""
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
owners = {}
|
||||
order = []
|
||||
for row in conn.execute(
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes FROM "
|
||||
+ _FROM
|
||||
+ " WHERE "
|
||||
+ _ELIGIBLE
|
||||
+ " ORDER BY p.id"
|
||||
):
|
||||
owners[row["id"]] = {
|
||||
"positive_text": row["text"],
|
||||
"title": row["title"],
|
||||
"category": row["category"],
|
||||
"tags": [],
|
||||
"rating": row["rating"],
|
||||
"notes": row["notes"],
|
||||
"variants": [],
|
||||
}
|
||||
order.append(row["id"])
|
||||
for row in conn.execute(
|
||||
"SELECT pt.prompt_id,t.name FROM prompt_tags pt "
|
||||
"JOIN tags t ON t.id=pt.tag_id ORDER BY pt.prompt_id,t.name"
|
||||
):
|
||||
if row["prompt_id"] in owners:
|
||||
owners[row["prompt_id"]]["tags"].append(row["name"])
|
||||
variants = {}
|
||||
for row in conn.execute(
|
||||
"SELECT pp.id,pp.positive_prompt_id,"
|
||||
"COALESCE(n.text,'') AS negative_text,"
|
||||
"pp.title,pp.category,pp.rating,pp.notes,pp.archived "
|
||||
"FROM prompt_pairs pp "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
||||
"ORDER BY pp.id"
|
||||
):
|
||||
variant = {
|
||||
"negative_text": row["negative_text"],
|
||||
"title": row["title"],
|
||||
"category": row["category"],
|
||||
"tags": [],
|
||||
"rating": row["rating"],
|
||||
"notes": row["notes"],
|
||||
"archived": bool(row["archived"]),
|
||||
}
|
||||
variants[row["id"]] = variant
|
||||
owners[row["positive_prompt_id"]]["variants"].append(variant)
|
||||
for row in conn.execute(
|
||||
"SELECT pt.pair_id,t.name FROM prompt_pair_tags pt "
|
||||
"JOIN tags t ON t.id=pt.tag_id ORDER BY pt.pair_id,t.name"
|
||||
):
|
||||
variants[row["pair_id"]]["tags"].append(row["name"])
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"kind": "prompt-manager-library",
|
||||
"positives": [owners[key] for key in order],
|
||||
}
|
||||
@@ -1,310 +0,0 @@
|
||||
"""Transactional native Library commands guarded by epoch and edit token."""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
from .library import _ELIGIBLE
|
||||
from .models import utc_now_iso
|
||||
|
||||
try:
|
||||
from ..utils.validators import (
|
||||
validate_category,
|
||||
validate_prompt_text,
|
||||
validate_tags,
|
||||
)
|
||||
except ImportError:
|
||||
from utils.validators import validate_category, validate_prompt_text, validate_tags
|
||||
|
||||
METADATA_FIELDS = frozenset({"title", "category", "tags", "rating", "notes"})
|
||||
MAX_TEXT_FIELD = 10000
|
||||
_UUID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
|
||||
_TOKEN = re.compile(r"[0-9a-f]{64}")
|
||||
# Parent metadata stored on the prompt row; title lives in prompt_library_state.
|
||||
_PROMPT_COLUMNS = ("category", "rating", "notes")
|
||||
|
||||
|
||||
class LibraryCommandError(Exception):
|
||||
"""A native command lost a race against storage or another writer.
|
||||
|
||||
``code`` is a wire code (``metadata-conflict``, ``storage-changed`` or
|
||||
``vocabulary-conflict`` for a tag or category that changed since it was read),
|
||||
``epoch`` the storage epoch read inside the transaction and ``positive``
|
||||
the current owner record when it may be disclosed.
|
||||
"""
|
||||
|
||||
def __init__(self, code: str, epoch: str, positive: dict | None = None):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.epoch = epoch
|
||||
self.positive = positive
|
||||
|
||||
|
||||
def validate_library_metadata(metadata: dict, *, require_title: bool = False) -> None:
|
||||
"""Reject anything but a nonempty allowlisted parent metadata object."""
|
||||
if not isinstance(metadata, dict) or not metadata:
|
||||
raise ValueError("metadata must be a nonempty object")
|
||||
unknown = set(metadata) - METADATA_FIELDS
|
||||
if unknown:
|
||||
raise ValueError("Unknown metadata fields: " + ", ".join(sorted(unknown)))
|
||||
if require_title and (
|
||||
not isinstance(metadata.get("title"), str) or not metadata["title"].strip()
|
||||
):
|
||||
raise ValueError("title is required")
|
||||
for name in ("title", "notes"):
|
||||
value = metadata.get(name)
|
||||
if value is not None and not isinstance(value, str):
|
||||
raise ValueError(f"{name} must be a string or null")
|
||||
if value is not None and len(value) > MAX_TEXT_FIELD:
|
||||
raise ValueError(f"{name} is too long")
|
||||
if metadata.get("category") is not None:
|
||||
validate_category(metadata["category"])
|
||||
if "tags" in metadata:
|
||||
if not isinstance(metadata["tags"], list):
|
||||
raise ValueError("tags must be a list of strings")
|
||||
validate_tags(metadata["tags"])
|
||||
rating = metadata.get("rating")
|
||||
if rating is not None and (type(rating) is not int or not 1 <= rating <= 5):
|
||||
raise ValueError("rating must be an integer between 1 and 5 or null")
|
||||
|
||||
|
||||
def normalize_library_metadata(metadata: dict) -> dict:
|
||||
"""Return validated metadata with category trimmed and blank stored as NULL."""
|
||||
if "category" not in metadata:
|
||||
return metadata
|
||||
category = metadata["category"]
|
||||
return {**metadata, "category": (category or "").strip() or None}
|
||||
|
||||
|
||||
def normalize_library_tags(tags: list[str]) -> list[str]:
|
||||
"""Trim and deduplicate validated tags, keeping first-seen order."""
|
||||
return list(dict.fromkeys(tag.strip() for tag in tags))
|
||||
|
||||
|
||||
def _validate_epoch(expected_epoch: str) -> None:
|
||||
if not isinstance(expected_epoch, str) or not _UUID.fullmatch(expected_epoch):
|
||||
raise ValueError("expected_epoch must be a canonical UUID")
|
||||
|
||||
|
||||
def _validate_guards(positive_id: int, expected_epoch: str, if_match: str) -> None:
|
||||
if type(positive_id) is not int or not 1 <= positive_id <= 2**63 - 1:
|
||||
raise ValueError("Invalid positive prompt ID")
|
||||
_validate_epoch(expected_epoch)
|
||||
if not isinstance(if_match, str) or not _TOKEN.fullmatch(if_match):
|
||||
raise ValueError("if_match must be 64 lowercase hexadecimal characters")
|
||||
|
||||
|
||||
def _validate_negative_text(negative_text: str) -> None:
|
||||
"""Require an explicit string; blank deliberately means the empty negative."""
|
||||
if not isinstance(negative_text, str):
|
||||
raise ValueError("negative_text must be explicit; use an empty string for none")
|
||||
if negative_text.strip():
|
||||
validate_prompt_text(negative_text)
|
||||
|
||||
|
||||
class LibraryCommandOperations:
|
||||
"""Connection-level parent commands mixed into PromptDatabase."""
|
||||
|
||||
def _library_epoch(self, conn: sqlite3.Connection, expected_epoch: str) -> str:
|
||||
row = conn.execute(
|
||||
"SELECT storage_epoch FROM execution_history_metadata WHERE singleton_id=1"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise sqlite3.DatabaseError("Missing execution history metadata")
|
||||
if row[0] != expected_epoch:
|
||||
raise LibraryCommandError("storage-changed", row[0])
|
||||
return row[0]
|
||||
|
||||
def _library_parent(
|
||||
self, conn: sqlite3.Connection, epoch: str, positive_id: int
|
||||
) -> dict | None:
|
||||
rows = self._library_parents(
|
||||
conn, epoch, _ELIGIBLE + " AND p.id=?", [positive_id]
|
||||
)
|
||||
return rows[0] if rows else None
|
||||
|
||||
def _ensure_library_parent(self, conn: sqlite3.Connection, prompt_id: int) -> None:
|
||||
"""Register explicit parent intent once; an existing row is never touched."""
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM prompt_library_state WHERE prompt_id=?", (prompt_id,)
|
||||
).fetchone():
|
||||
return
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_library_state(prompt_id,title) VALUES(?,NULL)",
|
||||
(prompt_id,),
|
||||
)
|
||||
|
||||
def _apply_library_metadata(
|
||||
self, conn: sqlite3.Connection, current: dict, metadata: dict
|
||||
) -> None:
|
||||
"""Write only fields that differ; stamp ``prompts.updated_at`` once."""
|
||||
prompt_id = current["id"]
|
||||
changed = False
|
||||
if "title" in metadata and metadata["title"] != current["title"]:
|
||||
conn.execute(
|
||||
"UPDATE prompt_library_state SET title=? WHERE prompt_id=?",
|
||||
(metadata["title"], prompt_id),
|
||||
)
|
||||
changed = True
|
||||
columns = {
|
||||
name: metadata[name]
|
||||
for name in _PROMPT_COLUMNS
|
||||
if name in metadata and metadata[name] != current[name]
|
||||
}
|
||||
if "tags" in metadata:
|
||||
tags = normalize_library_tags(metadata["tags"])
|
||||
if set(tags) != set(current["tags"]):
|
||||
self._sync_prompt_tags(conn, prompt_id, tags)
|
||||
changed = True
|
||||
if columns or changed:
|
||||
columns["updated_at"] = utc_now_iso()
|
||||
conn.execute(
|
||||
"UPDATE prompts SET "
|
||||
+ ",".join(f"{name}=?" for name in columns)
|
||||
+ " WHERE id=?",
|
||||
[*columns.values(), prompt_id],
|
||||
)
|
||||
|
||||
def update_library_parent(
|
||||
self,
|
||||
positive_id: int,
|
||||
*,
|
||||
expected_epoch: str,
|
||||
if_match: str,
|
||||
metadata: dict,
|
||||
) -> dict | None:
|
||||
"""Apply explicit parent metadata if the epoch and edit token still match.
|
||||
|
||||
Returns ``{library_epoch, positive}`` read before commit, or None when
|
||||
the positive is missing or not a Library parent. Raises
|
||||
LibraryCommandError for a rotated epoch or a stale token.
|
||||
"""
|
||||
_validate_guards(positive_id, expected_epoch, if_match)
|
||||
validate_library_metadata(metadata)
|
||||
metadata = normalize_library_metadata(metadata)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
current = self._library_parent(conn, epoch, positive_id)
|
||||
if current is None:
|
||||
return None
|
||||
if current["edit_token"] != if_match:
|
||||
raise LibraryCommandError("metadata-conflict", epoch, current)
|
||||
self._ensure_library_parent(conn, positive_id)
|
||||
self._apply_library_metadata(conn, current, metadata)
|
||||
positive = self._library_parent(conn, epoch, positive_id)
|
||||
if positive is None:
|
||||
raise sqlite3.DatabaseError("Library parent vanished during update")
|
||||
return {"library_epoch": epoch, "positive": positive}
|
||||
|
||||
def _recipe_negative(
|
||||
self, conn: sqlite3.Connection, negative_text: str
|
||||
) -> int | None:
|
||||
if not negative_text.strip():
|
||||
return None
|
||||
return self._pair_component_outcome(conn, negative_text)[0]
|
||||
|
||||
def _recipe_receipt(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
epoch: str,
|
||||
positive_id: int,
|
||||
pair_id: int,
|
||||
*,
|
||||
positive_created: bool,
|
||||
pair_created: bool,
|
||||
) -> dict:
|
||||
"""Read both owners inside the write transaction for the receipt."""
|
||||
positive = self._library_parent(conn, epoch, positive_id)
|
||||
pairs = self._read_pair_rows(conn, "pp.id = ?", [pair_id])
|
||||
if positive is None or not pairs:
|
||||
raise sqlite3.DatabaseError("Library recipe vanished during save")
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"positive": positive,
|
||||
"pair": pairs[0],
|
||||
"positive_created": positive_created,
|
||||
"pair_created": pair_created,
|
||||
# Parent metadata is applied only to a newly inserted positive.
|
||||
"parent_metadata_applied": positive_created,
|
||||
}
|
||||
|
||||
def create_library_recipe(
|
||||
self,
|
||||
positive_text: str,
|
||||
negative_text: str,
|
||||
*,
|
||||
expected_epoch: str,
|
||||
parent_metadata: dict,
|
||||
) -> dict:
|
||||
"""Create or reuse a positive and its ordered pair in one transaction.
|
||||
|
||||
Submitted parent metadata applies only when the positive component is
|
||||
newly inserted; a reused component keeps its own metadata. New pairs get
|
||||
empty pair metadata. Reused pairs keep theirs, including archive state.
|
||||
Every success registers the positive as a Library parent.
|
||||
"""
|
||||
_validate_epoch(expected_epoch)
|
||||
validate_prompt_text(positive_text)
|
||||
_validate_negative_text(negative_text)
|
||||
validate_library_metadata(parent_metadata, require_title=True)
|
||||
parent_metadata = normalize_library_metadata(parent_metadata)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
positive_id, positive_created = self._pair_component_outcome(
|
||||
conn, positive_text
|
||||
)
|
||||
negative_id = self._recipe_negative(conn, negative_text)
|
||||
pair_id, pair_created = self._save_pair_components(
|
||||
conn, positive_id, negative_id
|
||||
)
|
||||
self._ensure_library_parent(conn, positive_id)
|
||||
if positive_created:
|
||||
current = self._library_parent(conn, epoch, positive_id)
|
||||
if current is None:
|
||||
raise sqlite3.DatabaseError("New Library parent is not eligible")
|
||||
self._apply_library_metadata(conn, current, parent_metadata)
|
||||
return self._recipe_receipt(
|
||||
conn,
|
||||
epoch,
|
||||
positive_id,
|
||||
pair_id,
|
||||
positive_created=positive_created,
|
||||
pair_created=pair_created,
|
||||
)
|
||||
|
||||
def add_library_negative(
|
||||
self,
|
||||
positive_id: int,
|
||||
negative_text: str,
|
||||
*,
|
||||
expected_epoch: str,
|
||||
if_match: str,
|
||||
) -> dict | None:
|
||||
"""Create or reuse a pair under the exact stored positive.
|
||||
|
||||
The stored positive ID and text are used as they are; only the negative
|
||||
is resolved. Returns None when the positive is missing or not a Library
|
||||
parent, and raises LibraryCommandError for a rotated epoch or a stale
|
||||
token, in both cases before any insertion.
|
||||
"""
|
||||
_validate_guards(positive_id, expected_epoch, if_match)
|
||||
_validate_negative_text(negative_text)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
current = self._library_parent(conn, epoch, positive_id)
|
||||
if current is None:
|
||||
return None
|
||||
if current["edit_token"] != if_match:
|
||||
raise LibraryCommandError("metadata-conflict", epoch, current)
|
||||
negative_id = self._recipe_negative(conn, negative_text)
|
||||
pair_id, pair_created = self._save_pair_components(
|
||||
conn, positive_id, negative_id
|
||||
)
|
||||
self._ensure_library_parent(conn, positive_id)
|
||||
return self._recipe_receipt(
|
||||
conn,
|
||||
epoch,
|
||||
positive_id,
|
||||
pair_id,
|
||||
positive_created=False,
|
||||
pair_created=pair_created,
|
||||
)
|
||||
@@ -1,86 +0,0 @@
|
||||
"""Optional-old, required-current schema for Library parent state."""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
TABLE = "prompt_library_state"
|
||||
MAX_TITLE_LENGTH = 10_000
|
||||
_DDL = (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, title TEXT, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
|
||||
)
|
||||
_COLUMNS = (("prompt_id", "INTEGER", 0, 1), ("title", "TEXT", 0, 0))
|
||||
_FOREIGN_KEYS = {("prompt_id", "prompts", "id", "CASCADE")}
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def library_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> str | None:
|
||||
"""Reject unsupported Library state schemas and invalid persisted values.
|
||||
|
||||
Reads only: nothing is ever repaired. A missing table is acceptable for
|
||||
backups that predate it unless ``allow_missing`` is false.
|
||||
"""
|
||||
objects = conn.execute(
|
||||
"SELECT type,sql FROM sqlite_master WHERE name=? "
|
||||
"OR (type IN ('index','trigger') AND tbl_name=?)",
|
||||
(TABLE, TABLE),
|
||||
).fetchall()
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing Library state schema"
|
||||
tables = [sql for kind, sql in objects if kind == "table"]
|
||||
if len(tables) != 1 or len(objects) != 1:
|
||||
return "Library state name is reserved by an unsupported schema object"
|
||||
if _compact(tables[0] or "") != _compact(_DDL):
|
||||
return "Library state has an invalid table definition"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({TABLE})")
|
||||
)
|
||||
if columns != _COLUMNS:
|
||||
return "Library state has invalid columns"
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({TABLE})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS:
|
||||
return "Library state has invalid foreign key definitions"
|
||||
if conn.execute(f"PRAGMA foreign_key_check({TABLE})").fetchone():
|
||||
return "Library state contains foreign key violations"
|
||||
# length() stops at an embedded NUL, so bound by bytes in SQL (characters
|
||||
# never outnumber bytes, and UTF-8 uses at most 4 bytes per character) and
|
||||
# count the few remaining candidates as decoded characters.
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(title) NOT IN ('null','text') "
|
||||
f"OR length(CAST(title AS BLOB)) > {MAX_TITLE_LENGTH * 4} LIMIT 1"
|
||||
).fetchone():
|
||||
return "Library state contains invalid titles"
|
||||
for (raw,) in conn.execute(
|
||||
f"SELECT CAST(title AS BLOB) FROM {TABLE} "
|
||||
f"WHERE length(CAST(title AS BLOB)) > {MAX_TITLE_LENGTH}"
|
||||
):
|
||||
if len(bytes(raw).decode("utf-8", "replace")) > MAX_TITLE_LENGTH:
|
||||
return "Library state contains invalid titles"
|
||||
return None
|
||||
|
||||
|
||||
def migrate_library_state(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table atomically; validate before and after creating it."""
|
||||
conn.execute("SAVEPOINT prompt_library_state_schema")
|
||||
try:
|
||||
error = library_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
error = library_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE prompt_library_state_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO prompt_library_state_schema")
|
||||
conn.execute("RELEASE prompt_library_state_schema")
|
||||
raise
|
||||
@@ -1,37 +0,0 @@
|
||||
"""Narrow comparison tokens for a Library parent's own metadata projection."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
_DOMAIN = "prompt-manager-library-parent:v1\n"
|
||||
|
||||
|
||||
def canonical_token(domain: str, preimage: dict) -> str:
|
||||
"""SHA-256 of ``domain`` plus the canonical JSON of ``preimage``."""
|
||||
payload = json.dumps(
|
||||
preimage,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
)
|
||||
return hashlib.sha256((domain + payload).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def library_edit_token(epoch: str, positive: dict) -> str:
|
||||
"""Hash the parent's owner fields under the current storage epoch.
|
||||
|
||||
Timestamps, counts, pairs and previews are excluded, and an absent state
|
||||
row projects the same null title as a default one.
|
||||
"""
|
||||
preimage = {
|
||||
"epoch": epoch,
|
||||
"id": positive["id"],
|
||||
"text": positive["text"],
|
||||
"title": positive["title"],
|
||||
"category": positive["category"],
|
||||
"notes": positive["notes"],
|
||||
"rating": positive["rating"],
|
||||
"tags": sorted(set(positive["tags"])),
|
||||
}
|
||||
return canonical_token(_DOMAIN, preimage)
|
||||
@@ -1,136 +0,0 @@
|
||||
"""Atomic metadata enrichment for a single known generation context."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
|
||||
from .models import normalize_image_path
|
||||
from .operations import _coerce_file_info
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
def ingest_resolved_pair(
|
||||
db: Any, image_path: str, metadata: Dict[str, Any], resolution: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
state = resolution.get("state", "unresolved")
|
||||
if state != "resolved":
|
||||
return {"status": state, "pairs_added": 0, "links_added": 0}
|
||||
positive, negative = resolution["positive"], resolution["negative"]
|
||||
positive_text = positive.get("storage_text", positive["text"])
|
||||
negative_text = negative.get("storage_text", negative["text"])
|
||||
normalized = normalize_image_path(image_path)
|
||||
provenance = {"resolver_version": 1, "positive": positive, "negative": negative}
|
||||
encoded = json.dumps(
|
||||
provenance, sort_keys=True, separators=(",", ":"), allow_nan=False
|
||||
)
|
||||
with db._pair_transaction() as conn:
|
||||
images = conn.execute(
|
||||
"SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
|
||||
"JOIN prompts p ON p.id=gi.prompt_id "
|
||||
"WHERE gi.file_path=? OR gi.image_path=?",
|
||||
(normalized, image_path),
|
||||
).fetchall()
|
||||
if len(images) > 1 or (
|
||||
images and images[0]["positive_hash"] != generate_prompt_hash(positive_text)
|
||||
):
|
||||
return {"status": "conflict", "pairs_added": 0, "links_added": 0}
|
||||
existing = images[0] if images else None
|
||||
if existing:
|
||||
prior = conn.execute(
|
||||
"SELECT n.hash AS negative_hash FROM image_prompt_pairs ip "
|
||||
"JOIN prompt_pairs pp ON pp.id=ip.pair_id "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id WHERE ip.image_id=?",
|
||||
(existing["id"],),
|
||||
).fetchall()
|
||||
expected = (
|
||||
generate_prompt_hash(negative_text) if negative_text.strip() else None
|
||||
)
|
||||
if any(row["negative_hash"] != expected for row in prior):
|
||||
return {"status": "conflict", "pairs_added": 0, "links_added": 0}
|
||||
prior_pair = conn.execute(
|
||||
"SELECT pp.id FROM prompt_pairs pp "
|
||||
"JOIN prompts p ON p.id=pp.positive_prompt_id "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
||||
"WHERE p.hash=? AND n.hash IS ?",
|
||||
(
|
||||
generate_prompt_hash(positive_text),
|
||||
generate_prompt_hash(negative_text) if negative_text.strip() else None,
|
||||
),
|
||||
).fetchone()
|
||||
pair_id = db._save_pair(conn, positive_text, negative_text)
|
||||
pair = conn.execute(
|
||||
"SELECT * FROM prompt_pairs WHERE id=?", (pair_id,)
|
||||
).fetchone()
|
||||
pairs_added = int(prior_pair is None)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
snapshots = {
|
||||
column: json.dumps(metadata[key], sort_keys=True, allow_nan=False)
|
||||
for column, key in (
|
||||
("workflow_data", "workflow"),
|
||||
("prompt_metadata", "prompt"),
|
||||
("parameters", "parameters"),
|
||||
)
|
||||
if key in metadata
|
||||
}
|
||||
info = {
|
||||
"file_size": file_size,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"format": image_format,
|
||||
}
|
||||
snapshots.update(
|
||||
{key: value for key, value in info.items() if value is not None}
|
||||
)
|
||||
changed = False
|
||||
if existing:
|
||||
image_id = existing["id"]
|
||||
updates = {
|
||||
key: value for key, value in snapshots.items() if existing[key] != value
|
||||
}
|
||||
if updates:
|
||||
conn.execute(
|
||||
"UPDATE generated_images SET "
|
||||
+ ",".join(f"{key}=?" for key in updates)
|
||||
+ " WHERE id=?",
|
||||
[*updates.values(), image_id],
|
||||
)
|
||||
changed = True
|
||||
else:
|
||||
columns = ["prompt_id", "image_path", "filename", "file_path", *snapshots]
|
||||
values = [
|
||||
pair["positive_prompt_id"],
|
||||
image_path,
|
||||
os.path.basename(image_path),
|
||||
normalized,
|
||||
*snapshots.values(),
|
||||
]
|
||||
image_id = conn.execute(
|
||||
"INSERT INTO generated_images("
|
||||
+ ",".join(columns)
|
||||
+ ") VALUES("
|
||||
+ ",".join("?" for _ in columns)
|
||||
+ ")",
|
||||
values,
|
||||
).lastrowid
|
||||
linked = conn.execute(
|
||||
"SELECT 1 FROM image_prompt_pairs WHERE pair_id=? "
|
||||
"AND image_id=? AND provenance=?",
|
||||
(pair_id, image_id, encoded),
|
||||
).fetchone()
|
||||
db._link_image_to_pair(conn, pair_id, image_id, provenance)
|
||||
status = (
|
||||
"added"
|
||||
if existing is None
|
||||
else "enriched" if changed or not linked else "unchanged"
|
||||
)
|
||||
return {
|
||||
"status": status,
|
||||
"pairs_added": pairs_added,
|
||||
"links_added": int(not linked),
|
||||
"pair_id": pair_id,
|
||||
"image_id": image_id,
|
||||
}
|
||||
@@ -1,433 +0,0 @@
|
||||
"""Ordered reusable prompt pairs and their generation provenance."""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Dict, Iterator, List, Optional
|
||||
|
||||
from .models import pair_provenance_error, utc_now_iso
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
|
||||
class PairDependencyError(ValueError):
|
||||
"""An operation would change or remove content used by a saved pair."""
|
||||
|
||||
|
||||
def _validate_metadata(
|
||||
title: Optional[str],
|
||||
category: Optional[str],
|
||||
tags: Optional[List[str]],
|
||||
rating: Optional[int],
|
||||
notes: Optional[str],
|
||||
) -> Optional[List[str]]:
|
||||
for name, value in (("title", title), ("category", category), ("notes", notes)):
|
||||
if value is not None and not isinstance(value, str):
|
||||
raise ValueError(f"{name} must be a string")
|
||||
if rating is not None and (type(rating) is not int or not 1 <= rating <= 5):
|
||||
raise ValueError("Rating must be between 1 and 5")
|
||||
if tags is None:
|
||||
return None
|
||||
if not isinstance(tags, list) or any(not isinstance(tag, str) for tag in tags):
|
||||
raise ValueError("tags must be a list of strings")
|
||||
return list(dict.fromkeys(tag.strip() for tag in tags if tag.strip()))
|
||||
|
||||
|
||||
class PairOperations:
|
||||
"""Connection-level pair operations mixed into PromptDatabase."""
|
||||
|
||||
@contextmanager
|
||||
def _pair_transaction(self) -> Iterator[sqlite3.Connection]:
|
||||
conn = self.model.get_connection()
|
||||
if conn.in_transaction:
|
||||
raise RuntimeError("Pair operations require their own transaction")
|
||||
with conn:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
yield conn
|
||||
|
||||
def _ensure_prompts_unpaired(
|
||||
self, conn: sqlite3.Connection, prompt_ids: List[int]
|
||||
) -> None:
|
||||
if not prompt_ids:
|
||||
return
|
||||
placeholders = ",".join("?" for _ in prompt_ids)
|
||||
row = conn.execute(
|
||||
f"SELECT id FROM prompt_pairs WHERE positive_prompt_id IN ({placeholders}) "
|
||||
f"OR negative_prompt_id IN ({placeholders}) LIMIT 1",
|
||||
[*prompt_ids, *prompt_ids],
|
||||
).fetchone()
|
||||
if row:
|
||||
raise PairDependencyError(
|
||||
f"Prompt belongs to saved pair {row['id']}; delete the pair first "
|
||||
"or save changed text as a new prompt"
|
||||
)
|
||||
|
||||
def _pair_component(self, conn: sqlite3.Connection, text: str) -> int:
|
||||
return self._pair_component_outcome(conn, text)[0]
|
||||
|
||||
def _pair_component_outcome(
|
||||
self, conn: sqlite3.Connection, text: str
|
||||
) -> tuple[int, bool]:
|
||||
"""Resolve a component by its dedup hash; report whether it was inserted.
|
||||
|
||||
The first stored spelling wins: the hash ignores outer whitespace and
|
||||
case, internal whitespace stays significant. ``created`` comes from the
|
||||
INSERT result under the caller's held write transaction.
|
||||
"""
|
||||
prompt_hash = generate_prompt_hash(text)
|
||||
now = utc_now_iso()
|
||||
created = (
|
||||
conn.execute(
|
||||
"INSERT INTO prompts(text, hash, created_at, updated_at) "
|
||||
"VALUES(?,?,?,?) ON CONFLICT(hash) DO NOTHING",
|
||||
(text.strip(), prompt_hash, now, now),
|
||||
).rowcount
|
||||
== 1
|
||||
)
|
||||
row = conn.execute(
|
||||
"SELECT id FROM prompts WHERE hash = ?", (prompt_hash,)
|
||||
).fetchone()
|
||||
return row["id"], created
|
||||
|
||||
def _sync_pair_tags(
|
||||
self, conn: sqlite3.Connection, pair_id: int, tags: List[str]
|
||||
) -> None:
|
||||
conn.execute("DELETE FROM prompt_pair_tags WHERE pair_id = ?", (pair_id,))
|
||||
for tag_id in self._ensure_tags(conn, tags).values():
|
||||
conn.execute(
|
||||
"INSERT INTO prompt_pair_tags(pair_id,tag_id) VALUES(?,?)",
|
||||
(pair_id, tag_id),
|
||||
)
|
||||
|
||||
def save_prompt_pair(
|
||||
self,
|
||||
positive_text: str,
|
||||
negative_text: str = "",
|
||||
*,
|
||||
title: Optional[str] = None,
|
||||
category: Optional[str] = None,
|
||||
tags: Optional[List[str]] = None,
|
||||
rating: Optional[int] = None,
|
||||
notes: Optional[str] = None,
|
||||
) -> int:
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata."""
|
||||
with self._pair_transaction() as conn:
|
||||
return self._save_pair(
|
||||
conn,
|
||||
positive_text,
|
||||
negative_text,
|
||||
title=title,
|
||||
category=category,
|
||||
tags=tags,
|
||||
rating=rating,
|
||||
notes=notes,
|
||||
)
|
||||
|
||||
def _save_pair(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
positive_text: str,
|
||||
negative_text: str = "",
|
||||
*,
|
||||
title: Optional[str] = None,
|
||||
category: Optional[str] = None,
|
||||
tags: Optional[List[str]] = None,
|
||||
rating: Optional[int] = None,
|
||||
notes: Optional[str] = None,
|
||||
) -> int:
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata."""
|
||||
if not isinstance(positive_text, str) or not positive_text.strip():
|
||||
raise ValueError("Positive prompt text cannot be empty")
|
||||
if not isinstance(negative_text, str):
|
||||
raise ValueError("Negative prompt text must be a string; use '' for empty")
|
||||
# Validate metadata before any component insertion.
|
||||
_validate_metadata(title, category, tags, rating, notes)
|
||||
positive = self._pair_component(conn, positive_text)
|
||||
negative = (
|
||||
self._pair_component(conn, negative_text) if negative_text.strip() else None
|
||||
)
|
||||
return self._save_pair_components(
|
||||
conn,
|
||||
positive,
|
||||
negative,
|
||||
title=title,
|
||||
category=category,
|
||||
tags=tags,
|
||||
rating=rating,
|
||||
notes=notes,
|
||||
)[0]
|
||||
|
||||
def _save_pair_components(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
positive_id: int,
|
||||
negative_id: Optional[int],
|
||||
*,
|
||||
title: Optional[str] = None,
|
||||
category: Optional[str] = None,
|
||||
tags: Optional[List[str]] = None,
|
||||
rating: Optional[int] = None,
|
||||
notes: Optional[str] = None,
|
||||
) -> tuple[int, bool]:
|
||||
"""Create or reuse the ordered pair of existing component IDs.
|
||||
|
||||
Returns ``(pair_id, created)``. A reused pair keeps all of its metadata
|
||||
and archive state; metadata applies only to a newly inserted pair.
|
||||
"""
|
||||
tags = _validate_metadata(title, category, tags, rating, notes)
|
||||
existing = conn.execute(
|
||||
"SELECT id FROM prompt_pairs WHERE positive_prompt_id = ? "
|
||||
"AND negative_prompt_id IS ?",
|
||||
(positive_id, negative_id),
|
||||
).fetchone()
|
||||
if existing:
|
||||
return existing["id"], False
|
||||
now = utc_now_iso()
|
||||
pair_id = conn.execute(
|
||||
"INSERT INTO prompt_pairs(positive_prompt_id,negative_prompt_id,title,"
|
||||
"category,rating,notes,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)",
|
||||
(positive_id, negative_id, title, category, rating, notes, now, now),
|
||||
).lastrowid
|
||||
if tags:
|
||||
self._sync_pair_tags(conn, pair_id, tags)
|
||||
return pair_id, True
|
||||
|
||||
def _read_pair_rows(
|
||||
self, conn: sqlite3.Connection, where: str, params: List[Any], suffix: str = ""
|
||||
) -> List[Dict[str, Any]]:
|
||||
rows = conn.execute(
|
||||
"SELECT pp.*, p.text AS positive_text, "
|
||||
"COALESCE(n.text,'') AS negative_text,"
|
||||
"(SELECT COUNT(DISTINCT image_id) FROM image_prompt_pairs ip "
|
||||
"WHERE ip.pair_id=pp.id) AS image_count FROM prompt_pairs pp "
|
||||
"JOIN prompts p ON p.id=pp.positive_prompt_id "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id WHERE " + where + suffix,
|
||||
params,
|
||||
).fetchall()
|
||||
pairs = [dict(row) for row in rows]
|
||||
if not pairs:
|
||||
return pairs
|
||||
by_id = {pair["id"]: pair for pair in pairs}
|
||||
for pair in pairs:
|
||||
pair["archived"] = bool(pair["archived"])
|
||||
pair["tags"] = []
|
||||
placeholders = ",".join("?" for _ in by_id)
|
||||
for row in conn.execute(
|
||||
"SELECT pt.pair_id,t.name FROM prompt_pair_tags pt JOIN tags t ON "
|
||||
f"t.id=pt.tag_id WHERE pt.pair_id IN ({placeholders}) ORDER BY t.name",
|
||||
list(by_id),
|
||||
):
|
||||
by_id[row["pair_id"]]["tags"].append(row["name"])
|
||||
return pairs
|
||||
|
||||
def get_prompt_pair(self, pair_id: int) -> Optional[Dict[str, Any]]:
|
||||
"""Return components, pair metadata and image count, including archives."""
|
||||
rows = self._read_pair_rows(self.model.get_connection(), "pp.id = ?", [pair_id])
|
||||
return rows[0] if rows else None
|
||||
|
||||
def list_prompt_pairs(
|
||||
self,
|
||||
*,
|
||||
text: Optional[str] = None,
|
||||
category: Optional[str] = None,
|
||||
tags: Optional[List[str]] = None,
|
||||
include_archived: bool = False,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
sort: str = "newest",
|
||||
) -> Dict[str, Any]:
|
||||
"""List pairs with all-tag filtering and stable paginated ordering."""
|
||||
orders = {
|
||||
"newest": "pp.id DESC",
|
||||
"last_used_desc": "julianday(pp.last_used_at) DESC, pp.id DESC",
|
||||
}
|
||||
if sort not in orders:
|
||||
raise ValueError("Unknown pair sort order")
|
||||
if type(limit) is not int or not 1 <= limit <= 500:
|
||||
raise ValueError("limit must be between 1 and 500")
|
||||
if type(offset) is not int or offset < 0:
|
||||
raise ValueError("offset must be a nonnegative integer")
|
||||
tags = _validate_metadata(None, category, tags, None, None)
|
||||
if text is not None and not isinstance(text, str):
|
||||
raise ValueError("text must be a string")
|
||||
clauses, params = ["1=1"], []
|
||||
if not include_archived:
|
||||
clauses.append("pp.archived=0")
|
||||
if category is not None:
|
||||
clauses.append("pp.category = ?")
|
||||
params.append(category)
|
||||
if text:
|
||||
clauses.append("(p.text LIKE ? OR n.text LIKE ? OR pp.title LIKE ?)")
|
||||
params.extend([f"%{text}%"] * 3)
|
||||
for tag in tags or []:
|
||||
clauses.append(
|
||||
"EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t "
|
||||
"ON t.id=pt.tag_id WHERE pt.pair_id=pp.id AND t.name=?)"
|
||||
)
|
||||
params.append(tag)
|
||||
where = " AND ".join(clauses)
|
||||
conn = self.model.get_connection()
|
||||
total = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompt_pairs pp "
|
||||
"JOIN prompts p ON p.id=pp.positive_prompt_id "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id WHERE " + where,
|
||||
params,
|
||||
).fetchone()[0]
|
||||
pairs = self._read_pair_rows(
|
||||
conn,
|
||||
where,
|
||||
[*params, limit, offset],
|
||||
f" ORDER BY {orders[sort]} LIMIT ? OFFSET ?",
|
||||
)
|
||||
return {
|
||||
"pairs": pairs,
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"has_more": offset + len(pairs) < total,
|
||||
}
|
||||
|
||||
def update_prompt_pair_metadata(
|
||||
self,
|
||||
pair_id: int,
|
||||
*,
|
||||
title: Optional[str] = None,
|
||||
category: Optional[str] = None,
|
||||
tags: Optional[List[str]] = None,
|
||||
rating: Optional[int] = None,
|
||||
notes: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Update pair metadata explicitly; None preserves and empty values clear."""
|
||||
tags = _validate_metadata(title, category, tags, rating, notes)
|
||||
changes = {
|
||||
key: value
|
||||
for key, value in (
|
||||
("title", title),
|
||||
("category", category),
|
||||
("rating", rating),
|
||||
("notes", notes),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
if not changes and tags is None:
|
||||
return False
|
||||
with self._pair_transaction() as conn:
|
||||
if not conn.execute(
|
||||
"SELECT 1 FROM prompt_pairs WHERE id=?", (pair_id,)
|
||||
).fetchone():
|
||||
return False
|
||||
changes["updated_at"] = utc_now_iso()
|
||||
conn.execute(
|
||||
"UPDATE prompt_pairs SET "
|
||||
+ ",".join(f"{key}=?" for key in changes)
|
||||
+ " WHERE id=?",
|
||||
[*changes.values(), pair_id],
|
||||
)
|
||||
if tags is not None:
|
||||
self._sync_pair_tags(conn, pair_id, tags)
|
||||
return True
|
||||
|
||||
def archive_prompt_pair(self, pair_id: int, archived: bool = True) -> bool:
|
||||
"""Hide a reusable pair without changing components or generation history."""
|
||||
if type(archived) is not bool:
|
||||
raise ValueError("archived must be a boolean")
|
||||
with self._pair_transaction() as conn:
|
||||
return (
|
||||
conn.execute(
|
||||
"UPDATE prompt_pairs SET archived=?,updated_at=? WHERE id=?",
|
||||
(int(archived), utc_now_iso(), pair_id),
|
||||
).rowcount
|
||||
> 0
|
||||
)
|
||||
|
||||
def delete_prompt_pair(self, pair_id: int) -> bool:
|
||||
"""Delete unused pair metadata; generated pairs must be archived instead."""
|
||||
with self._pair_transaction() as conn:
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM image_prompt_pairs WHERE pair_id=? LIMIT 1", (pair_id,)
|
||||
).fetchone():
|
||||
raise PairDependencyError(
|
||||
"Pair has generated images; archive it instead"
|
||||
)
|
||||
return (
|
||||
conn.execute("DELETE FROM prompt_pairs WHERE id=?", (pair_id,)).rowcount
|
||||
> 0
|
||||
)
|
||||
|
||||
def record_prompt_pair_use(self, pair_id: int, times: int = 1) -> bool:
|
||||
"""Count pair uses independently of standalone prompt usage."""
|
||||
if type(times) is not int or times < 1:
|
||||
raise ValueError("times must be a positive integer")
|
||||
with self._pair_transaction() as conn:
|
||||
return (
|
||||
conn.execute(
|
||||
"UPDATE prompt_pairs SET run_count=run_count+?,last_used_at=? "
|
||||
"WHERE id=?",
|
||||
(times, utc_now_iso(), pair_id),
|
||||
).rowcount
|
||||
> 0
|
||||
)
|
||||
|
||||
def link_image_to_pair(
|
||||
self, pair_id: int, image_id: int, provenance: Dict[str, Any]
|
||||
) -> int:
|
||||
"""Associate an image with immutable run provenance, idempotently."""
|
||||
with self._pair_transaction() as conn:
|
||||
return self._link_image_to_pair(conn, pair_id, image_id, provenance)
|
||||
|
||||
def _link_image_to_pair(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
pair_id: int,
|
||||
image_id: int,
|
||||
provenance: Dict[str, Any],
|
||||
) -> int:
|
||||
"""Associate an image with immutable run provenance, idempotently."""
|
||||
if not isinstance(provenance, dict):
|
||||
raise ValueError("provenance must be an object")
|
||||
try:
|
||||
encoded = json.dumps(
|
||||
provenance, sort_keys=True, separators=(",", ":"), allow_nan=False
|
||||
)
|
||||
except (TypeError, ValueError, RecursionError) as error:
|
||||
raise ValueError("provenance must contain JSON values") from error
|
||||
error = pair_provenance_error(encoded)
|
||||
if error:
|
||||
raise ValueError(error)
|
||||
pair = conn.execute(
|
||||
"SELECT positive_prompt_id FROM prompt_pairs WHERE id=?", (pair_id,)
|
||||
).fetchone()
|
||||
image = conn.execute(
|
||||
"SELECT prompt_id FROM generated_images WHERE id=?", (image_id,)
|
||||
).fetchone()
|
||||
if not pair or not image:
|
||||
raise ValueError("Pair and image must exist")
|
||||
if pair["positive_prompt_id"] != image["prompt_id"]:
|
||||
raise ValueError("Image must belong to the pair's positive prompt")
|
||||
conn.execute(
|
||||
"INSERT INTO image_prompt_pairs(pair_id,image_id,provenance,created_at) "
|
||||
"VALUES(?,?,?,?) ON CONFLICT(pair_id,image_id,provenance) DO NOTHING",
|
||||
(pair_id, image_id, encoded, utc_now_iso()),
|
||||
)
|
||||
return conn.execute(
|
||||
"SELECT id FROM image_prompt_pairs "
|
||||
"WHERE pair_id=? AND image_id=? AND provenance=?",
|
||||
(pair_id, image_id, encoded),
|
||||
).fetchone()["id"]
|
||||
|
||||
def get_image_prompt_pairs(self, image_id: int) -> List[Dict[str, Any]]:
|
||||
"""Return every recorded pair association with decoded provenance."""
|
||||
rows = (
|
||||
self.model.get_connection()
|
||||
.execute(
|
||||
"SELECT * FROM image_prompt_pairs WHERE image_id=? ORDER BY id",
|
||||
(image_id,),
|
||||
)
|
||||
.fetchall()
|
||||
)
|
||||
return [
|
||||
{**dict(row), "provenance": json.loads(row["provenance"])} for row in rows
|
||||
]
|
||||
@@ -1,435 +0,0 @@
|
||||
"""Bounded tag and category reads, and guarded tag commands.
|
||||
|
||||
A count is the number of eligible Library parents that carry the value
|
||||
directly or through one active saved pair, which is exactly what
|
||||
``list_library(tags=[name])`` / ``list_library(category=value)`` return.
|
||||
|
||||
Tag commands run in one ``_pair_transaction``: the storage epoch is checked
|
||||
first, then the exact ``{id, name}`` identity, and the receipt is read on the
|
||||
same connection before commit. Owner rows and their timestamps never change.
|
||||
|
||||
Category consolidation is previewed in one ``library_snapshot`` and applied in
|
||||
one ``_pair_transaction`` that recomputes the owner projection and requires the
|
||||
preview's ``impact_token``; only rows carrying the exact source change.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
|
||||
from .library import _ELIGIBLE, _limit
|
||||
from .library_commands import _TOKEN, LibraryCommandError, _validate_epoch
|
||||
from .library_tokens import canonical_token
|
||||
from .models import utc_now_iso
|
||||
|
||||
try:
|
||||
from ..utils.validators import validate_category, validate_tags
|
||||
except ImportError:
|
||||
from utils.validators import validate_category, validate_tags
|
||||
|
||||
_CATEGORY_DOMAIN = "prompt-manager-vocabulary-category:v1\n"
|
||||
|
||||
|
||||
def _tag_rows(page: str) -> str:
|
||||
"""Count rows for the registry rows selected by ``page`` (id, name)."""
|
||||
# Eligible parent components carrying a tag in the page, plus the
|
||||
# positive parent of every active pair carrying it (always eligible).
|
||||
return (
|
||||
"WITH page AS (" + page + "),"
|
||||
"owners(tag_id,pid) AS ("
|
||||
"SELECT pt.tag_id,p.id FROM prompts p "
|
||||
"JOIN prompt_tags pt ON pt.prompt_id=p.id "
|
||||
"WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id "
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count "
|
||||
"FROM page LEFT JOIN owners ON owners.tag_id=page.id "
|
||||
"GROUP BY page.id ORDER BY page.id"
|
||||
)
|
||||
|
||||
|
||||
_TAG_PAGE = _tag_rows("SELECT id,name FROM tags WHERE id>? ORDER BY id LIMIT ?")
|
||||
_TAG_ONE = _tag_rows("SELECT id,name FROM tags WHERE id=?")
|
||||
# Every owner kind a tag can sit on, for the delete confirmation.
|
||||
_TAG_IMPACT = (
|
||||
"SELECT "
|
||||
"(SELECT COUNT(*) FROM prompts p JOIN prompt_tags pt ON pt.prompt_id=p.id "
|
||||
"WHERE pt.tag_id=:tag AND " + _ELIGIBLE + ") AS positive_prompts,"
|
||||
"(SELECT COUNT(*) FROM prompts p JOIN prompt_tags pt ON pt.prompt_id=p.id "
|
||||
"WHERE pt.tag_id=:tag AND NOT " + _ELIGIBLE + ") AS other_prompts,"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs pp JOIN prompt_pair_tags ppt "
|
||||
"ON ppt.pair_id=pp.id WHERE ppt.tag_id=:tag AND pp.archived=0) "
|
||||
"AS negative_variants,"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs pp JOIN prompt_pair_tags ppt "
|
||||
"ON ppt.pair_id=pp.id WHERE ppt.tag_id=:tag AND pp.archived!=0) "
|
||||
"AS archived_negative_variants"
|
||||
)
|
||||
_MAX_ID = 2**63 - 1
|
||||
|
||||
|
||||
def validate_vocabulary_tag_name(value: object) -> str:
|
||||
"""Return the trimmed name, applying tag rules plus the native comma ban."""
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("Tag name must be a string")
|
||||
validate_tags([value])
|
||||
name = value.strip()
|
||||
if "," in name:
|
||||
raise ValueError("Tag names cannot contain commas")
|
||||
return name
|
||||
|
||||
|
||||
def _validate_tag_id(tag_id: object) -> None:
|
||||
if type(tag_id) is not int or not 1 <= tag_id <= _MAX_ID:
|
||||
raise ValueError("Invalid tag ID")
|
||||
|
||||
|
||||
# The /library/options universe: nonblank values on eligible parents or on
|
||||
# active pairs, grouped exactly (whitespace and case variants stay distinct).
|
||||
_CATEGORY_GROUPS = (
|
||||
"WITH owners(value,pid) AS ("
|
||||
"SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL "
|
||||
"AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!=''),"
|
||||
"grouped AS (SELECT value,MIN(pid) AS first_pid,"
|
||||
"COUNT(DISTINCT pid) AS prompt_count FROM owners GROUP BY value) "
|
||||
)
|
||||
|
||||
|
||||
# Every owner of one exact category value, in id order, with the flag the
|
||||
# impact counts split on.
|
||||
_CATEGORY_COMPONENTS = (
|
||||
"SELECT p.id," + _ELIGIBLE + " AS eligible FROM prompts p "
|
||||
"WHERE p.category=? ORDER BY p.id"
|
||||
)
|
||||
_CATEGORY_PAIRS = "SELECT id,archived FROM prompt_pairs WHERE category=? ORDER BY id"
|
||||
_CATEGORY_EXISTS = (
|
||||
"SELECT EXISTS(SELECT 1 FROM prompts WHERE category=:v)"
|
||||
" OR EXISTS(SELECT 1 FROM prompt_pairs WHERE category=:v)"
|
||||
)
|
||||
|
||||
# Pairs after components, so a failing pair update rolls both back.
|
||||
_CATEGORY_UPDATES = (
|
||||
"UPDATE prompts SET category=?,updated_at=? WHERE category=?",
|
||||
"UPDATE prompt_pairs SET category=?,updated_at=? WHERE category=?",
|
||||
)
|
||||
|
||||
|
||||
def category_impact_token(
|
||||
epoch: str,
|
||||
source: str,
|
||||
target: str,
|
||||
target_exists: bool,
|
||||
components: list,
|
||||
pairs: list,
|
||||
) -> str:
|
||||
"""Hash a category projection like ``library_edit_token`` (no secret)."""
|
||||
preimage = {
|
||||
"epoch": epoch,
|
||||
"source": source,
|
||||
"target": target,
|
||||
"target_exists": bool(target_exists),
|
||||
"components": sorted([int(i), bool(flag)] for i, flag in components),
|
||||
"pairs": sorted([int(i), bool(flag)] for i, flag in pairs),
|
||||
}
|
||||
return canonical_token(_CATEGORY_DOMAIN, preimage)
|
||||
|
||||
|
||||
def _validate_category_change(source: object, target: object) -> str:
|
||||
"""Check the exact source and return the trimmed, different target."""
|
||||
if not isinstance(source, str) or not source.strip():
|
||||
raise ValueError("source must be a nonblank category value")
|
||||
if not isinstance(target, str):
|
||||
raise ValueError("Category must be a string")
|
||||
validate_category(target)
|
||||
target = target.strip()
|
||||
if not target:
|
||||
raise ValueError("New category name cannot be blank")
|
||||
if target == source:
|
||||
raise ValueError("New category name must differ from the current one")
|
||||
return target
|
||||
|
||||
|
||||
def _validate_impact_token(impact_token: object) -> None:
|
||||
if not isinstance(impact_token, str) or not _TOKEN.fullmatch(impact_token):
|
||||
raise ValueError("impact_token must be 64 lowercase hexadecimal characters")
|
||||
|
||||
|
||||
def _tag_after(after: object) -> int:
|
||||
if after is None:
|
||||
return 0
|
||||
if type(after) is not int or after <= 0:
|
||||
raise ValueError("Invalid vocabulary cursor")
|
||||
return after
|
||||
|
||||
|
||||
def _category_after(after: object) -> tuple[int, str] | None:
|
||||
if after is None:
|
||||
return None
|
||||
if (
|
||||
not isinstance(after, list)
|
||||
or len(after) != 2
|
||||
or type(after[0]) is not int
|
||||
or not isinstance(after[1], str)
|
||||
):
|
||||
raise ValueError("Invalid vocabulary cursor")
|
||||
return after[0], after[1]
|
||||
|
||||
|
||||
class VocabularyOperations:
|
||||
"""Vocabulary reads and tag commands.
|
||||
|
||||
``_`` helpers take an open ``library_snapshot`` or ``_pair_transaction``
|
||||
connection.
|
||||
"""
|
||||
|
||||
def _vocabulary_tag(self, conn: sqlite3.Connection, tag_id: int) -> dict | None:
|
||||
row = conn.execute(_TAG_ONE, (tag_id,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def _vocabulary_identity(
|
||||
self, conn: sqlite3.Connection, epoch: str, tag_id: int, name: str
|
||||
) -> dict:
|
||||
"""The stored tag, or vocabulary-conflict when it is gone or renamed."""
|
||||
tag = self._vocabulary_tag(conn, tag_id)
|
||||
if tag is None or tag["name"] != name:
|
||||
raise LibraryCommandError("vocabulary-conflict", epoch)
|
||||
return tag
|
||||
|
||||
@staticmethod
|
||||
def _tag_owner_counts(conn: sqlite3.Connection, tag_id: int) -> dict:
|
||||
components = conn.execute(
|
||||
"SELECT COUNT(DISTINCT prompt_id) FROM prompt_tags WHERE tag_id=?",
|
||||
(tag_id,),
|
||||
).fetchone()[0]
|
||||
pairs = conn.execute(
|
||||
"SELECT COUNT(DISTINCT pair_id) FROM prompt_pair_tags WHERE tag_id=?",
|
||||
(tag_id,),
|
||||
).fetchone()[0]
|
||||
return {"components_changed": components, "pairs_changed": pairs}
|
||||
|
||||
def create_vocabulary_tag(self, name: str, *, expected_epoch: str) -> dict:
|
||||
"""Insert an unused registry row, or reuse the exact existing name."""
|
||||
_validate_epoch(expected_epoch)
|
||||
name = validate_vocabulary_tag_name(name)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
row = conn.execute("SELECT id FROM tags WHERE name=?", (name,)).fetchone()
|
||||
created = row is None
|
||||
if created:
|
||||
tag_id = conn.execute(
|
||||
"INSERT INTO tags(name) VALUES(?)", (name,)
|
||||
).lastrowid
|
||||
else:
|
||||
tag_id = row[0]
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"created": created,
|
||||
"tag": self._vocabulary_tag(conn, tag_id),
|
||||
}
|
||||
|
||||
def rename_vocabulary_tag(
|
||||
self, tag_id: int, *, expected_epoch: str, name: str, new_name: str
|
||||
) -> dict:
|
||||
"""Rename ``{tag_id, name}``, merging into an existing exact ``new_name``.
|
||||
|
||||
Links move across components and all pairs, archived included; an
|
||||
owner already carrying the target keeps one link.
|
||||
"""
|
||||
_validate_tag_id(tag_id)
|
||||
_validate_epoch(expected_epoch)
|
||||
if not isinstance(name, str):
|
||||
raise ValueError("name must be the tag's current name")
|
||||
new_name = validate_vocabulary_tag_name(new_name)
|
||||
if new_name == name:
|
||||
raise ValueError("New name must differ from the current name")
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
self._vocabulary_identity(conn, epoch, tag_id, name)
|
||||
counts = self._tag_owner_counts(conn, tag_id)
|
||||
target = conn.execute(
|
||||
"SELECT id FROM tags WHERE name=?", (new_name,)
|
||||
).fetchone()
|
||||
if target is None:
|
||||
conn.execute("UPDATE tags SET name=? WHERE id=?", (new_name, tag_id))
|
||||
result_id = tag_id
|
||||
else:
|
||||
result_id = target[0]
|
||||
self._move_tag_links(conn, tag_id, result_id)
|
||||
conn.execute("DELETE FROM tags WHERE id=?", (tag_id,))
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"merged": target is not None,
|
||||
"tag": self._vocabulary_tag(conn, result_id),
|
||||
**counts,
|
||||
}
|
||||
|
||||
def delete_vocabulary_tag(
|
||||
self, tag_id: int, *, expected_epoch: str, name: str
|
||||
) -> dict:
|
||||
"""Remove the registry row ``{tag_id, name}``; its links cascade."""
|
||||
_validate_tag_id(tag_id)
|
||||
_validate_epoch(expected_epoch)
|
||||
if not isinstance(name, str):
|
||||
raise ValueError("name must be the tag's current name")
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
self._vocabulary_identity(conn, epoch, tag_id, name)
|
||||
counts = self._tag_owner_counts(conn, tag_id)
|
||||
conn.execute("DELETE FROM tags WHERE id=?", (tag_id,))
|
||||
return {"library_epoch": epoch, "deleted": True, **counts}
|
||||
|
||||
def vocabulary_tag_impact(self, tag_id: int) -> dict | None:
|
||||
"""Owner counts by kind for one tag, or None when it does not exist."""
|
||||
_validate_tag_id(tag_id)
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
if not conn.execute("SELECT 1 FROM tags WHERE id=?", (tag_id,)).fetchone():
|
||||
return None
|
||||
row = conn.execute(_TAG_IMPACT, {"tag": tag_id}).fetchone()
|
||||
return {"library_epoch": epoch, "impact": dict(row)}
|
||||
|
||||
@staticmethod
|
||||
def _category_projection(
|
||||
conn: sqlite3.Connection, epoch: str, source: str, target: str
|
||||
) -> dict:
|
||||
"""Owners of ``source``, impact counts and the token over them."""
|
||||
components = [
|
||||
[row[0], bool(row[1])]
|
||||
for row in conn.execute(_CATEGORY_COMPONENTS, (source,))
|
||||
]
|
||||
pairs = [
|
||||
[row[0], bool(row[1])] for row in conn.execute(_CATEGORY_PAIRS, (source,))
|
||||
]
|
||||
if not components and not pairs:
|
||||
raise LibraryCommandError("vocabulary-conflict", epoch)
|
||||
target_exists = bool(
|
||||
conn.execute(_CATEGORY_EXISTS, {"v": target}).fetchone()[0]
|
||||
)
|
||||
eligible = sum(flag for _id, flag in components)
|
||||
archived = sum(flag for _id, flag in pairs)
|
||||
return {
|
||||
"target_exists": target_exists,
|
||||
"impact": {
|
||||
"positive_prompts": eligible,
|
||||
"other_prompts": len(components) - eligible,
|
||||
"negative_variants": len(pairs) - archived,
|
||||
"archived_negative_variants": archived,
|
||||
},
|
||||
"impact_token": category_impact_token(
|
||||
epoch, source, target, target_exists, components, pairs
|
||||
),
|
||||
}
|
||||
|
||||
def preview_vocabulary_category(
|
||||
self, *, expected_epoch: str, source: str, target: str
|
||||
) -> dict:
|
||||
"""Read-only impact of renaming the exact ``source`` to ``target``."""
|
||||
_validate_epoch(expected_epoch)
|
||||
target = _validate_category_change(source, target)
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
if epoch != expected_epoch:
|
||||
raise LibraryCommandError("storage-changed", epoch)
|
||||
projection = self._category_projection(conn, epoch, source, target)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"source": source,
|
||||
"target": target,
|
||||
**projection,
|
||||
}
|
||||
|
||||
def apply_vocabulary_category(
|
||||
self, *, expected_epoch: str, source: str, target: str, impact_token: str
|
||||
) -> dict:
|
||||
"""Set the exact ``source`` to ``target`` on every component and pair.
|
||||
|
||||
The recomputed projection must still match the preview's token; a
|
||||
mismatch is ``vocabulary-conflict`` with nothing written.
|
||||
"""
|
||||
_validate_epoch(expected_epoch)
|
||||
target = _validate_category_change(source, target)
|
||||
_validate_impact_token(impact_token)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
projection = self._category_projection(conn, epoch, source, target)
|
||||
if projection["impact_token"] != impact_token:
|
||||
raise LibraryCommandError("vocabulary-conflict", epoch)
|
||||
stamp = utc_now_iso()
|
||||
for statement in _CATEGORY_UPDATES:
|
||||
conn.execute(statement, (target, stamp, source))
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"source": source,
|
||||
"target": target,
|
||||
"consolidated": projection["target_exists"],
|
||||
"impact": projection["impact"],
|
||||
}
|
||||
|
||||
def _vocabulary_tags(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
epoch: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
after: int | None = None,
|
||||
) -> dict:
|
||||
_limit(limit)
|
||||
start = _tag_after(after)
|
||||
total = conn.execute("SELECT COUNT(*) FROM tags").fetchone()[0]
|
||||
rows = [dict(row) for row in conn.execute(_TAG_PAGE, (start, limit + 1))]
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"tags": rows,
|
||||
"total": total,
|
||||
"has_more": has_more,
|
||||
"next_key": rows[-1]["id"] if has_more else None,
|
||||
}
|
||||
|
||||
def _vocabulary_categories(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
epoch: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
after: list | None = None,
|
||||
) -> dict:
|
||||
_limit(limit)
|
||||
key = _category_after(after)
|
||||
total = conn.execute(
|
||||
_CATEGORY_GROUPS + "SELECT COUNT(*) FROM grouped"
|
||||
).fetchone()[0]
|
||||
where, params = "", []
|
||||
if key is not None:
|
||||
where = " WHERE first_pid>? OR (first_pid=? AND value>?)"
|
||||
params = [key[0], key[0], key[1]]
|
||||
rows = conn.execute(
|
||||
_CATEGORY_GROUPS
|
||||
+ "SELECT value,first_pid,prompt_count FROM grouped"
|
||||
+ where
|
||||
+ " ORDER BY first_pid,value LIMIT ?",
|
||||
[*params, limit + 1],
|
||||
).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"categories": [
|
||||
{"value": row["value"], "prompt_count": row["prompt_count"]}
|
||||
for row in rows
|
||||
],
|
||||
"total": total,
|
||||
"has_more": has_more,
|
||||
"next_key": (
|
||||
[rows[-1]["first_pid"], rows[-1]["value"]] if has_more else None
|
||||
),
|
||||
}
|
||||
|
||||
def list_vocabulary_tags(self, *, limit: int = 100, after: int | None = None):
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
return self._vocabulary_tags(conn, epoch, limit=limit, after=after)
|
||||
|
||||
def list_vocabulary_categories(
|
||||
self, *, limit: int = 100, after: list | None = None
|
||||
) -> dict:
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
return self._vocabulary_categories(conn, epoch, limit=limit, after=after)
|
||||
@@ -1,84 +0,0 @@
|
||||
# Application Redesign Development Plan
|
||||
|
||||
The maintainer approved the [interactive concept](concepts/workspace/README.md)
|
||||
on 2026-10-17. It is a design reference with simulated operations, not production
|
||||
code. This plan proposes implementation order and acceptance gates.
|
||||
|
||||
## Delivery status
|
||||
|
||||
- Foundation merged in [#198](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/198).
|
||||
- [#200](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/200) delivers native Images browsing, physical-file details, infinite scroll, Grid/List/Masonry, Viewer.js and video playback. The workspace opens on Images; original application entry points remain available.
|
||||
- Shared role-aware sending and the physical-file prompt inspector merged in [#203](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/203) and [#206](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/206).
|
||||
- The native Prompt Library merged in [#210](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/210) for [#208](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/208): one positive parent, saved negative variants, full selected text and a shared filtered image gallery. It does not invent chronological runs from counters or scan dates.
|
||||
- [Durable history storage](prompt-history-storage.md) records job attempts and outcomes from [#212](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/212), frozen prompt contexts from [#213](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/213), and observed run-owned output locators from [#214](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/214). Those locators do not verify file bytes or establish gallery-image associations. That association contract and the chronological execution timeline remain pending under [#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209).
|
||||
- [#215](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/215) brings the shell, Images and natural-height Prompt Library presentation into the approved geometry and themes while retaining their existing operations. [#216](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/216) adds the shared current-file inspector with exact generation facts, scoped provenance, raw/download evidence and loaded-collection viewer controls, including a slideshow that pauses at native video. This delivery does not establish whole-application parity, change the default entry point or constitute a release. Recorded chronological run/context browsing continues under [#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209); image tagging/review, System consolidation and remaining organization capabilities continue under #196.
|
||||
|
||||
## Scope and compatibility
|
||||
|
||||
Rebuild the standalone application around one shared shell. Preserve existing
|
||||
node schemas, saved workflows, backend capabilities, pinned Viewer.js and the
|
||||
ComfyAssets toolkit. Keep existing application entry points usable during the
|
||||
transition. Do not change the running maintainer instance during automated tests.
|
||||
|
||||
Use the concept's feature inventory as a parity checklist. Each row needs an
|
||||
implementation issue, observable acceptance criteria and automated test coverage.
|
||||
Record any intentional behavior change explicitly; no silent feature removal.
|
||||
|
||||
## Proposed delivery sequence
|
||||
|
||||
Use a tracking issue and small issue-linked branches and PRs. Dependencies may
|
||||
require splitting these milestones further.
|
||||
|
||||
| Milestone | Deliverable and acceptance focus |
|
||||
| --- | --- |
|
||||
| 1. Foundation | Production shell, routing, shared theme and disposable page-test harness. Verify startup, navigation, dark/light changes, cleanup and existing entry points. Establish discovered-test and coverage baselines before changes. |
|
||||
| 2. Image workspace | Real API-backed filters, Grid/List/Masonry, readable thumbnails, selection and shared Viewer.js inspector. Continuous infinite scroll only, with accessible Load more fallback, stable ordering, retry and restored browsing context. Bound retained DOM and test large datasets. |
|
||||
| 3. Prompt library and history | Real positive/negative associations, selected full negative, run history and shared image filtering. Design any missing execution-history persistence and migration before wiring the timeline; never invent historical runs from incomplete metadata. |
|
||||
| 4. Send to ComfyUI | Shared paper-plane actions wherever prompt text appears; role-specific and pair sends, destination preview, acknowledgement and failure recovery. Cover explicit empty versus unknown negatives and connected inputs. Never queue or replace a workflow. |
|
||||
| 5. Editing and review | Prompt editing, vocabulary management, per-image manual/auto-tag approval and Needs review. Preserve cancellation semantics and distinguish tag approval from prompt-association decisions. |
|
||||
| 6. System and maintenance | Settings, sources/scans, integrations, jobs, diagnostics, backup/restore and maintenance impact previews. Cover existing capabilities before retiring their old controls. |
|
||||
| 7. Additional organization | Favorites independent of ratings, collections, saved views, Activity, scheduled backups and portable import previews. Specify persistence, validation and recovery separately for each capability. |
|
||||
| 8. Cutover | Complete the parity checklist, migration and upgrade validation, accessibility/performance checks, documentation and targeted maintainer live testing. Switch the default only after these pass; preserve compatible URLs. |
|
||||
|
||||
## TDD and test layers
|
||||
|
||||
For each behavior, first write and run a meaningful failing test. Implement the
|
||||
smallest working change, verify it passes, then refactor with the suite green.
|
||||
Include failing-before/passing-after evidence in the PR.
|
||||
|
||||
- **Unit:** filtering and selection state, cursor handling, prompt roles,
|
||||
formatting, validation, navigation and lifecycle cleanup.
|
||||
- **Integration:** real HTTP handlers and disposable SQLite/filesystem fixtures;
|
||||
persistence, migration from supported old schemas, idempotent scans, restore,
|
||||
rollback and reference-safe deletion. Mock inference and external services.
|
||||
- **Smoke:** shipped production pages start and navigate with real fixture APIs;
|
||||
test populated, empty and error states, assets and unexpected console failures.
|
||||
- **Browser workflows:** production HTML/JS across Chromium, Firefox, WebKit and
|
||||
touch; editing/reload, infinite append, viewer cleanup, tagging approval,
|
||||
keyboard access, delayed responses, cancellation and retries.
|
||||
- **Host integration:** pinned disposable CPU ComfyUI, Classic and Nodes 2.0,
|
||||
switching both ways and workflow reload for host-facing changes. Exercise
|
||||
independent and combined pack installations for shared bridge/toolkit changes.
|
||||
|
||||
Retain existing test discovery and coverage gates. Carry forward database
|
||||
injection and open-handle checks into new fixtures. Add new frontend modules to
|
||||
coverage measurement and its baseline; do not lower floors or replace existing
|
||||
regression coverage with mockup-only tests.
|
||||
|
||||
## GitHub and release gates
|
||||
|
||||
Extend existing required workflows: the Python 3.10–3.13 matrix on Linux,
|
||||
Windows and macOS; 85% Python coverage; formatting; frontend units and per-file
|
||||
coverage floors; vendor checks; browser workflows; real ComfyUI compatibility.
|
||||
Publish traces, screenshots, logs and coverage artifacts on failures. See
|
||||
[TESTING.md](../TESTING.md) for commands and fixture isolation requirements.
|
||||
|
||||
Every PR must map acceptance criteria to tests, report relevant checks and include
|
||||
screenshots for visual changes. All required checks must pass before merging.
|
||||
Use maintainer live review at meaningful visual and host-integration milestones;
|
||||
capture discovered regressions in automated tests. Notify the maintainer when
|
||||
Python changes require a ComfyUI restart.
|
||||
|
||||
Keep version/publication changes separate from intermediate UI work: the existing
|
||||
registry workflow publishes on main/master changes to `pyproject.toml` or manual
|
||||
dispatch. Do not trigger a release merely to preview a redesign milestone.
|
||||
@@ -1,138 +0,0 @@
|
||||
# Prompt Manager application redesign concept
|
||||
|
||||
Status: visual concept approved by the maintainer, 2026-10-17. Production
|
||||
implementation is progressing in tested slices; see the [development plan](../../application-redesign.md).
|
||||
The production workspace now has native Images browsing; this concept remains the
|
||||
approved target for the complete application.
|
||||
|
||||
## Review the mockup
|
||||
|
||||
Run `python tools/preview-prompt-timeline.py --bind 0.0.0.0 --port 8767` and
|
||||
open `/workspace/`. The original approved timeline remains available at `/`.
|
||||
The server exposes an explicit allowlist of concept files and pinned assets.
|
||||
All examples are synthetic; there are no application API calls, database access,
|
||||
real model downloads, or ComfyUI handoffs. Local file inspection previews an
|
||||
image in the browser without uploading it. Exports contain sample data only.
|
||||
|
||||
The sidebar opens Images, Prompt library, Tags & categories, Inspect a file,
|
||||
Activity, System and Design review. System includes Overview, Appearance,
|
||||
Sources & scans, Integrations, Auto-tagging, Maintenance, Backup & export, and
|
||||
Diagnostics & logs. The in-app Design review contains the feature map below.
|
||||
|
||||
## Proposed direction
|
||||
|
||||
One application shell replaces separate Admin, Gallery and Metadata shells.
|
||||
Use the approved shared ComfyAssets theme: ComfyUI dark/light, blue primary
|
||||
actions, green positive roles, red negative roles and errors. Preserve existing
|
||||
capabilities and services while consolidating controls and clarifying their scope.
|
||||
|
||||
Browsing is image-led with readable thumbnails and one metadata inspector.
|
||||
Prompt library groups by positive; negative variants and runs share its gallery.
|
||||
The full selected negative appears directly beneath the positive in history.
|
||||
Favorites express preference independently of the existing quality rating.
|
||||
Collections organize projects without duplicating recipe text; saved views keep
|
||||
search/filter context. New capabilities are proposals, not implemented features.
|
||||
|
||||
Maintenance moves out of the browsing toolbar. Checks, impact preview, backups,
|
||||
and apply steps have separate responsibilities. Distinguish prompt text reuse
|
||||
from image-file duplicates, and metadata enrichment from thumbnail generation.
|
||||
No repeated global thumbnail-generation prompt on opening the gallery.
|
||||
|
||||
## Send to ComfyUI
|
||||
|
||||
Use the same original paper-plane outline, blue accent and accessible role label
|
||||
on every positive/negative prompt block, including image details and the viewer.
|
||||
A pair action sends both. Preview destination and exact fields before applying;
|
||||
positive-only and negative-only actions leave the other role untouched. An
|
||||
explicitly empty negative means clear the target field; unknown metadata must
|
||||
never be interpreted as empty. Sending text neither queues execution nor
|
||||
replaces a workflow.
|
||||
|
||||
Production needs a shared host bridge with versioned payloads, origin validation,
|
||||
destination discovery, capability checks, disconnected-host handling, linked-input
|
||||
checks and acknowledgements. Preview actual target values and report success only
|
||||
after acknowledgement. Default to asking for a destination; never guess between
|
||||
multiple nodes or positive and negative encoder inputs. Verify Classic and Nodes
|
||||
2.0 with both packs installed independently and together.
|
||||
|
||||
## Existing feature inventory
|
||||
|
||||
| Current feature | Proposed home / behavior | Disposition |
|
||||
| --- | --- | --- |
|
||||
| Admin prompt cards / per-prompt gallery | Prompt library + one prompt history workspace | Retained, rebuilt |
|
||||
| Standalone image gallery / Admin image modal | One Images workspace + shared Viewer.js viewer | Consolidated |
|
||||
| Video playback, autoplay / mute / loop, viewer slideshow and transforms | Shared media viewer; playback controls in Appearance; same image navigation | Retained |
|
||||
| Metadata page / viewer sidebar / full-text popups | One reusable metadata inspector; inline complete prompt text | Consolidated |
|
||||
| Prompt add, edit, rating, notes, copy, delete | Recipe editor, shared details, contextual bulk actions; reference-safe archive | Retained, clarified |
|
||||
| Categories, tag rename / merge, autocomplete | Tags & categories; same vocabulary in every editor | Retained |
|
||||
| Grid/list, thumbnail size, columns, pages/infinite scroll | Infinite scroll only; one Appearance configuration and contextual layout / density overrides | Simplified |
|
||||
| Scan, rescan, monitoring directories, worker threads | Sources & scans; explicit metadata enrichment vs thumbnail work | Consolidated |
|
||||
| Ollama, HuggingFace, WD14 variants, thresholds, review | One Auto-tagging configuration and review flow | Retained |
|
||||
| Thumbnail quality, size limit, metadata cache, paths | Appearance + cache controls in Maintenance | Retained |
|
||||
| Prompt dedup, hash checks, consistency, unused prompts | Maintenance plan with dependency checks and impact preview | Retained, safer flow |
|
||||
| Image dedup, orphan image links, browser cache | Separate reviewed operations in Maintenance | Retained |
|
||||
| Database backup / restore, JSON export | Backup & export; recovery and portable recipes clearly distinguished | Retained |
|
||||
| Diagnostics, image-link checks, logs, rotation, log levels | System → Diagnostics & logs | Retained |
|
||||
| LoRA detection/import, trigger injection, CivitAI key | System → Integrations; positive-only triggers | Retained |
|
||||
| Statistics / counts | System overview + usage breakdown; context counts in the workspace | Retained |
|
||||
| Send prompts to ComfyUI from every prompt surface | Role-aware paper-plane action + destination / overwrite preview | New proposal |
|
||||
| Reusable negatives / approved branch timeline | Prompt history with selected negative directly beneath positive | Approved concept |
|
||||
| Favorites, saved searches, project collections | Sidebar views; collections do not duplicate recipe text | New proposal |
|
||||
| Ambiguous metadata review, source provenance, run comparisons | Needs review queue; recorded vs inferred evidence; pair/run selection | New / expanded |
|
||||
| Unified jobs, cancellation, resumable progress visibility | Activity center; cancellation distinguished from hiding progress | New proposal |
|
||||
| Scheduled backups, pre-change safety snapshots | Explicit recovery policy after storage configuration | New proposal |
|
||||
|
||||
## Implementation boundaries and test requirements
|
||||
|
||||
Keep Viewer.js 1.15.2 and the shared toolkit as reproducible pinned artifacts;
|
||||
this mock uses their actual checked-in files without modifying them. Preserve
|
||||
legacy routes during the transition, current node layouts/schemas, existing
|
||||
controllers where reusable, and all existing test/coverage gates.
|
||||
|
||||
The timeline needs persisted execution records to represent real runs. Saved
|
||||
views, collections, metadata review resolution, activity history, scheduled
|
||||
backups, portable JSON import, and the ComfyUI bridge need separate implementation
|
||||
and persistence work. Preserve source evidence and distinguish recorded from
|
||||
inferred metadata. Existing schema migration is automatic; old image enrichment
|
||||
remains an optional explicit scan. The one-time upgrade notice belongs in the
|
||||
new shell, with Scan and Later choices.
|
||||
|
||||
Implement in reviewed slices: shared application shell and inspector; reusable
|
||||
prompt blocks/send bridge; image workspace; grouped library/history; settings and
|
||||
maintenance; optional organization and workflow improvements. Write regressions
|
||||
first for production behavior. Retain Python, frontend unit, Chromium/Firefox/
|
||||
WebKit/touch and managed Classic/Nodes 2.0 tests. Add behavior tests for shared
|
||||
filters, navigation state, viewer cleanup, interrupted jobs, dependency-safe
|
||||
maintenance, backup/restore, paired metadata and correct send destinations.
|
||||
|
||||
Screenshots in `screenshots/` record the reviewed starting point. The approved
|
||||
standalone timeline in `../prompt-variant-timeline.html` remains unchanged.
|
||||
|
||||
## Review polish
|
||||
|
||||
History actions use a shared branch icon. Images can switch among Grid, List and
|
||||
Masonry; Masonry preserves each image's proportions and keeps DOM order while
|
||||
packing cards with a responsive layout. It does not require experimental native
|
||||
masonry support. Resize observers are disconnected when leaving the view.
|
||||
|
||||
Every image card includes a tagging action beside Info, including cards in Needs
|
||||
review. Choose WD14 SwinV2, WD14 ViT, Ollama or HuggingFace; adjust thresholds or
|
||||
instructions, generate sample suggestions, edit tags manually, and approve only
|
||||
the desired suggestions. Changes are staged until approval, apply to that image
|
||||
alone and do not resolve uncertain prompt associations. Changing models discards
|
||||
previous suggestions. Cancel preserves existing tags. All inference is simulated.
|
||||
|
||||
## Continuous browsing decision
|
||||
|
||||
The redesign uses infinite scroll only. Remove page numbers, previous/next page
|
||||
buttons, paged/infinite mode selection and results-per-page controls. Keep request
|
||||
batching internal. The mock appends 12 images at a time in Grid, List and Masonry;
|
||||
it preserves existing card nodes, focus and selection while appending. It also
|
||||
retains the loaded range when switching layouts or revisiting a search. A
|
||||
keyboard-accessible Load more action provides an alternate trigger for the same
|
||||
continuous stream, not pagination. Exhausted and empty results are explicit.
|
||||
|
||||
Production will need bounded rendering, stable cursors, scroll restoration and
|
||||
loading/error/retry behavior for large collections. Existing backend pagination
|
||||
can continue providing internal request batches; no production tests or routes
|
||||
are removed as part of this prototype.
|
||||
@@ -1,105 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
||||
<meta name="referrer" content="no-referrer" />
|
||||
<title>Images · Prompt Manager concept</title>
|
||||
<link rel="stylesheet" href="/toolkit/controls.css" />
|
||||
<link rel="stylesheet" href="/vendor/viewer.css" />
|
||||
<link rel="stylesheet" href="/workspace/style.css" />
|
||||
<script src="/vendor/viewer.js" defer></script>
|
||||
<script src="/workspace/app.js" type="module"></script>
|
||||
</head>
|
||||
<body class="ca-tools">
|
||||
<a class="skip" href="#main">Skip to workspace</a>
|
||||
<div class="app-shell">
|
||||
<aside class="sidebar">
|
||||
<a class="brand" href="#images"
|
||||
><span class="brand-mark">P<span>±</span></span
|
||||
><span>Prompt Manager<small>COMFYASSETS</small></span></a
|
||||
>
|
||||
<div class="sidebar-label">WORKSPACE</div>
|
||||
<nav id="primary-nav" aria-label="Workspace"></nav>
|
||||
<div class="sidebar-label">
|
||||
SAVED VIEWS <span class="new-label">PROPOSED</span>
|
||||
</div>
|
||||
<nav id="saved-nav" aria-label="Saved views"></nav>
|
||||
<div class="sidebar-label">
|
||||
COLLECTIONS <span class="new-label">PROPOSED</span>
|
||||
</div>
|
||||
<nav id="collections-nav" aria-label="Collections"></nav>
|
||||
<div class="sidebar-bottom">
|
||||
<nav id="system-nav" aria-label="Tools and system"></nav>
|
||||
<div class="connection">
|
||||
<i></i>
|
||||
<div>
|
||||
ComfyUI workspace<small
|
||||
>Demo connection · Classic / Nodes 2.0</small
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</aside>
|
||||
<div class="workspace">
|
||||
<header class="topbar">
|
||||
<div class="breadcrumbs">
|
||||
ComfyAssets <span>/</span> <span id="breadcrumb">Images</span>
|
||||
</div>
|
||||
<div class="top-actions">
|
||||
<span class="concept-badge">Interactive concept</span
|
||||
><button id="activity-open" type="button">
|
||||
Activity <span class="count" id="activity-count">2</span></button
|
||||
><label class="theme-label" for="theme"
|
||||
>Theme
|
||||
<select id="theme" aria-label="Theme">
|
||||
<option value="dark">Dark</option>
|
||||
<option value="light">Light</option>
|
||||
<option value="system">System</option>
|
||||
</select></label
|
||||
><a class="canvas-link" href="#send"
|
||||
>ComfyUI <span aria-hidden="true">↗</span></a
|
||||
>
|
||||
</div>
|
||||
</header>
|
||||
<main id="main" tabindex="-1"></main>
|
||||
<footer class="app-footer">
|
||||
<span
|
||||
>Design prototype · sample images and data · actions stay in this
|
||||
tab</span
|
||||
><button type="button" id="review-link">What changes & why</button>
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
<div id="toast" class="toast" role="status" aria-atomic="true"></div>
|
||||
<dialog id="modal" aria-labelledby="modal-title">
|
||||
<div class="dialog-header">
|
||||
<h2 id="modal-title"></h2>
|
||||
<button type="button" id="modal-close" aria-label="Close dialog">
|
||||
×
|
||||
</button>
|
||||
</div>
|
||||
<div id="modal-body"></div>
|
||||
</dialog>
|
||||
<dialog id="viewer-dialog" aria-labelledby="viewer-title">
|
||||
<div class="dialog-header">
|
||||
<div>
|
||||
<h2 id="viewer-title">Image details</h2>
|
||||
<p class="muted small">
|
||||
One viewer · image, prompts, generation data, workflow
|
||||
</p>
|
||||
</div>
|
||||
<button type="button" id="viewer-close" aria-label="Close image viewer">
|
||||
×
|
||||
</button>
|
||||
</div>
|
||||
<div class="viewer-layout">
|
||||
<div class="viewer-left">
|
||||
<div id="viewer-stage"></div>
|
||||
<div class="viewer-toolbar" id="viewer-toolbar"></div>
|
||||
</div>
|
||||
<aside id="viewer-details" aria-label="Image metadata"></aside>
|
||||
</div>
|
||||
</dialog>
|
||||
</body>
|
||||
</html>
|
||||
|
Before Width: | Height: | Size: 243 KiB |
|
Before Width: | Height: | Size: 365 KiB |
|
Before Width: | Height: | Size: 441 KiB |
|
Before Width: | Height: | Size: 440 KiB |
|
Before Width: | Height: | Size: 443 KiB |
|
Before Width: | Height: | Size: 478 KiB |
|
Before Width: | Height: | Size: 133 KiB |
|
Before Width: | Height: | Size: 67 KiB |
|
Before Width: | Height: | Size: 176 KiB |
|
Before Width: | Height: | Size: 156 KiB |
|
Before Width: | Height: | Size: 319 KiB |
|
Before Width: | Height: | Size: 150 KiB |
|
Before Width: | Height: | Size: 128 KiB |
@@ -1,84 +0,0 @@
|
||||
# Durable Prompt History
|
||||
|
||||
Status: the durable job ledger, immutable prompt contexts and run-owned output
|
||||
references are implemented for
|
||||
[#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209). Gallery
|
||||
image associations and the chronological timeline remain pending; #209 stays open.
|
||||
|
||||
## Recorded jobs
|
||||
|
||||
New jobs observed at the ComfyUI queue's dequeue boundary receive an immutable
|
||||
run identity and a host-session/queue-item identity. The public ComfyUI job UUID
|
||||
is a correlation field: submitting it twice records two attempts. Rejected graphs
|
||||
and jobs removed from the queue before dequeue create no execution rows. Cached
|
||||
jobs are recorded even when no PromptManager node runs, and headless jobs do not
|
||||
require a browser or WebSocket client.
|
||||
|
||||
The additive `execution_runs` table stores dequeue time, observed callback time,
|
||||
optional executor event times and an explicit outcome: `observed`, `succeeded`,
|
||||
`failed`, `interrupted` or `unknown`. Matching stored terminal events determine
|
||||
outcomes. Missing or conflicting terminal evidence remains unknown; a process
|
||||
crash can leave an observed row without an invented completion time. Bounded
|
||||
safe diagnostics retain node ID/type and exception type, never graph text,
|
||||
arbitrary extra data, credentials, exception messages, traceback, inputs or
|
||||
arbitrary output payloads. A bounded background writer keeps storage failures from
|
||||
changing host execution and reports capture gaps rather than inventing rows.
|
||||
|
||||
Existing databases begin with zero execution rows. Migration and backup restore
|
||||
preserve prompts, pairs, image links and aggregate counters; storage epochs keep
|
||||
stale queued writes from being replayed into restored data. There is no
|
||||
historical backfill from counters, scan dates, image timestamps or saved recipes.
|
||||
|
||||
[Testing](../TESTING.md#durable-execution-ledger) describes the real CPU host
|
||||
matrix, historical failure proof, privacy checks and cleanup. Python deployment
|
||||
requires restarting ComfyUI.
|
||||
|
||||
## What the existing UI proves
|
||||
|
||||
`prompts.run_count` and `prompt_pairs.run_count` remain aggregate usage counters,
|
||||
not completed-run counts. `last_used_at` is the latest aggregate update; legacy
|
||||
migration may have estimated counts from linked images. `generated_images`
|
||||
identifies linked files; image counts and insertion times are not run counts or
|
||||
execution times. Image/pair association creation time can reflect an old-image
|
||||
scan. A saved pair's creation time is its recipe date.
|
||||
|
||||
The native Prompt Library shows saved negative variants, linked images and
|
||||
aggregate uses. It does not expose the ledger as chronological dots or repeat-run
|
||||
branches. `PromptTracker` identities and PNG recipe evidence remain separate
|
||||
from durable host job identity.
|
||||
|
||||
## Immutable contexts and output references
|
||||
|
||||
Prompt contexts retain bounded raw/effective positive, negative and composition
|
||||
snapshots with nullable recipe navigation hints and explicit evidence provenance.
|
||||
Unknown text remains SQL NULL; a known empty negative remains empty text. Cached
|
||||
evidence belongs to its exact cache provider and attempt, rather than inheriting
|
||||
unrelated recent prompts. Deletion and restore can remove navigation hints without
|
||||
rewriting historical snapshots or rebinding them to current library rows.
|
||||
|
||||
Schema v3 adds `execution_output_capture` and `execution_run_outputs` without
|
||||
backfilling older runs or modifying gallery image links. At final queue completion,
|
||||
the writer attaches bounded output evidence from that attempt's final host history
|
||||
to its immutable run ID. The internal `get_execution_outputs(run_id)` reader
|
||||
returns state, reason, format version, source and ordered references. A complete
|
||||
capture with `none_reported` means the final host history reported no images;
|
||||
`not_captured`, partial, unavailable and conflicting evidence remain explicit.
|
||||
Failed and interrupted jobs can still have complete evidence for images reported
|
||||
before termination. Running jobs do not gain a final capture merely because an
|
||||
image node finished.
|
||||
|
||||
Each reference records output node ID, image ordinal, folder type, subfolder,
|
||||
filename, path style and an opaque configured-root key. A cached repeat can retain
|
||||
the same locators under a new run. Locators do not verify file bytes, continued
|
||||
existence, a newly generated file or membership in any prompt context: the reader
|
||||
reports file identity as `unverified`, generation and context membership as
|
||||
`unknown`, and temp outputs as transient. Capture never reads, resolves or serves
|
||||
the files and adds no public route. PNG metadata is not required. Same-path file
|
||||
replacement and existing gallery provenance retain their separate semantics.
|
||||
|
||||
## Remaining history work
|
||||
|
||||
Define a deduplicated run/gallery-image relationship that preserves existing image
|
||||
provenance and same-path replacement protection. Add a chronological timeline only
|
||||
after its navigation and evidence semantics can honestly represent prompt-level
|
||||
execution chronology, batched outputs and repeat-run branches.
|
||||
@@ -1,34 +0,0 @@
|
||||
# Prompt inspection and sending
|
||||
|
||||
The opt-in workspace at `/prompt_manager/workspace#/images` shows full positive
|
||||
and negative prompts in both Image details and the full-image preview. Existing
|
||||
gallery and metadata tools remain available during the application rebuild.
|
||||
|
||||
Each prompt shows its evidence source: recorded execution, workflow inference,
|
||||
or embedded generation parameters. Path-only saved-library associations are
|
||||
marked unverified and cannot be sent: a filename alone cannot prove that an image
|
||||
has not been replaced. Missing, ambiguous, and unreadable prompts remain visible
|
||||
as distinct states. An explicitly
|
||||
empty negative is a resolved value and can clear a destination's negative field.
|
||||
|
||||
Use the blue paper-plane action beside a prompt, or **Send prompt pair**. Keep a
|
||||
ComfyUI tab open under the same origin and deployment prefix. Choose a window and
|
||||
compatible Prompt Manager node, select **Preview changes**, review the complete
|
||||
before/after text and preserved prepend/append composition, then **Apply to
|
||||
ComfyUI**. The dialog reports success only after a matching acknowledgement.
|
||||
It does not load workflows or queue generation. If confirmation is lost, inspect
|
||||
ComfyUI and preview again before another attempt. Closing the dialog cannot undo
|
||||
an update already sent.
|
||||
|
||||
## Read-only API
|
||||
|
||||
`GET /prompt_manager/images/inspect/{relative_path}?root={root_index}` requires
|
||||
an explicit configured root, including when two roots contain the same filename.
|
||||
The response contains `image` identity and `prompts.positive` / `prompts.negative`.
|
||||
Each role has `status` (`resolved`, `absent`, `unresolved`, or `ambiguous`), `text`
|
||||
(string only when resolved, otherwise null), and `source`. Inspection does not
|
||||
index files, alter saved prompts, run scans, or decode image pixels. PNG text and
|
||||
video probe output are bounded; unsupported or malformed evidence remains
|
||||
unresolved rather than becoming sendable placeholder text.
|
||||
|
||||
Python route changes require restarting ComfyUI. Browser assets require a refresh.
|
||||
@@ -1,49 +0,0 @@
|
||||
# Prompt sending bridge
|
||||
|
||||
Issue #202 establishes the transport and host adapter before the redesigned
|
||||
inspector displays prompt text. Gallery paper-plane controls and normalized
|
||||
metadata follow in a separate slice; existing node layouts are unchanged.
|
||||
|
||||
## Shared transport
|
||||
|
||||
PromptManager consumes the exact Toolkit 0.2.0 artifact recorded in
|
||||
`web/js/shared/toolkit/provenance.json`. The shared module has no ComfyUI imports.
|
||||
Its client discovers explicit host sessions and compatible destinations, requests
|
||||
a preview, and applies that preview using a one-use, client-bound, expiring ticket.
|
||||
Success requires an acknowledgement with verified destination values.
|
||||
|
||||
BroadcastChannel connects independently opened tabs on the same origin and storage
|
||||
partition. Deployment prefixes have separate channel namespaces. Exact origin,
|
||||
protocol and peer checks reject unrelated messages. Same-origin scripts are
|
||||
trusted; this transport is not authentication against other scripts on that origin.
|
||||
Unsupported messaging is reported without breaking existing ComfyUI nodes.
|
||||
|
||||
An apply timeout or abort after posting means the outcome is unknown. It cannot
|
||||
prove the host cancelled the write. The sender must inspect a fresh destination
|
||||
preview before deciding to send again; automatic retry must not overwrite edits.
|
||||
|
||||
## PromptManager adapter
|
||||
|
||||
Classic, Text, Focused and Split accept positive text; +/- accepts either role or
|
||||
both. Plain CLIP encoders are not assigned roles by title or position. An omitted
|
||||
role remains untouched; an explicitly empty string clears only the selected
|
||||
field. Unknown/ambiguous metadata must not become an empty string.
|
||||
|
||||
Previews show original and requested text plus positive prepend/append composition.
|
||||
Connected composition is marked unknown rather than guessing upstream values.
|
||||
Connected destination fields reject writes. Applying rechecks graph, node, widgets,
|
||||
values and connections. Ordinary serialization cache synchronization is accepted
|
||||
only when it preserves live values. Both requested roles are checked before writing.
|
||||
|
||||
Widget values, visible inputs, serialization and composed previews are synchronized.
|
||||
Callback failures restore local fields; arbitrary external callback side effects
|
||||
cannot be undone. Sending never queues execution or loads/replaces a workflow.
|
||||
|
||||
## Lifecycle and validation
|
||||
|
||||
Page hiding closes the host and invalidates tickets; restored pages mount one fresh
|
||||
host. Client disposal clears pending timers/listeners and closes its channel.
|
||||
Shared protocol contracts run against the actual bundled file, alongside adapter
|
||||
units, cross-browser tab workflows and real Classic/Nodes 2.0 scenarios. Pinned
|
||||
PromptManager-only, KikoTools-only and combined disposable hosts verify coexistence
|
||||
and workflow reload without modifying either live installation.
|
||||
@@ -1,111 +0,0 @@
|
||||
# Reusable positive and negative prompts
|
||||
|
||||
Tracking: [#193](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/193).
|
||||
This work builds on the merged Split node in PR #192. Keep the feature unmerged
|
||||
until maintainer live testing is complete.
|
||||
|
||||
## Storage contract
|
||||
|
||||
Keep existing prompt IDs, text hashes, metadata, image links and node schemas.
|
||||
A component can serve either role. Ordered combinations refer to shared prompt
|
||||
rows; an explicitly empty negative uses a null reference. Missing or ambiguous
|
||||
metadata does not mean an empty negative. Pair metadata belongs to the
|
||||
combination; saving another combination does not overwrite component metadata.
|
||||
Existing trim/lowercase hash normalization remains unchanged.
|
||||
|
||||
Saved components are immutable while referenced. To change text, create or reuse
|
||||
a component and save a new combination. Archive combinations that have generation
|
||||
history. Pruning and legacy editing/deletion must respect these dependencies.
|
||||
Historical workflow snapshots remain intact, including their original text.
|
||||
|
||||
## Migration and recovery
|
||||
|
||||
Create pair tables after existing legacy rebuild migrations, in an atomic,
|
||||
repeatable migration. Never scan media during schema initialization. Test fresh,
|
||||
legacy and current databases, repeated startup, injected failures and retries.
|
||||
Preserve old backup compatibility. Migrate and validate a disposable backup copy
|
||||
before replacing live database pages. Reject broken pair schemas, foreign keys,
|
||||
triggers and views. Use SQLite backup APIs so committed WAL data is included.
|
||||
|
||||
## Metadata and scanning
|
||||
|
||||
Resolve positive and negative from the same sampler/output context. Track both
|
||||
node ID and output slot; bound graph traversal and text construction. Unknown
|
||||
conditioning operations and conflicting sampler candidates remain unresolved.
|
||||
Never guess that the first encoder or most recent negative belongs to an image.
|
||||
|
||||
Import high-confidence pairs atomically with their image association and source
|
||||
provenance. Repeated scans must not duplicate components, pairs or image links,
|
||||
regenerate thumbnails, or increment generation run counts. Preserve existing
|
||||
conflicting/manual associations for review. Report new, enriched, unchanged,
|
||||
ambiguous, unresolved and failed results separately.
|
||||
|
||||
## Live node
|
||||
|
||||
Use the shared toolkit and Split foundation. Positive and negative editors have
|
||||
equal size. Only positive receives prepend, append and automatic LoRA triggers.
|
||||
Keep upstream connections authoritative. Library selection previews both fields
|
||||
before an explicit load. Saving text changes creates another reusable combination.
|
||||
Verify Classic and Nodes 2.0 startup, switching both directions, reload, caching,
|
||||
linked inputs, popup behavior and lifecycle cleanup before maintainer testing.
|
||||
|
||||
## Delivery gate
|
||||
|
||||
Keep all existing coverage floors and test suites. Record regression failures
|
||||
before implementing each behavior. Tests use disposable state and the managed
|
||||
CPU host; never mutate the maintainer's running ComfyUI or personal database.
|
||||
Python changes require a ComfyUI restart. Live approval is required before merge.
|
||||
|
||||
## Live testing
|
||||
|
||||
1. Download a database backup from Admin before the first restart. The additive
|
||||
schema migration runs on startup; existing workflows and prompt IDs stay valid.
|
||||
2. Restart ComfyUI, reload the browser, and add **Prompt Manager (+/-)** from
|
||||
**ComfyAssets / Prompts**. Connect CLIP and the two conditioning outputs to the
|
||||
sampler's positive and negative inputs.
|
||||
3. Enter both texts, save a pair, then keep positive unchanged and change negative.
|
||||
Recent should show both combinations with distinct negative excerpts. Selecting
|
||||
one previews both; **Use pair** loads both together.
|
||||
4. Test category/tag autocomplete and metadata opt-in, resizing, light/dark,
|
||||
switching Classic / Nodes 2.0 both ways, and workflow reload. Connected fields
|
||||
stay authoritative; run the node to save connected metadata values.
|
||||
5. Queue the same recipe twice and change only negative once. Usage should count
|
||||
dequeued jobs even when ComfyUI caches the node. Prepend, append and LoRA triggers
|
||||
affect positive only. Gallery text is labelled as recorded during execution or
|
||||
recovered from workflow metadata.
|
||||
6. Run an output metadata scan to enrich existing images. Repeat it: pairs and
|
||||
links must remain deduplicated, with unchanged results rather than new links.
|
||||
Conflicting or ambiguous files need review and are preserved.
|
||||
|
||||
Fresh executions embed bounded text evidence in PNG metadata. A cached node may
|
||||
not emit fresh evidence, so its graph-derived text is explicitly marked inferred.
|
||||
Multiple distinct batch records without an image anchor remain unresolved.
|
||||
Rescanning after interruption is safe through idempotent per-file writes; this
|
||||
version does not persist a filesystem scan cursor across server restarts.
|
||||
The API can request cancellation with `POST /prompt_manager/scan/cancel`; the
|
||||
existing dashboard Hide button continues to hide progress without cancelling.
|
||||
|
||||
Pair APIs are under `/prompt_manager/pairs`: POST saves a combination, GET lists
|
||||
and filters, GET/PATCH/DELETE `/{id}` reads, edits metadata or removes an unused
|
||||
pair. Changing text creates another combination. Null metadata updates preserve
|
||||
existing values; empty strings/lists clear supported fields. Archive a pair with
|
||||
`PATCH {"archived":true}`. GET `/export` provides versioned JSON containing both
|
||||
texts and pair metadata; this is an export format, not a JSON import endpoint.
|
||||
|
||||
## Gallery redesign concept retained
|
||||
|
||||
The maintainer approved the interactive
|
||||
[negative variant timeline concept](concepts/prompt-variant-timeline.html) on
|
||||
2026-10-17. Preserve it for the later gallery redesign; it is not production UI
|
||||
and does not expand the current node work.
|
||||
|
||||
Keep one positive parent and one shared gallery, with red branches for reusable
|
||||
negative variants and smaller dots for repeat runs. Show the selected negative
|
||||
in full directly beneath the unchanged positive; selecting another variant or
|
||||
run updates that text and filters the shared gallery. Comparison shows both
|
||||
negatives. Distinguish explicitly empty negatives from an all-variants view and
|
||||
saved variants from executed runs. Production history needs persisted execution
|
||||
records; the prototype's timeline uses illustrative sample data.
|
||||
|
||||
The Admin upgrade/scan notice remains a separate follow-up: explain automatic
|
||||
schema migration and offer an optional metadata enrichment scan or Later.
|
||||
|
Before Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 86 KiB |
|
Before Width: | Height: | Size: 30 KiB |
|
Before Width: | Height: | Size: 70 KiB |
|
Before Width: | Height: | Size: 47 KiB |
@@ -1,13 +0,0 @@
|
||||
# Native Library visual evidence
|
||||
|
||||
Captured from the production workspace using disposable synthetic browser-test
|
||||
fixtures. No personal images, library, or running ComfyUI state was accessed.
|
||||
Solid-color media are intentional test images.
|
||||
|
||||
- `library-dark.png`: grouped positive shelf, dark appearance.
|
||||
- `library-light.png`: the same shelf, light appearance.
|
||||
- `history-selected-negative.png`: selected negative directly below its positive,
|
||||
saved-variant branches and the filtered shared gallery.
|
||||
- `history-mobile.png`: the same history at a 390-pixel viewport.
|
||||
|
||||
These screenshots document implemented behavior, rather than a design mockup.
|
||||
|
Before Width: | Height: | Size: 139 KiB |
|
Before Width: | Height: | Size: 162 KiB |
|
Before Width: | Height: | Size: 117 KiB |
|
Before Width: | Height: | Size: 116 KiB |
|
Before Width: | Height: | Size: 75 KiB |
|
Before Width: | Height: | Size: 74 KiB |
|
Before Width: | Height: | Size: 76 KiB |
|
Before Width: | Height: | Size: 75 KiB |
|
Before Width: | Height: | Size: 78 KiB |
|
Before Width: | Height: | Size: 102 KiB |
|
Before Width: | Height: | Size: 104 KiB |
|
Before Width: | Height: | Size: 79 KiB |
|
Before Width: | Height: | Size: 92 KiB |
|
Before Width: | Height: | Size: 94 KiB |
|
Before Width: | Height: | Size: 79 KiB |
|
Before Width: | Height: | Size: 100 KiB |
|
Before Width: | Height: | Size: 101 KiB |
|
Before Width: | Height: | Size: 80 KiB |
|
Before Width: | Height: | Size: 90 KiB |
|
Before Width: | Height: | Size: 91 KiB |
|
Before Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 106 KiB |
|
Before Width: | Height: | Size: 89 KiB |
|
Before Width: | Height: | Size: 98 KiB |
|
Before Width: | Height: | Size: 91 KiB |
|
Before Width: | Height: | Size: 105 KiB |
|
Before Width: | Height: | Size: 91 KiB |
|
Before Width: | Height: | Size: 101 KiB |
|
Before Width: | Height: | Size: 83 KiB |
|
Before Width: | Height: | Size: 101 KiB |
|
Before Width: | Height: | Size: 84 KiB |
|
Before Width: | Height: | Size: 112 KiB |
|
Before Width: | Height: | Size: 84 KiB |
|
Before Width: | Height: | Size: 99 KiB |
|
Before Width: | Height: | Size: 86 KiB |
|
Before Width: | Height: | Size: 110 KiB |
|
Before Width: | Height: | Size: 224 KiB |
|
Before Width: | Height: | Size: 231 KiB |
|
Before Width: | Height: | Size: 231 KiB |
|
Before Width: | Height: | Size: 54 KiB |
|
Before Width: | Height: | Size: 92 KiB |