Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1fb4ad5495 | ||
|
|
92be8770c3 | ||
|
|
7e1847955b | ||
|
|
ecb6b09813 | ||
|
|
cdb791d4a9 | ||
|
|
f924d7195d | ||
|
|
3789d85e58 | ||
|
|
fb6452934f | ||
|
|
395dc74cbe | ||
|
|
2f061904a2 | ||
|
|
f80e67c812 | ||
|
|
ab6fee3a82 | ||
|
|
f53a43152d | ||
|
|
fe63f01214 | ||
|
|
aab0fedc10 | ||
|
|
085e45e475 | ||
|
|
89572e03c6 | ||
|
|
60ce03ba4b | ||
|
|
ae3e191990 | ||
|
|
cd284432f5 | ||
|
|
c4eb487d02 | ||
|
|
a6d6129ec6 | ||
|
|
e8d96a0f55 | ||
|
|
d0134b2d13 | ||
|
|
9af3271dbb | ||
|
|
98ddded73c | ||
|
|
abd26bb275 | ||
|
|
b12a9a14ba | ||
|
|
bb744550f4 | ||
|
|
2650621a1b | ||
|
|
d6371b7fad | ||
|
|
75cd9c6e13 | ||
|
|
77f08782f4 | ||
|
|
62815c9e3e | ||
|
|
a1ec65442d | ||
|
|
e52cc39cd5 | ||
|
|
eff6ece6a4 | ||
|
|
d02a911748 | ||
|
|
bc3f09d844 | ||
|
|
e0b86712a8 | ||
|
|
66ca988d2a | ||
|
|
20dbc2df21 | ||
|
|
dd247e3b6f | ||
|
|
c416c84347 | ||
|
|
599dd828e6 | ||
|
|
a4a7b7129a | ||
|
|
f334004f8f | ||
|
|
0126705045 | ||
|
|
652458e995 | ||
|
|
89d94cdb2b | ||
|
|
b6e0439421 | ||
|
|
b5ca2eda94 | ||
|
|
8870496ea3 | ||
|
|
83199518df | ||
|
|
5e00ee479f | ||
|
|
2c54daf2d8 | ||
|
|
1ce2577733 | ||
|
|
2941a9b748 | ||
|
|
505c16a724 | ||
|
|
66e567cc95 | ||
|
|
dff08bc161 | ||
|
|
aad3a37bcf | ||
|
|
d03d0ed1c5 | ||
|
|
02190d03a8 | ||
|
|
40dc0e5417 | ||
|
|
ac378f07d3 | ||
|
|
827de84f40 | ||
|
|
9eb5e9453d | ||
|
|
f5856076ce | ||
|
|
4040007264 | ||
|
|
56d0a4904c | ||
|
|
e86c0dcf05 | ||
|
|
39877202b2 | ||
|
|
41027aa1b1 | ||
|
|
c7375573ed | ||
|
|
4f37af209c | ||
|
|
3ebd5e4166 | ||
|
|
d9f168c317 | ||
|
|
cdb771d722 | ||
|
|
55fa2c4870 | ||
|
|
11422d4584 | ||
|
|
b5a56bee0b | ||
|
|
95586e6395 | ||
|
|
5583159497 | ||
|
|
3f085d0724 | ||
|
|
e769845417 | ||
|
|
9a90c7677f | ||
|
|
65b15bbe75 | ||
|
|
95c4542c6f | ||
|
|
534c95248a | ||
|
|
e4dc781c26 | ||
|
|
d0339128e4 | ||
|
|
ef7e07c765 | ||
|
|
83fa80c330 | ||
|
|
e472a8bf74 | ||
|
|
44177974cb | ||
|
|
d11b354ed6 | ||
|
|
2f8b7cb306 | ||
|
|
37e4e08327 | ||
|
|
d2ea705a44 | ||
|
|
52a00370ea | ||
|
|
bdc38422a9 | ||
|
|
a985af506b | ||
|
|
a0fc5171e3 | ||
|
|
0652beba90 | ||
|
|
f35e010f79 | ||
|
|
d868745194 | ||
|
|
1bb3678f32 | ||
|
|
2f67b98287 | ||
|
|
f3b9f61f11 | ||
|
|
b1e7abd380 | ||
|
|
ae755d2ce0 | ||
|
|
4032f4b6ab | ||
|
|
88fa13689a | ||
|
|
cfe47d3cb0 | ||
|
|
fd949b870c | ||
|
|
80c6ec5340 | ||
|
|
c7ba481d3c | ||
|
|
9ac471eb72 | ||
|
|
5772de8d27 | ||
|
|
3be0ada6a1 | ||
|
|
48b1292115 | ||
|
|
fb9134f081 | ||
|
|
f9835aa4d6 | ||
|
|
863d717435 | ||
|
|
8252d9d226 | ||
|
|
28ad3a1a1f | ||
|
|
a905cc2316 | ||
|
|
9ab25f9aaa | ||
|
|
2964a3b632 | ||
|
|
9601837aa9 | ||
|
|
8bf759d6f8 | ||
|
|
fc3266d455 | ||
|
|
06099579e6 | ||
|
|
f68e92b3b6 |
@@ -1,22 +1,12 @@
|
||||
{
|
||||
"required_status_checks": {
|
||||
"strict": true,
|
||||
"strict": false,
|
||||
"contexts": [
|
||||
"py3.10 on ubuntu-latest",
|
||||
"py3.11 on ubuntu-latest",
|
||||
"py3.12 on ubuntu-latest",
|
||||
"py3.13 on ubuntu-latest",
|
||||
"py3.10 on windows-latest",
|
||||
"py3.11 on windows-latest",
|
||||
"py3.12 on windows-latest",
|
||||
"py3.13 on windows-latest",
|
||||
"py3.10 on macos-latest",
|
||||
"py3.11 on macos-latest",
|
||||
"py3.12 on macos-latest",
|
||||
"py3.13 on macos-latest",
|
||||
"coverage gate (85%)",
|
||||
"black and flake8",
|
||||
"frontend modules",
|
||||
"all tests",
|
||||
"viewerjs",
|
||||
"real ComfyUI compatibility",
|
||||
"bandit (high severity)"
|
||||
|
||||
@@ -4,6 +4,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
@@ -19,14 +20,17 @@ jobs:
|
||||
security:
|
||||
name: bandit (high severity)
|
||||
runs-on: ubuntu-latest
|
||||
# ceil(1.5 x slowest of three green main runs) + 3, never under 5 (test.yml): runs 40257217665,
|
||||
# 40254137380, 40240706583 took 23s / 16s / 23s -> 5 min.
|
||||
timeout-minutes: 5
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python 3.10
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.10"
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install bandit
|
||||
run: pip install bandit[toml]
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
name: Windows and macOS
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context
|
||||
# here is a required check. A red run is fixed at its cause before the workspace cutover (#334).
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
schedule:
|
||||
- cron: "17 3 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
# Every run gets its own group (run_id), so no main commit's run is ever cancelled or dropped as pending.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.run_id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PYTHONFAULTHANDLER: "1"
|
||||
|
||||
# Time limit rule as in test.yml: ceil(1.5 x the slowest of three recent green runs on main) + 3
|
||||
# minutes, never under 5, timed as whole-job wall time. One matrix job, so the slowest leg of each
|
||||
# run counts (tests/vendor/ci-workflow.test.mjs recomputes it, #493).
|
||||
# Measured 2026-10-29 on main runs 40935337099 (dd247e3), 40938407252 (dd247e3), 40941240624 (dd247e3):
|
||||
# slowest leg: 7m24s (py3.12 windows) / 7m31s (py3.13 windows) / 7m32s (py3.14 windows) -> 15 min
|
||||
# (windows 4m56s-7m32s, macos 4m39s-6m14s)
|
||||
# native-trash (#499) has no green main runs yet: its probe, run 40949370943 (2026-10-29), took
|
||||
# about 1 minute per macOS leg, so ceil(1.5 x 2) + 3 gives it 6 minutes (step 4). Recalibrate
|
||||
# after three green runs.
|
||||
# native-trash-linux (#543) has no green main runs yet: it installs the dev requirements and runs
|
||||
# one test module, about 1 minute by the same estimate, so ceil(1.5 x 2) + 3 gives it 6 minutes.
|
||||
# Recalibrate after three green runs.
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 15
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [windows-latest, macos-latest]
|
||||
python-version: ["3.12", "3.13", "3.14"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# The runner's TEMP is on C:, the OS disk, where a flush took about 6 ms
|
||||
# and a test database's create-to-delete about 111 ms; on the D: work
|
||||
# disk the same took 0.5 ms and 23 ms (macOS: 19 ms). Tests write their
|
||||
# databases under TEMP, so it moves to a folder on D: (measured in #487,
|
||||
# run 40915866444). The runner's TEMP was an 8.3 short name
|
||||
# (C:\Users\RUNNER~1\...), so the new one is spelled by its short name too
|
||||
# and paths still have to be resolved. D: makes no short names by default
|
||||
# (run 40922502209), so they are switched on for it first.
|
||||
- name: Keep test files on the work disk (Windows)
|
||||
if: runner.os == 'Windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
fsutil 8dot3name set D: 0
|
||||
$folder = New-Item -ItemType Directory -Force -Path (Join-Path $env:RUNNER_TEMP "PromptManagerTests")
|
||||
$short = (New-Object -ComObject Scripting.FileSystemObject).GetFolder($folder.FullName).ShortPath
|
||||
Write-Output "test TEMP: $short (long name $($folder.FullName))"
|
||||
if ($short -eq $folder.FullName) { Write-Output "::warning::D: gave the test TEMP no 8.3 short name" }
|
||||
"TEMP=$short" >> $env:GITHUB_ENV
|
||||
"TMP=$short" >> $env:GITHUB_ENV
|
||||
|
||||
- name: Run tests
|
||||
env:
|
||||
PM_TEST_SLOWEST: "40"
|
||||
run: python -m tests.run_suite -v
|
||||
|
||||
# The real Trash on both macOS CPUs (#499 design §10.2). It runs apart from the suite, so a red
|
||||
# suite never skips the native proof.
|
||||
native-trash:
|
||||
name: native trash on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 6
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [macos-latest, macos-15-intel]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Native trash proof
|
||||
timeout-minutes: 4
|
||||
env:
|
||||
PM_NATIVE_TRASH: macos
|
||||
run: python -m unittest tests.test_trash_macos_native tests.test_trash_macos -v
|
||||
|
||||
# The same tests as the fake mount table, on a real ext4 mount used as the output root (#543). A
|
||||
# runner can mount with sudo; the tests run as the normal user, so owner and mode checks are real.
|
||||
native-trash-linux:
|
||||
name: native trash on a real mount
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 6
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Mount an ext4 image
|
||||
run: |
|
||||
truncate -s 64M "$RUNNER_TEMP/drive.img"
|
||||
mkfs.ext4 -q "$RUNNER_TEMP/drive.img"
|
||||
mkdir "$RUNNER_TEMP/drive"
|
||||
sudo mount -o loop "$RUNNER_TEMP/drive.img" "$RUNNER_TEMP/drive"
|
||||
sudo rmdir "$RUNNER_TEMP/drive/lost+found"
|
||||
sudo chown "$(id -u):$(id -g)" "$RUNNER_TEMP/drive"
|
||||
findmnt "$RUNNER_TEMP/drive"
|
||||
|
||||
- name: Native trash proof
|
||||
timeout-minutes: 4
|
||||
run: PM_TRASH_MOUNT="$RUNNER_TEMP/drive" python -m unittest tests.test_trash_mount_root -v
|
||||
@@ -4,6 +4,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
@@ -19,15 +20,36 @@ concurrency:
|
||||
env:
|
||||
PYTHONFAULTHANDLER: "1"
|
||||
|
||||
# Every job has a time limit: ceil(1.5 x the slowest of three recent green runs on main) + 3
|
||||
# minutes, never under 5. The 3 minutes leave room for the Python runner's watchdog (120 s, see
|
||||
# tests/run_suite.py) to print its stack dump before GitHub cancels the job. A limit changes only
|
||||
# with new measurements written here (TESTING.md "Time limits and hangs").
|
||||
# Measured 2026-10-26 on runs 40257217681 (3be0ada), 40240706568 (fb9134f), 40228167328 (f9835aa):
|
||||
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
# 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
|
||||
# Windows and macOS moved to portability.yml (#420), which has its own measured limit (#493);
|
||||
# this Linux-only matrix keeps 27 minutes until it is re-measured.
|
||||
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
|
||||
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
|
||||
# perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s,
|
||||
# frontend modules 23s / 19s / 29s -> 5 min (the floor)
|
||||
# all tests: a result check without setup, 5 min (the floor)
|
||||
#
|
||||
# Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend
|
||||
# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on
|
||||
# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set.
|
||||
# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 27
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
||||
os: [ubuntu-latest]
|
||||
python-version: ${{ github.event_name == 'pull_request' && fromJSON('["3.12"]') || fromJSON('["3.12", "3.13", "3.14"]') }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
@@ -45,21 +67,26 @@ jobs:
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Run tests
|
||||
run: python -m unittest discover -s tests -v
|
||||
run: python -m tests.run_suite -v
|
||||
|
||||
comfyui:
|
||||
name: real ComfyUI compatibility
|
||||
# Merge queue, main and manual runs only. A job skipped by its condition reports Success, so this
|
||||
# required context never blocks a pull request; "all tests" fails if it is skipped anywhere else.
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
# Chromium and its system libraries come from the pinned Playwright image (tag = @playwright/test in
|
||||
# tools/vendor/package-lock.json; tests/vendor/browser-image.test.mjs checks it). The runner install
|
||||
# pulled them from Ubuntu's apt mirror on every run, and three runs (#349, #378, #382) crawled at
|
||||
# ~26 kB/s for 11-12 minutes until the job hit its limit after every test had passed.
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.64.1-noble
|
||||
options: --user 1001
|
||||
timeout-minutes: 24
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
cache-dependency-path: tools/vendor/package-lock.json
|
||||
- name: Read the reviewed runtime pins
|
||||
id: runtime
|
||||
run: |
|
||||
@@ -84,21 +111,42 @@ jobs:
|
||||
ref: ${{ steps.runtime.outputs.kikoRevision }}
|
||||
path: .ci-kikotools
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v7
|
||||
# The image's own Python and Node 24: setup-python/setup-node install the runner's toolcache builds,
|
||||
# whose entry points name host paths that do not exist in the container. The image has no ensurepip
|
||||
# and the job has no root, so a checksum-pinned pip wheel bootstraps a user-space venv. The image
|
||||
# Python must be the reviewed host Python, so a pin or image bump that disagrees fails here.
|
||||
- name: Python environment on the reviewed host Python
|
||||
env:
|
||||
PIP_WHEEL: https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl
|
||||
PIP_SHA256: 71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e
|
||||
PINNED_PYTHON: ${{ steps.runtime.outputs.pythonVersion }}
|
||||
run: |
|
||||
image="$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])')"
|
||||
[ "$image" = "$PINNED_PYTHON" ] || { echo "::error::image Python $image is not the pinned $PINNED_PYTHON"; exit 1; }
|
||||
venv="$RUNNER_TEMP/venv" wheel="$RUNNER_TEMP/pip-26.2.1-py3-none-any.whl"
|
||||
python3 -m venv --without-pip "$venv"
|
||||
curl -fsSL --retry 3 -o "$wheel" "$PIP_WHEEL"
|
||||
echo "$PIP_SHA256 $wheel" | sha256sum -c -
|
||||
"$venv/bin/python" "$wheel/pip" install --no-index --no-cache-dir "$wheel"
|
||||
echo "$venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python --version && node --version && node --version | grep -q '^v24\.'
|
||||
# pip's download cache (torch alone is ~200 MB); inside the container ~ is the job's HOME (/github/home).
|
||||
- name: Cache pip downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
python-version: ${{ steps.runtime.outputs.pythonVersion }}
|
||||
cache: pip
|
||||
cache-dependency-path: |
|
||||
requirements-dev.txt
|
||||
.ci-comfyui/requirements.txt
|
||||
tools/vendor/comfyui-runtime.json
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('requirements-dev.txt', '.ci-comfyui/requirements.txt', 'tools/vendor/comfyui-runtime.json') }}
|
||||
- name: Install CPU runtime and test dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}" --index-url https://download.pytorch.org/whl/cpu
|
||||
pip install -r .ci-comfyui/requirements.txt -r requirements-dev.txt "torch==${{ steps.runtime.outputs.torchVersion }}" "torchvision==${{ steps.runtime.outputs.torchvisionVersion }}"
|
||||
# npm's download cache, keyed on the lockfile. node_modules itself is still built fresh by npm ci.
|
||||
- name: Cache npm downloads
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- run: npx --prefix tools/vendor playwright install --with-deps chromium
|
||||
- name: Classic and Nodes 2.0 startup and compatibility
|
||||
env:
|
||||
PM_COMFY_ROOT: ${{ github.workspace }}/.ci-comfyui
|
||||
@@ -127,6 +175,7 @@ jobs:
|
||||
coverage:
|
||||
name: coverage gate (85%)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
@@ -143,7 +192,7 @@ jobs:
|
||||
|
||||
- name: Measure coverage
|
||||
run: |
|
||||
python -m coverage run -m unittest discover -s tests
|
||||
python -m coverage run -m tests.run_suite
|
||||
python -m coverage combine
|
||||
python -m coverage report --fail-under=85
|
||||
|
||||
@@ -154,6 +203,7 @@ jobs:
|
||||
name: perf budget
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
@@ -176,24 +226,31 @@ jobs:
|
||||
lint:
|
||||
name: black and flake8
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install black flake8
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
# The exact black and flake8 pins from requirements-dev.txt, so a new release cannot turn CI red.
|
||||
- run: pip install $(grep -E '^(black|flake8)==' requirements-dev.txt)
|
||||
- run: black --check .
|
||||
- run: flake8 .
|
||||
|
||||
js-test:
|
||||
name: frontend modules
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
cache-dependency-path: tools/vendor/package-lock.json
|
||||
|
||||
- name: Check shipped extension and page modules parse
|
||||
run: |
|
||||
@@ -207,3 +264,32 @@ jobs:
|
||||
|
||||
- name: Run JS tests
|
||||
run: node --test tests/js/*.test.js
|
||||
|
||||
# The required context "all tests". A dropped matrix leg never reports a check at all, so the eight
|
||||
# legs that pull requests skip cannot be required by name; this gate requires them instead. Every
|
||||
# job must succeed; real ComfyUI must be skipped on a pull request and succeed on every other event.
|
||||
all-tests:
|
||||
name: all tests
|
||||
needs: [test, comfyui, coverage, lint, js-test]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Require every test job for this event
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
TEST_RESULT: ${{ needs.test.result }}
|
||||
COMFYUI_RESULT: ${{ needs.comfyui.result }}
|
||||
COVERAGE_RESULT: ${{ needs.coverage.result }}
|
||||
LINT_RESULT: ${{ needs.lint.result }}
|
||||
JS_RESULT: ${{ needs.js-test.result }}
|
||||
run: |
|
||||
queue_only=success
|
||||
if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi
|
||||
echo "event: $EVENT_NAME (real ComfyUI must be $queue_only)"
|
||||
echo "test matrix: $TEST_RESULT"
|
||||
echo "comfyui: $COMFYUI_RESULT"
|
||||
echo "coverage: $COVERAGE_RESULT"
|
||||
echo "lint: $LINT_RESULT"
|
||||
echo "js-test: $JS_RESULT"
|
||||
test "$TEST_RESULT" = success && test "$COVERAGE_RESULT" = success && test "$LINT_RESULT" = success && test "$JS_RESULT" = success && test "$COMFYUI_RESULT" = "$queue_only"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
name: Viewer.js vendor and compatibility
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
push:
|
||||
@@ -26,6 +27,20 @@ concurrency:
|
||||
# coverage shards 7m34s/6m04s/5m22s (tests 6m38s/4m53s/4m04s), merge 54s, firefox 7m57s/5m21s (tests
|
||||
# 7m08s/4m34s), webkit 8m16s/5m03s (tests 7m26s/3m58s); a cache miss adds up to 5m to npm ci, which
|
||||
# the 20-minute limits absorb.
|
||||
# Hangs (#425): a job killed at its limit keeps no log and skips its upload steps, so each browser
|
||||
# test step has its own timeout-minutes (18) below the job's 20. At the step limit, SIGTERM makes
|
||||
# tools/vendor/running-tests-reporter.mjs print the tests still running and hand over to Playwright's
|
||||
# interrupt, which writes the blob report; the always() uploads then run.
|
||||
# Measured on main (runs 40725289094, 40719375442, 40700439399): setup before the test step <= 52s,
|
||||
# uploads <= 35s; test steps up to 14m42s (webkit shard 1/2) and 13m05s for coverage (shard 1/3:
|
||||
# 12m53s, node units and the viewer suite before 11.8m of workflows), so coverage is the tight one
|
||||
# in its 18 minutes. This holds only when the npm cache hits: a cold cache adds up to 5m to npm ci,
|
||||
# and then the job limit can still end the job before the step limit does.
|
||||
# #473: firefox/webkit shard 1/2 reached its 18-minute step limit (queue run 40866252975: webkit 622 tests
|
||||
# passed in 17m56s, 48.5 worker-minutes vs 28.0 in shard 2/2; recent shard 1/2 jobs 13.7m-18.9m, shard 2/2
|
||||
# 7.3m-12.2m), because --shard splits by test count and the slow concept specs sort into shard 1. The page
|
||||
# workflows now run in three shards per browser with the limits unchanged; the per-shard times are
|
||||
# re-measured after three green main runs.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
@@ -78,6 +93,7 @@ jobs:
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
@@ -86,6 +102,7 @@ jobs:
|
||||
path: coverage/js/raw/
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
# Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
|
||||
- name: Upload Chromium and touch traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
@@ -163,7 +180,9 @@ jobs:
|
||||
retention-days: 14
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/3)
|
||||
# Merge queue, main and manual runs only (#420); pull requests run the Chromium/touch coverage shards.
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -180,7 +199,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: [firefox, webkit]
|
||||
shard: [1, 2]
|
||||
shard: [1, 2, 3]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -207,8 +226,10 @@ jobs:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/3
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/3
|
||||
# Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
@@ -218,8 +239,9 @@ jobs:
|
||||
retention-days: 14
|
||||
|
||||
# Branch protection requires the exact context "viewerjs". This job keeps that
|
||||
# name and fails unless every frontend job above succeeded (every shard, the
|
||||
# coverage merge and every browser shard), including when one was cancelled or skipped.
|
||||
# name and fails unless every coverage shard and the coverage merge succeeded and
|
||||
# the browser shards succeeded, or, on a pull request only, were skipped (#420).
|
||||
# Any failure, cancellation or other skip fails it.
|
||||
viewerjs:
|
||||
needs: [frontend-coverage, frontend-coverage-merge, browser-workflows]
|
||||
if: always()
|
||||
@@ -228,11 +250,15 @@ jobs:
|
||||
steps:
|
||||
- name: Require every frontend job to succeed
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
COVERAGE_RESULT: ${{ needs.frontend-coverage.result }}
|
||||
MERGE_RESULT: ${{ needs.frontend-coverage-merge.result }}
|
||||
WORKFLOWS_RESULT: ${{ needs.browser-workflows.result }}
|
||||
run: |
|
||||
queue_only=success
|
||||
if [ "$EVENT_NAME" = pull_request ]; then queue_only=skipped; fi
|
||||
echo "event: $EVENT_NAME (browser-workflows must be $queue_only)"
|
||||
echo "frontend-coverage: $COVERAGE_RESULT"
|
||||
echo "frontend-coverage-merge: $MERGE_RESULT"
|
||||
echo "browser-workflows: $WORKFLOWS_RESULT"
|
||||
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = success
|
||||
test "$COVERAGE_RESULT" = success && test "$MERGE_RESULT" = success && test "$WORKFLOWS_RESULT" = "$queue_only"
|
||||
|
||||
@@ -211,3 +211,10 @@ coverage/
|
||||
|
||||
# Disposable upstream checkout for real-host CI
|
||||
.ci-comfyui/
|
||||
|
||||
# Old library files a Move library renamed (never deleted; #249 S6b)
|
||||
*.moved-to-*
|
||||
|
||||
# Per-install key behind the workspace scopes and cursors (#430)
|
||||
workspace-scope.key
|
||||
.workspace-scope.key.*.tmp
|
||||
|
||||
@@ -13,16 +13,18 @@ repos:
|
||||
language: system
|
||||
types: [python]
|
||||
|
||||
# Fails at the severity CI enforces (code-quality.yml: -lll, high only), never stricter.
|
||||
# pre-commit appends the staged files, so no shell wrapper can drop the first one.
|
||||
- id: bandit
|
||||
name: bandit
|
||||
entry: bash -c 'bandit "$@" -ll -q 2>&1; true'
|
||||
entry: bandit -lll -q
|
||||
language: system
|
||||
types: [python]
|
||||
exclude: ^(tests/|reference/)
|
||||
|
||||
- id: tests
|
||||
name: tests
|
||||
entry: python -m unittest discover tests/ -v
|
||||
entry: python -m tests.run_suite -q
|
||||
language: system
|
||||
pass_filenames: false
|
||||
stages: [pre-push]
|
||||
|
||||
@@ -22,22 +22,21 @@ Features:
|
||||
- Comprehensive logging and diagnostics
|
||||
"""
|
||||
|
||||
import re
|
||||
import atexit
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_version():
|
||||
"""Parse version from pyproject.toml"""
|
||||
"""Parse version from pyproject.toml (``utils/version.py``)."""
|
||||
from .utils.version import project_version
|
||||
|
||||
try:
|
||||
pyproject_path = Path(__file__).parent / "pyproject.toml"
|
||||
if pyproject_path.exists():
|
||||
content = pyproject_path.read_text()
|
||||
match = re.search(r'version\s*=\s*["\']([^"\']+)["\']', content)
|
||||
if match:
|
||||
return match.group(1)
|
||||
root = Path(__file__).parent
|
||||
except Exception:
|
||||
pass
|
||||
return "unknown"
|
||||
return "unknown"
|
||||
return project_version(root)
|
||||
|
||||
|
||||
from .prompt_manager import ( # noqa: E402
|
||||
@@ -72,6 +71,12 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
# are served by our own routes instead (see py/api).
|
||||
WEB_DIRECTORY = "web/comfy"
|
||||
|
||||
# Parts the package teardown stops (``_stop_background_work``). Each stays None
|
||||
# when its start-up block below fails, and the teardown skips it.
|
||||
api = None
|
||||
_global_prompt_tracker = None
|
||||
_global_image_monitor = None
|
||||
|
||||
# Add API routes (same pattern as ComfyUI_Assets)
|
||||
try:
|
||||
# Set extension URI
|
||||
@@ -87,10 +92,22 @@ try:
|
||||
routes = config.routes
|
||||
api = PromptManagerAPI()
|
||||
api.add_routes(routes)
|
||||
# Temp files of preview writes a crash interrupted (#317), off the
|
||||
# import path: the walk may cover a large thumbnails folder.
|
||||
threading.Thread(
|
||||
target=api.sweep_thumbnail_temps,
|
||||
name="PromptManagerPreviewTempSweep",
|
||||
daemon=True,
|
||||
).start()
|
||||
host_app = getattr(config.server_instance, "app", None)
|
||||
if host_app is not None and hasattr(host_app, "on_cleanup"):
|
||||
host_app.on_cleanup.append(api.cleanup_history_reads)
|
||||
host_app.on_cleanup.append(api.cleanup_annotation_captures)
|
||||
host_app.on_cleanup.append(api.cleanup_gallery_listings)
|
||||
if host_app is not None and hasattr(host_app, "on_startup"):
|
||||
# Build each output root's Images listing once the server's loop runs,
|
||||
# so the first open does not wait for the walk (#348).
|
||||
host_app.on_startup.append(api.warm_gallery_listings)
|
||||
if host_app is not None and hasattr(host_app, "on_shutdown"):
|
||||
# Open Needs review change streams end before the server waits on them.
|
||||
host_app.on_shutdown.append(api.close_review_events)
|
||||
@@ -127,18 +144,16 @@ try:
|
||||
|
||||
# The Needs review pass for ambiguous new outputs runs through the API's
|
||||
# capture store (the one the scan and the Tag dialog use): a linked output
|
||||
# is never queued and its pending items leave the queue. It stops before
|
||||
# that store is closed at shutdown.
|
||||
_review_api = globals().get("api")
|
||||
if _review_api is not None:
|
||||
_global_image_monitor.set_review_recorder(_review_api.record_review_file)
|
||||
_review_host = getattr(config.server_instance, "app", None)
|
||||
if _review_host is not None and hasattr(_review_host, "on_cleanup"):
|
||||
|
||||
async def _stop_review_recording(app, monitor=_global_image_monitor):
|
||||
monitor.set_review_recorder(None)
|
||||
|
||||
_review_host.on_cleanup.insert(0, _stop_review_recording)
|
||||
# is never queued and its pending items leave the queue. The package
|
||||
# teardown detaches it before that store is closed at shutdown.
|
||||
if api is not None:
|
||||
_global_image_monitor.set_review_recorder(api.record_review_file)
|
||||
# Every settled output's preview is queued for the API's background
|
||||
# worker (#249): the workspace never asks to generate thumbnails.
|
||||
_global_image_monitor.set_thumbnail_queue(api.queue_capture_thumbnail)
|
||||
# Every settled output walks its Images listing again (#348): the
|
||||
# API hops from the monitor's thread onto the server's loop.
|
||||
_global_image_monitor.set_output_listener(api.notify_output_settled)
|
||||
|
||||
# Start monitoring
|
||||
_global_image_monitor.start_monitoring()
|
||||
@@ -180,6 +195,26 @@ except Exception as e:
|
||||
f"Failed to register usage hook: {e}"
|
||||
)
|
||||
|
||||
# A cached node counts what its last execution of the same text counted (#347).
|
||||
try:
|
||||
from .py.config import server_instance
|
||||
from .utils.cached_runs import register_cached_runs
|
||||
|
||||
register_cached_runs(server_instance, _db_factory)
|
||||
except Exception as e:
|
||||
try:
|
||||
from .utils.logging_config import get_logger
|
||||
|
||||
get_logger("prompt_manager.init").error(
|
||||
f"Failed to register cached-run counting: {e}"
|
||||
)
|
||||
except Exception:
|
||||
import logging
|
||||
|
||||
logging.getLogger("prompt_manager.init").error(
|
||||
f"Failed to register cached-run counting: {e}"
|
||||
)
|
||||
|
||||
|
||||
def _history_db_factory() -> "PromptDatabase":
|
||||
"""Initialize history on its worker even if monitoring initialization failed."""
|
||||
@@ -208,6 +243,193 @@ except Exception:
|
||||
"Failed to register execution history"
|
||||
)
|
||||
|
||||
|
||||
# ── Package teardown (#301, #302) ────────────────────────────────────────────
|
||||
# ComfyUI never runs aiohttp's cleanup hooks (it never calls
|
||||
# ``runner.cleanup()``), so ``atexit`` is the live host's path: Ctrl+C, and a
|
||||
# normal exit. It is registered after the history hook, so at exit (newest
|
||||
# first) it runs before the history dispose. Hosts that do run ``on_cleanup``
|
||||
# (our test servers) run it there, and the tests' package unload calls it.
|
||||
|
||||
TEARDOWN_SECONDS = 10.0
|
||||
_clock = time.monotonic
|
||||
_background_work_lock = threading.Lock()
|
||||
_background_work_result = None
|
||||
|
||||
|
||||
def _teardown_step(name, stop, logger):
|
||||
"""Run one teardown step; a failure (even Ctrl+C) is logged and is False."""
|
||||
try:
|
||||
return bool(stop())
|
||||
except (Exception, KeyboardInterrupt):
|
||||
logger.error(f"Background work: stopping {name} failed", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def _teardown_logger():
|
||||
"""PromptManager's init logger; the standard one if that cannot load."""
|
||||
try:
|
||||
from .utils.logging_config import get_logger
|
||||
|
||||
return get_logger("prompt_manager.init")
|
||||
except Exception:
|
||||
import logging
|
||||
|
||||
return logging.getLogger("prompt_manager.init")
|
||||
|
||||
|
||||
def _stop_background_work():
|
||||
"""Stop the image monitor, the preview worker and the prompt cleanup
|
||||
thread against one TEARDOWN_SECONDS deadline, once.
|
||||
|
||||
Returns ``{"monitor", "thumbnails", "tracker"}`` booleans (True: it
|
||||
stopped), leaving out parts that never started. Later calls return a
|
||||
copy of the first result. A Ctrl+C inside a step is logged and the
|
||||
step is False; one between steps is re-raised after the result (parts
|
||||
not reached are False) is stored and the ``atexit`` entry removed.
|
||||
"""
|
||||
global _background_work_result
|
||||
with _background_work_lock:
|
||||
if _background_work_result is not None:
|
||||
return dict(_background_work_result)
|
||||
monitor, thumbnails, tracker = (
|
||||
_global_image_monitor,
|
||||
api,
|
||||
_global_prompt_tracker,
|
||||
)
|
||||
# A part whose global is set has started; False until its step says
|
||||
# otherwise, so an interrupt between steps leaves an honest result.
|
||||
started = (
|
||||
("monitor", monitor),
|
||||
("thumbnails", thumbnails),
|
||||
("tracker", tracker),
|
||||
)
|
||||
result = {name: False for name, part in started if part is not None}
|
||||
try:
|
||||
logger = _teardown_logger()
|
||||
began = _clock()
|
||||
deadline = began + TEARDOWN_SECONDS
|
||||
|
||||
def left(own_bound):
|
||||
return max(0.0, min(own_bound, deadline - _clock()))
|
||||
|
||||
if monitor is not None:
|
||||
|
||||
def stop_monitor():
|
||||
from .utils.image_monitor import OBSERVER_JOIN_SECONDS
|
||||
|
||||
# Callbacks first: an image in hand queues no preview into a
|
||||
# closing preview worker. The worker may have read the queue
|
||||
# callback already, so the API refuses new previews too (at
|
||||
# exit, Python 3.12+ refuses to start their worker thread).
|
||||
if thumbnails is not None:
|
||||
thumbnails.refuse_auto_thumbnails()
|
||||
monitor.set_review_recorder(None)
|
||||
monitor.set_thumbnail_queue(None)
|
||||
monitor.set_output_listener(None)
|
||||
return monitor.stop_monitoring(left(OBSERVER_JOIN_SECONDS))
|
||||
|
||||
result["monitor"] = _teardown_step(
|
||||
"the image monitor", stop_monitor, logger
|
||||
)
|
||||
if thumbnails is not None:
|
||||
|
||||
def close_previews():
|
||||
from .py.api.thumbnail_auto import CLOSE_WAIT_SECONDS
|
||||
|
||||
return thumbnails.close_auto_thumbnails(
|
||||
final=True, timeout=left(CLOSE_WAIT_SECONDS)
|
||||
)
|
||||
|
||||
result["thumbnails"] = _teardown_step(
|
||||
"the preview worker", close_previews, logger
|
||||
)
|
||||
if tracker is not None:
|
||||
|
||||
def stop_tracker():
|
||||
from .utils.prompt_tracker import TRACKER_JOIN_SECONDS
|
||||
|
||||
return tracker.shutdown(left(TRACKER_JOIN_SECONDS))
|
||||
|
||||
result["tracker"] = _teardown_step(
|
||||
"the prompt cleanup thread", stop_tracker, logger
|
||||
)
|
||||
elapsed = _clock() - began
|
||||
failed = [name for name, ok in result.items() if not ok]
|
||||
if failed:
|
||||
logger.warning(
|
||||
f"Background work not stopped in time ({elapsed:.1f} s): "
|
||||
+ ", ".join(failed)
|
||||
)
|
||||
else:
|
||||
parts = " ".join(f"{name}=ok" for name in result)
|
||||
logger.info(f"Background work stopped in {elapsed:.1f} s: {parts}")
|
||||
finally:
|
||||
_background_work_result = result
|
||||
atexit.unregister(_stop_background_work)
|
||||
return dict(result)
|
||||
|
||||
|
||||
async def _stop_background_work_on_cleanup(app):
|
||||
"""aiohttp ``on_cleanup``: no request is served then, so it runs inline."""
|
||||
_stop_background_work()
|
||||
|
||||
|
||||
def _kill_encoders_at_exit():
|
||||
"""Run by the exit hook before Python joins the executor threads (#376):
|
||||
a JPEG XL or ffmpeg encoder (no console on Windows, so Ctrl+C never
|
||||
reaches it) is killed instead of being waited for."""
|
||||
from .py.api.thumbnail_encode import shutdown_encoders
|
||||
|
||||
shutdown_encoders()
|
||||
|
||||
|
||||
def _abort_downloads_at_exit():
|
||||
"""Run by the exit hook (#376): huggingface_hub's Xet transfers ignore
|
||||
the chunk checkpoint, so they are aborted the way hub aborts on Ctrl+C."""
|
||||
import sys
|
||||
|
||||
if "huggingface_hub" not in sys.modules:
|
||||
return # no download ever ran: import nothing at exit
|
||||
from .py.autotag import abort_hub_downloads
|
||||
|
||||
abort_hub_downloads()
|
||||
|
||||
|
||||
try:
|
||||
# Before the executor join: running jobs stop at their next file, item or
|
||||
# chunk (#376). The teardown below still runs from atexit afterwards.
|
||||
from .utils import exit_signal as _exit_signal
|
||||
|
||||
_exit_signal.install(_kill_encoders_at_exit)
|
||||
_exit_signal.install(_abort_downloads_at_exit)
|
||||
except Exception as e:
|
||||
try:
|
||||
from .utils.logging_config import get_logger
|
||||
|
||||
get_logger("prompt_manager.init").error(
|
||||
f"Failed to register the exit signal: {e}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
atexit.register(_stop_background_work)
|
||||
from .py.config import server_instance as _teardown_server
|
||||
|
||||
_teardown_host = getattr(_teardown_server, "app", None)
|
||||
if _teardown_host is not None and hasattr(_teardown_host, "on_cleanup"):
|
||||
_teardown_host.on_cleanup.insert(0, _stop_background_work_on_cleanup)
|
||||
except Exception as e:
|
||||
try:
|
||||
from .utils.logging_config import get_logger
|
||||
|
||||
get_logger("prompt_manager.init").error(
|
||||
f"Failed to register the background work teardown: {e}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
|
||||
# Print startup message with loaded tools
|
||||
|
||||
|
After Width: | Height: | Size: 35 KiB |
|
After Width: | Height: | Size: 88 KiB |
|
After Width: | Height: | Size: 153 KiB |
|
After Width: | Height: | Size: 82 KiB |
|
After Width: | Height: | Size: 72 KiB |
|
After Width: | Height: | Size: 156 KiB |
|
After Width: | Height: | Size: 159 KiB |
|
After Width: | Height: | Size: 96 KiB |
|
After Width: | Height: | Size: 165 KiB |
|
After Width: | Height: | Size: 60 KiB |
|
After Width: | Height: | Size: 38 KiB |
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 42 KiB |
|
After Width: | Height: | Size: 39 KiB |
|
After Width: | Height: | Size: 42 KiB |
|
After Width: | Height: | Size: 40 KiB |
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 43 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 237 KiB |
|
After Width: | Height: | Size: 66 KiB |
|
After Width: | Height: | Size: 53 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 181 KiB |
|
After Width: | Height: | Size: 51 KiB |
|
After Width: | Height: | Size: 110 KiB |
@@ -19,6 +19,11 @@ links (``PromptDatabase._move_tag_links``).
|
||||
Cheap reads (spec amendment S1) treat a byte record as *current* only while
|
||||
the file's present ``(size, mtime_ns)`` equals the record's verified
|
||||
fingerprint; a file without a current record shows no image tags.
|
||||
|
||||
A write made from a tagging model's result passes ``mark_autotagged`` (#379):
|
||||
the byte record is then marked auto-tagged (``media_image_autotags``) in the
|
||||
same transaction, so a mark never exists without the write it records. Hand
|
||||
tags never mark.
|
||||
"""
|
||||
|
||||
import re
|
||||
@@ -34,6 +39,8 @@ from .annotation_schema import (
|
||||
MAX_LOCATOR_LENGTH,
|
||||
METADATA_TABLE,
|
||||
)
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .image_removal_schema import clear_marks
|
||||
from .library_commands import _UUID, LibraryCommandError, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .vocabulary import validate_vocabulary_tag_name
|
||||
@@ -96,12 +103,24 @@ _TAGS_OF = (
|
||||
"JOIN tags t ON t.id=mit.tag_id WHERE mit.byte_id IN ({marks}) "
|
||||
"ORDER BY mit.byte_id,mit.position"
|
||||
)
|
||||
_RECORDS = (
|
||||
"SELECT f.locator,b.id,b.size,b.mtime_ns,t.name " # nosec B608
|
||||
f"FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
f"LEFT JOIN {IMAGE_TAGS_TABLE} mit ON mit.byte_id=b.id "
|
||||
"LEFT JOIN tags t ON t.id=mit.tag_id "
|
||||
"ORDER BY f.locator,b.first_seen_at,b.id,mit.position"
|
||||
_MARK = f"INSERT OR IGNORE INTO {AUTOTAG_TABLE}(byte_id) VALUES(?)" # nosec B608
|
||||
_MARKED_OF = (
|
||||
f"SELECT byte_id FROM {AUTOTAG_TABLE} WHERE byte_id IN ({{marks}})" # nosec B608
|
||||
)
|
||||
_MARKED_LOCATORS = (
|
||||
f"SELECT DISTINCT f.locator FROM {AUTOTAG_TABLE} m " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.id=m.byte_id JOIN {FILES_TABLE} f ON f.id=b.file_id"
|
||||
)
|
||||
# Tag search (#379): the names that tag at least one image (the tag index
|
||||
# answers the EXISTS), then the locators carrying chosen names (by tag_id).
|
||||
_IMAGE_TAG_NAMES = (
|
||||
"SELECT t.id,t.name FROM tags t WHERE EXISTS " # nosec B608
|
||||
f"(SELECT 1 FROM {IMAGE_TAGS_TABLE} mit WHERE mit.tag_id=t.id)"
|
||||
)
|
||||
_TAG_LOCATORS = (
|
||||
f"SELECT DISTINCT f.locator FROM {IMAGE_TAGS_TABLE} mit " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.id=mit.byte_id "
|
||||
f"JOIN {FILES_TABLE} f ON f.id=b.file_id WHERE mit.tag_id IN ({{marks}})"
|
||||
)
|
||||
|
||||
|
||||
@@ -218,6 +237,9 @@ class AnnotationOperations:
|
||||
|
||||
@staticmethod
|
||||
def _annotation_owner(conn: sqlite3.Connection, locator: str, now: str) -> str:
|
||||
# A file with image tags or Needs review state is in the library: a
|
||||
# removal mark never hides it (#480).
|
||||
clear_marks(conn, [locator])
|
||||
row = conn.execute(_OWNER, (locator,)).fetchone()
|
||||
if row is not None:
|
||||
return row[0]
|
||||
@@ -301,14 +323,17 @@ class AnnotationOperations:
|
||||
expected_epoch: str,
|
||||
expected_tag_revision: int,
|
||||
tags: list,
|
||||
mark_autotagged: bool = False,
|
||||
) -> dict:
|
||||
"""Replace the set of these exact bytes, guarded by epoch and revision.
|
||||
|
||||
Creates the owner and byte record when the bytes are new (an explicit
|
||||
empty set is a real save). The identical set at the current revision,
|
||||
or one behind it (a lost-response retry), is ``replayed`` and writes
|
||||
nothing; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes.
|
||||
no tag; any other revision mismatch is ``tag-conflict`` with the
|
||||
current set and no writes. ``mark_autotagged`` marks the bytes
|
||||
auto-tagged on a saved or replayed receipt (the Tag image dialog's
|
||||
save after Generate suggestions).
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
@@ -329,6 +354,8 @@ class AnnotationOperations:
|
||||
):
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
return _receipt(epoch, sha256, current, revision, True)
|
||||
if expected_tag_revision != revision:
|
||||
raise AnnotationCommandError(
|
||||
@@ -337,6 +364,8 @@ class AnnotationOperations:
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
revision = self._write_image_tags(conn, byte_id, tags)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
return _receipt(epoch, sha256, tags, revision, False)
|
||||
|
||||
def add_image_tags(
|
||||
@@ -348,16 +377,22 @@ class AnnotationOperations:
|
||||
mtime_ns: int,
|
||||
expected_epoch: str,
|
||||
tags: list,
|
||||
mark_autotagged: bool = False,
|
||||
) -> dict:
|
||||
"""Union ``tags`` into the set of these exact bytes, keeping stored spelling.
|
||||
|
||||
``added`` counts the names that were new (case-insensitively). A union
|
||||
over ``MAX_IMAGE_TAGS`` is ``tag-limit`` with the current set and no
|
||||
writes; a union that adds nothing to an existing record is ``replayed``.
|
||||
``mark_autotagged`` (a tagging model's result) also marks the bytes
|
||||
auto-tagged, replayed or not; only such a call may pass no tags (the
|
||||
model found none: the image is marked and its set is unchanged).
|
||||
"""
|
||||
_validate_target(locator, sha256, size, mtime_ns)
|
||||
_validate_epoch(expected_epoch)
|
||||
tags = normalize_image_tags(tags)
|
||||
if not tags and not mark_autotagged:
|
||||
raise ValueError("tags must name at least one tag")
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
now = utc_now_iso()
|
||||
@@ -375,6 +410,8 @@ class AnnotationOperations:
|
||||
epoch,
|
||||
{"sha256": sha256, "tags": current, "tag_revision": revision},
|
||||
)
|
||||
if mark_autotagged:
|
||||
conn.execute(_MARK, (byte_id,))
|
||||
if not created and not added:
|
||||
if refingerprinted:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
@@ -403,6 +440,41 @@ class AnnotationOperations:
|
||||
conn.execute(_BUMP_INDEX)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
@staticmethod
|
||||
def _validated_stats(stats: object) -> dict:
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
return stats
|
||||
|
||||
@staticmethod
|
||||
def _current_byte_records(conn: sqlite3.Connection, stats: dict) -> dict:
|
||||
"""``{locator: byte_id}`` of each locator's current byte record.
|
||||
|
||||
The one rule every cheap read uses: a record is current while its
|
||||
verified ``(size, mtime_ns)`` equals the file's present fingerprint
|
||||
in ``stats``; the most recently verified wins (ties on ``(verified_at,
|
||||
byte_id)``). A locator without a current record is left out. Lookups
|
||||
go in chunks of 500 locators.
|
||||
"""
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
return {locator: byte_id for locator, (_key, byte_id) in chosen.items()}
|
||||
|
||||
def current_image_tags(self, stats: dict) -> dict:
|
||||
"""Image tags of the files whose current ``(size, mtime_ns)`` is verified.
|
||||
|
||||
@@ -411,29 +483,11 @@ class AnnotationOperations:
|
||||
exactly that verified fingerprint; the most recently verified record
|
||||
wins when several do. Lookups go in chunks of 500 locators.
|
||||
"""
|
||||
if not isinstance(stats, dict):
|
||||
raise ValueError(
|
||||
"stats must map locators to (size, mtime_ns) integer pairs"
|
||||
)
|
||||
for locator, fingerprint in stats.items():
|
||||
_validate_fingerprint(locator, fingerprint)
|
||||
locators = list(stats)
|
||||
chosen: dict[str, tuple[tuple[str, str], str]] = {}
|
||||
result: dict[str, list[str]] = {}
|
||||
stats = self._validated_stats(stats)
|
||||
with self._annotation_read() as conn:
|
||||
for start in range(0, len(locators), _LOOKUP_CHUNK):
|
||||
chunk = locators[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
for locator, byte_id, size, mtime_ns, verified_at in conn.execute(
|
||||
_FINGERPRINTS.format(marks=marks), chunk
|
||||
):
|
||||
if (size, mtime_ns) != stats[locator]:
|
||||
continue
|
||||
key = (verified_at, byte_id)
|
||||
if locator not in chosen or key > chosen[locator][0]:
|
||||
chosen[locator] = (key, byte_id)
|
||||
owners = {byte_id: locator for locator, (_key, byte_id) in chosen.items()}
|
||||
result = {locator: [] for locator in chosen}
|
||||
current = self._current_byte_records(conn, stats)
|
||||
owners = {byte_id: locator for locator, byte_id in current.items()}
|
||||
result: dict[str, list[str]] = {locator: [] for locator in current}
|
||||
byte_ids = list(owners)
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
@@ -442,20 +496,60 @@ class AnnotationOperations:
|
||||
result[owners[byte_id]].append(name)
|
||||
return result
|
||||
|
||||
def iter_image_tag_records(self) -> Iterator[tuple[str, int, int, list[str]]]:
|
||||
"""Stream every byte record as ``(locator, size, mtime_ns, tags)``.
|
||||
def autotag_state(self, stats: dict) -> dict:
|
||||
"""``{locator: bool}``: is each file's current byte record auto-tagged?
|
||||
|
||||
One SELECT ordered by locator, then by the bytes' first sighting, with
|
||||
each set's names in position order.
|
||||
``stats`` as for :meth:`current_image_tags`, whose rule picks the
|
||||
current record; a locator without one is False. One read snapshot.
|
||||
"""
|
||||
conn = self.model.get_connection()
|
||||
current: tuple | None = None
|
||||
for locator, byte_id, size, mtime_ns, name in conn.execute(_RECORDS):
|
||||
if current is None or current[0] != byte_id:
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
current = (byte_id, (locator, size, mtime_ns, []))
|
||||
if name is not None:
|
||||
current[1][3].append(name)
|
||||
if current is not None:
|
||||
yield current[1]
|
||||
stats = self._validated_stats(stats)
|
||||
with self._annotation_read() as conn:
|
||||
current = self._current_byte_records(conn, stats)
|
||||
byte_ids = list(dict.fromkeys(current.values()))
|
||||
marked: set[str] = set()
|
||||
for start in range(0, len(byte_ids), _LOOKUP_CHUNK):
|
||||
chunk = byte_ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
marked.update(
|
||||
row[0]
|
||||
for row in conn.execute(_MARKED_OF.format(marks=marks), chunk)
|
||||
)
|
||||
return {locator: current.get(locator) in marked for locator in stats}
|
||||
|
||||
def autotagged_locators(self) -> set:
|
||||
"""Locators with at least one auto-tagged byte record, current or not.
|
||||
|
||||
No fingerprint is checked (the Tag output images review's estimate).
|
||||
"""
|
||||
with self._annotation_read() as conn:
|
||||
return {row[0] for row in conn.execute(_MARKED_LOCATORS)}
|
||||
|
||||
def image_tag_ids(self, wanted, query: str) -> list[int]:
|
||||
"""Ids of the tag names on at least one image that a search wants.
|
||||
|
||||
A name counts when it is in ``wanted`` exactly (the Tags filter) or
|
||||
contains ``query`` case-folded (the text search; ``query`` is
|
||||
already folded). One read of the names that tag an image.
|
||||
"""
|
||||
wanted = set(wanted)
|
||||
with self._annotation_read() as conn:
|
||||
return [
|
||||
tag_id
|
||||
for tag_id, name in conn.execute(_IMAGE_TAG_NAMES)
|
||||
if name in wanted or (query and query in name.casefold())
|
||||
]
|
||||
|
||||
def image_tag_locators(self, tag_ids) -> set[str]:
|
||||
"""Locators with any byte record (current or not) carrying one of
|
||||
``tag_ids``, through the tag index, in chunks of 500 ids."""
|
||||
ids = list(dict.fromkeys(tag_ids))
|
||||
found: set[str] = set()
|
||||
with self._annotation_read() as conn:
|
||||
for start in range(0, len(ids), _LOOKUP_CHUNK):
|
||||
chunk = ids[start : start + _LOOKUP_CHUNK]
|
||||
marks = ",".join("?" * len(chunk))
|
||||
found.update(
|
||||
row[0]
|
||||
for row in conn.execute(_TAG_LOCATORS.format(marks=marks), chunk)
|
||||
)
|
||||
return found
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Auto-tagged marks (#379): which image bytes a tagging model has handled.
|
||||
|
||||
One table, ``media_image_autotags``: one row per byte record of the image
|
||||
annotation group (``media_annotation_bytes``) means "a tagging model looked at
|
||||
these exact bytes and its result was stored", or that the bytes were found to
|
||||
be untaggable. "Skip already tagged" skips an image whose current byte record
|
||||
is marked; hand tags never mark. Rows are written only by the image tag store
|
||||
(``database/annotations.py``), in the transaction of the tag write they
|
||||
record.
|
||||
|
||||
The group installs like ``image_favorites``: optional in old backups,
|
||||
required once a database is current, and never backfilled. Like the Needs
|
||||
review group it is valid only beside a valid image annotation group, because
|
||||
every mark cites one byte record (``ON DELETE CASCADE``). Validation reads
|
||||
only: the catalog compared after whitespace compaction, the columns and the
|
||||
foreign key through the PRAGMAs, then one bounded value pass and the foreign
|
||||
key check.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from .annotation_schema import BYTES_TABLE
|
||||
from .annotation_schema import TABLES as ANNOTATION_TABLES
|
||||
from .annotation_schema import _catalog as _annotation_catalog
|
||||
from .annotation_schema import _uuid, annotation_schema_error
|
||||
|
||||
AUTOTAG_TABLE = "media_image_autotags"
|
||||
_DDL = (
|
||||
f"CREATE TABLE {AUTOTAG_TABLE} (byte_id TEXT PRIMARY KEY "
|
||||
f"REFERENCES {BYTES_TABLE}(id) ON DELETE CASCADE) WITHOUT ROWID"
|
||||
)
|
||||
# (cid, name, declared type, not null, default, primary key position): the
|
||||
# key of a WITHOUT ROWID table is implicitly NOT NULL.
|
||||
_COLUMNS = [(0, "byte_id", "TEXT", 1, None, 1)]
|
||||
# (column, parent table, parent column, on delete)
|
||||
_FOREIGN_KEYS = {("byte_id", BYTES_TABLE, "id", "CASCADE")}
|
||||
|
||||
NEEDS_ANNOTATIONS = "Image auto-tag marks need the image annotation tables"
|
||||
INVALID_SCHEMA = "Image auto-tag marks have an invalid schema"
|
||||
INVALID_VALUES = "Image auto-tag marks contain invalid values"
|
||||
MISSING = "Missing image auto-tag marks schema"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def _catalog(conn: sqlite3.Connection) -> list:
|
||||
return conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? OR tbl_name=?",
|
||||
(AUTOTAG_TABLE, AUTOTAG_TABLE),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def autotag_without_annotations_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
"""Catalog only: the marks table present without its annotation tables.
|
||||
|
||||
Startup asks this before the annotation group installs, which would make
|
||||
a marks table copied without it look valid. It reads no rows.
|
||||
"""
|
||||
if not _catalog(conn):
|
||||
return None
|
||||
present = {
|
||||
name for kind, name, _sql in _annotation_catalog(conn) if kind == "table"
|
||||
}
|
||||
if present != set(ANNOTATION_TABLES):
|
||||
return NEEDS_ANNOTATIONS
|
||||
return None
|
||||
|
||||
|
||||
def autotag_schema_error(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
allow_missing: bool = True,
|
||||
annotations_valid: bool = False,
|
||||
) -> Optional[str]:
|
||||
"""Reject an unsupported marks table or invalid marks; reads only.
|
||||
|
||||
A database without the table is acceptable for backups that predate it
|
||||
unless ``allow_missing`` is false. A present table requires a present,
|
||||
valid image annotation group; ``annotations_valid`` says the caller has
|
||||
just validated it (startup's annotation install), so only its presence is
|
||||
checked again.
|
||||
"""
|
||||
objects = _catalog(conn)
|
||||
if not objects:
|
||||
return None if allow_missing else MISSING
|
||||
if annotations_valid:
|
||||
if autotag_without_annotations_error(conn):
|
||||
return NEEDS_ANNOTATIONS
|
||||
elif annotation_schema_error(conn, allow_missing=False):
|
||||
return NEEDS_ANNOTATIONS
|
||||
actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
|
||||
if actual != {("table", AUTOTAG_TABLE, _compact(_DDL))}:
|
||||
return INVALID_SCHEMA
|
||||
columns = [
|
||||
(row[0], row[1], row[2].upper(), row[3], row[4], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({AUTOTAG_TABLE})")
|
||||
]
|
||||
if columns != _COLUMNS:
|
||||
return INVALID_SCHEMA
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({AUTOTAG_TABLE})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS:
|
||||
return INVALID_SCHEMA
|
||||
# Only this module's constants are interpolated into the SQL below.
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {AUTOTAG_TABLE} " # nosec B608
|
||||
f"WHERE NOT ({_uuid('byte_id')}) LIMIT 1"
|
||||
).fetchone():
|
||||
return INVALID_VALUES
|
||||
if conn.execute(f"PRAGMA foreign_key_check({AUTOTAG_TABLE})").fetchone():
|
||||
return INVALID_VALUES
|
||||
return None
|
||||
|
||||
|
||||
def migrate_autotag_state(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table atomically; validate before and after creating it.
|
||||
|
||||
Runs after ``migrate_annotation_state`` (which has just validated the
|
||||
annotation group). Creates one empty table: no row is read or copied.
|
||||
"""
|
||||
conn.execute("SAVEPOINT media_autotag_schema")
|
||||
try:
|
||||
error = autotag_schema_error(conn, annotations_valid=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
error = autotag_schema_error(conn, allow_missing=False, annotations_valid=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE media_autotag_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO media_autotag_schema")
|
||||
conn.execute("RELEASE media_autotag_schema")
|
||||
raise
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Atomic replacement of one imported category (#221, #249 System S10).
|
||||
|
||||
The LoRA reference import used to delete its whole category and then save
|
||||
each record in its own transaction, so a failure part way left the previous
|
||||
collection deleted and the new one partial. ``replace_category_prompts``
|
||||
does both in one ``BEGIN IMMEDIATE`` transaction: the category's current
|
||||
prompts (with their tags and image links) are replaced by the staged
|
||||
records, or, on any error, nothing changes.
|
||||
|
||||
A staged record is ``{text, hash, tags, images}``. A record whose hash a
|
||||
prompt outside the category already has links its images to that prompt
|
||||
(the legacy import did the same); a second record with the hash of an
|
||||
earlier one links its images to that one. Neither is a new prompt.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
try:
|
||||
from .maintenance import PlanChanged
|
||||
from .pairs import PairDependencyError
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
except ImportError: # pragma: no cover - flat test imports
|
||||
from database.maintenance import PlanChanged
|
||||
from database.pairs import PairDependencyError
|
||||
from database.workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
|
||||
class EmptyReplacement(ValueError):
|
||||
"""The replacement has no record and would remove every current import."""
|
||||
|
||||
|
||||
# The category's prompt ids, as a subquery (one bound parameter).
|
||||
_IN_CATEGORY = "SELECT id FROM prompts WHERE category = ?"
|
||||
|
||||
# Each record's outcome and the result counter it adds to.
|
||||
_COUNTERS = {"imported": "imported", "linked": "linked", "duplicate": "duplicates"}
|
||||
|
||||
|
||||
def _placeholders(values: Sequence[Any]) -> str:
|
||||
return ",".join("?" * len(values))
|
||||
|
||||
|
||||
class CategoryImportOperations:
|
||||
"""Read the state of an imported category and replace it atomically."""
|
||||
|
||||
def category_import_state(self, category: str) -> Dict[str, Any]:
|
||||
"""``{ids, paired}``: the category's prompt ids (ascending) and how
|
||||
many of them a saved pair uses (those block a replacement)."""
|
||||
with self.model.get_connection() as conn:
|
||||
ids = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM prompts WHERE category = ? ORDER BY id",
|
||||
(category,),
|
||||
).fetchall()
|
||||
]
|
||||
paired = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompts WHERE category = ? AND id IN "
|
||||
"(SELECT positive_prompt_id FROM prompt_pairs UNION "
|
||||
"SELECT negative_prompt_id FROM prompt_pairs)",
|
||||
(category,),
|
||||
).fetchone()[0]
|
||||
return {"ids": ids, "paired": paired}
|
||||
|
||||
def existing_prompt_hashes(
|
||||
self, hashes: Sequence[str], exclude_category: str
|
||||
) -> set:
|
||||
"""The given hashes a prompt outside ``exclude_category`` already has."""
|
||||
found = set()
|
||||
wanted = list(dict.fromkeys(hashes))
|
||||
with self.model.get_connection() as conn:
|
||||
for start in range(0, len(wanted), 500):
|
||||
chunk = wanted[start : start + 500]
|
||||
rows = conn.execute(
|
||||
f"SELECT hash FROM prompts WHERE hash IN ({_placeholders(chunk)}) "
|
||||
"AND (category IS NULL OR category != ?)",
|
||||
[*chunk, exclude_category],
|
||||
).fetchall()
|
||||
found.update(row[0] for row in rows)
|
||||
return found
|
||||
|
||||
def replace_category_prompts(
|
||||
self,
|
||||
category: str,
|
||||
records: List[Dict[str, Any]],
|
||||
*,
|
||||
expected_ids: Optional[List[int]] = None,
|
||||
allow_empty: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Replace every prompt of ``category`` with ``records`` in one transaction.
|
||||
|
||||
Raises (and changes nothing): ``PlanChanged`` when ``expected_ids`` is
|
||||
given and the category's prompt ids differ from it;
|
||||
``PairDependencyError`` when a saved pair uses one of its prompts;
|
||||
``EmptyReplacement`` when ``records`` is empty while the category has
|
||||
prompts, unless ``allow_empty`` (an empty replacement that was
|
||||
reviewed). Any other error rolls everything back too.
|
||||
|
||||
Returns ``{removed, imported, linked, duplicates, images, rows}``:
|
||||
prompts removed and created, records linked to an existing prompt or
|
||||
to an earlier record with the same text, image links added, and per
|
||||
record ``{name, outcome}`` (``imported``/``linked``/``duplicate``).
|
||||
"""
|
||||
with self._pair_transaction() as conn:
|
||||
ids = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM prompts WHERE category = ? ORDER BY id",
|
||||
(category,),
|
||||
).fetchall()
|
||||
]
|
||||
if expected_ids is not None and ids != list(expected_ids):
|
||||
raise PlanChanged("The imported prompts changed since the review")
|
||||
if not records and ids and not allow_empty:
|
||||
raise EmptyReplacement(
|
||||
"Nothing to import; the previous imports were kept"
|
||||
)
|
||||
self._ensure_category_unpaired(conn, category)
|
||||
removed = self._delete_category_rows(conn, category, ids)
|
||||
result = {
|
||||
"removed": removed,
|
||||
"imported": 0,
|
||||
"linked": 0,
|
||||
"duplicates": 0,
|
||||
"images": 0,
|
||||
"rows": [],
|
||||
}
|
||||
created = {}
|
||||
for record in records:
|
||||
self._import_record(conn, category, record, created, result)
|
||||
return result
|
||||
|
||||
def _delete_category_rows(self, conn, category: str, ids: List[int]) -> int:
|
||||
"""Delete the category's prompts with their image links and tags.
|
||||
|
||||
No commit. The rows are selected by category (just verified against
|
||||
``ids`` under the write lock), so no statement binds one parameter per
|
||||
prompt and SQLite's variable limit never caps a collection.
|
||||
"""
|
||||
if not ids:
|
||||
return 0
|
||||
documents = documents_of_rows(conn, "prompt_id", ids) # chunked
|
||||
conn.execute(
|
||||
f"DELETE FROM generated_images WHERE prompt_id IN ({_IN_CATEGORY})",
|
||||
(category,),
|
||||
)
|
||||
release_documents(conn, documents)
|
||||
conn.execute(
|
||||
f"DELETE FROM prompt_tags WHERE prompt_id IN ({_IN_CATEGORY})",
|
||||
(category,),
|
||||
)
|
||||
return conn.execute(
|
||||
"DELETE FROM prompts WHERE category = ?", (category,)
|
||||
).rowcount
|
||||
|
||||
@staticmethod
|
||||
def _ensure_category_unpaired(conn, category: str) -> None:
|
||||
"""Raise PairDependencyError when a saved pair uses a category prompt."""
|
||||
row = conn.execute(
|
||||
f"SELECT id FROM prompt_pairs WHERE positive_prompt_id IN ({_IN_CATEGORY}) "
|
||||
f"OR negative_prompt_id IN ({_IN_CATEGORY}) LIMIT 1",
|
||||
(category, category),
|
||||
).fetchone()
|
||||
if row:
|
||||
raise PairDependencyError(
|
||||
f"Prompt belongs to saved pair {row[0]}; delete the pair first "
|
||||
"or save changed text as a new prompt"
|
||||
)
|
||||
|
||||
def _import_record(self, conn, category, record, created, result) -> None:
|
||||
"""Save one staged record on ``conn``, or link it to the prompt it repeats."""
|
||||
prompt_hash = record["hash"]
|
||||
name = record.get("name") or record["text"]
|
||||
if prompt_hash in created:
|
||||
prompt_id, outcome = created[prompt_hash], "duplicate"
|
||||
else:
|
||||
row = conn.execute(
|
||||
"SELECT id FROM prompts WHERE hash = ?", (prompt_hash,)
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
prompt_id, outcome = row[0], "linked"
|
||||
else:
|
||||
prompt_id = self._insert_prompt(
|
||||
conn,
|
||||
record["text"],
|
||||
category,
|
||||
list(record.get("tags") or []),
|
||||
None,
|
||||
None,
|
||||
prompt_hash,
|
||||
)
|
||||
outcome = "imported"
|
||||
created[prompt_hash] = prompt_id
|
||||
result[_COUNTERS[outcome]] += 1
|
||||
# Linked last to first: every workspace list shows a prompt's media
|
||||
# newest link first, so the record's first image (the LoRA cover,
|
||||
# #355) is the one a Library card shows.
|
||||
for image in reversed(record.get("images") or []):
|
||||
if self._insert_image_link(conn, prompt_id, image):
|
||||
result["images"] += 1
|
||||
result["rows"].append({"name": name, "outcome": outcome})
|
||||
@@ -9,8 +9,12 @@ IMMEDIATE`` transaction, so no row ever names a deleted file:
|
||||
already links the kept copy: then its saved-pair links join that record
|
||||
and it is deleted;
|
||||
- its favorite;
|
||||
- its collection memberships (#331), joined onto the kept copy's as their
|
||||
union under the same keys a favorite moves by;
|
||||
- its image tags (``media_image_tags`` of its byte record), joined onto the
|
||||
kept copy's byte record as their union, in order;
|
||||
- its own image category (#332), copied to the kept copy when that copy has
|
||||
none; the removed copy's row is kept, so a give-back loses nothing;
|
||||
- its pending Needs review items, settled: only an item still ``pending`` at
|
||||
revision 0 with no decision row is removed (``review._SETTLE_ITEM``), so a
|
||||
decided item is never touched.
|
||||
@@ -22,9 +26,12 @@ import os
|
||||
|
||||
from .annotation_schema import BYTES_TABLE, FILES_TABLE
|
||||
from .annotations import _BUMP_INDEX
|
||||
from .image_categories import COPY_SQL as COPY_CATEGORY_SQL
|
||||
from .image_categories import copy_image_category
|
||||
from .models import normalize_image_path, utc_now_iso
|
||||
from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM
|
||||
from .review_changes import notify_review_change
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
LINKS_BY_FILE_SQL = "SELECT id, prompt_id FROM generated_images WHERE file_path = ?"
|
||||
LEGACY_LINKS_SQL = (
|
||||
@@ -49,6 +56,15 @@ INSERT_FAVORITE_SQL = "INSERT OR IGNORE INTO image_favorites VALUES (?, ?, ?)"
|
||||
DELETE_FAVORITE_SQL = (
|
||||
"DELETE FROM image_favorites WHERE root_key = ? AND relative_path = ?"
|
||||
)
|
||||
COPY_MEMBERSHIPS_SQL = (
|
||||
"INSERT OR IGNORE INTO image_collection_items "
|
||||
"(collection_id, root_key, relative_path, added_at) "
|
||||
"SELECT collection_id, ?, ?, added_at FROM image_collection_items "
|
||||
"WHERE root_key = ? AND relative_path = ?"
|
||||
)
|
||||
DELETE_MEMBERSHIPS_SQL = (
|
||||
"DELETE FROM image_collection_items WHERE root_key = ? AND relative_path = ?"
|
||||
)
|
||||
COPY_BYTE_SQL = (
|
||||
f"SELECT b.id FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b " # nosec B608
|
||||
"ON b.file_id=f.id WHERE f.locator=? AND b.sha256=?"
|
||||
@@ -72,9 +88,12 @@ FILE_LINK_SQL = (
|
||||
DELETE_LINK_SQL,
|
||||
FAVORITE_SQL,
|
||||
DELETE_FAVORITE_SQL,
|
||||
COPY_MEMBERSHIPS_SQL,
|
||||
DELETE_MEMBERSHIPS_SQL,
|
||||
COPY_BYTE_SQL,
|
||||
COPY_TAG_COUNT_SQL,
|
||||
PENDING_COUNT_SQL,
|
||||
COPY_CATEGORY_SQL,
|
||||
)
|
||||
|
||||
|
||||
@@ -134,12 +153,17 @@ class DuplicateCopyOperations:
|
||||
by), ``targets`` (``{removed spelling: kept spelling}``, the spelling
|
||||
a moved record takes; else ``kept_path``), ``favorites`` (``(root key,
|
||||
removed key, kept root key, kept key)`` per listed root holding the
|
||||
file), ``removed_locator``,
|
||||
``kept_locator``, ``sha256`` and the kept copy's ``kept_size`` and
|
||||
``kept_mtime_ns``. Returns ``{moved, merged, favorites, tags,
|
||||
review}``.
|
||||
file; they move its favorite and its collection memberships),
|
||||
``removed_locator``, ``kept_locator``, ``sha256`` and the kept copy's
|
||||
``kept_size`` and ``kept_mtime_ns``; ``categories``: ``(removed file
|
||||
key, kept file key)`` pairs of the image categories (#332). Returns
|
||||
``{moved, merged, favorites, tags, review, collections, categories}``
|
||||
(``collections``: the memberships inserted on the kept copy;
|
||||
``categories``: the image categories copied to the kept copy).
|
||||
"""
|
||||
counts = dict.fromkeys(("moved", "merged", "favorites", "tags", "review"), 0)
|
||||
counts = dict.fromkeys(
|
||||
("moved", "merged", "favorites", "tags", "review", "collections"), 0
|
||||
)
|
||||
kept_keys = [normalize_image_path(path) for path in move["kept"]]
|
||||
with self._pair_transaction() as conn:
|
||||
for spelled in dict.fromkeys(move["removed"]):
|
||||
@@ -155,7 +179,9 @@ class DuplicateCopyOperations:
|
||||
break
|
||||
if existing is not None:
|
||||
conn.execute(MOVE_PAIR_LINKS_SQL, (existing[0], image_id))
|
||||
documents = documents_of_rows(conn, "id", [image_id])
|
||||
conn.execute(DELETE_LINK_SQL, (image_id,))
|
||||
release_documents(conn, documents)
|
||||
counts["merged"] += 1
|
||||
continue
|
||||
conn.execute(
|
||||
@@ -170,7 +196,12 @@ class DuplicateCopyOperations:
|
||||
counts["moved"] += 1
|
||||
for favorite in move.get("favorites", ()):
|
||||
counts["favorites"] += _move_favorite(conn, *favorite)
|
||||
counts["collections"] += _move_memberships(conn, *favorite)
|
||||
counts["tags"] = self._move_copy_tags(conn, move)
|
||||
counts["categories"] = sum(
|
||||
copy_image_category(conn, removed, kept)
|
||||
for removed, kept in move.get("categories", ())
|
||||
)
|
||||
for (item_id,) in conn.execute(
|
||||
_PENDING_OF_LOCATOR, (move["removed_locator"],)
|
||||
).fetchall():
|
||||
@@ -210,3 +241,12 @@ def _move_favorite(conn, root_key, removed_key, kept_root, kept_key):
|
||||
conn.execute(INSERT_FAVORITE_SQL, (kept_root, kept_key, row[0]))
|
||||
conn.execute(DELETE_FAVORITE_SQL, (root_key, removed_key))
|
||||
return 1
|
||||
|
||||
|
||||
def _move_memberships(conn, root_key, removed_key, kept_root, kept_key):
|
||||
"""Copy a removed copy's memberships to the kept copy; the rows inserted."""
|
||||
inserted = conn.execute(
|
||||
COPY_MEMBERSHIPS_SQL, (kept_root, kept_key, root_key, removed_key)
|
||||
).rowcount
|
||||
conn.execute(DELETE_MEMBERSHIPS_SQL, (root_key, removed_key))
|
||||
return inserted
|
||||
|
||||
@@ -7,6 +7,11 @@ from typing import Callable, ContextManager, Iterator
|
||||
|
||||
try:
|
||||
from ..utils.execution_history_events import RunFinish, encode_command
|
||||
from ..utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
stored_prompt_text,
|
||||
stored_prompt_text_sql,
|
||||
)
|
||||
from ..utils.execution_contexts import (
|
||||
CAPTURE_REASONS,
|
||||
ContextBundle,
|
||||
@@ -15,6 +20,11 @@ try:
|
||||
)
|
||||
except ImportError:
|
||||
from utils.execution_history_events import RunFinish, encode_command
|
||||
from utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
stored_prompt_text,
|
||||
stored_prompt_text_sql,
|
||||
)
|
||||
from utils.execution_contexts import (
|
||||
CAPTURE_REASONS,
|
||||
ContextBundle,
|
||||
@@ -134,7 +144,8 @@ def _insert_exact(
|
||||
|
||||
def _role_hints(role: str) -> str:
|
||||
return (
|
||||
f"SELECT r.{role}_prompt_id, tx.text_value "
|
||||
f"SELECT r.{role}_prompt_id, tx.text_value, "
|
||||
f"{stored_prompt_text_sql('tx.text_value')} "
|
||||
"FROM execution_run_contexts r "
|
||||
"JOIN execution_contexts c ON c.context_id = r.context_id "
|
||||
f"JOIN execution_prompt_snapshots s ON s.snapshot_id = c.{role}_snapshot_id "
|
||||
@@ -146,57 +157,78 @@ def _role_hints(role: str) -> str:
|
||||
|
||||
# Set form of the read-time prompt-hint rule in get_execution_contexts():
|
||||
# a stored role hint is live only in the current storage epoch, and only when
|
||||
# its exact raw snapshot text matches exactly one prompt, which is the hinted
|
||||
# one (_prompt_hint). Evaluated once per statement, never per prompt row.
|
||||
# the hinted prompt is the one match of _prompt_hint: by text (the raw
|
||||
# snapshot text or its stored form, outer whitespace stripped, #438), or, only
|
||||
# when no prompt has either text, by dedup hash (case folded too, #460).
|
||||
# The hash is a fallback, so a hint #438 kept is never made ambiguous.
|
||||
# Needs register_prompt_functions (pm_prompt_hash). The hint rows are the
|
||||
# outer loop (CROSS JOIN), so each prompts lookup is by index.
|
||||
# Defines the CTE ``live_hint_prompts(prompt_id)``; prefix it with WITH.
|
||||
LIVE_HINT_CTES = (
|
||||
"stored_hints(prompt_id, raw) AS ("
|
||||
"stored_hints(prompt_id, raw, stored) AS ("
|
||||
+ _role_hints("positive")
|
||||
+ " UNION "
|
||||
+ _role_hints("negative")
|
||||
+ "), unique_hint_texts(prompt_id, raw) AS ("
|
||||
"SELECT MIN(id), text COLLATE BINARY FROM prompts "
|
||||
"WHERE text COLLATE BINARY IN (SELECT raw FROM stored_hints) "
|
||||
"GROUP BY text COLLATE BINARY HAVING COUNT(*) = 1"
|
||||
+ "), hint_matches(prompt_id, raw, stored, match_id) AS ("
|
||||
"SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h "
|
||||
"CROSS JOIN prompts p ON p.text COLLATE BINARY IN (h.raw, h.stored) "
|
||||
"UNION ALL "
|
||||
"SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h "
|
||||
"CROSS JOIN prompts p ON p.hash = pm_prompt_hash(h.raw) "
|
||||
"WHERE NOT EXISTS (SELECT 1 FROM prompts q "
|
||||
"WHERE q.text COLLATE BINARY IN (h.raw, h.stored))"
|
||||
"), live_hint_prompts(prompt_id) AS ("
|
||||
"SELECT h.prompt_id FROM stored_hints h JOIN unique_hint_texts u "
|
||||
"ON u.prompt_id = h.prompt_id AND u.raw = h.raw COLLATE BINARY)"
|
||||
"SELECT prompt_id FROM hint_matches GROUP BY prompt_id, raw, stored "
|
||||
"HAVING COUNT(*) = 1 AND MIN(match_id) = prompt_id)"
|
||||
)
|
||||
LIVE_HINT_PROMPT_IDS_SQL = (
|
||||
"WITH " + LIVE_HINT_CTES + " SELECT DISTINCT prompt_id FROM live_hint_prompts"
|
||||
)
|
||||
|
||||
_BY_TEXT = "SELECT id FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 2"
|
||||
|
||||
|
||||
def _text_matches(conn: sqlite3.Connection, raw: str) -> list:
|
||||
return [row[0] for row in conn.execute(_BY_TEXT, (raw, stored_prompt_text(raw)))]
|
||||
|
||||
|
||||
def _prompt_hint(conn: sqlite3.Connection, raw: str | None) -> int | None:
|
||||
"""The one prompt that stores ``raw`` (#438), else the one it counts toward.
|
||||
|
||||
By text first: ``raw`` or ``raw.strip()`` (the exact form keeps hints of
|
||||
older rows that kept outer whitespace). Only when no prompt has either
|
||||
text, by the dedup hash, which also folds case (#460).
|
||||
"""
|
||||
if raw is None:
|
||||
return None
|
||||
rows = conn.execute(
|
||||
"SELECT id FROM prompts WHERE text=? COLLATE BINARY LIMIT 2", (raw,)
|
||||
).fetchall()
|
||||
return rows[0][0] if len(rows) == 1 else None
|
||||
matches = _text_matches(conn, raw)
|
||||
if not matches:
|
||||
matches = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM prompts WHERE hash=? LIMIT 2",
|
||||
(generate_prompt_hash(raw),),
|
||||
)
|
||||
]
|
||||
return matches[0] if len(matches) == 1 else None
|
||||
|
||||
|
||||
def _hints(
|
||||
conn: sqlite3.Connection, positive: str | None, negative: str | None
|
||||
) -> tuple:
|
||||
pos_id, neg_id = _prompt_hint(conn, positive), _prompt_hint(conn, negative)
|
||||
stored = None if negative is None else stored_prompt_text(negative)
|
||||
pair_id = None
|
||||
# A known negative that names no prompt because several share its text
|
||||
negative_ambiguous = (
|
||||
negative is not None
|
||||
and neg_id is None
|
||||
and conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE text=? COLLATE BINARY LIMIT 1", (negative,)
|
||||
).fetchone()
|
||||
is not None
|
||||
stored is not None and neg_id is None and bool(_text_matches(conn, negative))
|
||||
)
|
||||
if pos_id is not None and negative is not None and not negative_ambiguous:
|
||||
if pos_id is not None and stored is not None and not negative_ambiguous:
|
||||
rows = conn.execute(
|
||||
"SELECT pp.id FROM prompt_pairs pp LEFT JOIN prompts n ON "
|
||||
"n.id=pp.negative_prompt_id "
|
||||
"SELECT pp.id FROM prompt_pairs pp "
|
||||
"WHERE pp.positive_prompt_id=? AND ((pp.negative_prompt_id IS NULL AND "
|
||||
"?='') "
|
||||
"OR (pp.negative_prompt_id=? AND n.text=? COLLATE BINARY)) LIMIT 2",
|
||||
(pos_id, negative, neg_id, negative),
|
||||
"?='') OR pp.negative_prompt_id=?) LIMIT 2",
|
||||
(pos_id, stored, neg_id),
|
||||
).fetchall()
|
||||
if len(rows) == 1:
|
||||
pair_id = rows[0][0]
|
||||
|
||||
@@ -196,3 +196,14 @@ class ExecutionHistoryOperations:
|
||||
.fetchone()
|
||||
)
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
def rekey_history_restore(old_path: str, new_path: str) -> None:
|
||||
"""Move the observers of ``old_path`` to ``new_path`` (Move library)."""
|
||||
old, new = _database_identity(old_path), _database_identity(new_path)
|
||||
if old == new:
|
||||
return
|
||||
with _restore_lock:
|
||||
observers = _restore_observers.pop(old, None)
|
||||
if observers:
|
||||
_restore_observers.setdefault(new, {}).update(observers)
|
||||
|
||||
@@ -4,6 +4,8 @@ import sqlite3
|
||||
from contextlib import ExitStack
|
||||
from typing import Callable, ContextManager
|
||||
|
||||
from .run_attribution import notify_outputs_sealed
|
||||
|
||||
try:
|
||||
from ..utils.execution_history_events import RunFinish, encode_command
|
||||
from ..utils.execution_outputs import OutputBundle, validate_output_bundle
|
||||
@@ -113,6 +115,10 @@ class ExecutionOutputOperations:
|
||||
"VALUES(?,?,?,?,?,?,?,?)",
|
||||
[(run_id, *reference) for reference in prepared],
|
||||
)
|
||||
# Committed: images waiting for this run may now be attributed (#342)
|
||||
notify_outputs_sealed(
|
||||
run_id, [dict(zip(_REFERENCE_FIELDS, row)) for row in prepared]
|
||||
)
|
||||
return True
|
||||
|
||||
def get_execution_outputs(self, run_id: str) -> dict:
|
||||
|
||||
@@ -23,10 +23,10 @@ from pathlib import PurePosixPath, PureWindowsPath
|
||||
from urllib.parse import quote
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
from .models import normalize_image_path
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.hashing import register_prompt_functions
|
||||
from database.models import normalize_image_path
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.health")
|
||||
@@ -125,12 +125,8 @@ def _foreign_keys(conn, _roots):
|
||||
)
|
||||
|
||||
|
||||
def _prompt_hash(text):
|
||||
return generate_prompt_hash(text) if isinstance(text, str) else None
|
||||
|
||||
|
||||
def _canonical_hashes(conn, _roots):
|
||||
conn.create_function("pm_prompt_hash", 1, _prompt_hash, deterministic=True)
|
||||
register_prompt_functions(conn) # pm_prompt_hash, shared with history (#460)
|
||||
return _counted(
|
||||
f"Prompt {row[0]}: the stored hash does not match its text"
|
||||
for row in conn.execute(
|
||||
|
||||
@@ -15,6 +15,11 @@ from .execution_history import HistoryEpochChanged
|
||||
from .execution_outputs import _capture, _REFERENCE_COLUMNS
|
||||
from .library import _ELIGIBLE
|
||||
|
||||
try:
|
||||
from ..utils.hashing import prompt_key, stored_prompt_text
|
||||
except ImportError:
|
||||
from utils.hashing import prompt_key, stored_prompt_text
|
||||
|
||||
RunKey = tuple[int, str, int, str]
|
||||
|
||||
|
||||
@@ -100,14 +105,43 @@ _ORDER = (
|
||||
)
|
||||
# Stale-hint rows a newest-pair lookup inspects before it reports none.
|
||||
_PAIR_RUN_CANDIDATES = 16
|
||||
# The recorded texts of one scope, as a JSON array parameter (_scope_text_ids).
|
||||
_IN_SCOPE = "IN (SELECT value FROM json_each(?))"
|
||||
_MATCHING = (
|
||||
"SELECT DISTINCT rc.run_id FROM execution_prompt_snapshots s "
|
||||
"JOIN execution_contexts c ON c.positive_snapshot_id=s.snapshot_id "
|
||||
"JOIN execution_run_contexts rc ON rc.context_id=c.context_id "
|
||||
"WHERE s.raw_text_id=?"
|
||||
f"WHERE s.raw_text_id {_IN_SCOPE}"
|
||||
)
|
||||
|
||||
|
||||
def _scope_text_ids(conn: sqlite3.Connection, key: str) -> str:
|
||||
"""Every recorded raw text whose prompt key is ``key``, as a JSON array.
|
||||
|
||||
A run keeps its exact raw text, while the Library saves and counts it as
|
||||
the prompt with its key (outer whitespace #438, case #460). The values are
|
||||
compared, never only their digests; ``pm_prompt_key`` is Python's key.
|
||||
"""
|
||||
return json.dumps(
|
||||
[
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
# Only snapshot raw texts: effective and affix texts are skipped
|
||||
"SELECT t.text_id FROM (SELECT DISTINCT raw_text_id AS text_id "
|
||||
"FROM execution_prompt_snapshots) s CROSS JOIN "
|
||||
"execution_context_texts t WHERE t.text_id=s.text_id AND "
|
||||
"pm_prompt_key(t.text_value)=? COLLATE BINARY",
|
||||
(key,),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _scope_text_id(raw: str | None) -> str | None:
|
||||
"""The scope digest of a raw text: that of its prompt key."""
|
||||
return None if raw is None else content_digest("text", (prompt_key(raw),))
|
||||
|
||||
|
||||
def _limit(limit: int) -> None:
|
||||
if type(limit) is not int or not 1 <= limit <= 100:
|
||||
raise ValueError("History limit must be an integer from 1 to 100")
|
||||
@@ -255,8 +289,15 @@ class HistoryReadOperations:
|
||||
if saved is None:
|
||||
return None
|
||||
text = saved["text"]
|
||||
raw_id = None if text is None else content_digest("text", (text,))
|
||||
if after is not None and after.raw_text_id != raw_id:
|
||||
raw_id = _scope_text_id(text)
|
||||
# A position from before #460 names the scope by the stripped text
|
||||
# alone; it is the same scope, so a page open across the upgrade resumes.
|
||||
legacy_id = (
|
||||
None
|
||||
if text is None
|
||||
else content_digest("text", (stored_prompt_text(text),))
|
||||
)
|
||||
if after is not None and after.raw_text_id not in (raw_id, legacy_id):
|
||||
raise ValueError("History position text scope changed")
|
||||
group = {
|
||||
"positive_id": positive_id,
|
||||
@@ -273,18 +314,11 @@ class HistoryReadOperations:
|
||||
next_position=None,
|
||||
resume_position=None,
|
||||
)
|
||||
if (
|
||||
raw_id is None
|
||||
or conn.execute(
|
||||
"SELECT 1 FROM execution_context_texts WHERE text_id=? "
|
||||
"AND text_value=? COLLATE BINARY",
|
||||
(raw_id, text),
|
||||
).fetchone()
|
||||
is None
|
||||
):
|
||||
if raw_id is None:
|
||||
return result
|
||||
scope = _scope_text_ids(conn, prompt_key(text))
|
||||
where = f"r.run_id IN ({_MATCHING})"
|
||||
params = [raw_id]
|
||||
params = [scope]
|
||||
if after is not None:
|
||||
where += f" AND {_KEY}<=(?,?,?,?)"
|
||||
params.extend(after.upper)
|
||||
@@ -338,9 +372,9 @@ class HistoryReadOperations:
|
||||
"LEFT JOIN execution_prompt_snapshots n "
|
||||
"ON n.snapshot_id=c.negative_snapshot_id "
|
||||
"LEFT JOIN execution_context_texts t ON t.text_id=n.raw_text_id "
|
||||
f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id=? "
|
||||
f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id {_IN_SCOPE} "
|
||||
"ORDER BY rc.anchor_id COLLATE BINARY,rc.run_id COLLATE BINARY",
|
||||
(*by_run, raw_id),
|
||||
(*by_run, scope),
|
||||
)
|
||||
summaries = {}
|
||||
evidence_by_context = {}
|
||||
@@ -438,6 +472,7 @@ class HistoryReadOperations:
|
||||
if saved["text"] is None:
|
||||
return {"run": None}
|
||||
epoch = self.get_execution_epoch()
|
||||
key = prompt_key(saved["text"])
|
||||
candidates = conn.execute(
|
||||
"SELECT r.run_id,rc.anchor_id,rc.context_id,"
|
||||
"c.positive_snapshot_id,c.negative_snapshot_id "
|
||||
@@ -445,10 +480,11 @@ class HistoryReadOperations:
|
||||
"ON rc.run_id=r.run_id JOIN execution_contexts c "
|
||||
"ON c.context_id=rc.context_id JOIN execution_prompt_snapshots s "
|
||||
"ON s.snapshot_id=c.positive_snapshot_id "
|
||||
"WHERE s.raw_text_id=? AND rc.pair_id=? AND rc.reference_epoch=? "
|
||||
f"WHERE s.raw_text_id {_IN_SCOPE} AND rc.pair_id=? "
|
||||
"AND rc.reference_epoch=? "
|
||||
f"ORDER BY {_ORDER},rc.anchor_id COLLATE BINARY LIMIT ?",
|
||||
(
|
||||
content_digest("text", (saved["text"],)),
|
||||
_scope_text_ids(conn, key),
|
||||
pair_id,
|
||||
epoch,
|
||||
_PAIR_RUN_CANDIDATES,
|
||||
@@ -457,7 +493,11 @@ class HistoryReadOperations:
|
||||
for row in candidates:
|
||||
positive = _values(conn, row["positive_snapshot_id"])
|
||||
negative = _values(conn, row["negative_snapshot_id"])
|
||||
if positive is None or positive["raw"] != saved["text"]:
|
||||
if (
|
||||
positive is None
|
||||
or positive["raw"] is None
|
||||
or prompt_key(positive["raw"]) != key
|
||||
):
|
||||
continue
|
||||
hints = _hints(conn, positive["raw"], negative["raw"] if negative else None)
|
||||
if hints[2] == pair_id:
|
||||
@@ -477,11 +517,9 @@ class HistoryReadOperations:
|
||||
snapshots = {
|
||||
row["anchor_id"]: row
|
||||
for row in conn.execute(
|
||||
"SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id,"
|
||||
"p.raw_text_id AS positive_raw_text_id "
|
||||
"SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id "
|
||||
"FROM execution_run_contexts rc JOIN execution_contexts c "
|
||||
"ON c.context_id=rc.context_id LEFT JOIN execution_prompt_snapshots p "
|
||||
"ON p.snapshot_id=c.positive_snapshot_id WHERE rc.run_id=?",
|
||||
"ON c.context_id=rc.context_id WHERE rc.run_id=?",
|
||||
(run_id,),
|
||||
)
|
||||
}
|
||||
@@ -491,12 +529,12 @@ class HistoryReadOperations:
|
||||
context.update(
|
||||
{
|
||||
key: row[key]
|
||||
for key in (
|
||||
"positive_snapshot_id",
|
||||
"negative_snapshot_id",
|
||||
"positive_raw_text_id",
|
||||
)
|
||||
}
|
||||
for key in ("positive_snapshot_id", "negative_snapshot_id")
|
||||
},
|
||||
# The history scope this context belongs to (#438)
|
||||
positive_raw_text_id=_scope_text_id(
|
||||
(context["positive"] or {}).get("raw")
|
||||
),
|
||||
)
|
||||
pair = context["pair_id"]
|
||||
context["navigation"] = {
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
"""Image categories (#332): an image file's own category.
|
||||
|
||||
An image shows its own category when it has one, otherwise the category its
|
||||
link supplies (its recipe's, else its positive prompt's); this table holds
|
||||
only the own values. A row is keyed by the file key: the deepest configured
|
||||
output source that contains the file (its real, case-normalized path) and the
|
||||
path below it, case-folded where the OS folds case
|
||||
(``py/api/images.py`` ``image_category_key``). Without nested sources the key
|
||||
equals a favorite's key. Nothing follows a moved file: its row stays and is
|
||||
hidden until a file appears at the same path again.
|
||||
|
||||
Writes validate strictly (the route strips the value and applies
|
||||
``validate_category``). Startup checks the structure only, so a value saved
|
||||
under older rules never stops the library opening; restore also checks the
|
||||
values. Optional in old backups, required once a database is current.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Iterable
|
||||
|
||||
TABLE = "image_categories"
|
||||
INDEX = "idx_image_categories_category"
|
||||
MAX_KEY_LENGTH = 4096
|
||||
MAX_CATEGORY_LENGTH = 100
|
||||
_DDL = (
|
||||
f"CREATE TABLE {TABLE} (root_key TEXT NOT NULL, relative_path TEXT NOT NULL, "
|
||||
"category TEXT NOT NULL, updated_at TEXT NOT NULL, "
|
||||
"PRIMARY KEY (root_key, relative_path))"
|
||||
)
|
||||
_INDEX_DDL = f"CREATE INDEX {INDEX} ON {TABLE}(category)"
|
||||
_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
|
||||
_COLUMNS = (
|
||||
("root_key", "TEXT", 1, 1),
|
||||
("relative_path", "TEXT", 1, 2),
|
||||
("category", "TEXT", 1, 0),
|
||||
("updated_at", "TEXT", 1, 0),
|
||||
)
|
||||
# A category may not hold a character below U+0020 (validate_category).
|
||||
_CONTROL_GLOB = "*[" + "".join(chr(code) for code in range(1, 32)) + "]*"
|
||||
|
||||
CATEGORY_OF_SQL = (
|
||||
f"SELECT category FROM {TABLE} WHERE root_key = ? AND relative_path = ?"
|
||||
)
|
||||
CATEGORIES_OF_ROOT_SQL = (
|
||||
f"SELECT relative_path, category FROM {TABLE} WHERE root_key = ?"
|
||||
)
|
||||
UPSERT_SQL = (
|
||||
f"INSERT INTO {TABLE} (root_key, relative_path, category, updated_at) "
|
||||
"VALUES (?, ?, ?, ?) ON CONFLICT(root_key, relative_path) "
|
||||
"DO UPDATE SET category = excluded.category, updated_at = excluded.updated_at"
|
||||
)
|
||||
DELETE_SQL = f"DELETE FROM {TABLE} WHERE root_key = ? AND relative_path = ?"
|
||||
# The duplicate cleanup's copy (#332 MC11): the removed copy's row is copied
|
||||
# to the kept copy only when the kept copy has none; it is never deleted.
|
||||
COPY_SQL = (
|
||||
f"INSERT OR IGNORE INTO {TABLE} (root_key, relative_path, category, updated_at) "
|
||||
f"SELECT ?, ?, category, updated_at FROM {TABLE} "
|
||||
"WHERE root_key = ? AND relative_path = ?"
|
||||
)
|
||||
STATEMENTS = (CATEGORY_OF_SQL, CATEGORIES_OF_ROOT_SQL, UPSERT_SQL, DELETE_SQL, COPY_SQL)
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def _structure_error(
|
||||
conn: sqlite3.Connection, allow_missing: bool, allow_missing_index: bool = False
|
||||
) -> str | None:
|
||||
objects = conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? "
|
||||
"OR (type IN ('index','trigger') AND tbl_name=?)",
|
||||
(TABLE, TABLE),
|
||||
).fetchall()
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing image categories schema"
|
||||
tables = [sql for kind, _name, sql in objects if kind == "table"]
|
||||
others = {(kind, name) for kind, name, _sql in objects if kind != "table"}
|
||||
expected = {("index", _AUTOINDEX), ("index", INDEX)}
|
||||
# Before its install, a table without the category index is repaired by
|
||||
# CREATE INDEX IF NOT EXISTS rather than refused.
|
||||
if allow_missing_index and others == {("index", _AUTOINDEX)}:
|
||||
expected = others
|
||||
if len(tables) != 1 or others != expected:
|
||||
return "Image categories name is reserved by an unsupported schema object"
|
||||
if _compact(tables[0]) != _compact(_DDL):
|
||||
return "Image categories has an invalid table definition"
|
||||
index_sql = next((sql for kind, name, sql in objects if name == INDEX), None)
|
||||
if ("index", INDEX) in expected and _compact(index_sql) != _compact(_INDEX_DDL):
|
||||
return "Image categories has an invalid index"
|
||||
autoindex_sql = next(sql for kind, name, sql in objects if name == _AUTOINDEX)
|
||||
if autoindex_sql is not None:
|
||||
return "Image categories has an invalid index"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({TABLE})")
|
||||
)
|
||||
if columns != _COLUMNS:
|
||||
return "Image categories has invalid columns"
|
||||
return None
|
||||
|
||||
|
||||
def _values_error(conn: sqlite3.Connection) -> str | None:
|
||||
# Bytes bound characters from above (UTF-8 is at most 4 bytes each).
|
||||
limit = MAX_KEY_LENGTH * 4
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(root_key) != 'text' " # nosec B608
|
||||
"OR typeof(relative_path) != 'text' OR typeof(category) != 'text' "
|
||||
"OR typeof(updated_at) != 'text' "
|
||||
f"OR length(CAST(root_key AS BLOB)) > {limit} "
|
||||
f"OR length(CAST(relative_path AS BLOB)) > {limit} "
|
||||
"OR length(CAST(updated_at AS BLOB)) > 64 "
|
||||
"OR instr(CAST(category AS BLOB), x'00') > 0 "
|
||||
f"OR length(category) < 1 OR length(category) > {MAX_CATEGORY_LENGTH} "
|
||||
"OR category != TRIM(category) OR category GLOB ? LIMIT 1",
|
||||
(_CONTROL_GLOB,),
|
||||
).fetchone():
|
||||
return "Image categories contains invalid values"
|
||||
for root, path in conn.execute(
|
||||
f"SELECT CAST(root_key AS BLOB), CAST(relative_path AS BLOB) " # nosec B608
|
||||
f"FROM {TABLE} WHERE length(CAST(root_key AS BLOB)) > {MAX_KEY_LENGTH} "
|
||||
f"OR length(CAST(relative_path AS BLOB)) > {MAX_KEY_LENGTH}"
|
||||
):
|
||||
for raw in (root, path):
|
||||
if len(bytes(raw).decode("utf-8", "replace")) > MAX_KEY_LENGTH:
|
||||
return "Image categories contains invalid values"
|
||||
return None
|
||||
|
||||
|
||||
def image_categories_schema_error(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
allow_missing: bool = True,
|
||||
check_values: bool = True,
|
||||
) -> str | None:
|
||||
"""Reject an unsupported image categories schema (and, optionally, values).
|
||||
|
||||
Reads only. A missing table is acceptable for backups that predate it
|
||||
unless ``allow_missing`` is false. ``check_values`` also refuses a stored
|
||||
value the write path could never have written.
|
||||
"""
|
||||
error = _structure_error(conn, allow_missing)
|
||||
if error is not None or not check_values:
|
||||
return error
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone()
|
||||
return _values_error(conn) if exists else None
|
||||
|
||||
|
||||
def migrate_image_categories(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table and its index atomically; structure is checked
|
||||
before and after. Reads only ``sqlite_master`` and this table's PRAGMAs
|
||||
and writes only its own two objects (no backfill)."""
|
||||
conn.execute("SAVEPOINT image_categories_schema")
|
||||
try:
|
||||
error = _structure_error(conn, True, allow_missing_index=True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
conn.execute(
|
||||
_INDEX_DDL.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS", 1)
|
||||
)
|
||||
error = image_categories_schema_error(
|
||||
conn, allow_missing=False, check_values=False
|
||||
)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE image_categories_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO image_categories_schema")
|
||||
conn.execute("RELEASE image_categories_schema")
|
||||
raise
|
||||
|
||||
|
||||
def copy_image_category(conn: sqlite3.Connection, removed_key, kept_key) -> int:
|
||||
"""Give the kept copy the removed copy's category when it has none.
|
||||
|
||||
Runs inside the caller's transaction; never deletes the removed copy's
|
||||
row, so a give-back or a failed delete loses nothing. Returns 1 when a
|
||||
row was inserted, else 0.
|
||||
"""
|
||||
if tuple(removed_key) == tuple(kept_key):
|
||||
return 0
|
||||
return max(conn.execute(COPY_SQL, (*kept_key, *removed_key)).rowcount, 0)
|
||||
|
||||
|
||||
class ImageCategoryOperations:
|
||||
"""PromptDatabase mixin: set, clear and read images' own categories.
|
||||
|
||||
Never reads or writes ``prompts``, ``prompt_pairs`` or
|
||||
``generated_images``.
|
||||
"""
|
||||
|
||||
def image_category(self, root_key: str, relative_key: str) -> str | None:
|
||||
"""One file's own category, or None."""
|
||||
row = (
|
||||
self.model.get_connection()
|
||||
.execute(CATEGORY_OF_SQL, (root_key, relative_key))
|
||||
.fetchone()
|
||||
)
|
||||
return row[0] if row else None
|
||||
|
||||
def own_categories_of(self, root_keys: Iterable[str]) -> dict:
|
||||
"""``{(root key, relative key): category}`` under the given sources.
|
||||
|
||||
One primary-key range read per source.
|
||||
"""
|
||||
found = {}
|
||||
conn = self.model.get_connection()
|
||||
for root_key in dict.fromkeys(root_keys):
|
||||
for relative, category in conn.execute(CATEGORIES_OF_ROOT_SQL, (root_key,)):
|
||||
found[(root_key, relative)] = category
|
||||
return found
|
||||
|
||||
def set_image_categories(self, file_keys, category: str) -> dict:
|
||||
"""Set *category* on every file key, in one transaction.
|
||||
|
||||
``file_keys``: ``(root key, relative key)`` pairs from any number of
|
||||
sources. Returns ``{updated, unchanged}``; a file that already had
|
||||
exactly this value is ``unchanged`` and keeps its ``updated_at``.
|
||||
"""
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
keys = list(dict.fromkeys(tuple(key) for key in file_keys))
|
||||
counts = {"updated": 0, "unchanged": 0}
|
||||
now = utc_now_iso()
|
||||
with self._pair_transaction() as conn:
|
||||
for key in keys:
|
||||
row = conn.execute(CATEGORY_OF_SQL, key).fetchone()
|
||||
if row is not None and row[0] == category:
|
||||
counts["unchanged"] += 1
|
||||
continue
|
||||
conn.execute(UPSERT_SQL, (*key, category, now))
|
||||
counts["updated"] += 1
|
||||
clear_marks_of_keys(conn, keys) # #480: a categorized file is listed
|
||||
return counts
|
||||
|
||||
def clear_image_categories(self, file_keys) -> int:
|
||||
"""Remove the own category of every file key (one transaction).
|
||||
|
||||
Returns the number of rows removed.
|
||||
"""
|
||||
keys = list(dict.fromkeys(tuple(key) for key in file_keys))
|
||||
removed = 0
|
||||
with self._pair_transaction() as conn:
|
||||
for key in keys:
|
||||
removed += conn.execute(DELETE_SQL, key).rowcount
|
||||
return removed
|
||||
@@ -0,0 +1,492 @@
|
||||
"""Collections (#331): named, hand-picked groups of image files.
|
||||
|
||||
A member names its file exactly as a favorite does (``image_favorites.py``):
|
||||
the source folder's real, case-normalized path (``root_key``) and the path
|
||||
below it (``relative_path``), both server-side only. A member whose file is
|
||||
moved, renamed or deleted outside PromptManager is kept and simply not listed;
|
||||
nothing follows a file to a new place.
|
||||
|
||||
Names are trimmed with each run of whitespace collapsed to one space, 1 to 100
|
||||
characters without control characters; ``name_key`` (NFC of the case-folded
|
||||
NFC name) is UNIQUE, so names are unique ignoring case and Unicode form.
|
||||
|
||||
The two tables install like ``image_favorites`` and ``saved_views``: optional
|
||||
in old backups, required once a database is current, created empty with no
|
||||
backfill. Startup checks only the structure, so a value saved under older
|
||||
rules never stops the library from opening; restore checks the values too,
|
||||
but never recomputes ``name_key`` (case tables differ between Python
|
||||
versions).
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
import unicodedata
|
||||
from typing import Optional
|
||||
|
||||
from .image_favorites import MAX_KEY_LENGTH
|
||||
|
||||
COLLECTIONS_TABLE = "image_collections"
|
||||
ITEMS_TABLE = "image_collection_items"
|
||||
FILE_INDEX = "idx_image_collection_items_file"
|
||||
TABLES = (COLLECTIONS_TABLE, ITEMS_TABLE)
|
||||
MAX_NAME_LENGTH = 100
|
||||
_TIMESTAMP_BYTES = 64
|
||||
# A name key's byte bound: case folding expands a character to at most
|
||||
# three, each at most 4 UTF-8 bytes.
|
||||
_MAX_KEY_BYTES = MAX_NAME_LENGTH * 3 * 4
|
||||
_WHITESPACE = re.compile(r"\s+")
|
||||
|
||||
_TABLE_DDL = {
|
||||
COLLECTIONS_TABLE: (
|
||||
f"CREATE TABLE {COLLECTIONS_TABLE} (id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||||
"name TEXT NOT NULL, name_key TEXT NOT NULL UNIQUE, "
|
||||
"created_at TEXT NOT NULL)"
|
||||
),
|
||||
ITEMS_TABLE: (
|
||||
f"CREATE TABLE {ITEMS_TABLE} (collection_id INTEGER NOT NULL REFERENCES "
|
||||
f"{COLLECTIONS_TABLE}(id) ON DELETE CASCADE, root_key TEXT NOT NULL, "
|
||||
"relative_path TEXT NOT NULL, added_at TEXT NOT NULL, "
|
||||
"PRIMARY KEY (collection_id, root_key, relative_path))"
|
||||
),
|
||||
}
|
||||
_INDEX_DDL = f"CREATE INDEX {FILE_INDEX} ON {ITEMS_TABLE}(root_key, relative_path)"
|
||||
# (name, declared type, not null, default, primary key position)
|
||||
_COLUMNS = {
|
||||
COLLECTIONS_TABLE: (
|
||||
("id", "INTEGER", 0, None, 1),
|
||||
("name", "TEXT", 1, None, 0),
|
||||
("name_key", "TEXT", 1, None, 0),
|
||||
("created_at", "TEXT", 1, None, 0),
|
||||
),
|
||||
ITEMS_TABLE: (
|
||||
("collection_id", "INTEGER", 1, None, 1),
|
||||
("root_key", "TEXT", 1, None, 2),
|
||||
("relative_path", "TEXT", 1, None, 3),
|
||||
("added_at", "TEXT", 1, None, 0),
|
||||
),
|
||||
}
|
||||
# (column, parent table, parent column, on delete)
|
||||
_FOREIGN_KEYS = {
|
||||
COLLECTIONS_TABLE: set(),
|
||||
ITEMS_TABLE: {("collection_id", COLLECTIONS_TABLE, "id", "CASCADE")},
|
||||
}
|
||||
|
||||
|
||||
class CollectionInvalid(ValueError):
|
||||
"""A name, id or target breaks the write rules (a 400)."""
|
||||
|
||||
|
||||
def _has_control(text: str) -> bool:
|
||||
"""Any Unicode control character (category Cc: C0, DEL and C1)."""
|
||||
return any(unicodedata.category(char) == "Cc" for char in text)
|
||||
|
||||
|
||||
def normalize_collection_name(name: object) -> str:
|
||||
"""The stored spelling of *name*: trimmed, whitespace collapsed, checked.
|
||||
|
||||
Raises ``CollectionInvalid`` unless the result is 1 to 100 characters
|
||||
without control characters.
|
||||
"""
|
||||
if not isinstance(name, str):
|
||||
raise CollectionInvalid("Name must be a string")
|
||||
collapsed = _WHITESPACE.sub(" ", name).strip()
|
||||
if not 1 <= len(collapsed) <= MAX_NAME_LENGTH:
|
||||
raise CollectionInvalid(f"Name must be 1 to {MAX_NAME_LENGTH} characters")
|
||||
if _has_control(collapsed):
|
||||
raise CollectionInvalid("Name contains control characters")
|
||||
return collapsed
|
||||
|
||||
|
||||
def collection_name_key(name: str) -> str:
|
||||
"""The uniqueness key of a name: NFC of the case-folded NFC spelling."""
|
||||
return unicodedata.normalize("NFC", unicodedata.normalize("NFC", name).casefold())
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql).lower()
|
||||
|
||||
|
||||
def _expected_catalog() -> set:
|
||||
objects = {("table", name, _compact(sql)) for name, sql in _TABLE_DDL.items()}
|
||||
objects.add(("index", FILE_INDEX, _compact(_INDEX_DDL)))
|
||||
objects |= {("index", f"sqlite_autoindex_{table}_1", "") for table in TABLES}
|
||||
return objects
|
||||
|
||||
|
||||
def _catalog(conn: sqlite3.Connection) -> list:
|
||||
# sqlite_sequence (AUTOINCREMENT) is shared with other tables and ignored.
|
||||
return conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master "
|
||||
"WHERE name IN (?,?,?) OR tbl_name IN (?,?)",
|
||||
(*TABLES, FILE_INDEX, *TABLES),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
for table, expected in _COLUMNS.items():
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[4], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
)
|
||||
if columns != expected:
|
||||
return f"Image collections table {table} has invalid columns"
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({table})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS[table]:
|
||||
return f"Image collections table {table} has invalid foreign keys"
|
||||
return None
|
||||
|
||||
|
||||
def _decoded_too_long(conn, sql: str, limit: int) -> bool:
|
||||
"""Whether a byte-bounded candidate row decodes to over *limit* characters."""
|
||||
for row in conn.execute(sql):
|
||||
for raw in row:
|
||||
if len(bytes(raw).decode("utf-8", "replace")) > limit:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _value_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
# Only this module's constants are interpolated into the SQL below. Bytes
|
||||
# bound characters from above (UTF-8 is at most 4 bytes each); the few
|
||||
# candidates over the character bound are measured decoded in Python.
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {COLLECTIONS_TABLE} WHERE " # nosec B608
|
||||
"typeof(id) != 'integer' OR typeof(name) != 'text' "
|
||||
"OR typeof(name_key) != 'text' OR typeof(created_at) != 'text' "
|
||||
f"OR length(CAST(name AS BLOB)) > {MAX_NAME_LENGTH * 4} "
|
||||
"OR length(CAST(name_key AS BLOB)) = 0 "
|
||||
f"OR length(CAST(name_key AS BLOB)) > {_MAX_KEY_BYTES} "
|
||||
f"OR length(CAST(created_at AS BLOB)) > {_TIMESTAMP_BYTES} LIMIT 1"
|
||||
).fetchone():
|
||||
return "Image collections contains invalid values"
|
||||
# The key is not recomputed: Unicode case tables differ between Python
|
||||
# versions, and a key written by another version must stay restorable.
|
||||
# Its UNIQUE constraint keeps names distinct; writes compute it.
|
||||
for (name,) in conn.execute(f"SELECT name FROM {COLLECTIONS_TABLE}"): # nosec B608
|
||||
if (
|
||||
not 1 <= len(name) <= MAX_NAME_LENGTH
|
||||
or _has_control(name)
|
||||
or name != name.strip()
|
||||
):
|
||||
return "Image collections contains invalid names"
|
||||
limit = MAX_KEY_LENGTH * 4
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {ITEMS_TABLE} WHERE " # nosec B608
|
||||
"typeof(collection_id) != 'integer' OR typeof(root_key) != 'text' "
|
||||
"OR typeof(relative_path) != 'text' OR typeof(added_at) != 'text' "
|
||||
f"OR length(CAST(root_key AS BLOB)) > {limit} "
|
||||
f"OR length(CAST(relative_path AS BLOB)) > {limit} "
|
||||
f"OR length(CAST(added_at AS BLOB)) > {_TIMESTAMP_BYTES} LIMIT 1"
|
||||
).fetchone() or _decoded_too_long(
|
||||
conn,
|
||||
f"SELECT CAST(root_key AS BLOB), CAST(relative_path AS BLOB) " # nosec B608
|
||||
f"FROM {ITEMS_TABLE} WHERE length(CAST(root_key AS BLOB)) > {MAX_KEY_LENGTH} "
|
||||
f"OR length(CAST(relative_path AS BLOB)) > {MAX_KEY_LENGTH}",
|
||||
MAX_KEY_LENGTH,
|
||||
):
|
||||
return "Image collections contains invalid members"
|
||||
if conn.execute(f"PRAGMA foreign_key_check({ITEMS_TABLE})").fetchone():
|
||||
return "Image collections contains members of a missing collection"
|
||||
return None
|
||||
|
||||
|
||||
def image_collections_schema_error(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
allow_missing: bool = True,
|
||||
check_values: bool = True,
|
||||
) -> Optional[str]:
|
||||
"""Reject partial or unsupported collections schemas and invalid rows.
|
||||
|
||||
Reads only: nothing is ever repaired. A database without either table is
|
||||
acceptable for backups that predate them unless ``allow_missing`` is
|
||||
false. ``check_values=False`` (live startup) checks the structure only.
|
||||
"""
|
||||
objects = _catalog(conn)
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing image collections schema"
|
||||
present = {name for kind, name, _sql in objects if kind == "table"}
|
||||
if present != set(TABLES):
|
||||
return "Incomplete image collections schema"
|
||||
actual = {(kind, name, _compact(sql or "")) for kind, name, sql in objects}
|
||||
if actual != _expected_catalog():
|
||||
return "Image collections has an invalid schema definition"
|
||||
error = _structure_error(conn)
|
||||
if error or not check_values:
|
||||
return error
|
||||
return _value_error(conn)
|
||||
|
||||
|
||||
def migrate_image_collections(conn: sqlite3.Connection) -> None:
|
||||
"""Install both tables atomically; validate before and after creating them."""
|
||||
conn.execute("SAVEPOINT image_collections_schema")
|
||||
try:
|
||||
error = image_collections_schema_error(conn, check_values=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
for sql in _TABLE_DDL.values():
|
||||
conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
conn.execute(_INDEX_DDL.replace("INDEX", "INDEX IF NOT EXISTS", 1))
|
||||
error = image_collections_schema_error(
|
||||
conn, allow_missing=False, check_values=False
|
||||
)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE image_collections_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO image_collections_schema")
|
||||
conn.execute("RELEASE image_collections_schema")
|
||||
raise
|
||||
|
||||
|
||||
class CollectionMissing(LookupError):
|
||||
"""No collection has the id (a 404 ``collection-not-found``)."""
|
||||
|
||||
|
||||
class CollectionNameTaken(ValueError):
|
||||
"""Another collection already has the name's key (a 409 ``name-taken``)."""
|
||||
|
||||
def __init__(self, collection_id: int):
|
||||
super().__init__("Another collection has this name")
|
||||
self.collection_id = collection_id
|
||||
|
||||
|
||||
class CollectionFilesMissing(LookupError):
|
||||
"""A new name with nothing to insert (a 404 ``files-not-found``)."""
|
||||
|
||||
|
||||
LIST_SQL = f"SELECT id, name FROM {COLLECTIONS_TABLE} ORDER BY name_key, id"
|
||||
GET_SQL = f"SELECT id, name FROM {COLLECTIONS_TABLE} WHERE id = ?"
|
||||
BY_KEY_SQL = f"SELECT id, name FROM {COLLECTIONS_TABLE} WHERE name_key = ?"
|
||||
MEMBERS_SQL = (
|
||||
f"SELECT relative_path FROM {ITEMS_TABLE} "
|
||||
"WHERE collection_id = ? AND root_key = ?"
|
||||
)
|
||||
ROOT_MEMBERS_SQL = (
|
||||
f"SELECT collection_id, relative_path FROM {ITEMS_TABLE} WHERE root_key = ?"
|
||||
)
|
||||
COUNTS_SQL = (
|
||||
f"SELECT c.id, (SELECT COUNT(*) FROM {ITEMS_TABLE} i "
|
||||
f"WHERE i.collection_id = c.id) FROM {COLLECTIONS_TABLE} c"
|
||||
)
|
||||
FILE_SQL = (
|
||||
f"SELECT collection_id FROM {ITEMS_TABLE} "
|
||||
"WHERE root_key = ? AND relative_path = ? ORDER BY collection_id"
|
||||
)
|
||||
# Every read, for the EXPLAIN QUERY PLAN check: none scans the items table and
|
||||
# only the two whole-list reads (LIST_SQL, COUNTS_SQL) scan the collections.
|
||||
READ_SQL = (
|
||||
LIST_SQL,
|
||||
GET_SQL,
|
||||
BY_KEY_SQL,
|
||||
MEMBERS_SQL,
|
||||
ROOT_MEMBERS_SQL,
|
||||
COUNTS_SQL,
|
||||
FILE_SQL,
|
||||
)
|
||||
_INSERT_ITEM_SQL = f"INSERT OR IGNORE INTO {ITEMS_TABLE} VALUES (?, ?, ?, ?)"
|
||||
_DELETE_ITEM_SQL = (
|
||||
f"DELETE FROM {ITEMS_TABLE} "
|
||||
"WHERE collection_id = ? AND root_key = ? AND relative_path = ?"
|
||||
)
|
||||
|
||||
|
||||
def _collection_id(value: object) -> int:
|
||||
if type(value) is not int or not 1 <= value <= 999_999_999:
|
||||
raise CollectionInvalid("Collection id must be a positive integer")
|
||||
return value
|
||||
|
||||
|
||||
def _items(items: object) -> list:
|
||||
"""Distinct ``(root key, relative key)`` pairs, in order."""
|
||||
if not isinstance(items, (list, tuple)):
|
||||
raise CollectionInvalid("Items must be a list")
|
||||
found = []
|
||||
for item in items:
|
||||
if (
|
||||
not isinstance(item, (list, tuple))
|
||||
or len(item) != 2
|
||||
or not all(
|
||||
isinstance(part, str) and 1 <= len(part) <= MAX_KEY_LENGTH
|
||||
for part in item
|
||||
)
|
||||
):
|
||||
raise CollectionInvalid("Each item is a root key and a relative key")
|
||||
found.append((item[0], item[1]))
|
||||
return list(dict.fromkeys(found))
|
||||
|
||||
|
||||
def _row(row) -> dict:
|
||||
return {"id": row[0], "name": row[1]}
|
||||
|
||||
|
||||
class ImageCollectionOperations:
|
||||
"""PromptDatabase mixin: collections and their members.
|
||||
|
||||
Each write runs in its own ``BEGIN IMMEDIATE`` transaction
|
||||
(``_pair_transaction``), so two tabs creating one name are serialized.
|
||||
"""
|
||||
|
||||
def list_collections(self) -> list[dict]:
|
||||
"""Every collection ``{id, name}`` in name-key order, then id."""
|
||||
rows = self.model.get_connection().execute(LIST_SQL)
|
||||
return [_row(row) for row in rows]
|
||||
|
||||
def get_collection(self, collection_id: int) -> Optional[dict]:
|
||||
"""``{id, name}`` of one collection, or None."""
|
||||
row = (
|
||||
self.model.get_connection()
|
||||
.execute(GET_SQL, (_collection_id(collection_id),))
|
||||
.fetchone()
|
||||
)
|
||||
return _row(row) if row else None
|
||||
|
||||
def collection_exists(self, collection_id: int) -> bool:
|
||||
return self.get_collection(collection_id) is not None
|
||||
|
||||
def collection_members(self, collection_id: int, root_key: str) -> set[str]:
|
||||
"""The relative keys stored for one collection under one source."""
|
||||
rows = self.model.get_connection().execute(
|
||||
MEMBERS_SQL, (_collection_id(collection_id), root_key)
|
||||
)
|
||||
return {path for (path,) in rows}
|
||||
|
||||
def root_collection_members(self, root_key: str) -> dict[int, set[str]]:
|
||||
"""``{collection id: relative keys}`` stored under one source."""
|
||||
found: dict[int, set[str]] = {}
|
||||
for collection_id, path in self.model.get_connection().execute(
|
||||
ROOT_MEMBERS_SQL, (root_key,)
|
||||
):
|
||||
found.setdefault(collection_id, set()).add(path)
|
||||
return found
|
||||
|
||||
def collection_count_snapshot(self, root_keys: list) -> tuple[dict, dict]:
|
||||
"""``({root key: {id: relative keys}}, {id: stored rows})`` read in
|
||||
one read transaction, so a write between the reads cannot skew them.
|
||||
"""
|
||||
conn = self.model.get_connection()
|
||||
if conn.in_transaction:
|
||||
raise RuntimeError("Collection counts require their own transaction")
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
members = {}
|
||||
for root_key in dict.fromkeys(root_keys):
|
||||
found: dict[int, set[str]] = {}
|
||||
for collection_id, path in conn.execute(ROOT_MEMBERS_SQL, (root_key,)):
|
||||
found.setdefault(collection_id, set()).add(path)
|
||||
members[root_key] = found
|
||||
stored = {cid: count for cid, count in conn.execute(COUNTS_SQL)}
|
||||
finally:
|
||||
conn.rollback() # it only read
|
||||
return members, stored
|
||||
|
||||
def collection_counts(self) -> dict[int, int]:
|
||||
"""``{collection id: stored rows}`` for every collection."""
|
||||
rows = self.model.get_connection().execute(COUNTS_SQL)
|
||||
return {collection_id: count for collection_id, count in rows}
|
||||
|
||||
def file_collections(self, root_key: str, relative_path: str) -> list[int]:
|
||||
"""Ids of the collections holding one file."""
|
||||
rows = self.model.get_connection().execute(FILE_SQL, (root_key, relative_path))
|
||||
return [collection_id for (collection_id,) in rows]
|
||||
|
||||
def add_to_collection(
|
||||
self, *, items: object, collection_id: object = None, name: object = None
|
||||
) -> dict:
|
||||
"""Add files to an existing id or a name; create the name when needed.
|
||||
|
||||
A name whose key exists uses that collection; otherwise a collection
|
||||
is created only when at least one item is inserted. Returns
|
||||
``{collection, created, added, already}``.
|
||||
"""
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
if (collection_id is None) == (name is None):
|
||||
raise CollectionInvalid("Give either a collection id or a name")
|
||||
pairs = _items(items)
|
||||
if name is not None:
|
||||
spelled = normalize_collection_name(name)
|
||||
else:
|
||||
collection_id = _collection_id(collection_id)
|
||||
now = utc_now_iso()
|
||||
with self._pair_transaction() as conn:
|
||||
created = False
|
||||
if name is not None:
|
||||
row = conn.execute(
|
||||
BY_KEY_SQL, (collection_name_key(spelled),)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
if not pairs:
|
||||
raise CollectionFilesMissing("Nothing to add")
|
||||
cursor = conn.execute(
|
||||
f"INSERT INTO {COLLECTIONS_TABLE} " # nosec B608
|
||||
"(name, name_key, created_at) VALUES (?, ?, ?)",
|
||||
(spelled, collection_name_key(spelled), now),
|
||||
)
|
||||
row, created = (cursor.lastrowid, spelled), True
|
||||
else:
|
||||
row = conn.execute(GET_SQL, (collection_id,)).fetchone()
|
||||
if row is None:
|
||||
raise CollectionMissing("No such collection")
|
||||
collection = _row(row)
|
||||
added = sum(
|
||||
conn.execute(
|
||||
_INSERT_ITEM_SQL, (collection["id"], root, path, now)
|
||||
).rowcount
|
||||
for root, path in pairs
|
||||
)
|
||||
clear_marks_of_keys(conn, pairs) # #480: a collected file is listed
|
||||
return {
|
||||
"collection": collection,
|
||||
"created": created,
|
||||
"added": added,
|
||||
"already": len(pairs) - added,
|
||||
}
|
||||
|
||||
def remove_from_collection(self, collection_id: object, items: object) -> dict:
|
||||
"""Remove files from one collection; ``{collection, removed}``."""
|
||||
collection_id = _collection_id(collection_id)
|
||||
pairs = _items(items)
|
||||
with self._pair_transaction() as conn:
|
||||
row = conn.execute(GET_SQL, (collection_id,)).fetchone()
|
||||
if row is None:
|
||||
raise CollectionMissing("No such collection")
|
||||
removed = sum(
|
||||
conn.execute(_DELETE_ITEM_SQL, (collection_id, root, path)).rowcount
|
||||
for root, path in pairs
|
||||
)
|
||||
return {"collection": _row(row), "removed": removed}
|
||||
|
||||
def rename_collection(self, collection_id: object, name: object) -> dict:
|
||||
"""The renamed ``{id, name}``; a case variant of its own name is fine."""
|
||||
collection_id = _collection_id(collection_id)
|
||||
spelled = normalize_collection_name(name)
|
||||
key = collection_name_key(spelled)
|
||||
with self._pair_transaction() as conn:
|
||||
if conn.execute(GET_SQL, (collection_id,)).fetchone() is None:
|
||||
raise CollectionMissing("No such collection")
|
||||
taken = conn.execute(BY_KEY_SQL, (key,)).fetchone()
|
||||
if taken is not None and taken[0] != collection_id:
|
||||
raise CollectionNameTaken(taken[0])
|
||||
conn.execute(
|
||||
f"UPDATE {COLLECTIONS_TABLE} SET name = ?, name_key = ? " # nosec B608
|
||||
"WHERE id = ?",
|
||||
(spelled, key, collection_id),
|
||||
)
|
||||
return {"id": collection_id, "name": spelled}
|
||||
|
||||
def delete_collection(self, collection_id: object) -> bool:
|
||||
"""Delete one collection and its members (never a file). True."""
|
||||
collection_id = _collection_id(collection_id)
|
||||
with self._pair_transaction() as conn:
|
||||
cursor = conn.execute(
|
||||
f"DELETE FROM {COLLECTIONS_TABLE} WHERE id = ?", # nosec B608
|
||||
(collection_id,),
|
||||
)
|
||||
if cursor.rowcount == 0:
|
||||
raise CollectionMissing("No such collection")
|
||||
return True
|
||||
@@ -104,11 +104,14 @@ class ImageFavoriteOperations:
|
||||
"""Mark one file as a favorite; repeating it is a no-op. Returns True."""
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
|
||||
with self.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO image_favorites VALUES (?, ?, ?)",
|
||||
(root_key, relative_path, utc_now_iso()),
|
||||
)
|
||||
clear_marks_of_keys(conn, [(root_key, relative_path)]) # #480
|
||||
return True
|
||||
|
||||
def clear_image_favorite(self, root_key: str, relative_path: str) -> bool:
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
"""Remove a file's records from the library, and its removal mark (#480).
|
||||
|
||||
A *removal* describes one physical file the way the API resolved it from the
|
||||
walked listing (``py/api/image_delete.py``):
|
||||
|
||||
- ``spellings``: every path string records may name it by
|
||||
(``duplicate_spellings.Spellings.of``);
|
||||
- ``locator``: its private canonical locator;
|
||||
- ``favorites``: ``(root key, relative key)`` under every listed root holding
|
||||
it, the keys of its favorite and its collection memberships;
|
||||
- ``category``: its own category's file key, or None.
|
||||
|
||||
``remove_image_records`` deletes, in one ``BEGIN IMMEDIATE`` transaction:
|
||||
|
||||
- its image records (``generated_images``) under every spelling, plus legacy
|
||||
rows matched by ``image_path``; their saved-pair links go by cascade and the
|
||||
workflow documents they held are released;
|
||||
- the image tags and auto-tag marks of its byte records. The annotation file
|
||||
and byte rows go too, unless a Needs review item still cites them
|
||||
(``media_review_items.byte_id`` is ``ON DELETE RESTRICT``): a decided or
|
||||
revised item keeps its decision;
|
||||
- its pending, undecided Needs review items (settled as duplicate removal
|
||||
settles them);
|
||||
- its own category, favorite and collection memberships.
|
||||
|
||||
With ``mark`` it also writes the file's removal mark (library-only removal):
|
||||
the file stays on disk, hidden until its bytes change. Prompts, recorded runs
|
||||
and the media file itself are never touched here.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from .annotation_schema import BYTES_TABLE, FILES_TABLE
|
||||
from .annotations import _BUMP_INDEX
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .duplicate_copies import (
|
||||
DELETE_FAVORITE_SQL,
|
||||
DELETE_LINK_SQL,
|
||||
DELETE_MEMBERSHIPS_SQL,
|
||||
_file_link_rows,
|
||||
)
|
||||
from .image_categories import DELETE_SQL as DELETE_CATEGORY_SQL
|
||||
from .image_removal_schema import (
|
||||
META_TABLE,
|
||||
TABLE,
|
||||
bump_generation,
|
||||
clear_marks,
|
||||
has_marks_table,
|
||||
mark_of,
|
||||
write_mark as _write_mark,
|
||||
)
|
||||
from .library import display_title
|
||||
from .models import utc_now_iso
|
||||
from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM
|
||||
from .review_changes import notify_review_change
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
BYTES_OF_LOCATOR_SQL = (
|
||||
f"SELECT b.id FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b " # nosec B608
|
||||
"ON b.file_id=f.id WHERE f.locator=?"
|
||||
)
|
||||
TAG_COUNT_SQL = "SELECT COUNT(*) FROM media_image_tags WHERE byte_id=?"
|
||||
AUTOTAG_COUNT_SQL = (
|
||||
f"SELECT COUNT(*) FROM {AUTOTAG_TABLE} WHERE byte_id=?" # nosec B608
|
||||
)
|
||||
CLEAR_TAGS_SQL = "DELETE FROM media_image_tags WHERE byte_id=?"
|
||||
CLEAR_AUTOTAG_SQL = f"DELETE FROM {AUTOTAG_TABLE} WHERE byte_id=?" # nosec B608
|
||||
BUMP_TAG_REVISION_SQL = (
|
||||
f"UPDATE {BYTES_TABLE} SET tag_revision=tag_revision+1 WHERE id=?" # nosec B608
|
||||
)
|
||||
REVIEW_REFS_SQL = (
|
||||
f"SELECT COUNT(*) FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
"JOIN media_review_items i ON i.byte_id=b.id WHERE f.locator=?"
|
||||
)
|
||||
DELETE_FILE_SQL = f"DELETE FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
FILE_EXISTS_SQL = f"SELECT 1 FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
FAVORITE_EXISTS_SQL = (
|
||||
"SELECT COUNT(*) FROM image_favorites WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
MEMBERSHIP_COUNT_SQL = (
|
||||
"SELECT COUNT(*) FROM image_collection_items WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
CATEGORY_EXISTS_SQL = (
|
||||
"SELECT COUNT(*) FROM image_categories WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
PAIR_LINK_COUNT_SQL = "SELECT COUNT(*) FROM image_prompt_pairs WHERE image_id=?"
|
||||
PROMPT_LINKS_SQL = "SELECT id FROM generated_images WHERE prompt_id=?"
|
||||
PROMPT_TITLE_SQL = (
|
||||
"SELECT p.text, s.title FROM prompts p "
|
||||
"LEFT JOIN prompt_library_state s ON s.prompt_id=p.id WHERE p.id=?"
|
||||
)
|
||||
MARKS_SQL = f"SELECT locator, size, mtime_ns FROM {TABLE}" # nosec B608
|
||||
REVISION_SQL = f"SELECT revision FROM {META_TABLE} WHERE singleton_id=1" # nosec B608
|
||||
MAX_ORPHAN_TITLES = 3
|
||||
RECORD_KINDS = (
|
||||
"prompt_links",
|
||||
"pair_links",
|
||||
"image_tags",
|
||||
"auto_tagged",
|
||||
"categories",
|
||||
"collections",
|
||||
"favorites",
|
||||
"review_items",
|
||||
)
|
||||
|
||||
|
||||
def _link_ids(conn, removal):
|
||||
"""``{image id: prompt id}`` of the records naming the file, any spelling."""
|
||||
found = {}
|
||||
for spelled in dict.fromkeys(removal["spellings"]):
|
||||
for image_id, prompt_id in _file_link_rows(conn, spelled):
|
||||
found[image_id] = prompt_id
|
||||
return found
|
||||
|
||||
|
||||
def _byte_ids(conn, locator):
|
||||
return [row[0] for row in conn.execute(BYTES_OF_LOCATOR_SQL, (locator,))]
|
||||
|
||||
|
||||
def file_locator(path) -> str:
|
||||
"""The locator of an absolute path (``normcase(realpath)``)."""
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
|
||||
|
||||
def removed_from_library(conn, path) -> bool:
|
||||
"""Whether *path* is hidden by a removal mark whose bytes still match.
|
||||
|
||||
A mark whose file changed (other size or ``mtime_ns``) or is gone is
|
||||
stale: it is deleted on *conn* (the caller's write transaction), and the
|
||||
file is not hidden.
|
||||
"""
|
||||
locator = file_locator(path)
|
||||
mark = mark_of(conn, locator)
|
||||
if mark is None:
|
||||
return False
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except OSError:
|
||||
clear_marks(conn, [locator])
|
||||
return False
|
||||
if (info.st_size, info.st_mtime_ns) == tuple(mark):
|
||||
return True
|
||||
clear_marks(conn, [locator])
|
||||
return False
|
||||
|
||||
|
||||
def mark_hides(marks: dict, locator: str, path) -> bool:
|
||||
"""Whether a removal mark in *marks* hides the file at *path* (blocking).
|
||||
|
||||
*marks* is ``{locator: (size, mtime_ns)}``. The mark of *locator* hides
|
||||
the file while ``os.lstat(path)`` still has the size and ``mtime_ns`` it
|
||||
was removed with; a file that cannot be looked at is not hidden.
|
||||
"""
|
||||
mark = marks.get(locator)
|
||||
if mark is None:
|
||||
return False
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except OSError:
|
||||
return False
|
||||
return (info.st_size, info.st_mtime_ns) == tuple(mark)
|
||||
|
||||
|
||||
def write_mark(conn, locator, size, mtime_ns):
|
||||
_write_mark(conn, locator, size, mtime_ns, utc_now_iso())
|
||||
|
||||
|
||||
class ImageRemovalOperations:
|
||||
"""PromptDatabase mixin: remove a file's records; removal marks."""
|
||||
|
||||
def count_removal_records(self, removals) -> dict:
|
||||
"""Counts of what removing *removals* deletes, in one read snapshot.
|
||||
|
||||
``{records: {kind: n}, orphaned_prompts: {count, titles}}``: prompts
|
||||
whose every image record is among these files lose their last image
|
||||
(they stay in the Library).
|
||||
"""
|
||||
from .maintenance import _read_snapshot
|
||||
|
||||
def read(conn):
|
||||
totals = dict.fromkeys(RECORD_KINDS, 0)
|
||||
links = {}
|
||||
for removal in removals:
|
||||
found = _link_ids(conn, removal)
|
||||
links.update(found)
|
||||
for image_id in found:
|
||||
totals["pair_links"] += conn.execute(
|
||||
PAIR_LINK_COUNT_SQL, (image_id,)
|
||||
).fetchone()[0]
|
||||
tagged = autotagged = False
|
||||
for byte_id in _byte_ids(conn, removal["locator"]):
|
||||
tagged |= bool(
|
||||
conn.execute(TAG_COUNT_SQL, (byte_id,)).fetchone()[0]
|
||||
)
|
||||
autotagged |= bool(
|
||||
conn.execute(AUTOTAG_COUNT_SQL, (byte_id,)).fetchone()[0]
|
||||
)
|
||||
totals["image_tags"] += int(tagged)
|
||||
totals["auto_tagged"] += int(autotagged)
|
||||
totals["review_items"] += len(
|
||||
conn.execute(_PENDING_OF_LOCATOR, (removal["locator"],)).fetchall()
|
||||
)
|
||||
totals["favorites"] += int(
|
||||
any(
|
||||
conn.execute(FAVORITE_EXISTS_SQL, key).fetchone()[0]
|
||||
for key in removal["favorites"]
|
||||
)
|
||||
)
|
||||
totals["collections"] += sum(
|
||||
conn.execute(MEMBERSHIP_COUNT_SQL, key).fetchone()[0]
|
||||
for key in removal["favorites"]
|
||||
)
|
||||
if removal.get("category"):
|
||||
totals["categories"] += conn.execute(
|
||||
CATEGORY_EXISTS_SQL, removal["category"]
|
||||
).fetchone()[0]
|
||||
totals["prompt_links"] = len(links)
|
||||
return {
|
||||
"records": totals,
|
||||
"orphaned_prompts": self._orphaned_prompts(conn, links),
|
||||
}
|
||||
|
||||
return _read_snapshot(self.model.db_path, read)
|
||||
|
||||
@staticmethod
|
||||
def _orphaned_prompts(conn, links):
|
||||
removed = set(links)
|
||||
orphans = []
|
||||
for prompt_id in dict.fromkeys(links.values()):
|
||||
remaining = [
|
||||
row[0]
|
||||
for row in conn.execute(PROMPT_LINKS_SQL, (prompt_id,))
|
||||
if row[0] not in removed
|
||||
]
|
||||
if not remaining:
|
||||
orphans.append(prompt_id)
|
||||
titles = []
|
||||
for prompt_id in orphans[:MAX_ORPHAN_TITLES]:
|
||||
row = conn.execute(PROMPT_TITLE_SQL, (prompt_id,)).fetchone()
|
||||
if row is not None:
|
||||
titles.append(display_title((row[1],), row[0]) or "")
|
||||
return {"count": len(orphans), "titles": titles}
|
||||
|
||||
def remove_image_records(self, removal, *, mark=None) -> dict:
|
||||
"""Delete every record of one file (one transaction); see the module.
|
||||
|
||||
``mark``: ``(size, mtime_ns)`` to write the file's removal mark, or
|
||||
None (the file was moved to the trash). Returns the counts deleted.
|
||||
"""
|
||||
counts = dict.fromkeys(RECORD_KINDS, 0)
|
||||
locator = removal["locator"]
|
||||
with self._pair_transaction() as conn:
|
||||
links = _link_ids(conn, removal)
|
||||
if links:
|
||||
documents = documents_of_rows(conn, "id", list(links))
|
||||
for image_id in links:
|
||||
counts["pair_links"] += conn.execute(
|
||||
PAIR_LINK_COUNT_SQL, (image_id,)
|
||||
).fetchone()[0]
|
||||
conn.execute(DELETE_LINK_SQL, (image_id,))
|
||||
release_documents(conn, documents)
|
||||
counts["prompt_links"] = len(links)
|
||||
counts.update(self._remove_annotations(conn, locator))
|
||||
for key in removal["favorites"]:
|
||||
counts["favorites"] += conn.execute(DELETE_FAVORITE_SQL, key).rowcount
|
||||
counts["collections"] += conn.execute(
|
||||
DELETE_MEMBERSHIPS_SQL, key
|
||||
).rowcount
|
||||
if removal.get("category"):
|
||||
counts["categories"] += conn.execute(
|
||||
DELETE_CATEGORY_SQL, tuple(removal["category"])
|
||||
).rowcount
|
||||
if mark is not None:
|
||||
write_mark(conn, locator, *mark)
|
||||
if counts["review_items"]:
|
||||
notify_review_change(self.model.db_path)
|
||||
return counts
|
||||
|
||||
def _remove_annotations(self, conn, locator) -> dict:
|
||||
"""Tags, auto-tag marks, pending review items and, when free, the rows."""
|
||||
counts = {"image_tags": 0, "auto_tagged": 0, "review_items": 0}
|
||||
bytes_ = _byte_ids(conn, locator)
|
||||
for byte_id in bytes_:
|
||||
tags = conn.execute(CLEAR_TAGS_SQL, (byte_id,)).rowcount
|
||||
autotags = conn.execute(CLEAR_AUTOTAG_SQL, (byte_id,)).rowcount
|
||||
if tags:
|
||||
conn.execute(BUMP_TAG_REVISION_SQL, (byte_id,))
|
||||
counts["image_tags"] |= bool(tags)
|
||||
counts["auto_tagged"] |= bool(autotags)
|
||||
for (item_id,) in conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchall():
|
||||
counts["review_items"] += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
||||
if counts["review_items"]:
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
if conn.execute(FILE_EXISTS_SQL, (locator,)).fetchone():
|
||||
if not conn.execute(REVIEW_REFS_SQL, (locator,)).fetchone()[0]:
|
||||
conn.execute(DELETE_FILE_SQL, (locator,))
|
||||
conn.execute(_BUMP_INDEX)
|
||||
counts["image_tags"] = int(counts["image_tags"])
|
||||
counts["auto_tagged"] = int(counts["auto_tagged"])
|
||||
return counts
|
||||
|
||||
def removal_marks(self) -> dict:
|
||||
"""``{locator: (size, mtime_ns)}`` of every removal mark (one read)."""
|
||||
conn = self.model.get_connection()
|
||||
if not has_marks_table(conn):
|
||||
return {}
|
||||
return {row[0]: (row[1], row[2]) for row in conn.execute(MARKS_SQL)}
|
||||
|
||||
def removal_revision(self) -> int:
|
||||
conn = self.model.get_connection()
|
||||
if not has_marks_table(conn):
|
||||
return 0
|
||||
row = conn.execute(REVISION_SQL).fetchone()
|
||||
return row[0] if row else 0
|
||||
|
||||
def clear_removal_marks(self, locators) -> int:
|
||||
"""Show these files again (their marks go); the count removed."""
|
||||
locators = list(locators)
|
||||
if not locators:
|
||||
return 0
|
||||
with self._pair_transaction() as conn:
|
||||
removed = clear_marks(conn, locators)
|
||||
# #498: again after the commit, so a reader that cached the marks
|
||||
# between the in-transaction bump and the commit reloads them.
|
||||
bump_generation()
|
||||
return removed
|
||||
|
||||
def clear_hiding_marks(self, files) -> set:
|
||||
"""Delete the marks that still hide their file (#498, Return to library).
|
||||
|
||||
*files* are ``(locator, path)`` pairs. In one ``BEGIN IMMEDIATE``
|
||||
transaction each mark is read again and deleted only while the file at
|
||||
*path* still has its size and ``mtime_ns``. Returns the locators whose
|
||||
marks went.
|
||||
"""
|
||||
cleared = set()
|
||||
files = list(files)
|
||||
if not files:
|
||||
return cleared
|
||||
with self._pair_transaction() as conn:
|
||||
for locator, path in files:
|
||||
mark = mark_of(conn, locator)
|
||||
if mark is not None and mark_hides({locator: mark}, locator, path):
|
||||
cleared.add(locator)
|
||||
clear_marks(conn, sorted(cleared))
|
||||
bump_generation()
|
||||
return cleared
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Removal marks (#480): files removed from the library that stay on disk.
|
||||
|
||||
``image_removals`` holds one row per hidden file: its private canonical
|
||||
locator (the ``media_annotation_files.locator`` form) and the ``size`` and
|
||||
``mtime_ns`` it had when it was removed. A file at that locator with other
|
||||
bytes is a new file and shows again. ``image_removals_meta`` is one row, the
|
||||
marks revision, bumped in the same transaction as every mark change so the
|
||||
workspace listing caches can tell marks changed.
|
||||
|
||||
The group installs like ``image_favorites``: optional in old backups,
|
||||
required once a database is current. Older builds ignore it.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
from .annotation_schema import MAX_LOCATOR_LENGTH
|
||||
|
||||
TABLE = "image_removals"
|
||||
META_TABLE = "image_removals_meta"
|
||||
TABLES = (TABLE, META_TABLE)
|
||||
_TABLE_DDL = {
|
||||
TABLE: (
|
||||
f"CREATE TABLE {TABLE} (locator TEXT PRIMARY KEY, "
|
||||
"size INTEGER NOT NULL CHECK(size>=0), mtime_ns INTEGER NOT NULL, "
|
||||
"removed_at TEXT NOT NULL) WITHOUT ROWID"
|
||||
),
|
||||
META_TABLE: (
|
||||
f"CREATE TABLE {META_TABLE} (singleton_id INTEGER PRIMARY KEY "
|
||||
"CHECK(singleton_id=1), revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(revision>=0))"
|
||||
),
|
||||
}
|
||||
_SEED_META = (
|
||||
f"INSERT OR IGNORE INTO {META_TABLE} (singleton_id, revision) VALUES (1, 0)"
|
||||
)
|
||||
_COLUMNS = {
|
||||
TABLE: (
|
||||
("locator", "TEXT", 1, 1),
|
||||
("size", "INTEGER", 1, 0),
|
||||
("mtime_ns", "INTEGER", 1, 0),
|
||||
("removed_at", "TEXT", 1, 0),
|
||||
),
|
||||
META_TABLE: (
|
||||
("singleton_id", "INTEGER", 0, 1),
|
||||
("revision", "INTEGER", 1, 0),
|
||||
),
|
||||
}
|
||||
_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
|
||||
rows = conn.execute(
|
||||
"SELECT type,name,tbl_name,sql FROM sqlite_master WHERE name IN (?,?) "
|
||||
"OR tbl_name IN (?,?)",
|
||||
TABLES + TABLES,
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return None if allow_missing else "Missing image removal marks schema"
|
||||
tables = {name: sql for kind, name, _tbl, sql in rows if kind == "table"}
|
||||
others = {(kind, name) for kind, name, _tbl, _sql in rows if kind != "table"}
|
||||
if set(tables) != set(TABLES) or others - {("index", _AUTOINDEX)}:
|
||||
return "Image removal marks name is reserved by an unsupported schema object"
|
||||
for table in TABLES:
|
||||
if _compact(tables[table]) != _compact(_TABLE_DDL[table]):
|
||||
return "Image removal marks have an invalid table definition"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
)
|
||||
if columns != _COLUMNS[table]:
|
||||
return "Image removal marks have invalid columns"
|
||||
return None
|
||||
|
||||
|
||||
def _values_error(conn: sqlite3.Connection) -> str | None:
|
||||
limit = MAX_LOCATOR_LENGTH * 4
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(locator) != 'text' " # nosec B608
|
||||
f"OR length(CAST(locator AS BLOB)) > {limit} OR locator = '' "
|
||||
"OR typeof(size) != 'integer' OR typeof(mtime_ns) != 'integer' "
|
||||
"OR typeof(removed_at) != 'text' OR length(CAST(removed_at AS BLOB)) > 64 "
|
||||
"LIMIT 1"
|
||||
).fetchone():
|
||||
return "Image removal marks contain invalid values"
|
||||
meta = conn.execute(
|
||||
f"SELECT singleton_id, typeof(revision) FROM {META_TABLE}" # nosec B608
|
||||
).fetchall()
|
||||
if meta != [(1, "integer")]:
|
||||
return "Image removal marks have an invalid revision"
|
||||
return None
|
||||
|
||||
|
||||
def image_removals_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> str | None:
|
||||
"""Reject unsupported schemas and invalid persisted values (reads only)."""
|
||||
error = _structure_error(conn, allow_missing)
|
||||
if error:
|
||||
return error
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone()
|
||||
return _values_error(conn) if exists else None
|
||||
|
||||
|
||||
def migrate_image_removals(conn: sqlite3.Connection) -> None:
|
||||
"""Install both tables atomically; validate before and after."""
|
||||
conn.execute("SAVEPOINT image_removals_schema")
|
||||
try:
|
||||
error = image_removals_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
for sql in _TABLE_DDL.values():
|
||||
conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
conn.execute(_SEED_META)
|
||||
error = image_removals_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE image_removals_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO image_removals_schema")
|
||||
conn.execute("RELEASE image_removals_schema")
|
||||
raise
|
||||
|
||||
|
||||
_UNMARK_SQL = f"DELETE FROM {TABLE} WHERE locator=?" # nosec B608
|
||||
_MARK_SQL = (
|
||||
f"INSERT OR REPLACE INTO {TABLE} " # nosec B608
|
||||
"(locator, size, mtime_ns, removed_at) VALUES (?, ?, ?, ?)"
|
||||
)
|
||||
_MARK_OF_SQL = f"SELECT size, mtime_ns FROM {TABLE} WHERE locator=?" # nosec B608
|
||||
BUMP_MARKS_SQL = (
|
||||
f"UPDATE {META_TABLE} SET revision=revision+1 WHERE singleton_id=1" # nosec B608
|
||||
)
|
||||
|
||||
|
||||
# Bumped whenever this process writes or clears a mark (even in a transaction
|
||||
# that later rolls back): readers that cache the marks reload when it moves.
|
||||
_generation = 0
|
||||
|
||||
|
||||
def generation() -> int:
|
||||
"""The marks generation of this process (see ``_generation``)."""
|
||||
return _generation
|
||||
|
||||
|
||||
def _bump() -> None:
|
||||
global _generation
|
||||
_generation += 1
|
||||
|
||||
|
||||
def bump_generation() -> None:
|
||||
"""Move the generation after a commit (a cached read made before it reloads)."""
|
||||
_bump()
|
||||
|
||||
|
||||
def has_marks_table(conn: sqlite3.Connection) -> bool:
|
||||
return (
|
||||
conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def locator_of_key(root_key: str, relative_key: str) -> str:
|
||||
"""The locator of a favorite-style file key (root key + relative key)."""
|
||||
return os.path.normcase(os.path.join(root_key, *relative_key.split("/")))
|
||||
|
||||
|
||||
def mark_of(conn: sqlite3.Connection, locator: str):
|
||||
"""``(size, mtime_ns)`` of the file's removal mark, or None."""
|
||||
if not has_marks_table(conn):
|
||||
return None
|
||||
row = conn.execute(_MARK_OF_SQL, (locator,)).fetchone()
|
||||
return (row[0], row[1]) if row else None
|
||||
|
||||
|
||||
def write_mark(
|
||||
conn: sqlite3.Connection, locator: str, size: int, mtime_ns: int, now: str
|
||||
):
|
||||
conn.execute(_MARK_SQL, (locator, int(size), int(mtime_ns), now))
|
||||
conn.execute(BUMP_MARKS_SQL)
|
||||
_bump()
|
||||
|
||||
|
||||
def clear_marks(conn: sqlite3.Connection, locators) -> int:
|
||||
"""Delete the marks of *locators* on *conn* (no commit); the count removed.
|
||||
|
||||
Every record write for a file calls this (#480): a file with records is
|
||||
in the library, so it is never hidden.
|
||||
"""
|
||||
locators = list(dict.fromkeys(locators))
|
||||
if not locators or not has_marks_table(conn):
|
||||
return 0
|
||||
removed = 0
|
||||
for locator in locators:
|
||||
removed += conn.execute(_UNMARK_SQL, (locator,)).rowcount
|
||||
if removed:
|
||||
conn.execute(BUMP_MARKS_SQL)
|
||||
_bump()
|
||||
return removed
|
||||
|
||||
|
||||
def clear_marks_of_keys(conn: sqlite3.Connection, file_keys) -> int:
|
||||
"""``clear_marks`` for favorite-style ``(root key, relative key)`` pairs."""
|
||||
return clear_marks(conn, [locator_of_key(*key) for key in file_keys])
|
||||
@@ -5,7 +5,9 @@ import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from .library_folders import folder_condition, list_library_folders
|
||||
from .library_tokens import library_edit_token
|
||||
from .prompt_archive import ARCHIVED, SHELVED
|
||||
|
||||
_ELIGIBLE = """(EXISTS(SELECT 1 FROM prompt_pairs ep WHERE ep.positive_prompt_id=p.id)
|
||||
OR EXISTS(SELECT 1 FROM generated_images eg WHERE eg.prompt_id=p.id)
|
||||
@@ -105,6 +107,12 @@ ARCHIVE_SCOPES = {
|
||||
}
|
||||
|
||||
|
||||
# Shelves (#443): the parents on the shelf, or the archived ones.
|
||||
PARENT_SCOPES = {"active": SHELVED, "archived": ARCHIVED}
|
||||
# A positive pair owner that is on the shelf, as a condition on alias pp.
|
||||
SHELVED_PAIR = SHELVED.replace("p.id", "pp.positive_prompt_id")
|
||||
|
||||
|
||||
def _escape(value: str) -> str:
|
||||
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
@@ -121,7 +129,54 @@ def _archive(value: str) -> str:
|
||||
return ARCHIVE_SCOPES[value]
|
||||
|
||||
|
||||
def _filter(text: str, category: str | None, tags: list[str]) -> tuple[str, list]:
|
||||
# How the Tags filter matches (#444): every tag (AND), any of them (OR), or
|
||||
# parents without any tag.
|
||||
TAG_MODES = ("all", "any", "untagged")
|
||||
# A parent that no tag's Browse returns (#444 "Untagged prompts"): no tag of its
|
||||
# own and no active pair carrying one. The vocabulary's untagged count uses it too.
|
||||
UNTAGGED = (
|
||||
"NOT EXISTS(SELECT 1 FROM prompt_tags ut WHERE ut.prompt_id=p.id) "
|
||||
"AND NOT EXISTS(SELECT 1 FROM prompt_pairs up JOIN prompt_pair_tags upt "
|
||||
"ON upt.pair_id=up.id WHERE up.positive_prompt_id=p.id AND up.archived=0)"
|
||||
)
|
||||
|
||||
|
||||
def _tag_mode(tag_mode: object, tags: list[str]) -> str:
|
||||
if not isinstance(tag_mode, str) or tag_mode not in TAG_MODES:
|
||||
raise ValueError("tag_mode must be one of " + ", ".join(TAG_MODES))
|
||||
if tag_mode == "untagged" and tags:
|
||||
raise ValueError("Untagged prompts cannot also filter by tags")
|
||||
return tag_mode
|
||||
|
||||
|
||||
def _tag_conditions(tags: list[str], any_tag: bool) -> tuple[list, list, list]:
|
||||
"""Parent-arm and pair-arm tag conditions and the tags' parameters."""
|
||||
parent_exists = (
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name{test})"
|
||||
)
|
||||
pair_exists = (
|
||||
"EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.pair_id=pp.id AND t.name{test})"
|
||||
)
|
||||
if any_tag and tags:
|
||||
# One condition per arm: the card, or one active pair, carries any tag.
|
||||
test = " IN (" + ",".join("?" for _ in tags) + ")"
|
||||
return [parent_exists.format(test=test)], [pair_exists.format(test=test)], tags
|
||||
return (
|
||||
[parent_exists.format(test="=?") for _ in tags],
|
||||
[pair_exists.format(test="=?") for _ in tags],
|
||||
tags,
|
||||
)
|
||||
|
||||
|
||||
def _filter(
|
||||
text: str,
|
||||
category: str | None,
|
||||
tags: list[str],
|
||||
tag_mode: str = "all",
|
||||
folder_prefixes: list[str] | None = None,
|
||||
) -> tuple[str, list]:
|
||||
if not isinstance(text, str) or len(text) > 500:
|
||||
raise ValueError("Search text must be a string of at most 500 characters")
|
||||
if category is not None and (not isinstance(category, str) or len(category) > 200):
|
||||
@@ -132,6 +187,8 @@ def _filter(text: str, category: str | None, tags: list[str]) -> tuple[str, list
|
||||
or any(not isinstance(tag, str) or not tag or len(tag) > 200 for tag in tags)
|
||||
):
|
||||
raise ValueError("Invalid library tags")
|
||||
mode = _tag_mode(tag_mode, tags)
|
||||
untagged = mode == "untagged"
|
||||
parent, pair, parent_params, pair_params = [], ["pp.archived=0"], [], []
|
||||
if text:
|
||||
parent.append("(p.text LIKE ? ESCAPE '\\' OR ls.title LIKE ? ESCAPE '\\')")
|
||||
@@ -146,17 +203,11 @@ def _filter(text: str, category: str | None, tags: list[str]) -> tuple[str, list
|
||||
pair.append("pp.category=?")
|
||||
parent_params.append(category)
|
||||
pair_params.append(category)
|
||||
for tag in tags:
|
||||
parent.append(
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name=?)"
|
||||
)
|
||||
pair.append(
|
||||
"EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.pair_id=pp.id AND t.name=?)"
|
||||
)
|
||||
parent_params.append(tag)
|
||||
pair_params.append(tag)
|
||||
parent_tags, pair_tags, tag_params = _tag_conditions(tags, mode == "any")
|
||||
parent.extend(parent_tags)
|
||||
pair.extend(pair_tags)
|
||||
parent_params.extend(tag_params)
|
||||
pair_params.extend(tag_params)
|
||||
condition = (
|
||||
"("
|
||||
+ (" AND ".join(parent) or "1=1")
|
||||
@@ -168,7 +219,17 @@ def _filter(text: str, category: str | None, tags: list[str]) -> tuple[str, list
|
||||
+ " AND ".join(pair)
|
||||
+ ")"
|
||||
)
|
||||
return f"({_ELIGIBLE}) AND ({condition})", [*parent_params, *pair_params]
|
||||
if untagged:
|
||||
# A whole-card condition: inside the parent-or-pair OR an untagged pair
|
||||
# would let a tagged parent through.
|
||||
condition = f"({condition}) AND {UNTAGGED}"
|
||||
params = [*parent_params, *pair_params]
|
||||
if folder_prefixes:
|
||||
# A whole-card condition around both arms: images link to the parent.
|
||||
folder, folder_params = folder_condition(folder_prefixes)
|
||||
condition = f"({condition}) AND {folder}"
|
||||
params.extend(folder_params)
|
||||
return f"({_ELIGIBLE}) AND ({condition})", params
|
||||
|
||||
|
||||
class LibraryOperations:
|
||||
@@ -216,6 +277,8 @@ class LibraryOperations:
|
||||
rows = conn.execute(
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,p.created_at,"
|
||||
"p.run_count,p.last_used_at,"
|
||||
+ ARCHIVED
|
||||
+ " AS archived,"
|
||||
+ (f"{sort_expression} AS _sort," if sort_expression else "")
|
||||
+ "(SELECT COUNT(*) FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
|
||||
"AND v.archived=0) AS variant_count,"
|
||||
@@ -239,6 +302,7 @@ class LibraryOperations:
|
||||
# Usage is shown, never compared: the token hashes owner fields only.
|
||||
row["run_count"] = row["run_count"] if type(row["run_count"]) is int else 0
|
||||
row["last_used_at"] = utc_iso(row["last_used_at"])
|
||||
row["archived"] = bool(row["archived"])
|
||||
row["edit_token"] = library_edit_token(epoch, row)
|
||||
return result
|
||||
|
||||
@@ -251,14 +315,21 @@ class LibraryOperations:
|
||||
text: str = "",
|
||||
category: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
tag_mode: str = "all",
|
||||
folder_prefixes: list[str] | None = None,
|
||||
sort: str = "newest",
|
||||
limit: int = 40,
|
||||
after: dict | None = None,
|
||||
shelf: str = "active",
|
||||
) -> dict:
|
||||
_limit(limit)
|
||||
if sort not in _ORDERS:
|
||||
raise ValueError("Unknown library sort order")
|
||||
where, params = _filter(text, category, [] if tags is None else tags)
|
||||
if not isinstance(shelf, str) or shelf not in PARENT_SCOPES:
|
||||
raise ValueError("shelf must be active or archived")
|
||||
where, params = _filter(
|
||||
text, category, [] if tags is None else tags, tag_mode, folder_prefixes
|
||||
)
|
||||
if after is not None:
|
||||
if (
|
||||
not isinstance(after, dict)
|
||||
@@ -270,15 +341,18 @@ class LibraryOperations:
|
||||
ceiling = after["ceiling"]
|
||||
where += " AND p.id<=?"
|
||||
params.append(ceiling)
|
||||
# One SELECT counts the filtered parents and their active variants.
|
||||
total, variant_total = conn.execute(
|
||||
"SELECT COUNT(*),COALESCE(SUM((SELECT COUNT(*) FROM prompt_pairs v "
|
||||
"WHERE v.positive_prompt_id=p.id AND v.archived=0)),0) FROM "
|
||||
+ _FROM
|
||||
+ " WHERE "
|
||||
+ where,
|
||||
# One SELECT counts the filtered parents of this scope, their active
|
||||
# variants, and the filtered archived parents (Show archived, #443).
|
||||
scope = PARENT_SCOPES[shelf]
|
||||
total, variant_total, archived_total = conn.execute(
|
||||
f"SELECT COALESCE(SUM({scope}),0),"
|
||||
f"COALESCE(SUM(CASE WHEN {scope} THEN (SELECT COUNT(*) "
|
||||
"FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
|
||||
"AND v.archived=0) ELSE 0 END),0),"
|
||||
f"COALESCE(SUM({ARCHIVED}),0) FROM " + _FROM + " WHERE " + where,
|
||||
params,
|
||||
).fetchone()
|
||||
where += f" AND {scope}"
|
||||
expression, direction = _ORDERS[sort]
|
||||
if after is not None:
|
||||
operator = ">" if direction == "ASC" else "<"
|
||||
@@ -311,6 +385,9 @@ class LibraryOperations:
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"variant_total": variant_total,
|
||||
"archived_total": archived_total,
|
||||
# Echoed so a client can tell this read applied its tag match.
|
||||
"tag_mode": tag_mode,
|
||||
"has_more": has_more,
|
||||
"next_key": next_key,
|
||||
}
|
||||
@@ -321,9 +398,12 @@ class LibraryOperations:
|
||||
text: str = "",
|
||||
category: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
tag_mode: str = "all",
|
||||
folder_prefixes: list[str] | None = None,
|
||||
sort: str = "newest",
|
||||
limit: int = 40,
|
||||
after: dict | None = None,
|
||||
shelf: str = "active",
|
||||
) -> dict:
|
||||
with self.library_snapshot(include_ceiling=True) as (conn, epoch, ceiling):
|
||||
return self._list_library(
|
||||
@@ -333,9 +413,12 @@ class LibraryOperations:
|
||||
text=text,
|
||||
category=category,
|
||||
tags=tags,
|
||||
tag_mode=tag_mode,
|
||||
folder_prefixes=folder_prefixes,
|
||||
sort=sort,
|
||||
limit=limit,
|
||||
after=after,
|
||||
shelf=shelf,
|
||||
)
|
||||
|
||||
def _is_library_positive(self, conn: sqlite3.Connection, positive_id: int) -> bool:
|
||||
@@ -463,9 +546,13 @@ class LibraryOperations:
|
||||
"categories": [row["category"] for row in rows],
|
||||
}
|
||||
|
||||
def get_library_options(self) -> dict:
|
||||
def get_library_options(self, folder_roots=None) -> dict:
|
||||
"""Categories; with *folder_roots* (root spellings) also ``folders``."""
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
return self._get_library_options(conn, epoch)
|
||||
result = self._get_library_options(conn, epoch)
|
||||
if folder_roots is not None:
|
||||
result["folders"] = list_library_folders(conn, folder_roots, _ELIGIBLE)
|
||||
return result
|
||||
|
||||
def get_library_previews(self, positive_ids: list[int]) -> dict[int, list[dict]]:
|
||||
"""Read at most two links per parent in one bounded page query."""
|
||||
@@ -576,7 +663,9 @@ class LibraryOperations:
|
||||
owners = {}
|
||||
order = []
|
||||
for row in conn.execute(
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes FROM "
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,"
|
||||
+ ARCHIVED
|
||||
+ " AS archived FROM "
|
||||
+ _FROM
|
||||
+ " WHERE "
|
||||
+ _ELIGIBLE
|
||||
@@ -589,6 +678,7 @@ class LibraryOperations:
|
||||
"tags": [],
|
||||
"rating": row["rating"],
|
||||
"notes": row["notes"],
|
||||
"archived": bool(row["archived"]),
|
||||
"variants": [],
|
||||
}
|
||||
order.append(row["id"])
|
||||
@@ -633,8 +723,9 @@ class LibraryOperations:
|
||||
"""System › Overview counts from one snapshot, writing nothing.
|
||||
|
||||
``positive_prompts`` and ``negative_variants`` are the unfiltered
|
||||
Library's ``total`` and ``variant_total``: eligible parents and active
|
||||
pairs (every pair's positive is eligible). ``images`` counts indexed
|
||||
Library's ``total`` and ``variant_total``: eligible parents on the
|
||||
shelf (not archived, #443) and their active pairs (every pair's
|
||||
positive is eligible). ``images`` counts indexed
|
||||
``generated_images`` rows, not files on disk. An image has no rating
|
||||
of its own (it shows its linked prompt's), so ``rated_images`` counts
|
||||
the indexed images whose linked prompt is rated. ``review_pending`` is
|
||||
@@ -642,8 +733,14 @@ class LibraryOperations:
|
||||
"""
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
row = conn.execute(
|
||||
"SELECT (SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE + "),"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs WHERE archived=0),"
|
||||
"SELECT (SELECT COUNT(*) FROM prompts p WHERE "
|
||||
+ _ELIGIBLE
|
||||
+ " AND "
|
||||
+ SHELVED
|
||||
+ "),"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs pp WHERE pp.archived=0 AND "
|
||||
+ SHELVED_PAIR
|
||||
+ "),"
|
||||
"(SELECT COUNT(*) FROM generated_images),"
|
||||
"(SELECT COUNT(*) FROM generated_images gi JOIN prompts r "
|
||||
"ON r.id=gi.prompt_id WHERE r.rating IS NOT NULL),"
|
||||
|
||||
@@ -0,0 +1,473 @@
|
||||
"""Delete selected Library prompts; archive the paired ones (#443, D152).
|
||||
|
||||
A selected Library parent is classified, in this order, as ``not_found``
|
||||
(missing, or not a Library parent), ``protected`` (tagged ``__protected__``;
|
||||
never changed), ``already_archived``, ``archive`` (the positive or negative of
|
||||
any saved pair, archived pair or not: archiving hides it from the shelf and
|
||||
removes nothing) or ``delete`` (in no pair: deleted exactly as the legacy
|
||||
delete does, through ``delete_prompt_rows``). Every command checks the
|
||||
storage epoch first and runs in one ``BEGIN IMMEDIATE`` transaction.
|
||||
|
||||
The write executes what the user reviewed: it is given the reviewed archive
|
||||
and delete lists, and deletes only reviewed deletes the route backed up
|
||||
(``candidates``) whose fingerprint is unchanged since that backup.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Iterable
|
||||
|
||||
from .library import _ELIGIBLE
|
||||
from .library_commands import _validate_bulk_ids, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .prompt_archive import ARCHIVED as _ARCHIVED
|
||||
from .prompt_archive import TABLE as ARCHIVE
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
# Ids bound per statement: the pair check binds each id twice, which stays
|
||||
# under SQLite's historical 999-variable limit (maintenance.py ID_BATCH).
|
||||
ID_BATCH = 400
|
||||
_PROTECTED = (
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name='__protected__')"
|
||||
)
|
||||
_PAIRED = (
|
||||
"EXISTS(SELECT 1 FROM prompt_pairs pp "
|
||||
"WHERE pp.positive_prompt_id=p.id OR pp.negative_prompt_id=p.id)"
|
||||
)
|
||||
# #482: used as the negative of another positive's pair (archived or not):
|
||||
# never deleted, so that positive's variant stays (archive only).
|
||||
_NEGATIVE_OF_OTHER = (
|
||||
"EXISTS(SELECT 1 FROM prompt_pairs op "
|
||||
"WHERE op.negative_prompt_id=p.id AND op.positive_prompt_id!=p.id)"
|
||||
)
|
||||
# Classes in the order a prompt is tested; the first that holds wins.
|
||||
CLASSES = (
|
||||
"not_found",
|
||||
"protected",
|
||||
"already_archived",
|
||||
"not_archived",
|
||||
"archive",
|
||||
"delete",
|
||||
)
|
||||
# #492: the shelf a Delete… runs on. On the archived shelf an archived prompt is
|
||||
# classified by its pairs, and one restored since is ``not_archived``.
|
||||
SHELVES = ("active", "archived")
|
||||
|
||||
|
||||
def _check_shelf(shelf: str) -> bool:
|
||||
"""Whether ``shelf`` is the archived shelf (ValueError when unknown)."""
|
||||
if shelf not in SHELVES:
|
||||
raise ValueError("shelf must be active or archived")
|
||||
return shelf == "archived"
|
||||
|
||||
|
||||
_DIGEST = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def _batches(ids: list) -> Iterable[list]:
|
||||
for start in range(0, len(ids), ID_BATCH):
|
||||
yield ids[start : start + ID_BATCH]
|
||||
|
||||
|
||||
def delete_prompt_rows(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
|
||||
"""Delete prompts and their image link rows; return the prompts deleted.
|
||||
|
||||
The legacy delete's statements (``delete_prompt``, ``bulk_delete_prompts``)
|
||||
in the caller's transaction: image link rows first (``image_prompt_pairs``
|
||||
cascades, review decisions keep a NULL image), then the prompts (tags and
|
||||
Library state cascade), then the workflow documents no row references.
|
||||
Image files are never touched. The caller checks pairs first.
|
||||
"""
|
||||
documents = documents_of_rows(conn, "prompt_id", prompt_ids)
|
||||
deleted = 0
|
||||
for chunk in _batches(list(prompt_ids)):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
conn.execute(
|
||||
f"DELETE FROM generated_images WHERE prompt_id IN ({marks})", # nosec
|
||||
chunk,
|
||||
)
|
||||
deleted += conn.execute(
|
||||
f"DELETE FROM prompts WHERE id IN ({marks})", chunk # nosec B608
|
||||
).rowcount
|
||||
release_documents(conn, documents)
|
||||
return deleted
|
||||
|
||||
|
||||
class LibraryArchiveOperations:
|
||||
"""PromptDatabase mixin: classify, archive or delete, and restore parents."""
|
||||
|
||||
def _classify_library_delete(
|
||||
self, conn: sqlite3.Connection, prompt_ids: list[int], shelf: str = "active"
|
||||
) -> dict:
|
||||
on_archived_shelf = _check_shelf(shelf)
|
||||
found, deletable = {}, set()
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for row in conn.execute(
|
||||
f"SELECT p.id,{_PROTECTED},{_ARCHIVED},{_PAIRED}," # nosec B608
|
||||
f"{_NEGATIVE_OF_OTHER} "
|
||||
f"FROM prompts p WHERE p.id IN ({marks}) AND {_ELIGIBLE}",
|
||||
chunk,
|
||||
):
|
||||
protected, archived, paired, negative_of_other = row[1:5]
|
||||
if protected:
|
||||
found[row[0]] = "protected"
|
||||
elif on_archived_shelf and not archived:
|
||||
found[row[0]] = "not_archived"
|
||||
elif archived and not on_archived_shelf:
|
||||
found[row[0]] = "already_archived"
|
||||
elif paired:
|
||||
found[row[0]] = "archive"
|
||||
if not negative_of_other:
|
||||
deletable.add(row[0])
|
||||
else:
|
||||
found[row[0]] = "delete"
|
||||
classes = {name: [] for name in CLASSES}
|
||||
for pid in prompt_ids:
|
||||
classes[found.get(pid, "not_found")].append(pid)
|
||||
# #482: the archive ids Delete permanently may delete with their pairs.
|
||||
classes["deletable"] = [pid for pid in classes["archive"] if pid in deletable]
|
||||
return classes
|
||||
|
||||
@staticmethod
|
||||
def _image_links(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
|
||||
total = 0
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
total += conn.execute(
|
||||
"SELECT COUNT(*) FROM generated_images " # nosec B608
|
||||
f"WHERE prompt_id IN ({marks})",
|
||||
chunk,
|
||||
).fetchone()[0]
|
||||
return total
|
||||
|
||||
@staticmethod
|
||||
def _fingerprints(conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
|
||||
"""What a safety backup holds of each prompt a delete would remove.
|
||||
|
||||
The prompt row (usage included), its tag ids, its Library title and
|
||||
its image link rows (count and newest id), as a JSON-safe list.
|
||||
"""
|
||||
prints = {}
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for row in conn.execute(
|
||||
"SELECT p.id,p.text,p.hash,p.category,p.rating,p.notes," # nosec
|
||||
"p.updated_at,p.run_count,p.last_used_at,"
|
||||
"(SELECT group_concat(tag_id) FROM (SELECT tag_id FROM prompt_tags "
|
||||
"WHERE prompt_id=p.id ORDER BY tag_id)),"
|
||||
"(SELECT title FROM prompt_library_state WHERE prompt_id=p.id),"
|
||||
"(SELECT COUNT(*) FROM generated_images WHERE prompt_id=p.id),"
|
||||
"(SELECT MAX(id) FROM generated_images WHERE prompt_id=p.id) "
|
||||
f"FROM prompts p WHERE p.id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
prints[row[0]] = [
|
||||
value if isinstance(value, (int, str, type(None))) else repr(value)
|
||||
for value in tuple(row)[1:]
|
||||
]
|
||||
return prints
|
||||
|
||||
@staticmethod
|
||||
def _distinct_images(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
|
||||
"""Image rows of ``prompt_ids`` (each image once; #482 deletes them)."""
|
||||
return LibraryArchiveOperations._image_links(conn, prompt_ids)
|
||||
|
||||
@staticmethod
|
||||
def _pair_prints(conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
|
||||
"""#482: each prompt's own saved pairs, as the review saw them.
|
||||
|
||||
Per pair (by id): its negative, archived flag, metadata, usage, ordered
|
||||
tag ids and its image-pair links (count and newest id).
|
||||
"""
|
||||
prints = {pid: [] for pid in prompt_ids}
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for row in conn.execute(
|
||||
"SELECT pp.positive_prompt_id,pp.id,pp.negative_prompt_id," # nosec
|
||||
"pp.archived,pp.title,pp.category,pp.rating,pp.notes,pp.run_count,"
|
||||
"pp.last_used_at,pp.updated_at,"
|
||||
"(SELECT group_concat(tag_id) FROM (SELECT tag_id FROM "
|
||||
"prompt_pair_tags WHERE pair_id=pp.id ORDER BY tag_id)),"
|
||||
"(SELECT COUNT(*) FROM image_prompt_pairs WHERE pair_id=pp.id),"
|
||||
"(SELECT MAX(id) FROM image_prompt_pairs WHERE pair_id=pp.id) "
|
||||
f"FROM prompt_pairs pp WHERE pp.positive_prompt_id IN ({marks}) "
|
||||
"ORDER BY pp.id",
|
||||
chunk,
|
||||
):
|
||||
prints[row[0]].append(
|
||||
[
|
||||
v if isinstance(v, (int, str, type(None))) else repr(v)
|
||||
for v in tuple(row)[1:]
|
||||
]
|
||||
)
|
||||
return prints
|
||||
|
||||
@classmethod
|
||||
def _review_digests(cls, conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
|
||||
"""#482: one digest per deletable prompt of its row and its pairs."""
|
||||
prompts = cls._fingerprints(conn, prompt_ids)
|
||||
pairs = cls._pair_prints(conn, prompt_ids)
|
||||
return {
|
||||
pid: hashlib.sha256(
|
||||
json.dumps(
|
||||
[prompts.get(pid), pairs[pid]], separators=(",", ":")
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
for pid in prompt_ids
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _paired_counts(
|
||||
conn: sqlite3.Connection, prompt_ids: list[int], selected: list[int]
|
||||
) -> dict:
|
||||
"""#482: what Delete permanently would remove with ``prompt_ids``."""
|
||||
pairs = archived = variants = 0
|
||||
negatives = set()
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for archived_flag, negative, positive in conn.execute(
|
||||
"SELECT archived,negative_prompt_id,positive_prompt_id " # nosec
|
||||
f"FROM prompt_pairs WHERE positive_prompt_id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
pairs += 1
|
||||
archived += 1 if archived_flag else 0
|
||||
if negative is not None:
|
||||
variants += 1
|
||||
if negative != positive:
|
||||
negatives.add(negative)
|
||||
return {
|
||||
"pairs": pairs,
|
||||
"archived_pairs": archived,
|
||||
"variants": variants,
|
||||
"kept_negatives": len(negatives - set(selected)),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _checked_ids(prompt_ids: list[int], expected_epoch: str) -> list[int]:
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_bulk_ids(prompt_ids)
|
||||
return list(dict.fromkeys(prompt_ids))
|
||||
|
||||
def classify_library_delete(
|
||||
self, prompt_ids: list[int], *, expected_epoch: str, shelf: str = "active"
|
||||
) -> dict:
|
||||
"""What a delete of ``prompt_ids`` would do now (reads only).
|
||||
|
||||
Returns the id lists per class, in request order, ``image_links``
|
||||
(the image link rows the ``delete`` class would remove) and
|
||||
``fingerprints`` of the ``delete`` class (``_fingerprints``). On the
|
||||
archived ``shelf`` (#492) archived prompts are classified by their pairs.
|
||||
"""
|
||||
_check_shelf(shelf)
|
||||
prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
classes = self._classify_library_delete(conn, prompt_ids, shelf)
|
||||
deletable = classes["deletable"]
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
**classes,
|
||||
"image_links": self._image_links(conn, classes["delete"]),
|
||||
"fingerprints": self._fingerprints(conn, classes["delete"]),
|
||||
# #482: Delete permanently, with the deletable prompts' pairs.
|
||||
"review_digests": self._review_digests(conn, deletable),
|
||||
"paired_counts": self._paired_counts(conn, deletable, prompt_ids),
|
||||
"paired_image_links": self._distinct_images(conn, deletable),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _repair_hash(conn: sqlite3.Connection, prompt_id: int) -> None:
|
||||
"""Give an archived prompt its canonical hash, so ingestion finds it.
|
||||
|
||||
Scans, the monitor and node saves resolve prompts by hash; a stale or
|
||||
missing one would make them insert a visible copy (Q3). When another
|
||||
row already holds the canonical hash, lookups find that row and no new
|
||||
copy is made, so nothing is written.
|
||||
"""
|
||||
text, stored = conn.execute(
|
||||
"SELECT text,hash FROM prompts WHERE id=?", (prompt_id,)
|
||||
).fetchone()
|
||||
canonical = generate_prompt_hash(text) if isinstance(text, str) else None
|
||||
if canonical is None or stored == canonical:
|
||||
return
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE hash=? AND id!=?", (canonical, prompt_id)
|
||||
).fetchone():
|
||||
return
|
||||
conn.execute("UPDATE prompts SET hash=? WHERE id=?", (canonical, prompt_id))
|
||||
|
||||
def archive_or_delete_library_prompts(
|
||||
self,
|
||||
archive_ids: list[int],
|
||||
delete_ids: list[int],
|
||||
*,
|
||||
expected_epoch: str,
|
||||
candidates: dict | None,
|
||||
delete_paired: dict | None = None,
|
||||
shelf: str = "active",
|
||||
) -> dict:
|
||||
"""Archive and delete what the user reviewed, all or nothing.
|
||||
|
||||
``archive_ids`` and ``delete_ids`` are the reviewed classes.
|
||||
``candidates`` maps each reviewed delete the route backed up to its
|
||||
fingerprint then (None: every reviewed delete, unchecked; tests and
|
||||
callers without a backup). A reviewed delete is deleted only when it
|
||||
still classifies ``delete`` and its fingerprint is unchanged; one that
|
||||
is now paired, and a reviewed archive now unpaired, are archived (the
|
||||
restorable direction); anything else is kept and counted.
|
||||
|
||||
``delete_paired`` (#482) maps reviewed archive ids the user chose to
|
||||
Delete permanently to their review digest. Each is deleted with its
|
||||
own saved pairs only while it is still deletable and its digest is
|
||||
unchanged; otherwise it is archived and counted ``changed``.
|
||||
|
||||
On the archived ``shelf`` (#492) the ids are archived already: nothing
|
||||
is archived or rewritten, every reviewed id not deleted (protected or
|
||||
changed) counts ``changed``, and one restored since ``not_archived``.
|
||||
"""
|
||||
on_archived_shelf = _check_shelf(shelf)
|
||||
_validate_epoch(expected_epoch)
|
||||
for ids in (archive_ids, delete_ids):
|
||||
if not isinstance(ids, list) or any(
|
||||
type(pid) is not int or not 1 <= pid <= 2**63 - 1 for pid in ids
|
||||
):
|
||||
raise ValueError("prompt ids must be lists of positive integers")
|
||||
reviewed_delete = set(delete_ids)
|
||||
delete_paired = {} if delete_paired is None else delete_paired
|
||||
if not isinstance(delete_paired, dict) or any(
|
||||
type(pid) is not int
|
||||
or not isinstance(digest, str)
|
||||
or not _DIGEST.match(digest)
|
||||
for pid, digest in delete_paired.items()
|
||||
):
|
||||
raise ValueError("delete_paired must map prompt ids to review digests")
|
||||
if not set(delete_paired) <= set(archive_ids):
|
||||
raise ValueError("A permanent delete must be a reviewed archive id")
|
||||
if not archive_ids and not delete_ids:
|
||||
raise ValueError("Nothing to delete or archive")
|
||||
if reviewed_delete & set(archive_ids):
|
||||
raise ValueError("A prompt cannot be both archived and deleted")
|
||||
prompt_ids = list(dict.fromkeys([*archive_ids, *delete_ids]))
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
classes = self._classify_library_delete(conn, prompt_ids, shelf)
|
||||
now_prints = self._fingerprints(conn, classes["delete"])
|
||||
deletes, archives, changed = [], [], 0
|
||||
paired_now = set(classes["deletable"])
|
||||
chosen = [pid for pid in classes["archive"] if pid in delete_paired]
|
||||
digests = self._review_digests(
|
||||
conn, [pid for pid in chosen if pid in paired_now]
|
||||
)
|
||||
paired_deletes = []
|
||||
for pid in classes["archive"]:
|
||||
if pid not in delete_paired:
|
||||
archives.append(pid)
|
||||
elif digests.get(pid) == delete_paired[pid]:
|
||||
paired_deletes.append(pid)
|
||||
else:
|
||||
archives.append(pid) # changed since the review: restorable
|
||||
changed += 1
|
||||
for pid in classes["delete"]:
|
||||
if pid not in reviewed_delete:
|
||||
archives.append(pid) # reviewed as archive, now unpaired
|
||||
changed += 1 if pid in delete_paired else 0
|
||||
elif candidates is None or candidates.get(pid) == now_prints[pid]:
|
||||
deletes.append(pid)
|
||||
else:
|
||||
changed += 1
|
||||
changed += len(classes["protected"])
|
||||
if on_archived_shelf:
|
||||
# Already archived: the rest stay as they are, each one counted.
|
||||
archives = []
|
||||
changed = len(prompt_ids) - len(deletes) - len(paired_deletes)
|
||||
changed -= len(classes["not_found"]) + len(classes["not_archived"])
|
||||
image_links = self._image_links(conn, [*deletes, *paired_deletes])
|
||||
now = utc_now_iso()
|
||||
for pid in archives:
|
||||
self._repair_hash(conn, pid)
|
||||
conn.executemany(
|
||||
f"INSERT INTO {ARCHIVE}(prompt_id, archived_at) VALUES (?, ?)",
|
||||
[(pid, now) for pid in archives],
|
||||
)
|
||||
# A changed row is the newest edit the restore fingerprint reads.
|
||||
conn.executemany(
|
||||
"UPDATE prompts SET updated_at=? WHERE id=?",
|
||||
[(now, pid) for pid in archives],
|
||||
)
|
||||
# #482: the image-pair links (RESTRICT) go first, then the pairs
|
||||
# (pair tags CASCADE, review decisions SET NULL), then the prompts.
|
||||
pairs_removed = 0
|
||||
for chunk in _batches(paired_deletes):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
pairs_of = (
|
||||
"SELECT id FROM prompt_pairs " # nosec B608
|
||||
f"WHERE positive_prompt_id IN ({marks})"
|
||||
)
|
||||
conn.execute(
|
||||
f"DELETE FROM image_prompt_pairs WHERE pair_id IN ({pairs_of})",
|
||||
chunk,
|
||||
)
|
||||
pairs_removed += conn.execute(
|
||||
"DELETE FROM prompt_pairs " # nosec B608
|
||||
f"WHERE positive_prompt_id IN ({marks})",
|
||||
chunk,
|
||||
).rowcount
|
||||
deleted = delete_prompt_rows(conn, [*deletes, *paired_deletes])
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"deleted": deleted,
|
||||
"archived": len(archives),
|
||||
"already_archived": len(classes["already_archived"]),
|
||||
"not_archived": len(classes["not_archived"]),
|
||||
"not_found": len(classes["not_found"]),
|
||||
"changed": changed,
|
||||
"image_links": image_links,
|
||||
"pairs_removed": pairs_removed,
|
||||
}
|
||||
|
||||
def restore_library_prompts(
|
||||
self, prompt_ids: list[int], *, expected_epoch: str
|
||||
) -> dict:
|
||||
"""Put archived prompts back on the shelf, all or nothing.
|
||||
|
||||
Every requested id with an archive row is restored, whatever its
|
||||
protection or Library parent state: restoring only un-hides it.
|
||||
"""
|
||||
prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
restored, existing = [], set()
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for pid, archived in conn.execute(
|
||||
f"SELECT p.id,{_ARCHIVED} FROM prompts p " # nosec B608
|
||||
f"WHERE p.id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
existing.add(pid)
|
||||
if archived:
|
||||
restored.append(pid)
|
||||
now = utc_now_iso()
|
||||
conn.executemany(
|
||||
f"DELETE FROM {ARCHIVE} WHERE prompt_id=?", [(pid,) for pid in restored]
|
||||
)
|
||||
conn.executemany(
|
||||
"UPDATE prompts SET updated_at=? WHERE id=?",
|
||||
[(now, pid) for pid in restored],
|
||||
)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"restored": len(restored),
|
||||
"unchanged": len(existing) - len(restored),
|
||||
"not_found": len(prompt_ids) - len(existing),
|
||||
}
|
||||
@@ -21,6 +21,10 @@ _UUID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
|
||||
_TOKEN = re.compile(r"[0-9a-f]{64}")
|
||||
# Parent metadata stored on the prompt row; title lives in prompt_library_state.
|
||||
_PROMPT_COLUMNS = ("category", "rating", "notes")
|
||||
# Ids per IN (...) read: well under SQLite's oldest bound-parameter limit (999).
|
||||
_BULK_CHUNK = 500
|
||||
# The tags one prompt may carry: what the editor's save accepts (``validate_tags``).
|
||||
MAX_LIBRARY_TAGS = 20
|
||||
|
||||
|
||||
class LibraryCommandError(Exception):
|
||||
@@ -85,6 +89,13 @@ def _validate_epoch(expected_epoch: str) -> None:
|
||||
raise ValueError("expected_epoch must be a canonical UUID")
|
||||
|
||||
|
||||
def _validate_bulk_ids(prompt_ids: list[int]) -> None:
|
||||
if not isinstance(prompt_ids, list) or not prompt_ids:
|
||||
raise ValueError("prompt_ids must be a non-empty list")
|
||||
if any(type(pid) is not int or not 1 <= pid <= 2**63 - 1 for pid in prompt_ids):
|
||||
raise ValueError("prompt_ids must be positive integers")
|
||||
|
||||
|
||||
def _validate_guards(positive_id: int, expected_epoch: str, if_match: str) -> None:
|
||||
if type(positive_id) is not int or not 1 <= positive_id <= 2**63 - 1:
|
||||
raise ValueError("Invalid positive prompt ID")
|
||||
@@ -308,3 +319,134 @@ class LibraryCommandOperations:
|
||||
positive_created=False,
|
||||
pair_created=pair_created,
|
||||
)
|
||||
|
||||
def bulk_set_library_category(
|
||||
self, prompt_ids: list[int], category: str, *, expected_epoch: str
|
||||
) -> dict:
|
||||
"""Set one category on many Library parents, all or nothing (#437).
|
||||
|
||||
Guarded by the storage epoch the page read. Only rows whose category
|
||||
differs are written (their ``updated_at`` and edit token change) and
|
||||
registered as Library parents, as the editor's PATCH does; ids
|
||||
that are missing or are not Library parents count as not found.
|
||||
"""
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_bulk_ids(prompt_ids)
|
||||
prompt_ids = list(dict.fromkeys(prompt_ids))
|
||||
if not isinstance(category, str) or not category.strip():
|
||||
raise ValueError("Category is required")
|
||||
validate_category(category)
|
||||
category = category.strip()
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
current = {}
|
||||
for start in range(0, len(prompt_ids), _BULK_CHUNK):
|
||||
chunk = prompt_ids[start : start + _BULK_CHUNK]
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
current.update(
|
||||
conn.execute(
|
||||
f"SELECT p.id,p.category FROM prompts p " # nosec B608
|
||||
f"WHERE p.id IN ({marks}) AND {_ELIGIBLE}",
|
||||
chunk,
|
||||
).fetchall()
|
||||
)
|
||||
changed = [pid for pid, value in current.items() if value != category]
|
||||
for pid in changed:
|
||||
self._ensure_library_parent(conn, pid)
|
||||
now = utc_now_iso()
|
||||
conn.executemany(
|
||||
"UPDATE prompts SET category=?,updated_at=? WHERE id=?",
|
||||
[(category, now, pid) for pid in changed],
|
||||
)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"category": category,
|
||||
"updated": len(changed),
|
||||
"unchanged": len(current) - len(changed),
|
||||
"not_found": len(prompt_ids) - len(current),
|
||||
}
|
||||
|
||||
def bulk_add_library_tags(
|
||||
self, prompt_ids: list[int], tags: list[str], *, expected_epoch: str
|
||||
) -> dict:
|
||||
"""Add tags to many Library parents, all or nothing (#442).
|
||||
|
||||
Names are normalized once (trim, the vocabulary name rules, casefold
|
||||
repeats folded, at most 20). A parent is given only the names it lacks,
|
||||
compared with ``casefold`` against its own tags; registry rows are
|
||||
found and inserted by their exact name, and only for names some parent
|
||||
gets. A parent whose tags would pass 20 is skipped (``over_limit``) so
|
||||
the editor can still save it. Pair and image tags are not touched.
|
||||
"""
|
||||
from .annotations import ImageTagLimitError, normalize_image_tags
|
||||
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_bulk_ids(prompt_ids)
|
||||
prompt_ids = list(dict.fromkeys(prompt_ids))
|
||||
too_many = f"At most {MAX_LIBRARY_TAGS} tags can be added at once"
|
||||
try:
|
||||
tags = normalize_image_tags(tags)
|
||||
except ImageTagLimitError:
|
||||
# Its message names images; this is a prompt route.
|
||||
raise ValueError(too_many) from None
|
||||
if not tags:
|
||||
raise ValueError("tags must name at least one tag")
|
||||
if len(tags) > MAX_LIBRARY_TAGS:
|
||||
raise ValueError(too_many)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
held = self._library_parent_tags(conn, prompt_ids)
|
||||
additions, over_limit = {}, 0
|
||||
for pid, names in held.items():
|
||||
present = {name.casefold() for name in names}
|
||||
missing = [name for name in tags if name.casefold() not in present]
|
||||
if not missing:
|
||||
continue
|
||||
if len(names) + len(missing) > MAX_LIBRARY_TAGS:
|
||||
over_limit += 1
|
||||
continue
|
||||
additions[pid] = missing
|
||||
wanted = list(
|
||||
dict.fromkeys(n for names in additions.values() for n in names)
|
||||
)
|
||||
tag_ids = self._ensure_tags(conn, wanted)
|
||||
now = utc_now_iso()
|
||||
for pid, names in additions.items():
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO prompt_tags (prompt_id, tag_id) "
|
||||
"VALUES (?, ?)",
|
||||
[(pid, tag_ids[name]) for name in names],
|
||||
)
|
||||
self._ensure_library_parent(conn, pid)
|
||||
conn.execute("UPDATE prompts SET updated_at=? WHERE id=?", (now, pid))
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"tags": tags,
|
||||
"updated": len(additions),
|
||||
"unchanged": len(held) - len(additions) - over_limit,
|
||||
"not_found": len(prompt_ids) - len(held),
|
||||
"over_limit": over_limit,
|
||||
}
|
||||
|
||||
def _library_parent_tags(
|
||||
self, conn: sqlite3.Connection, prompt_ids: list[int]
|
||||
) -> dict[int, list[str]]:
|
||||
"""The Library parents among ``prompt_ids`` and their own tag names."""
|
||||
held: dict[int, list[str]] = {}
|
||||
for start in range(0, len(prompt_ids), _BULK_CHUNK):
|
||||
chunk = prompt_ids[start : start + _BULK_CHUNK]
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for (pid,) in conn.execute(
|
||||
f"SELECT p.id FROM prompts p WHERE p.id IN ({marks}) " # nosec B608
|
||||
f"AND {_ELIGIBLE}",
|
||||
chunk,
|
||||
):
|
||||
held[pid] = []
|
||||
for pid, name in conn.execute(
|
||||
"SELECT pt.prompt_id,t.name FROM prompt_tags pt " # nosec B608
|
||||
f"JOIN tags t ON t.id=pt.tag_id WHERE pt.prompt_id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
if pid in held:
|
||||
held[pid].append(name)
|
||||
return held
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Library output-folder filter (#444): folder names, root prefixes and listing.
|
||||
|
||||
A folder is a ``/``-joined path relative to an output root, as Images' folder
|
||||
filter and the legacy ``/prompt_manager/subfolders`` name them. A parent
|
||||
matches a folder when one of its linked images (``generated_images.prompt_id``,
|
||||
the link its previews and image count read) lies in that folder or below it,
|
||||
under any spelling of any output root.
|
||||
|
||||
Matching compares ``generated_images.file_path``, which is always
|
||||
``normcase(normpath(abspath(path)))`` (``models.normalize_image_path``), with
|
||||
prefixes built by the same normalization here, so case folding is the
|
||||
platform's own on both sides (Windows folds non-ASCII too, unlike SQLite's
|
||||
``lower()``) and the comparison is an exact ``substr`` with no wildcards.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
_MAX_FOLDER = 1024
|
||||
_MAX_PREFIXES = 64
|
||||
|
||||
|
||||
def normalize_folder(value: object, pathmod=os.path) -> str:
|
||||
"""The folder as ``a/b``, outer slashes dropped; empty means no filter.
|
||||
|
||||
``/`` always separates. ``\\`` separates only on Windows (``ntpath``): on
|
||||
POSIX it is an ordinary character of a folder's name, and the listing
|
||||
(``relative_folder``) names such a folder with it, so both agree.
|
||||
``.``/``..``/empty segments, control characters and (on Windows) a drive
|
||||
are refused: the value only ever names a folder below a root.
|
||||
"""
|
||||
if not isinstance(value, str) or len(value) > _MAX_FOLDER:
|
||||
raise ValueError(f"folder must be text of at most {_MAX_FOLDER} characters")
|
||||
windows = pathmod.sep == "\\"
|
||||
folder = (value.replace("\\", "/") if windows else value).strip("/")
|
||||
if not folder:
|
||||
return ""
|
||||
parts = folder.split("/")
|
||||
if (
|
||||
any(part in ("", ".", "..") for part in parts)
|
||||
or (windows and ":" in parts[0])
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in folder)
|
||||
):
|
||||
raise ValueError("folder must be a folder below an output root")
|
||||
return folder
|
||||
|
||||
|
||||
def _base(root: object, pathmod) -> str:
|
||||
return pathmod.normcase(pathmod.normpath(str(root)))
|
||||
|
||||
|
||||
def folder_prefixes(roots, folder: str, pathmod=os.path) -> list[str]:
|
||||
"""One ``file_path`` prefix per root spelling for a normalized folder."""
|
||||
prefixes = []
|
||||
for root in roots:
|
||||
joined = pathmod.join(_base(root, pathmod), *folder.split("/"))
|
||||
prefix = pathmod.normcase(pathmod.normpath(joined)) + pathmod.sep
|
||||
if prefix not in prefixes:
|
||||
prefixes.append(prefix)
|
||||
if not prefixes or len(prefixes) > _MAX_PREFIXES:
|
||||
raise ValueError("No output folder to filter by")
|
||||
return prefixes
|
||||
|
||||
|
||||
def folder_condition(prefixes: list[str]) -> tuple[str, list]:
|
||||
"""A condition on alias ``p``: one linked image under any prefix."""
|
||||
tests = " OR ".join("substr(fg.file_path,1,?)=?" for _ in prefixes)
|
||||
params = [value for prefix in prefixes for value in (len(prefix), prefix)]
|
||||
return (
|
||||
"EXISTS(SELECT 1 FROM generated_images fg WHERE fg.prompt_id=p.id "
|
||||
f"AND ({tests}))",
|
||||
params,
|
||||
)
|
||||
|
||||
|
||||
def relative_folder(image_path: str, roots, pathmod=os.path) -> str | None:
|
||||
"""The folder of *image_path* below the first root spelling holding it.
|
||||
|
||||
The spelling of the stored path is kept (its case on Windows); ``None``
|
||||
for an image directly in a root or outside every root.
|
||||
"""
|
||||
directory = pathmod.normpath(pathmod.dirname(image_path))
|
||||
folded = pathmod.normcase(directory)
|
||||
for root in roots:
|
||||
base = _base(root, pathmod)
|
||||
start = base if base.endswith(pathmod.sep) else base + pathmod.sep
|
||||
if not folded.startswith(start):
|
||||
continue
|
||||
# normcase keeps the length for ordinary names; else show the folded form.
|
||||
rest = (directory if len(directory) == len(folded) else folded)[len(start) :]
|
||||
parts = rest.split(pathmod.sep)
|
||||
if any(part in ("", ".", "..") for part in parts):
|
||||
return None
|
||||
return "/".join(parts)
|
||||
return None
|
||||
|
||||
|
||||
def list_library_folders(
|
||||
conn: sqlite3.Connection, roots, eligible: str, pathmod=os.path
|
||||
) -> list[str]:
|
||||
"""Every folder (and ancestor) holding an image of an eligible parent."""
|
||||
folders = set()
|
||||
for (image_path,) in conn.execute(
|
||||
"SELECT DISTINCT g.image_path FROM generated_images g "
|
||||
"JOIN prompts p ON p.id=g.prompt_id WHERE " + eligible
|
||||
):
|
||||
if not isinstance(image_path, str) or not image_path:
|
||||
continue
|
||||
folder = relative_folder(image_path, roots, pathmod)
|
||||
if folder is None:
|
||||
continue
|
||||
parts = folder.split("/")
|
||||
folders.update("/".join(parts[:index]) for index in range(1, len(parts) + 1))
|
||||
return sorted(folders, key=lambda value: (value.casefold(), value))
|
||||
@@ -0,0 +1,831 @@
|
||||
"""Move library (#249 S6b): copy the node-folder library, verify it, switch.
|
||||
|
||||
``plan_move`` checks a ``utils.library_location`` location can move and
|
||||
sizes the copy; ``move_library`` does it, in this order:
|
||||
|
||||
1. **Disk check** on the target's volume (the copy, the backups and
|
||||
config.json, plus a margin). The source is never renamed across folders,
|
||||
so another drive is simply a copy.
|
||||
2. **Copy** with SQLite's backup API from a private connection holding one
|
||||
read transaction: the per-table counts, the workflow documents' digest
|
||||
and ``PRAGMA data_version`` describe exactly the copied snapshot. The copy
|
||||
is a temp file in the target folder, made self-contained
|
||||
(``journal_mode=DELETE``), fsynced and owner-only.
|
||||
3. **Verify**: ``verify_database_file`` (integrity check, schema, every
|
||||
workflow document decompressed against its hash, #315), the same row
|
||||
count in every table and the same documents.
|
||||
4. **Copy files**: every pooled safety backup and pre-migration backup,
|
||||
checked by size and SHA-256 after the copy (their times are kept, so the
|
||||
pool prunes in the same order).
|
||||
5. **Publish** under the old database's write lock (``BEGIN IMMEDIATE``) and
|
||||
the config.json lock: if any other connection wrote since the snapshot
|
||||
(``data_version``), steps 2–4 run again (``attempts``, then
|
||||
``library-busy``). Else config.json is copied (owner-only, checked), a
|
||||
``pending`` marker is written, the temp files take their final names
|
||||
(``os.replace`` inside the target folder), the marker turns ``moved``
|
||||
and ``switch`` points the process at the new library while the old file
|
||||
is still locked, so no write can land in it.
|
||||
6. **Retire**: the old file's WAL is checkpointed and the old ``prompts.db``
|
||||
and ``config.json`` are renamed ``*.moved-to-<folder>`` (never deleted;
|
||||
the old backups stay where they are). A file still open elsewhere is left
|
||||
in place and reported (``old-in-use``): the marker already makes a restart
|
||||
use the new library.
|
||||
|
||||
Any failure before the marker says ``moved`` removes everything the move
|
||||
wrote in the target folder and leaves the old library in use, untouched.
|
||||
A crash at any point leaves either the old library in use (the marker is
|
||||
absent or ``pending``; the next move replaces its own leftovers) or the new
|
||||
one (``moved``). ``MoveFailed`` carries a code and a message without paths.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import time
|
||||
import types
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from .models import BUSY_TIMEOUT_SECONDS, PromptModel
|
||||
|
||||
try:
|
||||
from ..utils import library_location as ll
|
||||
from ..utils.logging_config import get_logger
|
||||
from ..utils.workspace_keys import KEY_FILE_NAME
|
||||
except ImportError:
|
||||
from utils import library_location as ll
|
||||
from utils.logging_config import get_logger
|
||||
from utils.workspace_keys import KEY_FILE_NAME
|
||||
|
||||
logger = get_logger("prompt_manager.database.library_move")
|
||||
|
||||
TEMP_PREFIX = ".pm-move-"
|
||||
MOVED_INFIX = ".moved-to-"
|
||||
DISK_MARGIN_BYTES = 16 * 2**20
|
||||
DEFAULT_ATTEMPTS = 3
|
||||
_COPY_CHUNK = 4 * 2**20
|
||||
# Backups that live beside the database and move with it.
|
||||
BACKUP_INFIXES = (".backup_", ".pre-workflow-storage_")
|
||||
|
||||
MESSAGES = {
|
||||
"not-movable": (
|
||||
"This library is not in the node folder, so there is nothing to move."
|
||||
),
|
||||
"same-folder": "The new folder is the node folder itself.",
|
||||
"target-invalid": "The new folder cannot hold the library.",
|
||||
"target-has-library": (
|
||||
"The new folder already holds a Prompt Manager library. Nothing was moved."
|
||||
),
|
||||
"target-conflict": (
|
||||
"The new folder already holds files with the same names. Nothing was moved."
|
||||
),
|
||||
"insufficient-space": (
|
||||
"There is not enough free space for the copy. Nothing was moved."
|
||||
),
|
||||
"copy-failed": "The copy could not be written or did not match. Nothing was moved.",
|
||||
"verify-failed": "The copy did not pass verification. Nothing was moved.",
|
||||
"library-busy": (
|
||||
"Prompt Manager kept writing to the library while it was copied. "
|
||||
"Nothing was moved; try again when no workflow is running."
|
||||
),
|
||||
"switch-failed": (
|
||||
"Prompt Manager could not switch to the verified copy, so the copy was "
|
||||
"removed. Nothing was moved."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class MoveFailed(Exception):
|
||||
"""A move that did not happen (or did not finish), with a coded reason."""
|
||||
|
||||
def __init__(self, code: str, message: Optional[str] = None):
|
||||
self.code = code
|
||||
self.message = message or MESSAGES.get(code, "The library was not moved.")
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MovePlan:
|
||||
location: "ll.LibraryLocation"
|
||||
db_path: str
|
||||
source_dir: str
|
||||
target_dir: str
|
||||
config_path: Optional[str]
|
||||
backups: Tuple[str, ...]
|
||||
db_bytes: int
|
||||
files_bytes: int
|
||||
|
||||
@property
|
||||
def required_bytes(self) -> int:
|
||||
return self.db_bytes + self.files_bytes + DISK_MARGIN_BYTES
|
||||
|
||||
@property
|
||||
def target_location(self) -> "ll.LibraryLocation":
|
||||
"""Where the library is once moved (an explicit config.json stays)."""
|
||||
source = self.location.target_source or ll.SOURCE_DEFAULT
|
||||
config = (
|
||||
self.location.config_path
|
||||
if self.location.config_explicit
|
||||
else os.path.join(self.target_dir, ll.CONFIG_NAME)
|
||||
)
|
||||
return ll.LibraryLocation(
|
||||
source,
|
||||
os.path.join(self.target_dir, ll.DATABASE_NAME),
|
||||
config,
|
||||
self.target_dir,
|
||||
self.target_dir,
|
||||
None,
|
||||
source,
|
||||
self.location.config_explicit,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MoveResult:
|
||||
location: "ll.LibraryLocation"
|
||||
counts: Dict[str, int]
|
||||
files: List[str]
|
||||
renamed: List[str] = field(default_factory=list)
|
||||
retire_problem: Optional[str] = None
|
||||
|
||||
|
||||
def _size(path: str) -> int:
|
||||
try:
|
||||
return os.path.getsize(path)
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def _free_bytes(path: str) -> int:
|
||||
return shutil.disk_usage(path).free
|
||||
|
||||
|
||||
def _existing_ancestor(path: str) -> str:
|
||||
current = os.path.abspath(path)
|
||||
while not os.path.isdir(current):
|
||||
parent = os.path.dirname(current)
|
||||
if parent == current:
|
||||
break
|
||||
current = parent
|
||||
return current
|
||||
|
||||
|
||||
def backup_names(db_path: str) -> Tuple[str, ...]:
|
||||
"""The pooled and pre-migration backups beside ``db_path`` (names)."""
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
base = os.path.basename(db_path)
|
||||
prefixes = tuple(base + infix for infix in BACKUP_INFIXES)
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return ()
|
||||
return tuple(
|
||||
sorted(
|
||||
name
|
||||
for name in names
|
||||
if name.startswith(prefixes) and os.path.isfile(os.path.join(folder, name))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def plan_move(location) -> MovePlan:
|
||||
"""Check ``location`` can move to its target; raises MoveFailed."""
|
||||
if not location.movable:
|
||||
raise MoveFailed("not-movable")
|
||||
target = os.path.abspath(location.target_dir)
|
||||
source_dir = os.path.dirname(os.path.abspath(location.db_path))
|
||||
if ll.same_folder(target, source_dir) or os.path.normcase(
|
||||
os.path.realpath(target)
|
||||
) == os.path.normcase(os.path.realpath(source_dir)):
|
||||
raise MoveFailed("same-folder")
|
||||
if os.path.exists(target) and not os.path.isdir(target):
|
||||
raise MoveFailed("target-invalid")
|
||||
backups = backup_names(location.db_path)
|
||||
# config.json moves with a node-folder library; one named by
|
||||
# PROMPT_MANAGER_CONFIG_PATH is the user's and stays where it is.
|
||||
config = None if location.config_explicit else location.config_path
|
||||
config = config if config and os.path.isfile(config) else None
|
||||
if ll.read_marker(target).get("state") != ll.MARKER_PENDING:
|
||||
if os.path.exists(os.path.join(target, ll.DATABASE_NAME)):
|
||||
raise MoveFailed("target-has-library")
|
||||
names = [ll.CONFIG_NAME] if config else []
|
||||
if any(os.path.exists(os.path.join(target, n)) for n in names + list(backups)):
|
||||
raise MoveFailed("target-conflict")
|
||||
db_bytes = _size(location.db_path) + _size(location.db_path + "-wal")
|
||||
files_bytes = sum(_size(os.path.join(source_dir, n)) for n in backups)
|
||||
files_bytes += _size(config) if config else 0
|
||||
return MovePlan(
|
||||
location,
|
||||
os.path.abspath(location.db_path),
|
||||
source_dir,
|
||||
target,
|
||||
config,
|
||||
backups,
|
||||
db_bytes,
|
||||
files_bytes,
|
||||
)
|
||||
|
||||
|
||||
def space_problem(plan: MovePlan) -> Optional[Tuple[int, int]]:
|
||||
"""``(required, free)`` when the target's volume is short, else None."""
|
||||
free = _free_bytes(_existing_ancestor(plan.target_dir))
|
||||
if free < plan.required_bytes:
|
||||
return plan.required_bytes, free
|
||||
return None
|
||||
|
||||
|
||||
# ── copy and verify ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _tables(conn) -> List[str]:
|
||||
return [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' "
|
||||
"AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _describe(conn) -> Tuple[Dict[str, int], str]:
|
||||
"""Row counts per table and a digest of the workflow documents."""
|
||||
counts = {}
|
||||
for table in _tables(conn):
|
||||
# A name read from sqlite_master, quoted as an identifier.
|
||||
quoted = '"' + table.replace('"', '""') + '"'
|
||||
sql = f"SELECT COUNT(*) FROM {quoted}" # nosec B608
|
||||
counts[table] = conn.execute(sql).fetchone()[0]
|
||||
digest = hashlib.sha256()
|
||||
if "workflow_documents" in counts:
|
||||
for row in conn.execute(
|
||||
"SELECT id, sha256, byte_length FROM workflow_documents ORDER BY id"
|
||||
):
|
||||
digest.update(repr((row[0], bytes(row[1]), row[2])).encode())
|
||||
return counts, digest.hexdigest()
|
||||
|
||||
|
||||
def _fsync_file(path: str) -> None:
|
||||
with open(path, "rb+") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
|
||||
def _fsync_dir(path: str) -> None:
|
||||
if os.name != "posix":
|
||||
return
|
||||
fd = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _private(path: str) -> None:
|
||||
if os.name == "posix":
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _copy_snapshot(source: sqlite3.Connection, temp: str):
|
||||
"""Back up ``source``'s current snapshot to ``temp``; return what it holds."""
|
||||
source.execute("BEGIN")
|
||||
try:
|
||||
version = source.execute("PRAGMA data_version").fetchone()[0]
|
||||
counts, documents = _describe(source)
|
||||
destination = sqlite3.connect(temp)
|
||||
try:
|
||||
source.backup(destination)
|
||||
finally:
|
||||
destination.close()
|
||||
finally:
|
||||
source.execute("COMMIT")
|
||||
with contextlib.closing(sqlite3.connect(temp)) as conn:
|
||||
conn.execute("PRAGMA journal_mode=DELETE").fetchone()
|
||||
_fsync_file(temp)
|
||||
_private(temp)
|
||||
return version, counts, documents
|
||||
|
||||
|
||||
def _verify_copy(temp: str, counts, documents) -> None:
|
||||
ok, reason = PromptModel.verify_database_file(temp)
|
||||
if not ok:
|
||||
logger.error(f"Library move: the copy did not verify: {reason}")
|
||||
detail = reason if reason.startswith("Workflow document") else None
|
||||
raise MoveFailed(
|
||||
"verify-failed",
|
||||
f"{MESSAGES['verify-failed']} {detail}" if detail else None,
|
||||
)
|
||||
uri = Path(os.path.abspath(temp)).as_uri() + "?mode=ro"
|
||||
try:
|
||||
with contextlib.closing(sqlite3.connect(uri, uri=True)) as conn:
|
||||
copied = _describe(conn)
|
||||
except sqlite3.Error as error:
|
||||
logger.error(f"Library move: the copy could not be read: {error}")
|
||||
raise MoveFailed("verify-failed") from error
|
||||
if copied != (counts, documents):
|
||||
logger.error("Library move: the copy's rows or documents differ")
|
||||
raise MoveFailed("verify-failed")
|
||||
|
||||
|
||||
def _hash_copy(source: str, destination: str) -> Tuple[int, str]:
|
||||
"""Copy ``source`` to ``destination`` (fsynced, owner-only, times kept)."""
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with open(source, "rb") as reader, open(destination, "wb") as writer:
|
||||
while True:
|
||||
chunk = reader.read(_COPY_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
writer.write(chunk)
|
||||
writer.flush()
|
||||
os.fsync(writer.fileno())
|
||||
stat = os.stat(source)
|
||||
os.utime(destination, ns=(stat.st_atime_ns, stat.st_mtime_ns))
|
||||
_private(destination)
|
||||
return size, digest.hexdigest()
|
||||
|
||||
|
||||
def _file_digest(path: str) -> Tuple[int, str]:
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with open(path, "rb") as reader:
|
||||
while True:
|
||||
chunk = reader.read(_COPY_CHUNK)
|
||||
if not chunk:
|
||||
return size, digest.hexdigest()
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
|
||||
|
||||
def _check_copies(copies) -> None:
|
||||
for source, temp, expected in copies:
|
||||
if _file_digest(temp) != expected or _file_digest(source) != expected:
|
||||
logger.error(
|
||||
f"Library move: {os.path.basename(source)} did not copy unchanged"
|
||||
)
|
||||
raise MoveFailed("copy-failed")
|
||||
|
||||
|
||||
def write_marker(
|
||||
directory: str, state: str, source_db: str, source_config: Optional[str] = None
|
||||
) -> str:
|
||||
"""Atomically write the move marker; return its path.
|
||||
|
||||
``from`` and ``config_from`` name the old files, so a restart after a
|
||||
crash between the switch and the rename can finish the rename.
|
||||
"""
|
||||
path = os.path.join(directory, ll.MARKER_NAME)
|
||||
temp = os.path.join(directory, f"{TEMP_PREFIX}{uuid.uuid4().hex}.marker")
|
||||
payload = json.dumps(
|
||||
{
|
||||
"version": 1,
|
||||
"state": state,
|
||||
"from": source_db,
|
||||
"config_from": source_config,
|
||||
"updated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
},
|
||||
indent=2,
|
||||
).encode("utf-8")
|
||||
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
_private(temp)
|
||||
os.replace(temp, path)
|
||||
except BaseException:
|
||||
_remove(temp)
|
||||
raise
|
||||
_fsync_dir(directory)
|
||||
return path
|
||||
|
||||
|
||||
def _remove(*paths) -> None:
|
||||
for path in paths:
|
||||
try:
|
||||
if path and os.path.lexists(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
logger.warning(f"Library move: could not remove {os.path.basename(path)}")
|
||||
|
||||
|
||||
def sweep_move_temps(directory: str) -> List[str]:
|
||||
"""Remove the temp files an interrupted move left in ``directory``."""
|
||||
removed = []
|
||||
try:
|
||||
names = os.listdir(directory)
|
||||
except OSError:
|
||||
return removed
|
||||
for name in names:
|
||||
if name.startswith(TEMP_PREFIX):
|
||||
_remove(os.path.join(directory, name))
|
||||
removed.append(name)
|
||||
return removed
|
||||
|
||||
|
||||
# ── the move ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _noop(*_args, **_kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def switch_process(old_db: str, location, detached: list) -> None:
|
||||
"""Point this process at ``location``: resolver, every model, observers.
|
||||
|
||||
Adds the old file's connections, detached and still open, to
|
||||
``detached`` as soon as they are detached, so the mover retires them
|
||||
(``PromptModel.retire_detached``) whether or not the rest succeeds.
|
||||
"""
|
||||
from .execution_history import rekey_history_restore
|
||||
from .review_changes import rekey_review_changes
|
||||
|
||||
ll.switch_to(location)
|
||||
detached.extend(PromptModel.repoint_instances(old_db, location.db_path))
|
||||
rekey_review_changes(old_db, location.db_path)
|
||||
rekey_history_restore(old_db, location.db_path)
|
||||
|
||||
|
||||
def unswitch_process(old_db: str, new_db: str) -> list:
|
||||
"""Undo ``switch_process`` (any part of it); returns detached connections."""
|
||||
from .execution_history import rekey_history_restore
|
||||
from .review_changes import rekey_review_changes
|
||||
|
||||
ll.clear_override()
|
||||
detached = PromptModel.repoint_instances(new_db, old_db)
|
||||
rekey_review_changes(new_db, old_db)
|
||||
rekey_history_restore(new_db, old_db)
|
||||
return detached
|
||||
|
||||
|
||||
class _Move:
|
||||
def __init__(self, plan, *, switch, config_lock, progress, checkpoint):
|
||||
self.plan = plan
|
||||
self.switch = switch
|
||||
self.config_lock = config_lock
|
||||
self.progress = progress
|
||||
self.checkpoint = checkpoint
|
||||
self.created_dir = False
|
||||
self.temps: List[str] = []
|
||||
self.published: List[str] = []
|
||||
self.marker: Optional[str] = None
|
||||
self.moved = False
|
||||
# Retired connections still in use; awaited once the locks are free.
|
||||
self.retiring: list = []
|
||||
|
||||
def temp(self, suffix: str) -> str:
|
||||
path = os.path.join(
|
||||
self.plan.target_dir, f"{TEMP_PREFIX}{uuid.uuid4().hex}{suffix}"
|
||||
)
|
||||
self.temps.append(path)
|
||||
return path
|
||||
|
||||
def prepare_target(self) -> None:
|
||||
target = self.plan.target_dir
|
||||
self.created_dir = not os.path.isdir(target)
|
||||
ll.ensure_private_dir(target)
|
||||
sweep_move_temps(target)
|
||||
if ll.read_marker(target).get("state") == ll.MARKER_PENDING:
|
||||
# An earlier move stopped before it switched: its files are ours.
|
||||
names = (ll.DATABASE_NAME, ll.CONFIG_NAME, KEY_FILE_NAME)
|
||||
for name in names + self.plan.backups:
|
||||
_remove(os.path.join(target, name))
|
||||
for suffix in ("-wal", "-shm", "-journal"):
|
||||
_remove(os.path.join(target, ll.DATABASE_NAME + suffix))
|
||||
|
||||
def copy_round(self, source):
|
||||
"""Steps 2–4 once: the verified temp copies and the snapshot version."""
|
||||
self.progress("Copying the database")
|
||||
temp_db = self.temp(".db")
|
||||
version, counts, documents = _copy_snapshot(source, temp_db)
|
||||
self.checkpoint("copied")
|
||||
self.progress("Verifying the copy")
|
||||
_verify_copy(temp_db, counts, documents)
|
||||
self.checkpoint("verified")
|
||||
copies = []
|
||||
for name in self.plan.backups:
|
||||
src = os.path.join(self.plan.source_dir, name)
|
||||
temp = self.temp(".bak")
|
||||
try:
|
||||
expected = _hash_copy(src, temp)
|
||||
except OSError as error:
|
||||
logger.error(f"Library move: copying {name} failed: {error}")
|
||||
raise MoveFailed("copy-failed") from error
|
||||
copies.append((src, temp, expected, name))
|
||||
if copies:
|
||||
self.progress(f"Copied {len(copies)} backup file(s)")
|
||||
self.checkpoint("files")
|
||||
_check_copies([(s, t, e) for s, t, e, _ in copies])
|
||||
return version, counts, temp_db, copies
|
||||
|
||||
def discard_round(self, temp_db, copies) -> None:
|
||||
_remove(temp_db, *[temp for _s, temp, _e, _n in copies])
|
||||
|
||||
def publish(self, temp_db, copies) -> List[str]:
|
||||
"""Under the locks: config.json, marker, final names, marker moved."""
|
||||
target = self.plan.target_dir
|
||||
files = []
|
||||
if self.plan.config_path:
|
||||
temp = self.temp(".config")
|
||||
try:
|
||||
expected = _hash_copy(self.plan.config_path, temp)
|
||||
except OSError as error:
|
||||
raise MoveFailed("copy-failed") from error
|
||||
_check_copies([(self.plan.config_path, temp, expected)])
|
||||
copies = copies + [(self.plan.config_path, temp, expected, ll.CONFIG_NAME)]
|
||||
copies = copies + self._key_copy()
|
||||
self.marker = write_marker(
|
||||
target, ll.MARKER_PENDING, self.plan.db_path, self.plan.config_path
|
||||
)
|
||||
self.checkpoint("pending")
|
||||
for _src, temp, _expected, name in copies:
|
||||
final = os.path.join(target, name)
|
||||
os.replace(temp, final)
|
||||
self.published.append(final)
|
||||
files.append(name)
|
||||
final_db = os.path.join(target, ll.DATABASE_NAME)
|
||||
os.replace(temp_db, final_db)
|
||||
self.published.append(final_db)
|
||||
files.append(ll.DATABASE_NAME)
|
||||
_fsync_dir(target)
|
||||
self.checkpoint("published")
|
||||
write_marker(target, ll.MARKER_MOVED, self.plan.db_path, self.plan.config_path)
|
||||
self.moved = True
|
||||
self.checkpoint("marked")
|
||||
return files
|
||||
|
||||
def _key_copy(self) -> list:
|
||||
"""The workspace key's verified copy (#430), or none: it is re-creatable.
|
||||
|
||||
Without it the moved library starts with a new key, so pages left open
|
||||
across the restart refresh once; that never fails the move.
|
||||
"""
|
||||
source = os.path.join(self.plan.source_dir, KEY_FILE_NAME)
|
||||
if not os.path.isfile(source):
|
||||
return []
|
||||
temp = self.temp(".key")
|
||||
try:
|
||||
expected = _hash_copy(source, temp)
|
||||
_check_copies([(source, temp, expected)])
|
||||
except (OSError, MoveFailed) as error:
|
||||
_remove(temp)
|
||||
logger.warning(
|
||||
f"Library move: the workspace key was not copied ({error}); "
|
||||
"open workspace pages refresh once after the restart"
|
||||
)
|
||||
return []
|
||||
return [(source, temp, expected, KEY_FILE_NAME)]
|
||||
|
||||
def cleanup(self) -> None:
|
||||
if self.moved:
|
||||
return
|
||||
_remove(*self.temps, *self.published, self.marker)
|
||||
if self.created_dir:
|
||||
try:
|
||||
os.rmdir(self.plan.target_dir)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _unique(path: str) -> str:
|
||||
candidate, number = path, 1
|
||||
while os.path.lexists(candidate):
|
||||
candidate = f"{path}_{number}"
|
||||
number += 1
|
||||
return candidate
|
||||
|
||||
|
||||
def retire_old_files(plan: MovePlan) -> Tuple[List[str], Optional[str]]:
|
||||
"""Rename the old database and config.json; ``(renamed, problem code)``.
|
||||
|
||||
Every connection to the old file must be closed. The WAL is checkpointed
|
||||
first; when it cannot be emptied (another process still has the file
|
||||
open) nothing is renamed (``old-in-use``).
|
||||
"""
|
||||
old = plan.db_path
|
||||
folder = os.path.basename(os.path.normpath(plan.target_dir))
|
||||
try:
|
||||
with contextlib.closing(sqlite3.connect(old, timeout=1)) as conn:
|
||||
busy = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0]
|
||||
except sqlite3.Error as error:
|
||||
logger.warning(
|
||||
f"Library move: the old database could not be checkpointed: {error}"
|
||||
)
|
||||
return [], "old-in-use"
|
||||
if busy or _size(old + "-wal") > 0:
|
||||
return [], "old-in-use"
|
||||
renamed = []
|
||||
try:
|
||||
for path in (old, plan.config_path):
|
||||
if not path or not os.path.isfile(path):
|
||||
continue
|
||||
new_name = _unique(path + MOVED_INFIX + folder)
|
||||
os.rename(path, new_name)
|
||||
renamed.append(os.path.basename(new_name))
|
||||
for suffix in ("-wal", "-shm"):
|
||||
if os.path.isfile(old + suffix) and _size(old + suffix) == 0:
|
||||
os.remove(old + suffix)
|
||||
except OSError as error:
|
||||
logger.warning(f"Library move: the old files could not be renamed: {error}")
|
||||
return renamed, "old-locked"
|
||||
return renamed, None
|
||||
|
||||
|
||||
def finish_interrupted_move(location) -> List[str]:
|
||||
"""Rename old files a crash left between the switch and the rename.
|
||||
|
||||
The marker of the folder in use says ``moved`` and names the old files;
|
||||
an old ``prompts.db`` (and config.json) still there under its own name is
|
||||
renamed ``*.moved-to-<folder>`` now, never deleted. Returns the new
|
||||
names (nothing when there is nothing to finish or the file is in use).
|
||||
"""
|
||||
if location.source not in (ll.SOURCE_DEFAULT, ll.SOURCE_ENVIRONMENT):
|
||||
return []
|
||||
marker = ll.read_marker(location.data_dir)
|
||||
old = marker.get("from")
|
||||
if marker.get("state") != ll.MARKER_MOVED or not isinstance(old, str):
|
||||
return []
|
||||
if not os.path.isfile(old) or os.path.basename(old) != ll.DATABASE_NAME:
|
||||
return []
|
||||
if ll.same_folder(os.path.dirname(old), location.data_dir):
|
||||
return []
|
||||
config = marker.get("config_from")
|
||||
if not (
|
||||
isinstance(config, str)
|
||||
and os.path.basename(config) == ll.CONFIG_NAME
|
||||
and os.path.isfile(config)
|
||||
and os.path.normcase(os.path.abspath(config))
|
||||
!= os.path.normcase(os.path.abspath(location.config_path))
|
||||
):
|
||||
config = None
|
||||
pending = types.SimpleNamespace(
|
||||
db_path=old, config_path=config, target_dir=location.data_dir
|
||||
)
|
||||
renamed, problem = retire_old_files(pending)
|
||||
if problem:
|
||||
logger.warning(f"Library move: the old files are still in use ({problem})")
|
||||
elif renamed:
|
||||
logger.info(f"Library move finished: renamed {', '.join(renamed)}")
|
||||
return renamed
|
||||
|
||||
|
||||
def move_library(
|
||||
plan: MovePlan,
|
||||
*,
|
||||
switch: Callable = switch_process,
|
||||
unswitch: Callable = unswitch_process,
|
||||
after_switch: Callable = _noop,
|
||||
config_lock=None,
|
||||
progress: Callable[[str], None] = _noop,
|
||||
checkpoint: Callable[[str], None] = _noop,
|
||||
attempts: int = DEFAULT_ATTEMPTS,
|
||||
) -> MoveResult:
|
||||
"""Move the library as planned (blocking; see the module docstring).
|
||||
|
||||
``switch(old_db, location, detached)`` points the process at the copy
|
||||
and adds the old connections it detached to ``detached``; if it raises,
|
||||
``unswitch(old_db, new_db)`` undoes it while both files are still
|
||||
locked, every detached connection is closed, the copy is removed and
|
||||
the move fails (``switch-failed``). ``after_switch()`` runs
|
||||
once the switch is final; it is best effort (logged, never fails the move).
|
||||
"""
|
||||
short = space_problem(plan)
|
||||
if short is not None:
|
||||
raise MoveFailed("insufficient-space")
|
||||
lock = config_lock if config_lock is not None else contextlib.nullcontext()
|
||||
move = _Move(
|
||||
plan,
|
||||
switch=switch,
|
||||
config_lock=lock,
|
||||
progress=progress,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
move.unswitch = unswitch
|
||||
location = plan.target_location
|
||||
try:
|
||||
move.prepare_target()
|
||||
files, counts = _copy_and_switch(move, location, attempts)
|
||||
except Exception:
|
||||
move.cleanup()
|
||||
raise
|
||||
try:
|
||||
after_switch()
|
||||
except Exception:
|
||||
logger.exception("Library move: a follow-up after the switch failed")
|
||||
from .execution_history import notify_history_restore
|
||||
|
||||
notify_history_restore(location.db_path)
|
||||
progress("Renaming the old files")
|
||||
renamed, problem = retire_old_files(plan)
|
||||
checkpoint("retired")
|
||||
return MoveResult(location, counts, files, renamed, problem)
|
||||
|
||||
|
||||
def _switch_guarded(move: _Move, location) -> list:
|
||||
"""Switch while the copy is locked too; undo it all if any part fails.
|
||||
|
||||
The copy's write lock keeps every repointed thread from writing to it
|
||||
until the switch is whole, so undoing a partial switch loses nothing.
|
||||
Returns the old file's detached connections (still open). On failure
|
||||
every detached connection is retired; those still in use are added to
|
||||
``move.retiring`` for the caller to await once the locks are released.
|
||||
"""
|
||||
plan = move.plan
|
||||
guard = sqlite3.connect(
|
||||
location.db_path, timeout=BUSY_TIMEOUT_SECONDS, isolation_level=None
|
||||
)
|
||||
try:
|
||||
guard.execute("BEGIN IMMEDIATE")
|
||||
detached: list = []
|
||||
try:
|
||||
move.switch(plan.db_path, location, detached)
|
||||
except Exception as error:
|
||||
logger.exception("Library move: switching failed; undoing it")
|
||||
move.moved = False # the copy and its marker are removed
|
||||
# Retire, never wait, here: a connection still in use may be
|
||||
# waiting on the very locks this thread holds.
|
||||
try:
|
||||
undone = move.unswitch(plan.db_path, location.db_path)
|
||||
except Exception:
|
||||
logger.exception("Library move: undoing the switch failed")
|
||||
else:
|
||||
_retire(move, undone)
|
||||
# The old file's connections a partial switch detached: the
|
||||
# models reopen it on next use, so close them now, not at GC.
|
||||
_retire(move, detached)
|
||||
raise MoveFailed("switch-failed") from error
|
||||
move.checkpoint("switched")
|
||||
return detached
|
||||
finally:
|
||||
if guard.in_transaction:
|
||||
guard.execute("ROLLBACK")
|
||||
guard.close()
|
||||
|
||||
|
||||
def _copy_and_switch(move: _Move, location, attempts: int):
|
||||
"""Copy, then switch under both locks; retired connections are awaited.
|
||||
|
||||
A connection that was inside a statement when it was retired is closed
|
||||
by its own thread once the statement ends, which may need the locks
|
||||
released first; the old files are renamed (and a failed copy removed)
|
||||
only after that, since Windows refuses to rename or delete an open file.
|
||||
"""
|
||||
try:
|
||||
return _copy_and_switch_locked(move, location, attempts)
|
||||
finally:
|
||||
try:
|
||||
PromptModel.await_detached(move.retiring, logger)
|
||||
except Exception:
|
||||
logger.exception("Library move: waiting for old connections failed")
|
||||
|
||||
|
||||
def _retire(move: _Move, connections: list) -> None:
|
||||
"""Retire ``connections``, adding those still in use to ``move.retiring``.
|
||||
|
||||
Never raises: a switch that completed must not be reported as failed
|
||||
because one of the old connections could not be retired (logged; the
|
||||
rename then reports the old files as still in use).
|
||||
"""
|
||||
try:
|
||||
move.retiring.extend(PromptModel.retire_detached(connections, logger))
|
||||
except Exception:
|
||||
logger.exception("Library move: retiring the old connections failed")
|
||||
|
||||
|
||||
def _copy_and_switch_locked(move: _Move, location, attempts: int):
|
||||
plan = move.plan
|
||||
for attempt in range(1, attempts + 1):
|
||||
source = sqlite3.connect(
|
||||
plan.db_path, timeout=BUSY_TIMEOUT_SECONDS, isolation_level=None
|
||||
)
|
||||
try:
|
||||
source.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}")
|
||||
version, counts, temp_db, copies = move.copy_round(source)
|
||||
source.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
if source.execute("PRAGMA data_version").fetchone()[0] != version:
|
||||
move.progress("The library changed during the copy; copying again")
|
||||
move.discard_round(temp_db, copies)
|
||||
continue
|
||||
move.checkpoint("locked")
|
||||
with move.config_lock:
|
||||
files = move.publish(temp_db, copies)
|
||||
detached = _switch_guarded(move, location)
|
||||
# Still holding the old file's lock: a write that was waiting
|
||||
# on it is interrupted and cannot commit, so it fails (and
|
||||
# runs again on the new library) instead of landing in the
|
||||
# old file. Its own thread closes the connection.
|
||||
_retire(move, detached)
|
||||
finally:
|
||||
if source.in_transaction:
|
||||
source.execute("ROLLBACK")
|
||||
return files, counts
|
||||
finally:
|
||||
source.close()
|
||||
raise MoveFailed("library-busy")
|
||||
@@ -15,6 +15,8 @@ change:
|
||||
``cleanup_orphaned_images``), counted only under a source that can be read
|
||||
now, as the health check's ``missing_files`` judges them
|
||||
(``health.link_states``). It deletes database rows only, never a file.
|
||||
It also forgets the removal marks (#480) that hide nothing any more: the
|
||||
file is gone or changed, under a readable source (#501, ``_stale_marks``).
|
||||
|
||||
``preview_maintenance`` reads one snapshot on its own ``mode=ro`` connection
|
||||
(``health.readonly_uri``) and closes it in ``finally``; files are looked at
|
||||
@@ -41,13 +43,23 @@ from pathlib import Path
|
||||
|
||||
from .execution_contexts import LIVE_HINT_CTES
|
||||
from .health import BATCH_ROWS, link_states, readable_roots, readonly_uri
|
||||
from .image_removal_schema import TABLE as MARKS_TABLE
|
||||
from .image_removal_schema import clear_marks, has_marks_table
|
||||
from .pairs import PairDependencyError
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
|
||||
MAINTENANCE_OPERATIONS = (
|
||||
"duplicate_prompts",
|
||||
"optimize",
|
||||
"unused_components",
|
||||
"missing_links",
|
||||
# Applied as its own batched job (database/workflow_storage.py).
|
||||
"workflow_storage",
|
||||
)
|
||||
MAX_DETAILS = 50 # detail rows a plan lists
|
||||
MAX_RESULT_ROWS = 200 # changed records an apply describes (the job lists 200)
|
||||
@@ -77,6 +89,12 @@ PROTECTIONS = (
|
||||
"with Library state (a title or an explicit parent)",
|
||||
"with Library state (a title or an explicit parent)",
|
||||
),
|
||||
(
|
||||
# #443: an archived prompt is restorable, never pruned.
|
||||
"EXISTS (SELECT 1 FROM prompt_archive pa WHERE pa.prompt_id = p.id)",
|
||||
"archived in the Library",
|
||||
"archived in the Library",
|
||||
),
|
||||
(
|
||||
"p.id IN (SELECT prompt_id FROM live_hint_prompts)",
|
||||
"used by recorded runs",
|
||||
@@ -85,7 +103,8 @@ PROTECTIONS = (
|
||||
)
|
||||
|
||||
# Exempt references: images, pairs, the __protected__ tag, Library parent
|
||||
# state, and live recorded-run hints (computed once, see LIVE_HINT_CTES).
|
||||
# state, the Library archive, and live recorded-run hints (computed once,
|
||||
# see LIVE_HINT_CTES).
|
||||
ORPHANED_PROMPTS_SQL = (
|
||||
"WITH "
|
||||
+ LIVE_HINT_CTES
|
||||
@@ -179,6 +198,8 @@ class MaintenanceDraft:
|
||||
protected: dict
|
||||
media: dict
|
||||
details: list = field(default_factory=list)
|
||||
# missing_links only: locators of removal marks whose file is gone (#501).
|
||||
marks: list = field(default_factory=list)
|
||||
|
||||
def public(self):
|
||||
"""The plan fields a response may carry (no fingerprint, no targets)."""
|
||||
@@ -260,24 +281,117 @@ def _orphaned_ids(conn):
|
||||
|
||||
|
||||
def _delete_ids(conn, table, ids):
|
||||
"""Delete ``table`` rows by id in batches; returns the rows removed."""
|
||||
"""Delete ``table`` rows by id in batches; returns the rows removed.
|
||||
|
||||
The workflow documents only those rows referenced go with them (a
|
||||
prompt's images go by cascade), in the same transaction.
|
||||
"""
|
||||
if table not in ("prompts", "generated_images"):
|
||||
raise ValueError(f"Unsupported table {table!r}")
|
||||
removed = 0
|
||||
for batch in _batches(ids):
|
||||
documents = documents_of_rows(
|
||||
conn, "id" if table == "generated_images" else "prompt_id", batch
|
||||
)
|
||||
cursor = conn.execute(
|
||||
f"DELETE FROM {table} WHERE id IN ({_marks(batch)})", # nosec B608
|
||||
batch,
|
||||
)
|
||||
removed += cursor.rowcount
|
||||
release_documents(conn, documents)
|
||||
return removed
|
||||
|
||||
|
||||
def _mark_rows(conn, only=None):
|
||||
"""``[(locator, size, mtime_ns)]`` of the removal marks (all, or those
|
||||
among *only*), sorted by locator."""
|
||||
if not has_marks_table(conn):
|
||||
return []
|
||||
sql = f"SELECT locator, size, mtime_ns FROM {MARKS_TABLE}" # nosec B608
|
||||
if only is None:
|
||||
return sorted(tuple(row) for row in conn.execute(sql))
|
||||
found = []
|
||||
for batch in _batches(sorted(only)):
|
||||
found.extend(
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
f"{sql} WHERE locator IN ({_marks(batch)})", batch # nosec B608
|
||||
)
|
||||
)
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def _lists_something(folder):
|
||||
"""Whether *folder* holds at least one entry (an unplugged drive's mount
|
||||
point, or an unbound volume, is an empty folder that reads fine)."""
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
return next(entries, None) is not None
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _name_listed(path):
|
||||
"""Whether *path*'s folder lists its name, compared under ``normcase``.
|
||||
|
||||
A locator is ``normcase``d; on a case-sensitive NTFS folder ``lstat`` of
|
||||
the lower-cased spelling fails for a file that is there.
|
||||
"""
|
||||
folder, name = os.path.split(path)
|
||||
wanted = os.path.normcase(name)
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
return any(os.path.normcase(entry.name) == wanted for entry in entries)
|
||||
except OSError:
|
||||
return True # cannot tell: keep the mark
|
||||
|
||||
|
||||
def _stale_mark(locator, size, mtime_ns):
|
||||
"""Whether the file of a mark is gone or has other bytes (blocking)."""
|
||||
try:
|
||||
info = os.lstat(locator)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return os.path.isdir(os.path.dirname(locator)) and not _name_listed(locator)
|
||||
except OSError:
|
||||
return False
|
||||
return (info.st_size, info.st_mtime_ns) != (size, mtime_ns)
|
||||
|
||||
|
||||
def _stale_marks(marks, roots):
|
||||
"""``[(locator, owner)]`` of the marks that hide nothing any more (#501).
|
||||
|
||||
*marks* are ``(locator, size, mtime_ns)`` rows. A mark is stale when its
|
||||
file was changed, or is gone from a folder that still exists. Only marks
|
||||
under a source that can be read now and lists something are judged (an
|
||||
unplugged drive's marks, or its empty mount point's, stay); any doubt
|
||||
keeps the mark. *owner* is the index of the longest source holding it.
|
||||
Locators are ``normcase(realpath)`` paths, so the sources are too.
|
||||
"""
|
||||
readable = readable_roots(roots)
|
||||
prefixes = [
|
||||
os.path.normcase(os.path.realpath(root)).rstrip("\\/") + os.sep
|
||||
for root in roots
|
||||
]
|
||||
judged = {}
|
||||
stale = []
|
||||
for locator, size, mtime_ns in marks:
|
||||
owners = [i for i, prefix in enumerate(prefixes) if locator.startswith(prefix)]
|
||||
if not owners:
|
||||
continue # under no source now (a source removed from the settings)
|
||||
owner = max(owners, key=lambda i: len(prefixes[i]))
|
||||
if owner not in judged:
|
||||
judged[owner] = readable[owner] and _lists_something(roots[owner])
|
||||
if judged[owner] and _stale_mark(locator, size, mtime_ns):
|
||||
stale.append((locator, owner))
|
||||
return stale
|
||||
|
||||
|
||||
def _read_snapshot(db_path, read):
|
||||
"""``read(conn)`` on one ``mode=ro`` snapshot; the connection always closes."""
|
||||
conn = sqlite3.connect(readonly_uri(db_path), uri=True)
|
||||
try:
|
||||
conn.row_factory = sqlite3.Row
|
||||
register_prompt_functions(conn) # LIVE_HINT_CTES calls them (#460)
|
||||
conn.execute("BEGIN") # one read snapshot for the whole plan
|
||||
return read(conn)
|
||||
finally:
|
||||
@@ -379,10 +493,13 @@ class MaintenanceOperations:
|
||||
conn, primary_id, merged_metadata, strict=strict
|
||||
)
|
||||
removed = 0
|
||||
# Images the lenient merge could not move go with their prompt.
|
||||
documents = documents_of_rows(conn, "prompt_id", duplicate_ids)
|
||||
for duplicate_id in duplicate_ids:
|
||||
conn.execute("DELETE FROM prompts WHERE id = ?", (duplicate_id,))
|
||||
removed += 1
|
||||
self.logger.debug(f"Removed duplicate prompt {duplicate_id}")
|
||||
release_documents(conn, documents)
|
||||
return removed, moved
|
||||
|
||||
@staticmethod
|
||||
@@ -408,9 +525,14 @@ class MaintenanceOperations:
|
||||
roots = [str(root) for root in roots]
|
||||
if operation == "optimize":
|
||||
return self._plan_optimize()
|
||||
if operation == "workflow_storage":
|
||||
return self._plan_workflow_storage()
|
||||
if operation == "missing_links":
|
||||
rows = _read_snapshot(self.model.db_path, self._link_rows)
|
||||
return self._plan_missing_links(rows, roots)
|
||||
rows, marks = _read_snapshot(
|
||||
self.model.db_path,
|
||||
lambda conn: (self._link_rows(conn), _mark_rows(conn)),
|
||||
)
|
||||
return self._plan_missing_links(rows, roots, _stale_marks(marks, roots))
|
||||
reader = (
|
||||
self._plan_duplicate_prompts
|
||||
if operation == "duplicate_prompts"
|
||||
@@ -541,13 +663,15 @@ class MaintenanceOperations:
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _links_fingerprint(candidates):
|
||||
def _links_fingerprint(candidates, stale=()):
|
||||
fingerprint = _Fingerprint("missing_links")
|
||||
for row, owner in candidates:
|
||||
fingerprint.add([row[0], row[1], row[3], owner])
|
||||
for locator, owner in stale:
|
||||
fingerprint.add(["mark", locator, owner])
|
||||
return fingerprint.hexdigest()
|
||||
|
||||
def _plan_missing_links(self, rows, roots):
|
||||
def _plan_missing_links(self, rows, roots, stale=()):
|
||||
readable = readable_roots(roots)
|
||||
candidates, unavailable, outside = [], 0, 0
|
||||
for row, owner, state in link_states(rows, roots, readable):
|
||||
@@ -575,29 +699,47 @@ class MaintenanceOperations:
|
||||
)
|
||||
)
|
||||
count = len(candidates)
|
||||
parts = []
|
||||
if count:
|
||||
parts.append(
|
||||
_plural(count, "link to a missing file", "links to missing files")
|
||||
)
|
||||
if stale:
|
||||
parts.append(
|
||||
_plural(
|
||||
len(stale),
|
||||
"removed image whose file is gone or changed",
|
||||
"removed images whose files are gone or changed",
|
||||
)
|
||||
)
|
||||
details = [
|
||||
{
|
||||
"label": _relative(row[1], roots[owner]),
|
||||
"detail": f"{_source_name(roots[owner], owner)} · the file is missing",
|
||||
}
|
||||
for row, owner in candidates
|
||||
] + [
|
||||
{
|
||||
"label": _relative(locator, os.path.realpath(roots[owner])),
|
||||
"detail": (
|
||||
f"{_source_name(roots[owner], owner)} · removed image, "
|
||||
"the file is gone or changed"
|
||||
),
|
||||
}
|
||||
for locator, owner in stale
|
||||
]
|
||||
return MaintenanceDraft(
|
||||
operation="missing_links",
|
||||
fingerprint=self._links_fingerprint(candidates),
|
||||
fingerprint=self._links_fingerprint(candidates, stale),
|
||||
targets=[row[0] for row, _owner in candidates],
|
||||
scope={
|
||||
"count": count,
|
||||
"label": (
|
||||
_plural(count, "link to a missing file", "links to missing files")
|
||||
if count
|
||||
else "No links to missing files"
|
||||
),
|
||||
"count": count + len(stale),
|
||||
"label": "; ".join(parts) or "No links to missing files",
|
||||
},
|
||||
protected={"count": unavailable + outside, "reasons": reasons},
|
||||
media={"deletes_files": False, "label": LINKS_ONLY},
|
||||
details=[
|
||||
{
|
||||
"label": _relative(row[1], roots[owner]),
|
||||
"detail": (
|
||||
f"{_source_name(roots[owner], owner)} · the file is missing"
|
||||
),
|
||||
}
|
||||
for row, owner in candidates[:MAX_DETAILS]
|
||||
],
|
||||
details=details[:MAX_DETAILS],
|
||||
marks=[locator for locator, _owner in stale],
|
||||
)
|
||||
|
||||
def _plan_optimize(self):
|
||||
@@ -647,16 +789,20 @@ class MaintenanceOperations:
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
def apply_maintenance(self, operation, fingerprint, targets, roots=()) -> dict:
|
||||
def apply_maintenance(
|
||||
self, operation, fingerprint, targets, roots=(), marks=()
|
||||
) -> dict:
|
||||
"""Apply a reviewed plan exactly, or raise PlanChanged (blocking).
|
||||
|
||||
Inside ``BEGIN IMMEDIATE`` the fingerprint is computed again on the
|
||||
write connection; when it differs nothing is written. ``targets`` and
|
||||
``roots`` are the plan's. Returns ``{changed, rows}``: how many
|
||||
records changed and up to ``MAX_RESULT_ROWS`` descriptions of them
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``).
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``). ``marks`` are a
|
||||
missing-links plan's stale removal marks; that apply also returns
|
||||
``marks``, how many it forgot.
|
||||
"""
|
||||
if operation not in MAINTENANCE_OPERATIONS:
|
||||
if operation not in MAINTENANCE_OPERATIONS or operation == "workflow_storage":
|
||||
raise ValueError(f"Unknown maintenance operation {operation!r}")
|
||||
if operation == "optimize":
|
||||
# VACUUM cannot run in a transaction; the plan binds the database.
|
||||
@@ -667,9 +813,12 @@ class MaintenanceOperations:
|
||||
appliers = {
|
||||
"duplicate_prompts": self._apply_duplicate_prompts,
|
||||
"unused_components": self._apply_unused_components,
|
||||
"missing_links": self._apply_missing_links,
|
||||
}
|
||||
with self._pair_transaction() as conn:
|
||||
if operation == "missing_links":
|
||||
return self._apply_missing_links(
|
||||
conn, fingerprint, targets, roots, list(marks)
|
||||
)
|
||||
return appliers[operation](conn, fingerprint, targets, roots)
|
||||
|
||||
def _apply_duplicate_prompts(self, conn, fingerprint, targets, _roots):
|
||||
@@ -698,7 +847,7 @@ class MaintenanceOperations:
|
||||
]
|
||||
return {"changed": changed, "rows": rows}
|
||||
|
||||
def _apply_missing_links(self, conn, fingerprint, targets, roots):
|
||||
def _apply_missing_links(self, conn, fingerprint, targets, roots, marks):
|
||||
rows = []
|
||||
for batch in _batches(sorted(targets)):
|
||||
rows.extend(
|
||||
@@ -715,10 +864,14 @@ class MaintenanceOperations:
|
||||
if state != "missing": # back, or its source went away
|
||||
raise PlanChanged("missing_links")
|
||||
candidates.append((row, owner))
|
||||
if self._links_fingerprint(candidates) != fingerprint:
|
||||
stale = _stale_marks(_mark_rows(conn, marks) if marks else [], roots)
|
||||
if [locator for locator, _owner in stale] != marks: # a mark went or is valid
|
||||
raise PlanChanged("missing_links")
|
||||
if self._links_fingerprint(candidates, stale) != fingerprint:
|
||||
raise PlanChanged("missing_links")
|
||||
changed = _delete_ids(conn, "generated_images", [row[0] for row in rows])
|
||||
return {
|
||||
"changed": changed,
|
||||
"marks": clear_marks(conn, marks),
|
||||
"rows": [{"image_path": row[1]} for row in rows[:MAX_RESULT_ROWS]],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
"""A SQLite connection that is never closed under the thread using it.
|
||||
|
||||
``PromptModel`` gives each thread its own connection, but Move library and
|
||||
``close_all`` must retire every thread's connection from one thread. Closing
|
||||
a ``sqlite3.Connection`` while its owner is inside a statement (Python runs
|
||||
``sqlite3_step`` without the GIL) frees memory the owner then reads: on
|
||||
Windows the process dies with "access violation", elsewhere the owner sees
|
||||
garbage such as a ``MemoryError``.
|
||||
|
||||
``ManagedConnection`` counts the calls its owner is inside. ``retire()`` is
|
||||
safe from any thread and never raises: an idle connection is closed at once;
|
||||
one in use is closed by its owner as the call ends. A retired connection
|
||||
refuses new work (cursors, statements, commits) with the error a closed
|
||||
connection raises.
|
||||
|
||||
What retiring a connection in use does and does not do:
|
||||
|
||||
- It calls ``sqlite3_interrupt`` (thread-safe). That ends a running
|
||||
statement at its next step, but it does not wake a statement that is
|
||||
waiting for another connection's lock: that one waits until the lock is
|
||||
released (or the busy timeout ends it), and only then is it interrupted.
|
||||
An interrupt that arrives between two statements, for example between the
|
||||
implicit ``BEGIN`` and the DML or between ``executemany`` rows, is lost.
|
||||
- Writes are kept out of a file the process has switched away from because
|
||||
a retired connection never commits: legacy-mode DML runs inside a
|
||||
transaction that only ``commit()`` (refused) or ``close()`` (rolls back)
|
||||
ends. A statement that commits by itself is not covered: DDL or a PRAGMA
|
||||
write outside a transaction, or an explicit ``COMMIT`` passed to
|
||||
``execute``/``executescript`` (``executescript`` also commits first, in C,
|
||||
on Python before 3.12). Managed connections must never be used that way
|
||||
for library writes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
from typing import Iterable, Iterator, List, Optional
|
||||
|
||||
import sqlite3
|
||||
|
||||
CLOSED_MESSAGE = "Cannot operate on a closed database."
|
||||
# Rows fetched per step when a cursor is iterated: the connection is marked
|
||||
# in use once per chunk instead of once per row (iteration costs about what
|
||||
# a plain cursor costs). A loop that stops early leaves up to this many rows
|
||||
# fetched; ``fetchone``/``fetchmany`` after a partial loop skip them.
|
||||
ITER_CHUNK = 128
|
||||
|
||||
|
||||
class ManagedCursor(sqlite3.Cursor):
|
||||
"""A cursor whose calls into SQLite count as its connection being in use."""
|
||||
|
||||
def execute(self, sql, parameters=(), /):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().execute(sql, parameters)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def executemany(self, sql, seq_of_parameters, /):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().executemany(sql, seq_of_parameters)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def executescript(self, sql_script, /):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().executescript(sql_script)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def fetchone(self):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().fetchone()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def fetchmany(self, *args, **kwargs):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().fetchmany(*args, **kwargs)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def fetchall(self):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().fetchall()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def __iter__(self) -> Iterator:
|
||||
return self._chunks()
|
||||
|
||||
def _chunks(self) -> Iterator:
|
||||
while True:
|
||||
rows = self.fetchmany(ITER_CHUNK)
|
||||
if not rows:
|
||||
return
|
||||
yield from rows
|
||||
|
||||
def __next__(self):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().__next__()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def close(self):
|
||||
conn = self.connection
|
||||
try:
|
||||
conn._enter()
|
||||
except sqlite3.ProgrammingError:
|
||||
return super().close() # the connection is closed or closing
|
||||
try:
|
||||
return super().close()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
|
||||
class ManagedConnection(sqlite3.Connection):
|
||||
"""A connection only its owner closes while a call into it is running."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# Guards _depth, _retired, _closing and _cursors. Re-entrant so that
|
||||
# code running in the closing thread while it holds the lock (a
|
||||
# cursor() during the cursor-set copy) is refused instead of hanging.
|
||||
self._use_lock = threading.RLock()
|
||||
self._depth = 0
|
||||
self._retired = False
|
||||
self._closing = False
|
||||
self._released = threading.Event()
|
||||
# Python before 3.12 keeps the file open after close() while a cursor
|
||||
# still holds a statement; these are closed first (see _close_now).
|
||||
self._cursors: "weakref.WeakSet[sqlite3.Cursor]" = weakref.WeakSet()
|
||||
self.close_logger: Optional[logging.Logger] = None
|
||||
|
||||
# ── in use ───────────────────────────────────────────────────────
|
||||
|
||||
def _enter(self) -> None:
|
||||
with self._use_lock:
|
||||
if self._retired:
|
||||
raise sqlite3.ProgrammingError(CLOSED_MESSAGE)
|
||||
self._depth += 1
|
||||
|
||||
def _leave(self) -> None:
|
||||
with self._use_lock:
|
||||
self._depth -= 1
|
||||
close_now = self._retired and self._depth == 0
|
||||
if close_now:
|
||||
self._close_now()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def using(self):
|
||||
"""Mark the connection in use for a call this class cannot wrap.
|
||||
|
||||
For example as the *target* of another connection's ``backup()``.
|
||||
"""
|
||||
self._enter()
|
||||
try:
|
||||
yield self
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
def cursor(self, factory=None):
|
||||
with self._use_lock:
|
||||
if self._retired:
|
||||
raise sqlite3.ProgrammingError(CLOSED_MESSAGE)
|
||||
cursor = super().cursor(factory or ManagedCursor)
|
||||
self._cursors.add(cursor)
|
||||
return cursor
|
||||
|
||||
def execute(self, sql, parameters=(), /):
|
||||
return self.cursor().execute(sql, parameters)
|
||||
|
||||
def executemany(self, sql, seq_of_parameters, /):
|
||||
return self.cursor().executemany(sql, seq_of_parameters)
|
||||
|
||||
def executescript(self, sql_script, /):
|
||||
return self.cursor().executescript(sql_script)
|
||||
|
||||
def commit(self):
|
||||
self._enter()
|
||||
try:
|
||||
return super().commit()
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
def rollback(self):
|
||||
self._enter() # a closed connection raises, as sqlite3's does
|
||||
try:
|
||||
return super().rollback()
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
def backup(self, target, **kwargs):
|
||||
self._enter()
|
||||
try:
|
||||
return super().backup(target, **kwargs)
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
def create_function(self, *args, **kwargs):
|
||||
self._enter()
|
||||
try:
|
||||
return super().create_function(*args, **kwargs)
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
@property
|
||||
def in_transaction(self) -> bool:
|
||||
self._enter()
|
||||
try:
|
||||
return super().in_transaction
|
||||
finally:
|
||||
self._leave()
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
# The C implementation calls its own commit/rollback, not these.
|
||||
if exc_type is None:
|
||||
try:
|
||||
self.commit()
|
||||
except BaseException:
|
||||
with contextlib.suppress(sqlite3.Error):
|
||||
self.rollback()
|
||||
raise
|
||||
else:
|
||||
self.rollback()
|
||||
return False
|
||||
|
||||
# ── closing ──────────────────────────────────────────────────────
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close now if idle, else as soon as the running call ends.
|
||||
|
||||
Unlike ``sqlite3.Connection.close``, a call from another thread while
|
||||
the owner is inside a call returns before the file is closed. A caller
|
||||
that must know the file is closed (before renaming or deleting it)
|
||||
waits for ``wait_closed`` (``PromptModel.close_detached`` does).
|
||||
"""
|
||||
self.retire()
|
||||
|
||||
def retire(self) -> bool:
|
||||
"""Refuse new work and close; True when the connection is closed already.
|
||||
|
||||
Safe from any thread, and never raises: failures are logged.
|
||||
"""
|
||||
try:
|
||||
with self._use_lock:
|
||||
if not self._retired:
|
||||
self._retired = True
|
||||
if self._depth:
|
||||
# Ends the running statement (see the module docstring
|
||||
# for what it does not end). The owner cannot start
|
||||
# closing the connection while this lock is held.
|
||||
self.interrupt()
|
||||
return False
|
||||
elif self._depth:
|
||||
return False
|
||||
else:
|
||||
return self._released.wait(0)
|
||||
except Exception:
|
||||
self._log_exception("Error retiring database connection")
|
||||
return self._released.wait(0)
|
||||
self._close_now()
|
||||
return True
|
||||
|
||||
def wait_closed(self, timeout: float) -> bool:
|
||||
return self._released.wait(timeout)
|
||||
|
||||
def _close_now(self) -> None:
|
||||
# Only ever called with no call into the connection running, and a
|
||||
# retired connection refuses new calls and cursors: its cursors are
|
||||
# idle, and the set is copied under the lock that guards it.
|
||||
with self._use_lock:
|
||||
if self._closing:
|
||||
return # another thread closes it and sets _released
|
||||
self._closing = True
|
||||
cursors = list(self._cursors)
|
||||
try:
|
||||
for cursor in cursors:
|
||||
with contextlib.suppress(sqlite3.Error):
|
||||
sqlite3.Cursor.close(cursor)
|
||||
super().close()
|
||||
except Exception:
|
||||
self._log_exception("Error closing database connection")
|
||||
finally:
|
||||
self._released.set()
|
||||
|
||||
def _log_exception(self, message: str) -> None:
|
||||
if self.close_logger is not None:
|
||||
self.close_logger.exception(message)
|
||||
|
||||
|
||||
def retire_connections(
|
||||
connections: Iterable[sqlite3.Connection], logger=None
|
||||
) -> List[ManagedConnection]:
|
||||
"""Retire every connection; return those whose owner has yet to close them.
|
||||
|
||||
Safe while holding a lock the owners wait on: nothing here waits for them.
|
||||
One connection failing to retire is logged and never skips the rest.
|
||||
"""
|
||||
pending = []
|
||||
for conn in connections:
|
||||
try:
|
||||
if isinstance(conn, ManagedConnection):
|
||||
if logger is not None:
|
||||
conn.close_logger = logger
|
||||
if not conn.retire():
|
||||
pending.append(conn)
|
||||
else:
|
||||
conn.close()
|
||||
except Exception:
|
||||
if logger is not None:
|
||||
logger.exception("Error closing database connection")
|
||||
return pending
|
||||
|
||||
|
||||
def await_closed(
|
||||
connections: Iterable[ManagedConnection], timeout: float, logger=None
|
||||
) -> bool:
|
||||
"""Wait until the owners closed ``connections`` (retired); False on timeout.
|
||||
|
||||
A connection still open after ``timeout`` (its owner is blocked, for
|
||||
example on another process's lock after a lost interrupt) is logged and
|
||||
left to close when its call ends; a Move library then reports the old
|
||||
files as still in use instead of renaming them.
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
still_open = 0
|
||||
for conn in connections:
|
||||
if not conn.wait_closed(max(0.0, deadline - time.monotonic())):
|
||||
still_open += 1
|
||||
if still_open and logger is not None:
|
||||
logger.warning(
|
||||
f"{still_open} database connection(s) still in use after "
|
||||
f"{timeout:g}s; they close when their statement ends"
|
||||
)
|
||||
return not still_open
|
||||
@@ -9,25 +9,53 @@ import math
|
||||
import sqlite3
|
||||
import os
|
||||
import threading
|
||||
import tempfile
|
||||
import weakref
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from typing import Optional, Set, Tuple
|
||||
|
||||
from .managed_connection import (
|
||||
ManagedConnection,
|
||||
await_closed,
|
||||
retire_connections,
|
||||
)
|
||||
from . import restore_stage
|
||||
from .library_schema import library_schema_error, migrate_library_state
|
||||
from .image_favorites import image_favorites_schema_error, migrate_image_favorites
|
||||
from .image_collections import (
|
||||
image_collections_schema_error,
|
||||
migrate_image_collections,
|
||||
)
|
||||
from .annotation_schema import annotation_schema_error, migrate_annotation_state
|
||||
from .autotag_schema import (
|
||||
autotag_schema_error,
|
||||
autotag_without_annotations_error,
|
||||
migrate_autotag_state,
|
||||
)
|
||||
from .review_schema import (
|
||||
migrate_review_state,
|
||||
review_schema_error,
|
||||
review_without_annotations_error,
|
||||
)
|
||||
from .saved_views import migrate_saved_views, saved_views_schema_error
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
from .image_categories import image_categories_schema_error, migrate_image_categories
|
||||
from .image_removal_schema import image_removals_schema_error, migrate_image_removals
|
||||
from .prompt_archive import migrate_prompt_archive, prompt_archive_schema_error
|
||||
from .workflow_documents import (
|
||||
DOCUMENT_ID_COLUMNS,
|
||||
migrate_workflow_documents,
|
||||
workflow_documents_schema_error,
|
||||
)
|
||||
|
||||
|
||||
from .execution_history_schema import (
|
||||
history_schema_error,
|
||||
migrate_execution_history,
|
||||
rotate_history_epoch,
|
||||
)
|
||||
|
||||
# Import logging system
|
||||
@@ -50,6 +78,19 @@ MAX_PAIR_PROVENANCE_LENGTH = 4_000_000
|
||||
MAX_PAIR_PROVENANCE_DEPTH = 64
|
||||
|
||||
|
||||
def _refuse_rebuild_with_documents(columns) -> None:
|
||||
"""The legacy generated_images rebuilds copy a fixed column list.
|
||||
|
||||
A file that needs one (``prompt_id`` TEXT, no ``file_path``) but already
|
||||
has the workflow document ids would lose them, so it is refused.
|
||||
"""
|
||||
if any(column in columns for column in DOCUMENT_ID_COLUMNS):
|
||||
raise sqlite3.DatabaseError(
|
||||
"This database needs an old image table rebuild but already has "
|
||||
"workflow document references; the rebuild would drop them"
|
||||
)
|
||||
|
||||
|
||||
def pair_provenance_error(encoded: object) -> Optional[str]:
|
||||
"""Validate the same bounded JSON object for live writes and restores."""
|
||||
if not isinstance(encoded, str):
|
||||
@@ -119,12 +160,15 @@ def configure_connection(conn: sqlite3.Connection) -> None:
|
||||
(common on Windows) let other writers' polling busy handlers starve past
|
||||
BUSY_TIMEOUT_SECONDS and fail with "database is locked" (#242). Without
|
||||
WAL (for example an in-memory database) SQLite's FULL default is kept.
|
||||
It also registers the prompt-key SQL functions (register_prompt_functions).
|
||||
"""
|
||||
journal_mode = conn.execute("PRAGMA journal_mode = WAL").fetchone()[0]
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}")
|
||||
if str(journal_mode).lower() == "wal":
|
||||
conn.execute("PRAGMA synchronous = NORMAL")
|
||||
# History reads and the live-hint rule call Python's prompt key (#460)
|
||||
register_prompt_functions(conn)
|
||||
|
||||
|
||||
class PromptModel:
|
||||
@@ -308,14 +352,28 @@ class PromptModel:
|
||||
migrate_library_state(conn)
|
||||
migrate_image_favorites(conn)
|
||||
migrate_saved_views(conn)
|
||||
migrate_image_collections(conn)
|
||||
# A review group without its annotation group is refused before the
|
||||
# annotation install could make it look valid (catalog only; the rows
|
||||
# are validated once, in migrate_review_state).
|
||||
review_error = review_without_annotations_error(conn)
|
||||
if review_error:
|
||||
raise sqlite3.DatabaseError(review_error)
|
||||
# The auto-tag marks cite annotation bytes the same way (#379).
|
||||
autotag_error = autotag_without_annotations_error(conn)
|
||||
if autotag_error:
|
||||
raise sqlite3.DatabaseError(autotag_error)
|
||||
migrate_annotation_state(conn)
|
||||
migrate_review_state(conn)
|
||||
migrate_autotag_state(conn)
|
||||
# Image categories (#332): independent of every table above.
|
||||
migrate_image_categories(conn)
|
||||
# The Library parent archive (#443): independent of every table above.
|
||||
migrate_prompt_archive(conn)
|
||||
# Removal marks (#480): independent of every table above.
|
||||
migrate_image_removals(conn)
|
||||
# Last: the older generated_images rebuilds run only before file_path.
|
||||
migrate_workflow_documents(conn)
|
||||
|
||||
def _migrate_prompt_pairs(self, conn: sqlite3.Connection) -> None:
|
||||
conn.execute("SAVEPOINT prompt_pair_schema")
|
||||
@@ -528,7 +586,10 @@ class PromptModel:
|
||||
"WHERE pp.positive_prompt_id IS NOT gi.prompt_id LIMIT 1"
|
||||
).fetchone():
|
||||
return "Image pair association has a different positive prompt owner"
|
||||
for row in conn.execute("SELECT provenance FROM image_prompt_pairs"):
|
||||
# fetchone, not iteration: a managed connection's cursor prefetches
|
||||
# ITER_CHUNK rows when iterated, and each provenance may be ~4 MB.
|
||||
cursor = conn.execute("SELECT provenance FROM image_prompt_pairs")
|
||||
while (row := cursor.fetchone()) is not None:
|
||||
error = pair_provenance_error(row[0])
|
||||
if error:
|
||||
return error
|
||||
@@ -587,19 +648,42 @@ class PromptModel:
|
||||
if conn is not None and self._local.generation == self._generation:
|
||||
return conn
|
||||
|
||||
# check_same_thread=False only so close_all can clean up other
|
||||
# threads' connections; each connection is still used by one thread.
|
||||
conn = sqlite3.connect(
|
||||
self.db_path, check_same_thread=False, timeout=BUSY_TIMEOUT_SECONDS
|
||||
)
|
||||
conn.row_factory = sqlite3.Row
|
||||
configure_connection(conn)
|
||||
while True:
|
||||
# The path and generation are read together, and the new
|
||||
# connection is kept only if no Move library repointed the model
|
||||
# meanwhile: otherwise it would write to the old file.
|
||||
with self._conn_lock:
|
||||
path, generation = self.db_path, self._generation
|
||||
# check_same_thread=False only so close_all and Move library can
|
||||
# retire other threads' connections; each connection is still
|
||||
# used by one thread, and ManagedConnection never closes it under
|
||||
# a running statement (see managed_connection).
|
||||
conn = sqlite3.connect(
|
||||
path,
|
||||
check_same_thread=False,
|
||||
timeout=BUSY_TIMEOUT_SECONDS,
|
||||
factory=ManagedConnection,
|
||||
)
|
||||
try:
|
||||
conn.row_factory = sqlite3.Row
|
||||
configure_connection(conn)
|
||||
except BaseException:
|
||||
conn.close()
|
||||
raise
|
||||
with self._conn_lock:
|
||||
if generation == self._generation:
|
||||
self._connections.add(conn)
|
||||
break
|
||||
conn.close()
|
||||
self._local.conn = conn
|
||||
self._local.generation = self._generation
|
||||
with self._conn_lock:
|
||||
self._connections.add(conn)
|
||||
self._local.generation = generation
|
||||
return conn
|
||||
|
||||
@property
|
||||
def generation(self) -> int:
|
||||
"""Bumped whenever every connection is closed or the model repointed."""
|
||||
return self._generation
|
||||
|
||||
@classmethod
|
||||
def close_all_instances(cls) -> None:
|
||||
"""Close every connection of every live model in this process."""
|
||||
@@ -608,22 +692,76 @@ class PromptModel:
|
||||
for model in models:
|
||||
model.close_all()
|
||||
|
||||
@classmethod
|
||||
def repoint_instances(cls, old_path: str, new_path: str) -> list:
|
||||
"""Point every model of ``old_path`` at ``new_path`` (Move library).
|
||||
|
||||
Every thread opens the new file on its next ``get_connection``. The
|
||||
old connections are detached, not closed, and returned: the caller
|
||||
retires them (``retire_detached``) while it still holds the old
|
||||
file's write lock. A write waiting on that lock is then interrupted
|
||||
and can never commit, so it fails (and, through
|
||||
``PromptDatabase.survive_library_switch``, runs again on the new file).
|
||||
"""
|
||||
old_key = os.path.normcase(os.path.realpath(old_path))
|
||||
with cls._instances_lock:
|
||||
models = list(cls._instances)
|
||||
with cls._init_lock:
|
||||
cls._initialized_paths.add(os.path.normcase(os.path.realpath(new_path)))
|
||||
detached = []
|
||||
for model in models:
|
||||
if model._schema_key() == old_key:
|
||||
detached.extend(model._detach_connections(new_path))
|
||||
return detached
|
||||
|
||||
@staticmethod
|
||||
def retire_detached(connections, logger=None) -> list:
|
||||
"""Retire detached connections without waiting for their owners.
|
||||
|
||||
Idle ones close now. One whose owner thread is inside a statement is
|
||||
never closed from here (that frees memory the owner is using: an
|
||||
access violation on Windows); it is interrupted, refuses to commit and
|
||||
is closed by its owner as the statement ends. Returns those still
|
||||
open, for ``await_detached`` once no lock the owners wait on is held.
|
||||
"""
|
||||
return retire_connections(connections, logger)
|
||||
|
||||
@staticmethod
|
||||
def await_detached(connections, logger=None) -> bool:
|
||||
"""Wait until retired connections are closed; False if some are not."""
|
||||
return await_closed(connections, BUSY_TIMEOUT_SECONDS, logger)
|
||||
|
||||
@staticmethod
|
||||
def close_detached(connections, logger=None) -> bool:
|
||||
"""Retire connections ``repoint_instances`` detached and await them.
|
||||
|
||||
Do not call it while holding a lock their owners may wait on; retire
|
||||
them there (``retire_detached``) and await them after releasing it.
|
||||
"""
|
||||
pending = retire_connections(connections, logger)
|
||||
return await_closed(pending, BUSY_TIMEOUT_SECONDS, logger)
|
||||
|
||||
def close_all(self) -> None:
|
||||
"""Close every thread's connection; they reopen lazily on next use.
|
||||
|
||||
Needed before deleting the database file (Windows keeps open files
|
||||
locked) and by tests that ran queries through executor threads.
|
||||
"""
|
||||
self.close_detached(self._detach_connections(), self.logger)
|
||||
|
||||
def _detach_connections(self, new_path: Optional[str] = None) -> list:
|
||||
"""Forget every connection (optionally pointing the model elsewhere).
|
||||
|
||||
Returns them for the caller to close; threads reopen on next use.
|
||||
"""
|
||||
with self._conn_lock:
|
||||
if new_path is not None:
|
||||
self.db_path = new_path
|
||||
connections = list(self._connections)
|
||||
self._connections.clear()
|
||||
self._generation += 1
|
||||
self._local.conn = None
|
||||
for conn in connections:
|
||||
try:
|
||||
conn.close()
|
||||
except sqlite3.Error as e:
|
||||
self.logger.warning(f"Error closing database connection: {e}")
|
||||
return connections
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the calling thread's connection, if it has one.
|
||||
@@ -637,10 +775,7 @@ class PromptModel:
|
||||
self._local.conn = None
|
||||
with self._conn_lock:
|
||||
self._connections.discard(conn)
|
||||
try:
|
||||
conn.close()
|
||||
except sqlite3.Error as e:
|
||||
self.logger.warning(f"Error closing database connection: {e}")
|
||||
self.close_detached([conn], self.logger)
|
||||
|
||||
def _migrate_workflow_name_removal(self, conn: sqlite3.Connection) -> None:
|
||||
"""
|
||||
@@ -711,6 +846,7 @@ class PromptModel:
|
||||
columns = {column[1]: column[2] for column in cursor.fetchall()}
|
||||
if columns.get("prompt_id") != "TEXT":
|
||||
return
|
||||
_refuse_rebuild_with_documents(columns)
|
||||
|
||||
self.logger.info("Migrating foreign key types: prompt_id TEXT -> INTEGER")
|
||||
with self._table_rebuild(conn):
|
||||
@@ -876,6 +1012,7 @@ class PromptModel:
|
||||
}
|
||||
if "file_path" in columns:
|
||||
return
|
||||
_refuse_rebuild_with_documents(columns)
|
||||
|
||||
self.logger.info("Migrating database: image uniqueness by full path")
|
||||
with self._table_rebuild(conn):
|
||||
@@ -1029,7 +1166,7 @@ class PromptModel:
|
||||
|
||||
@staticmethod
|
||||
def verify_database_file(
|
||||
path: str, *, require_library_state: bool = False
|
||||
path: str, *, require_library_state: bool = False, documents: bool = True
|
||||
) -> Tuple[bool, str]:
|
||||
"""
|
||||
Check that a file is a healthy prompts database before restoring it.
|
||||
@@ -1040,9 +1177,13 @@ class PromptModel:
|
||||
Args:
|
||||
path: File to inspect
|
||||
require_library_state: Also require the tables every current
|
||||
database has (Library state, image favorites, saved views,
|
||||
image annotations and the review queue), as a migrated
|
||||
database has (Library state, image favorites, image
|
||||
categories, saved views, collections, image annotations and
|
||||
the review queue), as a migrated
|
||||
restore stage must; old backups may lack them.
|
||||
documents: Read every workflow document and refuse a damaged
|
||||
one. A restore checks its upload without it and repairs
|
||||
damaged documents on the stage (``database/restore_stage.py``).
|
||||
|
||||
Returns:
|
||||
(ok, reason): reason is "ok" when the file is acceptable, otherwise
|
||||
@@ -1102,11 +1243,31 @@ class PromptModel:
|
||||
)
|
||||
if favorites_error:
|
||||
return False, favorites_error
|
||||
categories_error = image_categories_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if categories_error:
|
||||
return False, categories_error
|
||||
archive_error = prompt_archive_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if archive_error:
|
||||
return False, archive_error
|
||||
removals_error = image_removals_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if removals_error:
|
||||
return False, removals_error
|
||||
views_error = saved_views_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if views_error:
|
||||
return False, views_error
|
||||
collections_error = image_collections_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if collections_error:
|
||||
return False, collections_error
|
||||
annotation_error = annotation_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
@@ -1117,6 +1278,16 @@ class PromptModel:
|
||||
)
|
||||
if review_error:
|
||||
return False, review_error
|
||||
autotag_error = autotag_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if autotag_error:
|
||||
return False, autotag_error
|
||||
documents_error = workflow_documents_schema_error(
|
||||
conn, allow_missing=not require_library_state, documents=documents
|
||||
)
|
||||
if documents_error:
|
||||
return False, documents_error
|
||||
return True, "ok"
|
||||
except sqlite3.Error as e:
|
||||
return False, f"Invalid SQLite database: {e}"
|
||||
@@ -1139,21 +1310,51 @@ class PromptModel:
|
||||
Path(src_path).resolve().as_uri() + "?mode=ro", uri=True
|
||||
)
|
||||
try:
|
||||
source.backup(self.get_connection())
|
||||
target = self.get_connection()
|
||||
# The target is written under backup's step too: mark it in use
|
||||
# so a concurrent close_all retires it instead of closing it.
|
||||
with target.using():
|
||||
source.backup(target)
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
def stage_restore(self, src_path: str):
|
||||
"""Stage ``src_path`` beside this database for a restore (blocking).
|
||||
|
||||
Returns a ``StagedRestore`` (``database/restore_stage.py``): a
|
||||
migrated, repaired and verified copy in the library's folder, its
|
||||
digest and what it holds. Neither the live database nor the source
|
||||
is touched. Raises ValueError when either does not verify.
|
||||
"""
|
||||
return restore_stage.stage_restore(PromptModel, self.db_path, src_path)
|
||||
|
||||
def publish_staged(self, staged) -> str:
|
||||
"""Replace the live database with ``staged``; return the safety backup.
|
||||
|
||||
See ``restore_stage.publish_staged``: the staged bytes are rechecked,
|
||||
the current database is kept as ``<db_path>.backup_<stamp>`` and the
|
||||
WAL is truncated afterwards.
|
||||
"""
|
||||
backup_path = restore_stage.publish_staged(self, staged)
|
||||
self.logger.info("Database restored from a staged copy")
|
||||
from .execution_history import notify_history_restore
|
||||
|
||||
notify_history_restore(self.db_path)
|
||||
return backup_path
|
||||
|
||||
def restore_from_file(self, src_path: str) -> str:
|
||||
"""
|
||||
Replace the live database with the file at src_path.
|
||||
|
||||
A disposable SQLite backup of the source is migrated and verified first.
|
||||
Only then is the current database backed up next to itself as
|
||||
``<db_path>.backup_<YYYYmmdd_HHMMSS>`` (WAL content included), and the
|
||||
staged pages replace the live pages through SQLite's backup API.
|
||||
Migration failures never replace live content or modify the source.
|
||||
An online replacement failure leaves the live database intact and
|
||||
propagates; the completed safety backup remains available.
|
||||
``stage_restore`` then ``publish_staged``: a disposable copy of the
|
||||
source is migrated, repaired and verified in the library's folder
|
||||
first. Only then is the current database backed up next to itself as
|
||||
``<db_path>.backup_<YYYYmmdd_HHMMSS_ffffff>`` (WAL content included),
|
||||
and the staged pages replace the live pages through SQLite's backup
|
||||
API. Staging failures never replace live content or modify the
|
||||
source. An online replacement failure leaves the live database intact
|
||||
and propagates; the completed safety backup remains available. The
|
||||
stage is removed either way.
|
||||
|
||||
Args:
|
||||
src_path: SQLite file to restore
|
||||
@@ -1163,55 +1364,14 @@ class PromptModel:
|
||||
there was no database to back up.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the previous database could not be backed up
|
||||
ValueError: If the source or its migrated copy does not verify
|
||||
SafetyBackupFailed: If the previous database could not be backed up
|
||||
sqlite3.OperationalError: If SQLite refuses online replacement
|
||||
"""
|
||||
ok, reason = self.verify_database_file(src_path)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
with tempfile.TemporaryDirectory(prefix="prompt-manager-restore-") as directory:
|
||||
staged_path = str(Path(directory) / "staged.db")
|
||||
source = sqlite3.connect(
|
||||
Path(src_path).resolve().as_uri() + "?mode=ro", uri=True
|
||||
)
|
||||
try:
|
||||
with closing(sqlite3.connect(staged_path)) as destination:
|
||||
source.backup(destination)
|
||||
finally:
|
||||
source.close()
|
||||
staged = None
|
||||
try:
|
||||
staged = PromptModel(staged_path)
|
||||
rotate_history_epoch(staged.get_connection())
|
||||
ok, reason = self.verify_database_file(
|
||||
staged_path, require_library_state=True
|
||||
)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
staged.close_all()
|
||||
backup_path = ""
|
||||
if os.path.exists(self.db_path):
|
||||
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup_path = f"{self.db_path}.backup_{timestamp}"
|
||||
if not self.backup_database(backup_path):
|
||||
raise RuntimeError("Could not back up the current database")
|
||||
self.logger.info(f"Current database backed up to: {backup_path}")
|
||||
try:
|
||||
self._replace_contents_from(staged_path)
|
||||
except sqlite3.OperationalError as exc:
|
||||
exc._prompt_manager_restore_backup_created = bool(backup_path)
|
||||
raise
|
||||
with PromptModel._init_lock:
|
||||
PromptModel._initialized_paths.add(self._schema_key())
|
||||
self.logger.info(f"Database restored from {src_path}")
|
||||
from .execution_history import notify_history_restore
|
||||
|
||||
notify_history_restore(self.db_path)
|
||||
return backup_path
|
||||
finally:
|
||||
if staged is not None:
|
||||
staged.close_all()
|
||||
with PromptModel._init_lock:
|
||||
PromptModel._initialized_paths.discard(
|
||||
os.path.normcase(os.path.realpath(staged_path))
|
||||
)
|
||||
staged = self.stage_restore(src_path)
|
||||
try:
|
||||
backup_path = self.publish_staged(staged)
|
||||
finally:
|
||||
staged.discard()
|
||||
self.logger.info(f"Database restored from {src_path}")
|
||||
return backup_path
|
||||
|
||||
@@ -7,17 +7,25 @@ import json
|
||||
import math
|
||||
import datetime
|
||||
import os
|
||||
from typing import Optional, List, Dict, Any, Set, Tuple
|
||||
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
|
||||
|
||||
try:
|
||||
from ..utils.inspection_response import inherited_category
|
||||
except ImportError:
|
||||
from utils.inspection_response import inherited_category
|
||||
|
||||
from .models import PromptModel, normalize_image_path, utc_now_iso
|
||||
from .pairs import PairDependencyError, PairOperations # noqa: F401 (re-exported)
|
||||
from .library import LibraryOperations, titled
|
||||
from .image_favorites import ImageFavoriteOperations
|
||||
from .image_removal import ImageRemovalOperations, removed_from_library
|
||||
from .image_collections import ImageCollectionOperations
|
||||
from .annotations import AnnotationOperations
|
||||
from .review import ReviewOperations
|
||||
from .review_decisions import ReviewDecisionOperations
|
||||
from .saved_views import SavedViewOperations
|
||||
from .library_commands import LibraryCommandOperations
|
||||
from .library_archive import LibraryArchiveOperations, delete_prompt_rows
|
||||
from .vocabulary import VocabularyOperations
|
||||
from .execution_history import ExecutionHistoryOperations
|
||||
from .execution_contexts import ExecutionContextOperations
|
||||
@@ -25,6 +33,16 @@ from .execution_outputs import ExecutionOutputOperations
|
||||
from .history_reads import HistoryReadOperations
|
||||
from .health import HealthOperations, iter_consistency_issues
|
||||
from .duplicate_copies import DuplicateCopyOperations
|
||||
from .workflow_storage import WorkflowStorageOperations
|
||||
from .category_import import CategoryImportOperations, EmptyReplacement # noqa: F401
|
||||
from .image_categories import ImageCategoryOperations
|
||||
from .workflow_documents import (
|
||||
DocumentReader,
|
||||
documents_of_rows,
|
||||
release_documents,
|
||||
resolve_image_row,
|
||||
stored_values,
|
||||
)
|
||||
from .maintenance import ( # noqa: F401 (ORPHANED_PROMPTS_SQL is re-exported)
|
||||
ORPHANED_PROMPTS_SQL,
|
||||
_LINK_ROWS_SQL,
|
||||
@@ -38,6 +56,7 @@ try:
|
||||
from ..utils.logging_config import get_logger
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
from ..utils.json_strict import dumps_strict
|
||||
from ..utils import library_location
|
||||
except ImportError:
|
||||
import sys
|
||||
|
||||
@@ -46,6 +65,7 @@ except ImportError:
|
||||
from utils.logging_config import get_logger
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.json_strict import dumps_strict
|
||||
from utils import library_location
|
||||
|
||||
|
||||
# Subquery to fetch tags from junction table, embedded in SELECT statements
|
||||
@@ -91,40 +111,56 @@ def escape_like(term: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _resolve_db_path(db_path: Optional[str] = None) -> str:
|
||||
"""Resolve the database path from config, falling back to defaults.
|
||||
def _configured_default_path():
|
||||
"""config.json's ``database.default_path``, or None.
|
||||
|
||||
When no explicit path is given, reads PromptManagerConfig.DEFAULT_DB_PATH
|
||||
(set via config.json). Relative paths are resolved against the extension
|
||||
root directory so the database location is predictable regardless of the
|
||||
process working directory (important for Docker deployments).
|
||||
The package-qualified config wins over a standalone ``py.config`` (a
|
||||
custom-node package uses its own configuration).
|
||||
"""
|
||||
try:
|
||||
try:
|
||||
from ..py.config import PromptManagerConfig
|
||||
except ImportError:
|
||||
from py.config import PromptManagerConfig
|
||||
except Exception:
|
||||
return None
|
||||
return getattr(PromptManagerConfig, "DEFAULT_DB_PATH", None)
|
||||
|
||||
|
||||
def resolve_library_location():
|
||||
"""Where the library lives now (``utils.library_location``, #249 S6b).
|
||||
|
||||
The image sources that rule out a ``PROMPT_MANAGER_DATA_DIR`` are read by
|
||||
the resolver itself, exactly as ``config_location`` reads them.
|
||||
"""
|
||||
return library_location.resolve_library(_configured_default_path())
|
||||
|
||||
|
||||
def _resolve_db_path(db_path: Optional[str] = None) -> str:
|
||||
"""Resolve the database path through the one library resolver.
|
||||
|
||||
With no explicit path, ``utils.library_location.resolve_library`` decides
|
||||
(``PROMPT_MANAGER_DATA_DIR``, an explicit ``database.default_path``, a
|
||||
node-folder library not moved yet, else ComfyUI's ``user/__promptmanager``).
|
||||
An explicit relative path is resolved against the extension root, as
|
||||
before, so the location does not depend on the working directory.
|
||||
|
||||
Args:
|
||||
db_path: Explicit database path, or None to use config/default.
|
||||
db_path: Explicit database path, or None to use the resolver.
|
||||
|
||||
Returns:
|
||||
Absolute path to the SQLite database file.
|
||||
Absolute path to the SQLite database file; its folder exists (a new
|
||||
data folder is created owner-only).
|
||||
"""
|
||||
extension_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
if db_path is None:
|
||||
try:
|
||||
try:
|
||||
from ..py.config import PromptManagerConfig
|
||||
except ImportError:
|
||||
from py.config import PromptManagerConfig
|
||||
|
||||
db_path = PromptManagerConfig.DEFAULT_DB_PATH
|
||||
except Exception:
|
||||
db_path = "prompts.db"
|
||||
|
||||
if not os.path.isabs(db_path):
|
||||
db_path = resolve_library_location().db_path
|
||||
elif not os.path.isabs(db_path):
|
||||
extension_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
db_path = os.path.normpath(os.path.join(extension_root, db_path))
|
||||
|
||||
# Ensure parent directory exists for custom paths
|
||||
parent = os.path.dirname(db_path)
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
library_location.ensure_private_dir(parent)
|
||||
|
||||
return db_path
|
||||
|
||||
@@ -210,12 +246,17 @@ LEFT JOIN prompt_library_state pls ON pls.prompt_id = pp.positive_prompt_id
|
||||
WHERE gi.image_path IN ({marks}) OR gi.file_path IN ({marks}){fallback}"""
|
||||
# One image row describes a card only when no other row claims the same file
|
||||
# (as in get_image_link_summaries, several rows are ambiguous). The pair scope
|
||||
# and the linked positive follow _LINK_SUMMARY_SQL; category, rating, tags and
|
||||
# the searchable text and titles come from that positive (spec D12/D13). Every
|
||||
# and the linked positive follow _LINK_SUMMARY_SQL; rating, tags and the
|
||||
# searchable text and titles come from that positive (spec D12/D13). The raw
|
||||
# recipe and positive categories are read for the inherited category, which
|
||||
# Python decides (``inherited_category``, #332 MC4); SQL never filters by
|
||||
# category. Every
|
||||
# writer since the 3.2.4 migration stores file_path, so it is the path key.
|
||||
# The prompt-level conditions are uncorrelated IN sets SQLite builds once, and
|
||||
# the CASE evaluates the per-row ambiguity probe only for admitted rows.
|
||||
_LINK_FILTER_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id
|
||||
_LINK_FILTER_SQL = """SELECT gi.file_path AS file_path, p.rating AS rating,
|
||||
{hit} AS matched, p.id AS positive, pp.id IS NOT NULL AS has_pair,
|
||||
pp.category AS pair_category, p.category AS category
|
||||
FROM generated_images gi
|
||||
LEFT JOIN prompt_pairs pp ON pp.id = (
|
||||
SELECT CASE WHEN COUNT(DISTINCT ip.pair_id) = 1 THEN MIN(ip.pair_id) END
|
||||
@@ -224,10 +265,31 @@ JOIN prompts p ON p.id = COALESCE(pp.positive_prompt_id, gi.prompt_id)
|
||||
WHERE gi.file_path IS NOT NULL AND CASE WHEN {admit} THEN NOT EXISTS (
|
||||
SELECT 1 FROM generated_images other
|
||||
WHERE other.file_path = gi.file_path AND other.id <> gi.id) ELSE 0 END"""
|
||||
# _LINK_FILTER_SQL's rows as one JSON array, built by SQLite in one step (#354).
|
||||
# Python's sqlite3 releases and retakes the GIL around every step, so a read
|
||||
# that steps once per linked row waits on a busy ComfyUI thread thousands of
|
||||
# times. A BLOB value (no writer stores one) makes the value NULL (or
|
||||
# json_array refuses it), and the read streams instead. The raw categories are
|
||||
# put in only for a category read (#332), so plain reads stay as small as #354
|
||||
# measured them.
|
||||
_LINK_MATCHES_JSON_SQL = """SELECT CASE
|
||||
WHEN max({blob}) THEN NULL
|
||||
ELSE json_group_array(json_array({columns})) END
|
||||
FROM ({select})"""
|
||||
_LINK_PLAIN_COLUMNS = ("file_path", "rating", "matched", "positive")
|
||||
_LINK_CATEGORY_COLUMNS = _LINK_PLAIN_COLUMNS + ("has_pair", "pair_category", "category")
|
||||
# The columns a BLOB can reach: matched, positive and has_pair are computed.
|
||||
_LINK_STORED_COLUMNS = ("file_path", "rating", "pair_category", "category")
|
||||
# Set when this SQLite has no JSON functions: every later read streams.
|
||||
_one_step_unsupported = False
|
||||
# One warning per process when a read falls back to streaming.
|
||||
_one_step_warned = False
|
||||
# The linked positive of each given path (one unambiguous link, pair scope as
|
||||
# in _LINK_FILTER_SQL) with its category; {hit} as in link_filter_query. Used
|
||||
# for the few files whose current image tags meet a Tags filter (#249 B3).
|
||||
_LINK_FACTS_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id, p.category
|
||||
# in _LINK_FILTER_SQL) with the raw categories of its inherited category; {hit}
|
||||
# as in link_filter_query. Used for the few files whose current image tags meet
|
||||
# a Tags filter (#249 B3).
|
||||
_LINK_FACTS_SQL = """SELECT gi.file_path, p.rating, {hit}, p.id,
|
||||
pp.id IS NOT NULL, pp.category, p.category
|
||||
FROM generated_images gi
|
||||
LEFT JOIN prompt_pairs pp ON pp.id = (
|
||||
SELECT CASE WHEN COUNT(DISTINCT ip.pair_id) = 1 THEN MIN(ip.pair_id) END
|
||||
@@ -256,8 +318,24 @@ _LINK_FILTER_TAG = (
|
||||
)
|
||||
|
||||
|
||||
# One unambiguously linked image's facts for the Images filters, a plain tuple
|
||||
# (hit, rating, positive id, inherited category): built per linked row, so no
|
||||
# NamedTuple. The category is the inherited one (#332 MC4: the recipe's when
|
||||
# set, else the positive's), never the image's own category.
|
||||
LinkMatch = Tuple[bool, Optional[int], int, Any]
|
||||
|
||||
|
||||
def _link_match(rating, matched, positive, has_pair, pair_category, category):
|
||||
inherited, _scope = inherited_category(category, pair_category, bool(has_pair))
|
||||
return (bool(matched), rating, positive, inherited)
|
||||
|
||||
|
||||
def link_filter_query(
|
||||
q: str, category: Optional[str], tags: List[str], *, every_link: bool = False
|
||||
q: str,
|
||||
tags: List[str],
|
||||
*,
|
||||
every_link: bool = False,
|
||||
category: Optional[str] = None,
|
||||
) -> Tuple[str, Dict[str, Any]]:
|
||||
"""The workspace filter SELECT and its named parameters (all bound).
|
||||
|
||||
@@ -271,7 +349,9 @@ def link_filter_query(
|
||||
params["pattern"] = f"%{escape_like(q)}%"
|
||||
admitted = []
|
||||
if category is not None:
|
||||
admitted.append("p.category = :category")
|
||||
# Only narrows: an inherited category is one of the two raw values;
|
||||
# which one is decided in Python (inherited_category, #332 MC4).
|
||||
admitted.append("(p.category = :category OR pp.category = :category)")
|
||||
params["category"] = category
|
||||
for index, tag in enumerate(tags):
|
||||
admitted.append(_LINK_FILTER_TAG.format(index=index))
|
||||
@@ -282,6 +362,50 @@ def link_filter_query(
|
||||
return sql, params
|
||||
|
||||
|
||||
def link_matches_json_query(sql: str, categories: bool = False) -> str:
|
||||
"""*sql* from link_filter_query, wrapped to return its rows as one JSON value.
|
||||
|
||||
With *categories* each row also carries the raw values the inherited
|
||||
category is decided from; otherwise only path, rating, hit and positive.
|
||||
"""
|
||||
columns = _LINK_CATEGORY_COLUMNS if categories else _LINK_PLAIN_COLUMNS
|
||||
blob = " OR ".join(
|
||||
f"typeof({name}) = 'blob'" for name in _LINK_STORED_COLUMNS if name in columns
|
||||
)
|
||||
return _LINK_MATCHES_JSON_SQL.format(
|
||||
blob=blob, columns=", ".join(columns), select=sql
|
||||
)
|
||||
|
||||
|
||||
def _link_matches_from_rows(
|
||||
rows: Iterable[Sequence[Any]], categories: bool, wanted: Optional[Set[str]]
|
||||
) -> Dict[str, LinkMatch]:
|
||||
"""LinkMatch per wanted path from link rows (one-step or streamed).
|
||||
|
||||
Without *categories* the inherited category is None, as the caller then
|
||||
never compares it.
|
||||
"""
|
||||
matches: Dict[str, LinkMatch] = {}
|
||||
for path, rating, matched, positive, *raw in rows:
|
||||
if wanted is None or path in wanted:
|
||||
if categories:
|
||||
matches[path] = _link_match(rating, matched, positive, *raw[:3])
|
||||
else:
|
||||
matches[path] = (bool(matched), rating, positive, None)
|
||||
return matches
|
||||
|
||||
|
||||
def _streamed_link_matches(
|
||||
conn: sqlite3.Connection,
|
||||
sql: str,
|
||||
params: Dict[str, Any],
|
||||
wanted: Optional[Set[str]],
|
||||
categories: bool = False,
|
||||
) -> Dict[str, LinkMatch]:
|
||||
"""The link read stepped once per row (the fallback of the one-step read)."""
|
||||
return _link_matches_from_rows(conn.execute(sql, params), categories, wanted)
|
||||
|
||||
|
||||
_ASCII_FOLD = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
|
||||
|
||||
|
||||
@@ -326,12 +450,14 @@ class PromptDatabase(
|
||||
PairOperations,
|
||||
LibraryOperations,
|
||||
LibraryCommandOperations,
|
||||
LibraryArchiveOperations,
|
||||
VocabularyOperations,
|
||||
ExecutionHistoryOperations,
|
||||
ExecutionContextOperations,
|
||||
ExecutionOutputOperations,
|
||||
HistoryReadOperations,
|
||||
ImageFavoriteOperations,
|
||||
ImageCollectionOperations,
|
||||
AnnotationOperations,
|
||||
ReviewOperations,
|
||||
ReviewDecisionOperations,
|
||||
@@ -339,6 +465,10 @@ class PromptDatabase(
|
||||
HealthOperations,
|
||||
MaintenanceOperations,
|
||||
DuplicateCopyOperations,
|
||||
WorkflowStorageOperations,
|
||||
CategoryImportOperations,
|
||||
ImageCategoryOperations,
|
||||
ImageRemovalOperations,
|
||||
):
|
||||
"""Database operations class for managing prompts."""
|
||||
|
||||
@@ -356,6 +486,25 @@ class PromptDatabase(
|
||||
self.model = PromptModel(db_path)
|
||||
self.logger.debug("Database operations initialized successfully")
|
||||
|
||||
def survive_library_switch(self, write):
|
||||
"""Run ``write()``; once more if it failed while the library moved.
|
||||
|
||||
A background write (run counting, the image monitor) that was waiting
|
||||
on the old file when Move library switched fails when its connection
|
||||
is closed. When the model's generation changed during the attempt,
|
||||
the write runs again on a fresh connection, which is the new library;
|
||||
any other failure is raised as it was. ``write`` must open its own
|
||||
transaction (every PromptDatabase method does).
|
||||
"""
|
||||
generation = self.model.generation
|
||||
try:
|
||||
return write()
|
||||
except Exception:
|
||||
if self.model.generation == generation:
|
||||
raise
|
||||
self.logger.info("Running a write again on the moved library")
|
||||
return write()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the calling thread's database connection (see PromptModel.close)."""
|
||||
self.model.close()
|
||||
@@ -409,26 +558,9 @@ class PromptDatabase(
|
||||
|
||||
try:
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT INTO prompts (
|
||||
text, category, tags, rating, notes, hash, created_at,
|
||||
updated_at
|
||||
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
text.strip(),
|
||||
category,
|
||||
rating,
|
||||
notes,
|
||||
prompt_hash,
|
||||
datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
),
|
||||
prompt_id = self._insert_prompt(
|
||||
conn, text, category, tags, rating, notes, prompt_hash
|
||||
)
|
||||
prompt_id = cursor.lastrowid
|
||||
if tags:
|
||||
self._sync_prompt_tags(conn, prompt_id, tags)
|
||||
conn.commit()
|
||||
self.logger.debug(f"Successfully saved prompt with ID: {prompt_id}")
|
||||
return prompt_id
|
||||
@@ -444,6 +576,32 @@ class PromptDatabase(
|
||||
)
|
||||
return existing["id"]
|
||||
|
||||
def _insert_prompt(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
text: str,
|
||||
category: Optional[str],
|
||||
tags: Optional[List[str]],
|
||||
rating: Optional[int],
|
||||
notes: Optional[str],
|
||||
prompt_hash: Optional[str],
|
||||
) -> int:
|
||||
"""Insert one prompt row and its tags on ``conn`` (no commit)."""
|
||||
now = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT INTO prompts (
|
||||
text, category, tags, rating, notes, hash, created_at,
|
||||
updated_at
|
||||
) VALUES (?, ?, NULL, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(text.strip(), category, rating, notes, prompt_hash, now, now),
|
||||
)
|
||||
prompt_id = cursor.lastrowid
|
||||
if tags:
|
||||
self._sync_prompt_tags(conn, prompt_id, tags)
|
||||
return prompt_id
|
||||
|
||||
def record_prompt_use(self, prompt_id: int, times: int = 1) -> bool:
|
||||
"""
|
||||
Count runs of a prompt and mark it as most recently used.
|
||||
@@ -776,10 +934,12 @@ class PromptDatabase(
|
||||
"""
|
||||
with self._pair_transaction() as conn:
|
||||
self._ensure_prompts_unpaired(conn, [prompt_id])
|
||||
documents = documents_of_rows(conn, "prompt_id", [prompt_id])
|
||||
# First delete related images to avoid foreign key constraint
|
||||
conn.execute(
|
||||
"DELETE FROM generated_images WHERE prompt_id = ?", (prompt_id,)
|
||||
)
|
||||
release_documents(conn, documents)
|
||||
# Then delete the prompt
|
||||
cursor = conn.execute("DELETE FROM prompts WHERE id = ?", (prompt_id,))
|
||||
conn.commit()
|
||||
@@ -803,10 +963,12 @@ class PromptDatabase(
|
||||
return 0
|
||||
self._ensure_prompts_unpaired(conn, ids)
|
||||
placeholders = ",".join("?" * len(ids))
|
||||
documents = documents_of_rows(conn, "prompt_id", ids)
|
||||
conn.execute(
|
||||
f"DELETE FROM generated_images WHERE prompt_id IN ({placeholders})",
|
||||
ids,
|
||||
)
|
||||
release_documents(conn, documents)
|
||||
conn.execute(
|
||||
f"DELETE FROM prompt_tags WHERE prompt_id IN ({placeholders})",
|
||||
ids,
|
||||
@@ -1567,84 +1729,142 @@ class PromptDatabase(
|
||||
metadata: Optional metadata about the image
|
||||
|
||||
Returns:
|
||||
int: The ID of the created image record
|
||||
int: The ID of the created image record, or 0 when nothing was
|
||||
linked (invalid or unknown prompt, already linked, or an error,
|
||||
which is logged)
|
||||
"""
|
||||
# Validate that prompt_id is a valid integer and exists in prompts table
|
||||
try:
|
||||
# Convert prompt_id to integer if it's a string number
|
||||
if isinstance(prompt_id, str) and prompt_id.isdigit():
|
||||
prompt_id_int = int(prompt_id)
|
||||
elif isinstance(prompt_id, int):
|
||||
prompt_id_int = prompt_id
|
||||
else:
|
||||
# Handle temporary IDs like "temp_123456" - skip linking
|
||||
if isinstance(prompt_id, str) and prompt_id.startswith("temp_"):
|
||||
self.logger.debug(
|
||||
f"Skipping image linking for temporary prompt ID: {prompt_id}"
|
||||
)
|
||||
return 0
|
||||
else:
|
||||
self.logger.warning(
|
||||
f"Invalid prompt_id format: {prompt_id}, skipping image linking"
|
||||
)
|
||||
return 0
|
||||
|
||||
# Verify the prompt exists in the database
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(
|
||||
"SELECT id FROM prompts WHERE id = ?", (prompt_id_int,)
|
||||
)
|
||||
if not cursor.fetchone():
|
||||
self.logger.warning(
|
||||
f"Prompt ID {prompt_id_int} not found in database, skipping "
|
||||
"image linking"
|
||||
)
|
||||
return 0
|
||||
|
||||
# Proceed with linking
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
filename = os.path.basename(image_path)
|
||||
file_path = normalize_image_path(image_path)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
|
||||
# INSERT OR IGNORE skips a file already linked to this prompt
|
||||
# (same prompt_id + normalised full path)
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO generated_images
|
||||
(prompt_id, image_path, filename, file_path, file_size, width,
|
||||
height, format, workflow_data, prompt_metadata, parameters)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
prompt_id_int,
|
||||
image_path,
|
||||
filename,
|
||||
file_path,
|
||||
file_size,
|
||||
width,
|
||||
height,
|
||||
image_format,
|
||||
dumps_strict(metadata.get("workflow", {})),
|
||||
dumps_strict(metadata.get("prompt", {})),
|
||||
dumps_strict(metadata.get("parameters", {})),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
if cursor.rowcount == 0:
|
||||
self.logger.debug(
|
||||
f"Image {file_path} already linked to prompt {prompt_id_int}"
|
||||
)
|
||||
return 0
|
||||
|
||||
return cursor.lastrowid
|
||||
|
||||
return self.link_image_to_prompt_or_raise(prompt_id, image_path, metadata)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error linking image to prompt {prompt_id}: {e}")
|
||||
return 0
|
||||
|
||||
def link_image_to_prompt_or_raise(
|
||||
self, prompt_id: str, image_path: str, metadata: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""``link_image_to_prompt`` that raises database errors.
|
||||
|
||||
The image monitor links through this so a write that failed because
|
||||
Move library switched reaches ``survive_library_switch`` and runs
|
||||
again on the new library (#249 S6b). Each call is one transaction:
|
||||
a failed attempt rolls back, and a committed one does not raise.
|
||||
Returns 0 for an invalid or unknown prompt or an image already linked.
|
||||
"""
|
||||
# Validate that prompt_id is a valid integer and exists in prompts table
|
||||
# Convert prompt_id to integer if it's a string number
|
||||
if isinstance(prompt_id, str) and prompt_id.isdigit():
|
||||
prompt_id_int = int(prompt_id)
|
||||
elif isinstance(prompt_id, int):
|
||||
prompt_id_int = prompt_id
|
||||
else:
|
||||
# Handle temporary IDs like "temp_123456" - skip linking
|
||||
if isinstance(prompt_id, str) and prompt_id.startswith("temp_"):
|
||||
self.logger.debug(
|
||||
f"Skipping image linking for temporary prompt ID: {prompt_id}"
|
||||
)
|
||||
return 0
|
||||
else:
|
||||
self.logger.warning(
|
||||
f"Invalid prompt_id format: {prompt_id}, skipping image linking"
|
||||
)
|
||||
return 0
|
||||
|
||||
# Verify the prompt exists in the database
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(
|
||||
"SELECT id FROM prompts WHERE id = ?", (prompt_id_int,)
|
||||
)
|
||||
if not cursor.fetchone():
|
||||
self.logger.warning(
|
||||
f"Prompt ID {prompt_id_int} not found in database, skipping "
|
||||
"image linking"
|
||||
)
|
||||
return 0
|
||||
|
||||
# Proceed with linking
|
||||
image_id = self._insert_image_link(
|
||||
conn, prompt_id_int, image_path, metadata
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
if not image_id:
|
||||
self.logger.debug(
|
||||
f"Image {normalize_image_path(image_path)} already linked to "
|
||||
f"prompt {prompt_id_int}"
|
||||
)
|
||||
return 0
|
||||
|
||||
return image_id
|
||||
|
||||
def _insert_image_link(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
prompt_id_int: int,
|
||||
image_path: str,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> int:
|
||||
"""Insert one image link on ``conn`` (no commit).
|
||||
|
||||
Returns the new image row's id, or 0 when the file is already linked
|
||||
to this prompt (``INSERT OR IGNORE``).
|
||||
"""
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
if removed_from_library(conn, image_path):
|
||||
# #480: the user removed this exact file from the library; its
|
||||
# bytes are unchanged, so nothing links it again.
|
||||
self.logger.debug("Not linking a file removed from the library")
|
||||
return 0
|
||||
filename = os.path.basename(image_path)
|
||||
file_path = normalize_image_path(image_path)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
# Each workflow and prompt graph is stored once (new form).
|
||||
documents = stored_values(
|
||||
conn,
|
||||
{
|
||||
"workflow_data": dumps_strict(metadata.get("workflow", {})),
|
||||
"prompt_metadata": dumps_strict(metadata.get("prompt", {})),
|
||||
},
|
||||
)
|
||||
|
||||
# INSERT OR IGNORE skips a file already linked to this prompt
|
||||
# (same prompt_id + normalised full path)
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO generated_images
|
||||
(prompt_id, image_path, filename, file_path, file_size, width,
|
||||
height, format, workflow_data, prompt_metadata, parameters,
|
||||
workflow_document_id, prompt_document_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
prompt_id_int,
|
||||
image_path,
|
||||
filename,
|
||||
file_path,
|
||||
file_size,
|
||||
width,
|
||||
height,
|
||||
image_format,
|
||||
documents["workflow_data"],
|
||||
documents["prompt_metadata"],
|
||||
dumps_strict(metadata.get("parameters", {})),
|
||||
documents["workflow_document_id"],
|
||||
documents["prompt_document_id"],
|
||||
),
|
||||
)
|
||||
if cursor.rowcount == 0:
|
||||
# Already linked: a document made only for this row goes.
|
||||
release_documents(
|
||||
conn,
|
||||
(
|
||||
documents["workflow_document_id"],
|
||||
documents["prompt_document_id"],
|
||||
),
|
||||
)
|
||||
return 0
|
||||
return cursor.lastrowid
|
||||
|
||||
def get_prompt_images(self, prompt_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get all images associated with a prompt.
|
||||
@@ -1664,7 +1884,8 @@ class PromptDatabase(
|
||||
""",
|
||||
(prompt_id,),
|
||||
)
|
||||
return [self._image_row_to_dict(row) for row in cursor.fetchall()]
|
||||
reader = DocumentReader(conn)
|
||||
return [self._image_row_to_dict(row, reader) for row in cursor]
|
||||
|
||||
def get_recent_images(
|
||||
self, limit: int = 50, offset: int = 0
|
||||
@@ -1690,7 +1911,8 @@ class PromptDatabase(
|
||||
""",
|
||||
(int(limit), max(0, int(offset))),
|
||||
)
|
||||
return [self._image_row_to_dict(row) for row in cursor.fetchall()]
|
||||
reader = DocumentReader(conn)
|
||||
return [self._image_row_to_dict(row, reader) for row in cursor]
|
||||
|
||||
def get_all_images(
|
||||
self,
|
||||
@@ -1724,10 +1946,10 @@ class PromptDatabase(
|
||||
|
||||
with self.model.get_connection() as conn:
|
||||
cursor = conn.execute(sql, params)
|
||||
rows = cursor.fetchall()
|
||||
reader = DocumentReader(conn)
|
||||
result = []
|
||||
for row in rows:
|
||||
data = self._image_row_to_dict(row)
|
||||
for row in cursor:
|
||||
data = self._image_row_to_dict(row, reader)
|
||||
if row["_prompt_tags_list"]:
|
||||
data["prompt_tags"] = row["_prompt_tags_list"].split("|||")
|
||||
else:
|
||||
@@ -1765,7 +1987,8 @@ class PromptDatabase(
|
||||
max(0, int(offset)),
|
||||
),
|
||||
)
|
||||
return [self._image_row_to_dict(row) for row in cursor.fetchall()]
|
||||
reader = DocumentReader(conn)
|
||||
return [self._image_row_to_dict(row, reader) for row in cursor]
|
||||
|
||||
def get_image_by_id(self, image_id: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
@@ -1782,7 +2005,7 @@ class PromptDatabase(
|
||||
"SELECT * FROM generated_images WHERE id = ?", (image_id,)
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
return self._image_row_to_dict(row) if row else None
|
||||
return self._image_row_to_dict(row, DocumentReader(conn)) if row else None
|
||||
|
||||
def delete_image(self, image_id: int) -> bool:
|
||||
"""
|
||||
@@ -1795,9 +2018,11 @@ class PromptDatabase(
|
||||
bool: True if deletion was successful
|
||||
"""
|
||||
with self.model.get_connection() as conn:
|
||||
documents = documents_of_rows(conn, "id", [image_id])
|
||||
cursor = conn.execute(
|
||||
"DELETE FROM generated_images WHERE id = ?", (image_id,)
|
||||
)
|
||||
release_documents(conn, documents)
|
||||
conn.commit()
|
||||
return cursor.rowcount > 0
|
||||
|
||||
@@ -1831,12 +2056,16 @@ class PromptDatabase(
|
||||
cursor = conn.execute("SELECT id, image_path FROM generated_images")
|
||||
images = cursor.fetchall()
|
||||
|
||||
for image in images:
|
||||
if not os.path.exists(image["image_path"]):
|
||||
conn.execute(
|
||||
"DELETE FROM generated_images WHERE id = ?", (image["id"],)
|
||||
)
|
||||
removed_count += 1
|
||||
removed = [
|
||||
image["id"]
|
||||
for image in images
|
||||
if not os.path.exists(image["image_path"])
|
||||
]
|
||||
documents = documents_of_rows(conn, "id", removed)
|
||||
for image_id in removed:
|
||||
conn.execute("DELETE FROM generated_images WHERE id = ?", (image_id,))
|
||||
removed_count += 1
|
||||
release_documents(conn, documents)
|
||||
|
||||
conn.commit()
|
||||
|
||||
@@ -2124,14 +2353,10 @@ class PromptDatabase(
|
||||
Returns:
|
||||
Number of prompts actually deleted.
|
||||
"""
|
||||
count = 0
|
||||
with self._pair_transaction() as conn:
|
||||
self._ensure_prompts_unpaired(conn, prompt_ids)
|
||||
for pid in prompt_ids:
|
||||
conn.execute("DELETE FROM generated_images WHERE prompt_id = ?", (pid,))
|
||||
cursor = conn.execute("DELETE FROM prompts WHERE id = ?", (pid,))
|
||||
if cursor.rowcount > 0:
|
||||
count += 1
|
||||
# The workspace Library's delete runs the same statements (#443).
|
||||
count = delete_prompt_rows(conn, list(dict.fromkeys(prompt_ids)))
|
||||
conn.commit()
|
||||
return count
|
||||
|
||||
@@ -2205,7 +2430,8 @@ class PromptDatabase(
|
||||
cursor = conn.execute(
|
||||
"""SELECT gi.prompt_id, p.text, p.category, p.rating, p.notes,
|
||||
gi.workflow_data, gi.prompt_metadata, gi.generation_time,
|
||||
gi.parameters, gi.id, gi.image_path, gi.file_path
|
||||
gi.parameters, gi.id, gi.image_path, gi.file_path,
|
||||
gi.workflow_document_id, gi.prompt_document_id
|
||||
FROM generated_images gi
|
||||
JOIN prompts p ON gi.prompt_id = p.id
|
||||
WHERE gi.image_path = ? OR gi.file_path = ?
|
||||
@@ -2285,12 +2511,21 @@ class PromptDatabase(
|
||||
"text_source": source,
|
||||
}
|
||||
tags = self._get_prompt_tags(conn, row[0])
|
||||
texts = resolve_image_row(DocumentReader(conn), row)
|
||||
try:
|
||||
workflow = json.loads(row[5]) if row[5] else None
|
||||
workflow = (
|
||||
json.loads(texts["workflow_data"])
|
||||
if texts["workflow_data"]
|
||||
else None
|
||||
)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
workflow = None
|
||||
try:
|
||||
prompt_meta = json.loads(row[6]) if row[6] else None
|
||||
prompt_meta = (
|
||||
json.loads(texts["prompt_metadata"])
|
||||
if texts["prompt_metadata"]
|
||||
else None
|
||||
)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
prompt_meta = None
|
||||
return {
|
||||
@@ -2406,38 +2641,94 @@ class PromptDatabase(
|
||||
def get_image_link_matches(
|
||||
self,
|
||||
q: str,
|
||||
category: Optional[str],
|
||||
tags: List[str],
|
||||
*,
|
||||
every_link: bool = False,
|
||||
wanted: Optional[Set[str]] = None,
|
||||
) -> Dict[str, Tuple[bool, Optional[int], int]]:
|
||||
"""Map normalized paths of unambiguously linked images to
|
||||
(hit, rating, positive id).
|
||||
category: Optional[str] = None,
|
||||
) -> Dict[str, LinkMatch]:
|
||||
"""Map normalized paths of unambiguously linked images to a LinkMatch
|
||||
(hit, rating, positive id, inherited category).
|
||||
|
||||
Rows are limited to images whose linked positive has *category* and
|
||||
every tag in *tags* (exact names). ``hit`` says whether *q* occurs in
|
||||
Rows are limited to images whose linked positive has every tag in
|
||||
*tags* (exact names). The category filter is applied by the caller to
|
||||
the effective category (#332). ``hit`` says whether *q* occurs in
|
||||
the positive's text, Library title or a tag name; a pair's own title
|
||||
names its negative variant and is not searched.
|
||||
Without category or tags only hits are returned, unless *every_link*
|
||||
Without tags only hits are returned, unless *every_link*
|
||||
asks for every linked image (the rating sort and the positive count).
|
||||
The positive id is the card's linked positive. One SELECT whose rows
|
||||
are streamed; with *wanted* (the listing's normalized paths) only those
|
||||
paths are kept, so memory is bounded by the listing, not the database.
|
||||
The positive id is the card's linked positive. With *category* only
|
||||
links whose raw recipe or positive category is that value are read (a
|
||||
superset of the links that inherit it; the caller compares the
|
||||
inherited value) and the inherited category is decided per row;
|
||||
otherwise it is None, which keeps the plain reads cheap.
|
||||
|
||||
One SELECT, returned as one JSON value (#354): SQLite steps once, so
|
||||
the read takes the GIL back a constant two or three times instead of
|
||||
once per linked row while ComfyUI generates. Peak memory therefore
|
||||
grows with the database's matching linked rows (about 530 bytes each,
|
||||
more with the raw categories of a category read, freed on return); the
|
||||
result is bounded by *wanted* (the listing's normalized paths) when
|
||||
given. Where the JSON step fails (no JSON functions, a BLOB value, a
|
||||
JSON text over SQLite's length limit) the rows are streamed as before.
|
||||
"""
|
||||
sql, params = link_filter_query(q, category, tags, every_link=every_link)
|
||||
matches: Dict[str, Tuple[bool, Optional[int], int]] = {}
|
||||
sql, params = link_filter_query(
|
||||
q, tags, every_link=every_link, category=category
|
||||
)
|
||||
categories = category is not None
|
||||
with self.model.get_connection() as conn:
|
||||
for path, rating, matched, positive in conn.execute(sql, params):
|
||||
if wanted is None or path in wanted:
|
||||
matches[path] = (bool(matched), rating, positive)
|
||||
return matches
|
||||
rows = None
|
||||
if not _one_step_unsupported:
|
||||
rows = self._one_step_link_rows(conn, sql, params, categories)
|
||||
if rows is None:
|
||||
return _streamed_link_matches(conn, sql, params, wanted, categories)
|
||||
return _link_matches_from_rows(rows, categories, wanted)
|
||||
|
||||
def _one_step_link_rows(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
sql: str,
|
||||
params: Dict[str, Any],
|
||||
categories: bool = False,
|
||||
) -> Optional[List[List[Any]]]:
|
||||
"""*sql*'s rows from one JSON value, or None when this read must stream.
|
||||
|
||||
An interrupted read (Move library retiring the connection) is raised
|
||||
as before, never retried row by row.
|
||||
"""
|
||||
global _one_step_unsupported, _one_step_warned
|
||||
error: Optional[str] = None
|
||||
try:
|
||||
cursor = conn.execute(link_matches_json_query(sql, categories), params)
|
||||
try:
|
||||
((document,),) = cursor.fetchall()
|
||||
finally:
|
||||
cursor.close() # resets the statement: its snapshot ends now
|
||||
except (sqlite3.OperationalError, sqlite3.DataError) as failure:
|
||||
# DataError: the JSON text is longer than SQLITE_LIMIT_LENGTH.
|
||||
error = str(failure)
|
||||
if error == "interrupted":
|
||||
raise
|
||||
if error.startswith("no such function"):
|
||||
_one_step_unsupported = True
|
||||
else:
|
||||
if document is not None:
|
||||
return json.loads(document)
|
||||
error = "a BLOB value"
|
||||
if not _one_step_warned:
|
||||
_one_step_warned = True
|
||||
self.logger.warning(
|
||||
"Reading image links row by row (slower while ComfyUI "
|
||||
"generates); the one-step read failed: %s",
|
||||
error,
|
||||
)
|
||||
return None
|
||||
|
||||
def get_image_link_facts(
|
||||
self, paths: List[str], q: str = ""
|
||||
) -> Dict[str, Tuple[bool, Optional[int], int, Optional[str], Tuple[str, ...]]]:
|
||||
) -> Dict[str, Tuple[bool, Optional[int], int, Any, Tuple[str, ...]]]:
|
||||
"""Map each normalized path with one unambiguous link to its positive's
|
||||
(hit, rating, positive id, category, tag names).
|
||||
(hit, rating, positive id, inherited category, tag names).
|
||||
|
||||
The link is resolved as in get_image_link_matches, and ``hit`` says
|
||||
whether *q* occurs in the positive's text, Library title or a tag
|
||||
@@ -2448,7 +2739,7 @@ class PromptDatabase(
|
||||
if q:
|
||||
hit = _LINK_FILTER_HIT
|
||||
params["pattern"] = f"%{escape_like(q)}%"
|
||||
rows: Dict[str, Tuple[bool, Optional[int], int, Optional[str]]] = {}
|
||||
rows: Dict[str, LinkMatch] = {}
|
||||
names: Dict[int, List[str]] = {}
|
||||
unique = list(dict.fromkeys(paths))
|
||||
with self.model.get_connection() as conn:
|
||||
@@ -2457,10 +2748,8 @@ class PromptDatabase(
|
||||
bound = {f"path{i}": path for i, path in enumerate(chunk)}
|
||||
marks = ",".join(f":{key}" for key in bound)
|
||||
sql = _LINK_FACTS_SQL.format(hit=hit, marks=marks)
|
||||
for path, rating, matched, positive, category in conn.execute(
|
||||
sql, {**params, **bound}
|
||||
):
|
||||
rows[path] = (bool(matched), rating, positive, category)
|
||||
for path, *row in conn.execute(sql, {**params, **bound}):
|
||||
rows[path] = _link_match(*row)
|
||||
positives = sorted({row[2] for row in rows.values()})
|
||||
for start in range(0, len(positives), _FACTS_CHUNK):
|
||||
chunk = positives[start : start + _FACTS_CHUNK]
|
||||
@@ -2473,20 +2762,16 @@ class PromptDatabase(
|
||||
path: (*row, tuple(names.get(row[2], ()))) for path, row in rows.items()
|
||||
}
|
||||
|
||||
def get_image_paths_matching(
|
||||
self, q: str, category: Optional[str], tags: List[str]
|
||||
) -> Set[str]:
|
||||
"""Normalized paths whose linked positive matches *q*, *category* and *tags*.
|
||||
def get_image_paths_matching(self, q: str, tags: List[str]) -> Set[str]:
|
||||
"""Normalized paths whose linked positive matches *q* and *tags*.
|
||||
|
||||
See get_image_link_matches; an empty *q* matches every linked image
|
||||
the category and tags admit.
|
||||
the tags admit.
|
||||
"""
|
||||
return {
|
||||
path
|
||||
for path, (hit, _rating, _positive) in self.get_image_link_matches(
|
||||
q, category, tags
|
||||
).items()
|
||||
if hit or not q
|
||||
for path, match in self.get_image_link_matches(q, tags).items()
|
||||
if match[0] or not q
|
||||
}
|
||||
|
||||
def get_prompt_id_for_image(self, image_path: str) -> Optional[int]:
|
||||
@@ -2540,17 +2825,20 @@ class PromptDatabase(
|
||||
with self.model.get_connection() as conn:
|
||||
return list(iter_consistency_issues(conn))
|
||||
|
||||
def _image_row_to_dict(self, row: sqlite3.Row) -> Dict[str, Any]:
|
||||
def _image_row_to_dict(
|
||||
self, row: sqlite3.Row, reader: DocumentReader
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Convert an image database row to a dictionary with parsed JSON fields.
|
||||
|
||||
Args:
|
||||
row: SQLite row object
|
||||
reader: The workflow document reader of the row's connection
|
||||
|
||||
Returns:
|
||||
Dictionary representation of the row
|
||||
"""
|
||||
data = dict(row)
|
||||
data = resolve_image_row(reader, row)
|
||||
|
||||
# Parse JSON fields
|
||||
for field in ["workflow_data", "prompt_metadata", "parameters"]:
|
||||
|
||||
@@ -1,10 +1,24 @@
|
||||
"""Atomic metadata enrichment for a single known generation context."""
|
||||
"""Atomic metadata enrichment for a single known generation context.
|
||||
|
||||
Also the helpers both ingestion store paths share (#247): the file's link
|
||||
rows and the one-prompt-per-file rule (``file_link_rows``,
|
||||
``link_conflict``) and the marks a prompt created by ingestion carries
|
||||
(``Marks``, ``mark_created``).
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .models import normalize_image_path
|
||||
from .operations import _coerce_file_info
|
||||
from .workflow_documents import (
|
||||
TEXT_COLUMNS,
|
||||
DocumentReader,
|
||||
release_documents,
|
||||
resolve_image_row,
|
||||
stored_values,
|
||||
)
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
@@ -16,14 +30,99 @@ except ImportError:
|
||||
from utils.prompt_pair_resolver import RESOLVER_VERSION
|
||||
|
||||
|
||||
def blank_positive(resolution: Dict[str, Any]) -> bool:
|
||||
"""A resolved pair whose stored positive text is empty (#342).
|
||||
|
||||
A PromptManager node whose editor is empty encodes only its prepend and
|
||||
append text, which is never stored as a prompt: such a result names no
|
||||
prompt to link.
|
||||
"""
|
||||
if resolution.get("state") != "resolved":
|
||||
return False
|
||||
positive = resolution.get("positive") or {}
|
||||
text = positive.get("storage_text", positive.get("text"))
|
||||
return not (isinstance(text, str) and text.strip())
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Marks:
|
||||
"""What a prompt ingestion creates carries, so it can be found and removed.
|
||||
|
||||
``note`` is formatted with ``name``, the file's name.
|
||||
"""
|
||||
|
||||
category: str
|
||||
tags: Tuple[str, ...]
|
||||
note: str
|
||||
|
||||
|
||||
SCAN_MARKS = Marks("scanned", ("auto-scanned",), "Auto-scanned from {name}")
|
||||
RUN_MARKS = Marks("scanned", ("auto-scanned",), "Saved from a workflow run: {name}")
|
||||
|
||||
|
||||
def mark_created(conn: Any, db: Any, prompt_id: int, marks: Marks, name: str) -> None:
|
||||
"""Apply ``marks`` to a prompt row the caller just created on ``conn``."""
|
||||
conn.execute(
|
||||
"UPDATE prompts SET category = ?, notes = ? WHERE id = ?",
|
||||
(marks.category, marks.note.format(name=name), prompt_id),
|
||||
)
|
||||
db._sync_prompt_tags(conn, prompt_id, list(marks.tags))
|
||||
|
||||
|
||||
def file_link_rows(conn: Any, image_path: str) -> List[Any]:
|
||||
"""The file's link rows, each with its prompt's hash.
|
||||
|
||||
Matched under the path and its resolved spelling (an output folder
|
||||
reached through a symlink or junction), each normalized and raw;
|
||||
deduplicated by row id.
|
||||
"""
|
||||
rows: Dict[Any, Any] = {}
|
||||
for spelling in dict.fromkeys((image_path, _resolved(image_path))):
|
||||
for row in conn.execute(
|
||||
"SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
|
||||
"JOIN prompts p ON p.id=gi.prompt_id "
|
||||
"WHERE gi.file_path=? OR gi.image_path=?",
|
||||
(normalize_image_path(spelling), spelling),
|
||||
).fetchall():
|
||||
rows.setdefault(row["id"], row)
|
||||
return list(rows.values())
|
||||
|
||||
|
||||
def _resolved(path: str) -> str:
|
||||
try:
|
||||
return os.path.realpath(path)
|
||||
except (OSError, ValueError):
|
||||
return path
|
||||
|
||||
|
||||
def link_conflict(rows: List[Any], prompt_hash: str) -> bool:
|
||||
"""One prompt per file: more than one row, or a row of another prompt."""
|
||||
return len(rows) > 1 or bool(rows and rows[0]["positive_hash"] != prompt_hash)
|
||||
|
||||
|
||||
def ingest_resolved_pair(
|
||||
db: Any, image_path: str, metadata: Dict[str, Any], resolution: Dict[str, Any]
|
||||
db: Any,
|
||||
image_path: str,
|
||||
metadata: Dict[str, Any],
|
||||
resolution: Dict[str, Any],
|
||||
*,
|
||||
marks: Optional[Marks] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Store a resolved pair and link the file to it in one transaction.
|
||||
|
||||
With ``marks`` (ingestion from files), each pair component this call
|
||||
creates carries them; without (the PromptManager node path) components
|
||||
are created bare, as always.
|
||||
"""
|
||||
state = resolution.get("state", "unresolved")
|
||||
if state == "resolved" and blank_positive(resolution):
|
||||
state = "unresolved"
|
||||
if state != "resolved":
|
||||
return {"status": state, "pairs_added": 0, "links_added": 0}
|
||||
with db._pair_transaction() as conn:
|
||||
return _ingest_resolved_pair_in(conn, db, image_path, metadata, resolution)
|
||||
return _ingest_resolved_pair_in(
|
||||
conn, db, image_path, metadata, resolution, marks=marks
|
||||
)
|
||||
|
||||
|
||||
def _conflict() -> Dict[str, Any]:
|
||||
@@ -38,6 +137,7 @@ def _ingest_resolved_pair_in(
|
||||
resolution: Dict[str, Any],
|
||||
*,
|
||||
provenance: Dict[str, Any] | None = None,
|
||||
marks: Optional[Marks] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""``ingest_resolved_pair`` inside the caller's open ``_pair_transaction``.
|
||||
|
||||
@@ -46,7 +146,8 @@ def _ingest_resolved_pair_in(
|
||||
negative is a ``conflict`` and nothing is written. The caller owns the
|
||||
transaction, so its own later failure rolls the link back too.
|
||||
``provenance`` replaces the link's default resolver provenance (Needs
|
||||
review records the user's selection there).
|
||||
review records the user's selection there). ``marks`` as in
|
||||
``ingest_resolved_pair``.
|
||||
"""
|
||||
positive, negative = resolution["positive"], resolution["negative"]
|
||||
positive_text = positive.get("storage_text", positive["text"])
|
||||
@@ -59,15 +160,8 @@ def _ingest_resolved_pair_in(
|
||||
"negative": negative,
|
||||
}
|
||||
encoded = dumps_strict(provenance, sort_keys=True, separators=(",", ":"))
|
||||
images = conn.execute(
|
||||
"SELECT gi.*,p.hash AS positive_hash FROM generated_images gi "
|
||||
"JOIN prompts p ON p.id=gi.prompt_id "
|
||||
"WHERE gi.file_path=? OR gi.image_path=?",
|
||||
(normalized, image_path),
|
||||
).fetchall()
|
||||
if len(images) > 1 or (
|
||||
images and images[0]["positive_hash"] != generate_prompt_hash(positive_text)
|
||||
):
|
||||
images = file_link_rows(conn, image_path)
|
||||
if link_conflict(images, generate_prompt_hash(positive_text)):
|
||||
return _conflict()
|
||||
existing = images[0] if images else None
|
||||
if existing:
|
||||
@@ -92,7 +186,12 @@ def _ingest_resolved_pair_in(
|
||||
generate_prompt_hash(negative_text) if negative_text.strip() else None,
|
||||
),
|
||||
).fetchone()
|
||||
pair_id = db._save_pair(conn, positive_text, negative_text)
|
||||
if marks is None:
|
||||
pair_id = db._save_pair(conn, positive_text, negative_text)
|
||||
else:
|
||||
pair_id = _save_marked_pair(
|
||||
conn, db, positive_text, negative_text, marks, os.path.basename(image_path)
|
||||
)
|
||||
pair = conn.execute("SELECT * FROM prompt_pairs WHERE id=?", (pair_id,)).fetchone()
|
||||
pairs_added = int(prior_pair is None)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
@@ -116,25 +215,35 @@ def _ingest_resolved_pair_in(
|
||||
changed = False
|
||||
if existing:
|
||||
image_id = existing["id"]
|
||||
# Compared as stored text, whichever form the row is in.
|
||||
current = resolve_image_row(DocumentReader(conn), existing)
|
||||
updates = {
|
||||
key: value for key, value in snapshots.items() if existing[key] != value
|
||||
key: value for key, value in snapshots.items() if current[key] != value
|
||||
}
|
||||
if updates:
|
||||
replaced = [
|
||||
existing[reference]
|
||||
for column, reference in TEXT_COLUMNS
|
||||
if column in updates
|
||||
]
|
||||
updates = _document_form(conn, updates)
|
||||
conn.execute(
|
||||
"UPDATE generated_images SET "
|
||||
+ ",".join(f"{key}=?" for key in updates)
|
||||
+ " WHERE id=?",
|
||||
[*updates.values(), image_id],
|
||||
)
|
||||
release_documents(conn, replaced)
|
||||
changed = True
|
||||
else:
|
||||
columns = ["prompt_id", "image_path", "filename", "file_path", *snapshots]
|
||||
stored = _document_form(conn, snapshots)
|
||||
columns = ["prompt_id", "image_path", "filename", "file_path", *stored]
|
||||
values = [
|
||||
pair["positive_prompt_id"],
|
||||
image_path,
|
||||
os.path.basename(image_path),
|
||||
normalized,
|
||||
*snapshots.values(),
|
||||
*stored.values(),
|
||||
]
|
||||
image_id = conn.execute(
|
||||
"INSERT INTO generated_images("
|
||||
@@ -162,3 +271,38 @@ def _ingest_resolved_pair_in(
|
||||
"pair_id": pair_id,
|
||||
"image_id": image_id,
|
||||
}
|
||||
|
||||
|
||||
def _save_marked_pair(
|
||||
conn: Any,
|
||||
db: Any,
|
||||
positive_text: str,
|
||||
negative_text: str,
|
||||
marks: Marks,
|
||||
name: str,
|
||||
) -> int:
|
||||
"""``_save_pair`` that marks each component it creates."""
|
||||
texts = [positive_text] + ([negative_text] if negative_text.strip() else [])
|
||||
ids = []
|
||||
for text in texts:
|
||||
prompt_id, created = db._pair_component_outcome(conn, text)
|
||||
if created:
|
||||
mark_created(conn, db, prompt_id, marks, name)
|
||||
ids.append(prompt_id)
|
||||
negative_id = ids[1] if len(ids) > 1 else None
|
||||
return db._save_pair_components(conn, ids[0], negative_id)[0]
|
||||
|
||||
|
||||
def _document_form(conn: Any, values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""``values`` with the workflow and prompt graph texts stored once.
|
||||
|
||||
Each of those columns becomes NULL beside its document id (new form);
|
||||
every other column is unchanged.
|
||||
"""
|
||||
texts = {
|
||||
column: values[column]
|
||||
for column, _reference in TEXT_COLUMNS
|
||||
if column in values
|
||||
}
|
||||
rest = {key: value for key, value in values.items() if key not in texts}
|
||||
return {**rest, **stored_values(conn, texts)}
|
||||
|
||||
@@ -112,8 +112,14 @@ class PairOperations:
|
||||
tags: Optional[List[str]] = None,
|
||||
rating: Optional[int] = None,
|
||||
notes: Optional[str] = None,
|
||||
fill_new_positive: bool = False,
|
||||
) -> int:
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata."""
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata.
|
||||
|
||||
``fill_new_positive`` also gives the positive prompt the category and
|
||||
tags, only if this save inserted its row (#546). An existing prompt
|
||||
never changes, so a cleared category stays cleared.
|
||||
"""
|
||||
with self._pair_transaction() as conn:
|
||||
return self._save_pair(
|
||||
conn,
|
||||
@@ -124,6 +130,7 @@ class PairOperations:
|
||||
tags=tags,
|
||||
rating=rating,
|
||||
notes=notes,
|
||||
fill_new_positive=fill_new_positive,
|
||||
)
|
||||
|
||||
def _save_pair(
|
||||
@@ -137,6 +144,7 @@ class PairOperations:
|
||||
tags: Optional[List[str]] = None,
|
||||
rating: Optional[int] = None,
|
||||
notes: Optional[str] = None,
|
||||
fill_new_positive: bool = False,
|
||||
) -> int:
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata."""
|
||||
if not isinstance(positive_text, str) or not positive_text.strip():
|
||||
@@ -144,8 +152,13 @@ class PairOperations:
|
||||
if not isinstance(negative_text, str):
|
||||
raise ValueError("Negative prompt text must be a string; use '' for empty")
|
||||
# Validate metadata before any component insertion.
|
||||
_validate_metadata(title, category, tags, rating, notes)
|
||||
positive = self._pair_component(conn, positive_text)
|
||||
tags = _validate_metadata(title, category, tags, rating, notes)
|
||||
positive, created = self._pair_component_outcome(conn, positive_text)
|
||||
if created and fill_new_positive:
|
||||
conn.execute(
|
||||
"UPDATE prompts SET category = ? WHERE id = ?", (category, positive)
|
||||
)
|
||||
self._sync_prompt_tags(conn, positive, tags or [])
|
||||
negative = (
|
||||
self._pair_component(conn, negative_text) if negative_text.strip() else None
|
||||
)
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Library parent archive (#443, D152): prompts hidden from the shelf, restorable.
|
||||
|
||||
A row means the Library parent is archived; no row means it is on the shelf.
|
||||
Nothing else about the prompt changes: its text, metadata, tags, run count,
|
||||
image links, pairs and recorded runs stay. Restore deletes the row. Deleting
|
||||
the prompt (any path) drops the row by cascade.
|
||||
|
||||
A table of its own, not a column: ``prompt_library_state`` is validated by its
|
||||
exact definition, so a new column there would make older builds refuse the
|
||||
upgraded database, while older builds ignore an unknown table (they show the
|
||||
archived prompts again and keep the rows). Optional in old backups, required
|
||||
once a database is current.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
TABLE = "prompt_archive"
|
||||
MAX_STAMP_BYTES = 64
|
||||
# What ``utc_now_iso`` writes starts with a date; anything else was not written here.
|
||||
_STAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T*"
|
||||
_DDL = (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT NOT NULL, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
|
||||
)
|
||||
_COLUMNS = (("prompt_id", "INTEGER", 0, 1), ("archived_at", "TEXT", 1, 0))
|
||||
_FOREIGN_KEYS = {("prompt_id", "prompts", "id", "CASCADE")}
|
||||
# A predicate on ``p`` (the prompts alias): the parent is not archived.
|
||||
SHELVED = f"NOT EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
|
||||
ARCHIVED = f"EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
|
||||
objects = conn.execute(
|
||||
"SELECT type,sql FROM sqlite_master WHERE name=? "
|
||||
"OR (type IN ('index','trigger') AND tbl_name=?)",
|
||||
(TABLE, TABLE),
|
||||
).fetchall()
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing prompt archive schema"
|
||||
tables = [sql for kind, sql in objects if kind == "table"]
|
||||
if len(tables) != 1 or len(objects) != 1:
|
||||
return "Prompt archive name is reserved by an unsupported schema object"
|
||||
if _compact(tables[0]) != _compact(_DDL):
|
||||
return "Prompt archive has an invalid table definition"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({TABLE})")
|
||||
)
|
||||
if columns != _COLUMNS:
|
||||
return "Prompt archive has invalid columns"
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({TABLE})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS:
|
||||
return "Prompt archive has invalid foreign key definitions"
|
||||
return None
|
||||
|
||||
|
||||
def prompt_archive_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> str | None:
|
||||
"""Reject an unsupported archive schema or values; reads only.
|
||||
|
||||
A missing table is acceptable for backups that predate it unless
|
||||
``allow_missing`` is false.
|
||||
"""
|
||||
error = _structure_error(conn, allow_missing)
|
||||
if error is not None:
|
||||
return error
|
||||
if not conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone():
|
||||
return None
|
||||
if conn.execute(f"PRAGMA foreign_key_check({TABLE})").fetchone():
|
||||
return "Prompt archive contains foreign key violations"
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(archived_at)!='text' " # nosec B608
|
||||
f"OR length(CAST(archived_at AS BLOB))>{MAX_STAMP_BYTES} "
|
||||
f"OR archived_at NOT GLOB '{_STAMP_GLOB}' LIMIT 1"
|
||||
).fetchone():
|
||||
return "Prompt archive contains invalid values"
|
||||
return None
|
||||
|
||||
|
||||
def migrate_prompt_archive(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table atomically; validate before and after (no backfill)."""
|
||||
conn.execute("SAVEPOINT prompt_archive_schema")
|
||||
try:
|
||||
error = _structure_error(conn, True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
error = _structure_error(conn, False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE prompt_archive_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO prompt_archive_schema")
|
||||
conn.execute("RELEASE prompt_archive_schema")
|
||||
raise
|
||||
@@ -0,0 +1,376 @@
|
||||
"""A restore in two steps: stage the upload, then publish it (#249 System S8).
|
||||
|
||||
**Stage** (``stage_restore``) never touches the live database or the source.
|
||||
In a new folder ``.pm-restore-*`` in the library's own folder (#318: never
|
||||
the system temp folder, which may be another disk), it:
|
||||
|
||||
1. checks the source (``verify_database_file`` without reading every
|
||||
workflow document: a damaged one is repaired below, not refused);
|
||||
2. copies it with SQLite's backup API (a rollback-journal file and a WAL file
|
||||
alike) and opens the copy as a ``PromptModel``, which migrates an older
|
||||
schema;
|
||||
3. repairs damaged workflow documents (``repair_damaged_documents``),
|
||||
rotates the history epoch and counts what the copy holds;
|
||||
4. makes the copy one self-contained file and verifies it fully (every
|
||||
current table, every document);
|
||||
5. records the copy's SHA-256, so publishing binds exactly these bytes.
|
||||
|
||||
**Publish** (``publish_staged``) re-hashes the stage (``StageChanged`` if it
|
||||
differs), keeps the current database as a uniquely named, standalone safety
|
||||
backup beside it (``database/safety_backup.py``; not verified, because the
|
||||
database being replaced may be the damaged one), copies the stage over the
|
||||
live pages through the backup API (no file is swapped under open handles; an
|
||||
``OperationalError`` there leaves the live data intact, #226), then
|
||||
checkpoints the WAL with TRUNCATE so the copied pages do not leave a
|
||||
database-sized ``-wal`` behind. A reader that keeps a snapshot past the busy
|
||||
wait is logged, not fatal: the restore itself is done.
|
||||
|
||||
Every handle on the stage is closed before its folder is removed (Windows).
|
||||
``sweep_restore_temps`` removes what an interrupted restore left (startup).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
from . import workflow_documents as wd
|
||||
from .execution_history_schema import rotate_history_epoch
|
||||
from .safety_backup import take_safety_backup
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.restore_stage")
|
||||
|
||||
STAGE_PREFIX = ".pm-restore-"
|
||||
UPLOAD_PREFIX = ".pm-restore-upload-"
|
||||
STAGED_NAME = "staged.db"
|
||||
# How long the post-restore TRUNCATE checkpoint waits for readers to leave.
|
||||
CHECKPOINT_BUSY_MS = 30_000
|
||||
_HASH_CHUNK = 1024 * 1024
|
||||
SUMMARY_KEYS = (
|
||||
"prompts",
|
||||
"positives",
|
||||
"pairs",
|
||||
"archived_pairs",
|
||||
"image_links",
|
||||
"runs",
|
||||
"saved_views",
|
||||
"collections",
|
||||
)
|
||||
|
||||
|
||||
class StageChanged(ValueError):
|
||||
"""The staged file is no longer the bytes that were reviewed."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StagedRestore:
|
||||
"""A verified, migrated copy of an upload, ready to publish."""
|
||||
|
||||
path: str
|
||||
digest: str
|
||||
size: int
|
||||
summary: Dict[str, int]
|
||||
schema: Dict[str, bool]
|
||||
documents: Dict[str, int] = field(
|
||||
default_factory=lambda: {"repaired": 0, "cleared": 0, "images": 0}
|
||||
)
|
||||
|
||||
@property
|
||||
def directory(self) -> str:
|
||||
return os.path.dirname(self.path)
|
||||
|
||||
def discard(self) -> None:
|
||||
"""Remove the stage folder (idempotent; its handles are all closed)."""
|
||||
remove_stage_folder(self.directory)
|
||||
|
||||
|
||||
def remove_stage_folder(directory: str) -> None:
|
||||
"""Remove a stage folder and its files, logging what cannot be removed."""
|
||||
if not directory or not os.path.isdir(directory):
|
||||
return
|
||||
for name in os.listdir(directory):
|
||||
try:
|
||||
os.remove(os.path.join(directory, name))
|
||||
except OSError as error:
|
||||
logger.warning(f"Could not remove restore stage file {name}: {error}")
|
||||
try:
|
||||
os.rmdir(directory)
|
||||
except OSError as error:
|
||||
logger.warning(f"Could not remove the restore stage folder: {error}")
|
||||
|
||||
|
||||
def file_digest(path: str) -> str:
|
||||
"""``sha256:<hex>`` of the file's bytes (blocking)."""
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(_HASH_CHUNK), b""):
|
||||
digest.update(chunk)
|
||||
return "sha256:" + digest.hexdigest()
|
||||
|
||||
|
||||
def _readonly(path: str) -> sqlite3.Connection:
|
||||
return sqlite3.connect(Path(path).resolve().as_uri() + "?mode=ro", uri=True)
|
||||
|
||||
|
||||
def _schema(path: str) -> list:
|
||||
"""The file's schema objects, to tell whether staging migrated it."""
|
||||
with closing(_readonly(path)) as conn:
|
||||
return sorted(
|
||||
conn.execute(
|
||||
"SELECT type, name, tbl_name, sql FROM sqlite_master "
|
||||
"WHERE name NOT LIKE 'sqlite_%'"
|
||||
).fetchall(),
|
||||
key=repr,
|
||||
)
|
||||
|
||||
|
||||
def _count(conn: sqlite3.Connection, table: str, where: str = "") -> int:
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
||||
).fetchone()
|
||||
if not exists:
|
||||
return 0
|
||||
# ``table`` and ``where`` are constants of library_summary, never input.
|
||||
return conn.execute(f"SELECT COUNT(*) FROM {table} {where}").fetchone()[0] # nosec
|
||||
|
||||
|
||||
def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
|
||||
"""What a library holds, for a restore's review (reads only).
|
||||
|
||||
``positives`` are the Library's prompts (``library_counts``'s
|
||||
``positive_prompts``); ``pairs`` the active saved pairs and
|
||||
``archived_pairs`` the archived ones; ``image_links`` the indexed image
|
||||
rows; ``runs`` the recorded executions; ``collections`` the image
|
||||
collections (#331).
|
||||
"""
|
||||
from .library import _ELIGIBLE
|
||||
from .prompt_archive import SHELVED, TABLE as ARCHIVE
|
||||
|
||||
has_state = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND "
|
||||
"name='prompt_library_state'"
|
||||
).fetchone()
|
||||
has_pairs = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='prompt_pairs'"
|
||||
).fetchone()
|
||||
has_archive = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (ARCHIVE,)
|
||||
).fetchone()
|
||||
if has_state and has_pairs:
|
||||
# Archived parents (#443) are off the shelf, as in library_counts.
|
||||
positives = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompts p WHERE "
|
||||
+ _ELIGIBLE
|
||||
+ (" AND " + SHELVED if has_archive else "")
|
||||
).fetchone()[0]
|
||||
else:
|
||||
positives = _count(conn, "prompts")
|
||||
return {
|
||||
"prompts": _count(conn, "prompts"),
|
||||
"positives": positives,
|
||||
"pairs": _count(conn, "prompt_pairs", "WHERE archived=0"),
|
||||
"archived_pairs": _count(conn, "prompt_pairs", "WHERE archived<>0"),
|
||||
"image_links": _count(conn, "generated_images"),
|
||||
"runs": _count(conn, "execution_runs"),
|
||||
"saved_views": _count(conn, "saved_views"),
|
||||
"collections": _count(conn, "image_collections"),
|
||||
}
|
||||
|
||||
|
||||
# The newest row of each table a restore would overwrite: with the counts,
|
||||
# a write after the review (a new run, image, prompt or edit) changes it.
|
||||
_FINGERPRINT_SQL = (
|
||||
("prompts", "SELECT MAX(id), MAX(updated_at), MAX(last_used_at) FROM prompts"),
|
||||
("generated_images", "SELECT MAX(id) FROM generated_images"),
|
||||
("prompt_pairs", "SELECT MAX(id), MAX(updated_at) FROM prompt_pairs"),
|
||||
("execution_runs", "SELECT COUNT(*), MAX(rowid) FROM execution_runs"),
|
||||
("saved_views", "SELECT MAX(rowid) FROM saved_views"),
|
||||
("image_collections", "SELECT MAX(rowid) FROM image_collections"),
|
||||
)
|
||||
|
||||
|
||||
def live_fingerprint(conn: sqlite3.Connection):
|
||||
"""``(summary, fingerprint)`` of a live library, read in one snapshot.
|
||||
|
||||
The fingerprint changes when rows are added to or removed from the
|
||||
tables a restore replaces, or when a prompt, pair, run or saved view is
|
||||
the newest edited (counts plus the newest row and edit of each table);
|
||||
the apply refuses with ``plan-changed`` then. It does not see every
|
||||
edit (a tag, favorite or collection membership change on an older row);
|
||||
those are kept in the restore's safety backup. A read transaction the
|
||||
caller already has is used and left open; one opened here is ended here.
|
||||
"""
|
||||
import json
|
||||
|
||||
opened = not conn.in_transaction
|
||||
if opened:
|
||||
conn.execute("BEGIN") # one read snapshot for every count
|
||||
try:
|
||||
summary = library_summary(conn)
|
||||
newest = []
|
||||
for table, sql in _FINGERPRINT_SQL:
|
||||
try:
|
||||
newest.append([table, list(conn.execute(sql).fetchone())])
|
||||
except sqlite3.OperationalError:
|
||||
newest.append([table, None]) # no such table or column
|
||||
finally:
|
||||
if opened:
|
||||
conn.rollback() # it only read
|
||||
text = json.dumps([summary, newest], sort_keys=True, default=str)
|
||||
return summary, "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _copy(src_path: str, staged_path: str) -> None:
|
||||
source = _readonly(src_path)
|
||||
try:
|
||||
with closing(sqlite3.connect(staged_path)) as destination:
|
||||
source.backup(destination)
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
|
||||
def _standalone(path: str) -> None:
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
conn.execute("PRAGMA journal_mode=DELETE").fetchone()
|
||||
|
||||
|
||||
def stage_restore(model_cls, db_path: str, src_path: str) -> StagedRestore:
|
||||
"""Stage ``src_path`` beside ``db_path`` (blocking); see the module docstring.
|
||||
|
||||
Raises ValueError with a readable reason when the source or the migrated
|
||||
stage does not verify; the stage folder is then already removed.
|
||||
"""
|
||||
ok, reason = model_cls.verify_database_file(src_path, documents=False)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
before = _schema(src_path)
|
||||
with closing(_readonly(src_path)) as conn:
|
||||
library_state = bool(
|
||||
conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND "
|
||||
"name='prompt_library_state'"
|
||||
).fetchone()
|
||||
)
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
directory = tempfile.mkdtemp(prefix=STAGE_PREFIX, dir=folder)
|
||||
staged_path = os.path.join(directory, STAGED_NAME)
|
||||
staged_key = os.path.normcase(os.path.realpath(staged_path))
|
||||
try:
|
||||
_copy(src_path, staged_path)
|
||||
staged = None
|
||||
try:
|
||||
staged = model_cls(staged_path)
|
||||
conn = staged.get_connection()
|
||||
documents = wd.repair_damaged_documents(conn)
|
||||
rotate_history_epoch(conn)
|
||||
summary = library_summary(conn)
|
||||
finally:
|
||||
if staged is not None:
|
||||
staged.close_all()
|
||||
_standalone(staged_path)
|
||||
ok, reason = model_cls.verify_database_file(
|
||||
staged_path, require_library_state=True
|
||||
)
|
||||
if not ok:
|
||||
raise ValueError(reason)
|
||||
result = StagedRestore(
|
||||
path=staged_path,
|
||||
digest=file_digest(staged_path),
|
||||
size=os.path.getsize(staged_path),
|
||||
summary=summary,
|
||||
schema={
|
||||
"migrated": _schema(staged_path) != before,
|
||||
"library_state": library_state,
|
||||
},
|
||||
documents=documents,
|
||||
)
|
||||
except BaseException:
|
||||
remove_stage_folder(directory)
|
||||
raise
|
||||
finally:
|
||||
with model_cls._init_lock:
|
||||
model_cls._initialized_paths.discard(staged_key)
|
||||
return result
|
||||
|
||||
|
||||
def checkpoint_truncate(conn: sqlite3.Connection) -> bool:
|
||||
"""Checkpoint the WAL and truncate it to 0 bytes; False when a reader held it."""
|
||||
previous = conn.execute("PRAGMA busy_timeout").fetchone()[0]
|
||||
try:
|
||||
conn.execute(f"PRAGMA busy_timeout = {int(CHECKPOINT_BUSY_MS)}")
|
||||
busy = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0]
|
||||
finally:
|
||||
conn.execute(f"PRAGMA busy_timeout = {int(previous)}")
|
||||
return not busy
|
||||
|
||||
|
||||
def publish_staged(model, staged: StagedRestore) -> str:
|
||||
"""Publish ``staged`` over ``model``'s database; return the safety backup path.
|
||||
|
||||
"" when there was no database to back up. Raises StageChanged before
|
||||
anything is written when the stage's bytes changed since staging, and
|
||||
SafetyBackupFailed when the current database could not be kept. An
|
||||
``OperationalError`` from the page copy propagates with
|
||||
``_prompt_manager_restore_backup_created`` set; the live data is intact.
|
||||
"""
|
||||
if not os.path.isfile(staged.path) or file_digest(staged.path) != staged.digest:
|
||||
raise StageChanged("The staged restore changed since it was reviewed")
|
||||
backup_path = ""
|
||||
if os.path.exists(model.db_path):
|
||||
backup_path = take_safety_backup(model, verify=False)
|
||||
logger.info(f"Current database backed up to: {os.path.basename(backup_path)}")
|
||||
try:
|
||||
model._replace_contents_from(staged.path)
|
||||
except sqlite3.OperationalError as exc:
|
||||
exc._prompt_manager_restore_backup_created = bool(backup_path)
|
||||
raise
|
||||
with model._init_lock:
|
||||
model._initialized_paths.add(model._schema_key())
|
||||
_truncate_wal(model)
|
||||
return backup_path
|
||||
|
||||
|
||||
def _truncate_wal(model) -> Optional[bool]:
|
||||
"""TRUNCATE-checkpoint the live WAL after a publish (never raises)."""
|
||||
try:
|
||||
conn = model.get_connection()
|
||||
with conn.using():
|
||||
truncated = checkpoint_truncate(conn)
|
||||
except sqlite3.Error as error:
|
||||
logger.warning(f"The database log could not be truncated: {error}")
|
||||
return None
|
||||
if not truncated:
|
||||
logger.warning(
|
||||
"The database log could not be truncated after the restore: a "
|
||||
"reader kept it open. It shrinks at the next checkpoint with no "
|
||||
"reader."
|
||||
)
|
||||
return truncated
|
||||
|
||||
|
||||
def sweep_restore_temps(db_path: str) -> list:
|
||||
"""Remove the stages and uploads an interrupted restore left (startup)."""
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
removed = []
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return removed
|
||||
for name in names:
|
||||
path = os.path.join(folder, name)
|
||||
if name.startswith(UPLOAD_PREFIX) and os.path.isfile(path):
|
||||
try:
|
||||
os.remove(path)
|
||||
removed.append(name)
|
||||
except OSError:
|
||||
continue
|
||||
elif name.startswith(STAGE_PREFIX) and os.path.isdir(path):
|
||||
remove_stage_folder(path)
|
||||
if not os.path.exists(path):
|
||||
removed.append(name)
|
||||
return removed
|
||||
@@ -62,3 +62,14 @@ def notify_review_change(db_path: str) -> None:
|
||||
callback()
|
||||
except Exception:
|
||||
_logger.warning("Needs review change observer failed", exc_info=True)
|
||||
|
||||
|
||||
def rekey_review_changes(old_path: str, new_path: str) -> None:
|
||||
"""Move the observers of ``old_path`` to ``new_path`` (Move library)."""
|
||||
old, new = _identity(old_path), _identity(new_path)
|
||||
if old == new:
|
||||
return
|
||||
with _lock:
|
||||
observers = _observers.pop(old, None)
|
||||
if observers:
|
||||
_observers.setdefault(new, {}).update(observers)
|
||||
|
||||
@@ -5,8 +5,12 @@ A decision is one row of ``media_review_decisions`` for one item revision.
|
||||
ingester's compatible-only rules (``_ingest_resolved_pair_in``) in the same
|
||||
``_pair_transaction`` as the decision row, the item's new state and revision
|
||||
and the review index revision; any conflict or later failure rolls all of it
|
||||
back. ``unresolved`` records the decision and links nothing. No decision
|
||||
counts a run, touches usage or calls an execution writer.
|
||||
back. ``manual`` (#536) is the same but for a Library prompt the reviewer chose:
|
||||
its link's provenance names a manual choice and carries no prompt sides, so it
|
||||
cannot pose as file evidence, and the decision's ``candidate_hash`` (the table
|
||||
requires one) is the digest of that choice. ``unresolved`` records the decision
|
||||
and links nothing. No decision counts a run, touches usage or calls an execution
|
||||
writer.
|
||||
|
||||
Callers verify the file first (the capture must be the item's bytes and the
|
||||
current file must still hash to them); nothing here reads the filesystem.
|
||||
@@ -17,6 +21,7 @@ import sqlite3
|
||||
|
||||
from .annotation_schema import BYTES_TABLE
|
||||
from .annotations import _new_id
|
||||
from .library import _ELIGIBLE, _FROM, SHELVED
|
||||
from .library_commands import LibraryCommandError
|
||||
from .models import utc_now_iso
|
||||
from .review import (
|
||||
@@ -30,9 +35,28 @@ from .review import (
|
||||
same_name_rows,
|
||||
)
|
||||
from .review_changes import notify_review_change
|
||||
from .review_schema import DECISIONS_TABLE, ITEMS_TABLE, canonical_evidence
|
||||
from .review_schema import (
|
||||
DECISIONS_TABLE,
|
||||
ITEMS_TABLE,
|
||||
canonical_evidence,
|
||||
evidence_digest,
|
||||
)
|
||||
|
||||
DECISION_KIND = "user-selection"
|
||||
MANUAL_KIND = "manual-link"
|
||||
_MANUAL_POSITIVE = (
|
||||
f"SELECT p.id,p.text FROM {_FROM} " # nosec B608
|
||||
f"WHERE p.id=? AND {_ELIGIBLE} AND {SHELVED}"
|
||||
)
|
||||
_MANUAL_PAIR = (
|
||||
"SELECT COALESCE(n.text,'') AS negative FROM prompt_pairs pp "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
||||
"WHERE pp.id=? AND pp.positive_prompt_id=? AND pp.archived=0"
|
||||
)
|
||||
_ARCHIVED_BARE_PAIR = (
|
||||
"SELECT 1 FROM prompt_pairs WHERE positive_prompt_id=? "
|
||||
"AND negative_prompt_id IS NULL AND archived=1"
|
||||
)
|
||||
_INSERT_DECISION = (
|
||||
f"INSERT INTO {DECISIONS_TABLE}(id,item_id,decision_revision," # nosec B608
|
||||
"created_at,outcome,candidate_hash,evidence,pair_id,image_id) "
|
||||
@@ -59,6 +83,8 @@ class ReviewCommandError(LibraryCommandError):
|
||||
``fingerprint`` the current dependency fingerprint, when known;
|
||||
``changed`` marks an association conflict caused by links that changed
|
||||
since the reviewer read them (rather than an incompatible choice).
|
||||
``prompt-missing``: the Library prompt or recipe of a manual link is gone,
|
||||
archived or not that prompt's.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -93,6 +119,7 @@ class ReviewDecisionOperations:
|
||||
sha256: str,
|
||||
outcome: str,
|
||||
candidate: dict | None = None,
|
||||
manual: dict | None = None,
|
||||
image_path: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
expected_fingerprint: str | None = None,
|
||||
@@ -101,7 +128,9 @@ class ReviewDecisionOperations:
|
||||
) -> dict:
|
||||
"""Record the decision; return ``{item, total, pair}``.
|
||||
|
||||
``selected`` needs the evidence ``candidate`` (with its ``hash``), the
|
||||
``selected`` needs the evidence ``candidate`` (with its ``hash``) or
|
||||
``manual`` (``{positive_id, pair_id}``; a null ``pair_id`` is no
|
||||
negative), the
|
||||
``image_path`` the scan and the monitor link under, the captured
|
||||
``metadata`` and the ``expected_fingerprint`` the reviewer saw.
|
||||
``aliases`` are the other spellings legacy rows of the same physical
|
||||
@@ -115,9 +144,12 @@ class ReviewDecisionOperations:
|
||||
if outcome not in ("selected", "unresolved"):
|
||||
raise ValueError("outcome must be selected or unresolved")
|
||||
if outcome == "selected" and (
|
||||
not isinstance(candidate, dict) or not image_path
|
||||
(not isinstance(candidate, dict)) == (not isinstance(manual, dict))
|
||||
or not image_path
|
||||
):
|
||||
raise ValueError("A selected decision needs a candidate and a file")
|
||||
raise ValueError(
|
||||
"A selected decision needs one candidate or manual choice and a file"
|
||||
)
|
||||
try:
|
||||
with self._pair_transaction() as conn:
|
||||
decided = self._decide_in(
|
||||
@@ -128,6 +160,7 @@ class ReviewDecisionOperations:
|
||||
sha256=sha256,
|
||||
outcome=outcome,
|
||||
candidate=candidate,
|
||||
manual=manual,
|
||||
image_path=image_path,
|
||||
metadata=metadata or {},
|
||||
expected_fingerprint=expected_fingerprint,
|
||||
@@ -158,13 +191,24 @@ class ReviewDecisionOperations:
|
||||
raise _Rollback(ReviewCommandError("file-changed", epoch))
|
||||
decision_id = _new_id()
|
||||
pair = None
|
||||
candidate = command["candidate"] if command["outcome"] == "selected" else None
|
||||
selected = command["outcome"] == "selected"
|
||||
candidate = command["candidate"] if selected else None
|
||||
manual = command["manual"] if selected else None
|
||||
candidate_hash = candidate["hash"] if candidate is not None else None
|
||||
if candidate is not None:
|
||||
pair = self._apply_candidate(
|
||||
conn, epoch, current, decision_id, total, **command
|
||||
)
|
||||
elif manual is not None:
|
||||
pair = self._apply_manual(
|
||||
conn, epoch, current, decision_id, total, **command
|
||||
)
|
||||
manual = {"positive_id": manual["positive_id"], "pair_id": pair["id"]}
|
||||
candidate_hash = evidence_digest(
|
||||
canonical_evidence({"kind": MANUAL_KIND, **manual})
|
||||
)
|
||||
evidence = {
|
||||
"kind": DECISION_KIND,
|
||||
"kind": MANUAL_KIND if manual is not None else DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"outcome": command["outcome"],
|
||||
"reviewed_digest": command["sha256"],
|
||||
@@ -174,6 +218,8 @@ class ReviewDecisionOperations:
|
||||
"dependency_fingerprint": command["expected_fingerprint"],
|
||||
"pair": pair,
|
||||
}
|
||||
if manual is not None:
|
||||
evidence["manual"] = manual
|
||||
now = utc_now_iso()
|
||||
self._insert_review_decision(
|
||||
conn,
|
||||
@@ -183,7 +229,7 @@ class ReviewDecisionOperations:
|
||||
current["revision"] + 1,
|
||||
now,
|
||||
command["outcome"],
|
||||
candidate["hash"] if candidate is not None else None,
|
||||
candidate_hash,
|
||||
canonical_evidence(evidence),
|
||||
pair["id"] if pair else None,
|
||||
pair["image_id"] if pair else None,
|
||||
@@ -204,6 +250,77 @@ class ReviewDecisionOperations:
|
||||
self, conn, epoch, current, decision_id, total, **command
|
||||
) -> dict:
|
||||
"""Link the candidate's pair, compatible-only; raise on any conflict."""
|
||||
candidate = command["candidate"]
|
||||
positive, negative = candidate["positive"], candidate["negative"]
|
||||
provenance = {
|
||||
"kind": DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"reviewed_digest": command["sha256"],
|
||||
"candidate_hash": candidate["hash"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
}
|
||||
return self._link_compatible(
|
||||
conn, epoch, current, total, positive, negative, provenance, command
|
||||
)
|
||||
|
||||
def _apply_manual(self, conn, epoch, current, decision_id, total, **command):
|
||||
"""Link the chosen Library prompt, compatible-only; raise otherwise."""
|
||||
missing = _Rollback(
|
||||
ReviewCommandError("prompt-missing", epoch, item=current, total=total)
|
||||
)
|
||||
positive_id = command["manual"]["positive_id"]
|
||||
pair_id = command["manual"]["pair_id"]
|
||||
row = conn.execute(_MANUAL_POSITIVE, (positive_id,)).fetchone()
|
||||
if row is None:
|
||||
raise missing
|
||||
positive_text, negative_text = row[1], ""
|
||||
if pair_id is not None:
|
||||
chosen = conn.execute(_MANUAL_PAIR, (pair_id, positive_id)).fetchone()
|
||||
if chosen is None:
|
||||
raise missing
|
||||
negative_text = chosen[0]
|
||||
elif conn.execute(_ARCHIVED_BARE_PAIR, (positive_id,)).fetchone():
|
||||
raise missing
|
||||
provenance = {
|
||||
"kind": MANUAL_KIND,
|
||||
"decision_id": decision_id,
|
||||
"reviewed_digest": command["sha256"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"positive_id": positive_id,
|
||||
"pair_id": pair_id,
|
||||
}
|
||||
# The sides name no provenance: only the ingester reads them.
|
||||
pair = self._link_compatible(
|
||||
conn,
|
||||
epoch,
|
||||
current,
|
||||
total,
|
||||
{"text": positive_text},
|
||||
{"text": negative_text},
|
||||
provenance,
|
||||
command,
|
||||
)
|
||||
# A prompt whose stored hash is stale reaches another row: never link that.
|
||||
reached = conn.execute(
|
||||
"SELECT positive_prompt_id FROM prompt_pairs WHERE id=?", (pair["id"],)
|
||||
).fetchone()[0]
|
||||
if reached != positive_id or pair_id not in (None, pair["id"]):
|
||||
raise missing
|
||||
return pair
|
||||
|
||||
def _link_compatible(
|
||||
self,
|
||||
conn,
|
||||
epoch,
|
||||
current,
|
||||
total,
|
||||
positive,
|
||||
negative,
|
||||
provenance,
|
||||
command,
|
||||
) -> dict:
|
||||
# Imported here: pair_ingestion imports operations, which composes this.
|
||||
from .pair_ingestion import _ingest_resolved_pair_in
|
||||
|
||||
@@ -242,17 +359,6 @@ class ReviewDecisionOperations:
|
||||
# Enrich the one row of this file under the spelling it was
|
||||
# linked with, so the ingester's compatibility rules see it.
|
||||
image_path = rows[0][1]
|
||||
candidate = command["candidate"]
|
||||
positive, negative = candidate["positive"], candidate["negative"]
|
||||
provenance = {
|
||||
"kind": DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"reviewed_digest": command["sha256"],
|
||||
"candidate_hash": candidate["hash"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
}
|
||||
result = _ingest_resolved_pair_in(
|
||||
conn,
|
||||
self,
|
||||
|
||||
@@ -229,13 +229,15 @@ def evidence_offers_choice(evidence: Any) -> bool:
|
||||
"""
|
||||
if not isinstance(evidence, dict):
|
||||
return False
|
||||
# The saver anchor's one pair (#390) is a suggestion to confirm.
|
||||
needed = 1 if evidence.get("anchor") == "saver" else MIN_CHOOSABLE
|
||||
if evidence.get("limited"):
|
||||
total = evidence.get("candidate_total")
|
||||
return type(total) is int and total >= MIN_CHOOSABLE
|
||||
return type(total) is int and total >= needed
|
||||
candidates = evidence.get("candidates")
|
||||
if not isinstance(candidates, list):
|
||||
return False
|
||||
return sum(1 for c in candidates if choosable(c)) >= MIN_CHOOSABLE
|
||||
return sum(1 for c in candidates if choosable(c)) >= needed
|
||||
|
||||
|
||||
def evidence_digest(text: str) -> str:
|
||||
@@ -380,15 +382,21 @@ def _structure_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
|
||||
|
||||
def _evidence_error(conn: sqlite3.Connection) -> Optional[str]:
|
||||
"""Stream every evidence document once; one row is held at a time."""
|
||||
for evidence_hash, text in conn.execute(
|
||||
"""Stream every evidence document once; one row is held at a time.
|
||||
|
||||
Rows are read with ``fetchone``: iterating a managed connection's cursor
|
||||
would prefetch ``ITER_CHUNK`` (128) rows (``database/managed_connection.py``).
|
||||
"""
|
||||
items = conn.execute(
|
||||
f"SELECT evidence_hash, evidence FROM {ITEMS_TABLE}" # nosec B608
|
||||
):
|
||||
)
|
||||
while (row := items.fetchone()) is not None:
|
||||
evidence_hash, text = row[0], row[1]
|
||||
if not _is_canonical(text) or evidence_digest(text) != evidence_hash:
|
||||
return f"Review queue table {ITEMS_TABLE} contains invalid evidence"
|
||||
for (text,) in conn.execute(
|
||||
f"SELECT evidence FROM {DECISIONS_TABLE}" # nosec B608
|
||||
):
|
||||
decisions = conn.execute(f"SELECT evidence FROM {DECISIONS_TABLE}") # nosec B608
|
||||
while (row := decisions.fetchone()) is not None:
|
||||
text = row[0]
|
||||
if not _is_canonical(text):
|
||||
return f"Review queue table {DECISIONS_TABLE} contains invalid evidence"
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
"""Attribute an output file to a stored prompt through recorded history (#342).
|
||||
|
||||
Execution history records, per run, the files ComfyUI reported
|
||||
(``execution_run_outputs``) and the PromptManager texts the run used
|
||||
(``execution_run_contexts``, whose ``positive_prompt_id`` hint is set only
|
||||
while the raw text matches exactly one stored prompt). A file is attributed
|
||||
only when all of this holds:
|
||||
|
||||
- exactly one recorded run reported the file's output location (same output
|
||||
root, subfolder and name);
|
||||
- the file was last written while that run was executing, from its dequeue
|
||||
to its terminal event (else the finish callback): a later run that history
|
||||
did not record may have reused the name;
|
||||
- every positive context of that run names the same stored prompt.
|
||||
|
||||
Anything else returns None: the caller leaves the file unlinked. Read-only.
|
||||
|
||||
``recorded_output`` applies the first two rules and returns, for the
|
||||
inspector (#386), the one output node (save node) that run reported the file
|
||||
from and the texts PromptManager recorded at execution per sampler.
|
||||
|
||||
Lookups are batched (``stored_run_prompts``): one parameterized query per
|
||||
chunk of at most ``LOOKUP_CHUNK`` distinct file names, matched in memory. The
|
||||
history schema is left exactly as it is (no extra index), so older builds can
|
||||
still open the library.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import weakref
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
try:
|
||||
from ..utils.execution_outputs import snapshot_output_roots
|
||||
from ..utils.library_location import import_folder_paths
|
||||
except ImportError:
|
||||
from utils.execution_outputs import snapshot_output_roots
|
||||
from utils.library_location import import_folder_paths
|
||||
|
||||
# File times may trail or lead the recorded run times a little (coarse file
|
||||
# system clocks, the write finishing just after the host reported it).
|
||||
MTIME_SLACK_MS = 2000
|
||||
|
||||
# Distinct file names per lookup query: well under SQLite's bound-parameter
|
||||
# limit (999 on old builds), so a scan batch costs one query per chunk.
|
||||
LOOKUP_CHUNK = 500
|
||||
|
||||
_RUNS_FOR_NAMES = (
|
||||
"SELECT DISTINCT o.filename, o.subfolder, o.root_key, o.path_style, o.run_id, "
|
||||
"r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at), "
|
||||
"o.output_node_id "
|
||||
"FROM execution_run_outputs o JOIN execution_runs r ON r.run_id = o.run_id "
|
||||
"WHERE o.folder_type = 'output' AND o.filename IN ({marks})"
|
||||
)
|
||||
|
||||
|
||||
def host_output_directory() -> Optional[str]:
|
||||
"""ComfyUI's output directory, or None outside ComfyUI."""
|
||||
folder_paths = import_folder_paths()
|
||||
if folder_paths is None:
|
||||
return None
|
||||
try:
|
||||
value = folder_paths.get_output_directory()
|
||||
except Exception:
|
||||
return None
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
|
||||
def _inside(image_path: str, output_root: str) -> Optional[str]:
|
||||
"""``image_path`` relative to ``output_root``, or None when outside it."""
|
||||
try:
|
||||
relative = os.path.relpath(image_path, output_root)
|
||||
except ValueError: # another drive on Windows
|
||||
return None
|
||||
if os.path.isabs(relative) or relative == os.pardir:
|
||||
return None
|
||||
if relative.startswith(os.pardir + os.sep):
|
||||
return None
|
||||
return relative
|
||||
|
||||
|
||||
def _resolved(path: str) -> str:
|
||||
try:
|
||||
return os.path.realpath(path)
|
||||
except (OSError, ValueError):
|
||||
return os.path.abspath(path)
|
||||
|
||||
|
||||
def _location(image_path: str, output_root: str, path_style: str):
|
||||
"""(root key, subfolder, filename) as history records them, or None.
|
||||
|
||||
The root key is always the host's own spelling of its output folder,
|
||||
because history recorded that one. The file may be spelled through the
|
||||
same folder or through its resolved path (a symlinked or junction output
|
||||
folder, as scan roots are): the lexical spelling is tried first, then
|
||||
both sides resolved.
|
||||
"""
|
||||
root_key = snapshot_output_roots(path_style, output_root, output_root).output_key
|
||||
if root_key is None:
|
||||
return None
|
||||
relative = _inside(os.path.abspath(image_path), os.path.abspath(output_root))
|
||||
if relative is None:
|
||||
relative = _inside(_resolved(image_path), _resolved(output_root))
|
||||
if relative is None:
|
||||
return None
|
||||
subfolder, filename = os.path.split(relative)
|
||||
if not filename:
|
||||
return None
|
||||
return root_key, subfolder, filename
|
||||
|
||||
|
||||
def in_output_folder(image_path: str, output_root: Optional[str]) -> bool:
|
||||
"""Whether history could ever attribute ``image_path`` (inside the host's
|
||||
output folder, in either spelling)."""
|
||||
if not image_path or not output_root:
|
||||
return False
|
||||
style = "windows" if os.name == "nt" else "posix"
|
||||
return _location(image_path, output_root, style) is not None
|
||||
|
||||
|
||||
def _written_during(image_path: str, started: Any, finished: Any) -> bool:
|
||||
if not isinstance(started, int) or not isinstance(finished, int):
|
||||
return False
|
||||
try:
|
||||
written = os.stat(image_path).st_mtime_ns // 1_000_000
|
||||
except OSError:
|
||||
return False
|
||||
return started - MTIME_SLACK_MS <= written <= finished + MTIME_SLACK_MS
|
||||
|
||||
|
||||
def stored_run_prompt(
|
||||
db: Any,
|
||||
image_path: str,
|
||||
output_root: Optional[str],
|
||||
path_style: Optional[str] = None,
|
||||
) -> Optional[int]:
|
||||
"""The stored prompt id history attributes to ``image_path``, else None."""
|
||||
if not image_path:
|
||||
return None
|
||||
return stored_run_prompts(db, [image_path], output_root, path_style).get(image_path)
|
||||
|
||||
|
||||
def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]:
|
||||
"""``{(filename, subfolder, root_key, style): {run_id: (start, end, nodes)}}``.
|
||||
|
||||
``nodes`` is the set of output node ids that reported the location in that
|
||||
run. One query per ``LOOKUP_CHUNK`` names; None without history tables.
|
||||
"""
|
||||
found: Dict[tuple, dict] = {}
|
||||
conn = db.model.get_connection()
|
||||
for start in range(0, len(names), LOOKUP_CHUNK):
|
||||
chunk = names[start : start + LOOKUP_CHUNK]
|
||||
query = _RUNS_FOR_NAMES.format(marks=",".join("?" for _ in chunk))
|
||||
try:
|
||||
rows = conn.execute(query, chunk).fetchall() # nosec B608
|
||||
except sqlite3.Error:
|
||||
return None # no history tables (a library without history)
|
||||
for name, subfolder, root_key, style, run_id, began, ended, node in rows:
|
||||
runs = found.setdefault((name, subfolder, root_key, style), {})
|
||||
runs.setdefault(run_id, (began, ended, set()))[2].add(node)
|
||||
return found
|
||||
|
||||
|
||||
def recorded_runs(
|
||||
db: Any,
|
||||
image_paths: Iterable[str],
|
||||
output_root: Optional[str],
|
||||
path_style: Optional[str],
|
||||
) -> Dict[str, tuple]:
|
||||
"""``{path: (run_id, output node ids)}`` for each file history attributes.
|
||||
|
||||
A file is attributed when exactly one recorded run reported its location
|
||||
and it was last written while that run executed.
|
||||
"""
|
||||
paths = list(dict.fromkeys(path for path in image_paths if path))
|
||||
if not paths or not output_root:
|
||||
return {}
|
||||
style = path_style or ("windows" if os.name == "nt" else "posix")
|
||||
located = {}
|
||||
for path in paths:
|
||||
location = _location(path, output_root, style)
|
||||
if location is not None:
|
||||
located[path] = location
|
||||
if not located:
|
||||
return {}
|
||||
names = sorted({filename for _key, _sub, filename in located.values()})
|
||||
runs = _runs_by_location(db, names)
|
||||
if not runs:
|
||||
return {}
|
||||
attributed = {}
|
||||
for path, (root_key, subfolder, filename) in located.items():
|
||||
reported = runs.get((filename, subfolder, root_key, style), {})
|
||||
if len(reported) != 1:
|
||||
continue
|
||||
((run_id, (began, ended, nodes)),) = reported.items()
|
||||
if _written_during(path, began, ended):
|
||||
attributed[path] = (run_id, nodes)
|
||||
return attributed
|
||||
|
||||
|
||||
def named_prompt(db: Any, run_id: str) -> Optional[int]:
|
||||
"""The one stored prompt every positive context of ``run_id`` names, else None."""
|
||||
contexts = db.get_execution_contexts(run_id).get("contexts") or []
|
||||
named = {
|
||||
context.get("positive_prompt_id")
|
||||
for context in contexts
|
||||
if context.get("positive") is not None
|
||||
}
|
||||
if len(named) != 1:
|
||||
return None
|
||||
(prompt_id,) = named
|
||||
return prompt_id if isinstance(prompt_id, int) else None
|
||||
|
||||
|
||||
def stored_run_prompts(
|
||||
db: Any,
|
||||
image_paths: Iterable[str],
|
||||
output_root: Optional[str],
|
||||
path_style: Optional[str] = None,
|
||||
) -> Dict[str, Optional[int]]:
|
||||
"""``{path: stored prompt id or None}`` for many files in few queries.
|
||||
|
||||
The output lookup is one query per ``LOOKUP_CHUNK`` distinct names; each
|
||||
run that alone reported a file is then read once for its contexts.
|
||||
"""
|
||||
paths = list(dict.fromkeys(path for path in image_paths if path))
|
||||
result: Dict[str, Optional[int]] = dict.fromkeys(paths)
|
||||
prompts: Dict[str, Optional[int]] = {}
|
||||
for path, (run_id, _nodes) in recorded_runs(
|
||||
db, paths, output_root, path_style
|
||||
).items():
|
||||
if run_id not in prompts:
|
||||
prompts[run_id] = named_prompt(db, run_id)
|
||||
result[path] = prompts[run_id]
|
||||
return result
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RecordedOutput:
|
||||
"""What recorded history knows about the run that wrote one file (#386).
|
||||
|
||||
``saver_id`` is the one output node the run reported the file from (None
|
||||
when several did). ``texts`` maps each sampler anchor of the run to the
|
||||
effective texts PromptManager recorded at execution for its roles
|
||||
(``{"positive": text or None, "negative": text or None}``); a role whose
|
||||
text was only inferred from the graph is None.
|
||||
"""
|
||||
|
||||
saver_id: Optional[str]
|
||||
texts: Dict[str, Dict[str, Optional[str]]]
|
||||
|
||||
|
||||
def _recorded_texts(db: Any, run_id: str) -> Dict[str, Dict[str, Optional[str]]]:
|
||||
texts = {}
|
||||
for context in db.get_execution_contexts(run_id).get("contexts") or []:
|
||||
evidence = context.get("evidence") or {}
|
||||
roles = {}
|
||||
for role in ("positive", "negative"):
|
||||
values = context.get(role) or {}
|
||||
proof = (evidence.get(role) or {}).get("effective") or {}
|
||||
value = values.get("effective")
|
||||
recorded = proof.get("provenance") == "recorded"
|
||||
roles[role] = value if recorded and isinstance(value, str) else None
|
||||
texts[context["anchor_id"]] = roles
|
||||
return texts
|
||||
|
||||
|
||||
def recorded_output(
|
||||
db: Any,
|
||||
image_path: str,
|
||||
output_root: Optional[str],
|
||||
path_style: Optional[str] = None,
|
||||
) -> Optional[RecordedOutput]:
|
||||
"""The recorded run facts for ``image_path``, else None (#386).
|
||||
|
||||
The file must be attributed to one run (``recorded_runs``: one run
|
||||
reported its location and it was written during that run). Read-only; it
|
||||
never links anything.
|
||||
"""
|
||||
if not image_path:
|
||||
return None
|
||||
conn = db.model.get_connection()
|
||||
own_transaction = not conn.in_transaction
|
||||
if own_transaction: # outputs and texts from one state of history
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
found = recorded_runs(db, [image_path], output_root, path_style)
|
||||
if image_path not in found:
|
||||
return None
|
||||
run_id, nodes = found[image_path]
|
||||
saver = next(iter(nodes)) if len(nodes) == 1 else None
|
||||
return RecordedOutput(saver, _recorded_texts(db, run_id))
|
||||
finally:
|
||||
if own_transaction:
|
||||
conn.rollback()
|
||||
|
||||
|
||||
_LOGGER = logging.getLogger("prompt_manager.run_attribution")
|
||||
|
||||
|
||||
def recorded_output_lookup(
|
||||
db: Any, *, close: bool = False, reference: Optional[dict] = None
|
||||
) -> Callable[[Any], Any]:
|
||||
"""``lookup(path) -> RecordedOutput or None`` for the inspect routes (#386).
|
||||
|
||||
Uses ComfyUI's output folder. Any failure (no history, a locked library)
|
||||
is logged and names no node, so the file is judged from its own metadata.
|
||||
``close`` closes the calling thread's connection afterwards, for routes
|
||||
that close theirs after reading. ``reference`` (a history occurrence's
|
||||
output reference) spells the file as history recorded it: the route's
|
||||
target is case-normalized on Windows, and history matches names exactly.
|
||||
"""
|
||||
|
||||
def lookup(path: Any) -> Optional[RecordedOutput]:
|
||||
root = host_output_directory()
|
||||
if reference is not None:
|
||||
if reference.get("folder_type") != "output" or not root:
|
||||
return None
|
||||
path = os.path.join(
|
||||
root, reference.get("subfolder") or "", reference.get("filename") or ""
|
||||
)
|
||||
try:
|
||||
return recorded_output(db, str(path), root)
|
||||
except (sqlite3.Error, OSError, ValueError):
|
||||
_LOGGER.warning("Recorded run lookup failed", exc_info=True)
|
||||
return None
|
||||
except Exception:
|
||||
_LOGGER.exception("Recorded run lookup failed unexpectedly")
|
||||
return None
|
||||
finally:
|
||||
if close:
|
||||
db.model.close()
|
||||
|
||||
return lookup
|
||||
|
||||
|
||||
_SEALED_LISTENERS: List[weakref.WeakMethod] = []
|
||||
_SEALED_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def subscribe_outputs_sealed(method: Callable[[str, List[dict]], None]) -> None:
|
||||
"""Call bound ``method(run_id, references)`` after a run's outputs are recorded.
|
||||
|
||||
Held weakly: a listener that is garbage collected is dropped.
|
||||
"""
|
||||
with _SEALED_LOCK:
|
||||
_SEALED_LISTENERS.append(weakref.WeakMethod(method))
|
||||
|
||||
|
||||
def notify_outputs_sealed(run_id: str, references: Iterable[dict]) -> None:
|
||||
"""Tell listeners a run's output references were just committed.
|
||||
|
||||
A listener failure is logged and never reaches the history writer.
|
||||
"""
|
||||
references = list(references)
|
||||
with _SEALED_LOCK:
|
||||
alive = [ref for ref in _SEALED_LISTENERS if ref() is not None]
|
||||
_SEALED_LISTENERS[:] = alive
|
||||
listeners = [ref() for ref in alive]
|
||||
for listener in listeners:
|
||||
if listener is None:
|
||||
continue
|
||||
try:
|
||||
listener(run_id, references)
|
||||
except Exception:
|
||||
_LOGGER.warning("Output seal listener failed", exc_info=True)
|
||||
@@ -0,0 +1,185 @@
|
||||
"""Safety backups beside the live database: one writer for every caller.
|
||||
|
||||
A safety backup is ``<db>.<label>_<YYYYmmdd_HHMMSS_ffffff>[_n]`` in the
|
||||
library's own folder. It is written by SQLite's backup API into a temp file
|
||||
beside the database (``SAFETY_TEMP_PREFIX``, swept at startup after a crash),
|
||||
made one self-contained file (rollback journal), optionally verified, fsynced,
|
||||
made private (``0o600`` on POSIX) and only then given its unique name, so a
|
||||
backup name never carries a half-written file. The names are unique within
|
||||
one second (microseconds, then a counter): on Windows two backups in the same
|
||||
second must not overwrite each other.
|
||||
|
||||
Maintenance applies take a verified backup (``py/api/maintenance.py``); a
|
||||
restore takes an unverified one, because the database it replaces may be the
|
||||
damaged one the user is recovering from (``database/restore_stage.py``).
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from contextlib import closing
|
||||
|
||||
SAFETY_TEMP_PREFIX = ".pm-safety-"
|
||||
_NAME_ATTEMPTS = 1000
|
||||
|
||||
|
||||
class SafetyBackupFailed(Exception):
|
||||
"""The safety backup could not be written, verified or put in place.
|
||||
|
||||
``reason`` is the verify reason when the copy did not verify.
|
||||
"""
|
||||
|
||||
def __init__(self, message, reason=None):
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def _now():
|
||||
return datetime.datetime.now()
|
||||
|
||||
|
||||
def remove_quietly(*paths):
|
||||
"""Remove each existing path, ignoring errors (cleanup after a failure)."""
|
||||
for path in paths:
|
||||
try:
|
||||
if path and os.path.exists(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def backup_names(db_path, label="backup"):
|
||||
"""Candidate names, unique within one second (microseconds, then ``_n``)."""
|
||||
base = f"{db_path}.{label}_{_now().strftime('%Y%m%d_%H%M%S_%f')}"
|
||||
for attempt in range(_NAME_ATTEMPTS):
|
||||
yield base if attempt == 0 else f"{base}_{attempt}"
|
||||
|
||||
|
||||
def reserve_backup_name(db_path, label="backup"):
|
||||
"""Create and return an unused ``<db>.<label>_<stamp>[_n]`` (O_EXCL)."""
|
||||
for candidate in backup_names(db_path, label):
|
||||
try:
|
||||
fd = os.open(candidate, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||
except FileExistsError:
|
||||
continue
|
||||
os.close(fd)
|
||||
return candidate
|
||||
raise SafetyBackupFailed("No free backup name")
|
||||
|
||||
|
||||
def place_backup(temp, db_path, label="backup"):
|
||||
"""Give the finished ``temp`` its unique backup name; return that path.
|
||||
|
||||
``os.link`` is atomic and refuses an existing name, so no empty or half
|
||||
file ever carries a backup name, even across a crash. A filesystem
|
||||
without hard links falls back to reserving the name (``O_EXCL``) and
|
||||
``os.replace``. The temp file is removed by the caller.
|
||||
"""
|
||||
for candidate in backup_names(db_path, label):
|
||||
try:
|
||||
os.link(temp, candidate)
|
||||
except FileExistsError:
|
||||
continue
|
||||
except (OSError, NotImplementedError):
|
||||
break # no hard links here
|
||||
else:
|
||||
return candidate
|
||||
else:
|
||||
raise SafetyBackupFailed("No free backup name")
|
||||
final = reserve_backup_name(db_path, label)
|
||||
try:
|
||||
os.replace(temp, final)
|
||||
except OSError:
|
||||
remove_quietly(final)
|
||||
raise
|
||||
return final
|
||||
|
||||
|
||||
def sweep_safety_temps(db_path):
|
||||
"""Remove the temp copies an interrupted safety backup left (blocking).
|
||||
|
||||
Called once at startup, before any job can take a backup. Returns the
|
||||
names removed.
|
||||
"""
|
||||
folder = os.path.dirname(os.path.abspath(db_path))
|
||||
removed = []
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return removed
|
||||
for name in names:
|
||||
if not (name.startswith(SAFETY_TEMP_PREFIX) and ".tmp" in name):
|
||||
continue
|
||||
try:
|
||||
os.remove(os.path.join(folder, name))
|
||||
removed.append(name)
|
||||
except OSError:
|
||||
continue
|
||||
return removed
|
||||
|
||||
|
||||
def make_standalone(path):
|
||||
"""Make the copy one self-contained file (rollback journal, not WAL).
|
||||
|
||||
The backup API copies the live database's WAL flag; a WAL file opened
|
||||
read-only later (a verify, a restore preview) would leave ``-wal`` and
|
||||
``-shm`` sidecars beside it.
|
||||
"""
|
||||
try:
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
conn.execute("PRAGMA journal_mode=DELETE").fetchone()
|
||||
except sqlite3.Error as exc:
|
||||
raise SafetyBackupFailed("The backup could not be finished") from exc
|
||||
|
||||
|
||||
def fsync_file(path):
|
||||
with open(path, "rb+") as handle:
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
|
||||
def safety_temp(db_path):
|
||||
"""A new, closed temp file beside the database (``SAFETY_TEMP_PREFIX``)."""
|
||||
fd, temp = tempfile.mkstemp(
|
||||
dir=os.path.dirname(os.path.abspath(db_path)),
|
||||
prefix=SAFETY_TEMP_PREFIX,
|
||||
suffix=".tmp",
|
||||
)
|
||||
os.close(fd)
|
||||
return temp
|
||||
|
||||
|
||||
def take_safety_backup(model, label="backup", *, verify=True):
|
||||
"""Write (and by default verify) a safety backup; return its path (blocking).
|
||||
|
||||
The copy is made in a temp file beside the database and only renamed
|
||||
onto its final, unique name ``<db>.<label>_<stamp>`` once it is complete
|
||||
(and verified), so a backup file on disk is always a whole one. ``label``
|
||||
"backup" joins the pruned pool; another label (the workflow storage
|
||||
migration's ``pre-workflow-storage``) is kept until the user removes it.
|
||||
Raises SafetyBackupFailed, leaving no file.
|
||||
"""
|
||||
db_path = os.path.abspath(model.db_path)
|
||||
temp = safety_temp(db_path)
|
||||
final = None
|
||||
try:
|
||||
if not model.backup_database(temp):
|
||||
raise SafetyBackupFailed("The backup could not be written")
|
||||
make_standalone(temp)
|
||||
if verify:
|
||||
ok, reason = model.verify_database_file(temp)
|
||||
if not ok:
|
||||
raise SafetyBackupFailed("The backup did not verify", reason)
|
||||
fsync_file(temp)
|
||||
if os.name != "nt":
|
||||
os.chmod(temp, 0o600)
|
||||
final = place_backup(temp, db_path, label)
|
||||
return final
|
||||
except OSError as exc:
|
||||
remove_quietly(final)
|
||||
raise SafetyBackupFailed("The backup could not be put in place") from exc
|
||||
except SafetyBackupFailed:
|
||||
remove_quietly(final)
|
||||
raise
|
||||
finally:
|
||||
remove_quietly(temp, temp + "-journal", temp + "-wal", temp + "-shm")
|
||||
@@ -33,8 +33,14 @@ MAX_TAG_LENGTH = 50
|
||||
_IMAGE_SORTS = ("newest", "oldest", "name", "name-desc", "rating")
|
||||
_LIBRARY_SORTS = ("newest", "oldest", "name", "rating")
|
||||
_MEDIA = ("all", "image", "video")
|
||||
# How a Library view's Tags match (#444): every tag, any of them, or parents
|
||||
# without tags ("untagged" never goes with tags; the listing refuses both).
|
||||
_TAG_MODES = ("all", "any", "untagged")
|
||||
_CONTROL = re.compile(r"[\x00-\x1f\x7f]")
|
||||
_SOURCE = re.compile(r"all|0|[1-9][0-9]{0,8}")
|
||||
# A collection id as the listing's ``collection`` query takes it (#331): none
|
||||
# ("") or 1 to 9 digits without a leading zero.
|
||||
_COLLECTION = re.compile(r"|[1-9][0-9]{0,8}")
|
||||
FALLBACK_NAME = "Saved view"
|
||||
|
||||
|
||||
@@ -55,12 +61,18 @@ _STATE_FIELDS = {
|
||||
"tags": ("tags", None, ()),
|
||||
# The Favorites page's filter (#249 D51): a view saved there reopens it.
|
||||
"favorites": ("bool", None, False),
|
||||
# A view saved on a collection page (#331 D140) reopens that
|
||||
# collection; never together with favorites.
|
||||
"collection": ("collection", None, ""),
|
||||
},
|
||||
"library": {
|
||||
"q": ("text", 500, ""),
|
||||
"category": ("text", 100, ""),
|
||||
"sort": ("choice", _LIBRARY_SORTS, "newest"),
|
||||
"tags": ("tags", None, ()),
|
||||
# #444: the Tags match and the output folder (relative, "/"-joined).
|
||||
"tag_mode": ("choice", _TAG_MODES, "all"),
|
||||
"folder": ("text", 1024, ""),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -125,6 +137,8 @@ def _validate_value(key: str, kind: str, limit, value: object) -> None:
|
||||
raise SavedViewInvalid(f"State '{key}' is not a supported value")
|
||||
if kind == "source" and not _SOURCE.fullmatch(value):
|
||||
raise SavedViewInvalid(f"State '{key}' must be 'all' or a source index")
|
||||
if kind == "collection" and not _COLLECTION.fullmatch(value):
|
||||
raise SavedViewInvalid(f"State '{key}' must be empty or a collection id")
|
||||
if kind == "text" and (len(value) > limit or _CONTROL.search(value)):
|
||||
raise SavedViewInvalid(f"State '{key}' is too long or has control characters")
|
||||
|
||||
@@ -142,6 +156,10 @@ def encode_state(page: object, state: object) -> str:
|
||||
)
|
||||
for key, (kind, limit, _default) in fields.items():
|
||||
_validate_value(key, kind, limit, state[key])
|
||||
if state.get("favorites") is True and state.get("collection"):
|
||||
raise SavedViewInvalid("A view cannot filter by favorites and a collection")
|
||||
if state.get("tag_mode") == "untagged" and state.get("tags"):
|
||||
raise SavedViewInvalid("A view cannot list untagged prompts and filter tags")
|
||||
encoded = json.dumps(
|
||||
{key: state[key] for key in fields}, ensure_ascii=False, separators=(",", ":")
|
||||
)
|
||||
@@ -181,6 +199,13 @@ def normalize_state(page: str, raw: object) -> dict:
|
||||
except SavedViewInvalid:
|
||||
value = default
|
||||
state[key] = list(value) if kind == "tags" else value
|
||||
# Never written by this build; a stored row with both opens the collection,
|
||||
# as the listing does for a URL with both (#331 D140).
|
||||
if state.get("favorites") is True and state.get("collection"):
|
||||
state["favorites"] = False
|
||||
# Never written by this build either: the tags win.
|
||||
if state.get("tag_mode") == "untagged" and state.get("tags"):
|
||||
state["tag_mode"] = "all"
|
||||
return state
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Link an output file to the prompt of one text, creating it when new (#247).
|
||||
|
||||
The store step of ingestion from file metadata when the decision is a
|
||||
positive text (``utils.file_prompt``). The prompt insert and the link run in
|
||||
one ``BEGIN IMMEDIATE`` transaction: two writers never store one text twice,
|
||||
a failed link leaves no new prompt behind, and a file stays linked to one
|
||||
prompt (a file already linked to another prompt is a ``conflict``).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from .pair_ingestion import Marks, file_link_rows, link_conflict, mark_created
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
_LOGGER = logging.getLogger("prompt_manager.database.scanned_prompts")
|
||||
|
||||
|
||||
def store_scanned_prompt_or_raise(
|
||||
db: Any, path: str, canonical: Dict[str, Any], text: str, marks: Marks
|
||||
) -> Tuple[str, Optional[int]]:
|
||||
"""``(status, prompt id)`` after linking ``path`` to ``text``'s prompt.
|
||||
|
||||
``added`` (prompt created, carrying ``marks``), ``linked`` (stored text
|
||||
reused), ``unchanged`` (already linked to it) or ``conflict`` (linked to
|
||||
another prompt; nothing written, id None). Database errors are raised
|
||||
and roll the whole store back, so a caller can run it again.
|
||||
"""
|
||||
with db._pair_transaction() as conn:
|
||||
rows = file_link_rows(conn, path)
|
||||
if link_conflict(rows, generate_prompt_hash(text)):
|
||||
return "conflict", None
|
||||
if rows:
|
||||
return "unchanged", rows[0]["prompt_id"]
|
||||
prompt_id, created = db._pair_component_outcome(conn, text)
|
||||
if created:
|
||||
mark_created(conn, db, prompt_id, marks, os.path.basename(path))
|
||||
db._insert_image_link(conn, prompt_id, path, canonical)
|
||||
return ("added" if created else "linked"), prompt_id
|
||||
|
||||
|
||||
def store_scanned_prompt(
|
||||
db: Any, path: str, canonical: Dict[str, Any], text: str, marks: Marks
|
||||
) -> Tuple[str, Optional[int]]:
|
||||
"""The scan's form: an error is logged and reported as ``("found", None)``.
|
||||
|
||||
A write that failed because Move library switched runs again on the new
|
||||
library (``survive_library_switch``); the rerun re-checks the link rows.
|
||||
"""
|
||||
try:
|
||||
return db.survive_library_switch(
|
||||
lambda: store_scanned_prompt_or_raise(db, path, canonical, text, marks)
|
||||
)
|
||||
except Exception as error:
|
||||
_LOGGER.error(
|
||||
f"Failed to store the prompt of {os.path.basename(path)}: {error}"
|
||||
)
|
||||
return "found", None
|
||||
|
||||
|
||||
def link_stored_prompt_or_raise(
|
||||
db: Any, path: str, canonical: Dict[str, Any], prompt_id: int
|
||||
) -> Optional[str]:
|
||||
"""Link ``path`` to the stored prompt ``prompt_id`` (recorded history, #342).
|
||||
|
||||
``linked``, ``unchanged`` (already linked to it), ``conflict`` (linked to
|
||||
another prompt; nothing written) or None when the prompt is gone. One
|
||||
``BEGIN IMMEDIATE`` transaction; database errors are raised.
|
||||
"""
|
||||
with db._pair_transaction() as conn:
|
||||
rows = file_link_rows(conn, path)
|
||||
if len(rows) > 1 or (rows and rows[0]["prompt_id"] != prompt_id):
|
||||
return "conflict"
|
||||
if rows:
|
||||
return "unchanged"
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE id = ?", (prompt_id,)
|
||||
).fetchone()
|
||||
if exists is None:
|
||||
return None
|
||||
db._insert_image_link(conn, prompt_id, path, canonical)
|
||||
return "linked"
|
||||
@@ -13,12 +13,15 @@ same connection before commit. Owner rows and their timestamps never change.
|
||||
|
||||
Category consolidation is previewed in one ``library_snapshot`` and applied in
|
||||
one ``_pair_transaction`` that recomputes the owner projection and requires the
|
||||
preview's ``impact_token``; only rows carrying the exact source change.
|
||||
preview's ``impact_token``; only rows carrying the exact source change. Image
|
||||
rows holding the source (``image_categories``, #332) are owners too: they are
|
||||
counted, hashed into the token and renamed with the prompts and pairs.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
|
||||
from .library import _ELIGIBLE, _limit
|
||||
from .library import _ELIGIBLE, SHELVED_PAIR, UNTAGGED, _escape, _limit
|
||||
from .prompt_archive import SHELVED
|
||||
from .library_commands import _TOKEN, LibraryCommandError, _validate_epoch
|
||||
from .library_tokens import canonical_token
|
||||
from .models import utc_now_iso
|
||||
@@ -36,30 +39,76 @@ _IMAGE_FILES = (
|
||||
)
|
||||
|
||||
|
||||
def _tag_rows(page: str) -> str:
|
||||
def _tag_rows(page: str, order: str = "page.id") -> str:
|
||||
"""Count rows for the registry rows selected by ``page`` (id, name)."""
|
||||
# Eligible parent components carrying a tag in the page, plus the
|
||||
# positive parent of every active pair carrying it (always eligible).
|
||||
# Eligible parent components on the shelf carrying a tag in the page,
|
||||
# plus the positive parent of every active pair carrying it (always
|
||||
# eligible) when that parent is on the shelf: archived parents (#443)
|
||||
# are hidden with their pairs, so Browse lists what was counted.
|
||||
return (
|
||||
"WITH page AS (" + page + "),"
|
||||
"owners(tag_id,pid) AS ("
|
||||
"SELECT pt.tag_id,p.id FROM prompts p "
|
||||
"JOIN prompt_tags pt ON pt.prompt_id=p.id "
|
||||
"WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " "
|
||||
"AND " + SHELVED + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id "
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"WHERE pp.archived=0 AND " + SHELVED_PAIR + " "
|
||||
"AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"images(tag_id,n) AS (" + _IMAGE_FILES + " "
|
||||
"WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count,"
|
||||
"COALESCE(MAX(images.n),0) AS image_count "
|
||||
"FROM page LEFT JOIN owners ON owners.tag_id=page.id "
|
||||
"LEFT JOIN images ON images.tag_id=page.id "
|
||||
"GROUP BY page.id ORDER BY page.id"
|
||||
"GROUP BY page.id ORDER BY " + order
|
||||
)
|
||||
|
||||
|
||||
_TAG_PAGE = _tag_rows("SELECT id,name FROM tags WHERE id>? ORDER BY id LIMIT ?")
|
||||
# #444 tag sorts: sort -> (key expression on tags/counted, direction). Ties and
|
||||
# "created" follow the id, so every order is total and keyset pages are exact.
|
||||
TAG_SORTS = {
|
||||
"created": (None, "ASC"),
|
||||
"name": ("name COLLATE NOCASE", "ASC"),
|
||||
"name-desc": ("name COLLATE NOCASE", "DESC"),
|
||||
"prompts": ("prompt_count", "DESC"),
|
||||
"prompts-asc": ("prompt_count", "ASC"),
|
||||
}
|
||||
_MAX_SEARCH = 200
|
||||
_TAG_SEARCH = "name LIKE ? ESCAPE '\\'"
|
||||
# Every matching tag ranked by the prompt count its row shows (the owners rule
|
||||
# of ``_tag_rows``), then one keyset page of it with its image counts.
|
||||
_RANKED_TAG_PAGE = (
|
||||
"WITH owners(tag_id,pid) AS ("
|
||||
"SELECT pt.tag_id,p.id FROM prompts p "
|
||||
"JOIN prompt_tags pt ON pt.prompt_id=p.id WHERE " + _ELIGIBLE + " "
|
||||
"AND " + SHELVED + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id WHERE pp.archived=0 "
|
||||
"AND " + SHELVED_PAIR + "),"
|
||||
"counted AS (SELECT t.id,t.name,COUNT(DISTINCT o.pid) AS prompt_count "
|
||||
"FROM tags t LEFT JOIN owners o ON o.tag_id=t.id WHERE {search} "
|
||||
"GROUP BY t.id),"
|
||||
"page AS (SELECT id,name,prompt_count FROM counted WHERE {after} "
|
||||
"ORDER BY prompt_count {direction},id LIMIT ?),"
|
||||
"images(tag_id,n) AS (" + _IMAGE_FILES + " "
|
||||
"WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
|
||||
"SELECT page.id,page.name,page.prompt_count,"
|
||||
"COALESCE(images.n,0) AS image_count FROM page "
|
||||
"LEFT JOIN images ON images.tag_id=page.id "
|
||||
"ORDER BY page.prompt_count {direction},page.id"
|
||||
)
|
||||
_UNTAGGED_COUNT = (
|
||||
"SELECT COUNT(*) FROM prompts p WHERE ("
|
||||
+ _ELIGIBLE
|
||||
+ ") AND "
|
||||
+ UNTAGGED
|
||||
# Untagged prompts opens the shelf, which hides archived parents (#443).
|
||||
+ " AND "
|
||||
+ SHELVED
|
||||
)
|
||||
_TAG_ONE = _tag_rows("SELECT id,name FROM tags WHERE id=?")
|
||||
# Every owner kind a tag can sit on, for the delete confirmation.
|
||||
_TAG_IMPACT = (
|
||||
@@ -97,16 +146,34 @@ def _validate_tag_id(tag_id: object) -> None:
|
||||
raise ValueError("Invalid tag ID")
|
||||
|
||||
|
||||
# The /library/options universe: nonblank values on eligible parents or on
|
||||
# active pairs, grouped exactly (whitespace and case variants stay distinct).
|
||||
_CATEGORY_GROUPS = (
|
||||
"WITH owners(value,pid) AS ("
|
||||
"SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL "
|
||||
"AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
# Category values in Tags & categories (#332 §7): the prompt-backed values
|
||||
# (nonblank values on eligible parents or on active pairs; ``prompt_count``
|
||||
# counts only parents on the shelf, so a value only archived parents (#443)
|
||||
# hold stays listed with 0, as a tag does; grouped exactly, so
|
||||
# whitespace and case variants stay distinct; /library/options lists the same
|
||||
# prompt-backed set with its own query) plus the values only image rows hold.
|
||||
# A prompt-backed value is ordered by its oldest owning positive; an image-only
|
||||
# value sorts after all of them (first_pid is SQLite's INTEGER maximum, above
|
||||
# any prompt id), then by value. ``image_count`` counts stored rows, including
|
||||
# files missing on disk. No FULL OUTER JOIN, so it runs on SQLite 3.31+.
|
||||
_IMAGE_ONLY_FIRST_PID = 2**63 - 1
|
||||
_CATEGORY_ROWS = (
|
||||
"WITH owners(value,pid,shelved) AS ("
|
||||
"SELECT p.category,p.id," + SHELVED + " FROM prompts p "
|
||||
"WHERE p.category IS NOT NULL AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT pp.category,pp.positive_prompt_id," + SHELVED_PAIR + " "
|
||||
"FROM prompt_pairs pp "
|
||||
"WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!=''),"
|
||||
"grouped AS (SELECT value,MIN(pid) AS first_pid,"
|
||||
"COUNT(DISTINCT pid) AS prompt_count FROM owners GROUP BY value) "
|
||||
"image_values(value,image_count) AS ("
|
||||
"SELECT category,COUNT(*) FROM image_categories GROUP BY category),"
|
||||
"grouped(value,first_pid,prompt_count) AS ("
|
||||
"SELECT value,MIN(pid),COUNT(DISTINCT CASE WHEN shelved THEN pid END) "
|
||||
"FROM owners GROUP BY value "
|
||||
"UNION ALL SELECT value," + str(_IMAGE_ONLY_FIRST_PID) + ",0 FROM image_values "
|
||||
"WHERE value NOT IN (SELECT value FROM owners)),"
|
||||
"counted AS (SELECT g.value,g.first_pid,g.prompt_count,"
|
||||
"COALESCE(i.image_count,0) AS image_count FROM grouped g "
|
||||
"LEFT JOIN image_values i ON i.value=g.value) "
|
||||
)
|
||||
|
||||
|
||||
@@ -117,15 +184,23 @@ _CATEGORY_COMPONENTS = (
|
||||
"WHERE p.category=? ORDER BY p.id"
|
||||
)
|
||||
_CATEGORY_PAIRS = "SELECT id,archived FROM prompt_pairs WHERE category=? ORDER BY id"
|
||||
# Image rows holding one exact value (#332 DC7), an index search.
|
||||
_IMAGE_CATEGORY_ROWIDS = (
|
||||
"SELECT rowid FROM image_categories WHERE category=? ORDER BY rowid"
|
||||
)
|
||||
_CATEGORY_EXISTS = (
|
||||
"SELECT EXISTS(SELECT 1 FROM prompts WHERE category=:v)"
|
||||
" OR EXISTS(SELECT 1 FROM prompt_pairs WHERE category=:v)"
|
||||
" OR EXISTS(SELECT 1 FROM image_categories WHERE category=:v)"
|
||||
)
|
||||
|
||||
# Pairs after components, so a failing pair update rolls both back.
|
||||
# Pairs after components and image rows last, so a failing update rolls every
|
||||
# earlier one back. An image row is keyed by its file, never by its value, so
|
||||
# merging into a value other files already hold cannot conflict.
|
||||
_CATEGORY_UPDATES = (
|
||||
"UPDATE prompts SET category=?,updated_at=? WHERE category=?",
|
||||
"UPDATE prompt_pairs SET category=?,updated_at=? WHERE category=?",
|
||||
"UPDATE image_categories SET category=?,updated_at=? WHERE category=?",
|
||||
)
|
||||
|
||||
|
||||
@@ -136,8 +211,15 @@ def category_impact_token(
|
||||
target_exists: bool,
|
||||
components: list,
|
||||
pairs: list,
|
||||
*,
|
||||
images: list = (),
|
||||
) -> str:
|
||||
"""Hash a category projection like ``library_edit_token`` (no secret)."""
|
||||
"""Hash a category projection like ``library_edit_token`` (no secret).
|
||||
|
||||
``images`` are the rowids of the image rows holding ``source`` (#332 DC7).
|
||||
With none the preimage has no ``images`` key, so a prompt-only token keeps
|
||||
the value it had before image categories existed.
|
||||
"""
|
||||
preimage = {
|
||||
"epoch": epoch,
|
||||
"source": source,
|
||||
@@ -146,6 +228,8 @@ def category_impact_token(
|
||||
"components": sorted([int(i), bool(flag)] for i, flag in components),
|
||||
"pairs": sorted([int(i), bool(flag)] for i, flag in pairs),
|
||||
}
|
||||
if images:
|
||||
preimage["images"] = sorted(int(i) for i in images)
|
||||
return canonical_token(_CATEGORY_DOMAIN, preimage)
|
||||
|
||||
|
||||
@@ -177,6 +261,28 @@ def _tag_after(after: object) -> int:
|
||||
return after
|
||||
|
||||
|
||||
def _tag_search(search: object) -> str:
|
||||
if not isinstance(search, str) or len(search) > _MAX_SEARCH:
|
||||
raise ValueError(f"Tag search must be text of at most {_MAX_SEARCH} characters")
|
||||
return search.strip()
|
||||
|
||||
|
||||
def _sorted_tag_after(after: object, sort: str) -> tuple | None:
|
||||
"""The ``[key, id]`` cursor of a name or prompt-count sort."""
|
||||
if after is None:
|
||||
return None
|
||||
key_type = str if sort.startswith("name") else int
|
||||
if (
|
||||
not isinstance(after, list)
|
||||
or len(after) != 2
|
||||
or type(after[0]) is not key_type
|
||||
or type(after[1]) is not int
|
||||
or after[1] <= 0
|
||||
):
|
||||
raise ValueError("Invalid vocabulary cursor")
|
||||
return after[0], after[1]
|
||||
|
||||
|
||||
def _category_after(after: object) -> tuple[int, str] | None:
|
||||
if after is None:
|
||||
return None
|
||||
@@ -325,7 +431,8 @@ class VocabularyOperations:
|
||||
pairs = [
|
||||
[row[0], bool(row[1])] for row in conn.execute(_CATEGORY_PAIRS, (source,))
|
||||
]
|
||||
if not components and not pairs:
|
||||
images = [row[0] for row in conn.execute(_IMAGE_CATEGORY_ROWIDS, (source,))]
|
||||
if not components and not pairs and not images:
|
||||
raise LibraryCommandError("vocabulary-conflict", epoch)
|
||||
target_exists = bool(
|
||||
conn.execute(_CATEGORY_EXISTS, {"v": target}).fetchone()[0]
|
||||
@@ -339,9 +446,10 @@ class VocabularyOperations:
|
||||
"other_prompts": len(components) - eligible,
|
||||
"negative_variants": len(pairs) - archived,
|
||||
"archived_negative_variants": archived,
|
||||
"images": len(images),
|
||||
},
|
||||
"impact_token": category_impact_token(
|
||||
epoch, source, target, target_exists, components, pairs
|
||||
epoch, source, target, target_exists, components, pairs, images=images
|
||||
),
|
||||
}
|
||||
|
||||
@@ -365,7 +473,8 @@ class VocabularyOperations:
|
||||
def apply_vocabulary_category(
|
||||
self, *, expected_epoch: str, source: str, target: str, impact_token: str
|
||||
) -> dict:
|
||||
"""Set the exact ``source`` to ``target`` on every component and pair.
|
||||
"""Set the exact ``source`` to ``target`` on every component, pair
|
||||
and image row.
|
||||
|
||||
The recomputed projection must still match the preview's token; a
|
||||
mismatch is ``vocabulary-conflict`` with nothing written.
|
||||
@@ -395,22 +504,80 @@ class VocabularyOperations:
|
||||
epoch: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
after: int | None = None,
|
||||
after: object = None,
|
||||
search: str = "",
|
||||
sort: str = "created",
|
||||
) -> dict:
|
||||
"""One page of tags matching ``search`` (a literal, case-insensitive
|
||||
substring) in ``sort`` order, with the untagged parents' count."""
|
||||
_limit(limit)
|
||||
start = _tag_after(after)
|
||||
total = conn.execute("SELECT COUNT(*) FROM tags").fetchone()[0]
|
||||
rows = [dict(row) for row in conn.execute(_TAG_PAGE, (start, limit + 1))]
|
||||
search = _tag_search(search)
|
||||
if not isinstance(sort, str) or sort not in TAG_SORTS:
|
||||
raise ValueError("sort must be one of " + ", ".join(TAG_SORTS))
|
||||
where, params = "1=1", []
|
||||
if search:
|
||||
where, params = _TAG_SEARCH, [f"%{_escape(search)}%"]
|
||||
total, untagged = conn.execute(
|
||||
f"SELECT (SELECT COUNT(*) FROM tags WHERE {where}),({_UNTAGGED_COUNT})",
|
||||
params,
|
||||
).fetchone()
|
||||
rows = [
|
||||
dict(row) for row in self._tag_page(conn, where, params, sort, after, limit)
|
||||
]
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
key, _direction = TAG_SORTS[sort]
|
||||
next_key = None
|
||||
if has_more:
|
||||
last = rows[-1]
|
||||
next_key = (
|
||||
last["id"]
|
||||
if key is None
|
||||
else [
|
||||
last["prompt_count" if key == "prompt_count" else "name"],
|
||||
last["id"],
|
||||
]
|
||||
)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"tags": rows,
|
||||
"total": total,
|
||||
"untagged_count": untagged,
|
||||
# Echoed so a client can tell this read applied its search and sort.
|
||||
"search": search,
|
||||
"sort": sort,
|
||||
"has_more": has_more,
|
||||
"next_key": rows[-1]["id"] if has_more else None,
|
||||
"next_key": next_key,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _tag_page(conn, where, params, sort, after, limit):
|
||||
key, direction = TAG_SORTS[sort]
|
||||
if key is None:
|
||||
page = (
|
||||
f"SELECT id,name FROM tags WHERE {where} AND id>? ORDER BY id LIMIT ?"
|
||||
)
|
||||
return conn.execute(
|
||||
_tag_rows(page), [*params, _tag_after(after), limit + 1]
|
||||
)
|
||||
cursor = _sorted_tag_after(after, sort)
|
||||
operator = ">" if direction == "ASC" else "<"
|
||||
keyset, keys = "1=1", []
|
||||
if cursor is not None:
|
||||
keyset = f"({key}{operator}? OR ({key}=? AND id>?))"
|
||||
keys = [cursor[0], cursor[0], cursor[1]]
|
||||
if key == "prompt_count":
|
||||
sql = _RANKED_TAG_PAGE.format(
|
||||
search=where, after=keyset, direction=direction
|
||||
)
|
||||
return conn.execute(sql, [*params, *keys, limit + 1])
|
||||
page = (
|
||||
f"SELECT id,name FROM tags WHERE {where} AND {keyset} "
|
||||
f"ORDER BY {key} {direction},id LIMIT ?"
|
||||
)
|
||||
order = f"page.{key} {direction},page.id"
|
||||
return conn.execute(_tag_rows(page, order), [*params, *keys, limit + 1])
|
||||
|
||||
def _vocabulary_categories(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
@@ -422,15 +589,15 @@ class VocabularyOperations:
|
||||
_limit(limit)
|
||||
key = _category_after(after)
|
||||
total = conn.execute(
|
||||
_CATEGORY_GROUPS + "SELECT COUNT(*) FROM grouped"
|
||||
_CATEGORY_ROWS + "SELECT COUNT(*) FROM counted"
|
||||
).fetchone()[0]
|
||||
where, params = "", []
|
||||
if key is not None:
|
||||
where = " WHERE first_pid>? OR (first_pid=? AND value>?)"
|
||||
params = [key[0], key[0], key[1]]
|
||||
rows = conn.execute(
|
||||
_CATEGORY_GROUPS
|
||||
+ "SELECT value,first_pid,prompt_count FROM grouped"
|
||||
_CATEGORY_ROWS
|
||||
+ "SELECT value,first_pid,prompt_count,image_count FROM counted"
|
||||
+ where
|
||||
+ " ORDER BY first_pid,value LIMIT ?",
|
||||
[*params, limit + 1],
|
||||
@@ -440,7 +607,11 @@ class VocabularyOperations:
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"categories": [
|
||||
{"value": row["value"], "prompt_count": row["prompt_count"]}
|
||||
{
|
||||
"value": row["value"],
|
||||
"prompt_count": row["prompt_count"],
|
||||
"image_count": row["image_count"],
|
||||
}
|
||||
for row in rows
|
||||
],
|
||||
"total": total,
|
||||
@@ -450,9 +621,18 @@ class VocabularyOperations:
|
||||
),
|
||||
}
|
||||
|
||||
def list_vocabulary_tags(self, *, limit: int = 100, after: int | None = None):
|
||||
def list_vocabulary_tags(
|
||||
self,
|
||||
*,
|
||||
limit: int = 100,
|
||||
after: object = None,
|
||||
search: str = "",
|
||||
sort: str = "created",
|
||||
):
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
return self._vocabulary_tags(conn, epoch, limit=limit, after=after)
|
||||
return self._vocabulary_tags(
|
||||
conn, epoch, limit=limit, after=after, search=search, sort=sort
|
||||
)
|
||||
|
||||
def list_vocabulary_categories(
|
||||
self, *, limit: int = 100, after: list | None = None
|
||||
|
||||
@@ -0,0 +1,652 @@
|
||||
"""Each workflow and prompt graph stored once, compressed (#249 S6 carry-over).
|
||||
|
||||
``generated_images.workflow_data`` (the UI workflow) and ``prompt_metadata``
|
||||
(the API prompt graph) used to hold the same large JSON text again for every
|
||||
image of a run. Now each distinct text is one ``workflow_documents`` row:
|
||||
|
||||
- ``sha256``: the SHA-256 of the text's exact UTF-8 bytes (unique key);
|
||||
- ``byte_length``: the length of those bytes;
|
||||
- ``data``: ``zlib.compress`` of those bytes.
|
||||
|
||||
An image row references its texts through ``workflow_document_id`` and
|
||||
``prompt_document_id``. Two forms coexist until the migration has rewritten
|
||||
every row, and both stay readable forever:
|
||||
|
||||
- old form: the text inline in ``workflow_data`` / ``prompt_metadata``, no id;
|
||||
- new form: the inline column NULL and the document id set.
|
||||
|
||||
The text is stored and hashed exactly as written, never parsed and
|
||||
re-serialized, because a JSON round trip can change bytes (key order,
|
||||
spacing, number spelling, duplicate keys, ComfyUI's ``NaN`` tokens).
|
||||
|
||||
``read_stored_text`` is the one decoder. Inline text, when present, wins (a
|
||||
row an older app version rewrote inline stays right); otherwise the document
|
||||
is read by its rowid, decompressed within its recorded length and checked
|
||||
against its length and hash. A document that fails a check reads as
|
||||
unreadable (None, logged), never as other text. ``resolve_image_row`` puts
|
||||
the texts back into a ``generated_images`` row and drops the id keys, so
|
||||
every payload keeps its shape.
|
||||
|
||||
No trigger: every restore validator refuses triggers in a restored file. A
|
||||
document is removed in code: ``release_documents`` in the same transaction
|
||||
that drops a row's last reference, and ``sweep_documents`` (at startup, after
|
||||
the migration and before Reclaim space) for anything a path missed.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
import zlib
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Dict, Iterable, Optional, Set
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.workflow_documents")
|
||||
|
||||
TABLE = "workflow_documents"
|
||||
# (inline text column, document id column) on generated_images.
|
||||
TEXT_COLUMNS = (
|
||||
("workflow_data", "workflow_document_id"),
|
||||
("prompt_metadata", "prompt_document_id"),
|
||||
)
|
||||
DOCUMENT_ID_COLUMNS = tuple(reference for _column, reference in TEXT_COLUMNS)
|
||||
INDEXES = {
|
||||
"workflow_document_id": "idx_image_workflow_document",
|
||||
"prompt_document_id": "idx_image_prompt_document",
|
||||
}
|
||||
COMPRESSION_LEVEL = 6
|
||||
# A text larger than this stays inline: a document is decompressed in one
|
||||
# piece, and its recorded length bounds that (a restored file cannot make a
|
||||
# small row expand without limit).
|
||||
MAX_DOCUMENT_BYTES = 256 * 2**20
|
||||
ID_BATCH = 400 # ids bound per statement (under SQLite's old 999 limit)
|
||||
|
||||
_DDL = (
|
||||
f"CREATE TABLE {TABLE} (id INTEGER PRIMARY KEY, sha256 BLOB NOT NULL UNIQUE, "
|
||||
"byte_length INTEGER NOT NULL, data BLOB NOT NULL)"
|
||||
)
|
||||
_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
|
||||
_REFERENCE_TYPE = f"INTEGER REFERENCES {TABLE}(id)"
|
||||
|
||||
_UNREFERENCED = (
|
||||
"NOT EXISTS (SELECT 1 FROM generated_images g "
|
||||
"WHERE g.workflow_document_id=workflow_documents.id) "
|
||||
"AND NOT EXISTS (SELECT 1 FROM generated_images g "
|
||||
"WHERE g.prompt_document_id=workflow_documents.id)"
|
||||
)
|
||||
LOOKUP_SQL = (
|
||||
"SELECT id, sha256, byte_length, data FROM workflow_documents WHERE sha256=?"
|
||||
)
|
||||
REPAIR_SQL = "UPDATE workflow_documents SET byte_length=?, data=? WHERE id=?"
|
||||
ALL_DOCUMENTS_SQL = "SELECT sha256, byte_length, data, id FROM workflow_documents"
|
||||
READ_SQL = "SELECT sha256, byte_length, data FROM workflow_documents WHERE id=?"
|
||||
INSERT_SQL = (
|
||||
"INSERT INTO workflow_documents (sha256, byte_length, data) VALUES (?, ?, ?)"
|
||||
)
|
||||
# Built from constants only; the ids are bound as "?" marks.
|
||||
RELEASE_SQL = "DELETE FROM workflow_documents WHERE id IN ({marks}) AND "
|
||||
RELEASE_SQL += _UNREFERENCED
|
||||
# The sweep walks the ids through the small unique index, not the blobs.
|
||||
SWEEP_SQL = (
|
||||
"DELETE FROM workflow_documents WHERE id IN (SELECT id FROM workflow_documents "
|
||||
"INDEXED BY sqlite_autoindex_workflow_documents_1 WHERE "
|
||||
)
|
||||
SWEEP_SQL += _UNREFERENCED + ")"
|
||||
IMAGE_DOCUMENTS_SQL = (
|
||||
"SELECT workflow_document_id, prompt_document_id FROM generated_images WHERE "
|
||||
)
|
||||
|
||||
# Statements whose query plans tests/test_workflow_documents.py records.
|
||||
QUERY_PLAN_STATEMENTS = {
|
||||
"lookup by hash": (LOOKUP_SQL, (b"\0" * 32,)),
|
||||
"read by id": (READ_SQL, (1,)),
|
||||
"release": (RELEASE_SQL.format(marks="?,?"), (1, 2)),
|
||||
"sweep": (SWEEP_SQL, ()),
|
||||
"documents of images": (IMAGE_DOCUMENTS_SQL + "id IN (?,?)", (1, 2)),
|
||||
"documents of prompts": (IMAGE_DOCUMENTS_SQL + "prompt_id IN (?,?)", (1, 2)),
|
||||
}
|
||||
|
||||
|
||||
class DocumentMismatch(Exception):
|
||||
"""A document would not give back exactly the text it was made from."""
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
# ── schema ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _reference_columns(conn: sqlite3.Connection) -> Dict[str, tuple]:
|
||||
return {
|
||||
row[1]: tuple(row)
|
||||
for row in conn.execute("PRAGMA table_info(generated_images)")
|
||||
if row[1] in DOCUMENT_ID_COLUMNS
|
||||
}
|
||||
|
||||
|
||||
def _reference_error(conn: sqlite3.Connection, columns: Dict[str, tuple]):
|
||||
for name, info in columns.items():
|
||||
# (cid, name, type, notnull, default, pk)
|
||||
if (str(info[2]).upper(), info[3], info[4], info[5]) != ("INTEGER", 0, None, 0):
|
||||
return "Workflow document references have an invalid column"
|
||||
keys = {
|
||||
(row[3], row[2], row[4])
|
||||
for row in conn.execute("PRAGMA foreign_key_list(generated_images)")
|
||||
if row[3] in DOCUMENT_ID_COLUMNS
|
||||
}
|
||||
if keys != {(name, TABLE, "id") for name in DOCUMENT_ID_COLUMNS}:
|
||||
return "Workflow document references have invalid foreign keys"
|
||||
for column, index in INDEXES.items():
|
||||
info = conn.execute(f'PRAGMA index_info("{index}")').fetchall()
|
||||
owner = conn.execute(
|
||||
"SELECT tbl_name FROM sqlite_master WHERE type='index' AND name=?",
|
||||
(index,),
|
||||
).fetchone()
|
||||
if [row[2] for row in info] != [column] or not owner:
|
||||
return "Workflow document references are missing their index"
|
||||
if owner[0] != "generated_images":
|
||||
return "Workflow document references are missing their index"
|
||||
return None
|
||||
|
||||
|
||||
def _value_error(conn: sqlite3.Connection):
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM workflow_documents WHERE typeof(sha256)!='blob' "
|
||||
"OR length(sha256)!=32 OR typeof(byte_length)!='integer' OR byte_length<0 "
|
||||
"OR byte_length>? OR typeof(data)!='blob' LIMIT 1",
|
||||
(MAX_DOCUMENT_BYTES,),
|
||||
).fetchone():
|
||||
return "Workflow documents contain invalid values"
|
||||
for column in DOCUMENT_ID_COLUMNS:
|
||||
# typeof() reads no overflow page; a dangling id is a rowid probe.
|
||||
# ``column`` is one of DOCUMENT_ID_COLUMNS, never caller data.
|
||||
sql = (
|
||||
f"SELECT 1 FROM generated_images g WHERE typeof(g.{column}) " # nosec B608
|
||||
f"NOT IN ('null','integer') OR (g.{column} IS NOT NULL AND NOT EXISTS "
|
||||
f"(SELECT 1 FROM workflow_documents d WHERE d.id=g.{column})) LIMIT 1"
|
||||
)
|
||||
if conn.execute(sql).fetchone():
|
||||
return "Workflow document references point at missing documents"
|
||||
return None
|
||||
|
||||
|
||||
def damaged_message(document_id: int) -> str:
|
||||
"""The verify reason for a damaged document: which one, and what to do."""
|
||||
return (
|
||||
f"Workflow document {document_id} is damaged: its text no longer matches "
|
||||
"its checksum, and the images that use it show no workflow. To fix it, "
|
||||
"restore a backup from before the damage, or store a new image with the "
|
||||
"same workflow (that repairs it)."
|
||||
)
|
||||
|
||||
|
||||
def _damaged_document(conn: sqlite3.Connection) -> Optional[int]:
|
||||
"""The first document failing its length or hash check, or None."""
|
||||
for row in conn.execute(ALL_DOCUMENTS_SQL):
|
||||
if _decode(row[:3], row[3], quiet=True) is None:
|
||||
return row[3]
|
||||
return None
|
||||
|
||||
|
||||
def workflow_documents_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True, documents: bool = False
|
||||
) -> Optional[str]:
|
||||
"""Reject an unsupported document schema or invalid stored values (reads only).
|
||||
|
||||
A database from before the dedupe has neither the table nor the id
|
||||
columns; that is acceptable unless ``allow_missing`` is false. Half of
|
||||
the schema never is.
|
||||
"""
|
||||
objects = conn.execute(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name=? "
|
||||
"OR (type IN ('index','trigger') AND tbl_name=?)",
|
||||
(TABLE, TABLE),
|
||||
).fetchall()
|
||||
has_images = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='generated_images'"
|
||||
).fetchone()
|
||||
columns = _reference_columns(conn) if has_images else {}
|
||||
if not objects and not columns:
|
||||
if allow_missing or not has_images:
|
||||
return None
|
||||
return "Missing workflow document schema"
|
||||
if not objects or set(columns) != set(DOCUMENT_ID_COLUMNS):
|
||||
return "Incomplete workflow document schema"
|
||||
tables = [sql for kind, _name, sql in objects if kind == "table"]
|
||||
others = {(kind, name, sql) for kind, name, sql in objects if kind != "table"}
|
||||
if len(tables) != 1 or others != {("index", _AUTOINDEX, None)}:
|
||||
return "Workflow documents name is reserved by an unsupported schema object"
|
||||
# The exact definition fixes the columns, their types and constraints.
|
||||
if _compact(tables[0]) != _compact(_DDL):
|
||||
return "Workflow documents have an invalid table definition"
|
||||
error = _reference_error(conn, columns) or _value_error(conn)
|
||||
if error is None and documents:
|
||||
# Restore and backup verification only: every document is read.
|
||||
damaged = _damaged_document(conn)
|
||||
if damaged is not None:
|
||||
return damaged_message(damaged)
|
||||
return error
|
||||
|
||||
|
||||
def migrate_workflow_documents(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table, the id columns and their indexes atomically.
|
||||
|
||||
Runs after every older migration (the table rebuilds only run on
|
||||
databases without ``file_path``, before this). Validates before and
|
||||
after, then sweeps documents no row references.
|
||||
"""
|
||||
conn.execute("SAVEPOINT workflow_documents_schema")
|
||||
try:
|
||||
error = workflow_documents_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
present = _reference_columns(conn)
|
||||
for column in DOCUMENT_ID_COLUMNS:
|
||||
if column not in present:
|
||||
conn.execute(
|
||||
f"ALTER TABLE generated_images ADD COLUMN {column} "
|
||||
f"{_REFERENCE_TYPE}"
|
||||
)
|
||||
for column, index in INDEXES.items():
|
||||
conn.execute(
|
||||
f"CREATE INDEX IF NOT EXISTS {index} ON generated_images({column})"
|
||||
)
|
||||
error = workflow_documents_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
sweep_documents(conn)
|
||||
conn.execute("RELEASE workflow_documents_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO workflow_documents_schema")
|
||||
conn.execute("RELEASE workflow_documents_schema")
|
||||
raise
|
||||
|
||||
|
||||
# ── write ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def storable(text: Any) -> bool:
|
||||
"""Whether ``text`` can be a document (text within MAX_DOCUMENT_BYTES)."""
|
||||
if not isinstance(text, str):
|
||||
return False
|
||||
# UTF-8 takes at most 4 bytes a character: most texts need no encoding.
|
||||
if len(text) * 4 <= MAX_DOCUMENT_BYTES:
|
||||
return True
|
||||
return len(text.encode("utf-8")) <= MAX_DOCUMENT_BYTES
|
||||
|
||||
|
||||
def store_text(
|
||||
conn: sqlite3.Connection, text: Optional[str], verified: Optional[set] = None
|
||||
) -> Optional[int]:
|
||||
"""The id of the document holding exactly ``text``; None for None.
|
||||
|
||||
Reuses the document with the same hash, or inserts a new one whose
|
||||
compressed bytes were checked to decompress to ``text``'s bytes. A reused
|
||||
document is decompressed and checked first; a damaged one (a restored
|
||||
file's corruption under a valid hash) is repaired from ``text``, whose
|
||||
hash is its key, so every row that references it reads right again.
|
||||
``verified``, when given, holds the ids already checked (a migration
|
||||
batch), which are then not read again. Opens a write transaction first
|
||||
when none is open, so a concurrent ``release_documents`` cannot remove a
|
||||
reused document before the caller references it. Raises TypeError for a
|
||||
non-text value and ValueError for a text over MAX_DOCUMENT_BYTES.
|
||||
"""
|
||||
if text is None:
|
||||
return None
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("Only text is stored as a workflow document")
|
||||
raw = text.encode("utf-8")
|
||||
if len(raw) > MAX_DOCUMENT_BYTES:
|
||||
raise ValueError("Text is too large for a workflow document")
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
if not conn.in_transaction:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
row = conn.execute(LOOKUP_SQL, (digest,)).fetchone()
|
||||
if row is not None and verified is not None and row[0] in verified:
|
||||
return row[0]
|
||||
data = zlib.compress(raw, COMPRESSION_LEVEL)
|
||||
if zlib.decompress(data) != raw: # pragma: no cover - zlib is exact
|
||||
raise DocumentMismatch("Compression did not round-trip")
|
||||
if row is None:
|
||||
document_id = conn.execute(INSERT_SQL, (digest, len(raw), data)).lastrowid
|
||||
else:
|
||||
document_id = row[0]
|
||||
if _decode(row[1:], document_id, quiet=True) != text:
|
||||
logger.warning(f"Workflow document {document_id} was damaged; repaired")
|
||||
conn.execute(REPAIR_SQL, (len(raw), data, document_id))
|
||||
if verified is not None:
|
||||
verified.add(document_id)
|
||||
return document_id
|
||||
|
||||
|
||||
def stored_values(
|
||||
conn: sqlite3.Connection, texts: Dict[str, Optional[str]]
|
||||
) -> Dict[str, Any]:
|
||||
"""New-form column values for ``{text column: text}`` (writers).
|
||||
|
||||
Each text becomes ``{column: None, id column: document id}``; None
|
||||
stays ``{column: None, id column: None}``; a value that cannot be a
|
||||
document (not text, or too large) stays inline with no id.
|
||||
"""
|
||||
references = dict(TEXT_COLUMNS)
|
||||
values: Dict[str, Any] = {}
|
||||
for column, text in texts.items():
|
||||
reference = references[column]
|
||||
if text is None or not storable(text):
|
||||
values[column], values[reference] = text, None
|
||||
else:
|
||||
values[column], values[reference] = None, store_text(conn, text)
|
||||
return values
|
||||
|
||||
|
||||
# ── read ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _decode(row, document_id, quiet=False) -> Optional[str]:
|
||||
"""``(sha256, byte_length, data)`` as checked text, or None (logged)."""
|
||||
if row is None:
|
||||
logger.warning(f"Workflow document {document_id} is missing")
|
||||
return None
|
||||
digest, length, data = bytes(row[0]), row[1], row[2]
|
||||
try:
|
||||
if not isinstance(length, int) or not 0 <= length <= MAX_DOCUMENT_BYTES:
|
||||
raise ValueError("invalid length")
|
||||
inflater = zlib.decompressobj()
|
||||
raw = inflater.decompress(bytes(data), length + 1)
|
||||
if len(raw) != length or inflater.unconsumed_tail or not inflater.eof:
|
||||
raise ValueError("length differs")
|
||||
if hashlib.sha256(raw).digest() != digest:
|
||||
raise ValueError("hash differs")
|
||||
return raw.decode("utf-8")
|
||||
except (zlib.error, ValueError, TypeError) as error:
|
||||
if not quiet:
|
||||
logger.warning(f"Workflow document {document_id} is unreadable: {error}")
|
||||
return None
|
||||
|
||||
|
||||
def read_stored_text(
|
||||
conn: sqlite3.Connection, inline: Any, document_id: Optional[int]
|
||||
) -> Any:
|
||||
"""The stored text of one column: inline when present, else the document.
|
||||
|
||||
Returns None when there is neither, or when the document is missing or
|
||||
fails its checks (logged). An inline value is returned unchanged.
|
||||
"""
|
||||
if inline is not None:
|
||||
return inline
|
||||
if document_id is None:
|
||||
return None
|
||||
return _decode(conn.execute(READ_SQL, (document_id,)).fetchone(), document_id)
|
||||
|
||||
|
||||
class DocumentReader:
|
||||
"""``read_stored_text`` with a small memo for list reads on one connection."""
|
||||
|
||||
MEMO = 32
|
||||
|
||||
def __init__(self, conn: sqlite3.Connection):
|
||||
self.conn = conn
|
||||
self._memo: "OrderedDict[int, Optional[str]]" = OrderedDict()
|
||||
|
||||
def text(self, inline: Any, document_id: Optional[int]) -> Any:
|
||||
if inline is not None or document_id is None:
|
||||
return read_stored_text(self.conn, inline, document_id)
|
||||
if document_id in self._memo:
|
||||
self._memo.move_to_end(document_id)
|
||||
return self._memo[document_id]
|
||||
text = read_stored_text(self.conn, None, document_id)
|
||||
self._memo[document_id] = text
|
||||
if len(self._memo) > self.MEMO:
|
||||
self._memo.popitem(last=False)
|
||||
return text
|
||||
|
||||
|
||||
def resolve_image_row(reader: DocumentReader, row) -> Dict[str, Any]:
|
||||
"""A ``generated_images`` row as a dict with its texts read back.
|
||||
|
||||
The id columns are dropped, so a payload built from it has the shape it
|
||||
had before the dedupe. Columns the row did not select stay absent.
|
||||
"""
|
||||
data = dict(row)
|
||||
for column, reference in TEXT_COLUMNS:
|
||||
has_reference = reference in data
|
||||
document_id = data.pop(reference, None)
|
||||
if column in data or has_reference:
|
||||
data[column] = reader.text(data.get(column), document_id)
|
||||
return data
|
||||
|
||||
|
||||
# ── garbage ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _chunks(values):
|
||||
values = list(values)
|
||||
for start in range(0, len(values), ID_BATCH):
|
||||
yield values[start : start + ID_BATCH]
|
||||
|
||||
|
||||
def image_documents(conn: sqlite3.Connection, where: str, params) -> Set[int]:
|
||||
"""The document ids the image rows matching ``where`` reference.
|
||||
|
||||
``where`` is a fixed condition written in this package (``id IN (...)``,
|
||||
``prompt_id IN (...)``), never caller data; values go in ``params``.
|
||||
"""
|
||||
ids: Set[int] = set()
|
||||
sql = IMAGE_DOCUMENTS_SQL + where
|
||||
for row in conn.execute(sql, list(params)): # nosec B608
|
||||
ids.update(value for value in row if value is not None)
|
||||
return ids
|
||||
|
||||
|
||||
def documents_of_rows(
|
||||
conn: sqlite3.Connection, column: str, values: Iterable[Any]
|
||||
) -> Set[int]:
|
||||
"""Document ids of the image rows whose ``column`` is in ``values``.
|
||||
|
||||
``column`` is ``id`` or ``prompt_id`` (the rows a prompt delete cascades to).
|
||||
"""
|
||||
if column not in ("id", "prompt_id"):
|
||||
raise ValueError(f"Unsupported column {column!r}")
|
||||
ids: Set[int] = set()
|
||||
for chunk in _chunks(values):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
ids |= image_documents(conn, f"{column} IN ({marks})", chunk)
|
||||
return ids
|
||||
|
||||
|
||||
def release_documents(conn: sqlite3.Connection, ids: Iterable[Optional[int]]) -> int:
|
||||
"""Delete the documents in ``ids`` that no image row references now.
|
||||
|
||||
Runs in the caller's transaction, right after the delete or update that
|
||||
dropped the references. Returns how many documents were removed.
|
||||
"""
|
||||
removed = 0
|
||||
for chunk in _chunks(sorted({i for i in ids if i is not None})):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
# Only "?" marks are interpolated.
|
||||
sql = RELEASE_SQL.format(marks=marks)
|
||||
removed += conn.execute(sql, chunk).rowcount # nosec B608
|
||||
return removed
|
||||
|
||||
|
||||
def sweep_documents(conn: sqlite3.Connection) -> int:
|
||||
"""Delete every document no image row references; returns the count."""
|
||||
return conn.execute(SWEEP_SQL).rowcount
|
||||
|
||||
|
||||
# ── repair (a restore's stage) ────────────────────────────────────────
|
||||
|
||||
INLINE_TEXTS_SQL = (
|
||||
"SELECT workflow_data, prompt_metadata FROM generated_images "
|
||||
"WHERE workflow_data IS NOT NULL OR prompt_metadata IS NOT NULL"
|
||||
)
|
||||
REKEY_SQL = "UPDATE workflow_documents SET sha256=?, byte_length=? WHERE id=?"
|
||||
|
||||
|
||||
def _rows(conn: sqlite3.Connection, sql: str, params=()):
|
||||
"""Each row of ``sql``, read one at a time.
|
||||
|
||||
Iterating a managed connection's cursor prefetches ``ITER_CHUNK`` (128)
|
||||
rows, and a document or inline text can be megabytes.
|
||||
"""
|
||||
cursor = conn.execute(sql, params)
|
||||
while (row := cursor.fetchone()) is not None:
|
||||
yield row
|
||||
|
||||
|
||||
def _inflated(data) -> Optional[bytes]:
|
||||
"""The complete text bytes ``data`` inflates to, or None.
|
||||
|
||||
Complete means the zlib stream ends (its Adler-32 checked) within
|
||||
MAX_DOCUMENT_BYTES with nothing after it: the body is intact even when
|
||||
the row's recorded hash or length is not.
|
||||
"""
|
||||
try:
|
||||
inflater = zlib.decompressobj()
|
||||
raw = inflater.decompress(bytes(data), MAX_DOCUMENT_BYTES + 1)
|
||||
if not inflater.eof or inflater.unconsumed_tail or inflater.unused_data:
|
||||
return None
|
||||
if len(raw) > MAX_DOCUMENT_BYTES:
|
||||
return None
|
||||
raw.decode("utf-8")
|
||||
return raw
|
||||
except (zlib.error, TypeError, UnicodeDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _one_column_off(row, raw: bytes) -> bool:
|
||||
"""Whether only the recorded hash or only the recorded length is wrong.
|
||||
|
||||
The two columns check the body independently. When the body inflates
|
||||
completely and one of them still agrees with it, that column vouches for
|
||||
the body and the other one is the damaged value. When both disagree, the
|
||||
body itself is the suspect (another valid stream), so it is not trusted.
|
||||
"""
|
||||
hash_ok = hashlib.sha256(raw).digest() == bytes(row[0])
|
||||
length_ok = row[1] == len(raw)
|
||||
return hash_ok != length_ok
|
||||
|
||||
|
||||
def _inline_copies(conn: sqlite3.Connection, wanted: Dict[bytes, int]):
|
||||
"""``{document id: text}`` for each wanted hash an inline row still holds."""
|
||||
found: Dict[int, str] = {}
|
||||
for row in _rows(conn, INLINE_TEXTS_SQL):
|
||||
for text in row:
|
||||
if not isinstance(text, str) or not storable(text):
|
||||
continue
|
||||
document_id = wanted.get(hashlib.sha256(text.encode("utf-8")).digest())
|
||||
if document_id is not None and document_id not in found:
|
||||
found[document_id] = text
|
||||
if len(found) == len(wanted):
|
||||
break
|
||||
return found
|
||||
|
||||
|
||||
def _rekey(conn: sqlite3.Connection, document_id: int, raw: bytes) -> Optional[int]:
|
||||
"""Give an intact body its true key, or merge it into the document that has it.
|
||||
|
||||
Returns the holder's id when its own body was damaged and has just been
|
||||
rewritten from ``raw`` (it is repaired, never to be cleared), else None.
|
||||
"""
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
holder = conn.execute(LOOKUP_SQL, (digest,)).fetchone()
|
||||
if holder is None or holder[0] == document_id:
|
||||
conn.execute(REKEY_SQL, (digest, len(raw), document_id))
|
||||
return None
|
||||
for column in DOCUMENT_ID_COLUMNS:
|
||||
# ``column`` is one of DOCUMENT_ID_COLUMNS, never caller data.
|
||||
conn.execute(
|
||||
f"UPDATE generated_images SET {column}=? WHERE {column}=?", # nosec B608
|
||||
(holder[0], document_id),
|
||||
)
|
||||
conn.execute("DELETE FROM workflow_documents WHERE id=?", (document_id,))
|
||||
if _decode(holder[1:], holder[0], quiet=True) is None:
|
||||
conn.execute(
|
||||
REPAIR_SQL, (len(raw), zlib.compress(raw, COMPRESSION_LEVEL), holder[0])
|
||||
)
|
||||
return holder[0]
|
||||
return None
|
||||
|
||||
|
||||
def repair_damaged_documents(conn: sqlite3.Connection) -> Dict[str, int]:
|
||||
"""Repair or clear every document failing its checks; commit (writes).
|
||||
|
||||
For a restore's stage only, never the live library. In order:
|
||||
|
||||
1. A document whose body inflates completely and whose recorded hash or
|
||||
length (not both) disagrees is re-keyed from its text, or merged
|
||||
into the document that already holds that text (``_one_column_off``).
|
||||
2. A damaged body whose exact text an inline row still holds (the same
|
||||
SHA-256, its key) is stored again from that text.
|
||||
3. Any other damaged document is unreadable for good: its images'
|
||||
references are cleared (they already showed no workflow) and the
|
||||
document is removed, so the stage verifies.
|
||||
|
||||
Returns ``{repaired, cleared, images}``: documents repaired (1 and 2),
|
||||
documents cleared and the distinct image rows whose reference was
|
||||
cleared. Rows are read one at a time.
|
||||
"""
|
||||
damaged: Dict[bytes, int] = {}
|
||||
intact: Dict[int, bytes] = {}
|
||||
for row in _rows(conn, ALL_DOCUMENTS_SQL):
|
||||
if _decode(row[:3], row[3], quiet=True) is None:
|
||||
raw = _inflated(row[2])
|
||||
if raw is not None and _one_column_off(row, raw):
|
||||
intact[row[3]] = raw
|
||||
else:
|
||||
damaged[bytes(row[0])] = row[3]
|
||||
result = {"repaired": 0, "cleared": 0, "images": 0}
|
||||
if not damaged and not intact:
|
||||
return result
|
||||
with conn:
|
||||
healed = []
|
||||
for document_id, raw in intact.items():
|
||||
holder = _rekey(conn, document_id, raw)
|
||||
if holder is not None and holder in damaged.values():
|
||||
# Rewritten from the merged body: repaired, not to be cleared
|
||||
# (it now also holds the merged images' references).
|
||||
damaged = {k: v for k, v in damaged.items() if v != holder}
|
||||
healed.append(holder)
|
||||
copies = _inline_copies(conn, damaged) if damaged else {}
|
||||
for document_id, text in copies.items():
|
||||
raw = text.encode("utf-8")
|
||||
data = zlib.compress(raw, COMPRESSION_LEVEL)
|
||||
conn.execute(REPAIR_SQL, (len(raw), data, document_id))
|
||||
lost = sorted(set(damaged.values()) - set(copies))
|
||||
images: Set[int] = set()
|
||||
for chunk in _chunks(lost):
|
||||
marks = ",".join("?" * len(chunk))
|
||||
# ``marks`` is only "?" placeholders; the ids are bound.
|
||||
for (image_id,) in _rows(
|
||||
conn,
|
||||
"SELECT id FROM generated_images WHERE " # nosec B608
|
||||
f"workflow_document_id IN ({marks}) OR prompt_document_id IN ({marks})",
|
||||
chunk + chunk,
|
||||
):
|
||||
images.add(image_id)
|
||||
for column in DOCUMENT_ID_COLUMNS:
|
||||
conn.execute(
|
||||
f"UPDATE generated_images SET {column}=NULL " # nosec B608
|
||||
f"WHERE {column} IN ({marks})",
|
||||
chunk,
|
||||
)
|
||||
conn.execute(
|
||||
f"DELETE FROM workflow_documents WHERE id IN ({marks})", # nosec B608
|
||||
chunk,
|
||||
)
|
||||
for document_id in list(intact) + healed + list(copies):
|
||||
logger.warning(f"Workflow document {document_id} was damaged; repaired")
|
||||
if lost:
|
||||
logger.warning(
|
||||
f"{len(lost)} damaged workflow document(s) could not be repaired; "
|
||||
f"{len(images)} image(s) show no workflow"
|
||||
)
|
||||
result.update(
|
||||
repaired=len(intact) + len(healed) + len(copies),
|
||||
cleared=len(lost),
|
||||
images=len(images),
|
||||
)
|
||||
return result
|
||||
@@ -0,0 +1,455 @@
|
||||
"""The workflow storage migration and Reclaim space (#249 S6 carry-over).
|
||||
|
||||
``migrate_workflow_storage`` rewrites old-form image rows (workflow and
|
||||
prompt graph text inline) to the new form (a ``workflow_documents`` id,
|
||||
``database/workflow_documents.py``) in batches:
|
||||
|
||||
- each batch is one ``BEGIN IMMEDIATE`` transaction over at most
|
||||
``BATCH_ROWS`` rows or about ``BATCH_BYTES`` of text, found by a rowid
|
||||
range search (``NEXT_IDS_SQL``) and read one row at a time, so memory is
|
||||
bounded by one row plus the batch's distinct documents;
|
||||
- each text is stored (or its document reused) and read back through the
|
||||
one accessor before its row changes; a text that would not come back
|
||||
identical fails the batch (``DocumentMismatch``) and rolls it back whole;
|
||||
- a value that is not text (or is over ``MAX_DOCUMENT_BYTES``) stays inline
|
||||
and is counted as protected;
|
||||
- each row runs in its own savepoint: one that cannot be rewritten (text
|
||||
SQLite cannot decode, a text that would not come back) stays inline and
|
||||
is listed, and the rest of the batch goes on;
|
||||
- a batch ends after ``BATCH_SECONDS``, and the write lock is then left
|
||||
free for ``BATCH_PAUSE``, so other writers are never starved.
|
||||
|
||||
A crash or a stop between batches leaves committed batches in the new form
|
||||
and the rest in the old form; both read the same, and a rerun continues
|
||||
from the first old-form row. Only database rows change: no image or video
|
||||
file is opened.
|
||||
|
||||
``reclaim_space`` is the VACUUM afterwards: it sweeps unreferenced
|
||||
documents, rebuilds the file and checkpoints the WAL with TRUNCATE, so the
|
||||
freed pages go back to the disk.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
from contextlib import closing
|
||||
|
||||
from .workflow_documents import (
|
||||
TEXT_COLUMNS,
|
||||
DocumentMismatch,
|
||||
read_stored_text,
|
||||
release_documents,
|
||||
storable,
|
||||
store_text,
|
||||
sweep_documents,
|
||||
)
|
||||
|
||||
OPERATION = "workflow_storage"
|
||||
# The first apply's backup: ``<db>.pre-workflow-storage_<stamp>``, outside the
|
||||
# pruned ``<db>.backup_*`` pool. Only one is ever made; later applies take a
|
||||
# pooled safety backup.
|
||||
PRE_MIGRATION_LABEL = "pre-workflow-storage"
|
||||
BATCH_ROWS = 200
|
||||
BATCH_BYTES = 16 * 2**20 # characters of text rewritten per transaction
|
||||
# A batch holds the write lock: it ends after this long, and the loop then
|
||||
# leaves the lock free for BATCH_PAUSE, longer than SQLite's longest busy
|
||||
# poll (100 ms), so the image monitor and the run counting get in between.
|
||||
BATCH_SECONDS = 0.5
|
||||
BATCH_PAUSE = 0.15
|
||||
|
||||
|
||||
def _row_error(error) -> bool:
|
||||
"""Whether ``error`` is one row's own cause, which keeps only that row inline.
|
||||
|
||||
Text SQLite cannot decode for Python, and a text that would not come
|
||||
back identical. Anything else (a full disk, an I/O error, a malformed
|
||||
database, a bug) fails the whole batch. The decode error is matched by
|
||||
its message: ``sqlite_errorcode`` needs Python 3.11.
|
||||
"""
|
||||
if isinstance(error, (DocumentMismatch, UnicodeError)):
|
||||
return True
|
||||
return isinstance(error, sqlite3.OperationalError) and str(error).startswith(
|
||||
"Could not decode"
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.workflow_storage")
|
||||
_clock = time.monotonic
|
||||
|
||||
# typeof() reads a column's type from the record header: no overflow page.
|
||||
NEXT_IDS_SQL = (
|
||||
"SELECT id FROM generated_images WHERE id>? AND "
|
||||
"(typeof(workflow_data)!='null' OR typeof(prompt_metadata)!='null') "
|
||||
"ORDER BY id LIMIT ?"
|
||||
)
|
||||
ROW_SQL = (
|
||||
"SELECT workflow_data, prompt_metadata, workflow_document_id, "
|
||||
"prompt_document_id FROM generated_images WHERE id=?"
|
||||
)
|
||||
UPDATE_SQL = (
|
||||
"UPDATE generated_images SET {assignments} WHERE id=?" # columns are constants
|
||||
)
|
||||
# One pass over the record headers: rows with inline text, and inline
|
||||
# values that are not text (they stay inline).
|
||||
INLINE_COUNTS_SQL = (
|
||||
"SELECT "
|
||||
"SUM(typeof(workflow_data)='text' OR typeof(prompt_metadata)='text'), "
|
||||
"SUM(typeof(workflow_data) NOT IN ('null','text')) "
|
||||
"+ SUM(typeof(prompt_metadata) NOT IN ('null','text')) "
|
||||
"FROM generated_images"
|
||||
)
|
||||
# octet_length() (SQLite 3.43+) reads no overflow page either.
|
||||
INLINE_BYTES_SQL = (
|
||||
"SELECT SUM(octet_length(workflow_data)), SUM(octet_length(prompt_metadata)) "
|
||||
"FROM generated_images"
|
||||
)
|
||||
|
||||
QUERY_PLAN_STATEMENTS = {
|
||||
"next old-form rows": (NEXT_IDS_SQL, (0, BATCH_ROWS)),
|
||||
"one row": (ROW_SQL, (1,)),
|
||||
}
|
||||
|
||||
|
||||
def _rewrite_row(conn, image_id, row, verified, checked):
|
||||
"""Store the row's inline texts as documents; returns (changed, protected, size).
|
||||
|
||||
``verified`` maps document ids already read back in this batch to their
|
||||
text, and ``checked`` holds the reused ids ``store_text`` already
|
||||
checked, so a document shared by many rows is decompressed once.
|
||||
"""
|
||||
changes, replaced = {}, []
|
||||
protected = size = 0
|
||||
for index, (column, reference) in enumerate(TEXT_COLUMNS):
|
||||
value, current = row[index], row[index + len(TEXT_COLUMNS)]
|
||||
if value is None:
|
||||
continue
|
||||
if not storable(value):
|
||||
protected += 1
|
||||
continue
|
||||
document_id = store_text(conn, value, checked)
|
||||
if document_id not in verified:
|
||||
verified[document_id] = read_stored_text(conn, None, document_id)
|
||||
if verified[document_id] != value:
|
||||
raise DocumentMismatch(f"Image {image_id}: {column} would change")
|
||||
changes[column], changes[reference] = None, document_id
|
||||
if current is not None and current != document_id:
|
||||
replaced.append(current)
|
||||
size += len(value)
|
||||
if changes:
|
||||
assignments = ",".join(f"{key}=?" for key in changes)
|
||||
conn.execute(
|
||||
UPDATE_SQL.format(assignments=assignments),
|
||||
[*changes.values(), image_id],
|
||||
)
|
||||
release_documents(conn, replaced)
|
||||
return bool(changes), protected, size
|
||||
|
||||
|
||||
def migrate_batch(
|
||||
conn,
|
||||
after_id,
|
||||
max_rows=BATCH_ROWS,
|
||||
max_bytes=BATCH_BYTES,
|
||||
max_seconds=BATCH_SECONDS,
|
||||
):
|
||||
"""Rewrite the next batch after ``after_id`` in one transaction.
|
||||
|
||||
The batch ends after ``max_rows`` rows, about ``max_bytes`` of text or
|
||||
``max_seconds``, whichever comes first. Each row runs in its own
|
||||
savepoint: a row that cannot be rewritten (``_row_error``) is rolled
|
||||
back alone, stays inline and is listed in ``errors``; anything else rolls
|
||||
the whole batch back and propagates. Returns ``{last_id, rows,
|
||||
protected, errors, done}``: the last row looked at, rows rewritten,
|
||||
inline values kept (not text), the ids kept by an error and whether
|
||||
there was nothing left to look at.
|
||||
"""
|
||||
if conn.in_transaction:
|
||||
raise RuntimeError("The migration needs its own transaction")
|
||||
started = _clock()
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
ids = [row[0] for row in conn.execute(NEXT_IDS_SQL, (after_id, max_rows))]
|
||||
result = {
|
||||
"last_id": after_id,
|
||||
"rows": 0,
|
||||
"protected": 0,
|
||||
"errors": [],
|
||||
"done": not ids,
|
||||
}
|
||||
verified, checked, size = {}, set(), 0
|
||||
for image_id in ids:
|
||||
conn.execute("SAVEPOINT workflow_row")
|
||||
try:
|
||||
row = conn.execute(ROW_SQL, (image_id,)).fetchone()
|
||||
changed, protected, row_size = _rewrite_row(
|
||||
conn, image_id, row, verified, checked
|
||||
)
|
||||
except Exception as error:
|
||||
if not _row_error(error):
|
||||
raise
|
||||
conn.execute("ROLLBACK TO workflow_row")
|
||||
conn.execute("RELEASE workflow_row")
|
||||
verified.clear() # the row's documents were rolled back too
|
||||
checked.clear()
|
||||
logger.warning(f"Image record {image_id} kept inline: {error}")
|
||||
result["errors"].append(image_id)
|
||||
changed, protected, row_size = False, 0, 0
|
||||
else:
|
||||
conn.execute("RELEASE workflow_row")
|
||||
result["rows"] += int(changed)
|
||||
result["protected"] += protected
|
||||
result["last_id"] = image_id
|
||||
size += row_size
|
||||
if size >= max_bytes or _clock() - started >= max_seconds:
|
||||
break
|
||||
conn.commit()
|
||||
return result
|
||||
except BaseException:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def pre_migration_backups(db_path):
|
||||
"""The kept pre-migration backups beside ``db_path``, earliest first."""
|
||||
db_path = os.path.abspath(db_path)
|
||||
prefix = os.path.basename(db_path) + "." + PRE_MIGRATION_LABEL + "_"
|
||||
folder = os.path.dirname(db_path)
|
||||
try:
|
||||
names = os.listdir(folder)
|
||||
except OSError:
|
||||
return []
|
||||
return sorted(
|
||||
name
|
||||
for name in names
|
||||
if name.startswith(prefix) and os.path.isfile(os.path.join(folder, name))
|
||||
)
|
||||
|
||||
|
||||
def _file_bytes(db_path):
|
||||
return sum(
|
||||
os.path.getsize(path)
|
||||
for path in (db_path, db_path + "-wal")
|
||||
if os.path.exists(path)
|
||||
)
|
||||
|
||||
|
||||
class WorkflowStorageOperations:
|
||||
"""PromptDatabase mixin: the workflow storage migration and Reclaim space."""
|
||||
|
||||
def migrate_workflow_storage(
|
||||
self,
|
||||
*,
|
||||
on_batch=None,
|
||||
should_stop=None,
|
||||
max_rows=BATCH_ROWS,
|
||||
max_bytes=BATCH_BYTES,
|
||||
) -> dict:
|
||||
"""Rewrite every old-form row, batch by batch (blocking).
|
||||
|
||||
``on_batch(progress)`` runs after each committed batch with the
|
||||
running totals ``{rows, protected, batches, last_id}``;
|
||||
``should_stop()`` is asked before each batch. Each committed batch is
|
||||
followed by ``BATCH_PAUSE`` with the write lock free. Returns
|
||||
``{rows, protected, batches, stopped, errors, documents_removed}``
|
||||
(``errors``: the ids of rows kept inline by an error). Unreferenced
|
||||
documents are swept once every row was looked at.
|
||||
"""
|
||||
totals = {"rows": 0, "protected": 0, "batches": 0, "last_id": 0}
|
||||
errors = []
|
||||
stopped = False
|
||||
while True:
|
||||
if should_stop is not None and should_stop():
|
||||
stopped = True
|
||||
break
|
||||
batch = self.migrate_workflow_batch(totals["last_id"], max_rows, max_bytes)
|
||||
if batch["done"]:
|
||||
break
|
||||
time.sleep(BATCH_PAUSE) # leave the write lock to other writers
|
||||
errors += batch["errors"]
|
||||
totals["rows"] += batch["rows"]
|
||||
totals["protected"] += batch["protected"]
|
||||
totals["batches"] += 1
|
||||
totals["last_id"] = batch["last_id"]
|
||||
if on_batch is not None:
|
||||
on_batch(dict(totals))
|
||||
removed = 0 if stopped else self.sweep_workflow_documents()
|
||||
return {
|
||||
"rows": totals["rows"],
|
||||
"protected": totals["protected"],
|
||||
"batches": totals["batches"],
|
||||
"stopped": stopped,
|
||||
"errors": errors,
|
||||
"documents_removed": removed,
|
||||
}
|
||||
|
||||
def migrate_workflow_batch(
|
||||
self, after_id, max_rows=BATCH_ROWS, max_bytes=BATCH_BYTES
|
||||
) -> dict:
|
||||
"""One batch on the calling thread's connection (``migrate_batch``)."""
|
||||
return migrate_batch(self.model.get_connection(), after_id, max_rows, max_bytes)
|
||||
|
||||
def sweep_workflow_documents(self) -> int:
|
||||
"""Remove every document no image row references (one transaction)."""
|
||||
conn = self.model.get_connection()
|
||||
with conn:
|
||||
return sweep_documents(conn)
|
||||
|
||||
def workflow_storage_status(self) -> dict:
|
||||
"""``workflow_storage_counts`` on one read-only snapshot (blocking)."""
|
||||
from .maintenance import _read_snapshot
|
||||
|
||||
return _read_snapshot(self.model.db_path, self.workflow_storage_counts)
|
||||
|
||||
def workflow_storage_counts(self, conn) -> dict:
|
||||
"""``{rows, protected, documents, inline_bytes}`` on ``conn`` (reads only).
|
||||
|
||||
``inline_bytes`` is None where SQLite has no ``octet_length``.
|
||||
"""
|
||||
rows, protected = conn.execute(INLINE_COUNTS_SQL).fetchone()
|
||||
documents = conn.execute("SELECT COUNT(*) FROM workflow_documents").fetchone()
|
||||
inline_bytes = None
|
||||
if sqlite3.sqlite_version_info >= (3, 43, 0):
|
||||
row = conn.execute(INLINE_BYTES_SQL).fetchone()
|
||||
inline_bytes = sum(value or 0 for value in row)
|
||||
return {
|
||||
"rows": rows or 0,
|
||||
"protected": protected or 0,
|
||||
"documents": documents[0],
|
||||
"inline_bytes": inline_bytes,
|
||||
}
|
||||
|
||||
def reclaim_space(self) -> dict:
|
||||
"""Sweep, VACUUM and truncate the WAL (blocking); file bytes before/after.
|
||||
|
||||
The caller holds the registry's ``exclusive`` slot. Raises whatever
|
||||
stopped VACUUM; the database is then unchanged.
|
||||
"""
|
||||
db_path = self.model.db_path
|
||||
removed = self.sweep_workflow_documents()
|
||||
before = _file_bytes(db_path)
|
||||
self.model.vacuum_database()
|
||||
with closing(sqlite3.connect(db_path)) as checkpoint:
|
||||
busy = checkpoint.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()[0]
|
||||
return {
|
||||
"before_bytes": before,
|
||||
"after_bytes": _file_bytes(db_path),
|
||||
"documents_removed": removed,
|
||||
"checkpoint_busy": bool(busy),
|
||||
}
|
||||
|
||||
# ── preview ───────────────────────────────────────────────────────
|
||||
|
||||
def _plan_workflow_storage(self):
|
||||
"""The reviewed plan of the migration (one read-only snapshot)."""
|
||||
from .maintenance import (
|
||||
MaintenanceDraft,
|
||||
_Fingerprint,
|
||||
_megabytes,
|
||||
_plural,
|
||||
_read_snapshot,
|
||||
)
|
||||
|
||||
def read(conn):
|
||||
sizes = [
|
||||
conn.execute(f"PRAGMA {name}").fetchone()[0]
|
||||
for name in ("page_size", "page_count", "freelist_count")
|
||||
]
|
||||
return sizes, self.workflow_storage_counts(conn)
|
||||
|
||||
(page_size, page_count, free_pages), counts = _read_snapshot(
|
||||
self.model.db_path, read
|
||||
)
|
||||
size = page_size * page_count
|
||||
details = [
|
||||
{
|
||||
"label": "Database file",
|
||||
"detail": f"{_megabytes(size)}, of which "
|
||||
f"{_megabytes(free_pages * page_size)} is free pages",
|
||||
},
|
||||
{
|
||||
"label": "Image records",
|
||||
"detail": f"{_plural(counts['rows'], 'record')} keep their workflow "
|
||||
"or prompt graph inline; each distinct text is stored once, "
|
||||
"compressed",
|
||||
},
|
||||
]
|
||||
if not counts["rows"]:
|
||||
details[1]["detail"] = NOTHING_TO_DO
|
||||
if counts["inline_bytes"] is not None:
|
||||
details.append(
|
||||
{
|
||||
"label": "Inline text",
|
||||
"detail": f"{_megabytes(counts['inline_bytes'])} now",
|
||||
}
|
||||
)
|
||||
if counts["documents"]:
|
||||
details.append(
|
||||
{
|
||||
"label": "Already stored once",
|
||||
"detail": _plural(counts["documents"], "workflow document"),
|
||||
}
|
||||
)
|
||||
name = os.path.basename(self.model.db_path)
|
||||
kept = pre_migration_backups(self.model.db_path)
|
||||
backup = (
|
||||
POOLED_BACKUP_NOTE.format(
|
||||
name=f"{name}.backup_<date and time>", kept=kept[0]
|
||||
)
|
||||
if kept
|
||||
else BACKUP_NOTE.format(
|
||||
name=f"{name}.{PRE_MIGRATION_LABEL}_<date and time>"
|
||||
)
|
||||
)
|
||||
details += [
|
||||
{"label": "Backup", "detail": backup},
|
||||
{"label": "Reclaim space", "detail": RECLAIM_NOTE},
|
||||
{"label": "Older versions", "detail": OLDER_VERSIONS_NOTE},
|
||||
]
|
||||
protected = counts["protected"]
|
||||
return MaintenanceDraft(
|
||||
operation=OPERATION,
|
||||
fingerprint=_Fingerprint(OPERATION).hexdigest(),
|
||||
targets=[],
|
||||
scope={
|
||||
"count": counts["rows"],
|
||||
"label": (
|
||||
_plural(counts["rows"], "image record") + " to rewrite"
|
||||
if counts["rows"]
|
||||
else "Nothing to rewrite"
|
||||
),
|
||||
},
|
||||
protected={
|
||||
"count": protected,
|
||||
"reasons": (
|
||||
[f"{_plural(protected, 'stored value')} not text; kept as is"]
|
||||
if protected
|
||||
else []
|
||||
),
|
||||
},
|
||||
media={"deletes_files": False, "label": MEDIA_NOTE},
|
||||
details=details,
|
||||
)
|
||||
|
||||
|
||||
MEDIA_NOTE = "No image or video file is opened or changed; only the database."
|
||||
RECLAIM_NOTE = (
|
||||
"When every record is rewritten, Reclaim space runs alone to give the freed "
|
||||
"space back to the disk (VACUUM). The database is locked while it runs."
|
||||
)
|
||||
BACKUP_NOTE = (
|
||||
"Always taken first, whatever the safety backup setting: {name} beside "
|
||||
"the database, verified, and kept until you remove it (it is never pruned)."
|
||||
)
|
||||
POOLED_BACKUP_NOTE = (
|
||||
"A safety backup is taken first, whatever the safety backup setting: {name} "
|
||||
"among the pooled backups; the pre-migration backup {kept} is already kept."
|
||||
)
|
||||
NOTHING_TO_DO = "Every image record already stores its workflow once."
|
||||
OLDER_VERSIONS_NOTE = (
|
||||
"This version stores the workflows of images it records in the new form, "
|
||||
"whether or not this job runs, and this job rewrites the rest. An older "
|
||||
"Prompt Manager version shows those images without their workflow until you "
|
||||
"update again. The way back to an older version is the earliest "
|
||||
"pre-workflow-storage backup."
|
||||
)
|
||||
@@ -12,7 +12,8 @@ code. This plan proposes implementation order and acceptance gates.
|
||||
- The native Prompt Library merged in [#210](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/210) for [#208](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/208): one positive parent, saved negative variants, full selected text and a shared filtered image gallery. It does not invent chronological runs from counters or scan dates.
|
||||
- [Durable history storage](prompt-history-storage.md) records job attempts and outcomes from [#212](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/212), frozen prompt contexts from [#213](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/213), and observed run-owned output locators from [#214](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/214). Those locators do not verify file bytes or establish gallery-image associations. That association contract and the chronological execution timeline remain pending under [#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209).
|
||||
- [#215](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/215) brings the shell, Images and natural-height Prompt Library presentation into the approved geometry and themes while retaining their existing operations. [#216](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/216) adds the shared current-file inspector with exact generation facts, scoped provenance, raw/download evidence and loaded-collection viewer controls, including a slideshow that pauses at native video. This delivery does not establish whole-application parity, change the default entry point or constitute a release. Recorded chronological run/context browsing continues under [#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209); image tagging/review, System consolidation and remaining organization capabilities continue under #196.
|
||||
- Local Inspect (#249, PRs L1–L2) replaces the `#/inspect` handoff with the concept's native "Inspect a file" page: one local image or video previews inline and opens the shared viewer and full inspector on a record read entirely in the browser, with nothing uploaded, imported or stored. The legacy metadata page (`web/metadata.html`, served at `/prompt_manager/gallery.html`) stays reachable from Overview; its capabilities map to the shared inspector, except its one-click COPY ALL summary, whose ruled replacement ("Copy settings summary") is pending. Recorded deviations D60–D66 are in [TESTING.md](../TESTING.md). Local roles equal to the server's resolver follow in PR L3.
|
||||
- Local Inspect (#249, PRs L1–L3) replaces the `#/inspect` handoff with the concept's native "Inspect a file" page: one local image or video previews inline and opens the shared viewer and full inspector on a record read entirely in the browser, with nothing uploaded, imported or stored. The legacy metadata page (`web/metadata.html`, served at `/prompt_manager/gallery.html`) stays reachable from Overview; its capabilities map to the shared inspector, including its one-click COPY ALL summary, which is the shared inspector's "Copy settings summary" (D117, LI-4). PR L3 ([#286](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/286)) shipped that button and local roles equal to the server's resolver (`web/js/workspace/local-resolver.mjs`, TESTING.md "Local roles (#249 PR L3, decision LI-1)"). Recorded deviations D60–D66 and D117 are in [TESTING.md](../TESTING.md).
|
||||
- Collections ([#331](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/331), three PRs) adds named, hand-picked groups of images, stored in the library database by file location as favorites are: a COLLECTIONS sidebar group whose rows open Images filtered to one collection (`#/collection/<id>`) with rename and delete, Add to collection and Remove from collection in the Images selection bar, a Collections… checklist in the shared inspector for the side panel and the viewer, and saved views that keep a collection. Membership survives restarts, source reordering and System › Maintenance duplicate cleanup; no file is copied, moved or changed.
|
||||
|
||||
## Scope and compatibility
|
||||
|
||||
|
||||
@@ -18,9 +18,13 @@ protocol and peer checks reject unrelated messages. Same-origin scripts are
|
||||
trusted; this transport is not authentication against other scripts on that origin.
|
||||
Unsupported messaging is reported without breaking existing ComfyUI nodes.
|
||||
|
||||
Apply works in one click: without a preview, the sender requests the ticket itself
|
||||
and applies it. Preview remains optional, and shows the exact change first.
|
||||
|
||||
An apply timeout or abort after posting means the outcome is unknown. It cannot
|
||||
prove the host cancelled the write. The sender must inspect a fresh destination
|
||||
preview before deciding to send again; automatic retry must not overwrite edits.
|
||||
preview or choose the destination again before sending again; automatic retry must
|
||||
not overwrite edits. An error the host returned is a refusal: nothing was written.
|
||||
|
||||
## PromptManager adapter
|
||||
|
||||
@@ -31,7 +35,9 @@ field. Unknown/ambiguous metadata must not become an empty string.
|
||||
|
||||
Previews show original and requested text plus positive prepend/append composition.
|
||||
Connected composition is marked unknown rather than guessing upstream values.
|
||||
Connected destination fields reject writes. Applying rechecks graph, node, widgets,
|
||||
Connected destination fields reject writes. The sender compares the destination's revision
|
||||
(node object and graph version) from discovery with the one at the preview, and refuses if the workflow changed.
|
||||
Applying rechecks graph, node, widgets,
|
||||
values and connections. Ordinary serialization cache synchronization is accepted
|
||||
only when it preserves live values. Both requested roles are checked before writing.
|
||||
|
||||
|
||||
@@ -26,13 +26,13 @@ except ImportError:
|
||||
InputTypeDict = dict
|
||||
|
||||
try:
|
||||
from .prompt_manager_base import PromptManagerBase
|
||||
from .prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
|
||||
except ImportError:
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from prompt_manager_base import PromptManagerBase
|
||||
from prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
|
||||
|
||||
|
||||
class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
@@ -115,6 +115,7 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
)
|
||||
},
|
||||
),
|
||||
"save_to_library": SAVE_TO_LIBRARY_INPUT,
|
||||
},
|
||||
"hidden": {
|
||||
# The queued graph and this node's id: used to tell the positive
|
||||
@@ -152,6 +153,7 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
append_text: str = "",
|
||||
prompt: Optional[Dict[str, Any]] = None,
|
||||
unique_id: Optional[str] = None,
|
||||
save_to_library: bool = True,
|
||||
) -> Tuple[Any]:
|
||||
"""
|
||||
Encode the text prompt and save it to the database.
|
||||
@@ -164,6 +166,7 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
search_text: Text to search for in past prompts
|
||||
prepend_text: Text to prepend to the main prompt
|
||||
append_text: Text to append to the main prompt
|
||||
save_to_library: Off: nothing is saved, counted or tracked (#90)
|
||||
|
||||
Returns:
|
||||
Tuple containing the conditioning for the diffusion model and the final
|
||||
@@ -205,7 +208,9 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
# Save prompt to database and set execution context for gallery tracking
|
||||
prompt_id = None
|
||||
extended_tags = []
|
||||
if storage_text and storage_text.strip():
|
||||
if not save_to_library:
|
||||
self._skip_not_saved(prompt, unique_id)
|
||||
elif storage_text and storage_text.strip():
|
||||
self.logger.debug(f"Processing prompt text: {storage_text[:100]}...")
|
||||
|
||||
# Add prepend/append info to tags if they exist
|
||||
@@ -238,6 +243,9 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
except Exception as e:
|
||||
# Log error but don't fail the encoding
|
||||
self.logger.warning(f"Failed to save prompt to database: {e}")
|
||||
self._mark_untracked(prompt, unique_id)
|
||||
else:
|
||||
self._skip_empty_prompt(prompt, unique_id)
|
||||
|
||||
# Perform standard CLIP text encoding using the combined text
|
||||
self.logger.debug(
|
||||
@@ -246,6 +254,11 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
tokens = clip.tokenize(encoding_text)
|
||||
conditioning = clip.encode_from_tokens_scheduled(tokens)
|
||||
|
||||
self.logger.info(f"CLIP encoding completed, text: {repr(encoding_text)[:80]}")
|
||||
result = (conditioning, encoding_text)
|
||||
if not save_to_library:
|
||||
return result # #90: no metadata registration, no history text
|
||||
|
||||
# Register with ComfyUI integration for standard metadata compatibility
|
||||
node_id = f"promptmanager_{int(time.time() * 1000)}"
|
||||
self.comfyui_integration.register_prompt(
|
||||
@@ -259,12 +272,15 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
"append_text": append_text.strip() if append_text else None,
|
||||
},
|
||||
)
|
||||
|
||||
self.logger.info(f"CLIP encoding completed, text: {repr(encoding_text)[:80]}")
|
||||
result = (conditioning, encoding_text)
|
||||
record_runtime_context(
|
||||
type(self).__name__, text, prepend_text, append_text, encoding_text
|
||||
)
|
||||
self._record_cached_item(
|
||||
unique_id,
|
||||
(encoding_text,),
|
||||
storage_text,
|
||||
fields=((text, prepend_text, append_text),),
|
||||
)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -23,8 +23,16 @@ try:
|
||||
from .utils.image_monitor import get_image_monitor
|
||||
from .utils.prompt_tracker import get_prompt_tracker
|
||||
from .utils.hashing import generate_prompt_hash
|
||||
from .utils.prompt_graph import resolve_text, run_prompt_nodes
|
||||
from .utils.prompt_graph import resolve_text, run_prompt_nodes, save_setting
|
||||
from .utils.run_keys import executing_run_id, prompt_node_key
|
||||
from .utils.usage_tracking import PENDING_FIRST_USE
|
||||
from .utils.cached_runs import (
|
||||
CACHED_RUNS,
|
||||
Role,
|
||||
Seen,
|
||||
own_counts,
|
||||
stored_facts,
|
||||
)
|
||||
except ImportError:
|
||||
import sys
|
||||
|
||||
@@ -34,8 +42,32 @@ except ImportError:
|
||||
from utils.image_monitor import get_image_monitor
|
||||
from utils.prompt_tracker import get_prompt_tracker
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.prompt_graph import resolve_text, run_prompt_nodes
|
||||
from utils.prompt_graph import resolve_text, run_prompt_nodes, save_setting
|
||||
from utils.run_keys import executing_run_id, prompt_node_key
|
||||
from utils.usage_tracking import PENDING_FIRST_USE
|
||||
from utils.cached_runs import (
|
||||
CACHED_RUNS,
|
||||
Role,
|
||||
Seen,
|
||||
own_counts,
|
||||
stored_facts,
|
||||
)
|
||||
|
||||
|
||||
# The optional input every PromptManager node declares last (#90): declared
|
||||
# last so no widget value of a workflow saved before it shifts.
|
||||
SAVE_TO_LIBRARY_INPUT = (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"label_on": "save",
|
||||
"label_off": "skip",
|
||||
"tooltip": (
|
||||
"Save this node's prompt to the library, count its runs and link its "
|
||||
"images. Off: PromptManager ignores the run; the output is unchanged."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class PromptManagerBase:
|
||||
@@ -69,8 +101,10 @@ class PromptManagerBase:
|
||||
"""Save the prompt to the SQLite database.
|
||||
|
||||
Runs are normally counted by the queue hook (utils/usage_tracking.py), so a
|
||||
save only counts one when asked to (batch items whose text the hook could not
|
||||
see) or when the hook queued this prompt before it existed.
|
||||
save only counts one when asked to (text the hook could not see, unless
|
||||
this job already counted the node from ComfyUI's cache) or when the hook
|
||||
queued this prompt before it existed. The two are independent: the
|
||||
pending catch-up is written even when ``count_run`` is False.
|
||||
|
||||
Args:
|
||||
text: The prompt text
|
||||
@@ -138,7 +172,9 @@ class PromptManagerBase:
|
||||
resolved = resolve_text(prompt_graph, node_id)
|
||||
return {
|
||||
"is_run_prompt": node_id in run_prompt_nodes(prompt_graph),
|
||||
# The hook skips a switch it cannot read (#90): the node counts
|
||||
"seen_at_queue": resolved is not None
|
||||
and save_setting(prompt_graph, node_id) is True
|
||||
and generate_prompt_hash(resolved) == generate_prompt_hash(text),
|
||||
"known": True,
|
||||
}
|
||||
@@ -165,17 +201,137 @@ class PromptManagerBase:
|
||||
batch_item = (
|
||||
role["known"] and role["is_run_prompt"] and not role["seen_at_queue"]
|
||||
)
|
||||
prompt_id = self._save_prompt_to_database(
|
||||
text=text, category=category, tags=tags, count_run=batch_item
|
||||
counts = own_counts(
|
||||
type(self).__name__,
|
||||
stored_facts(text),
|
||||
Role(
|
||||
frozenset({"text"} if role["known"] and role["is_run_prompt"] else ())
|
||||
),
|
||||
Seen(text=role["seen_at_queue"]),
|
||||
)
|
||||
if prompt_id and role["is_run_prompt"]:
|
||||
prompt_id = self._save_prompt_to_database(
|
||||
text=text,
|
||||
category=category,
|
||||
tags=tags,
|
||||
count_run=bool(counts) and not self._replayed(unique_id),
|
||||
)
|
||||
if role["is_run_prompt"] and not prompt_id:
|
||||
# The save failed: an older entry of this key must not stand in (#342)
|
||||
self._mark_untracked(prompt_graph, unique_id)
|
||||
elif role["is_run_prompt"]:
|
||||
self.prompt_tracker.set_current_prompt(
|
||||
prompt_text=encoding_text,
|
||||
additional_data={**additional_data, "prompt_id": prompt_id},
|
||||
push_to_queue=batch_item or not role["known"],
|
||||
**self._run_identity(prompt_graph, unique_id),
|
||||
)
|
||||
return prompt_id
|
||||
|
||||
def _cached_runs(self) -> Any:
|
||||
return getattr(self, "cached_runs", CACHED_RUNS)
|
||||
|
||||
def _replayed(self, unique_id: Any) -> bool:
|
||||
"""This job already counted the node from ComfyUI's cache (#347).
|
||||
|
||||
A RAM-pressure cache can free the entry after ``execution_cached``, so
|
||||
the node executes in a job that replayed it: its own count is skipped.
|
||||
"""
|
||||
try:
|
||||
return unique_id is not None and self._cached_runs().replayed(unique_id)
|
||||
except Exception as error:
|
||||
self.logger.debug(f"Could not check the cached-run replay: {error}")
|
||||
return False
|
||||
|
||||
def _record_cached_item(
|
||||
self,
|
||||
unique_id: Any,
|
||||
outputs: tuple,
|
||||
text: Any,
|
||||
negative_text: Any = None,
|
||||
fields: Optional[tuple] = None,
|
||||
) -> None:
|
||||
"""Remember this execution's outputs and stored text for cached jobs (#347).
|
||||
|
||||
``fields`` are the ``(raw, prepend, append)`` texts this execution gave
|
||||
``record_runtime_context`` for each output, so a cached job's history
|
||||
can record them too (#403).
|
||||
"""
|
||||
if unique_id is None:
|
||||
return
|
||||
try:
|
||||
self._cached_runs().record_item(
|
||||
unique_id, outputs, stored_facts(text, negative_text), fields
|
||||
)
|
||||
except Exception as error:
|
||||
self.logger.debug(f"Could not record the cached-run receipt: {error}")
|
||||
|
||||
@staticmethod
|
||||
def _run_identity(prompt_graph: Any, unique_id: Any) -> Dict[str, Any]:
|
||||
"""The tracker's proof that an image belongs to this execution (#342)."""
|
||||
return {
|
||||
"run_key": prompt_node_key(prompt_graph, unique_id),
|
||||
"run_id": executing_run_id(),
|
||||
}
|
||||
|
||||
def _skip_not_saved(self, prompt_graph: Any = None, unique_id: Any = None) -> None:
|
||||
"""``save_to_library`` is off: nothing is saved, counted or tracked (#90).
|
||||
|
||||
A literal off makes the node no run prompt (``utils.prompt_graph``), so
|
||||
no image looks up its run key. A switch wired from a node the graph
|
||||
cannot read keeps the node a run prompt: its run key is marked like an
|
||||
empty editor's, so an earlier run's entry of that key never links this
|
||||
run's images.
|
||||
"""
|
||||
message = (
|
||||
"Save to library is off for this node: its prompt is not saved, the "
|
||||
"run is not counted and its images are not linked"
|
||||
)
|
||||
if isinstance(prompt_graph, dict) and unique_id is not None:
|
||||
self._mark_untracked(prompt_graph, unique_id, message)
|
||||
else:
|
||||
self.logger.info(message)
|
||||
|
||||
def _skip_empty_prompt(self, prompt_graph: Any, unique_id: Any) -> None:
|
||||
"""The ``text`` editor is empty: save and track nothing (#342).
|
||||
|
||||
Prepend and append text are deliberately never stored as the prompt, so
|
||||
an empty editor has no prompt to save, count or link. For the run's
|
||||
positive node this is logged at info and the node's run key is marked
|
||||
empty (``_mark_untracked``).
|
||||
"""
|
||||
self._mark_untracked(
|
||||
prompt_graph,
|
||||
unique_id,
|
||||
"Prompt text is empty (prepend and append text are not stored): "
|
||||
"nothing saved or tracked, and this run's images stay unlinked",
|
||||
)
|
||||
|
||||
def _mark_untracked(
|
||||
self, prompt_graph: Any, unique_id: Any, message: Optional[str] = None
|
||||
) -> None:
|
||||
"""This execution of a positive node has no prompt to link (#342).
|
||||
|
||||
An empty editor, or a save or tracking failure. The node's run key is
|
||||
marked with a placeholder, so no earlier prompt of that key can be
|
||||
linked to this run's images, and a batch item keeps its queue place.
|
||||
``message`` is logged at info for a positive node.
|
||||
"""
|
||||
try:
|
||||
role = self._prompt_role(prompt_graph, unique_id, "")
|
||||
if not role["is_run_prompt"]:
|
||||
if message:
|
||||
self.logger.debug("Empty text on a non-positive node; not saved")
|
||||
return
|
||||
if message:
|
||||
self.logger.info(message)
|
||||
self.prompt_tracker.mark_empty(
|
||||
push_to_queue=role["known"],
|
||||
**self._run_identity(prompt_graph, unique_id),
|
||||
)
|
||||
except Exception as error:
|
||||
# Tracking must never block generation
|
||||
self.logger.warning(f"Could not mark the untracked prompt: {error}")
|
||||
|
||||
def _record_use(self, prompt_id: int, times: int = 1) -> None:
|
||||
"""Count runs; a tracking failure must never block generation."""
|
||||
try:
|
||||
@@ -213,8 +369,10 @@ class PromptManagerBase:
|
||||
|
||||
cache = get_trigger_cache()
|
||||
|
||||
# Lazy-load cache on first use
|
||||
if not cache.is_loaded and IntegrationConfig.LORA_MANAGER_PATH:
|
||||
# Lazy-load cache on first use; a failed load waits before the next try
|
||||
# (#355: never a rescan of LoraManager's folders on every run).
|
||||
retry_due = getattr(cache, "retry_due", lambda: True)
|
||||
if not cache.is_loaded and IntegrationConfig.LORA_MANAGER_PATH and retry_due():
|
||||
cache.load(IntegrationConfig.LORA_MANAGER_PATH)
|
||||
|
||||
modified, injected = inject_trigger_words(text, cache)
|
||||
|
||||
@@ -12,9 +12,21 @@ else:
|
||||
|
||||
try:
|
||||
from .prompt_manager import IO, InputTypeDict, PromptManagerFocused
|
||||
from .utils.cached_runs import (
|
||||
PAIR,
|
||||
Role,
|
||||
Seen,
|
||||
own_counts,
|
||||
pair_components,
|
||||
stored_facts,
|
||||
)
|
||||
from .utils.hashing import generate_prompt_hash
|
||||
from .utils.pair_execution import record_pair_execution
|
||||
from .utils.prompt_graph import intrinsic_pair_nodes, run_prompt_components
|
||||
from .utils.prompt_graph import (
|
||||
intrinsic_pair_nodes,
|
||||
run_prompt_components,
|
||||
save_setting,
|
||||
)
|
||||
from .utils.prompt_pair_resolver import resolve_text_input
|
||||
from .utils.usage_tracking import (
|
||||
PENDING_FIRST_USE,
|
||||
@@ -23,9 +35,21 @@ try:
|
||||
)
|
||||
except ImportError:
|
||||
from prompt_manager import IO, InputTypeDict, PromptManagerFocused
|
||||
from utils.cached_runs import (
|
||||
PAIR,
|
||||
Role,
|
||||
Seen,
|
||||
own_counts,
|
||||
pair_components,
|
||||
stored_facts,
|
||||
)
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from utils.pair_execution import record_pair_execution
|
||||
from utils.prompt_graph import intrinsic_pair_nodes, run_prompt_components
|
||||
from utils.prompt_graph import (
|
||||
intrinsic_pair_nodes,
|
||||
run_prompt_components,
|
||||
save_setting,
|
||||
)
|
||||
from utils.prompt_pair_resolver import resolve_text_input
|
||||
from utils.usage_tracking import (
|
||||
PENDING_FIRST_USE,
|
||||
@@ -74,8 +98,13 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
prompt: Optional[Dict[str, Any]] = None,
|
||||
unique_id: Optional[str] = None,
|
||||
extra_pnginfo: Optional[Dict[str, Any]] = None,
|
||||
save_to_library: bool = True,
|
||||
) -> Tuple[Any, Any, str, str]:
|
||||
"""Encode both sides once; database and tracking failures never block CLIP."""
|
||||
"""Encode both sides once; database and tracking failures never block CLIP.
|
||||
|
||||
With ``save_to_library`` off nothing is saved, counted, tracked or
|
||||
recorded (#90); the four outputs are the same.
|
||||
"""
|
||||
if clip is None:
|
||||
raise RuntimeError("ERROR: clip input is invalid: None")
|
||||
if not isinstance(text, str) or not isinstance(negative_text, str):
|
||||
@@ -93,6 +122,10 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
negative_conditioning = clip.encode_from_tokens_scheduled(
|
||||
clip.tokenize(negative_text)
|
||||
)
|
||||
result = positive_conditioning, negative_conditioning, positive, negative_text
|
||||
if not save_to_library:
|
||||
self._skip_not_saved(prompt, unique_id)
|
||||
return result
|
||||
record_pair_execution(
|
||||
extra_pnginfo,
|
||||
unique_id,
|
||||
@@ -117,85 +150,29 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
)
|
||||
is_positive = "text" in used_fields
|
||||
pair_used = node_id in intrinsic_pair_nodes(prompt) if known else True
|
||||
try:
|
||||
pair_id = self.db.save_prompt_pair(
|
||||
text,
|
||||
negative_text,
|
||||
category=category.strip() or None,
|
||||
tags=self._parse_tags(tags),
|
||||
)
|
||||
pair = self.db.get_prompt_pair(pair_id)
|
||||
prompt_id = pair["positive_prompt_id"]
|
||||
resolved_positive = (
|
||||
resolve_text_input(prompt, node_id, "text") if known else None
|
||||
)
|
||||
resolved_negative = (
|
||||
resolve_text_input(prompt, node_id, "negative_text") if known else None
|
||||
)
|
||||
positive_seen = resolved_positive is not None and generate_prompt_hash(
|
||||
resolved_positive
|
||||
) == generate_prompt_hash(text)
|
||||
pair_seen = (
|
||||
positive_seen
|
||||
and resolved_negative is not None
|
||||
and generate_prompt_hash(resolved_negative)
|
||||
== generate_prompt_hash(negative_text)
|
||||
)
|
||||
positive_pending = getattr(self, "pending_first_use", PENDING_FIRST_USE)
|
||||
pair_pending = getattr(
|
||||
self, "pending_pair_first_use", PENDING_PAIR_FIRST_USE
|
||||
)
|
||||
counted = set()
|
||||
for field, component_text, resolved_text, component_id in (
|
||||
("text", text, resolved_positive, prompt_id),
|
||||
(
|
||||
"negative_text",
|
||||
if not text.strip():
|
||||
# Nothing to store: prepend/append are never the stored prompt (#342)
|
||||
self._skip_empty_prompt(prompt, unique_id)
|
||||
else:
|
||||
try:
|
||||
pair_id, prompt_id = self._save_and_track_pair(
|
||||
text,
|
||||
negative_text,
|
||||
resolved_negative,
|
||||
pair["negative_prompt_id"],
|
||||
),
|
||||
):
|
||||
component_hash = generate_prompt_hash(component_text)
|
||||
if (
|
||||
field not in used_fields
|
||||
or not component_id
|
||||
or component_hash in counted
|
||||
):
|
||||
continue
|
||||
counted.add(component_hash)
|
||||
seen_at_queue = (
|
||||
resolved_text is not None
|
||||
and generate_prompt_hash(resolved_text) == component_hash
|
||||
positive,
|
||||
category=category,
|
||||
tags=tags,
|
||||
prepend_text=prepend_text,
|
||||
append_text=append_text,
|
||||
prompt=prompt,
|
||||
unique_id=unique_id,
|
||||
used_fields=used_fields,
|
||||
is_positive=is_positive,
|
||||
pair_used=pair_used,
|
||||
)
|
||||
runs = positive_pending.consume(component_hash) + int(not seen_at_queue)
|
||||
if runs:
|
||||
self.db.record_prompt_use(component_id, times=runs)
|
||||
if pair_used:
|
||||
runs = pair_pending.consume(pair_run_key(text, negative_text)) + int(
|
||||
not pair_seen
|
||||
)
|
||||
if runs:
|
||||
self.db.record_prompt_pair_use(pair_id, times=runs)
|
||||
if is_positive:
|
||||
self.prompt_tracker.set_current_prompt(
|
||||
prompt_text=positive,
|
||||
additional_data={
|
||||
"prompt_id": prompt_id,
|
||||
"pair_id": pair_id if pair_used else None,
|
||||
"positive_text": positive,
|
||||
"negative_text": negative_text,
|
||||
"positive_storage_text": text.strip(),
|
||||
"negative_storage_text": negative_text.strip(),
|
||||
"prepend_text": prepend_text.strip(),
|
||||
"append_text": append_text.strip(),
|
||||
"final_text": positive,
|
||||
"category": category.strip() or None,
|
||||
"tags": self._parse_tags(tags) or [],
|
||||
},
|
||||
push_to_queue=not positive_seen or (pair_used and not pair_seen),
|
||||
)
|
||||
except Exception as error:
|
||||
self.logger.warning(f"Could not save or track paired prompt: {error}")
|
||||
except Exception as error:
|
||||
self.logger.warning(f"Could not save or track paired prompt: {error}")
|
||||
if is_positive:
|
||||
self._mark_untracked(prompt, unique_id)
|
||||
try:
|
||||
self.comfyui_integration.register_prompt(
|
||||
f"promptmanager_pair_{time.time_ns()}",
|
||||
@@ -209,7 +186,6 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
)
|
||||
except Exception as error:
|
||||
self.logger.warning(f"Could not register paired metadata: {error}")
|
||||
result = positive_conditioning, negative_conditioning, positive, negative_text
|
||||
record_runtime_context(
|
||||
type(self).__name__,
|
||||
text,
|
||||
@@ -221,8 +197,115 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
record_runtime_context(
|
||||
type(self).__name__, negative_text, "", "", negative_text, role="negative"
|
||||
)
|
||||
self._record_cached_item(
|
||||
unique_id,
|
||||
(positive, negative_text),
|
||||
text,
|
||||
negative_text,
|
||||
fields=((text, prepend_text, append_text), (negative_text, "", "")),
|
||||
)
|
||||
return result
|
||||
|
||||
def _save_and_track_pair(
|
||||
self,
|
||||
text: str,
|
||||
negative_text: str,
|
||||
positive: str,
|
||||
*,
|
||||
category: str,
|
||||
tags: str,
|
||||
prepend_text: str,
|
||||
append_text: str,
|
||||
prompt: Optional[Dict[str, Any]],
|
||||
unique_id: Optional[str],
|
||||
used_fields: set,
|
||||
is_positive: bool,
|
||||
pair_used: bool,
|
||||
) -> Tuple[Optional[int], Optional[int]]:
|
||||
"""Save the recipe, count its runs and register the positive for linking."""
|
||||
known = isinstance(prompt, dict) and unique_id is not None
|
||||
node_id = str(unique_id)
|
||||
pair_id = self.db.save_prompt_pair(
|
||||
text,
|
||||
negative_text,
|
||||
category=category.strip() or None,
|
||||
tags=self._parse_tags(tags),
|
||||
# The Library shows the prompt's own category and tags, not the pair's.
|
||||
fill_new_positive=True,
|
||||
)
|
||||
pair = self.db.get_prompt_pair(pair_id)
|
||||
prompt_id = pair["positive_prompt_id"]
|
||||
resolved_positive = (
|
||||
resolve_text_input(prompt, node_id, "text") if known else None
|
||||
)
|
||||
resolved_negative = (
|
||||
resolve_text_input(prompt, node_id, "negative_text") if known else None
|
||||
)
|
||||
if known and save_setting(prompt, node_id) is not True:
|
||||
# The hook skips a switch it cannot read (#90): the node counts
|
||||
resolved_positive = resolved_negative = None
|
||||
positive_seen = resolved_positive is not None and generate_prompt_hash(
|
||||
resolved_positive
|
||||
) == generate_prompt_hash(text)
|
||||
pair_seen = (
|
||||
positive_seen
|
||||
and resolved_negative is not None
|
||||
and generate_prompt_hash(resolved_negative)
|
||||
== generate_prompt_hash(negative_text)
|
||||
)
|
||||
negative_seen = resolved_negative is not None and generate_prompt_hash(
|
||||
resolved_negative
|
||||
) == generate_prompt_hash(negative_text)
|
||||
positive_pending = getattr(self, "pending_first_use", PENDING_FIRST_USE)
|
||||
pair_pending = getattr(self, "pending_pair_first_use", PENDING_PAIR_FIRST_USE)
|
||||
facts = stored_facts(text, negative_text)
|
||||
role = Role(frozenset(used_fields), pair_used)
|
||||
# Own counts are skipped when this job already counted the node from
|
||||
# ComfyUI's cache (#347); the pending first-use catch-up never is.
|
||||
counts = (
|
||||
[]
|
||||
if self._replayed(unique_id)
|
||||
else own_counts(
|
||||
PAIR, facts, role, Seen(positive_seen, negative_seen, pair_seen)
|
||||
)
|
||||
)
|
||||
component_ids = {"text": prompt_id, "negative_text": pair["negative_prompt_id"]}
|
||||
for field, component_hash in pair_components(facts, role):
|
||||
component_id = component_ids[field]
|
||||
if not component_id:
|
||||
continue
|
||||
runs = positive_pending.consume(component_hash) + int(
|
||||
("prompt", component_hash) in counts
|
||||
)
|
||||
if runs:
|
||||
self.db.record_prompt_use(component_id, times=runs)
|
||||
if pair_used:
|
||||
runs = pair_pending.consume(pair_run_key(text, negative_text)) + int(
|
||||
("pair", facts.positive_hash, facts.negative_hash) in counts
|
||||
)
|
||||
if runs:
|
||||
self.db.record_prompt_pair_use(pair_id, times=runs)
|
||||
if is_positive:
|
||||
self.prompt_tracker.set_current_prompt(
|
||||
prompt_text=positive,
|
||||
additional_data={
|
||||
"prompt_id": prompt_id,
|
||||
"pair_id": pair_id if pair_used else None,
|
||||
"positive_text": positive,
|
||||
"negative_text": negative_text,
|
||||
"positive_storage_text": text.strip(),
|
||||
"negative_storage_text": negative_text.strip(),
|
||||
"prepend_text": prepend_text.strip(),
|
||||
"append_text": append_text.strip(),
|
||||
"final_text": positive,
|
||||
"category": category.strip() or None,
|
||||
"tags": self._parse_tags(tags) or [],
|
||||
},
|
||||
push_to_queue=not positive_seen or (pair_used and not pair_seen),
|
||||
**self._run_identity(prompt, unique_id),
|
||||
)
|
||||
return pair_id, prompt_id
|
||||
|
||||
@classmethod
|
||||
def IS_CHANGED(
|
||||
cls,
|
||||
|
||||
@@ -26,13 +26,13 @@ except ImportError:
|
||||
InputTypeDict = dict
|
||||
|
||||
try:
|
||||
from .prompt_manager_base import PromptManagerBase
|
||||
from .prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
|
||||
except ImportError:
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from prompt_manager_base import PromptManagerBase
|
||||
from prompt_manager_base import SAVE_TO_LIBRARY_INPUT, PromptManagerBase
|
||||
|
||||
|
||||
class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
@@ -115,6 +115,7 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
),
|
||||
},
|
||||
),
|
||||
"save_to_library": SAVE_TO_LIBRARY_INPUT,
|
||||
},
|
||||
"hidden": {
|
||||
# The queued graph and this node's id: used to tell the positive
|
||||
@@ -149,6 +150,7 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
append_text: str = "",
|
||||
prompt: Optional[Dict[str, Any]] = None,
|
||||
unique_id: Optional[str] = None,
|
||||
save_to_library: bool = True,
|
||||
) -> Tuple[str]:
|
||||
"""
|
||||
Process the text prompt and save it to the database.
|
||||
@@ -160,6 +162,7 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
search_text: Text to search for in past prompts
|
||||
prepend_text: Text to prepend to the main prompt
|
||||
append_text: Text to append to the main prompt
|
||||
save_to_library: Off: nothing is saved, counted or tracked (#90)
|
||||
|
||||
Returns:
|
||||
Tuple containing the final processed text string
|
||||
@@ -184,7 +187,9 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
# Save prompt to database and set execution context for gallery tracking
|
||||
prompt_id = None
|
||||
extended_tags = []
|
||||
if storage_text and storage_text.strip():
|
||||
if not save_to_library:
|
||||
self._skip_not_saved(prompt, unique_id)
|
||||
elif storage_text and storage_text.strip():
|
||||
self.logger.debug(f"Processing prompt text: {storage_text[:100]}...")
|
||||
|
||||
# Add prepend/append info to tags if they exist
|
||||
@@ -216,6 +221,14 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
except Exception as e:
|
||||
# Log error but don't fail the processing
|
||||
self.logger.warning(f"Failed to save prompt to database: {e}")
|
||||
self._mark_untracked(prompt, unique_id)
|
||||
else:
|
||||
self._skip_empty_prompt(prompt, unique_id)
|
||||
|
||||
self.logger.debug(f"Text processing completed: {final_text[:100]}...")
|
||||
result = (final_text,)
|
||||
if not save_to_library:
|
||||
return result # #90: no metadata registration, no history text
|
||||
|
||||
# Register with ComfyUI integration for standard metadata compatibility
|
||||
node_id = f"promptmanagertext_{int(time.time() * 1000)}"
|
||||
@@ -230,12 +243,15 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
"append_text": append_text.strip() if append_text else None,
|
||||
},
|
||||
)
|
||||
|
||||
self.logger.debug(f"Text processing completed: {final_text[:100]}...")
|
||||
result = (final_text,)
|
||||
record_runtime_context(
|
||||
type(self).__name__, text, prepend_text, append_text, final_text
|
||||
)
|
||||
self._record_cached_item(
|
||||
unique_id,
|
||||
(final_text,),
|
||||
storage_text,
|
||||
fields=((text, prepend_text, append_text),),
|
||||
)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -5,16 +5,24 @@ Split into domain-specific mixins for maintainability:
|
||||
- ImageRoutesMixin: gallery, thumbnails, image serving and linking
|
||||
- AdminRoutesMixin: duplicates, stats, settings, diagnostics, maintenance, backup, scan
|
||||
- LoggingRoutesMixin: log management endpoints
|
||||
- LogCleanupMixin: the reviewed rotated-log cleanup (a maintenance plan)
|
||||
- DiagnosticsReportMixin: the shareable diagnostic report, without secrets or paths
|
||||
- AutotagRoutesMixin: auto-tagging model management and tagging
|
||||
- JobRoutesMixin: the session job registry behind Activity (py/api/jobs.py)
|
||||
- ImageTagJobsMixin: bulk Add tags / Auto-tag as image-tags jobs
|
||||
- ThumbnailJobsMixin: thumbnail plan, thumbnail job and the gallery stream on it
|
||||
- ThumbnailConvertMixin: Convert outdated thumbnails (#317), a thumbnail job mode
|
||||
- AutoThumbnailMixin: previews made on first view, on capture and after a scan
|
||||
- ReviewRoutesMixin: the Needs review queue, item detail and decisions
|
||||
- ReviewRecordingMixin: Needs review recording for the scan and the monitor
|
||||
- ReviewEventsMixin: the Needs review change stream an open page re-reads on
|
||||
- SystemRoutesMixin: System health check, Overview and usage statistics
|
||||
- LibraryLocationRoutesMixin: where the library lives, Move library, Keep it here
|
||||
- MaintenanceRoutesMixin: reviewed maintenance plans (preview, apply)
|
||||
- LibraryDeleteRoutesMixin: the Library's Delete… (archive paired) and Restore
|
||||
- WorkflowStorageRoutesMixin: store each workflow once, then Reclaim space
|
||||
- DuplicateFilesMixin: duplicate image files as find and removal jobs
|
||||
- LoraImportRoutesMixin: the reviewed, staged LoRA reference import job
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -26,38 +34,56 @@ import math
|
||||
import ntpath
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
from PIL import Image
|
||||
|
||||
from .prompts import PromptRoutesMixin
|
||||
from .revalidation import revalidated_response
|
||||
from .pairs import PairRoutesMixin
|
||||
from .library import LibraryRoutesMixin
|
||||
from .library_delete import LibraryDeleteRoutesMixin
|
||||
from .vocabulary import VocabularyRoutesMixin
|
||||
from .annotations import AnnotationRoutesMixin
|
||||
from .review import ReviewRoutesMixin
|
||||
from .review_recording import ReviewRecordingMixin
|
||||
from .review_events import ReviewEventsMixin
|
||||
from .saved_views import SavedViewRoutesMixin
|
||||
from .collections import CollectionRoutesMixin
|
||||
from .history import HistoryRoutesMixin
|
||||
from .images import ImageRoutesMixin
|
||||
from .gallery_listing import WALK_STOPPED, RootListing, WalkOutcome
|
||||
from .admin import AdminRoutesMixin
|
||||
from .settings_routes import SettingsRoutesMixin
|
||||
from .job_routes import JobRoutesMixin
|
||||
from .image_tag_jobs import ImageTagJobsMixin
|
||||
from .thumbnail_jobs import ThumbnailJobsMixin
|
||||
from .thumbnail_convert import ThumbnailConvertMixin
|
||||
from .thumbnail_auto import AutoThumbnailMixin
|
||||
from .system import SystemRoutesMixin
|
||||
from .library_location import LibraryLocationRoutesMixin
|
||||
from .maintenance import MaintenanceRoutesMixin
|
||||
from .backup_restore import BackupRestoreRoutesMixin
|
||||
from .workflow_storage import WorkflowStorageRoutesMixin
|
||||
from .duplicate_files import DuplicateFilesMixin
|
||||
from .maintenance_plans import PlanStore
|
||||
from .jobs import JobRegistry
|
||||
from .logging_routes import LoggingRoutesMixin
|
||||
from .log_cleanup import LogCleanupMixin
|
||||
from .diagnostics_report import DiagnosticsReportMixin
|
||||
from .autotag_routes import AutotagRoutesMixin
|
||||
from .lora_integration import LoraIntegrationMixin
|
||||
from .lora_import_routes import LoraImportRoutesMixin
|
||||
from .image_categories import ImageCategoryRoutesMixin
|
||||
from .image_delete import ImageDeleteMixin
|
||||
from .image_return import ImageReturnMixin
|
||||
from .source_rescope import SourceRescopeMixin
|
||||
|
||||
try:
|
||||
from ...database.operations import PromptDatabase
|
||||
from ...utils.file_prompt import parse_comfyui_prompt
|
||||
from ...utils.logging_config import get_logger
|
||||
from ...utils.video_metadata import is_video_path, read_video_metadata
|
||||
except ImportError:
|
||||
@@ -67,6 +93,7 @@ except ImportError:
|
||||
0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
)
|
||||
from database.operations import PromptDatabase
|
||||
from utils.file_prompt import parse_comfyui_prompt
|
||||
from utils.logging_config import get_logger
|
||||
from utils.video_metadata import is_video_path, read_video_metadata
|
||||
|
||||
@@ -76,6 +103,18 @@ def _get_project_root():
|
||||
return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
# How long an executor thread waits for the loop to drop the listings.
|
||||
_OFF_LOOP_DROP_WAIT = 30
|
||||
|
||||
|
||||
def _running_loop():
|
||||
"""The event loop running in this thread, or None."""
|
||||
try:
|
||||
return asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return None
|
||||
|
||||
|
||||
def _public_path(path) -> str:
|
||||
"""Render a server path for an API response without revealing the layout.
|
||||
|
||||
@@ -266,6 +305,10 @@ async def _gzip_middleware(request, handler):
|
||||
if len(compressed) >= len(body):
|
||||
return response
|
||||
|
||||
# A length the handler declared is the uncompressed one: drop it so the
|
||||
# compressed body's own length is sent (#329: a stale length made the
|
||||
# browser wait for bytes that never came).
|
||||
response.headers.pop("Content-Length", None)
|
||||
response.body = compressed
|
||||
response.headers["Content-Encoding"] = "gzip"
|
||||
response.headers["Vary"] = "Accept-Encoding"
|
||||
@@ -275,12 +318,14 @@ async def _gzip_middleware(request, handler):
|
||||
class PromptManagerAPI(
|
||||
PairRoutesMixin,
|
||||
LibraryRoutesMixin,
|
||||
LibraryDeleteRoutesMixin,
|
||||
VocabularyRoutesMixin,
|
||||
AnnotationRoutesMixin,
|
||||
ReviewRoutesMixin,
|
||||
ReviewRecordingMixin,
|
||||
ReviewEventsMixin,
|
||||
SavedViewRoutesMixin,
|
||||
CollectionRoutesMixin,
|
||||
HistoryRoutesMixin,
|
||||
PromptRoutesMixin,
|
||||
ImageRoutesMixin,
|
||||
@@ -289,12 +334,24 @@ class PromptManagerAPI(
|
||||
JobRoutesMixin,
|
||||
ImageTagJobsMixin,
|
||||
ThumbnailJobsMixin,
|
||||
ThumbnailConvertMixin,
|
||||
AutoThumbnailMixin,
|
||||
SystemRoutesMixin,
|
||||
LibraryLocationRoutesMixin,
|
||||
MaintenanceRoutesMixin,
|
||||
BackupRestoreRoutesMixin,
|
||||
WorkflowStorageRoutesMixin,
|
||||
DuplicateFilesMixin,
|
||||
LoggingRoutesMixin,
|
||||
LogCleanupMixin,
|
||||
DiagnosticsReportMixin,
|
||||
AutotagRoutesMixin,
|
||||
LoraIntegrationMixin,
|
||||
LoraImportRoutesMixin,
|
||||
ImageCategoryRoutesMixin,
|
||||
ImageDeleteMixin,
|
||||
ImageReturnMixin,
|
||||
SourceRescopeMixin,
|
||||
):
|
||||
"""REST API handler for PromptManager operations and web interface.
|
||||
|
||||
@@ -322,13 +379,28 @@ class PromptManagerAPI(
|
||||
# Private media captures of the annotation routes (py/api/annotations.py),
|
||||
# created on first use with their restore subscription.
|
||||
self._captures = None
|
||||
# Capture tokens Generate suggestions ran on: a save of one marks the
|
||||
# image auto-tagged (#379).
|
||||
self._suggested_tokens = set()
|
||||
self._captures_unsubscribe = None
|
||||
# Set by close_annotation_captures (shutdown): no store is recreated.
|
||||
self._captures_closed = False
|
||||
self._cached_output_dir = None # Lazy-cached by _find_comfyui_output_dir()
|
||||
self._html_cache = {} # Cached HTML file contents keyed by path
|
||||
self._gallery_cache = {} # dict: path_str -> (files, timestamp)
|
||||
self._gallery_cache_ttl = 30 # Cache TTL in seconds
|
||||
# One shared listing per output root, keyed by str(root), and the
|
||||
# background walks refreshing them (py/api/gallery_listing.py, #348).
|
||||
self._root_listings: dict[str, RootListing] = {}
|
||||
self._listing_tasks: set = set()
|
||||
# The loop the walks run on, set by the startup warm-up or the first
|
||||
# walk; the monitor's thread hops onto it (notify_output_settled).
|
||||
self._listing_loop = None
|
||||
# Set by cleanup_gallery_listings: no background walk starts after it.
|
||||
self._listings_closed = False
|
||||
# The workspace caches rebuilt behind a stored walk that changed the
|
||||
# listing, once the host's startup warm-up has run (#348).
|
||||
self._workspace_warm_enabled = False
|
||||
self._workspace_warm_task = None
|
||||
self._workspace_warm_again = False
|
||||
# Workspace listing values and sort orders derived from a cached scan
|
||||
# (py/api/images.py _workspace_listed/_workspace_orders).
|
||||
self._workspace_listed_cache = {}
|
||||
@@ -359,17 +431,30 @@ class PromptManagerAPI(
|
||||
# Reviewed maintenance plans under single-use tokens; a restart
|
||||
# forgets them (py/api/maintenance.py).
|
||||
self._maintenance_plans = PlanStore()
|
||||
# #480: the removal marks the workspace listing hides, read once now so
|
||||
# a listing page reads no table until a mark or the database changes.
|
||||
try:
|
||||
self._removal_marks()
|
||||
except Exception:
|
||||
self.logger.warning("Removal marks could not be read", exc_info=True)
|
||||
|
||||
# A safety backup interrupted by a crash leaves its temp copy behind.
|
||||
try:
|
||||
from .maintenance import sweep_safety_temps
|
||||
|
||||
from .backup_restore import restore_stage
|
||||
|
||||
swept = sweep_safety_temps(self.db.model.db_path)
|
||||
swept += restore_stage.sweep_restore_temps(self.db.model.db_path)
|
||||
if swept:
|
||||
self.logger.info(f"Removed {len(swept)} interrupted backup copies")
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not sweep interrupted backup copies: {e}")
|
||||
|
||||
# A crash between Move library's switch and its rename left the old
|
||||
# node-folder files under their own names (#249 S6b): rename them now.
|
||||
self.finish_interrupted_library_move()
|
||||
|
||||
# Run cleanup on initialization to remove any existing duplicates
|
||||
try:
|
||||
removed = self.db.cleanup_duplicates()
|
||||
@@ -441,23 +526,122 @@ class PromptManagerAPI(
|
||||
return await loop.run_in_executor(None, call)
|
||||
return await loop.run_in_executor(None, func, *args)
|
||||
|
||||
def invalidate_gallery_cache(self):
|
||||
"""Invalidate the cached gallery file listing.
|
||||
def _root_listing(self, root) -> RootListing:
|
||||
"""The shared listing record of one output root (created empty)."""
|
||||
return self._root_listings.setdefault(str(root), RootListing())
|
||||
|
||||
Called by the image monitor when new files are detected. The workspace
|
||||
keeps its per-file values and card facts, so the next request rebuilds
|
||||
only new or changed files (see ImageRoutesMixin._workspace_listed and
|
||||
_card_facts); its sort orders
|
||||
are dropped, as the rescan replaces them anyway, and so is the
|
||||
favorites lookup, which holds the old scan list.
|
||||
def invalidate_gallery_cache(self, retire=False):
|
||||
"""Drop every root's listing, so the next read waits for a new walk.
|
||||
|
||||
Called after files are deleted through the app (duplicate cleanup and
|
||||
the legacy duplicate delete), never for thumbnail writes. A walk
|
||||
running now is not stored (its generation moved on). The workspace
|
||||
keeps its per-file values and card facts, so the next request
|
||||
rebuilds only new or changed files (see ImageRoutesMixin.
|
||||
_workspace_listed and _card_facts); its sort orders are dropped, as
|
||||
the walk replaces them anyway, and so is the favorites lookup, which
|
||||
holds the old list.
|
||||
|
||||
``retire`` (the roots changed): no background walk of a root starts
|
||||
again until a reader or the warm-up asks for it.
|
||||
"""
|
||||
self._gallery_cache = {}
|
||||
loop = self._listing_loop
|
||||
if loop is not None and loop.is_running() and _running_loop() is not loop:
|
||||
# Called from an executor thread (a settings save, a test seed):
|
||||
# listing records are touched only on the loop, and the caller
|
||||
# expects the drop done when this returns.
|
||||
dropped = threading.Event()
|
||||
|
||||
def drop():
|
||||
try:
|
||||
self._drop_listings(retire)
|
||||
finally:
|
||||
dropped.set()
|
||||
|
||||
loop.call_soon_threadsafe(drop)
|
||||
if not dropped.wait(_OFF_LOOP_DROP_WAIT):
|
||||
self.logger.warning("The Images listing drop is still queued")
|
||||
return
|
||||
self._drop_listings(retire)
|
||||
|
||||
def _drop_listings(self, retire=False):
|
||||
for record in list(self._root_listings.values()):
|
||||
record.drop()
|
||||
if retire:
|
||||
# The roots changed: a removed root must not come back through
|
||||
# a pending timer, a trailing walk or a monitor mark.
|
||||
record.retired = True
|
||||
record.dirty = False
|
||||
handle, record.rewalk_handle = record.rewalk_handle, None
|
||||
if handle is not None:
|
||||
handle.cancel()
|
||||
self._workspace_order_cache = {}
|
||||
self._favorite_listing_cache = {}
|
||||
|
||||
async def warm_gallery_listings(self, app=None):
|
||||
"""Start one background walk per output root (aiohttp on_startup).
|
||||
|
||||
Registered by the root ``__init__.py``, so the first Images open after
|
||||
ComfyUI starts finds the listing built. It never fails startup: a
|
||||
root that cannot be walked is retried by the first read.
|
||||
"""
|
||||
# Each stored walk that changes a listing then rebuilds the workspace's
|
||||
# per-file values, orders and favorites behind it (_warm_workspace).
|
||||
self._workspace_warm_enabled = True
|
||||
self._listing_loop = asyncio.get_running_loop()
|
||||
self._listings_closed = False
|
||||
try:
|
||||
for root in self._get_all_output_dirs():
|
||||
self._root_listing(root).retired = False
|
||||
self._schedule_walk(root)
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not warm the Images listing: {e}")
|
||||
|
||||
def notify_output_settled(self, path):
|
||||
"""The image monitor settled an output at *path* (any thread).
|
||||
|
||||
Called on the monitor's worker thread with the resolved path. It only
|
||||
hands the path to the event loop (``output_settled``) and never
|
||||
touches a listing from this thread; before the loop is known, or once
|
||||
the listings are cleaned up, it does nothing.
|
||||
"""
|
||||
loop = self._listing_loop
|
||||
if loop is None or self._listings_closed or not loop.is_running():
|
||||
return
|
||||
try:
|
||||
loop.call_soon_threadsafe(self.output_settled, path)
|
||||
except RuntimeError:
|
||||
pass # the loop closed meanwhile (shutdown)
|
||||
|
||||
async def cleanup_gallery_listings(self, app=None):
|
||||
"""Stop the background listing walks (aiohttp on_cleanup)."""
|
||||
self._workspace_warm_enabled = False
|
||||
# No mark, deferred walk or trailing walk starts a walk from now on.
|
||||
self._listings_closed = True
|
||||
# Before any await, so no delayed refresh can start a walk meanwhile.
|
||||
for record in self._root_listings.values():
|
||||
handle, record.rewalk_handle = record.rewalk_handle, None
|
||||
if handle is not None:
|
||||
handle.cancel()
|
||||
warm, self._workspace_warm_task = self._workspace_warm_task, None
|
||||
if warm is not None:
|
||||
warm.cancel()
|
||||
await asyncio.gather(warm, return_exceptions=True)
|
||||
tasks = list(self._listing_tasks)
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
self._listing_tasks.clear()
|
||||
# A walk cancelled before its first step never ran its own cleanup.
|
||||
for record in self._root_listings.values():
|
||||
flight, record.flight = record.flight, None
|
||||
if flight is not None and not flight.done():
|
||||
flight.set_result(WalkOutcome.failed(RuntimeError(WALK_STOPPED)))
|
||||
|
||||
def reset_gallery_roots_cache(self):
|
||||
"""Drop every cached listing and derived workspace value (roots changed)."""
|
||||
self.invalidate_gallery_cache()
|
||||
self.invalidate_gallery_cache(retire=True)
|
||||
self._workspace_listed_cache = {}
|
||||
self._workspace_card_cache = {}
|
||||
|
||||
@@ -483,45 +667,51 @@ class PromptManagerAPI(
|
||||
|
||||
@routes.get("/prompt_manager/workspace")
|
||||
async def serve_workspace_ui(request: web.Request) -> web.Response:
|
||||
return self._serve_html_page("workspace.html", "Workspace")
|
||||
return self._serve_html_page(request, "workspace.html", "Workspace")
|
||||
|
||||
@routes.get("/prompt_manager/web")
|
||||
async def serve_web_ui(request):
|
||||
return self._serve_html_page("index.html", "Web UI", cache=False)
|
||||
return self._serve_html_page(request, "index.html", "Web UI", cache=False)
|
||||
|
||||
@routes.get("/prompt_manager/gallery.html")
|
||||
async def serve_gallery_ui(request):
|
||||
return self._serve_html_page("metadata.html", "Gallery")
|
||||
return self._serve_html_page(request, "metadata.html", "Gallery")
|
||||
|
||||
@routes.get("/prompt_manager/admin")
|
||||
async def serve_admin_ui(request):
|
||||
return self._serve_html_page("admin.html", "Admin UI")
|
||||
return self._serve_html_page(request, "admin.html", "Admin UI")
|
||||
|
||||
@routes.get("/prompt_manager/gallery")
|
||||
async def serve_gallery_admin_ui(request):
|
||||
return self._serve_html_page("gallery.html", "Gallery")
|
||||
return self._serve_html_page(request, "gallery.html", "Gallery")
|
||||
|
||||
# ── Static file serving ───────────────────────────────────────
|
||||
|
||||
@routes.get("/prompt_manager/lib/{filepath:.*}")
|
||||
async def serve_lib_static(request):
|
||||
"""Serve static library files (JS, CSS) from web/lib directory."""
|
||||
return self._serve_static_file("lib", request.match_info.get("filepath"))
|
||||
return self._serve_static_file(
|
||||
request, "lib", request.match_info.get("filepath")
|
||||
)
|
||||
|
||||
@routes.get("/prompt_manager/js/{filepath:.*}")
|
||||
async def serve_js_static(request):
|
||||
"""Serve static JavaScript files from web/js directory."""
|
||||
return self._serve_static_file("js", request.match_info.get("filepath"))
|
||||
return self._serve_static_file(
|
||||
request, "js", request.match_info.get("filepath")
|
||||
)
|
||||
|
||||
# ── Register domain-specific routes from mixins ───────────────
|
||||
|
||||
self._register_pair_routes(routes)
|
||||
self._register_library_routes(routes)
|
||||
self._register_library_delete_routes(routes)
|
||||
self._register_vocabulary_routes(routes)
|
||||
self._register_annotation_routes(routes)
|
||||
self._register_review_routes(routes)
|
||||
self._register_review_event_routes(routes)
|
||||
self._register_saved_view_routes(routes)
|
||||
self._register_collection_routes(routes)
|
||||
self._register_history_routes(routes)
|
||||
self._register_prompt_routes(routes)
|
||||
self._register_image_routes(routes)
|
||||
@@ -529,11 +719,19 @@ class PromptManagerAPI(
|
||||
self._register_settings_routes(routes)
|
||||
self._register_job_routes(routes)
|
||||
self._register_system_routes(routes)
|
||||
self._register_diagnostics_report_routes(routes)
|
||||
self._register_library_location_routes(routes)
|
||||
self._register_maintenance_routes(routes)
|
||||
self._register_backup_restore_routes(routes)
|
||||
self._register_duplicate_routes(routes)
|
||||
self._register_logging_routes(routes)
|
||||
self._register_autotag_routes(routes)
|
||||
self._register_lora_routes(routes)
|
||||
self._register_lora_import_routes(routes)
|
||||
self._register_image_category_routes(routes)
|
||||
self._register_image_delete_routes(routes)
|
||||
self._register_image_return_routes(routes)
|
||||
self._register_rescope_routes(routes)
|
||||
|
||||
# Register gzip compression middleware (once)
|
||||
global _gzip_registered
|
||||
@@ -549,8 +747,11 @@ class PromptManagerAPI(
|
||||
|
||||
self.logger.info("All routes registered with decorator pattern")
|
||||
|
||||
def _serve_static_file(self, subdir, filepath):
|
||||
"""Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing."""
|
||||
def _serve_static_file(self, request, subdir, filepath):
|
||||
"""Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing.
|
||||
|
||||
A served file is revalidated on every reuse (#478, ``revalidation``).
|
||||
"""
|
||||
root = os.path.join(_get_project_root(), "web", subdir)
|
||||
exact = STATIC_EXACT_FILES.get((subdir, filepath))
|
||||
# An exact file passes the same containment check, its extension only.
|
||||
@@ -565,10 +766,13 @@ class PromptManagerAPI(
|
||||
return web.Response(text="Not Found", status=404)
|
||||
with open(file_path, "rb") as f:
|
||||
content = f.read()
|
||||
return web.Response(body=content, content_type=content_type)
|
||||
return revalidated_response(request, content, content_type)
|
||||
|
||||
def _serve_html_page(self, filename, title, cache=True):
|
||||
"""Serve ``web/<filename>``; 404 when missing, 500 when unreadable."""
|
||||
def _serve_html_page(self, request, filename, title, cache=True):
|
||||
"""Serve ``web/<filename>``; 404 when missing, 500 when unreadable.
|
||||
|
||||
A served page is revalidated on every reuse (#478, ``revalidation``).
|
||||
"""
|
||||
html_path = os.path.join(_get_project_root(), "web", filename)
|
||||
try:
|
||||
if cache and html_path in self._html_cache:
|
||||
@@ -585,7 +789,9 @@ class PromptManagerAPI(
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
return web.Response(text=content, content_type="text/html", charset="utf-8")
|
||||
return revalidated_response(
|
||||
request, content.encode("utf-8"), "text/html", charset="utf-8"
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception(f"Failed to load {title}")
|
||||
return web.Response(
|
||||
@@ -598,7 +804,12 @@ class PromptManagerAPI(
|
||||
# ── Shared utilities used by multiple mixins ──────────────────────
|
||||
|
||||
def _enrich_images(self, images):
|
||||
"""Add url and thumbnail_url to a flat list of image dicts."""
|
||||
"""Add url and thumbnail_url to a flat list of image dicts.
|
||||
|
||||
Each dict without ``is_video`` also gets ``is_video`` and
|
||||
``media_type`` from its extension (#355: a linked LoraManager video
|
||||
outside the output folders is shown as a video, not a picture).
|
||||
"""
|
||||
from urllib.parse import quote as url_quote
|
||||
|
||||
output_dirs = self._get_all_output_dirs()
|
||||
@@ -608,6 +819,11 @@ class PromptManagerAPI(
|
||||
if not image_path_str:
|
||||
continue
|
||||
|
||||
if "is_video" not in image:
|
||||
video = is_video_path(image_path_str)
|
||||
image["is_video"] = video
|
||||
image["media_type"] = "video" if video else "image"
|
||||
|
||||
img_path = Path(image_path_str)
|
||||
|
||||
# Set fallback url via image ID
|
||||
@@ -624,16 +840,20 @@ class PromptManagerAPI(
|
||||
f"{url_quote(rel_path.as_posix(), safe='/')}"
|
||||
)
|
||||
|
||||
# Check for thumbnail
|
||||
rel_no_ext = rel_path.with_suffix("")
|
||||
thumb_rel = (
|
||||
f"thumbnails/{rel_no_ext.as_posix()}_thumb{rel_path.suffix}"
|
||||
)
|
||||
# Any preview the file owns, the current format first (#317)
|
||||
from . import thumbnail_settings
|
||||
from .prompts import thumbnail_url_for
|
||||
|
||||
thumbnail_url = thumbnail_url_for(
|
||||
thumb_rel, output_path / thumb_rel
|
||||
existing = thumbnail_settings.existing_thumbnail(
|
||||
output_path, rel_path
|
||||
)
|
||||
thumbnail_url = None
|
||||
if existing is not None:
|
||||
thumb_rel = (
|
||||
thumbnail_settings.thumbnail_stem_rel(rel_path)
|
||||
+ existing.suffix
|
||||
)
|
||||
thumbnail_url = thumbnail_url_for(thumb_rel, existing)
|
||||
if thumbnail_url:
|
||||
image["thumbnail_url"] = thumbnail_url
|
||||
break # Found matching root, stop searching
|
||||
@@ -845,278 +1065,4 @@ class PromptManagerAPI(
|
||||
|
||||
def _parse_comfyui_prompt(self, metadata):
|
||||
"""Parse ComfyUI and A1111 prompt data from metadata."""
|
||||
result = {
|
||||
"prompt": None,
|
||||
"workflow": None,
|
||||
"parameters": {},
|
||||
"positive_prompt": None,
|
||||
"negative_prompt": None,
|
||||
}
|
||||
|
||||
# Check for A1111/Forge style parameters first.
|
||||
# The chunk is one block: a possibly multi-line positive prompt, then a
|
||||
# "Negative prompt:" section, then a trailing "Steps: ..." settings line.
|
||||
# Taking lines[0] as the positive prompt truncates any multi-line prompt
|
||||
# to one line, so split on the settings line instead.
|
||||
if "parameters" in metadata:
|
||||
import re as _re
|
||||
|
||||
params = metadata["parameters"]
|
||||
body = params
|
||||
settings = ""
|
||||
_sm = _re.search(r"\nSteps:\s", body)
|
||||
if _sm:
|
||||
settings = body[_sm.start() + 1 :].strip()
|
||||
body = body[: _sm.start()]
|
||||
negative = None
|
||||
_nm = _re.search(r"(?im)^\s*negative prompt:\s*", body)
|
||||
if _nm:
|
||||
negative = body[_nm.end() :].strip()
|
||||
body = body[: _nm.start()]
|
||||
elif body.lower().lstrip().startswith("negative prompt:"):
|
||||
negative = body.lower().lstrip()[len("negative prompt:") :].strip()
|
||||
body = ""
|
||||
result["positive_prompt"] = body.strip() or None
|
||||
if negative:
|
||||
result["negative_prompt"] = negative
|
||||
# Store raw parameters too
|
||||
result["parameters"]["parameters"] = params
|
||||
if settings:
|
||||
for _f in ("Steps", "Sampler", "Schedule type", "CFG scale", "Seed"):
|
||||
_fm = _re.search(
|
||||
r"(?i)(?:^|,\s*)" + _re.escape(_f) + r":\s*([^,]+)", settings
|
||||
)
|
||||
if _fm:
|
||||
result["parameters"][_f.lower().replace(" ", "_")] = _fm.group(
|
||||
1
|
||||
).strip()
|
||||
|
||||
# If no A1111 format found, proceed with ComfyUI parsing
|
||||
if result["positive_prompt"] is None:
|
||||
# Check for direct prompt field
|
||||
if "prompt" in metadata:
|
||||
try:
|
||||
prompt_data = json.loads(metadata["prompt"])
|
||||
result["prompt"] = prompt_data
|
||||
except json.JSONDecodeError:
|
||||
result["prompt"] = metadata["prompt"]
|
||||
|
||||
# Check for workflow
|
||||
if "workflow" in metadata:
|
||||
try:
|
||||
workflow_data = json.loads(metadata["workflow"])
|
||||
result["workflow"] = workflow_data
|
||||
except json.JSONDecodeError:
|
||||
result["workflow"] = metadata["workflow"]
|
||||
|
||||
# Check for other common ComfyUI fields
|
||||
common_fields = [
|
||||
"positive",
|
||||
"negative",
|
||||
"steps",
|
||||
"cfg",
|
||||
"sampler",
|
||||
"scheduler",
|
||||
"seed",
|
||||
]
|
||||
for field in common_fields:
|
||||
if field in metadata:
|
||||
try:
|
||||
result["parameters"][field] = json.loads(metadata[field])
|
||||
except json.JSONDecodeError:
|
||||
result["parameters"][field] = metadata[field]
|
||||
|
||||
return result
|
||||
|
||||
def _extract_readable_prompt(self, parsed_data):
|
||||
"""Extract human-readable prompt text from ComfyUI/A1111 data."""
|
||||
|
||||
def safe_to_string(value):
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
elif isinstance(value, list):
|
||||
return " ".join(str(item) for item in value if item)
|
||||
elif value is not None:
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
# First check if we already extracted a positive prompt (A1111 format)
|
||||
if parsed_data.get("positive_prompt"):
|
||||
return parsed_data["positive_prompt"]
|
||||
|
||||
# Check if prompt is already a string
|
||||
if isinstance(parsed_data.get("prompt"), str):
|
||||
return parsed_data["prompt"]
|
||||
|
||||
# Check if prompt is a simple value that can be converted
|
||||
if parsed_data.get("prompt") and not isinstance(
|
||||
parsed_data.get("prompt"), dict
|
||||
):
|
||||
return safe_to_string(parsed_data["prompt"])
|
||||
|
||||
prompt_data = parsed_data.get("prompt")
|
||||
if isinstance(prompt_data, dict):
|
||||
# Use the enhanced logic from parse-metadata.py
|
||||
positive_prompt = self._extract_positive_prompt_from_comfyui_data(
|
||||
prompt_data
|
||||
)
|
||||
if positive_prompt:
|
||||
return positive_prompt
|
||||
|
||||
# Check workflow data if available
|
||||
workflow_data = parsed_data.get("workflow")
|
||||
if isinstance(workflow_data, dict):
|
||||
positive_prompt = self._extract_positive_prompt_from_comfyui_data(
|
||||
workflow_data
|
||||
)
|
||||
if positive_prompt:
|
||||
return positive_prompt
|
||||
|
||||
# Check parameters for positive prompt
|
||||
if parsed_data.get("parameters", {}).get("positive"):
|
||||
return safe_to_string(parsed_data["parameters"]["positive"])
|
||||
|
||||
return None
|
||||
|
||||
def _get_node_inputs(self, node):
|
||||
"""Safely get inputs from a node, handling both dict and list formats.
|
||||
|
||||
Old format: inputs is a dict with direct key-value pairs
|
||||
inputs = {"text": "my prompt", "seed": 123}
|
||||
|
||||
New format: inputs is a list of connection objects
|
||||
inputs = [
|
||||
{"name": "text", "type": "STRING", "link": null,
|
||||
"widget": {"name": "text"}},
|
||||
{"name": "clip", "type": "CLIP", "link": 11}
|
||||
]
|
||||
"""
|
||||
if not isinstance(node, dict):
|
||||
return {}
|
||||
|
||||
inputs = node.get("inputs", {})
|
||||
|
||||
# If inputs is already a dict, return it
|
||||
if isinstance(inputs, dict):
|
||||
return inputs
|
||||
|
||||
# If inputs is a list, convert to dict format
|
||||
if isinstance(inputs, list):
|
||||
inputs_dict = {}
|
||||
for input_item in inputs:
|
||||
if isinstance(input_item, dict) and "name" in input_item:
|
||||
name = input_item["name"]
|
||||
inputs_dict[name] = input_item
|
||||
return inputs_dict
|
||||
|
||||
return {}
|
||||
|
||||
def _find_text_in_node(self, node):
|
||||
"""Try to find text content in a node using various strategies.
|
||||
|
||||
Handles both old and new workflow formats.
|
||||
"""
|
||||
if not isinstance(node, dict):
|
||||
return None
|
||||
|
||||
# Strategy 1: Check normalized inputs for 'text' field
|
||||
inputs = self._get_node_inputs(node)
|
||||
if "text" in inputs and isinstance(inputs["text"], str):
|
||||
return inputs["text"]
|
||||
|
||||
# Strategy 2: For text encoder nodes, check widgets_values
|
||||
class_type = node.get("class_type", node.get("type", ""))
|
||||
text_encoder_types = [
|
||||
"CLIPTextEncode",
|
||||
"CLIPTextEncodeSDXL",
|
||||
"CLIPTextEncodeSDXLRefiner",
|
||||
"CLIPTextEncodeFlux",
|
||||
"PromptManager",
|
||||
"PromptManagerText",
|
||||
"BNK_CLIPTextEncoder",
|
||||
"Text Encoder",
|
||||
"CLIP Text Encode",
|
||||
]
|
||||
|
||||
if any(
|
||||
encoder_type.lower() in class_type.lower()
|
||||
for encoder_type in text_encoder_types
|
||||
):
|
||||
widgets_values = node.get("widgets_values", [])
|
||||
if widgets_values and len(widgets_values) > 0:
|
||||
if isinstance(widgets_values[0], str) and widgets_values[0].strip():
|
||||
return widgets_values[0]
|
||||
|
||||
return None
|
||||
|
||||
def _extract_positive_prompt_from_comfyui_data(self, data):
|
||||
"""Extract positive prompt from ComfyUI data.
|
||||
|
||||
Handles both old and new formats.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
|
||||
# Build nodes dictionary
|
||||
nodes_by_id = {}
|
||||
if "nodes" in data:
|
||||
# Handle nodes array format
|
||||
for node in data["nodes"]:
|
||||
if isinstance(node, dict):
|
||||
nid = node.get("id")
|
||||
if nid is not None:
|
||||
nodes_by_id[nid] = node
|
||||
else:
|
||||
# Handle flat dictionary format (node_id -> node_data)
|
||||
for nid_str, node in data.items():
|
||||
try:
|
||||
nid = int(nid_str)
|
||||
except (ValueError, TypeError):
|
||||
nid = nid_str
|
||||
if isinstance(node, dict):
|
||||
if "id" in node:
|
||||
nid = node["id"]
|
||||
nodes_by_id[nid] = node
|
||||
|
||||
if not nodes_by_id:
|
||||
return None
|
||||
|
||||
# First, try to find positive/negative connection pattern
|
||||
pos_id = None
|
||||
for node in nodes_by_id.values():
|
||||
if isinstance(node, dict):
|
||||
inputs = self._get_node_inputs(node)
|
||||
if "positive" in inputs and "negative" in inputs:
|
||||
try:
|
||||
pos_input = inputs["positive"]
|
||||
if isinstance(pos_input, list) and len(pos_input) > 0:
|
||||
pos_id = int(pos_input[0])
|
||||
break
|
||||
except (ValueError, TypeError, IndexError):
|
||||
continue
|
||||
|
||||
# Get text from the positive node
|
||||
if pos_id is not None and pos_id in nodes_by_id:
|
||||
text_val = self._find_text_in_node(nodes_by_id[pos_id])
|
||||
if text_val:
|
||||
return text_val
|
||||
|
||||
# Fallback: find any text encoder node with text content
|
||||
text_nodes = []
|
||||
for node in nodes_by_id.values():
|
||||
if isinstance(node, dict):
|
||||
text_val = self._find_text_in_node(node)
|
||||
if text_val:
|
||||
# Try to determine if this is positive or negative
|
||||
node_title = node.get("title", "").lower()
|
||||
if "neg" not in node_title and "negative" not in node_title:
|
||||
# Prioritize non-negative prompts
|
||||
text_nodes.insert(0, text_val)
|
||||
else:
|
||||
text_nodes.append(text_val)
|
||||
|
||||
# Return the first positive-looking prompt
|
||||
if text_nodes:
|
||||
return text_nodes[0]
|
||||
|
||||
return None
|
||||
return parse_comfyui_prompt(metadata)
|
||||
|
||||
@@ -10,12 +10,15 @@ filesystem or the database.
|
||||
|
||||
import math
|
||||
|
||||
WD14_MODELS = frozenset({"wd14-swinv2", "wd14-vit"})
|
||||
LLM_MODELS = frozenset({"gguf", "hf"})
|
||||
MODEL_TYPES = WD14_MODELS | LLM_MODELS
|
||||
|
||||
MAX_INSTRUCTION_CHARS = 4000
|
||||
MAX_INSTRUCTION_BYTES = 16 * 1024
|
||||
from ..tagging_defaults import ( # noqa: F401 - re-exported for the route
|
||||
LLM_MODEL_TYPES as LLM_MODELS,
|
||||
MAX_INSTRUCTION_BYTES,
|
||||
MAX_INSTRUCTION_CHARS,
|
||||
MODEL_TYPES,
|
||||
WD14_MODEL_TYPES as WD14_MODELS,
|
||||
validate_template,
|
||||
validate_threshold,
|
||||
)
|
||||
|
||||
SUGGEST_FIELDS = frozenset(
|
||||
{"target", "capture_token", "expected_epoch", "model_type", "options"}
|
||||
@@ -24,7 +27,7 @@ _COMMON = frozenset({"suggestion_limit", "use_gpu"})
|
||||
_WD14_REQUIRED = frozenset({"general_threshold", "character_threshold"})
|
||||
_LLM_REQUIRED = frozenset({"instructions"})
|
||||
|
||||
UNAVAILABLE_HINT = "Download it from the legacy Auto-tag page."
|
||||
UNAVAILABLE_HINT = "Download it in System › Auto-tagging."
|
||||
|
||||
|
||||
class OptionError(ValueError):
|
||||
@@ -32,27 +35,17 @@ class OptionError(ValueError):
|
||||
|
||||
|
||||
def _threshold(name: str, value) -> float:
|
||||
if type(value) not in (int, float) or not math.isfinite(value):
|
||||
raise OptionError(f"{name} must be a finite number")
|
||||
if not 0 <= value <= 1:
|
||||
raise OptionError(f"{name} must be between 0 and 1")
|
||||
return float(value)
|
||||
try:
|
||||
return validate_threshold(name, value)
|
||||
except ValueError as error:
|
||||
raise OptionError(str(error)) from None
|
||||
|
||||
|
||||
def _instructions(value) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise OptionError("instructions must be non-empty text")
|
||||
if len(value) > MAX_INSTRUCTION_CHARS:
|
||||
raise OptionError(
|
||||
f"instructions must be at most {MAX_INSTRUCTION_CHARS} characters"
|
||||
)
|
||||
try:
|
||||
encoded = value.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
raise OptionError("instructions must be valid Unicode text") from None
|
||||
if len(encoded) > MAX_INSTRUCTION_BYTES:
|
||||
raise OptionError("instructions are too long")
|
||||
return value
|
||||
return validate_template(value)
|
||||
except ValueError as error:
|
||||
raise OptionError(str(error)) from None
|
||||
|
||||
|
||||
def validate_options(model_type, options, *, default_limit: int, max_limit: int):
|
||||
@@ -93,6 +86,23 @@ def unavailable_message(model_name: str) -> str:
|
||||
return f"{model_name} is not downloaded. {UNAVAILABLE_HINT}"
|
||||
|
||||
|
||||
def model_refusal(status: dict, model_type: str, model_name: str):
|
||||
"""Why *model_type* cannot run, from ``get_models_status()``, else None.
|
||||
|
||||
Not downloaded first (``unavailable_message``), then a backend package
|
||||
that cannot be imported (#424). Both are a 409 ``model-unavailable``.
|
||||
"""
|
||||
entry = status.get(model_type) or {}
|
||||
if not entry.get("downloaded"):
|
||||
return unavailable_message(model_name)
|
||||
missing = entry.get("missing_packages") or []
|
||||
if missing:
|
||||
from ..autotag import missing_packages_message
|
||||
|
||||
return missing_packages_message(model_type, [str(name) for name in missing])
|
||||
return None
|
||||
|
||||
|
||||
def run_suggestions(service, capture_path: str, model_type: str, options: dict):
|
||||
"""Blocking: load (or swap to) *model_type*, then suggest for the capture.
|
||||
|
||||
|
||||
@@ -21,13 +21,18 @@ from aiohttp import web
|
||||
from .annotation_suggest import (
|
||||
SUGGEST_FIELDS,
|
||||
OptionError,
|
||||
model_refusal,
|
||||
run_suggestions,
|
||||
suggestion_rows,
|
||||
unavailable_message,
|
||||
validate_options,
|
||||
)
|
||||
from .autotag_routes import model_busy_response
|
||||
from .images import _forbidden, _has_traversal, _unavailable_link, _verified_title
|
||||
|
||||
# The target check is shared with the collection routes (#331); both names
|
||||
# stay importable from here.
|
||||
from .media_target import MAX_TARGET_PATH # noqa: F401
|
||||
from .media_target import media_target as _target
|
||||
from .images import _unavailable_link, _verified_title
|
||||
from .prompts import bad_request
|
||||
from .. import model_gate
|
||||
|
||||
@@ -40,7 +45,7 @@ try:
|
||||
)
|
||||
from ...database.execution_history import subscribe_history_restore
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...utils.media_capture import CAPTURE_EXTENSIONS, CaptureError, CaptureStore
|
||||
from ...utils.media_capture import CaptureError, CaptureStore
|
||||
except ImportError:
|
||||
from database.annotations import (
|
||||
MAX_IMAGE_TAGS,
|
||||
@@ -50,15 +55,13 @@ except ImportError:
|
||||
)
|
||||
from database.execution_history import subscribe_history_restore
|
||||
from database.library_commands import LibraryCommandError
|
||||
from utils.media_capture import CAPTURE_EXTENSIONS, CaptureError, CaptureStore
|
||||
from utils.media_capture import CaptureError, CaptureStore
|
||||
|
||||
# Request bodies of the annotation routes are small (a target plus a tag set).
|
||||
MAX_BODY_BYTES = 64 * 1024
|
||||
MAX_TARGET_PATH = 1024
|
||||
CAPTURE_URL = "/prompt_manager/image-annotations/capture/"
|
||||
_PREVIEW_CHUNK = 64 * 1024
|
||||
|
||||
_TARGET_FIELDS = frozenset({"root_index", "root_scope", "path"})
|
||||
_OBSERVE_FIELDS = frozenset({"target"})
|
||||
_SAVE_FIELDS = frozenset(
|
||||
{
|
||||
@@ -188,37 +191,6 @@ def _capture_error(exc: CaptureError) -> web.Response:
|
||||
return _coded(exc.code, _MESSAGES.get(exc.code) or str(exc))
|
||||
|
||||
|
||||
def _target(data: object):
|
||||
"""``((index, scope, relative), None)`` for a target, else ``(None, error)``.
|
||||
|
||||
Only the listing identity is accepted: an int root index, the root's
|
||||
64-hex scope and a bounded root-relative POSIX media path, never an
|
||||
absolute, drive-qualified, backslash or ``..`` path. Nothing here touches
|
||||
the filesystem or the database.
|
||||
"""
|
||||
if not isinstance(data, dict) or set(data) != _TARGET_FIELDS:
|
||||
return None, bad_request("target must name root_index, root_scope and path")
|
||||
index, scope, relative = data["root_index"], data["root_scope"], data["path"]
|
||||
if type(index) is not int or index < 0:
|
||||
return None, bad_request("Invalid gallery root")
|
||||
if not isinstance(scope, str) or _HEX64.fullmatch(scope) is None:
|
||||
return None, bad_request("Invalid gallery root scope")
|
||||
if not isinstance(relative, str) or not relative or len(relative) > MAX_TARGET_PATH:
|
||||
return None, bad_request("path must be a relative media path")
|
||||
parts = relative.split("/")
|
||||
if (
|
||||
"\0" in relative
|
||||
or "\\" in relative
|
||||
or relative.startswith("/")
|
||||
or _has_traversal(relative)
|
||||
or any(part in ("", ".", "..") or ":" in part for part in parts)
|
||||
):
|
||||
return None, _forbidden()
|
||||
if os.path.splitext(parts[-1])[1].lower() not in CAPTURE_EXTENSIONS:
|
||||
return None, _forbidden("Only media files can be tagged")
|
||||
return (index, scope, relative), None
|
||||
|
||||
|
||||
def _expires_at(seconds: float | None) -> str:
|
||||
moment = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
|
||||
seconds=max(0.0, seconds or 0.0)
|
||||
@@ -296,6 +268,7 @@ class AnnotationRoutesMixin:
|
||||
self._captures_closed = True
|
||||
unsubscribe, self._captures_unsubscribe = self._captures_unsubscribe, None
|
||||
store, self._captures = self._captures, None
|
||||
self._suggested_tokens = set()
|
||||
if unsubscribe is not None:
|
||||
unsubscribe()
|
||||
if store is not None:
|
||||
@@ -515,6 +488,7 @@ class AnnotationRoutesMixin:
|
||||
store = self._captures
|
||||
if store is not None and _TOKEN.fullmatch(token) is not None:
|
||||
store.release(token)
|
||||
self._suggested_tokens.discard(token)
|
||||
return web.json_response(
|
||||
{"success": True}, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
@@ -564,6 +538,9 @@ class AnnotationRoutesMixin:
|
||||
return _coded("capture-expired", _OTHER_FILE)
|
||||
if facts.sha256 != capture.sha256:
|
||||
return _coded("file-changed")
|
||||
# A save after Generate suggestions ran on this capture marks the
|
||||
# image auto-tagged (#379), so a Skip batch leaves the review alone.
|
||||
suggested = token in self._suggested_tokens
|
||||
try:
|
||||
receipt = await self._run_in_executor(
|
||||
self.db.replace_image_tags,
|
||||
@@ -574,6 +551,7 @@ class AnnotationRoutesMixin:
|
||||
expected_epoch=expected_epoch,
|
||||
expected_tag_revision=revision,
|
||||
tags=tags,
|
||||
mark_autotagged=suggested,
|
||||
)
|
||||
except AnnotationCommandError as exc:
|
||||
current = exc.current or {}
|
||||
@@ -590,10 +568,23 @@ class AnnotationRoutesMixin:
|
||||
except Exception:
|
||||
self.logger.exception("Image tag save failed")
|
||||
return _error(500, "save-failed", "The tags could not be saved.")
|
||||
# Dropped only after a saved or replayed receipt: a retry after a
|
||||
# tag-conflict still marks.
|
||||
self._suggested_tokens.discard(token)
|
||||
return web.json_response(
|
||||
{"success": True, **receipt}, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
|
||||
def _remember_suggested(self, store, token: str) -> None:
|
||||
"""Note that Generate ran on *token*; forget tokens whose capture is
|
||||
gone. ``expires_in`` does not reset a capture's idle clock."""
|
||||
self._suggested_tokens = {
|
||||
kept
|
||||
for kept in self._suggested_tokens
|
||||
if store.expires_in(kept) is not None
|
||||
}
|
||||
self._suggested_tokens.add(token)
|
||||
|
||||
# -- suggest ---------------------------------------------------------------
|
||||
|
||||
async def suggest_image_tags(self, request: web.Request) -> web.Response:
|
||||
@@ -604,7 +595,7 @@ class AnnotationRoutesMixin:
|
||||
it, including a legacy batch), never downloads (``model-unavailable``)
|
||||
and never writes.
|
||||
"""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
from ..autotag import MODELS, MissingBackendPackages, get_autotag_service
|
||||
from ..config import MAX_SUGGESTION_LIMIT, AutotagConfig
|
||||
|
||||
data, error = await self._annotation_body(request, SUGGEST_FIELDS)
|
||||
@@ -642,9 +633,10 @@ class AnnotationRoutesMixin:
|
||||
return error
|
||||
service = get_autotag_service()
|
||||
status = await self._run_in_executor(service.get_models_status)
|
||||
if not status.get(model_type, {}).get("downloaded"):
|
||||
name = MODELS.get(model_type, {}).get("name", model_type)
|
||||
return _coded("model-unavailable", unavailable_message(name))
|
||||
name = MODELS.get(model_type, {}).get("name", model_type)
|
||||
refusal = model_refusal(status, model_type, name)
|
||||
if refusal is not None:
|
||||
return _coded("model-unavailable", refusal)
|
||||
owner = model_gate.new_owner("image-annotations")
|
||||
if not model_gate.try_acquire(owner):
|
||||
return model_busy_response()
|
||||
@@ -663,6 +655,9 @@ class AnnotationRoutesMixin:
|
||||
options,
|
||||
on_skipped=functools.partial(store.release_use, token),
|
||||
)
|
||||
except MissingBackendPackages as exc:
|
||||
# A package gone since the status read (load_model logged it).
|
||||
return _coded("model-unavailable", str(exc))
|
||||
except Exception:
|
||||
self.logger.exception("Tag suggestion failed")
|
||||
return _error(
|
||||
@@ -675,6 +670,7 @@ class AnnotationRoutesMixin:
|
||||
finally:
|
||||
if not started:
|
||||
store.release_use(token)
|
||||
self._remember_suggested(store, token)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
"""Collections (#331): named, hand-picked groups of images.
|
||||
|
||||
Routes under ``/prompt_manager/collections``. A file is named by its listing
|
||||
identity ``{root_index, root_scope, path}`` (``media_target.py``) and stored
|
||||
under the favorites' keys (``_favorite_root_key``, ``_favorite_relative_key``),
|
||||
which never leave the server. A stale scope (the sources were reordered) is a
|
||||
409 ``source-changed`` before anything is written. Responses are
|
||||
``{"success": bool, ...}``; an unexpected failure is a logged 500 with a
|
||||
generic message, never the exception text.
|
||||
"""
|
||||
|
||||
import hmac
|
||||
import re
|
||||
from functools import wraps
|
||||
from typing import Callable
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .images import (
|
||||
_favorite_relative_key,
|
||||
_favorite_root_key,
|
||||
_forbidden,
|
||||
_json_body_or_error,
|
||||
collection_not_found,
|
||||
)
|
||||
from .media_target import media_target
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
from ...database.image_collections import (
|
||||
CollectionFilesMissing,
|
||||
CollectionInvalid,
|
||||
CollectionMissing,
|
||||
CollectionNameTaken,
|
||||
normalize_collection_name,
|
||||
)
|
||||
from ...database.image_favorites import MAX_KEY_LENGTH
|
||||
except ImportError:
|
||||
from database.image_collections import (
|
||||
CollectionFilesMissing,
|
||||
CollectionInvalid,
|
||||
CollectionMissing,
|
||||
CollectionNameTaken,
|
||||
normalize_collection_name,
|
||||
)
|
||||
from database.image_favorites import MAX_KEY_LENGTH
|
||||
|
||||
BASE = "/prompt_manager/collections"
|
||||
# The most files one add or remove names (the bulk tagging cap, MC12).
|
||||
MAX_ITEMS = 500
|
||||
MAX_ID = 999_999_999
|
||||
_PATH_ID = re.compile(r"[1-9][0-9]{0,8}")
|
||||
_ROOT_INDEX = re.compile(r"[0-9]{1,9}")
|
||||
_ADD_TARGETS = (frozenset({"collection_id", "items"}), frozenset({"name", "items"}))
|
||||
_REMOVE_FIELDS = frozenset({"collection_id", "items"})
|
||||
_RENAME_FIELDS = frozenset({"name"})
|
||||
_MEDIA_MESSAGE = "Only media files can be added to a collection"
|
||||
READ_FAILED = "Collections could not be read."
|
||||
CHANGE_FAILED = "The collection could not be changed."
|
||||
SOURCES_CHANGED = "The sources changed. Refresh the images and try again."
|
||||
FILES_NOT_FOUND = (
|
||||
"None of the selected images could be found. Refresh the images and try again."
|
||||
)
|
||||
NAME_TAKEN = "Another collection already has this name."
|
||||
|
||||
|
||||
class _Refused(Exception):
|
||||
"""A request refused before any write; carries its response."""
|
||||
|
||||
def __init__(self, response: web.Response):
|
||||
super().__init__(response.status)
|
||||
self.response = response
|
||||
|
||||
|
||||
def _coded(status: int, code: str, message: str, **extra) -> web.Response:
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": message, **extra},
|
||||
status=status,
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
|
||||
def _collection_response(failure: str) -> Callable:
|
||||
"""Map the database's refusals to their status; anything else to a 500."""
|
||||
|
||||
def decorate(handler: Callable) -> Callable:
|
||||
@wraps(handler)
|
||||
async def wrapped(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
return await handler(self, request)
|
||||
except _Refused as refused:
|
||||
return refused.response
|
||||
except CollectionInvalid as exc:
|
||||
return bad_request(str(exc))
|
||||
except CollectionMissing:
|
||||
return collection_not_found()
|
||||
except CollectionFilesMissing:
|
||||
return _coded(404, "files-not-found", FILES_NOT_FOUND)
|
||||
except CollectionNameTaken as exc:
|
||||
return _coded(
|
||||
409, "name-taken", NAME_TAKEN, collection_id=exc.collection_id
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Collection request failed")
|
||||
return web.json_response(
|
||||
{"success": False, "error": failure}, status=500
|
||||
)
|
||||
|
||||
return wrapped
|
||||
|
||||
return decorate
|
||||
|
||||
|
||||
def _path_id(request: web.Request) -> int:
|
||||
raw = request.match_info["collection_id"]
|
||||
if not raw.isascii() or _PATH_ID.fullmatch(raw) is None:
|
||||
raise _Refused(bad_request("Invalid collection id"))
|
||||
return int(raw)
|
||||
|
||||
|
||||
def _body_id(value: object) -> int:
|
||||
if type(value) is not int or not 1 <= value <= MAX_ID:
|
||||
raise _Refused(bad_request("Invalid collection id"))
|
||||
return value
|
||||
|
||||
|
||||
async def _body(request: web.Request, allowed: tuple) -> dict:
|
||||
data, error = await _json_body_or_error(request)
|
||||
if error is not None:
|
||||
raise _Refused(error)
|
||||
if frozenset(data) not in allowed:
|
||||
names = " or ".join(", ".join(sorted(fields)) for fields in allowed)
|
||||
raise _Refused(bad_request(f"Request must contain exactly {names}"))
|
||||
return data
|
||||
|
||||
|
||||
class CollectionRoutesMixin:
|
||||
"""Mixin providing the collection routes (#331)."""
|
||||
|
||||
def _register_collection_routes(self, routes: web.RouteTableDef) -> None:
|
||||
# The fixed paths go first: "/items" must never match "/{collection_id}".
|
||||
routes.get(BASE)(self.list_collections)
|
||||
routes.post(BASE + "/items")(self.add_collection_items)
|
||||
routes.delete(BASE + "/items")(self.remove_collection_items)
|
||||
routes.get(BASE + "/membership")(self.collection_membership)
|
||||
routes.patch(BASE + "/{collection_id}")(self.rename_collection)
|
||||
routes.delete(BASE + "/{collection_id}")(self.delete_collection)
|
||||
|
||||
# -- reads -------------------------------------------------------------
|
||||
|
||||
@_collection_response(READ_FAILED)
|
||||
async def list_collections(self, request: web.Request) -> web.Response:
|
||||
"""Every collection with its listed ``count`` and its ``missing`` rows.
|
||||
|
||||
``count`` is the members listed under every configured source (one
|
||||
primary-key range read per source, matched against its cached
|
||||
listing); ``missing`` the stored rows beyond it. When a scan or a read
|
||||
fails both are null for every collection and the names still return.
|
||||
"""
|
||||
collections = await self._run_in_executor(self.db.list_collections)
|
||||
counts = None
|
||||
if collections:
|
||||
try:
|
||||
counts = await self._collection_counts()
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"Collection counts could not be read", exc_info=True
|
||||
)
|
||||
rows = []
|
||||
for collection in collections:
|
||||
listed, stored = (None, None) if counts is None else counts
|
||||
count = None if listed is None else listed.get(collection["id"], 0)
|
||||
stored_rows = None if stored is None else stored.get(collection["id"], 0)
|
||||
rows.append(
|
||||
{
|
||||
**collection,
|
||||
"count": count,
|
||||
# Clamped: never below zero, whatever the snapshot saw.
|
||||
"missing": None if count is None else max(0, stored_rows - count),
|
||||
}
|
||||
)
|
||||
return web.json_response({"success": True, "collections": rows})
|
||||
|
||||
async def _collection_counts(self) -> tuple[dict, dict]:
|
||||
"""``({id: listed members}, {id: stored rows})`` over every source.
|
||||
|
||||
The scans come from the gallery cache; the root keys (realpath), the
|
||||
listings and one read snapshot of every member and stored count run
|
||||
in the executor, so a write between two reads cannot skew them.
|
||||
"""
|
||||
roots = list(self._get_all_output_dirs())
|
||||
root_keys = await self._run_in_executor(
|
||||
lambda: [_favorite_root_key(root) for root in roots]
|
||||
)
|
||||
distinct = {} # one folder configured twice counts once
|
||||
for root, root_key in zip(roots, root_keys):
|
||||
distinct.setdefault(root_key, root)
|
||||
scans = {
|
||||
root_key: await self._get_gallery_files(root)
|
||||
for root_key, root in distinct.items()
|
||||
}
|
||||
return await self._run_in_executor(self._count_members, distinct, scans)
|
||||
|
||||
def _count_members(self, roots: dict, scans: dict) -> tuple[dict, dict]:
|
||||
"""Blocking: match one snapshot's members against each listing."""
|
||||
members, stored = self.db.collection_count_snapshot(list(roots))
|
||||
listed: dict[int, int] = {}
|
||||
for root_key, root in roots.items():
|
||||
listing = self._favorite_listing(root, scans[root_key])
|
||||
for collection_id, keys in members.get(root_key, {}).items():
|
||||
found = sum(1 for key in keys if key in listing)
|
||||
listed[collection_id] = listed.get(collection_id, 0) + found
|
||||
return listed, stored
|
||||
|
||||
@_collection_response(READ_FAILED)
|
||||
async def collection_membership(self, request: web.Request) -> web.Response:
|
||||
"""GET ``?root_index&root_scope&path``: the ids holding that file."""
|
||||
query = request.query
|
||||
if len(query) != 3 or set(query) != {"root_index", "root_scope", "path"}:
|
||||
# len() counts a repeated key (?path=a&path=b) twice.
|
||||
return bad_request("Query must name root_index, root_scope and path once")
|
||||
raw_index = query["root_index"]
|
||||
if not raw_index.isascii() or _ROOT_INDEX.fullmatch(raw_index) is None:
|
||||
return bad_request("Invalid gallery root")
|
||||
roots = list(self._get_all_output_dirs())
|
||||
target = {
|
||||
"root_index": int(raw_index),
|
||||
"root_scope": query["root_scope"],
|
||||
"path": query["path"],
|
||||
}
|
||||
root_key, key = await self._run_in_executor(
|
||||
self._collection_key, target, roots, {}
|
||||
)
|
||||
ids = await self._run_in_executor(self.db.file_collections, root_key, key)
|
||||
return web.json_response({"success": True, "collection_ids": ids})
|
||||
|
||||
# -- writes ------------------------------------------------------------
|
||||
|
||||
@_collection_response(CHANGE_FAILED)
|
||||
async def add_collection_items(self, request: web.Request) -> web.Response:
|
||||
"""POST ``{collection_id | name, items}``: add the listed files.
|
||||
|
||||
Only files the source's listing holds are added; the rest count in
|
||||
``not_found``. A new name with nothing to add creates nothing (404
|
||||
``files-not-found``).
|
||||
"""
|
||||
data = await _body(request, _ADD_TARGETS)
|
||||
target = {}
|
||||
if "name" in data:
|
||||
target["name"] = normalize_collection_name(data["name"])
|
||||
else:
|
||||
target["collection_id"] = _body_id(data["collection_id"])
|
||||
roots = list(self._get_all_output_dirs())
|
||||
keys = await self._run_in_executor(self._collection_items, data["items"], roots)
|
||||
found, not_found = [], 0
|
||||
listings = {}
|
||||
for index, root_key, key in keys:
|
||||
if index not in listings:
|
||||
root = roots[index]
|
||||
files = await self._get_gallery_files(root)
|
||||
listings[index] = await self._run_in_executor(
|
||||
self._favorite_listing, root, files
|
||||
)
|
||||
if key in listings[index]:
|
||||
found.append((root_key, key))
|
||||
else:
|
||||
not_found += 1
|
||||
result = await self._run_in_executor(
|
||||
lambda: self.db.add_to_collection(items=found, **target)
|
||||
)
|
||||
return web.json_response({"success": True, **result, "not_found": not_found})
|
||||
|
||||
@_collection_response(CHANGE_FAILED)
|
||||
async def remove_collection_items(self, request: web.Request) -> web.Response:
|
||||
"""DELETE ``{collection_id, items}``: listed or not, every named file."""
|
||||
data = await _body(request, (_REMOVE_FIELDS,))
|
||||
collection_id = _body_id(data["collection_id"])
|
||||
roots = list(self._get_all_output_dirs())
|
||||
keys = await self._run_in_executor(self._collection_items, data["items"], roots)
|
||||
result = await self._run_in_executor(
|
||||
self.db.remove_from_collection,
|
||||
collection_id,
|
||||
[(root_key, key) for _index, root_key, key in keys],
|
||||
)
|
||||
return web.json_response({"success": True, **result})
|
||||
|
||||
@_collection_response(CHANGE_FAILED)
|
||||
async def rename_collection(self, request: web.Request) -> web.Response:
|
||||
"""PATCH ``{name}``: a case variant of its own name is allowed."""
|
||||
collection_id = _path_id(request)
|
||||
data = await _body(request, (_RENAME_FIELDS,))
|
||||
name = normalize_collection_name(data["name"])
|
||||
collection = await self._run_in_executor(
|
||||
self.db.rename_collection, collection_id, name
|
||||
)
|
||||
return web.json_response({"success": True, "collection": collection})
|
||||
|
||||
@_collection_response(CHANGE_FAILED)
|
||||
async def delete_collection(self, request: web.Request) -> web.Response:
|
||||
"""DELETE: the collection and its members; files are never touched."""
|
||||
collection_id = _path_id(request)
|
||||
await self._run_in_executor(self.db.delete_collection, collection_id)
|
||||
return web.json_response({"success": True, "deleted": True})
|
||||
|
||||
# -- request steps -----------------------------------------------------
|
||||
|
||||
def _collection_items(self, items: object, roots: list) -> list[tuple]:
|
||||
"""``(index, root key, relative key)`` per distinct file, in order.
|
||||
|
||||
Blocking (realpath per root, once per request): run in the executor.
|
||||
Every item is checked before anything is written: 400 for a malformed
|
||||
shape, 403 for an unsafe path, 409 for a stale scope. Two spellings
|
||||
that fold to one key (Windows) count once.
|
||||
"""
|
||||
if not isinstance(items, list) or not 1 <= len(items) <= MAX_ITEMS:
|
||||
raise _Refused(bad_request(f"items must list 1 to {MAX_ITEMS} images"))
|
||||
keys, sources = {}, {}
|
||||
for item in items:
|
||||
index = item.get("root_index") if isinstance(item, dict) else None
|
||||
root_key, key = self._collection_key(item, roots, sources)
|
||||
keys.setdefault((root_key, key), index)
|
||||
return [(index, root_key, key) for (root_key, key), index in keys.items()]
|
||||
|
||||
def _collection_key(
|
||||
self, item: object, roots: list, sources: dict
|
||||
) -> tuple[str, str]:
|
||||
"""The stored keys of one target, or raise its refusal (blocking).
|
||||
|
||||
*sources* caches ``{index: (scope, root key)}`` for one request, so
|
||||
each root's realpath runs once however many items name it.
|
||||
"""
|
||||
target, error = media_target(item, media_message=_MEDIA_MESSAGE)
|
||||
if error is not None:
|
||||
raise _Refused(error)
|
||||
index, scope, relative = target
|
||||
if index >= len(roots):
|
||||
raise _Refused(_coded(409, "source-changed", SOURCES_CHANGED))
|
||||
if index not in sources:
|
||||
sources[index] = (
|
||||
self._gallery_root_scope(roots[index], index),
|
||||
_favorite_root_key(roots[index]),
|
||||
)
|
||||
current, root_key = sources[index]
|
||||
if not hmac.compare_digest(scope, current):
|
||||
raise _Refused(_coded(409, "source-changed", SOURCES_CHANGED))
|
||||
key = _favorite_relative_key(relative)
|
||||
if max(len(root_key), len(key)) > MAX_KEY_LENGTH:
|
||||
# A stored key past the schema bound would fail the next restore.
|
||||
raise _Refused(_forbidden("Path is too long to add to a collection"))
|
||||
return root_key, key
|
||||
@@ -0,0 +1,291 @@
|
||||
"""The shareable diagnostic report (#329 System S12).
|
||||
|
||||
``GET /prompt_manager/system/diagnostics`` answers a report built on the
|
||||
server from an explicit list of fields: the project and Viewer.js versions,
|
||||
Python and the platform, the four checks of the legacy ``GET /diagnostics``
|
||||
(one helper, ``admin.legacy_checks``), the database's public info, source
|
||||
counts, the log configuration and statistics, the last health summary and
|
||||
the configuration keys that only ``config.json`` holds. It never copies
|
||||
``PromptManagerConfig.get_config()``: no path leaf is listed, and the CivitAI
|
||||
key is reported only as ``has_civitai_api_key``.
|
||||
|
||||
Every path in the report goes through ``report_path``: relative to the
|
||||
closest public anchor that is not a filesystem root, else ``"outside
|
||||
ComfyUI"``, never a basename that could be a user name. As a backstop,
|
||||
``scrub`` replaces any string that still looks like an absolute path, holds
|
||||
the saved key or the home folder (with or without its root), and counts each
|
||||
replacement in ``redacted`` (0 for a correct builder).
|
||||
|
||||
The report opens the database, so it takes a registry lease: it is refused
|
||||
(409 ``busy``) while an ``exclusive`` job (a restore) runs, and a restore
|
||||
waits for it. It is built off the event loop.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import ntpath
|
||||
import os
|
||||
import platform
|
||||
import posixpath
|
||||
import re
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import admin as _admin
|
||||
from .job_routes import busy_response
|
||||
from .jobs import JobBusy
|
||||
from .system import health_summary
|
||||
|
||||
try:
|
||||
from ...utils.version import project_version
|
||||
except ImportError: # loaded as the top-level ``py`` package (tests)
|
||||
from utils.version import project_version
|
||||
|
||||
DIAGNOSTICS_LEASE = "Diagnostics"
|
||||
_ROOT = Path(__file__).resolve().parents[2]
|
||||
VIEWERJS_PROVENANCE = str(_ROOT / "web" / "lib" / "viewerjs" / "provenance.json")
|
||||
OUTSIDE = "outside ComfyUI"
|
||||
REMOVED = "[removed]"
|
||||
# The CivitAI key is searched for only when it is long enough to be a key.
|
||||
MIN_KEY_LENGTH = 8
|
||||
_WINDOWS_ABSOLUTE = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
|
||||
|
||||
def anchored_path(canonical, anchors, home, pathmod=os.path):
|
||||
"""``canonical`` relative to the closest usable anchor, else ``outside ComfyUI``.
|
||||
|
||||
All three are already canonical (``normcase``) in ``pathmod``'s spelling.
|
||||
A filesystem root is never usable, nor is a strict ancestor of ``home``
|
||||
(``/home``, ``C:\\Users``), whose relative paths would start with a user
|
||||
name; so a path inside the home folder needs an anchor at or below it.
|
||||
"""
|
||||
flavor = PureWindowsPath if pathmod is ntpath else PurePosixPath
|
||||
target = flavor(canonical)
|
||||
home_path = flavor(home) if home else None
|
||||
usable = []
|
||||
for anchor in map(flavor, anchors):
|
||||
if anchor.parent == anchor:
|
||||
continue
|
||||
if home_path is not None and home_path != anchor:
|
||||
if home_path.is_relative_to(anchor):
|
||||
continue
|
||||
if target.is_relative_to(anchor):
|
||||
usable.append(anchor)
|
||||
if not usable:
|
||||
return OUTSIDE
|
||||
anchor = max(usable, key=lambda a: len(a.parts))
|
||||
return target.relative_to(anchor).as_posix() or "."
|
||||
|
||||
|
||||
def _canonical(path):
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
|
||||
|
||||
def report_path(path):
|
||||
"""``path`` relative to a public anchor (POSIX style), else ``outside ComfyUI``."""
|
||||
if not path:
|
||||
return ""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
try:
|
||||
canonical = _canonical(path)
|
||||
anchors = [_canonical(anchor) for anchor in GalleryConfig.path_anchors()]
|
||||
home = os.path.expanduser("~")
|
||||
home = _canonical(home) if home and home != "~" else None
|
||||
except (OSError, ValueError):
|
||||
return OUTSIDE
|
||||
return anchored_path(canonical, anchors, home)
|
||||
|
||||
|
||||
def _looks_absolute(text):
|
||||
return (
|
||||
posixpath.isabs(text)
|
||||
or ntpath.isabs(text)
|
||||
or _WINDOWS_ABSOLUTE.match(text) is not None
|
||||
or text.startswith("\\\\")
|
||||
)
|
||||
|
||||
|
||||
# The home folder is matched only as whole path parts, never inside a word.
|
||||
_PART = r"(?<![\w.-]){}(?![\w.-])"
|
||||
MIN_HOME_NAME = 3
|
||||
|
||||
|
||||
def _home_patterns():
|
||||
"""``(pattern, paths_only)`` for the home folder: in full, rootless, its name.
|
||||
|
||||
Accepted residual risk: only this process's own home folder is known, so
|
||||
another user's home (an output folder under /home/<them> while ComfyUI
|
||||
runs as a service account) can still be named by a report path.
|
||||
"""
|
||||
home = os.path.expanduser("~")
|
||||
if not home or home == "~" or len(home) < 4 or home in ("/", "\\"):
|
||||
return ()
|
||||
spellings = {home, home.replace("\\", "/")}
|
||||
relative = os.path.splitdrive(home)[1].strip("\\/")
|
||||
for spelling in (relative, relative.replace("\\", "/")):
|
||||
if "/" in spelling or "\\" in spelling:
|
||||
spellings.add(spelling)
|
||||
patterns = [
|
||||
(re.compile(_PART.format(re.escape(spelling)), re.IGNORECASE), False)
|
||||
for spelling in spellings
|
||||
]
|
||||
name = re.split(r"[\\/]", relative)[-1] if relative else ""
|
||||
if len(name) >= MIN_HOME_NAME:
|
||||
# The bare name only inside a path ("info/x"), not a word ("INFO").
|
||||
patterns.append(
|
||||
(re.compile(_PART.format(re.escape(name)), re.IGNORECASE), True)
|
||||
)
|
||||
return tuple(patterns)
|
||||
|
||||
|
||||
def scrub(report, key):
|
||||
"""``report`` with leaked paths and the key replaced, counted in ``redacted``."""
|
||||
key = key if isinstance(key, str) and len(key) >= MIN_KEY_LENGTH else None
|
||||
homes = _home_patterns()
|
||||
count = 0
|
||||
|
||||
def clean(value):
|
||||
nonlocal count
|
||||
if isinstance(value, str):
|
||||
if (
|
||||
_looks_absolute(value)
|
||||
or (key and key in value)
|
||||
or any(
|
||||
pattern.search(value)
|
||||
for pattern, paths_only in homes
|
||||
if not paths_only or "/" in value or "\\" in value
|
||||
)
|
||||
):
|
||||
count += 1
|
||||
return REMOVED
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return {name: clean(item) for name, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [clean(item) for item in value]
|
||||
return value
|
||||
|
||||
cleaned = clean(report)
|
||||
cleaned["redacted"] = int(cleaned.get("redacted") or 0) + count
|
||||
return cleaned
|
||||
|
||||
|
||||
def viewerjs_version(path=None):
|
||||
"""The pinned Viewer.js version from its provenance file, or ``unknown``."""
|
||||
try:
|
||||
with open(path or VIEWERJS_PROVENANCE, encoding="utf-8") as handle:
|
||||
version = json.load(handle).get("version")
|
||||
except (OSError, ValueError, AttributeError):
|
||||
return "unknown"
|
||||
return version if isinstance(version, str) and version else "unknown"
|
||||
|
||||
|
||||
def public_config():
|
||||
"""The configuration keys only ``config.json`` holds, from an explicit list."""
|
||||
from ..config import GalleryConfig, IntegrationConfig, PromptManagerConfig
|
||||
|
||||
return {
|
||||
"database": {
|
||||
"enable_duplicate_detection": (
|
||||
PromptManagerConfig.ENABLE_DUPLICATE_DETECTION
|
||||
),
|
||||
"enable_auto_save": PromptManagerConfig.ENABLE_AUTO_SAVE,
|
||||
},
|
||||
"web_ui": {
|
||||
"result_timeout": PromptManagerConfig.RESULT_TIMEOUT,
|
||||
"show_test_button": PromptManagerConfig.SHOW_TEST_BUTTON,
|
||||
"webui_display_mode": PromptManagerConfig.WEBUI_DISPLAY_MODE,
|
||||
"infinite_scroll": PromptManagerConfig.INFINITE_SCROLL,
|
||||
},
|
||||
"performance": {
|
||||
"max_search_results": PromptManagerConfig.MAX_SEARCH_RESULTS,
|
||||
"enable_fuzzy_search": PromptManagerConfig.ENABLE_FUZZY_SEARCH,
|
||||
"auto_backup_interval": PromptManagerConfig.AUTO_BACKUP_INTERVAL,
|
||||
"worker_threads": PromptManagerConfig.WORKER_THREADS,
|
||||
},
|
||||
"gallery": {
|
||||
"processing_delay": GalleryConfig.PROCESSING_DELAY,
|
||||
"prompt_timeout": GalleryConfig.PROMPT_TIMEOUT,
|
||||
"cleanup_interval": GalleryConfig.CLEANUP_INTERVAL,
|
||||
"auto_cleanup_missing_files": GalleryConfig.AUTO_CLEANUP_MISSING_FILES,
|
||||
"max_image_age_days": GalleryConfig.MAX_IMAGE_AGE_DAYS,
|
||||
"max_concurrent_processing": GalleryConfig.MAX_CONCURRENT_PROCESSING,
|
||||
"metadata_extraction_timeout": GalleryConfig.METADATA_EXTRACTION_TIMEOUT,
|
||||
# config.json gallery.monitoring.save_foreign_runs (#247)
|
||||
"save_foreign_runs": GalleryConfig.SAVE_FOREIGN_RUNS,
|
||||
},
|
||||
"integrations": {
|
||||
"lora_manager_enabled": bool(IntegrationConfig.LORA_MANAGER_ENABLED),
|
||||
"has_civitai_api_key": bool(IntegrationConfig.CIVITAI_API_KEY),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _source_counts(sources):
|
||||
configured = sources.get("configured") or []
|
||||
effective = sources.get("effective") or []
|
||||
return {
|
||||
"configured": len(configured),
|
||||
"effective": len(effective),
|
||||
"watching": sum(1 for row in effective if row.get("watch") == "watching"),
|
||||
"unavailable": sum(1 for row in configured if row.get("valid") is False),
|
||||
}
|
||||
|
||||
|
||||
class DiagnosticsReportMixin:
|
||||
"""Mixin providing ``GET /prompt_manager/system/diagnostics``."""
|
||||
|
||||
def _register_diagnostics_report_routes(self, routes):
|
||||
@routes.get("/prompt_manager/system/diagnostics")
|
||||
async def diagnostics_report_route(request):
|
||||
return await self.get_diagnostics_report(request)
|
||||
|
||||
async def get_diagnostics_report(self, request):
|
||||
"""The report, built off the loop under a lease (409 ``busy`` in a restore)."""
|
||||
from ..config import IntegrationConfig
|
||||
|
||||
try:
|
||||
async with self._jobs.lease(DIAGNOSTICS_LEASE):
|
||||
report = await self._run_in_executor(self.build_report)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
except Exception:
|
||||
self.logger.exception("The diagnostic report could not be built")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": None,
|
||||
"error": "The diagnostic report could not be built.",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
report = scrub(report, IntegrationConfig.CIVITAI_API_KEY)
|
||||
return web.json_response(report, headers={"Cache-Control": "no-store"})
|
||||
|
||||
def build_report(self):
|
||||
"""The report's fields (blocking: reads the database and the log folder)."""
|
||||
db = self.db
|
||||
manager = self._get_logger_manager()
|
||||
stats = dict(manager.get_log_stats())
|
||||
stats["log_directory"] = report_path(stats.get("log_directory"))
|
||||
info = dict(db.model.get_database_info() or {})
|
||||
if "database_path" in info:
|
||||
info["database_path"] = report_path(info["database_path"])
|
||||
sources, _monitor = self._source_status()
|
||||
return {
|
||||
"success": True,
|
||||
"generated_at": datetime.datetime.now().replace(microsecond=0).isoformat(),
|
||||
"version": project_version(_ROOT),
|
||||
"python": platform.python_version(),
|
||||
"platform": {"system": platform.system(), "release": platform.release()},
|
||||
"viewerjs": viewerjs_version(VIEWERJS_PROVENANCE),
|
||||
"checks": _admin.legacy_checks(db.model.db_path, report_path),
|
||||
"database_info": info,
|
||||
"sources": _source_counts(sources),
|
||||
"logging": {"config": dict(manager.get_config()), "stats": stats},
|
||||
"health": health_summary(self._current_health_report()),
|
||||
"config": public_config(),
|
||||
"redacted": 0,
|
||||
}
|
||||
@@ -588,7 +588,10 @@ class DuplicateFilesMixin:
|
||||
job.update(total=len(targets), status_line="Removing duplicate files")
|
||||
spellings = await self._run_in_executor(self._source_spellings)
|
||||
removed, failure, keepers = 0, None, {}
|
||||
totals = dict.fromkeys(("moved", "merged", "favorites", "tags", "review"), 0)
|
||||
totals = dict.fromkeys(
|
||||
("moved", "merged", "favorites", "tags", "review", "collections"), 0
|
||||
)
|
||||
totals["categories"] = 0 # image categories kept on the kept copies (#332)
|
||||
for done, target in enumerate(targets, start=1):
|
||||
if failure is not None:
|
||||
self._record_target(job, "kept", target, NOT_ATTEMPTED)
|
||||
@@ -631,10 +634,18 @@ class DuplicateFilesMixin:
|
||||
job.log_line(
|
||||
f"Moved {_plural(totals['favorites'], 'favorite')} to the kept copies"
|
||||
)
|
||||
if totals["collections"]:
|
||||
job.log_line(
|
||||
f"Moved {_plural(totals['collections'], 'collection membership')} "
|
||||
"to the kept copies"
|
||||
)
|
||||
if totals["tags"]:
|
||||
job.log_line(
|
||||
f"Added {_plural(totals['tags'], 'image tag')} to the kept copies"
|
||||
)
|
||||
if totals.get("categories"):
|
||||
kept = _plural(totals["categories"], "image category", "image categories")
|
||||
job.log_line(f"Kept {kept} on the kept copies")
|
||||
if totals["review"]:
|
||||
job.log_line(
|
||||
f"Settled {_plural(totals['review'], 'Needs review item')} "
|
||||
|
||||
@@ -16,6 +16,11 @@ from pathlib import Path
|
||||
from .admin import DELETABLE_MEDIA_EXTENSIONS
|
||||
from .maintenance import DUPLICATE_FILES
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import in_trash_below_roots, is_trash_folder
|
||||
except ImportError:
|
||||
from utils.media_folders import in_trash_below_roots, is_trash_folder
|
||||
|
||||
# Keep entries one review takes; with paths of a few hundred characters the
|
||||
# list stays inside the preview body cap (DUPLICATE_PREVIEW_BODY_MAX, 1 MiB).
|
||||
MAX_KEEP = 2000
|
||||
@@ -197,6 +202,8 @@ def file_problem(
|
||||
parts = Path(os.path.relpath(lexical, base)).parts
|
||||
if "thumbnails" in parts or ".." in parts:
|
||||
return OUTSIDE_SOURCE
|
||||
if any(is_trash_folder(part) for part in parts[:-1]):
|
||||
return OUTSIDE_SOURCE # never removed from a trash folder (#543)
|
||||
if os.path.splitext(lexical)[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||||
return OUTSIDE_SOURCE
|
||||
try:
|
||||
@@ -212,6 +219,8 @@ def file_problem(
|
||||
try:
|
||||
if not _lexically_inside(_canonical_path(path), [_canonical_path(root)]):
|
||||
return OUTSIDE_SOURCE # a linked folder on the way
|
||||
if in_trash_below_roots(path, [root]):
|
||||
return OUTSIDE_SOURCE # a linked folder into a trash folder
|
||||
if keeper is not None:
|
||||
same = keeper_problem(path, root, info, keeper, keeper_root)
|
||||
if same:
|
||||
|
||||
@@ -18,7 +18,7 @@ Blocking: each lookup stats the file's folders once (cached).
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from .images import _favorite_relative_key, _favorite_root_key
|
||||
from .images import _favorite_relative_key, _favorite_root_key, image_category_key
|
||||
|
||||
|
||||
def _identity(path):
|
||||
@@ -78,6 +78,15 @@ class Spellings:
|
||||
)
|
||||
return spelled
|
||||
|
||||
@property
|
||||
def listed_roots(self):
|
||||
"""The listed roots (the output sources), in order."""
|
||||
return [root for root, _identity in self._listed]
|
||||
|
||||
def category_key(self, path):
|
||||
"""*path*'s image category file key (#332), or None outside every source."""
|
||||
return image_category_key(self.listed_roots, path)
|
||||
|
||||
def favorites(self, path):
|
||||
"""``{listed root: path below it (POSIX)}`` for each listed root holding it."""
|
||||
path = os.path.abspath(str(path))
|
||||
@@ -95,7 +104,8 @@ def copy_move(target, kept_stat, spellings):
|
||||
|
||||
A record found under a folder spelling moves to the kept copy under
|
||||
the same spelling when that folder holds it too, else to its real
|
||||
path. Its favorite moves under every listed root holding the file.
|
||||
path. Its favorite and its collection memberships move under every
|
||||
listed root holding the file.
|
||||
"""
|
||||
removed = spellings.of(target["abs"])
|
||||
kept = spellings.of(target["keeper_abs"])
|
||||
@@ -125,9 +135,23 @@ def copy_move(target, kept_stat, spellings):
|
||||
"targets": targets,
|
||||
"kept_path": target["keeper_abs"],
|
||||
"favorites": favorites,
|
||||
"categories": _category_pairs(target, spellings),
|
||||
"removed_locator": target["locator"],
|
||||
"kept_locator": target["keeper_locator"],
|
||||
"sha256": target["hash"],
|
||||
"kept_size": kept_stat.st_size,
|
||||
"kept_mtime_ns": kept_stat.st_mtime_ns,
|
||||
}
|
||||
|
||||
|
||||
def _category_pairs(target, spellings):
|
||||
"""``[(removed file key, kept file key)]`` for the image category (#332).
|
||||
|
||||
From the two absolute paths, not the favorites tuples: those follow the
|
||||
listed root, which differs from the file key when sources nest.
|
||||
"""
|
||||
removed = spellings.category_key(target["abs"])
|
||||
kept = spellings.category_key(target["keeper_abs"])
|
||||
if removed is None or kept is None:
|
||||
return []
|
||||
return [(removed, kept)]
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
"""One shared listing per output root, and the pure helpers behind it (#348).
|
||||
|
||||
The API keeps one ``RootListing`` per output root (``_root_listings`` in
|
||||
``py/api/__init__.py``). The record is read and written only on the event
|
||||
loop; the walk itself, the comparison with the previous listing and both
|
||||
folder sets run in the executor through ``walk_root``. Routes live in
|
||||
``py/api/images.py`` (``_listing``); this module has no aiohttp import so
|
||||
tests can exercise it without a server.
|
||||
|
||||
A settled output the image monitor reports is added to its root's stored
|
||||
listing without a walk (``output_settled``, ``insert_entry``), in the place
|
||||
a walk would list it. When that cannot be exact, the root is marked dirty
|
||||
(``mark_root_dirty``) instead; a dirty root gets one background walk, no
|
||||
sooner than ``rewalk_interval`` after its previous walk started (PR 2).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from bisect import bisect_left
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import PurePath
|
||||
from typing import Callable, NamedTuple
|
||||
|
||||
# A listing older than this (seconds) is served as it is, and one background
|
||||
# walk replaces it. Read as ``gallery_listing.REFRESH_AFTER`` so tests can
|
||||
# patch it.
|
||||
REFRESH_AFTER = 30
|
||||
|
||||
# The stale read is served at once; its refresh walk starts this many seconds
|
||||
# later, so the walk does not share the GIL with the page's own requests
|
||||
# (#348 live test: reads during the walk took 2.2 s instead of 0.07 s).
|
||||
REFRESH_DELAY = 1.0
|
||||
|
||||
# No background walk of a root starts sooner than rewalk_interval(last walk's
|
||||
# duration) after its previous walk started: FACTOR times that duration, at
|
||||
# least FLOOR and at most CAP seconds. Walks up to CAP / FACTOR (3 s) then
|
||||
# hold the GIL at most ~20% of the time; the cap keeps a new output listed
|
||||
# within ~20 s, at the cost of a higher share (back to back from 15 s) on
|
||||
# slower walks. Read through this module so tests can patch them.
|
||||
REWALK_FLOOR = 5
|
||||
REWALK_FACTOR = 5
|
||||
REWALK_CAP = 15
|
||||
|
||||
# A deferred walk is never armed sooner than this: no zero-delay loop.
|
||||
MIN_REWALK_DELAY = 0.5
|
||||
|
||||
# A reader with no listing gives up (500) after this many walks in a row were
|
||||
# discarded by invalidations made while they ran.
|
||||
MAX_DISCARDED_WALKS = 3
|
||||
|
||||
WALK_STOPPED = "The listing walk was stopped"
|
||||
|
||||
STORED = "stored"
|
||||
DISCARDED = "discarded"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class WalkOutcome(NamedTuple):
|
||||
"""What one walk ended with; a walk's flight future resolves to this."""
|
||||
|
||||
kind: str
|
||||
error: BaseException | None = None
|
||||
|
||||
@classmethod
|
||||
def stored(cls) -> "WalkOutcome":
|
||||
return cls(STORED)
|
||||
|
||||
@classmethod
|
||||
def discarded(cls) -> "WalkOutcome":
|
||||
return cls(DISCARDED)
|
||||
|
||||
@classmethod
|
||||
def failed(cls, error: BaseException) -> "WalkOutcome":
|
||||
return cls(FAILED, error)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RootListing:
|
||||
"""The shared listing of one output root.
|
||||
|
||||
A record is never deleted: ``drop()`` clears the listing and bumps
|
||||
``generation``, so a walk that started before a drop can never be stored
|
||||
after it.
|
||||
"""
|
||||
|
||||
# (path, mtime) newest first, as _scan_gallery_files_sync returns it.
|
||||
files: list | None = None
|
||||
folders: frozenset | None = None
|
||||
folders_with_ancestors: frozenset | None = None
|
||||
# Monotonic time the last walk finished, stored or failed.
|
||||
walked_at: float | None = None
|
||||
# Number of the last walk started; a running walk is always that one.
|
||||
walk_seq: int = 0
|
||||
# Number of the walk whose files are stored (Rescan waits on it).
|
||||
stored_seq: int = 0
|
||||
generation: int = 0
|
||||
# The running walk's future (resolves to a WalkOutcome), or None.
|
||||
flight: asyncio.Future | None = None
|
||||
# True while walks keep failing: one warning per streak.
|
||||
failing: bool = False
|
||||
# The one deferred background walk not started yet (loop.call_later), or
|
||||
# None: a stale refresh or a dirty root's rewalk, whichever is due first.
|
||||
rewalk_handle: asyncio.TimerHandle | None = None
|
||||
# normcase(realpath(root)), computed by the walk in the executor: monitor
|
||||
# paths are matched against it without file I/O on the loop.
|
||||
resolved_root: str | None = None
|
||||
# Monotonic start and duration of the last walk (rewalk_interval).
|
||||
last_started_at: float | None = None
|
||||
last_duration: float | None = None
|
||||
# A settled output was reported under this root; cleared when a walk starts.
|
||||
dirty: bool = False
|
||||
# Dropped because the output roots changed: no background walk until a
|
||||
# reader (or the warm-up) asks for this root again. The monitor keeps
|
||||
# watching a removed root until ComfyUI restarts.
|
||||
retired: bool = False
|
||||
# Relative paths of settled outputs waiting to be added to ``files``
|
||||
# without a walk, and the one task adding them (``output_settled``).
|
||||
pending_adds: list = field(default_factory=list)
|
||||
add_task: asyncio.Task | None = None
|
||||
# Settled outputs added to the stored listing since the last walk stored.
|
||||
patched: int = 0
|
||||
|
||||
def drop(self) -> None:
|
||||
self.files = None
|
||||
self.folders = None
|
||||
self.folders_with_ancestors = None
|
||||
self.walked_at = None
|
||||
self.pending_adds = []
|
||||
self.patched = 0
|
||||
self.generation += 1
|
||||
|
||||
|
||||
def folder_sets(root, files) -> tuple[frozenset, frozenset]:
|
||||
"""``(folders, folders_with_ancestors)`` of the files listed under *root*.
|
||||
|
||||
Folders keep the spelling ``/gallery/subfolders`` has always answered:
|
||||
``str(relative.parent)`` (native separators) for the folders holding
|
||||
files, plus every ancestor joined by ``/``.
|
||||
"""
|
||||
# From the path strings: relative_to per file was half of a large walk.
|
||||
sep, prefix = _root_prefix(root)
|
||||
folders = set()
|
||||
for path, _mtime in files:
|
||||
rel_dir = _folder_of(root, path, sep, prefix)
|
||||
if rel_dir:
|
||||
folders.add(rel_dir)
|
||||
with_ancestors = set(folders)
|
||||
for folder in folders:
|
||||
with_ancestors.update(_ancestors(folder))
|
||||
return frozenset(folders), frozenset(with_ancestors)
|
||||
|
||||
|
||||
def add_folder(root, path, folders, with_ancestors) -> tuple[frozenset, frozenset]:
|
||||
"""Both folder sets once *path* is listed too (the same sets when it adds none)."""
|
||||
sep, prefix = _root_prefix(root)
|
||||
rel_dir = _folder_of(root, path, sep, prefix)
|
||||
if not rel_dir or rel_dir in folders:
|
||||
return folders, with_ancestors
|
||||
return folders | {rel_dir}, with_ancestors | {rel_dir, *_ancestors(rel_dir)}
|
||||
|
||||
|
||||
def _root_prefix(root) -> tuple[str, str]:
|
||||
# The separator is the path flavour's own (tests use PureWindowsPath).
|
||||
sep = str(type(root)("a", "b"))[1]
|
||||
prefix = str(root)
|
||||
if not prefix.endswith(sep):
|
||||
prefix += sep
|
||||
return sep, prefix
|
||||
|
||||
|
||||
def _folder_of(root, path, sep, prefix) -> str:
|
||||
"""The folder ``/gallery/subfolders`` names for *path*; "" for the root."""
|
||||
text = str(path)
|
||||
if text.startswith(prefix):
|
||||
rel_dir = text[len(prefix) :].rpartition(sep)[0]
|
||||
else:
|
||||
rel_dir = str(path.relative_to(root).parent)
|
||||
return "" if rel_dir == "." else rel_dir
|
||||
|
||||
|
||||
def _ancestors(folder: str) -> list[str]:
|
||||
parts = folder.replace("\\", "/").split("/")
|
||||
return ["/".join(parts[:i]) for i in range(1, len(parts))]
|
||||
|
||||
|
||||
def relative_part(resolved_root: str, path, pathmod=os.path) -> str | None:
|
||||
"""The part of the resolved *path* below *resolved_root*, as a walk spells it.
|
||||
|
||||
Kept in *path*'s own case (``realpath`` gives a file's on-disk case on
|
||||
Windows) with the native separator. None when *path* is not below the
|
||||
root, or when the case folding changed the length so the slice would
|
||||
not be exact.
|
||||
"""
|
||||
sep = pathmod.sep
|
||||
text = str(path)
|
||||
if sep == "\\":
|
||||
text = _plain(text.replace("/", sep), pathmod)
|
||||
folded = pathmod.normcase(text)
|
||||
prefix = _plain(resolved_root, pathmod)
|
||||
if not prefix.endswith(sep):
|
||||
prefix += sep
|
||||
if len(folded) != len(text) or not folded.startswith(prefix):
|
||||
return None
|
||||
relative = text[len(prefix) :]
|
||||
if any(part in ("", ".", "..") for part in relative.split(sep)):
|
||||
return None
|
||||
return relative
|
||||
|
||||
|
||||
def insert_entry(files: list, entry: tuple, max_files: int) -> list | None:
|
||||
"""A new listing with *entry* ``(path, mtime)`` where a walk would list it.
|
||||
|
||||
*files* is newest first; an entry for the same path is replaced. Returns
|
||||
*files* itself when it already holds *entry*, and None when the place is
|
||||
not exact: another file has the same mtime (a walk orders those by its
|
||||
own visit order), or the listing is at the walk's *max_files* cap.
|
||||
Blocking (one pass over *files*): run it in the executor.
|
||||
"""
|
||||
path, mtime = entry
|
||||
text = str(path)
|
||||
kept = files
|
||||
for index, (listed, listed_mtime) in enumerate(files):
|
||||
if str(listed) == text:
|
||||
if listed_mtime == mtime:
|
||||
return files
|
||||
kept = files[:index] + files[index + 1 :]
|
||||
break
|
||||
else:
|
||||
if len(files) >= max_files:
|
||||
return None
|
||||
index = bisect_left(kept, -mtime, key=lambda item: -item[1])
|
||||
if index < len(kept) and kept[index][1] == mtime:
|
||||
return None
|
||||
return [*kept[:index], entry, *kept[index:]]
|
||||
|
||||
|
||||
# A listing that differs from the cached one by more entries than this is
|
||||
# rebuilt in full by the workspace caches (listing_delta).
|
||||
MAX_DELTA = 256
|
||||
|
||||
|
||||
def listing_delta(old: list, new: list, limit: int = MAX_DELTA):
|
||||
"""``(added, removed)`` entries from *old* to *new*, or None past *limit*.
|
||||
|
||||
A listing patched by ``insert_entry`` shares every unchanged entry object
|
||||
with the one it came from, so entries are told apart by identity: a walk
|
||||
builds new tuples, shares none and is rebuilt in full. *old* must still
|
||||
be alive (the
|
||||
caller's cache holds it), so no identity was reused.
|
||||
"""
|
||||
if abs(len(new) - len(old)) > limit:
|
||||
return None
|
||||
old_ids = set(map(id, old))
|
||||
added = []
|
||||
for entry in new:
|
||||
if id(entry) not in old_ids:
|
||||
added.append(entry)
|
||||
if len(added) > limit:
|
||||
return None
|
||||
if new and len(added) == len(new):
|
||||
return None # nothing shared: a walk
|
||||
if len(old) + len(added) == len(new):
|
||||
return added, []
|
||||
new_ids = set(map(id, new))
|
||||
return added, [entry for entry in old if id(entry) not in new_ids]
|
||||
|
||||
|
||||
def rewalk_interval(last_duration) -> float:
|
||||
"""Seconds from a walk's start to the next background walk of its root."""
|
||||
scaled = REWALK_FACTOR * (last_duration or 0.0)
|
||||
return min(REWALK_CAP, max(REWALK_FLOOR, scaled))
|
||||
|
||||
|
||||
def root_contains(resolved_root: str, path, pathmod=os.path) -> bool:
|
||||
"""True when the resolved *path* lies under *resolved_root* (no file I/O).
|
||||
|
||||
*resolved_root* is already ``normcase``d; *path* is folded the same way,
|
||||
so Windows matches case-insensitively and with either separator. The
|
||||
match ends at a separator: ``/out2/x.png`` is not under ``/out``.
|
||||
"""
|
||||
target = _plain(pathmod.normcase(str(path)), pathmod)
|
||||
prefix = _plain(resolved_root, pathmod)
|
||||
if not prefix.endswith(pathmod.sep):
|
||||
prefix += pathmod.sep
|
||||
return target.startswith(prefix)
|
||||
|
||||
|
||||
def _plain(path: str, pathmod) -> str:
|
||||
"""*path* (normcased or not) without a Windows extended-length prefix.
|
||||
|
||||
``realpath`` may keep ``\\\\?\\`` on a file past MAX_PATH while its shorter
|
||||
root resolves without it.
|
||||
"""
|
||||
if pathmod.sep != "\\":
|
||||
return path
|
||||
if path[:8].lower() == "\\\\?\\unc\\":
|
||||
return "\\\\" + path[8:]
|
||||
if path.startswith("\\\\?\\"):
|
||||
return path[4:]
|
||||
return path
|
||||
|
||||
|
||||
def resolve_root(root) -> str:
|
||||
"""``normcase(realpath(root))``, the spelling monitor paths are matched in.
|
||||
|
||||
File I/O (realpath follows links): call it in the executor.
|
||||
"""
|
||||
return os.path.normcase(os.path.realpath(str(root)))
|
||||
|
||||
|
||||
def same_listing(old, new) -> bool:
|
||||
"""True when two listings hold the same ``(path, mtime)`` pairs in order."""
|
||||
if old is None or new is None:
|
||||
return False
|
||||
return old is new or old == new
|
||||
|
||||
|
||||
def walk_root(
|
||||
scan_fn: Callable[[PurePath], list], root, previous_files
|
||||
) -> tuple[list, frozenset, frozenset, str]:
|
||||
"""The single executor job of a walk.
|
||||
|
||||
Returns ``(files, folders, with_ancestors, resolved_root)``.
|
||||
|
||||
Opens the root first, so a missing or unreadable root raises ``OSError``
|
||||
instead of listing nothing. Returns *previous_files* itself when nothing
|
||||
changed, which keeps the identity-keyed caches built on it warm.
|
||||
"""
|
||||
with os.scandir(root):
|
||||
pass
|
||||
files = scan_fn(root)
|
||||
if same_listing(previous_files, files):
|
||||
files = previous_files
|
||||
folders, with_ancestors = folder_sets(root, files)
|
||||
return files, folders, with_ancestors, resolve_root(root)
|
||||
@@ -5,6 +5,7 @@ import base64
|
||||
import binascii
|
||||
from functools import wraps
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Callable
|
||||
@@ -20,16 +21,23 @@ from .history_current import (
|
||||
_OpenedCurrentMedia,
|
||||
_current_media_response,
|
||||
_prepare_current_media,
|
||||
accepts_jxl,
|
||||
_reauthorize_current_target,
|
||||
candidate_locator,
|
||||
resolve_current_candidate,
|
||||
snapshot_current_output_scope,
|
||||
)
|
||||
from .inspection_service import CurrentFileChanged, inspect_current_file
|
||||
from .inspection_service import (
|
||||
CurrentFileChanged,
|
||||
image_tags_reader,
|
||||
inspect_current_file,
|
||||
)
|
||||
from .images import image_category_key
|
||||
|
||||
try:
|
||||
from ...database.execution_history import HistoryEpochChanged
|
||||
from ...database.history_gallery import read_history_gallery
|
||||
from ...database.run_attribution import recorded_output_lookup
|
||||
from ...database.history_reads import (
|
||||
HistoryCaptureChanged,
|
||||
OutputPosition,
|
||||
@@ -37,10 +45,15 @@ try:
|
||||
WindowPosition,
|
||||
)
|
||||
from ...utils.history_read_fence import HistoryReadFence
|
||||
from ...utils.inspection_response import encode_history_inspection_response
|
||||
from ...utils.inspection_response import (
|
||||
OWN_CATEGORY_UNREAD,
|
||||
apply_own_category,
|
||||
encode_history_inspection_response,
|
||||
)
|
||||
except ImportError:
|
||||
from database.execution_history import HistoryEpochChanged
|
||||
from database.history_gallery import read_history_gallery
|
||||
from database.run_attribution import recorded_output_lookup
|
||||
from database.history_reads import (
|
||||
HistoryCaptureChanged,
|
||||
OutputPosition,
|
||||
@@ -48,7 +61,11 @@ except ImportError:
|
||||
WindowPosition,
|
||||
)
|
||||
from utils.history_read_fence import HistoryReadFence
|
||||
from utils.inspection_response import encode_history_inspection_response
|
||||
from utils.inspection_response import (
|
||||
OWN_CATEGORY_UNREAD,
|
||||
apply_own_category,
|
||||
encode_history_inspection_response,
|
||||
)
|
||||
|
||||
|
||||
_RUN_MAX_BYTES = 8 * 1024 * 1024
|
||||
@@ -142,6 +159,18 @@ def _query(request: web.Request) -> None:
|
||||
raise ValueError("Duplicate history query parameter")
|
||||
|
||||
|
||||
def _jxl_flag(request: web.Request) -> bool:
|
||||
"""The cover's ``jxl`` flag (#317): ``1`` when the workspace decoded a
|
||||
JPEG XL image (a browser may not list ``image/jxl`` in Accept), ``0`` or
|
||||
absent otherwise; anything else is a bad request."""
|
||||
value = request.query.get("jxl")
|
||||
if value is None:
|
||||
return False
|
||||
if value not in ("0", "1"):
|
||||
raise ValueError("Invalid jxl flag")
|
||||
return value == "1"
|
||||
|
||||
|
||||
def _page_limit(request: web.Request) -> int:
|
||||
limit = _decimal(request.query.get("limit", "40"), minimum=1)
|
||||
if limit > 100:
|
||||
@@ -662,6 +691,32 @@ class HistoryRoutesMixin:
|
||||
) -> web.Response:
|
||||
return await self._history_current(request, "inspection")
|
||||
|
||||
def _history_own_category(self, reference: dict, scope):
|
||||
"""The recorded output's own category (#332), inside the snapshot.
|
||||
|
||||
The path is the symlink-resolved output folder plus the receipt's
|
||||
subfolder and filename, keyed by ``image_category_key`` over the
|
||||
configured output sources as Images keys it. A temp output or a path
|
||||
no source holds has none. A failed read is OWN_CATEGORY_UNREAD.
|
||||
"""
|
||||
if (
|
||||
reference.get("folder_type") != "output"
|
||||
or scope.canonical_output is None
|
||||
or candidate_locator(reference, scope) is None
|
||||
):
|
||||
return None
|
||||
path = os.path.join(
|
||||
str(scope.canonical_output), reference["subfolder"], reference["filename"]
|
||||
)
|
||||
try:
|
||||
key = image_category_key(self._get_all_output_dirs(), path)
|
||||
return None if key is None else self.db.image_category(*key)
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"History image category could not be read", exc_info=True
|
||||
)
|
||||
return OWN_CATEGORY_UNREAD
|
||||
|
||||
async def _history_current(
|
||||
self, request: web.Request, kind: str
|
||||
) -> web.StreamResponse:
|
||||
@@ -686,6 +741,7 @@ class HistoryRoutesMixin:
|
||||
if scope.current_scope != current_scope:
|
||||
raise _current_stale()
|
||||
result = db.get_history_occurrence(run_id, node, ordinal)
|
||||
own = None
|
||||
if result is not None:
|
||||
receipt = result["receipt"]
|
||||
path = candidate_locator(receipt["reference"], scope)
|
||||
@@ -693,6 +749,10 @@ class HistoryRoutesMixin:
|
||||
db.model.get_connection(),
|
||||
(path,) if path is not None else (),
|
||||
)
|
||||
if kind == "inspection":
|
||||
own = self._history_own_category(
|
||||
receipt["reference"], scope
|
||||
)
|
||||
finally:
|
||||
db.model.close()
|
||||
if result is None:
|
||||
@@ -711,9 +771,27 @@ class HistoryRoutesMixin:
|
||||
resolved.canonical_target,
|
||||
source_label=resolved.public["source_label"],
|
||||
saved=saved.get("saved"),
|
||||
library=saved.get(
|
||||
"library",
|
||||
{"status": "unavailable" if saved_unavailable else "none"},
|
||||
library=apply_own_category(
|
||||
saved.get(
|
||||
"library",
|
||||
{
|
||||
"status": (
|
||||
"unavailable" if saved_unavailable else "none"
|
||||
)
|
||||
},
|
||||
),
|
||||
own,
|
||||
),
|
||||
# The file's own image tags, keyed as the Tag dialog
|
||||
# keys them: the normcased realpath (#457).
|
||||
image_tags=image_tags_reader(
|
||||
db,
|
||||
str(resolved.canonical_target),
|
||||
self.logger,
|
||||
close=True,
|
||||
),
|
||||
recorded=recorded_output_lookup(
|
||||
db, close=True, reference=receipt["reference"]
|
||||
),
|
||||
)
|
||||
_reauthorize_current_target(resolved, scope)
|
||||
@@ -734,7 +812,10 @@ class HistoryRoutesMixin:
|
||||
raise HistoryReadError(422, "response-limit") from None
|
||||
else:
|
||||
prepared = _prepare_current_media(
|
||||
resolved, scope, thumbnail=kind == "thumbnail"
|
||||
resolved,
|
||||
scope,
|
||||
thumbnail=kind == "thumbnail",
|
||||
**media_options,
|
||||
)
|
||||
_check_current(scope, self, fence, epoch, generation)
|
||||
return prepared
|
||||
@@ -761,25 +842,45 @@ class HistoryRoutesMixin:
|
||||
_check_current(scope, self, fence, epoch, generation)
|
||||
raise
|
||||
|
||||
pending = asyncio.create_task(self._run_in_executor(project))
|
||||
try:
|
||||
prepared = await asyncio.shield(pending)
|
||||
except asyncio.CancelledError:
|
||||
pending.add_done_callback(_close_prepared_result)
|
||||
raise
|
||||
except (
|
||||
HistoryReadError,
|
||||
HistoryEpochChanged,
|
||||
sqlite3.Error,
|
||||
OSError,
|
||||
ValueError,
|
||||
):
|
||||
raise
|
||||
except Exception as error:
|
||||
self.logger.warning(
|
||||
"Current history media unavailable (%s)", type(error).__name__
|
||||
)
|
||||
raise _current_unavailable("temporary_unavailable") from None
|
||||
async def execute():
|
||||
pending = asyncio.create_task(self._run_in_executor(project))
|
||||
try:
|
||||
return await asyncio.shield(pending)
|
||||
except asyncio.CancelledError:
|
||||
pending.add_done_callback(_close_prepared_result)
|
||||
raise
|
||||
except (
|
||||
HistoryReadError,
|
||||
HistoryEpochChanged,
|
||||
sqlite3.Error,
|
||||
OSError,
|
||||
ValueError,
|
||||
):
|
||||
raise
|
||||
except Exception as error:
|
||||
self.logger.warning(
|
||||
"Current history media unavailable (%s)", type(error).__name__
|
||||
)
|
||||
raise _current_unavailable("temporary_unavailable") from None
|
||||
|
||||
# A missing preview (#249) is noted on the first read, made through
|
||||
# the serve route's admission and single flight, then read again. A
|
||||
# video whose preview cannot be made is the placeholder frame.
|
||||
needed = []
|
||||
media_options = {}
|
||||
if kind == "thumbnail":
|
||||
media_options = {
|
||||
"ensure_preview": lambda *preview: needed.append(preview),
|
||||
"placeholder": self._video_placeholder_bytes(),
|
||||
"accepts_jxl": _jxl_flag(request) or accepts_jxl(request),
|
||||
}
|
||||
prepared = await execute()
|
||||
if needed:
|
||||
if not isinstance(prepared, bytes):
|
||||
prepared.close()
|
||||
await self._admit_preview(*needed[0])
|
||||
media_options["ensure_preview"] = None
|
||||
prepared = await execute()
|
||||
if isinstance(prepared, bytes):
|
||||
return _success_response(prepared)
|
||||
return await _current_media_response(request, prepared)
|
||||
|
||||
@@ -5,6 +5,7 @@ from dataclasses import dataclass
|
||||
import errno
|
||||
import hashlib
|
||||
import hmac
|
||||
import io
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
@@ -14,7 +15,10 @@ from typing import BinaryIO
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import thumbnail_settings
|
||||
from .images import _canonical, _public_path, SERVABLE_EXTENSIONS, VIDEO_EXTENSIONS
|
||||
from .images import in_trash_below_roots
|
||||
from .images import existing_media_thumbnail, media_thumbnail_rel
|
||||
|
||||
try:
|
||||
from ...utils.execution_outputs import (
|
||||
@@ -195,6 +199,9 @@ def resolve_current_candidate(
|
||||
if namespace is None or not namespace.is_dir():
|
||||
return _unavailable("root_unavailable")
|
||||
target = _canonical(lexical)
|
||||
if in_trash_below_roots(target, [namespace]):
|
||||
# Every kind and the outputs listing come through here (#543).
|
||||
return _unavailable("missing")
|
||||
if not _authorized(target, namespace, scope):
|
||||
return _unavailable("not_authorized")
|
||||
if target.suffix.lower() not in SERVABLE_EXTENSIONS:
|
||||
@@ -238,14 +245,23 @@ def resolve_current_candidate(
|
||||
|
||||
|
||||
def _open_current_target(
|
||||
target: Path, namespace: Path, scope: CurrentOutputScope
|
||||
target: Path, namespace: Path, scope: CurrentOutputScope, preview: bool = False
|
||||
) -> BinaryIO:
|
||||
"""Open only the canonical target and verify authority against the actual handle."""
|
||||
"""Open only the canonical target and verify authority against the actual handle.
|
||||
|
||||
A *preview* (a file under ``thumbnails/``) may also be JPEG XL (#317)."""
|
||||
if _canonical(target) != target:
|
||||
raise CurrentLocationUnavailable("not_authorized", stale=True)
|
||||
if in_trash_below_roots(target, [namespace]):
|
||||
# Also at the open: the cover chooses its preview after the candidate
|
||||
# check, and a preview can be a link into a trash folder (#543).
|
||||
raise CurrentLocationUnavailable("missing")
|
||||
if not _authorized(target, namespace, scope):
|
||||
raise CurrentLocationUnavailable("not_authorized")
|
||||
if target.suffix.lower() not in SERVABLE_EXTENSIONS:
|
||||
suffix = target.suffix.lower()
|
||||
if suffix not in SERVABLE_EXTENSIONS and not (
|
||||
preview and suffix in thumbnail_settings.THUMBNAIL_EXTENSIONS
|
||||
):
|
||||
raise CurrentLocationUnavailable("unsupported_media")
|
||||
flags = (
|
||||
os.O_RDONLY
|
||||
@@ -309,34 +325,107 @@ class _OpenedCurrentMedia:
|
||||
self.handle.close()
|
||||
|
||||
|
||||
def _prepare_current_media(
|
||||
resolved: ResolvedCurrentCandidate, scope: CurrentOutputScope, *, thumbnail: bool
|
||||
) -> _OpenedCurrentMedia:
|
||||
def _preview_writable(root: Path, preview: Path) -> bool:
|
||||
"""A preview may be written: ``thumbnails`` and every link segment of the
|
||||
preview's folder resolve inside *root* (the serve route's rule, #249)."""
|
||||
folder = root / "thumbnails"
|
||||
return _canonical(folder).is_relative_to(_canonical(root)) and _canonical(
|
||||
preview
|
||||
).is_relative_to(_canonical(folder))
|
||||
|
||||
|
||||
def _is_jxl(path) -> bool:
|
||||
return Path(path).suffix.lower() == ".jxl"
|
||||
|
||||
|
||||
def _preview_without_jxl(root: Path, relative: Path) -> Path | None:
|
||||
"""The first existing preview *relative* owns that is not JPEG XL."""
|
||||
return next(
|
||||
(
|
||||
path
|
||||
for path in thumbnail_settings.owned_thumbnails(root, relative)
|
||||
if not _is_jxl(path)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def accepts_jxl(request) -> bool:
|
||||
"""Whether the request's ``Accept`` lists ``image/jxl``."""
|
||||
accept = request.headers.get("Accept", "")
|
||||
return any(
|
||||
part.split(";", 1)[0].strip().lower() == "image/jxl"
|
||||
for part in accept.split(",")
|
||||
)
|
||||
|
||||
|
||||
def _placeholder_media(placeholder: bytes) -> "_OpenedCurrentMedia":
|
||||
return _OpenedCurrentMedia(io.BytesIO(placeholder), len(placeholder), "image/jpeg")
|
||||
|
||||
|
||||
def _cover_target(resolved, ensure_preview, placeholder, accepts_jxl):
|
||||
"""What the cover opens: a preview path, the placeholder media, or None
|
||||
for the original (see ``_prepare_current_media``)."""
|
||||
target = resolved.canonical_target
|
||||
relative = target.relative_to(resolved.gallery_root)
|
||||
is_video = target.suffix.lower() in VIDEO_EXTENSIONS
|
||||
# #317: any preview the file owns (the current format first); a missing
|
||||
# one is made under the current format's name.
|
||||
preview = existing_media_thumbnail(resolved.gallery_root, relative)
|
||||
if preview is not None and not accepts_jxl and _is_jxl(preview):
|
||||
preview = _preview_without_jxl(resolved.gallery_root, relative)
|
||||
if preview is None:
|
||||
if is_video and placeholder is not None:
|
||||
return _placeholder_media(placeholder)
|
||||
return None
|
||||
return _canonical(preview)
|
||||
if preview is not None:
|
||||
return _canonical(preview)
|
||||
preview = resolved.gallery_root / media_thumbnail_rel(relative)
|
||||
if ensure_preview is not None and _preview_writable(resolved.gallery_root, preview):
|
||||
ensure_preview(target, preview, is_video)
|
||||
if is_video and placeholder is not None:
|
||||
return _placeholder_media(placeholder)
|
||||
return None
|
||||
|
||||
|
||||
def _prepare_current_media(
|
||||
resolved: ResolvedCurrentCandidate,
|
||||
scope: CurrentOutputScope,
|
||||
*,
|
||||
thumbnail: bool,
|
||||
ensure_preview=None,
|
||||
placeholder: bytes | None = None,
|
||||
accepts_jxl: bool = True,
|
||||
) -> _OpenedCurrentMedia:
|
||||
"""Open the current file, or its preview for ``thumbnail``.
|
||||
|
||||
A missing preview that may be written is handed to
|
||||
``ensure_preview(src, dst, is_video)`` (the route makes it and reads
|
||||
again, #249). Without one an image's original is opened and a video is
|
||||
the ``placeholder`` JPEG, never the video inside an ``<img>``.
|
||||
|
||||
The cover's URL names no format, so a JPEG XL preview goes only to a
|
||||
browser whose ``Accept`` lists ``image/jxl`` (*accepts_jxl*, review M3).
|
||||
Another one gets the file's preview in another format when there is one,
|
||||
else the original image or a video's placeholder: never a broken image.
|
||||
"""
|
||||
target = resolved.canonical_target
|
||||
is_preview = False
|
||||
if thumbnail:
|
||||
relative = target.relative_to(resolved.gallery_root)
|
||||
extension = (
|
||||
".jpg"
|
||||
if target.suffix.lower() in VIDEO_EXTENSIONS
|
||||
else target.suffix.lower()
|
||||
)
|
||||
preview = (
|
||||
resolved.gallery_root
|
||||
/ "thumbnails"
|
||||
/ f"{relative.with_suffix('').as_posix()}_thumb{extension}"
|
||||
)
|
||||
try:
|
||||
preview.stat()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
else:
|
||||
target = _canonical(preview)
|
||||
handle = _open_current_target(target, resolved.namespace, scope)
|
||||
chosen = _cover_target(resolved, ensure_preview, placeholder, accepts_jxl)
|
||||
if isinstance(chosen, _OpenedCurrentMedia):
|
||||
return chosen
|
||||
if chosen is not None:
|
||||
target, is_preview = chosen, True
|
||||
handle = _open_current_target(target, resolved.namespace, scope, is_preview)
|
||||
try:
|
||||
return _OpenedCurrentMedia(
|
||||
handle,
|
||||
os.fstat(handle.fileno()).st_size,
|
||||
mimetypes.guess_type(target.name)[0] or "application/octet-stream",
|
||||
thumbnail_settings.content_type(target)
|
||||
or mimetypes.guess_type(target.name)[0]
|
||||
or "application/octet-stream",
|
||||
)
|
||||
except BaseException:
|
||||
handle.close()
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
"""Image category routes (#332): an image file's own category.
|
||||
|
||||
``PUT /prompt_manager/images/category`` sets the own category of up to 500
|
||||
listed files; ``DELETE`` removes it (listed or not, so a missing file's
|
||||
category can still be removed); ``GET /prompt_manager/images/categories``
|
||||
lists the own-category values of files the sources list. Targets are the
|
||||
Images listing identity (``{root_index, root_scope, path}``, checked by
|
||||
``media_target``) of an output source; LoRA folders are not sources here.
|
||||
Instead of ``items`` a request may name ``filter``, the Images listing's own
|
||||
query ("Select all N matching", #392), resolved now up to 500 images.
|
||||
Each file is stored under its file key (``images.image_category_key``).
|
||||
These routes never read or write prompts, recipes or image links.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .image_tag_jobs import MAX_JOB_TARGETS, _parse_targets
|
||||
from .images import (
|
||||
_favorite_relative_key,
|
||||
_favorite_root_key,
|
||||
_forbidden,
|
||||
_json_body_or_error,
|
||||
image_category_key,
|
||||
)
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
from ...database.image_categories import MAX_CATEGORY_LENGTH, MAX_KEY_LENGTH
|
||||
from ...database.models import normalize_image_path
|
||||
from ...utils.validators import validate_category
|
||||
except ImportError:
|
||||
from database.image_categories import MAX_CATEGORY_LENGTH, MAX_KEY_LENGTH
|
||||
from database.models import normalize_image_path
|
||||
from utils.validators import validate_category
|
||||
|
||||
CATEGORY_MEDIA_MESSAGE = "Only media files can have a category"
|
||||
_PUT_FIELDS = frozenset({"category"})
|
||||
_DELETE_FIELDS = frozenset()
|
||||
TOO_MANY = f"Select at most {MAX_JOB_TARGETS} images."
|
||||
SOURCE_CHANGED_MESSAGE = "The sources changed. Refresh the images and try again."
|
||||
WRITE_FAILED = "The category could not be changed."
|
||||
READ_FAILED = "Image categories could not be read."
|
||||
|
||||
|
||||
def _coded(status: int, code: str, message: str) -> web.Response:
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": message},
|
||||
status=status,
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
|
||||
def _category_value(raw) -> tuple[str | None, web.Response | None]:
|
||||
"""The stripped category, or a 400 (an empty value is a 400: use DELETE)."""
|
||||
if not isinstance(raw, str):
|
||||
return None, bad_request("category must be a string")
|
||||
value = raw.strip()
|
||||
if not value:
|
||||
return None, bad_request("category must not be empty")
|
||||
if len(value) > MAX_CATEGORY_LENGTH:
|
||||
return None, bad_request(
|
||||
f"Category cannot exceed {MAX_CATEGORY_LENGTH} characters"
|
||||
)
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
validate_category(value)
|
||||
except UnicodeEncodeError:
|
||||
return None, bad_request("category must be valid text")
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
return value, None
|
||||
|
||||
|
||||
class _KeyRefused(Exception):
|
||||
"""A target has no storable file key (outside every source, or too long)."""
|
||||
|
||||
def __init__(self, message: str = "Access denied"):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
|
||||
|
||||
def _file_key(roots, root, relative) -> tuple[str, str]:
|
||||
"""The target's file key (blocking: resolves the sources)."""
|
||||
key = image_category_key(roots, Path(root) / relative)
|
||||
if key is None:
|
||||
raise _KeyRefused()
|
||||
if max(len(key[0]), len(key[1])) > MAX_KEY_LENGTH:
|
||||
raise _KeyRefused("Path is too long")
|
||||
return key
|
||||
|
||||
|
||||
def _nested_sources(root: Path, sources) -> list[tuple[str, str]]:
|
||||
"""``[(relative key prefix, source root key)]`` of sources inside *root*.
|
||||
|
||||
Deepest first, so the first prefix that matches is the file's source.
|
||||
"""
|
||||
base = os.path.join(os.path.normcase(os.path.realpath(str(root))), "")
|
||||
nested = []
|
||||
for source in sources:
|
||||
real = os.path.realpath(str(source))
|
||||
if not os.path.normcase(real).startswith(base):
|
||||
continue
|
||||
relative = os.path.relpath(real, os.path.realpath(str(root)))
|
||||
nested.append(
|
||||
(
|
||||
_favorite_relative_key(relative.replace(os.sep, "/")),
|
||||
_favorite_root_key(source),
|
||||
)
|
||||
)
|
||||
nested.sort(key=lambda item: item[0].count("/"), reverse=True)
|
||||
return nested
|
||||
|
||||
|
||||
def _source_prefix(relative_key: str, nested) -> str | None:
|
||||
"""The prefix of the deepest nested source holding *relative_key*."""
|
||||
for prefix, _source_key in nested:
|
||||
if relative_key.startswith(prefix + "/"):
|
||||
return prefix
|
||||
return None
|
||||
|
||||
|
||||
class ImageCategoryRoutesMixin:
|
||||
"""Routes for images' own categories (#332)."""
|
||||
|
||||
def _register_image_category_routes(self, routes):
|
||||
# Fixed paths beside DELETE /prompt_manager/images/{image_id}, as the
|
||||
# favorite routes are. Registered after it, they still win: aiohttp
|
||||
# (3.10+) looks fixed paths up before dynamic resources
|
||||
# (test_delete_category_never_reaches_delete_image).
|
||||
@routes.put("/prompt_manager/images/category")
|
||||
async def set_image_category_route(request):
|
||||
return await self.set_image_category(request)
|
||||
|
||||
@routes.delete("/prompt_manager/images/category")
|
||||
async def clear_image_category_route(request):
|
||||
return await self.clear_image_category(request)
|
||||
|
||||
@routes.get("/prompt_manager/images/categories")
|
||||
async def list_image_categories_route(request):
|
||||
return await self.list_image_categories(request)
|
||||
|
||||
# -- reading ---------------------------------------------------------------
|
||||
|
||||
def _own_category_map(self, sources, listings) -> dict:
|
||||
"""``{normalized listed path: own category}`` for the listed files (blocking).
|
||||
|
||||
One primary-key range read per source; each stored row is matched to
|
||||
the listing of every selected root that holds its source, so the work
|
||||
follows the stored rows, not the files. A file under a nested source
|
||||
is keyed by the deepest one (``image_category_key``). A row whose file
|
||||
is gone matches nothing. A failed read raises.
|
||||
"""
|
||||
sources = list(sources)
|
||||
rows = self.db.own_categories_of(_favorite_root_key(s) for s in sources)
|
||||
if not rows:
|
||||
return {}
|
||||
by_source: dict[str, dict[str, str]] = {}
|
||||
for (root_key, relative), category in rows.items():
|
||||
by_source.setdefault(root_key, {})[relative] = category
|
||||
found = {}
|
||||
for _index, root, files in listings:
|
||||
listing = self._favorite_listing(root, files)
|
||||
root_key = _favorite_root_key(root)
|
||||
# normalize_image_path(root / listed) without a call per file: the
|
||||
# listed spellings are clean relative POSIX paths. Windows abspath
|
||||
# also drops a component's trailing dots and spaces, so it keeps
|
||||
# the call there.
|
||||
base = os.path.join(normalize_image_path(str(root)), "")
|
||||
|
||||
def normalized(listed, base=base, root=root):
|
||||
if os.name == "nt":
|
||||
return normalize_image_path(str(root / listed))
|
||||
return base + os.path.normcase(listed.replace("/", os.sep))
|
||||
|
||||
nested = [
|
||||
item for item in _nested_sources(root, sources) if item[1] != root_key
|
||||
]
|
||||
for relative, category in by_source.get(root_key, {}).items():
|
||||
listed = listing.get(relative)
|
||||
if listed is not None and _source_prefix(relative, nested) is None:
|
||||
found[normalized(listed)] = category
|
||||
for prefix, source_key in nested:
|
||||
for relative, category in by_source.get(source_key, {}).items():
|
||||
full = f"{prefix}/{relative}"
|
||||
listed = listing.get(full)
|
||||
if listed is not None and _source_prefix(full, nested) == prefix:
|
||||
found[normalized(listed)] = category
|
||||
return found
|
||||
|
||||
async def _source_listings(self, roots) -> list[tuple]:
|
||||
return [
|
||||
(index, root, await self._get_gallery_files(root))
|
||||
for index, root in enumerate(roots)
|
||||
]
|
||||
|
||||
async def list_image_categories(self, request):
|
||||
"""GET: the own-category values of listed files, sorted."""
|
||||
try:
|
||||
roots = list(self._get_all_output_dirs())
|
||||
listings = await self._source_listings(roots)
|
||||
found = await self._run_in_executor(self._own_category_map, roots, listings)
|
||||
except Exception:
|
||||
self.logger.exception("Image categories read failed")
|
||||
return _coded(500, "categories-failed", READ_FAILED)
|
||||
return web.json_response(
|
||||
{"success": True, "categories": sorted(set(found.values()))},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
# -- writing ---------------------------------------------------------------
|
||||
|
||||
async def _filter_targets(self, raw):
|
||||
"""``(targets, None)`` for an Images listing filter, else ``(None, error)``.
|
||||
|
||||
"Select all N matching" (#392) sends the listing's own query; it is
|
||||
resolved now, up to 500 images (a 400 ``too-many-images`` above).
|
||||
"""
|
||||
from .images import ListingSourceChanged, parse_listing_filter
|
||||
|
||||
try:
|
||||
selection = await self.resolve_listing_filter(
|
||||
parse_listing_filter(raw), MAX_JOB_TARGETS
|
||||
)
|
||||
except ListingSourceChanged:
|
||||
return None, _coded(409, "source-changed", SOURCE_CHANGED_MESSAGE)
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
if selection.truncated:
|
||||
return None, _coded(400, "too-many-images", TOO_MANY)
|
||||
return tuple(selection.targets), None
|
||||
|
||||
async def _category_request(self, request, fields: frozenset, data=None):
|
||||
"""``(data, targets, roots, None)`` or ``(None, None, None, error)``.
|
||||
|
||||
The images are ``items`` (1 to 500 targets) or ``filter`` (all images
|
||||
an Images listing filter admits), besides *fields*. *data* is the body
|
||||
when the caller has already read it (a body is read once).
|
||||
"""
|
||||
if data is None:
|
||||
data, error = await _json_body_or_error(request)
|
||||
if error is not None:
|
||||
return None, None, None, error
|
||||
names = set(data) - fields
|
||||
if set(data) & fields != fields or names not in ({"items"}, {"filter"}):
|
||||
return (
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
bad_request(
|
||||
"Request must contain "
|
||||
+ ", ".join(sorted(fields))
|
||||
+ (" and " if fields else "")
|
||||
+ "either items or filter"
|
||||
),
|
||||
)
|
||||
if "filter" in data:
|
||||
targets, error = await self._filter_targets(data["filter"])
|
||||
if error is not None:
|
||||
return None, None, None, error
|
||||
return data, targets, list(self._get_all_output_dirs()), None
|
||||
items = data["items"]
|
||||
if not isinstance(items, list) or not 1 <= len(items) <= MAX_JOB_TARGETS:
|
||||
return (
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
bad_request(f"items must list 1 to {MAX_JOB_TARGETS} images"),
|
||||
)
|
||||
targets, error = _parse_targets(items, media_message=CATEGORY_MEDIA_MESSAGE)
|
||||
if error is not None:
|
||||
return None, None, None, error
|
||||
try:
|
||||
for _index, _scope, relative in targets:
|
||||
relative.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
return None, None, None, bad_request("path must be valid text")
|
||||
roots = list(self._get_all_output_dirs())
|
||||
for index, scope in dict.fromkeys((t[0], t[1]) for t in targets):
|
||||
if index >= len(roots):
|
||||
return None, None, None, bad_request("Invalid gallery root")
|
||||
error = self._gallery_scope_error(
|
||||
{"root_scope": scope, "root": str(index)}, roots
|
||||
)
|
||||
if error is not None:
|
||||
if error.status == 409:
|
||||
error = _coded(409, "source-changed", SOURCE_CHANGED_MESSAGE)
|
||||
return None, None, None, error
|
||||
return data, targets, roots, None
|
||||
|
||||
def _listed_file_keys(self, roots, targets, files) -> tuple[list, list]:
|
||||
"""``(listed keys, unlisted keys)`` of the targets (blocking).
|
||||
|
||||
A target counts as listed when its source's listing holds it; its key
|
||||
then follows the listed spelling. Targets folding to one key count
|
||||
once, and a key that is listed through any target is listed.
|
||||
"""
|
||||
listings, listed, missing = {}, {}, {}
|
||||
for index, _scope, relative in targets:
|
||||
root = roots[index]
|
||||
if index not in listings:
|
||||
listings[index] = self._favorite_listing(root, files[index])
|
||||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||||
key = _file_key(roots, root, spelled or relative)
|
||||
(missing if spelled is None else listed)[key] = True
|
||||
return list(listed), [key for key in missing if key not in listed]
|
||||
|
||||
async def set_image_category(self, request):
|
||||
"""PUT ``{items, category}``: set the own category of listed files."""
|
||||
data, targets, roots, error = await self._category_request(request, _PUT_FIELDS)
|
||||
if error is not None:
|
||||
return error
|
||||
category, error = _category_value(data["category"])
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
files = {}
|
||||
for index in dict.fromkeys(t[0] for t in targets):
|
||||
files[index] = await self._get_gallery_files(roots[index])
|
||||
listed_keys, missing_keys = await self._run_in_executor(
|
||||
self._listed_file_keys, roots, targets, files
|
||||
)
|
||||
except _KeyRefused as refused:
|
||||
return _forbidden(refused.message)
|
||||
except Exception:
|
||||
self.logger.exception("Image category update failed")
|
||||
return _coded(500, "category-failed", WRITE_FAILED)
|
||||
try:
|
||||
if not listed_keys:
|
||||
return _coded(
|
||||
404,
|
||||
"files-not-found",
|
||||
"None of the selected images could be found.",
|
||||
)
|
||||
counts = await self._run_in_executor(
|
||||
self.db.set_image_categories, listed_keys, category
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Image category update failed")
|
||||
return _coded(500, "category-failed", WRITE_FAILED)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"category": category,
|
||||
"updated": counts["updated"],
|
||||
"unchanged": counts["unchanged"],
|
||||
"not_found": len(missing_keys),
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
async def clear_image_category(self, request):
|
||||
"""DELETE ``{items}``: remove the own category, listed or not."""
|
||||
_data, targets, roots, error = await self._category_request(
|
||||
request, _DELETE_FIELDS
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
keys = await self._run_in_executor(
|
||||
lambda: list(
|
||||
dict.fromkeys(_file_key(roots, roots[i], r) for i, _s, r in targets)
|
||||
)
|
||||
)
|
||||
cleared = await self._run_in_executor(self.db.clear_image_categories, keys)
|
||||
except _KeyRefused as refused:
|
||||
return _forbidden(refused.message)
|
||||
except Exception:
|
||||
self.logger.exception("Image category removal failed")
|
||||
return _coded(500, "category-failed", WRITE_FAILED)
|
||||
return web.json_response(
|
||||
{"success": True, "cleared": cleared},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
@@ -0,0 +1,521 @@
|
||||
"""Delete selected images: remove them from the library, or trash them (#480).
|
||||
|
||||
``POST /prompt_manager/images/delete/preview {items | filter}`` reviews the
|
||||
selection for both modes at once and answers a reviewed plan (the System ›
|
||||
Maintenance contract, ``py/api/maintenance.py``): the token, the counts of
|
||||
the records each file loses, the refused files with a coded reason (both
|
||||
modes, and the trash-only ones), the prompts that lose their last image and
|
||||
the safety backup. It writes nothing. ``items`` are Images listing identities
|
||||
(at most 500); ``filter`` is "Select all N matching". Each target is matched
|
||||
to its entry in the current listing (``not-listed`` otherwise), so every
|
||||
later step uses the walked spelling, never the request's text.
|
||||
|
||||
``POST /prompt_manager/maintenance/apply {token, mode}`` runs the plan as the
|
||||
``image_delete`` job (admission ``records``, not cancellable): the sources
|
||||
are checked, the promised safety backup is taken, then each file is checked
|
||||
again (containment and fingerprint) and
|
||||
|
||||
- ``mode="library"``: its records are deleted and its removal mark written
|
||||
in one transaction; the file stays on disk, hidden;
|
||||
- ``mode="trash"``: it is moved to the trash first (``utils/trash.py``); a
|
||||
refused move leaves the file and its records alone; after a move its
|
||||
records are deleted.
|
||||
|
||||
No path here unlinks a media file. A failed records write stops the job; the
|
||||
files after it are not attempted.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat as stat_module
|
||||
from pathlib import Path
|
||||
|
||||
from .admin import DELETABLE_MEDIA_EXTENSIONS, _remove_thumbnail
|
||||
from .duplicate_review import (
|
||||
CHANGED_CONTENT,
|
||||
GONE,
|
||||
LINK,
|
||||
OUTSIDE_SOURCE,
|
||||
UNREADABLE,
|
||||
file_problem,
|
||||
)
|
||||
from .images import _favorite_relative_key, _favorite_root_key
|
||||
from .jobs import IMAGE_DELETE_GROUP_NAMES, IMAGE_DELETE_JOB_KIND, JobBusy
|
||||
from .job_routes import busy_response
|
||||
from .maintenance import MAINTENANCE_PREVIEW_LEASE, _fail_job, _failure, backup_note
|
||||
|
||||
try:
|
||||
from ...database import image_removal_schema
|
||||
from ...database.image_removal import mark_hides
|
||||
from ...database.annotation_schema import MAX_LOCATOR_LENGTH
|
||||
from ...utils import trash
|
||||
from ...utils.media_folders import is_trash_folder
|
||||
except ImportError:
|
||||
from database import image_removal_schema
|
||||
from database.image_removal import mark_hides
|
||||
from database.annotation_schema import MAX_LOCATOR_LENGTH
|
||||
from utils import trash
|
||||
from utils.media_folders import is_trash_folder
|
||||
|
||||
IMAGE_DELETE = "delete_images"
|
||||
MODES = ("library", "trash")
|
||||
TITLES = {
|
||||
"library": "Delete images (remove from the library)",
|
||||
"trash": "Delete images (move to the trash)",
|
||||
}
|
||||
MAX_DETAIL_ROWS = 50
|
||||
# The system moved the file but its place in a trash could not be shown (#499).
|
||||
TRASH_UNCONFIRMED = "trash-unconfirmed"
|
||||
RECORDS_FAILED_CODE = "records-failed"
|
||||
|
||||
# Reason codes (design §4.4) and the sentence each shows.
|
||||
REASONS = {
|
||||
"not-listed": "No longer listed in Images.",
|
||||
"outside": "Outside the output folders.",
|
||||
"link": "A link, not a file.",
|
||||
"gone": "The file no longer exists.",
|
||||
"unreadable": "The file can't be read.",
|
||||
"thumbnail": "A generated preview, not an image.",
|
||||
"not-media": "Not a media file.",
|
||||
"trash-folder": "Inside a trash folder.",
|
||||
"path-too-long": "The path is too long.",
|
||||
"changed": "The file changed after the review.",
|
||||
trash.NO_TRASH: "This drive has no trash.",
|
||||
trash.NETWORK: "Network drives can't use the trash.",
|
||||
trash.TRASH_IN_ROOT: "This drive's trash is inside an output folder.",
|
||||
trash.LOCKED: "The file is open in another program. Close it and try again.",
|
||||
trash.FAILED: "The file could not be moved to the trash.",
|
||||
trash.NO_PERMISSION: (
|
||||
"PromptManager isn't allowed to move this file (no permission, or it is "
|
||||
"locked in Finder)."
|
||||
),
|
||||
trash.CLOUD_SYNC: (
|
||||
"Files in iCloud Drive or a cloud-synced folder can't be moved to the "
|
||||
"trash here."
|
||||
),
|
||||
TRASH_UNCONFIRMED: (
|
||||
"Moved by macOS, but its place in the trash could not be confirmed."
|
||||
),
|
||||
trash.WINDOWS_RECYCLE: (
|
||||
"Moving files to the Recycle Bin isn't available on Windows: files larger "
|
||||
"than the bin's capacity would be deleted permanently."
|
||||
),
|
||||
"not-attempted": "Not attempted: the job stopped after an error.",
|
||||
}
|
||||
_PROBLEM_CODES = {
|
||||
OUTSIDE_SOURCE: "outside",
|
||||
LINK: "link",
|
||||
GONE: "gone",
|
||||
UNREADABLE: "unreadable",
|
||||
CHANGED_CONTENT: "changed",
|
||||
}
|
||||
SOURCES_CHANGED = "The sources changed since the review; nothing was changed."
|
||||
RECORDS_FAILED = "The image records could not be removed; the job stopped."
|
||||
|
||||
|
||||
def lexical_problem(relative: str) -> str | None:
|
||||
"""The refusal code a root-relative POSIX path earns before any file access."""
|
||||
parts = relative.split("/")
|
||||
if "thumbnails" in parts[:-1]:
|
||||
return "thumbnail"
|
||||
if any(is_trash_folder(part) for part in parts[:-1]):
|
||||
return "trash-folder"
|
||||
if os.path.splitext(parts[-1])[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||||
return "not-media"
|
||||
return None
|
||||
|
||||
|
||||
def file_check(path, root, relative, *, expected=None):
|
||||
"""``(code, lstat)``: why *path* may not be deleted, or ``(None, lstat)``.
|
||||
|
||||
Text first (inside the root, not a preview or trash folder, a media
|
||||
file), then ``duplicate_review.file_problem``: a regular file, not a
|
||||
link, whose resolved path is still inside the resolved root. With
|
||||
*expected* (``{size, mtime_ns, inode}`` from the review) the file must
|
||||
still be those bytes. Every handle is closed when this returns.
|
||||
"""
|
||||
code = lexical_problem(relative)
|
||||
if code:
|
||||
return code, None
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return "gone", None
|
||||
except OSError:
|
||||
return "unreadable", None
|
||||
if getattr(info, "st_file_attributes", 0) & getattr(
|
||||
stat_module, "FILE_ATTRIBUTE_REPARSE_POINT", 0
|
||||
):
|
||||
return "link", info
|
||||
if expected is None:
|
||||
problem = file_problem(path, root, None, info.st_size, hash_check=False)
|
||||
else:
|
||||
problem = file_problem(
|
||||
path,
|
||||
root,
|
||||
None,
|
||||
expected["size"],
|
||||
hash_check=False,
|
||||
mtime_ns=expected["mtime_ns"],
|
||||
inode=expected["inode"],
|
||||
)
|
||||
if problem:
|
||||
return _PROBLEM_CODES.get(problem, "outside"), info
|
||||
if len(os.path.normcase(os.path.realpath(path))) > MAX_LOCATOR_LENGTH:
|
||||
return "path-too-long", info
|
||||
return None, info
|
||||
|
||||
|
||||
def _row(target, code):
|
||||
return {
|
||||
"path": target["rel"],
|
||||
"root_label": target["root_label"],
|
||||
"code": code,
|
||||
"detail": REASONS[code],
|
||||
}
|
||||
|
||||
|
||||
def _summary(rows):
|
||||
counts = {}
|
||||
for row in rows:
|
||||
counts[row["code"]] = counts.get(row["code"], 0) + 1
|
||||
return {"count": len(rows), "counts": counts, "rows": rows[:MAX_DETAIL_ROWS]}
|
||||
|
||||
|
||||
def _fingerprint(targets) -> str:
|
||||
material = [
|
||||
(t["root_index"], t["rel"], t["size"], t["mtime_ns"], t["inode"])
|
||||
for t in targets
|
||||
]
|
||||
return hashlib.sha256(json.dumps(material).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class ImageDeleteMixin:
|
||||
"""The Images Delete… review and its apply job (#480)."""
|
||||
|
||||
def _register_image_delete_routes(self, routes):
|
||||
# A fixed path beside the dynamic /images/{image_id} routes: aiohttp
|
||||
# (3.10+) looks fixed paths up first (see image_categories.py).
|
||||
@routes.post("/prompt_manager/images/delete/preview")
|
||||
async def preview_image_delete_route(request):
|
||||
return await self.preview_image_delete(request)
|
||||
|
||||
# ── review ────────────────────────────────────────────────────────
|
||||
|
||||
async def preview_image_delete(self, request):
|
||||
"""Review the selection under the maintenance preview lease."""
|
||||
try:
|
||||
async with self._jobs.lease(MAINTENANCE_PREVIEW_LEASE):
|
||||
return await self._preview_image_delete(request)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _preview_image_delete(self, request):
|
||||
_data, targets, roots, error = await self._category_request(
|
||||
request, frozenset()
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
db = self.db
|
||||
identity = self._database_identity(db)
|
||||
try:
|
||||
files = {}
|
||||
for index in dict.fromkeys(t[0] for t in targets):
|
||||
files[index] = await self._get_gallery_files(roots[index])
|
||||
review = await self._run_in_executor(
|
||||
self._review_image_delete, db, roots, targets, files
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Image delete review failed")
|
||||
return _failure("The review failed. Check server logs for details.", 500)
|
||||
return await self._plan_response(
|
||||
db,
|
||||
identity,
|
||||
IMAGE_DELETE,
|
||||
_fingerprint(review["targets"]),
|
||||
review["public"],
|
||||
{"targets": review["targets"]},
|
||||
)
|
||||
|
||||
def _review_image_delete(self, db, roots, targets, files):
|
||||
"""Every target checked now (blocking); ``{targets, public}``."""
|
||||
listings, seen = {}, set()
|
||||
acted, refused, trash_refused = [], [], []
|
||||
spellings = self._source_spellings()
|
||||
root_texts = [str(root) for root in roots]
|
||||
for index, _scope, relative in targets:
|
||||
root = roots[index]
|
||||
if index not in listings:
|
||||
listings[index] = self._favorite_listing(root, files[index])
|
||||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||||
target = {
|
||||
"root_index": index,
|
||||
"root_scope": _scope,
|
||||
"root": str(root),
|
||||
"root_label": self._public_path(root),
|
||||
"rel": spelled or relative,
|
||||
}
|
||||
if spelled is None:
|
||||
refused.append(_row(target, "not-listed"))
|
||||
continue
|
||||
absolute = os.path.join(str(root), *spelled.split("/"))
|
||||
code, info = file_check(absolute, str(root), spelled)
|
||||
if code:
|
||||
refused.append(_row(target, code))
|
||||
continue
|
||||
identity = (info.st_dev, info.st_ino)
|
||||
if identity in seen:
|
||||
continue # one physical file named twice (nested roots, hardlinks)
|
||||
seen.add(identity)
|
||||
target.update(self._removal_of(absolute, info, spellings))
|
||||
target["trash"] = trash.trash_available(absolute, root_texts)
|
||||
if target["trash"]:
|
||||
trash_refused.append(_row(target, target["trash"]))
|
||||
acted.append(target)
|
||||
counted = db.count_removal_records([t["removal"] for t in acted])
|
||||
library_count = len(acted)
|
||||
public = {
|
||||
"operation": IMAGE_DELETE,
|
||||
"scope": {"count": library_count, "label": "images"},
|
||||
"library": {"count": library_count},
|
||||
"trash": {"count": library_count - len(trash_refused)},
|
||||
"protected": {"count": len(refused), "reasons": []},
|
||||
"refused": _summary(refused),
|
||||
"trash_refused": _summary(trash_refused),
|
||||
"records": counted["records"],
|
||||
"orphaned_prompts": counted["orphaned_prompts"],
|
||||
"media": {"deletes_files": False, "label": "Chosen in the review"},
|
||||
"details": [],
|
||||
}
|
||||
return {"targets": acted, "public": public}
|
||||
|
||||
@staticmethod
|
||||
def _removal_of(absolute, info, spellings):
|
||||
favorites = [
|
||||
(_favorite_root_key(root), _favorite_relative_key(relative))
|
||||
for root, relative in spellings.favorites(absolute).items()
|
||||
]
|
||||
return {
|
||||
"abs": absolute,
|
||||
"size": info.st_size,
|
||||
"mtime_ns": info.st_mtime_ns,
|
||||
"inode": [info.st_dev, info.st_ino],
|
||||
"removal": {
|
||||
"spellings": list(spellings.of(absolute).values()),
|
||||
"locator": os.path.normcase(os.path.realpath(absolute)),
|
||||
"favorites": favorites,
|
||||
"category": spellings.category_key(absolute),
|
||||
},
|
||||
}
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
def _start_image_delete(self, db, plan, mode, backup_on):
|
||||
"""Register the delete job for a reviewed plan (on the loop)."""
|
||||
return self._jobs.start(
|
||||
IMAGE_DELETE_JOB_KIND,
|
||||
TITLES[mode],
|
||||
admission="records",
|
||||
cancellable=False,
|
||||
scope={"operation": IMAGE_DELETE, "mode": mode},
|
||||
groups=IMAGE_DELETE_GROUP_NAMES,
|
||||
runner=lambda job: self._run_image_delete(job, db, plan, mode, backup_on),
|
||||
)
|
||||
|
||||
async def _run_image_delete(self, job, db, plan, mode, backup_on):
|
||||
targets = plan.payload["targets"]
|
||||
unchanged = await self._run_in_executor(self._sources_unchanged, targets)
|
||||
if not unchanged:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return
|
||||
try:
|
||||
await self._delete_images(job, db, targets, mode, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
|
||||
async def _delete_images(self, job, db, targets, mode, backup):
|
||||
job.update(total=len(targets), status_line=TITLES[mode])
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
done_group = "trashed" if mode == "trash" else "removed"
|
||||
changed, failed = 0, None
|
||||
for done, target in enumerate(targets, start=1):
|
||||
if failed:
|
||||
self._record_delete(job, "not_changed", target, "not-attempted")
|
||||
continue
|
||||
group, code, label = await self._run_in_executor(
|
||||
self._delete_one_image, db, target, mode, roots
|
||||
)
|
||||
if group == "errors":
|
||||
failed = code or RECORDS_FAILED_CODE
|
||||
text = REASONS[code] if code else RECORDS_FAILED
|
||||
job.record(
|
||||
"errors",
|
||||
{
|
||||
"path": target["rel"],
|
||||
"root": target["root_label"],
|
||||
"root_index": target["root_index"],
|
||||
"root_scope": target["root_scope"],
|
||||
"detail": text,
|
||||
},
|
||||
f"! {target['root_label']}/{target['rel']} · {text}",
|
||||
)
|
||||
elif group == done_group:
|
||||
changed += 1
|
||||
detail = (
|
||||
f"Moved to the {label}" if label else "Removed from the library"
|
||||
)
|
||||
self._record_delete(job, group, target, None, detail)
|
||||
else:
|
||||
self._record_delete(job, group, target, code)
|
||||
job.update(processed=done, progress=int(done / len(targets) * 100))
|
||||
if changed:
|
||||
self.invalidate_gallery_cache()
|
||||
verb = "Moved" if mode == "trash" else "Removed"
|
||||
where = "to the trash" if mode == "trash" else "from the library"
|
||||
line = f"{verb} {changed:,} of {len(targets):,} images {where}"
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
if failed:
|
||||
# "records-failed" names every stop at an errors row (the page's
|
||||
# "Stopped after N of M" toast); the row says which error it was.
|
||||
job.scope["failure"] = RECORDS_FAILED_CODE
|
||||
job.publish({"type": "error", "code": failed, "message": line})
|
||||
job.finish("failed")
|
||||
|
||||
@staticmethod
|
||||
def _record_delete(job, group, target, code, detail=None):
|
||||
text = detail or REASONS[code]
|
||||
row = {
|
||||
"path": target["rel"],
|
||||
"root": target["root_label"],
|
||||
"root_index": target["root_index"],
|
||||
"root_scope": target["root_scope"],
|
||||
"code": code,
|
||||
"detail": text,
|
||||
}
|
||||
mark = {"removed": "-", "trashed": "-"}.get(group, "=")
|
||||
job.record(
|
||||
group, row, f"{mark} {target['root_label']}/{target['rel']} · {text}"
|
||||
)
|
||||
|
||||
def _delete_one_image(self, db, target, mode, roots):
|
||||
"""``(group, code, trash label)`` for one reviewed file (blocking)."""
|
||||
code, _info = file_check(
|
||||
target["abs"],
|
||||
target["root"],
|
||||
target["rel"],
|
||||
expected={
|
||||
"size": target["size"],
|
||||
"mtime_ns": target["mtime_ns"],
|
||||
"inode": target["inode"],
|
||||
},
|
||||
)
|
||||
if code:
|
||||
return "not_changed", code, None
|
||||
label = None
|
||||
if mode == "trash":
|
||||
try:
|
||||
label = trash.move_to_trash(target["abs"], roots)
|
||||
except trash.TrashRefused as refused:
|
||||
code = refused.code if refused.code in REASONS else trash.FAILED
|
||||
self.logger.info(
|
||||
"Not moved to the trash (%s): %s %s",
|
||||
code,
|
||||
target["rel"],
|
||||
refused.detail,
|
||||
)
|
||||
return "not_changed", code, None
|
||||
except trash.TrashUnconfirmed as unconfirmed:
|
||||
# The file left its folder: keep its records, stop the job.
|
||||
self.logger.warning(
|
||||
"Trash move not confirmed: %s %s", target["rel"], unconfirmed.detail
|
||||
)
|
||||
return "errors", TRASH_UNCONFIRMED, None
|
||||
mark = None
|
||||
else:
|
||||
mark = (target["size"], target["mtime_ns"])
|
||||
try:
|
||||
db.remove_image_records(target["removal"], mark=mark)
|
||||
except Exception:
|
||||
self.logger.exception("Removing an image's records failed")
|
||||
return "errors", None, label
|
||||
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
|
||||
return ("trashed" if mode == "trash" else "removed"), None, label
|
||||
|
||||
# ── the workspace listing ─────────────────────────────────────────
|
||||
|
||||
def _removal_marks(self) -> dict:
|
||||
"""The removal marks, cached until a mark changes or the database does.
|
||||
|
||||
A listing page reads no table while nothing changed: the cache follows
|
||||
this process's marks generation and the database identity (a restore).
|
||||
"""
|
||||
key = (self._database_identity(self.db), image_removal_schema.generation())
|
||||
cached = getattr(self, "_marks_cache", None)
|
||||
if cached is None or cached[0] != key:
|
||||
cached = (key, self.db.removal_marks())
|
||||
self._marks_cache = cached
|
||||
return cached[1]
|
||||
|
||||
def _hidden_entries(self, listings) -> set:
|
||||
"""Normalized paths of listed files a removal mark hides (blocking).
|
||||
|
||||
Only marked locators are looked at, and only those files are
|
||||
``lstat``'ed: a mark hides a file while its size and ``mtime_ns`` are
|
||||
still the ones it was removed with.
|
||||
"""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return set()
|
||||
located = self._workspace_locators(listings)
|
||||
return {
|
||||
entry.normalized
|
||||
for locator, entry in ((locator, located.get(locator)) for locator in marks)
|
||||
if entry is not None and mark_hides(marks, locator, entry.path)
|
||||
}
|
||||
|
||||
def _removed_paths(self, paths) -> set:
|
||||
"""The absolute paths among *paths* a removal mark hides (blocking)."""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return set()
|
||||
return {
|
||||
str(path)
|
||||
for path in paths
|
||||
if mark_hides(marks, os.path.normcase(os.path.realpath(str(path))), path)
|
||||
}
|
||||
|
||||
def _removal_check_under(self, root):
|
||||
"""``hidden(path) -> bool`` for files a walk of *root* found (blocking).
|
||||
|
||||
A walk never follows a link below *root*, so a file's locator is
|
||||
``normcase(join(realpath(root), relative))``: no ``realpath`` per
|
||||
file, and an ``lstat`` only for a marked one. Without marks it
|
||||
answers False at once.
|
||||
"""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return lambda _path: False
|
||||
root = os.fspath(root)
|
||||
real = os.path.realpath(root)
|
||||
|
||||
def hidden(path):
|
||||
relative = os.path.relpath(os.fspath(path), root)
|
||||
locator = os.path.normcase(os.path.join(real, relative))
|
||||
return mark_hides(marks, locator, path)
|
||||
|
||||
return hidden
|
||||
|
||||
def _without_removed(self, root, paths) -> list:
|
||||
"""*paths* found by a walk of *root*, less the hidden ones (blocking)."""
|
||||
removed = self._removal_check_under(root)
|
||||
return [path for path in paths if not removed(path)]
|
||||
|
||||
|
||||
def parse_mode(value):
|
||||
"""The apply's ``mode``, or None when it is not one of MODES."""
|
||||
return value if isinstance(value, str) and value in MODES else None
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Return removed images to the library (#498).
|
||||
|
||||
``POST /prompt_manager/images/removed/return {items | filter}`` names files a
|
||||
removal mark hides (#480, "Remove from the library only"): ``items`` are 1 to
|
||||
500 listing identities, ``filter`` the removed view's listing query
|
||||
(``removed: "1"`` plus the view's other parameters, at most 500 images). Only
|
||||
this route takes ``removed``: every other bulk route refuses it, because an
|
||||
explicit write there would show a file again without re-reading it.
|
||||
|
||||
The route answers 202 ``{job, returned, not_removed}`` after starting a
|
||||
``scan`` job titled "Return images to the library" (admission ``records``:
|
||||
409 ``busy`` beside a scan, restore or delete, nothing changed). The job
|
||||
first deletes the marks that still hide their file (same size and
|
||||
``mtime_ns``), in one transaction, then scans exactly those files with the
|
||||
scan's own pipeline: prompts, links, pairs and Needs review items come back
|
||||
from the files' embedded metadata. Image tags, auto-tag marks, the own
|
||||
category, collection memberships and the favorite were deleted by the remove
|
||||
and do not come back. No file is moved, written or deleted.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .image_categories import TOO_MANY, _coded
|
||||
from .images import (
|
||||
ListingSourceChanged,
|
||||
_favorite_relative_key,
|
||||
_json_body_or_error,
|
||||
parse_listing_filter,
|
||||
)
|
||||
from .image_tag_jobs import MAX_JOB_TARGETS
|
||||
from .jobs import JobBusy
|
||||
from .job_routes import busy_response
|
||||
|
||||
try:
|
||||
from ...database.image_removal import mark_hides
|
||||
except ImportError:
|
||||
from database.image_removal import mark_hides
|
||||
|
||||
NONE_REMOVED = "None of the selected images are removed from the library."
|
||||
SOURCE_CHANGED = "The image sources changed. Reload Images and try again."
|
||||
|
||||
|
||||
def _bad(message):
|
||||
return web.json_response({"success": False, "error": message}, status=400)
|
||||
|
||||
|
||||
class ImageReturnMixin:
|
||||
"""Return to library (#498): clear the marks, then scan those files."""
|
||||
|
||||
def _register_image_return_routes(self, routes):
|
||||
# A fixed path beside the dynamic /images/{image_id} routes.
|
||||
@routes.post("/prompt_manager/images/removed/return")
|
||||
async def return_removed_images_route(request):
|
||||
return await self.return_removed_images(request)
|
||||
|
||||
async def return_removed_images(self, request):
|
||||
data, error = await _json_body_or_error(request)
|
||||
if error is not None:
|
||||
return error
|
||||
if set(data) == {"filter"}:
|
||||
targets, roots, error = await self._removed_filter_targets(data["filter"])
|
||||
else:
|
||||
_data, targets, roots, error = await self._category_request(
|
||||
request, frozenset(), data=data
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
files = {}
|
||||
for index in dict.fromkeys(t[0] for t in targets):
|
||||
files[index] = await self._get_gallery_files(roots[index])
|
||||
returned, not_removed = await self._run_in_executor(
|
||||
self._hidden_targets, roots, targets, files
|
||||
)
|
||||
if not returned:
|
||||
return _coded(400, "no-images", NONE_REMOVED)
|
||||
try:
|
||||
job = self._start_scan_job(
|
||||
files="all", output_dirs=list(roots), returned=returned
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"job": job.summary(),
|
||||
"returned": len(returned),
|
||||
"not_removed": not_removed,
|
||||
},
|
||||
status=202,
|
||||
)
|
||||
|
||||
async def _removed_filter_targets(self, raw):
|
||||
"""``(targets, roots, None)`` for the removed view's filter, else an error."""
|
||||
if not isinstance(raw, dict) or raw.get("removed") != "1":
|
||||
return None, None, _bad('filter must hold removed: "1"')
|
||||
rest = {key: value for key, value in raw.items() if key != "removed"}
|
||||
try:
|
||||
parse_listing_filter(rest)
|
||||
selection = await self.resolve_listing_filter(
|
||||
{**rest, "removed": "1"}, MAX_JOB_TARGETS
|
||||
)
|
||||
except ListingSourceChanged:
|
||||
return None, None, _coded(409, "source-changed", SOURCE_CHANGED)
|
||||
except ValueError as exc:
|
||||
return None, None, _bad(str(exc))
|
||||
if selection.truncated:
|
||||
return None, None, _coded(400, "too-many-images", TOO_MANY)
|
||||
if not selection.targets:
|
||||
return None, None, _coded(400, "no-images", NONE_REMOVED)
|
||||
return tuple(selection.targets), list(self._get_all_output_dirs()), None
|
||||
|
||||
def _hidden_targets(self, roots, targets, files):
|
||||
"""``([(path, locator)], not_removed)``: the targets a mark hides now.
|
||||
|
||||
Each target is matched to its listed spelling (the walk's). One
|
||||
physical file named twice (nested roots) is returned once; a target
|
||||
that is not listed or not hidden counts in ``not_removed``.
|
||||
"""
|
||||
marks = self._removal_marks()
|
||||
listings, seen, returned, not_removed = {}, set(), [], 0
|
||||
for index, _scope, relative in targets:
|
||||
root = roots[index]
|
||||
if index not in listings:
|
||||
listings[index] = self._favorite_listing(root, files[index])
|
||||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||||
if spelled is None:
|
||||
not_removed += 1
|
||||
continue
|
||||
path = os.path.join(str(root), *spelled.split("/"))
|
||||
locator = os.path.normcase(os.path.realpath(path))
|
||||
if locator in seen:
|
||||
continue
|
||||
if not mark_hides(marks, locator, path):
|
||||
not_removed += 1
|
||||
continue
|
||||
seen.add(locator)
|
||||
returned.append((Path(path), locator))
|
||||
return returned, not_removed
|
||||
|
||||
def _clear_returned_marks(self, returned) -> list:
|
||||
"""The paths whose marks this deleted (blocking; one transaction)."""
|
||||
cleared = self.db.clear_hiding_marks(
|
||||
[(locator, str(path)) for path, locator in returned]
|
||||
)
|
||||
return [path for path, locator in returned if locator in cleared]
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
``POST /prompt_manager/jobs/image-tags`` starts one ``image-tags`` job over
|
||||
1..500 listed images, named by the listing identity ``{root_index,
|
||||
root_scope, path}``. The job is record-changing (it writes image tag sets and
|
||||
root_scope, path}``, or over every image a listing ``filter`` admits (at most
|
||||
10,000, resolved at start in the listing's order; "Select all N matching",
|
||||
#392). The job is record-changing (it writes image tag sets and
|
||||
new vocabulary names), so it never runs beside the metadata scan.
|
||||
|
||||
- **Add tags** (``mode: "add"``) hashes each current file and unions a fixed
|
||||
@@ -18,13 +20,21 @@ the suggestion worker and the storage call ``add_image_tags``. Files are
|
||||
handled one at a time and the job stops at a file boundary when cancelled,
|
||||
keeping what it already wrote. Rows and log lines carry root-relative paths
|
||||
and public root labels only.
|
||||
|
||||
Auto-tag marks each image it ran the model on as auto-tagged (#379), with the
|
||||
union of its tags. The ``autotag`` job ("Tag output images", started by
|
||||
``POST /jobs/autotag``) runs the same per-file loop over every still image the
|
||||
output folders list (:meth:`ImageTagJobsMixin._run_autotag_batch`): image
|
||||
tags only, never a prompt tag or prompt text.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import functools
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import NamedTuple
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -32,7 +42,7 @@ from .. import model_gate
|
||||
from .admin import _plural, _time_left
|
||||
from .annotation_suggest import (
|
||||
OptionError,
|
||||
unavailable_message,
|
||||
model_refusal,
|
||||
validate_options,
|
||||
)
|
||||
from .annotations import (
|
||||
@@ -45,10 +55,26 @@ from .annotations import (
|
||||
_submit,
|
||||
_target,
|
||||
)
|
||||
from .autotag_routes import model_busy_response
|
||||
from .images import _public_path
|
||||
from .autotag_routes import BATCH_CANCELLED, MODEL_BUSY_ERROR, model_busy_response
|
||||
from .images import (
|
||||
MAX_LISTING_SELECTION,
|
||||
CollectionMissing,
|
||||
ListingSourceChanged,
|
||||
collection_not_found,
|
||||
_current_fingerprint,
|
||||
_listing_posix,
|
||||
_public_path,
|
||||
parse_listing_filter,
|
||||
)
|
||||
from .job_routes import busy_response
|
||||
from .jobs import IMAGE_TAG_GROUP_NAMES, IMAGE_TAG_JOB_KIND, JobBusy
|
||||
from .jobs import (
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
AUTOTAG_GROUP_NAMES,
|
||||
IMAGE_TAG_GROUP_NAMES,
|
||||
IMAGE_TAG_JOB_KIND,
|
||||
JobBusy,
|
||||
)
|
||||
from .maintenance import disk_plan, short_space_message
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
@@ -75,6 +101,11 @@ except ImportError:
|
||||
from utils.video_metadata import is_video_path
|
||||
|
||||
MAX_JOB_TARGETS = 500
|
||||
# A filter start resolves on the server, so the request body does not bound it.
|
||||
MAX_FILTER_TARGETS = MAX_LISTING_SELECTION
|
||||
NO_IMAGES = "No images match this view."
|
||||
FILTER_LISTING_FAILED = "The matching images could not be listed. Try again."
|
||||
TOO_MANY_IMAGES = "Narrow the view to at most {cap:,} images."
|
||||
# 500 maximal targets plus the options: a path of MAX_TARGET_PATH code points
|
||||
# takes at most 6 bytes each on the wire (a \uXXXX escape; raw UTF-8 is at most
|
||||
# 4), and the root index and scope fit in 256. The annotation routes' 64 KiB
|
||||
@@ -88,9 +119,10 @@ ADD_TITLE = "Add tags to images"
|
||||
AUTO_TITLE = "Auto-tag images"
|
||||
GATE_OWNER_KIND = "image-tag-job"
|
||||
|
||||
# Each mode's fields besides the images, named by ``targets`` or ``filter``.
|
||||
_FIELDS = {
|
||||
"add": frozenset({"mode", "targets", "tags"}),
|
||||
"auto": frozenset({"mode", "targets", "model_type", "options"}),
|
||||
"add": frozenset({"mode", "tags"}),
|
||||
"auto": frozenset({"mode", "model_type", "options"}),
|
||||
}
|
||||
_VERBS = {"add": "Add tags", "auto": "Auto-tag"}
|
||||
_LOG_PREFIXES = {"tagged": "+", "unchanged": "=", "skipped": "-", "errors": "!"}
|
||||
@@ -107,6 +139,14 @@ READ_FAILED = "The file could not be read."
|
||||
SOURCE_CHANGED = "The source changed. Refresh the image collection."
|
||||
STORAGE_RESTORED = "Prompt storage was restored; start the job again."
|
||||
BUSY_RETRY = "The file could not be read now. Run the job again for it."
|
||||
# Tag output images (#379): the per-image batch.
|
||||
ALREADY_AUTOTAGGED = "Already auto-tagged"
|
||||
LISTING_FAILED = "The output folders could not be listed. Try again."
|
||||
LISTING_STATUS = "Listing output images…"
|
||||
_INTERNAL_ERROR = "An internal error occurred. Check server logs for details."
|
||||
# The pre-pass stats and looks up this many targets at a time, checking for a
|
||||
# cancel between chunks.
|
||||
PREPASS_CHUNK = 500
|
||||
|
||||
_monotonic = time.monotonic # the job's clock (a test can fake it)
|
||||
|
||||
@@ -141,6 +181,8 @@ class _Spec:
|
||||
tags: tuple = ()
|
||||
model_type: str | None = None
|
||||
options: dict | None = None
|
||||
# A listing filter the start resolves into ``targets`` (#392).
|
||||
filter: dict | None = None
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
@@ -153,13 +195,17 @@ class _Spec:
|
||||
return scope
|
||||
|
||||
|
||||
def _parse_targets(raw):
|
||||
"""Unique validated targets in request order, or ``(None, error)``."""
|
||||
def _parse_targets(raw, **target_options):
|
||||
"""Unique validated targets in request order, or ``(None, error)``.
|
||||
|
||||
``target_options`` reach the target check (the image category routes
|
||||
pass their own ``media_message``, #332).
|
||||
"""
|
||||
if not isinstance(raw, list) or not 1 <= len(raw) <= MAX_JOB_TARGETS:
|
||||
return None, bad_request(f"targets must list 1 to {MAX_JOB_TARGETS} images")
|
||||
seen, targets = set(), []
|
||||
for item in raw:
|
||||
target, error = _target(item)
|
||||
target, error = _target(item, **target_options)
|
||||
if error is not None:
|
||||
return None, error
|
||||
if target not in seen:
|
||||
@@ -199,21 +245,35 @@ def parse_image_tag_request(data: dict):
|
||||
"""A :class:`_Spec` for a start body, or ``(None, error response)``.
|
||||
|
||||
``{mode:"add", targets, tags}`` or ``{mode:"auto", targets, model_type,
|
||||
options}`` exactly; options follow the Tag dialog's Generate rules.
|
||||
options}`` exactly; options follow the Tag dialog's Generate rules. A
|
||||
``filter`` (a listing query object, parse_listing_filter) may stand in for
|
||||
``targets``; the start resolves it.
|
||||
"""
|
||||
fields = _FIELDS.get(data.get("mode"))
|
||||
if fields is None:
|
||||
return None, bad_request('mode must be "add" or "auto"')
|
||||
fields = fields | {"filter" if "filter" in data else "targets"}
|
||||
if set(data) != fields:
|
||||
return None, bad_request(
|
||||
"Request must contain exactly " + ", ".join(sorted(fields))
|
||||
)
|
||||
targets, error = _parse_targets(data["targets"])
|
||||
if error is not None:
|
||||
return None, error
|
||||
listing_filter, targets = None, ()
|
||||
if "filter" in data:
|
||||
try:
|
||||
listing_filter = parse_listing_filter(data["filter"])
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
else:
|
||||
targets, error = _parse_targets(data["targets"])
|
||||
if error is not None:
|
||||
return None, error
|
||||
if data["mode"] == "add":
|
||||
return _parse_add(targets, data)
|
||||
return _parse_auto(targets, data)
|
||||
spec, error = _parse_add(targets, data)
|
||||
else:
|
||||
spec, error = _parse_auto(targets, data)
|
||||
if spec is not None and listing_filter is not None:
|
||||
spec = dataclasses.replace(spec, filter=listing_filter)
|
||||
return spec, error
|
||||
|
||||
|
||||
def _capture_outcome(exc: CaptureError, root, relative: str):
|
||||
@@ -246,6 +306,92 @@ async def _capacity_retried(job, attempt):
|
||||
return await attempt()
|
||||
|
||||
|
||||
class Refusal(NamedTuple):
|
||||
"""Why a Tag output images start was refused.
|
||||
|
||||
``response`` is the native route's JSON answer; ``message`` the sentence
|
||||
a legacy stream frame carries.
|
||||
"""
|
||||
|
||||
status: int
|
||||
code: str
|
||||
message: str
|
||||
response: web.Response
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AutotagRun:
|
||||
"""What one Tag output images job runs with (validated at its start)."""
|
||||
|
||||
model_type: str
|
||||
existing: str
|
||||
options: dict
|
||||
keep_in_memory: bool
|
||||
backup_on: bool
|
||||
owner: object
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _BatchTarget:
|
||||
"""One still image of the batch: the job target and its private keys."""
|
||||
|
||||
target: tuple # (root_index, root_scope, relative): the image-tags target
|
||||
path: str # the listed file, for the pre-pass stat (never sent out)
|
||||
locator: str # the image tag key: normcase(join(realpath(root), relative))
|
||||
label: str # the root's public label
|
||||
|
||||
|
||||
def _still_targets(api, index, root, files, seen: set) -> list:
|
||||
"""Worker thread: one root's listed still images as _BatchTargets.
|
||||
|
||||
Newest first (the listing's order). A locator an earlier root already
|
||||
produced (overlapping roots) is dropped; ``seen`` collects them.
|
||||
"""
|
||||
scope = api._gallery_root_scope(root, index)
|
||||
label = _public_path(root)
|
||||
real = os.path.realpath(root)
|
||||
prefix = os.path.join(str(root), "")
|
||||
found = []
|
||||
for path, _mtime in files:
|
||||
relative = _listing_posix(root, prefix, path)
|
||||
if is_video_path(relative):
|
||||
continue
|
||||
locator = os.path.normcase(os.path.join(real, *relative.split("/")))
|
||||
if locator in seen:
|
||||
continue
|
||||
seen.add(locator)
|
||||
found.append(_BatchTarget((index, scope, relative), str(path), locator, label))
|
||||
return found
|
||||
|
||||
|
||||
def _prepass_chunk(db, chunk, skip_marked: bool, max_file: int):
|
||||
"""Worker thread: ``(remaining, skips)`` for a chunk of _BatchTargets.
|
||||
|
||||
Stats each file (no open, no hash): gone is "File not found", over the
|
||||
capture limit "Too large to verify"; under "Skip already tagged" a file
|
||||
whose current bytes are marked is "Already auto-tagged". ``skips`` are
|
||||
``(target, detail)`` pairs in target order.
|
||||
"""
|
||||
stats, skips, alive = {}, [], []
|
||||
for item in chunk:
|
||||
fingerprint = _current_fingerprint(item.path)
|
||||
if fingerprint is None:
|
||||
skips.append((item, FILE_NOT_FOUND))
|
||||
elif fingerprint[0] > max_file:
|
||||
skips.append((item, TOO_LARGE))
|
||||
else:
|
||||
stats[item.locator] = fingerprint
|
||||
alive.append(item)
|
||||
marked = db.autotag_state(stats) if skip_marked and stats else {}
|
||||
remaining = []
|
||||
for item in alive:
|
||||
if marked.get(item.locator):
|
||||
skips.append((item, ALREADY_AUTOTAGGED))
|
||||
else:
|
||||
remaining.append(item)
|
||||
return remaining, skips
|
||||
|
||||
|
||||
def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
"""Worker thread: load (or swap to) *model_type*; never downloads."""
|
||||
if service.is_model_loaded() and service.get_loaded_model_type() == model_type:
|
||||
@@ -254,6 +400,25 @@ def _load_model(service, model_type: str, use_gpu: bool) -> None:
|
||||
raise RuntimeError(f"{model_type} could not be loaded")
|
||||
|
||||
|
||||
def _refusal(status: int, code: str, message: str) -> Refusal:
|
||||
response = web.json_response(
|
||||
{"success": False, "code": code, "error": message}, status=status
|
||||
)
|
||||
return Refusal(status, code, message, response)
|
||||
|
||||
|
||||
def _busy_refusal(busy: JobBusy) -> Refusal:
|
||||
title = getattr(busy.running, "title", "") or "Another job"
|
||||
return Refusal(409, "busy", f"{title} already running", busy_response(busy))
|
||||
|
||||
|
||||
def _batch_cancelled(job) -> str:
|
||||
"""Say the batch was cancelled; its end state (the runner ends the job)."""
|
||||
job.update(status_line=BATCH_CANCELLED)
|
||||
job.log_line(BATCH_CANCELLED)
|
||||
return "cancelled"
|
||||
|
||||
|
||||
class ImageTagJobsMixin:
|
||||
"""The ``image-tags`` job kind: start route and runner."""
|
||||
|
||||
@@ -270,6 +435,10 @@ class ImageTagJobsMixin:
|
||||
spec, error = parse_image_tag_request(data)
|
||||
if error is not None:
|
||||
return error
|
||||
if spec.filter is not None:
|
||||
spec, error = await self._resolve_job_filter(spec)
|
||||
if error is not None:
|
||||
return error
|
||||
roots = await self._run_in_executor(self._job_roots, spec.targets)
|
||||
if any(root is None for root, _label in roots.values()):
|
||||
return _coded("source-changed")
|
||||
@@ -301,6 +470,250 @@ class ImageTagJobsMixin:
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
||||
|
||||
# -- Tag output images (#379) ---------------------------------------------
|
||||
|
||||
async def _autotag_targets(self) -> list:
|
||||
"""Every still image the output folders list, as _BatchTargets.
|
||||
|
||||
Root by root in configured order, each newest first (the shared
|
||||
listing, ``_get_gallery_files``, used as it is even when stale). Videos
|
||||
are not targets; a file under two overlapping roots is one target.
|
||||
"""
|
||||
seen: set = set()
|
||||
found = []
|
||||
for index, root in enumerate(list(self._get_all_output_dirs())):
|
||||
files = await self._get_gallery_files(root)
|
||||
found.extend(
|
||||
await self._run_in_executor(
|
||||
_still_targets, self, index, root, files, seen
|
||||
)
|
||||
)
|
||||
# #480: a file removed from the library is never tagged by a batch.
|
||||
hidden = await self._run_in_executor(
|
||||
self._removed_paths, [target.path for target in found]
|
||||
)
|
||||
return [target for target in found if target.path not in hidden]
|
||||
|
||||
async def _start_autotag(
|
||||
self, *, model_type, existing, options, keep_in_memory, title, owner_kind
|
||||
):
|
||||
"""Register a Tag output images job: ``(job, None)`` or ``(None, Refusal)``.
|
||||
|
||||
``options`` are already validated (``validate_options``). The checks
|
||||
run in this order: 409 ``model-unavailable``; 507
|
||||
``insufficient-space`` (only when the safety backup is on); 409
|
||||
``busy`` (another records job); 409 ``model-busy``. No await follows
|
||||
the busy check. The gate is taken before the job registers and
|
||||
released when the task ends, also before its runner ever starts.
|
||||
"""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
|
||||
backup_on = self._safety_backup_policy() == "on"
|
||||
status = await self._run_in_executor(get_autotag_service().get_models_status)
|
||||
_disk, short, _size = await self._run_in_executor(
|
||||
functools.partial(
|
||||
disk_plan, self.db.model.db_path, backup=backup_on, vacuum=False
|
||||
)
|
||||
)
|
||||
label = MODELS[model_type]["label"]
|
||||
message = model_refusal(status, model_type, label)
|
||||
if message is not None:
|
||||
return None, _refusal(409, "model-unavailable", message)
|
||||
if backup_on and short is not None:
|
||||
return None, _refusal(507, "insufficient-space", short_space_message(short))
|
||||
busy = self._jobs.blocker(AUTOTAG_BATCH_JOB_KIND, admission="records")
|
||||
if busy is not None:
|
||||
return None, _busy_refusal(JobBusy(busy))
|
||||
owner = model_gate.new_owner(owner_kind)
|
||||
if not model_gate.try_acquire(owner):
|
||||
response = model_busy_response()
|
||||
return None, Refusal(409, "model-busy", MODEL_BUSY_ERROR, response)
|
||||
run = AutotagRun(
|
||||
model_type, existing, options, keep_in_memory, backup_on, owner
|
||||
)
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
title,
|
||||
admission="records",
|
||||
cancellable=True,
|
||||
scope={"model_type": model_type, "existing": existing, "label": label},
|
||||
groups=AUTOTAG_GROUP_NAMES,
|
||||
runner=functools.partial(self._run_autotag_batch, run),
|
||||
)
|
||||
except BaseException as refused:
|
||||
model_gate.release(owner)
|
||||
if isinstance(refused, JobBusy):
|
||||
return None, _busy_refusal(refused)
|
||||
raise
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return job, None
|
||||
|
||||
async def _run_autotag_batch(self, run: AutotagRun, job) -> None:
|
||||
"""The ``autotag`` job: backup, model, listing, pre-pass, per-image loop.
|
||||
|
||||
``job.cancelled`` is checked between phases (and per pre-pass chunk
|
||||
and per image). Each phase publishes ``{"type": "phase", "phase"}``.
|
||||
With ``keep_in_memory`` false the model is unloaded under the gate.
|
||||
The job ends only after that cleanup and the gate release, so a job
|
||||
that reads as ended never still holds the gate.
|
||||
"""
|
||||
state = "failed"
|
||||
try:
|
||||
state = await self._autotag_batch_phases(run, job)
|
||||
except Exception:
|
||||
self.logger.exception("Tag output images failed")
|
||||
job.update(status_line=_INTERNAL_ERROR)
|
||||
job.log_line(f"Tagging failed · {_INTERNAL_ERROR}")
|
||||
finally:
|
||||
try:
|
||||
if not run.keep_in_memory:
|
||||
await self._unload_batch_model(run.owner)
|
||||
if run.backup_on:
|
||||
await self._prune_backup_pool(job, self.db)
|
||||
finally:
|
||||
model_gate.release(run.owner)
|
||||
job.publish({"type": state, "progress": job.progress})
|
||||
job.finish(state)
|
||||
|
||||
async def _unload_batch_model(self, owner) -> None:
|
||||
from ..autotag import get_autotag_service
|
||||
|
||||
service = get_autotag_service()
|
||||
try:
|
||||
if service.is_model_loaded():
|
||||
await model_gate.run(owner, service.unload_model)
|
||||
except Exception:
|
||||
self.logger.exception("The tagging model could not be unloaded")
|
||||
|
||||
async def _autotag_batch_phases(self, run: AutotagRun, job) -> str:
|
||||
"""Run the phases; the job's end state (the caller ends the job)."""
|
||||
job.publish({"type": "phase", "phase": "backup"})
|
||||
ok, _backup = await self._safety_backup_step(job, self.db, run.backup_on)
|
||||
if not ok:
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
job.publish({"type": "phase", "phase": "model"})
|
||||
job.update(status_line="Loading the tagging model")
|
||||
spec = _Spec("auto", (), model_type=run.model_type, options=run.options)
|
||||
service = await self._prepare_tagging_model(job, spec, run.owner)
|
||||
if service is None:
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
job.publish({"type": "phase", "phase": "listing"})
|
||||
job.update(status_line=LISTING_STATUS)
|
||||
epoch = await self._run_in_executor(self.db.get_execution_epoch)
|
||||
try:
|
||||
targets = await self._autotag_targets()
|
||||
except Exception:
|
||||
self.logger.exception("Tag output images could not list the outputs")
|
||||
job.update(status_line=LISTING_FAILED)
|
||||
job.log_line(f"! {LISTING_FAILED}")
|
||||
return "failed"
|
||||
total = len(targets)
|
||||
job.scope["images"] = total
|
||||
job.update(total=total)
|
||||
policy = (
|
||||
"skip images already auto-tagged" if run.existing == "skip" else "merge"
|
||||
)
|
||||
job.log_line(
|
||||
f"Tagging started · {job.scope.get('label') or run.model_type} · "
|
||||
f"{_plural(total, 'image')} · {policy}"
|
||||
)
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
remaining = await self._autotag_prepass(job, targets, run.existing == "skip")
|
||||
if remaining is None:
|
||||
return _batch_cancelled(job)
|
||||
done = total - len(remaining)
|
||||
spec = dataclasses.replace(
|
||||
spec, targets=tuple(item.target for item in remaining)
|
||||
)
|
||||
loop_run = _Run(job, spec, epoch, run.owner, service)
|
||||
if not await self._tag_each_file(loop_run, done_before=done):
|
||||
job.update(status_line=STORAGE_RESTORED)
|
||||
job.log_line(f"Tagging stopped · {STORAGE_RESTORED}")
|
||||
return "failed"
|
||||
if job.cancelled:
|
||||
return _batch_cancelled(job)
|
||||
finished = f"Tagged {_plural(job.counters.get('tagged', 0), 'image')}"
|
||||
job.update(status_line=finished, progress=100)
|
||||
job.log_line(finished)
|
||||
return "finished"
|
||||
|
||||
async def _autotag_prepass(self, job, targets, skip_marked: bool):
|
||||
"""The targets left for the model, or None when cancelled meanwhile.
|
||||
|
||||
Opens and hashes nothing: one stat per file and one indexed read per
|
||||
chunk. Skips are counted with ``record_counted`` (exact counters, rows
|
||||
up to the cap, no log line each).
|
||||
"""
|
||||
max_file = self._capture_store().max_file
|
||||
remaining, skipped = [], 0
|
||||
total = len(targets)
|
||||
for start in range(0, total, PREPASS_CHUNK):
|
||||
if job.cancelled:
|
||||
return None
|
||||
chunk = targets[start : start + PREPASS_CHUNK]
|
||||
left, skips = await self._run_in_executor(
|
||||
_prepass_chunk, self.db, chunk, skip_marked, max_file
|
||||
)
|
||||
remaining.extend(left)
|
||||
if skips:
|
||||
job.record_counted(
|
||||
"skipped",
|
||||
len(skips),
|
||||
[
|
||||
{
|
||||
"path": item.target[2],
|
||||
"root": item.label,
|
||||
"root_index": item.target[0],
|
||||
"detail": detail,
|
||||
}
|
||||
for item, detail in skips
|
||||
],
|
||||
)
|
||||
skipped += len(skips)
|
||||
job.update(processed=skipped, progress=skipped * 100 // total)
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
if skipped:
|
||||
job.log_line(f"Skipped {_plural(skipped, 'image')} before tagging")
|
||||
return remaining
|
||||
|
||||
async def _resolve_job_filter(self, spec: _Spec):
|
||||
"""``spec`` with its filter resolved into targets, or ``(None, error)``.
|
||||
|
||||
The listing's own identities, in its order, kept only where a target
|
||||
could have named them (a media path the job can tag).
|
||||
"""
|
||||
cap = MAX_FILTER_TARGETS
|
||||
try:
|
||||
selection = await self.resolve_listing_filter(spec.filter, cap)
|
||||
except ListingSourceChanged:
|
||||
return None, _coded("source-changed")
|
||||
except CollectionMissing:
|
||||
return None, collection_not_found()
|
||||
except ValueError as exc:
|
||||
return None, bad_request(str(exc))
|
||||
except Exception:
|
||||
# Never echo it: it can carry SQL or database text.
|
||||
self.logger.error("A tagging filter could not be resolved", exc_info=True)
|
||||
return None, _error(500, "listing-failed", FILTER_LISTING_FAILED)
|
||||
if selection.truncated:
|
||||
return None, _error(400, "too-many-images", TOO_MANY_IMAGES.format(cap=cap))
|
||||
targets = []
|
||||
for index, scope, relative in selection.targets:
|
||||
target, error = _target(
|
||||
{"root_index": index, "root_scope": scope, "path": relative}
|
||||
)
|
||||
if error is None:
|
||||
targets.append(target)
|
||||
if not targets:
|
||||
return None, _error(400, "no-images", NO_IMAGES)
|
||||
return dataclasses.replace(spec, targets=tuple(targets)), None
|
||||
|
||||
# -- runner ----------------------------------------------------------------
|
||||
|
||||
def _job_roots(self, targets) -> dict:
|
||||
@@ -331,28 +744,39 @@ class ImageTagJobsMixin:
|
||||
job.finish("failed")
|
||||
return
|
||||
if not await self._tag_each_file(run):
|
||||
job.update(status_line=STORAGE_RESTORED)
|
||||
job.log_line(f"{verb} stopped · {STORAGE_RESTORED}")
|
||||
job.finish("failed")
|
||||
return
|
||||
state = "cancelled" if job.cancelled else "finished"
|
||||
job.log_line(f"{verb} {state} · {_plural(job.processed, 'file')} processed")
|
||||
# An ended job never keeps the loop's "about X min left" (#379).
|
||||
ended = f"{verb} {state} · {_plural(job.processed, 'file')} processed"
|
||||
job.update(status_line=ended)
|
||||
job.log_line(ended)
|
||||
job.publish({"type": state, "progress": job.progress})
|
||||
job.finish(state)
|
||||
finally:
|
||||
if owner is not None:
|
||||
model_gate.release(owner)
|
||||
|
||||
async def _tag_each_file(self, run: _Run) -> bool:
|
||||
"""Tag the targets in order; False when the job had to stop."""
|
||||
job, total = run.job, len(run.spec.targets)
|
||||
async def _tag_each_file(self, run: _Run, *, done_before: int = 0) -> bool:
|
||||
"""Tag the targets in order; False when the job had to stop.
|
||||
|
||||
``done_before`` files were already counted (the batch's pre-pass): file
|
||||
n of this loop is "file done_before + n" of the job, and the time left
|
||||
uses this loop's rate only.
|
||||
"""
|
||||
job, count = run.job, len(run.spec.targets)
|
||||
total = done_before + count
|
||||
run.roots = await self._run_in_executor(self._job_roots, run.spec.targets)
|
||||
started = _monotonic()
|
||||
for number, target in enumerate(run.spec.targets, 1):
|
||||
if job.cancelled:
|
||||
break
|
||||
# The scan's estimate (D56), with each finished file as one batch.
|
||||
left = _time_left(number - 1, total, number - 1, _monotonic() - started)
|
||||
job.update(status_line=f"Processing file {number:,} of {total:,}{left}")
|
||||
left = _time_left(number - 1, count, number - 1, _monotonic() - started)
|
||||
shown = done_before + number
|
||||
job.update(status_line=f"Processing file {shown:,} of {total:,}{left}")
|
||||
job.publish({"type": "progress", "progress": job.progress})
|
||||
stopped = False
|
||||
try:
|
||||
@@ -363,20 +787,20 @@ class ImageTagJobsMixin:
|
||||
self.logger.exception("Image tagging failed for one file")
|
||||
group, detail = "errors", SAVE_FAILED
|
||||
self._record_tag_outcome(run, target, group, detail)
|
||||
job.update(processed=number, progress=number * 100 // total)
|
||||
job.update(processed=shown, progress=shown * 100 // total)
|
||||
if stopped:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _prepare_tagging_model(self, job, spec: _Spec, owner):
|
||||
"""The loaded service, or None (logged) when the job cannot run."""
|
||||
from ..autotag import MODELS, get_autotag_service
|
||||
from ..autotag import MODELS, MissingBackendPackages, get_autotag_service
|
||||
|
||||
service = get_autotag_service()
|
||||
status = await self._run_in_executor(service.get_models_status)
|
||||
if not status.get(spec.model_type, {}).get("downloaded"):
|
||||
name = MODELS.get(spec.model_type, {}).get("name", spec.model_type)
|
||||
message = unavailable_message(name)
|
||||
name = MODELS.get(spec.model_type, {}).get("name", spec.model_type)
|
||||
message = model_refusal(status, spec.model_type, name)
|
||||
if message is not None:
|
||||
job.update(status_line=message)
|
||||
job.log_line(f"! {message}")
|
||||
return None
|
||||
@@ -388,6 +812,11 @@ class ImageTagJobsMixin:
|
||||
spec.model_type,
|
||||
spec.options["use_gpu"],
|
||||
)
|
||||
except MissingBackendPackages as exc:
|
||||
# A package gone since the status read (load_model logged it).
|
||||
job.update(status_line=str(exc))
|
||||
job.log_line(f"! {exc}")
|
||||
return None
|
||||
except Exception:
|
||||
self.logger.exception("The auto-tag job could not load its model")
|
||||
job.update(status_line=INFERENCE_FAILED)
|
||||
@@ -439,9 +868,17 @@ class ImageTagJobsMixin:
|
||||
store.release(capture.token)
|
||||
|
||||
async def _auto_tag_capture(self, run: _Run, store, capture, root, relative):
|
||||
"""Suggest on the capture, then union only onto the same current bytes."""
|
||||
"""Suggest on the capture, then union only onto the same current bytes.
|
||||
|
||||
The bytes are marked auto-tagged with the union, also when the model
|
||||
returned no names; captured bytes no model can read are marked too
|
||||
(and stay skipped), so a "Skip already tagged" run passes them by.
|
||||
"""
|
||||
if not capture.inferable:
|
||||
return "skipped", NOT_A_STILL
|
||||
group, detail = await self._union_image_tags(
|
||||
capture, [], run.epoch, mark_autotagged=True
|
||||
)
|
||||
return (group, detail) if group == "errors" else ("skipped", NOT_A_STILL)
|
||||
if store.acquire(capture.token) is None:
|
||||
raise _JobStopped # every capture was invalidated (restore)
|
||||
spec = run.spec
|
||||
@@ -467,21 +904,30 @@ class ImageTagJobsMixin:
|
||||
if facts.locator != capture.locator or facts.sha256 != capture.sha256:
|
||||
return "skipped", FILE_CHANGED
|
||||
names = [str(name) for name, _score in pairs]
|
||||
if not names:
|
||||
return "unchanged", NO_NEW_TAGS
|
||||
return await self._union_image_tags(facts, names, run.epoch)
|
||||
return await self._union_image_tags(
|
||||
facts, names, run.epoch, mark_autotagged=True
|
||||
)
|
||||
|
||||
async def _union_image_tags(self, facts, tags: list, epoch: str):
|
||||
"""``add_image_tags`` on the verified bytes, as a (group, detail)."""
|
||||
async def _union_image_tags(
|
||||
self, facts, tags: list, epoch: str, *, mark_autotagged: bool = False
|
||||
):
|
||||
"""``add_image_tags`` on the verified bytes, as a (group, detail).
|
||||
|
||||
``mark_autotagged`` (a model's result) marks the bytes auto-tagged in
|
||||
the same write; then ``tags`` may be empty.
|
||||
"""
|
||||
try:
|
||||
receipt = await self._run_in_executor(
|
||||
self.db.add_image_tags,
|
||||
locator=facts.locator,
|
||||
sha256=facts.sha256,
|
||||
size=facts.size,
|
||||
mtime_ns=facts.mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
tags=tags,
|
||||
functools.partial(
|
||||
self.db.add_image_tags,
|
||||
locator=facts.locator,
|
||||
sha256=facts.sha256,
|
||||
size=facts.size,
|
||||
mtime_ns=facts.mtime_ns,
|
||||
expected_epoch=epoch,
|
||||
tags=tags,
|
||||
mark_autotagged=mark_autotagged,
|
||||
)
|
||||
)
|
||||
except LibraryCommandError as exc:
|
||||
if exc.code == "storage-changed":
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
from typing import Any, Callable
|
||||
|
||||
try:
|
||||
from ...utils.prompt_inspector import (
|
||||
@@ -93,6 +93,35 @@ def _in_host_temp(path: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def image_tags_reader(
|
||||
db: Any, locator: str, logger: Any, *, close: bool = False
|
||||
) -> Callable[[int, int], dict]:
|
||||
"""The inspector's ``image_tags`` reader for one file *locator* (S1).
|
||||
|
||||
The returned function takes the inspection's own stable stat and answers
|
||||
``current`` only for a byte record verified at exactly that
|
||||
``(size, mtime_ns)``; a failed read is ``unavailable``, never an error.
|
||||
``close`` closes the calling thread's connection afterwards, for routes
|
||||
that close theirs after reading (History).
|
||||
"""
|
||||
|
||||
def read(size: int, mtime_ns: int) -> dict:
|
||||
try:
|
||||
found = db.current_image_tags({locator: (size, mtime_ns)})
|
||||
except Exception:
|
||||
logger.warning("Inspection image tags could not be read", exc_info=True)
|
||||
return {"state": "unavailable", "tags": []}
|
||||
finally:
|
||||
if close:
|
||||
db.model.close()
|
||||
tags = found.get(locator)
|
||||
if tags is None:
|
||||
return {"state": "none", "tags": []}
|
||||
return {"state": "current", "tags": list(tags)}
|
||||
|
||||
return read
|
||||
|
||||
|
||||
def inspect_current_file(
|
||||
path: Path,
|
||||
*,
|
||||
@@ -100,18 +129,23 @@ def inspect_current_file(
|
||||
saved: dict | None,
|
||||
library: dict,
|
||||
image_tags: Callable[[int, int], dict] | None = None,
|
||||
recorded: Callable[[Path], Any] | None = None,
|
||||
) -> dict:
|
||||
"""Current-file evidence read between two equal stats of *path*.
|
||||
|
||||
*image_tags*, when given, is called once with that stable
|
||||
``(st_size, st_mtime_ns)`` and its facet is returned as ``image_tags``
|
||||
(spec amendment S1: tags of a byte record verified at exactly that stat).
|
||||
*recorded*, when given, is asked within the same stat window for what
|
||||
recorded run history knows about *path* (``RecordedOutput`` or None: the
|
||||
save node that wrote it and the texts PromptManager recorded, #386).
|
||||
"""
|
||||
before = _current_file_facts(path)
|
||||
try:
|
||||
inspection = read_inspection_file(path)
|
||||
except MetadataReadError:
|
||||
inspection = None
|
||||
run = recorded(path) if recorded is not None and inspection is not None else None
|
||||
after = _current_file_facts(path)
|
||||
fields = ("st_dev", "st_ino", "st_mode", "st_size", "st_mtime_ns", "st_ctime_ns")
|
||||
if any(getattr(before, key) != getattr(after, key) for key in fields):
|
||||
@@ -129,8 +163,15 @@ def inspect_current_file(
|
||||
profile = (
|
||||
PREVIEW_INSPECTION_PROFILE if _in_host_temp(path) else INSPECTION_PROFILE
|
||||
)
|
||||
# The file name anchors on the saver that wrote it (#381); recorded run
|
||||
# history names that saver when the file alone cannot (#386).
|
||||
evidence = inspect_prompt_evidence(
|
||||
inspection.metadata, role_saved, profile=profile
|
||||
inspection.metadata,
|
||||
role_saved,
|
||||
profile=profile,
|
||||
file_name=path.name,
|
||||
recorded_saver_id=run.saver_id if run is not None else None,
|
||||
recorded_texts=run.texts if run is not None else None,
|
||||
)
|
||||
return {
|
||||
"file": {
|
||||
|
||||