Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d0bfed3614 | ||
|
|
f8bc68b5d1 | ||
|
|
4ff6d6f134 |
@@ -1,9 +1,9 @@
|
||||
name: Windows
|
||||
name: Windows and macOS
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Non-blocking Windows runs (#420; macOS dropped from CI in #571): never on pull requests or in the merge queue, and no context
|
||||
# Non-blocking portability runs (#420): never on pull requests or in the merge queue, and no context
|
||||
# here is a required check. A red run is fixed at its cause before the workspace cutover (#334).
|
||||
on:
|
||||
push:
|
||||
@@ -21,23 +21,19 @@ env:
|
||||
PYTHONFAULTHANDLER: "1"
|
||||
|
||||
# Time limit rule as in test.yml: ceil(1.5 x the slowest of three recent green runs on main) + 3
|
||||
# minutes, never under 5, timed as whole-job wall time. One matrix job, so the slowest leg of each
|
||||
# run counts (tests/vendor/ci-workflow.test.mjs recomputes it, #493).
|
||||
# Measured 2026-10-29 on main runs 40935337099 (dd247e3), 40938407252 (dd247e3), 40941240624 (dd247e3):
|
||||
# slowest leg: 7m24s (py3.12 windows) / 7m31s (py3.13 windows) / 7m32s (py3.14 windows) -> 15 min
|
||||
# (windows 4m56s-7m32s)
|
||||
# native-trash-linux (#543) has no green main runs yet: it installs the dev requirements and runs
|
||||
# one test module, about 1 minute by the same estimate, so ceil(1.5 x 2) + 3 gives it 6 minutes.
|
||||
# Recalibrate after three green runs.
|
||||
# minutes, never under 5. Measured 2026-10-26 in test.yml on runs 40257217681 (3be0ada),
|
||||
# 40240706568 (fb9134f), 40228167328 (f9835aa), before these legs moved here in #420:
|
||||
# slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 27
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [windows-latest]
|
||||
os: [windows-latest, macos-latest]
|
||||
python-version: ["3.12", "3.13", "3.14"]
|
||||
|
||||
steps:
|
||||
@@ -55,63 +51,5 @@ jobs:
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# The runner's TEMP is on C:, the OS disk, where a flush took about 6 ms
|
||||
# and a test database's create-to-delete about 111 ms; on the D: work
|
||||
# disk the same took 0.5 ms and 23 ms. Tests write their
|
||||
# databases under TEMP, so it moves to a folder on D: (measured in #487,
|
||||
# run 40915866444). The runner's TEMP was an 8.3 short name
|
||||
# (C:\Users\RUNNER~1\...), so the new one is spelled by its short name too
|
||||
# and paths still have to be resolved. D: makes no short names by default
|
||||
# (run 40922502209), so they are switched on for it first.
|
||||
- name: Keep test files on the work disk (Windows)
|
||||
if: runner.os == 'Windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
fsutil 8dot3name set D: 0
|
||||
$folder = New-Item -ItemType Directory -Force -Path (Join-Path $env:RUNNER_TEMP "PromptManagerTests")
|
||||
$short = (New-Object -ComObject Scripting.FileSystemObject).GetFolder($folder.FullName).ShortPath
|
||||
Write-Output "test TEMP: $short (long name $($folder.FullName))"
|
||||
if ($short -eq $folder.FullName) { Write-Output "::warning::D: gave the test TEMP no 8.3 short name" }
|
||||
"TEMP=$short" >> $env:GITHUB_ENV
|
||||
"TMP=$short" >> $env:GITHUB_ENV
|
||||
|
||||
- name: Run tests
|
||||
env:
|
||||
PM_TEST_SLOWEST: "40"
|
||||
run: python -m tests.run_suite -v
|
||||
|
||||
# The same tests as the fake mount table, on a real ext4 mount used as the output root (#543). A
|
||||
# runner can mount with sudo; the tests run as the normal user, so owner and mode checks are real.
|
||||
native-trash-linux:
|
||||
name: native trash on a real mount
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 6
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
- name: Mount an ext4 image
|
||||
run: |
|
||||
truncate -s 64M "$RUNNER_TEMP/drive.img"
|
||||
mkfs.ext4 -q "$RUNNER_TEMP/drive.img"
|
||||
mkdir "$RUNNER_TEMP/drive"
|
||||
sudo mount -o loop "$RUNNER_TEMP/drive.img" "$RUNNER_TEMP/drive"
|
||||
sudo rmdir "$RUNNER_TEMP/drive/lost+found"
|
||||
sudo chown "$(id -u):$(id -g)" "$RUNNER_TEMP/drive"
|
||||
findmnt "$RUNNER_TEMP/drive"
|
||||
|
||||
- name: Native trash proof
|
||||
timeout-minutes: 4
|
||||
run: PM_TRASH_MOUNT="$RUNNER_TEMP/drive" python -m unittest tests.test_trash_mount_root -v
|
||||
|
||||
@@ -28,8 +28,8 @@ env:
|
||||
# py matrix, slowest py3.10 on windows-latest: 12m43s / 15m54s / 14m53s -> 27 min
|
||||
# (ubuntu 5m27s-6m27s, macos 6m14s-8m30s, windows 11m42s-15m54s)
|
||||
# 3.10 and 3.11 left the matrix and 3.14 joined it in #356; re-measure after three green main runs.
|
||||
# Windows moved to portability.yml (#420; macOS left CI in #571), which has its own measured limit (#493);
|
||||
# this Linux-only matrix keeps 27 minutes until it is re-measured.
|
||||
# Windows and macOS moved to portability.yml (#420), which keeps this 27-minute limit; this
|
||||
# Linux-only matrix keeps it too until it is re-measured.
|
||||
# real ComfyUI compatibility: 13m32s / 8m55s / 11m46s -> 24 min
|
||||
# coverage gate (85%): 7m26s / 7m22s / 7m54s -> 15 min
|
||||
# perf budget 30s / 29s / 27s, black and flake8 39s / 37s / 39s,
|
||||
@@ -39,7 +39,7 @@ env:
|
||||
# Pull requests run a fast subset (#420): Python 3.12 on ubuntu-latest, coverage, lint and frontend
|
||||
# modules. The merge queue (merge_group), pushes to main and manual runs add Python 3.13 and 3.14 on
|
||||
# Linux and the real ComfyUI job. "all tests" is the required gate that holds the queue to that set.
|
||||
# Windows runs in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
||||
# Windows and macOS run in portability.yml (push to main, nightly, on demand) and never gate a merge.
|
||||
jobs:
|
||||
test:
|
||||
name: py${{ matrix.python-version }} on ${{ matrix.os }}
|
||||
|
||||
@@ -36,18 +36,9 @@ concurrency:
|
||||
# 12m53s, node units and the viewer suite before 11.8m of workflows), so coverage is the tight one
|
||||
# in its 18 minutes. This holds only when the npm cache hits: a cold cache adds up to 5m to npm ci,
|
||||
# and then the job limit can still end the job before the step limit does.
|
||||
# #473: firefox/webkit shard 1/2 reached its 18-minute step limit (queue run 40866252975: webkit 622 tests
|
||||
# passed in 17m56s, 48.5 worker-minutes vs 28.0 in shard 2/2; recent shard 1/2 jobs 13.7m-18.9m, shard 2/2
|
||||
# 7.3m-12.2m), because --shard splits by test count and the slow concept specs sort into shard 1. The page
|
||||
# workflows now run in three shards per browser with the limits unchanged; the per-shard times are
|
||||
# re-measured after three green main runs.
|
||||
# #563: coverage shards reached their 18-minute step limit three times on 2026-10-31 with no failing
|
||||
# test (PR #560 shard 2/3: 814 passed in 17.9m; queue run 41206495451 shard 1/3: 827 passed; PR #564
|
||||
# shard 2/3 twice), against recent green shards of 11m-17m. Coverage now runs in four shards with the
|
||||
# limits unchanged; the per-shard times are re-measured after three green main runs.
|
||||
jobs:
|
||||
frontend-coverage:
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/4)
|
||||
name: frontend coverage (shard ${{ matrix.shard }}/3)
|
||||
runs-on: ubuntu-latest
|
||||
# Layout comparisons depend on the fonts each browser falls back to, so browser jobs run in the
|
||||
# same pinned Playwright image as local runs (tag = @playwright/test in tools/vendor/package-lock.json;
|
||||
@@ -63,7 +54,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4]
|
||||
shard: [1, 2, 3]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -92,13 +83,13 @@ jobs:
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
# Shard 1 also runs the node units and the viewer suite, so the vendor pins and the tooling tests
|
||||
# run on the lightest shard instead.
|
||||
- if: matrix.shard == 4
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor check
|
||||
- if: matrix.shard == 4
|
||||
- if: matrix.shard == 3
|
||||
run: npm run --prefix tools/vendor test
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/4
|
||||
- name: Page workflows with coverage, shard ${{ matrix.shard }}/3
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/4
|
||||
run: npm run --prefix tools/vendor test:coverage:shard -- ${{ matrix.shard }}/3
|
||||
- name: Upload raw coverage
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
@@ -140,13 +131,13 @@ jobs:
|
||||
with:
|
||||
pattern: coverage-raw-*
|
||||
path: ${{ runner.temp }}/coverage-raw
|
||||
# coverage:merge only checks that raw files exist; merging three of four shards would report
|
||||
# coverage:merge only checks that raw files exist; merging two of three shards would report
|
||||
# partial coverage, so a missing or empty shard artifact fails here.
|
||||
- name: Verify every shard's raw coverage arrived
|
||||
shell: bash
|
||||
env:
|
||||
STAGED_RAW: ${{ runner.temp }}/coverage-raw
|
||||
EXPECTED_SHARDS: 4
|
||||
EXPECTED_SHARDS: 3
|
||||
run: |
|
||||
found=0
|
||||
for shard in $(seq 1 "$EXPECTED_SHARDS"); do
|
||||
@@ -184,7 +175,7 @@ jobs:
|
||||
retention-days: 14
|
||||
|
||||
browser-workflows:
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/3)
|
||||
name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2)
|
||||
# Merge queue, main and manual runs only (#420); pull requests run the Chromium/touch coverage shards.
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
@@ -203,7 +194,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: [firefox, webkit]
|
||||
shard: [1, 2, 3]
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
# The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache
|
||||
@@ -230,9 +221,9 @@ jobs:
|
||||
path: ~/.npm
|
||||
key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }}
|
||||
- run: npm ci --prefix tools/vendor --ignore-scripts
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/3
|
||||
- name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2
|
||||
timeout-minutes: 18
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/3
|
||||
run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2
|
||||
# Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step.
|
||||
- name: Upload browser traces
|
||||
if: always()
|
||||
|
||||
@@ -92,20 +92,10 @@ try:
|
||||
routes = config.routes
|
||||
api = PromptManagerAPI()
|
||||
api.add_routes(routes)
|
||||
|
||||
# The thumbnail location's startup check (#320), then the temp files of
|
||||
# preview writes a crash interrupted (#317), off the import path: the
|
||||
# walk may cover a large thumbnails folder.
|
||||
# #320: the configured thumbnail location is in force before any request
|
||||
# (no I/O); the thread below checks it.
|
||||
api.use_preview_location()
|
||||
|
||||
def _preview_startup():
|
||||
api.start_preview_location()
|
||||
api.sweep_thumbnail_temps()
|
||||
|
||||
# Temp files of preview writes a crash interrupted (#317), off the
|
||||
# import path: the walk may cover a large thumbnails folder.
|
||||
threading.Thread(
|
||||
target=_preview_startup,
|
||||
target=api.sweep_thumbnail_temps,
|
||||
name="PromptManagerPreviewTempSweep",
|
||||
daemon=True,
|
||||
).start()
|
||||
@@ -130,7 +120,6 @@ except Exception as e:
|
||||
logger = get_logger("prompt_manager.init")
|
||||
logger.error(f"Failed to register API routes: {e}")
|
||||
except Exception:
|
||||
# logging failed too; a report must never stop the node loading
|
||||
pass
|
||||
|
||||
# Shared database, set up with the monitoring system below. Stays None when that
|
||||
@@ -290,11 +279,11 @@ def _teardown_logger():
|
||||
|
||||
|
||||
def _stop_background_work():
|
||||
"""Stop the image monitor, the preview worker, the job worker threads
|
||||
and the prompt cleanup thread against one TEARDOWN_SECONDS deadline, once.
|
||||
"""Stop the image monitor, the preview worker and the prompt cleanup
|
||||
thread against one TEARDOWN_SECONDS deadline, once.
|
||||
|
||||
Returns ``{"monitor", "thumbnails", "workers", "tracker"}`` booleans
|
||||
(True: it stopped), leaving out parts that never started. Later calls return a
|
||||
Returns ``{"monitor", "thumbnails", "tracker"}`` booleans (True: it
|
||||
stopped), leaving out parts that never started. Later calls return a
|
||||
copy of the first result. A Ctrl+C inside a step is logged and the
|
||||
step is False; one between steps is re-raised after the result (parts
|
||||
not reached are False) is stored and the ``atexit`` entry removed.
|
||||
@@ -308,14 +297,11 @@ def _stop_background_work():
|
||||
api,
|
||||
_global_prompt_tracker,
|
||||
)
|
||||
from .utils import parallel
|
||||
|
||||
# A part whose global is set has started; False until its step says
|
||||
# otherwise, so an interrupt between steps leaves an honest result.
|
||||
started = (
|
||||
("monitor", monitor),
|
||||
("thumbnails", thumbnails),
|
||||
("workers", parallel.job_pool_started() or None),
|
||||
("tracker", tracker),
|
||||
)
|
||||
result = {name: False for name, part in started if part is not None}
|
||||
@@ -358,16 +344,6 @@ def _stop_background_work():
|
||||
result["thumbnails"] = _teardown_step(
|
||||
"the preview worker", close_previews, logger
|
||||
)
|
||||
if "workers" in result:
|
||||
|
||||
def stop_workers():
|
||||
from .utils.parallel import JOB_POOL_JOIN_SECONDS
|
||||
|
||||
return parallel.shutdown_job_pool(left(JOB_POOL_JOIN_SECONDS))
|
||||
|
||||
result["workers"] = _teardown_step(
|
||||
"the job worker threads", stop_workers, logger
|
||||
)
|
||||
if tracker is not None:
|
||||
|
||||
def stop_tracker():
|
||||
@@ -435,7 +411,6 @@ except Exception as e:
|
||||
f"Failed to register the exit signal: {e}"
|
||||
)
|
||||
except Exception:
|
||||
# logging failed too; a report must never stop the node loading
|
||||
pass
|
||||
|
||||
try:
|
||||
@@ -453,14 +428,16 @@ except Exception as e:
|
||||
f"Failed to register the background work teardown: {e}"
|
||||
)
|
||||
except Exception:
|
||||
# logging failed too; a report must never stop the node loading
|
||||
pass
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
|
||||
# One startup line. ASCII only: a Windows console using a legacy code page
|
||||
# cannot print emoji, and a UnicodeEncodeError here would abort loading the node.
|
||||
print(
|
||||
f"\033[94m[ComfyUI-PromptManager] Version:\033[0m {get_version()}"
|
||||
f" - {len(NODE_CLASS_MAPPINGS)} tools loaded"
|
||||
)
|
||||
# Print startup message with loaded tools
|
||||
print()
|
||||
print(f"\033[94m[ComfyUI-PromptManager] Version:\033[0m {get_version()}")
|
||||
for node_key, display_name in NODE_DISPLAY_NAME_MAPPINGS.items():
|
||||
# ASCII only: a Windows console using a legacy code page cannot print emoji,
|
||||
# and a UnicodeEncodeError here would abort loading the whole node.
|
||||
print(f" \033[94mLoaded:\033[0m {display_name}")
|
||||
print(f"\033[94mTotal: {len(NODE_CLASS_MAPPINGS)} tools loaded\033[0m")
|
||||
print()
|
||||
|
||||
|
Before Width: | Height: | Size: 35 KiB |
|
Before Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 153 KiB |
|
Before Width: | Height: | Size: 82 KiB |
|
Before Width: | Height: | Size: 72 KiB |
|
Before Width: | Height: | Size: 156 KiB |
|
Before Width: | Height: | Size: 159 KiB |
|
Before Width: | Height: | Size: 96 KiB |
|
Before Width: | Height: | Size: 165 KiB |
|
Before Width: | Height: | Size: 60 KiB |
|
Before Width: | Height: | Size: 38 KiB |
|
Before Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 42 KiB |
|
Before Width: | Height: | Size: 39 KiB |
|
Before Width: | Height: | Size: 42 KiB |
|
Before Width: | Height: | Size: 40 KiB |
|
Before Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 43 KiB |
|
Before Width: | Height: | Size: 54 KiB |
|
Before Width: | Height: | Size: 237 KiB |
|
Before Width: | Height: | Size: 66 KiB |
|
Before Width: | Height: | Size: 53 KiB |
|
Before Width: | Height: | Size: 54 KiB |
|
Before Width: | Height: | Size: 181 KiB |
|
Before Width: | Height: | Size: 51 KiB |
|
Before Width: | Height: | Size: 110 KiB |
@@ -40,7 +40,6 @@ from .annotation_schema import (
|
||||
METADATA_TABLE,
|
||||
)
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .image_removal_schema import clear_marks
|
||||
from .library_commands import _UUID, LibraryCommandError, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .vocabulary import validate_vocabulary_tag_name
|
||||
@@ -237,9 +236,6 @@ class AnnotationOperations:
|
||||
|
||||
@staticmethod
|
||||
def _annotation_owner(conn: sqlite3.Connection, locator: str, now: str) -> str:
|
||||
# A file with image tags or Needs review state is in the library: a
|
||||
# removal mark never hides it (#480).
|
||||
clear_marks(conn, [locator])
|
||||
row = conn.execute(_OWNER, (locator,)).fetchone()
|
||||
if row is not None:
|
||||
return row[0]
|
||||
|
||||
@@ -7,11 +7,6 @@ from typing import Callable, ContextManager, Iterator
|
||||
|
||||
try:
|
||||
from ..utils.execution_history_events import RunFinish, encode_command
|
||||
from ..utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
stored_prompt_text,
|
||||
stored_prompt_text_sql,
|
||||
)
|
||||
from ..utils.execution_contexts import (
|
||||
CAPTURE_REASONS,
|
||||
ContextBundle,
|
||||
@@ -20,11 +15,6 @@ try:
|
||||
)
|
||||
except ImportError:
|
||||
from utils.execution_history_events import RunFinish, encode_command
|
||||
from utils.hashing import (
|
||||
generate_prompt_hash,
|
||||
stored_prompt_text,
|
||||
stored_prompt_text_sql,
|
||||
)
|
||||
from utils.execution_contexts import (
|
||||
CAPTURE_REASONS,
|
||||
ContextBundle,
|
||||
@@ -144,8 +134,7 @@ def _insert_exact(
|
||||
|
||||
def _role_hints(role: str) -> str:
|
||||
return (
|
||||
f"SELECT r.{role}_prompt_id, tx.text_value, "
|
||||
f"{stored_prompt_text_sql('tx.text_value')} "
|
||||
f"SELECT r.{role}_prompt_id, tx.text_value "
|
||||
"FROM execution_run_contexts r "
|
||||
"JOIN execution_contexts c ON c.context_id = r.context_id "
|
||||
f"JOIN execution_prompt_snapshots s ON s.snapshot_id = c.{role}_snapshot_id "
|
||||
@@ -157,78 +146,57 @@ def _role_hints(role: str) -> str:
|
||||
|
||||
# Set form of the read-time prompt-hint rule in get_execution_contexts():
|
||||
# a stored role hint is live only in the current storage epoch, and only when
|
||||
# the hinted prompt is the one match of _prompt_hint: by text (the raw
|
||||
# snapshot text or its stored form, outer whitespace stripped, #438), or, only
|
||||
# when no prompt has either text, by dedup hash (case folded too, #460).
|
||||
# The hash is a fallback, so a hint #438 kept is never made ambiguous.
|
||||
# Needs register_prompt_functions (pm_prompt_hash). The hint rows are the
|
||||
# outer loop (CROSS JOIN), so each prompts lookup is by index.
|
||||
# its exact raw snapshot text matches exactly one prompt, which is the hinted
|
||||
# one (_prompt_hint). Evaluated once per statement, never per prompt row.
|
||||
# Defines the CTE ``live_hint_prompts(prompt_id)``; prefix it with WITH.
|
||||
LIVE_HINT_CTES = (
|
||||
"stored_hints(prompt_id, raw, stored) AS ("
|
||||
"stored_hints(prompt_id, raw) AS ("
|
||||
+ _role_hints("positive")
|
||||
+ " UNION "
|
||||
+ _role_hints("negative")
|
||||
+ "), hint_matches(prompt_id, raw, stored, match_id) AS ("
|
||||
"SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h "
|
||||
"CROSS JOIN prompts p ON p.text COLLATE BINARY IN (h.raw, h.stored) "
|
||||
"UNION ALL "
|
||||
"SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h "
|
||||
"CROSS JOIN prompts p ON p.hash = pm_prompt_hash(h.raw) "
|
||||
"WHERE NOT EXISTS (SELECT 1 FROM prompts q "
|
||||
"WHERE q.text COLLATE BINARY IN (h.raw, h.stored))"
|
||||
+ "), unique_hint_texts(prompt_id, raw) AS ("
|
||||
"SELECT MIN(id), text COLLATE BINARY FROM prompts "
|
||||
"WHERE text COLLATE BINARY IN (SELECT raw FROM stored_hints) "
|
||||
"GROUP BY text COLLATE BINARY HAVING COUNT(*) = 1"
|
||||
"), live_hint_prompts(prompt_id) AS ("
|
||||
"SELECT prompt_id FROM hint_matches GROUP BY prompt_id, raw, stored "
|
||||
"HAVING COUNT(*) = 1 AND MIN(match_id) = prompt_id)"
|
||||
"SELECT h.prompt_id FROM stored_hints h JOIN unique_hint_texts u "
|
||||
"ON u.prompt_id = h.prompt_id AND u.raw = h.raw COLLATE BINARY)"
|
||||
)
|
||||
LIVE_HINT_PROMPT_IDS_SQL = (
|
||||
"WITH " + LIVE_HINT_CTES + " SELECT DISTINCT prompt_id FROM live_hint_prompts"
|
||||
)
|
||||
|
||||
_BY_TEXT = "SELECT id FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 2"
|
||||
|
||||
|
||||
def _text_matches(conn: sqlite3.Connection, raw: str) -> list:
|
||||
return [row[0] for row in conn.execute(_BY_TEXT, (raw, stored_prompt_text(raw)))]
|
||||
|
||||
|
||||
def _prompt_hint(conn: sqlite3.Connection, raw: str | None) -> int | None:
|
||||
"""The one prompt that stores ``raw`` (#438), else the one it counts toward.
|
||||
|
||||
By text first: ``raw`` or ``raw.strip()`` (the exact form keeps hints of
|
||||
older rows that kept outer whitespace). Only when no prompt has either
|
||||
text, by the dedup hash, which also folds case (#460).
|
||||
"""
|
||||
if raw is None:
|
||||
return None
|
||||
matches = _text_matches(conn, raw)
|
||||
if not matches:
|
||||
matches = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT id FROM prompts WHERE hash=? LIMIT 2",
|
||||
(generate_prompt_hash(raw),),
|
||||
)
|
||||
]
|
||||
return matches[0] if len(matches) == 1 else None
|
||||
rows = conn.execute(
|
||||
"SELECT id FROM prompts WHERE text=? COLLATE BINARY LIMIT 2", (raw,)
|
||||
).fetchall()
|
||||
return rows[0][0] if len(rows) == 1 else None
|
||||
|
||||
|
||||
def _hints(
|
||||
conn: sqlite3.Connection, positive: str | None, negative: str | None
|
||||
) -> tuple:
|
||||
pos_id, neg_id = _prompt_hint(conn, positive), _prompt_hint(conn, negative)
|
||||
stored = None if negative is None else stored_prompt_text(negative)
|
||||
pair_id = None
|
||||
# A known negative that names no prompt because several share its text
|
||||
negative_ambiguous = (
|
||||
stored is not None and neg_id is None and bool(_text_matches(conn, negative))
|
||||
negative is not None
|
||||
and neg_id is None
|
||||
and conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE text=? COLLATE BINARY LIMIT 1", (negative,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
if pos_id is not None and stored is not None and not negative_ambiguous:
|
||||
if pos_id is not None and negative is not None and not negative_ambiguous:
|
||||
rows = conn.execute(
|
||||
"SELECT pp.id FROM prompt_pairs pp "
|
||||
"SELECT pp.id FROM prompt_pairs pp LEFT JOIN prompts n ON "
|
||||
"n.id=pp.negative_prompt_id "
|
||||
"WHERE pp.positive_prompt_id=? AND ((pp.negative_prompt_id IS NULL AND "
|
||||
"?='') OR pp.negative_prompt_id=?) LIMIT 2",
|
||||
(pos_id, stored, neg_id),
|
||||
"?='') "
|
||||
"OR (pp.negative_prompt_id=? AND n.text=? COLLATE BINARY)) LIMIT 2",
|
||||
(pos_id, negative, neg_id, negative),
|
||||
).fetchall()
|
||||
if len(rows) == 1:
|
||||
pair_id = rows[0][0]
|
||||
|
||||
@@ -23,10 +23,10 @@ from pathlib import PurePosixPath, PureWindowsPath
|
||||
from urllib.parse import quote
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
from .models import normalize_image_path
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
from utils.hashing import generate_prompt_hash
|
||||
from database.models import normalize_image_path
|
||||
|
||||
logger = logging.getLogger("prompt_manager.database.health")
|
||||
@@ -125,8 +125,12 @@ def _foreign_keys(conn, _roots):
|
||||
)
|
||||
|
||||
|
||||
def _prompt_hash(text):
|
||||
return generate_prompt_hash(text) if isinstance(text, str) else None
|
||||
|
||||
|
||||
def _canonical_hashes(conn, _roots):
|
||||
register_prompt_functions(conn) # pm_prompt_hash, shared with history (#460)
|
||||
conn.create_function("pm_prompt_hash", 1, _prompt_hash, deterministic=True)
|
||||
return _counted(
|
||||
f"Prompt {row[0]}: the stored hash does not match its text"
|
||||
for row in conn.execute(
|
||||
|
||||
@@ -15,11 +15,6 @@ from .execution_history import HistoryEpochChanged
|
||||
from .execution_outputs import _capture, _REFERENCE_COLUMNS
|
||||
from .library import _ELIGIBLE
|
||||
|
||||
try:
|
||||
from ..utils.hashing import prompt_key, stored_prompt_text
|
||||
except ImportError:
|
||||
from utils.hashing import prompt_key, stored_prompt_text
|
||||
|
||||
RunKey = tuple[int, str, int, str]
|
||||
|
||||
|
||||
@@ -105,43 +100,14 @@ _ORDER = (
|
||||
)
|
||||
# Stale-hint rows a newest-pair lookup inspects before it reports none.
|
||||
_PAIR_RUN_CANDIDATES = 16
|
||||
# The recorded texts of one scope, as a JSON array parameter (_scope_text_ids).
|
||||
_IN_SCOPE = "IN (SELECT value FROM json_each(?))"
|
||||
_MATCHING = (
|
||||
"SELECT DISTINCT rc.run_id FROM execution_prompt_snapshots s "
|
||||
"JOIN execution_contexts c ON c.positive_snapshot_id=s.snapshot_id "
|
||||
"JOIN execution_run_contexts rc ON rc.context_id=c.context_id "
|
||||
f"WHERE s.raw_text_id {_IN_SCOPE}"
|
||||
"WHERE s.raw_text_id=?"
|
||||
)
|
||||
|
||||
|
||||
def _scope_text_ids(conn: sqlite3.Connection, key: str) -> str:
|
||||
"""Every recorded raw text whose prompt key is ``key``, as a JSON array.
|
||||
|
||||
A run keeps its exact raw text, while the Library saves and counts it as
|
||||
the prompt with its key (outer whitespace #438, case #460). The values are
|
||||
compared, never only their digests; ``pm_prompt_key`` is Python's key.
|
||||
"""
|
||||
return json.dumps(
|
||||
[
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
# Only snapshot raw texts: effective and affix texts are skipped
|
||||
"SELECT t.text_id FROM (SELECT DISTINCT raw_text_id AS text_id "
|
||||
"FROM execution_prompt_snapshots) s CROSS JOIN "
|
||||
"execution_context_texts t WHERE t.text_id=s.text_id AND "
|
||||
"pm_prompt_key(t.text_value)=? COLLATE BINARY",
|
||||
(key,),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _scope_text_id(raw: str | None) -> str | None:
|
||||
"""The scope digest of a raw text: that of its prompt key."""
|
||||
return None if raw is None else content_digest("text", (prompt_key(raw),))
|
||||
|
||||
|
||||
def _limit(limit: int) -> None:
|
||||
if type(limit) is not int or not 1 <= limit <= 100:
|
||||
raise ValueError("History limit must be an integer from 1 to 100")
|
||||
@@ -289,15 +255,8 @@ class HistoryReadOperations:
|
||||
if saved is None:
|
||||
return None
|
||||
text = saved["text"]
|
||||
raw_id = _scope_text_id(text)
|
||||
# A position from before #460 names the scope by the stripped text
|
||||
# alone; it is the same scope, so a page open across the upgrade resumes.
|
||||
legacy_id = (
|
||||
None
|
||||
if text is None
|
||||
else content_digest("text", (stored_prompt_text(text),))
|
||||
)
|
||||
if after is not None and after.raw_text_id not in (raw_id, legacy_id):
|
||||
raw_id = None if text is None else content_digest("text", (text,))
|
||||
if after is not None and after.raw_text_id != raw_id:
|
||||
raise ValueError("History position text scope changed")
|
||||
group = {
|
||||
"positive_id": positive_id,
|
||||
@@ -314,11 +273,18 @@ class HistoryReadOperations:
|
||||
next_position=None,
|
||||
resume_position=None,
|
||||
)
|
||||
if raw_id is None:
|
||||
if (
|
||||
raw_id is None
|
||||
or conn.execute(
|
||||
"SELECT 1 FROM execution_context_texts WHERE text_id=? "
|
||||
"AND text_value=? COLLATE BINARY",
|
||||
(raw_id, text),
|
||||
).fetchone()
|
||||
is None
|
||||
):
|
||||
return result
|
||||
scope = _scope_text_ids(conn, prompt_key(text))
|
||||
where = f"r.run_id IN ({_MATCHING})"
|
||||
params = [scope]
|
||||
params = [raw_id]
|
||||
if after is not None:
|
||||
where += f" AND {_KEY}<=(?,?,?,?)"
|
||||
params.extend(after.upper)
|
||||
@@ -372,9 +338,9 @@ class HistoryReadOperations:
|
||||
"LEFT JOIN execution_prompt_snapshots n "
|
||||
"ON n.snapshot_id=c.negative_snapshot_id "
|
||||
"LEFT JOIN execution_context_texts t ON t.text_id=n.raw_text_id "
|
||||
f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id {_IN_SCOPE} "
|
||||
f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id=? "
|
||||
"ORDER BY rc.anchor_id COLLATE BINARY,rc.run_id COLLATE BINARY",
|
||||
(*by_run, scope),
|
||||
(*by_run, raw_id),
|
||||
)
|
||||
summaries = {}
|
||||
evidence_by_context = {}
|
||||
@@ -472,7 +438,6 @@ class HistoryReadOperations:
|
||||
if saved["text"] is None:
|
||||
return {"run": None}
|
||||
epoch = self.get_execution_epoch()
|
||||
key = prompt_key(saved["text"])
|
||||
candidates = conn.execute(
|
||||
"SELECT r.run_id,rc.anchor_id,rc.context_id,"
|
||||
"c.positive_snapshot_id,c.negative_snapshot_id "
|
||||
@@ -480,11 +445,10 @@ class HistoryReadOperations:
|
||||
"ON rc.run_id=r.run_id JOIN execution_contexts c "
|
||||
"ON c.context_id=rc.context_id JOIN execution_prompt_snapshots s "
|
||||
"ON s.snapshot_id=c.positive_snapshot_id "
|
||||
f"WHERE s.raw_text_id {_IN_SCOPE} AND rc.pair_id=? "
|
||||
"AND rc.reference_epoch=? "
|
||||
"WHERE s.raw_text_id=? AND rc.pair_id=? AND rc.reference_epoch=? "
|
||||
f"ORDER BY {_ORDER},rc.anchor_id COLLATE BINARY LIMIT ?",
|
||||
(
|
||||
_scope_text_ids(conn, key),
|
||||
content_digest("text", (saved["text"],)),
|
||||
pair_id,
|
||||
epoch,
|
||||
_PAIR_RUN_CANDIDATES,
|
||||
@@ -493,11 +457,7 @@ class HistoryReadOperations:
|
||||
for row in candidates:
|
||||
positive = _values(conn, row["positive_snapshot_id"])
|
||||
negative = _values(conn, row["negative_snapshot_id"])
|
||||
if (
|
||||
positive is None
|
||||
or positive["raw"] is None
|
||||
or prompt_key(positive["raw"]) != key
|
||||
):
|
||||
if positive is None or positive["raw"] != saved["text"]:
|
||||
continue
|
||||
hints = _hints(conn, positive["raw"], negative["raw"] if negative else None)
|
||||
if hints[2] == pair_id:
|
||||
@@ -517,9 +477,11 @@ class HistoryReadOperations:
|
||||
snapshots = {
|
||||
row["anchor_id"]: row
|
||||
for row in conn.execute(
|
||||
"SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id "
|
||||
"SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id,"
|
||||
"p.raw_text_id AS positive_raw_text_id "
|
||||
"FROM execution_run_contexts rc JOIN execution_contexts c "
|
||||
"ON c.context_id=rc.context_id WHERE rc.run_id=?",
|
||||
"ON c.context_id=rc.context_id LEFT JOIN execution_prompt_snapshots p "
|
||||
"ON p.snapshot_id=c.positive_snapshot_id WHERE rc.run_id=?",
|
||||
(run_id,),
|
||||
)
|
||||
}
|
||||
@@ -529,12 +491,12 @@ class HistoryReadOperations:
|
||||
context.update(
|
||||
{
|
||||
key: row[key]
|
||||
for key in ("positive_snapshot_id", "negative_snapshot_id")
|
||||
},
|
||||
# The history scope this context belongs to (#438)
|
||||
positive_raw_text_id=_scope_text_id(
|
||||
(context["positive"] or {}).get("raw")
|
||||
),
|
||||
for key in (
|
||||
"positive_snapshot_id",
|
||||
"negative_snapshot_id",
|
||||
"positive_raw_text_id",
|
||||
)
|
||||
}
|
||||
)
|
||||
pair = context["pair_id"]
|
||||
context["navigation"] = {
|
||||
|
||||
@@ -221,7 +221,6 @@ class ImageCategoryOperations:
|
||||
sources. Returns ``{updated, unchanged}``; a file that already had
|
||||
exactly this value is ``unchanged`` and keeps its ``updated_at``.
|
||||
"""
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
keys = list(dict.fromkeys(tuple(key) for key in file_keys))
|
||||
@@ -235,7 +234,6 @@ class ImageCategoryOperations:
|
||||
continue
|
||||
conn.execute(UPSERT_SQL, (*key, category, now))
|
||||
counts["updated"] += 1
|
||||
clear_marks_of_keys(conn, keys) # #480: a categorized file is listed
|
||||
return counts
|
||||
|
||||
def clear_image_categories(self, file_keys) -> int:
|
||||
|
||||
@@ -402,7 +402,6 @@ class ImageCollectionOperations:
|
||||
is created only when at least one item is inserted. Returns
|
||||
``{collection, created, added, already}``.
|
||||
"""
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
if (collection_id is None) == (name is None):
|
||||
@@ -439,7 +438,6 @@ class ImageCollectionOperations:
|
||||
).rowcount
|
||||
for root, path in pairs
|
||||
)
|
||||
clear_marks_of_keys(conn, pairs) # #480: a collected file is listed
|
||||
return {
|
||||
"collection": collection,
|
||||
"created": created,
|
||||
|
||||
@@ -104,14 +104,11 @@ class ImageFavoriteOperations:
|
||||
"""Mark one file as a favorite; repeating it is a no-op. Returns True."""
|
||||
from .models import utc_now_iso # models imports this module's schema
|
||||
|
||||
from .image_removal_schema import clear_marks_of_keys
|
||||
|
||||
with self.model.get_connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO image_favorites VALUES (?, ?, ?)",
|
||||
(root_key, relative_path, utc_now_iso()),
|
||||
)
|
||||
clear_marks_of_keys(conn, [(root_key, relative_path)]) # #480
|
||||
return True
|
||||
|
||||
def clear_image_favorite(self, root_key: str, relative_path: str) -> bool:
|
||||
|
||||
@@ -1,348 +0,0 @@
|
||||
"""Remove a file's records from the library, and its removal mark (#480).
|
||||
|
||||
A *removal* describes one physical file the way the API resolved it from the
|
||||
walked listing (``py/api/image_delete.py``):
|
||||
|
||||
- ``spellings``: every path string records may name it by
|
||||
(``duplicate_spellings.Spellings.of``);
|
||||
- ``locator``: its private canonical locator;
|
||||
- ``favorites``: ``(root key, relative key)`` under every listed root holding
|
||||
it, the keys of its favorite and its collection memberships;
|
||||
- ``category``: its own category's file key, or None.
|
||||
|
||||
``remove_image_records`` deletes, in one ``BEGIN IMMEDIATE`` transaction:
|
||||
|
||||
- its image records (``generated_images``) under every spelling, plus legacy
|
||||
rows matched by ``image_path``; their saved-pair links go by cascade and the
|
||||
workflow documents they held are released;
|
||||
- the image tags and auto-tag marks of its byte records. The annotation file
|
||||
and byte rows go too, unless a Needs review item still cites them
|
||||
(``media_review_items.byte_id`` is ``ON DELETE RESTRICT``): a decided or
|
||||
revised item keeps its decision;
|
||||
- its pending, undecided Needs review items (settled as duplicate removal
|
||||
settles them);
|
||||
- its own category, favorite and collection memberships.
|
||||
|
||||
With ``mark`` it also writes the file's removal mark (library-only removal):
|
||||
the file stays on disk, hidden until its bytes change. Prompts, recorded runs
|
||||
and the media file itself are never touched here.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from .annotation_schema import BYTES_TABLE, FILES_TABLE
|
||||
from .annotations import _BUMP_INDEX
|
||||
from .autotag_schema import AUTOTAG_TABLE
|
||||
from .duplicate_copies import (
|
||||
DELETE_FAVORITE_SQL,
|
||||
DELETE_LINK_SQL,
|
||||
DELETE_MEMBERSHIPS_SQL,
|
||||
_file_link_rows,
|
||||
)
|
||||
from .image_categories import DELETE_SQL as DELETE_CATEGORY_SQL
|
||||
from .image_removal_schema import (
|
||||
META_TABLE,
|
||||
TABLE,
|
||||
bump_generation,
|
||||
clear_marks,
|
||||
has_marks_table,
|
||||
mark_of,
|
||||
write_mark as _write_mark,
|
||||
)
|
||||
from .library import display_title
|
||||
from .models import utc_now_iso
|
||||
from .review import _BUMP_REVIEW_INDEX, _PENDING_OF_LOCATOR, _SETTLE_ITEM
|
||||
from .review_changes import notify_review_change
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
BYTES_OF_LOCATOR_SQL = (
|
||||
f"SELECT b.id FROM {FILES_TABLE} f JOIN {BYTES_TABLE} b " # nosec B608
|
||||
"ON b.file_id=f.id WHERE f.locator=?"
|
||||
)
|
||||
TAG_COUNT_SQL = "SELECT COUNT(*) FROM media_image_tags WHERE byte_id=?"
|
||||
AUTOTAG_COUNT_SQL = (
|
||||
f"SELECT COUNT(*) FROM {AUTOTAG_TABLE} WHERE byte_id=?" # nosec B608
|
||||
)
|
||||
CLEAR_TAGS_SQL = "DELETE FROM media_image_tags WHERE byte_id=?"
|
||||
CLEAR_AUTOTAG_SQL = f"DELETE FROM {AUTOTAG_TABLE} WHERE byte_id=?" # nosec B608
|
||||
BUMP_TAG_REVISION_SQL = (
|
||||
f"UPDATE {BYTES_TABLE} SET tag_revision=tag_revision+1 WHERE id=?" # nosec B608
|
||||
)
|
||||
REVIEW_REFS_SQL = (
|
||||
f"SELECT COUNT(*) FROM {FILES_TABLE} f " # nosec B608
|
||||
f"JOIN {BYTES_TABLE} b ON b.file_id=f.id "
|
||||
"JOIN media_review_items i ON i.byte_id=b.id WHERE f.locator=?"
|
||||
)
|
||||
DELETE_FILE_SQL = f"DELETE FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
FILE_EXISTS_SQL = f"SELECT 1 FROM {FILES_TABLE} WHERE locator=?" # nosec B608
|
||||
FAVORITE_EXISTS_SQL = (
|
||||
"SELECT COUNT(*) FROM image_favorites WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
MEMBERSHIP_COUNT_SQL = (
|
||||
"SELECT COUNT(*) FROM image_collection_items WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
CATEGORY_EXISTS_SQL = (
|
||||
"SELECT COUNT(*) FROM image_categories WHERE root_key=? AND relative_path=?"
|
||||
)
|
||||
PAIR_LINK_COUNT_SQL = "SELECT COUNT(*) FROM image_prompt_pairs WHERE image_id=?"
|
||||
PROMPT_LINKS_SQL = "SELECT id FROM generated_images WHERE prompt_id=?"
|
||||
PROMPT_TITLE_SQL = (
|
||||
"SELECT p.text, s.title FROM prompts p "
|
||||
"LEFT JOIN prompt_library_state s ON s.prompt_id=p.id WHERE p.id=?"
|
||||
)
|
||||
MARKS_SQL = f"SELECT locator, size, mtime_ns FROM {TABLE}" # nosec B608
|
||||
REVISION_SQL = f"SELECT revision FROM {META_TABLE} WHERE singleton_id=1" # nosec B608
|
||||
MAX_ORPHAN_TITLES = 3
|
||||
RECORD_KINDS = (
|
||||
"prompt_links",
|
||||
"pair_links",
|
||||
"image_tags",
|
||||
"auto_tagged",
|
||||
"categories",
|
||||
"collections",
|
||||
"favorites",
|
||||
"review_items",
|
||||
)
|
||||
|
||||
|
||||
def _link_ids(conn, removal):
|
||||
"""``{image id: prompt id}`` of the records naming the file, any spelling."""
|
||||
found = {}
|
||||
for spelled in dict.fromkeys(removal["spellings"]):
|
||||
for image_id, prompt_id in _file_link_rows(conn, spelled):
|
||||
found[image_id] = prompt_id
|
||||
return found
|
||||
|
||||
|
||||
def _byte_ids(conn, locator):
|
||||
return [row[0] for row in conn.execute(BYTES_OF_LOCATOR_SQL, (locator,))]
|
||||
|
||||
|
||||
def file_locator(path) -> str:
|
||||
"""The locator of an absolute path (``normcase(realpath)``)."""
|
||||
return os.path.normcase(os.path.realpath(str(path)))
|
||||
|
||||
|
||||
def removed_from_library(conn, path) -> bool:
|
||||
"""Whether *path* is hidden by a removal mark whose bytes still match.
|
||||
|
||||
A mark whose file changed (other size or ``mtime_ns``) or is gone is
|
||||
stale: it is deleted on *conn* (the caller's write transaction), and the
|
||||
file is not hidden.
|
||||
"""
|
||||
locator = file_locator(path)
|
||||
mark = mark_of(conn, locator)
|
||||
if mark is None:
|
||||
return False
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except OSError:
|
||||
clear_marks(conn, [locator])
|
||||
return False
|
||||
if (info.st_size, info.st_mtime_ns) == tuple(mark):
|
||||
return True
|
||||
clear_marks(conn, [locator])
|
||||
return False
|
||||
|
||||
|
||||
def mark_hides(marks: dict, locator: str, path) -> bool:
|
||||
"""Whether a removal mark in *marks* hides the file at *path* (blocking).
|
||||
|
||||
*marks* is ``{locator: (size, mtime_ns)}``. The mark of *locator* hides
|
||||
the file while ``os.lstat(path)`` still has the size and ``mtime_ns`` it
|
||||
was removed with; a file that cannot be looked at is not hidden.
|
||||
"""
|
||||
mark = marks.get(locator)
|
||||
if mark is None:
|
||||
return False
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except OSError:
|
||||
return False
|
||||
return (info.st_size, info.st_mtime_ns) == tuple(mark)
|
||||
|
||||
|
||||
def write_mark(conn, locator, size, mtime_ns):
|
||||
_write_mark(conn, locator, size, mtime_ns, utc_now_iso())
|
||||
|
||||
|
||||
class ImageRemovalOperations:
|
||||
"""PromptDatabase mixin: remove a file's records; removal marks."""
|
||||
|
||||
def count_removal_records(self, removals) -> dict:
|
||||
"""Counts of what removing *removals* deletes, in one read snapshot.
|
||||
|
||||
``{records: {kind: n}, orphaned_prompts: {count, titles}}``: prompts
|
||||
whose every image record is among these files lose their last image
|
||||
(they stay in the Library).
|
||||
"""
|
||||
from .maintenance import _read_snapshot
|
||||
|
||||
def read(conn):
|
||||
totals = dict.fromkeys(RECORD_KINDS, 0)
|
||||
links = {}
|
||||
for removal in removals:
|
||||
found = _link_ids(conn, removal)
|
||||
links.update(found)
|
||||
for image_id in found:
|
||||
totals["pair_links"] += conn.execute(
|
||||
PAIR_LINK_COUNT_SQL, (image_id,)
|
||||
).fetchone()[0]
|
||||
tagged = autotagged = False
|
||||
for byte_id in _byte_ids(conn, removal["locator"]):
|
||||
tagged |= bool(
|
||||
conn.execute(TAG_COUNT_SQL, (byte_id,)).fetchone()[0]
|
||||
)
|
||||
autotagged |= bool(
|
||||
conn.execute(AUTOTAG_COUNT_SQL, (byte_id,)).fetchone()[0]
|
||||
)
|
||||
totals["image_tags"] += int(tagged)
|
||||
totals["auto_tagged"] += int(autotagged)
|
||||
totals["review_items"] += len(
|
||||
conn.execute(_PENDING_OF_LOCATOR, (removal["locator"],)).fetchall()
|
||||
)
|
||||
totals["favorites"] += int(
|
||||
any(
|
||||
conn.execute(FAVORITE_EXISTS_SQL, key).fetchone()[0]
|
||||
for key in removal["favorites"]
|
||||
)
|
||||
)
|
||||
totals["collections"] += sum(
|
||||
conn.execute(MEMBERSHIP_COUNT_SQL, key).fetchone()[0]
|
||||
for key in removal["favorites"]
|
||||
)
|
||||
if removal.get("category"):
|
||||
totals["categories"] += conn.execute(
|
||||
CATEGORY_EXISTS_SQL, removal["category"]
|
||||
).fetchone()[0]
|
||||
totals["prompt_links"] = len(links)
|
||||
return {
|
||||
"records": totals,
|
||||
"orphaned_prompts": self._orphaned_prompts(conn, links),
|
||||
}
|
||||
|
||||
return _read_snapshot(self.model.db_path, read)
|
||||
|
||||
@staticmethod
|
||||
def _orphaned_prompts(conn, links):
|
||||
removed = set(links)
|
||||
orphans = []
|
||||
for prompt_id in dict.fromkeys(links.values()):
|
||||
remaining = [
|
||||
row[0]
|
||||
for row in conn.execute(PROMPT_LINKS_SQL, (prompt_id,))
|
||||
if row[0] not in removed
|
||||
]
|
||||
if not remaining:
|
||||
orphans.append(prompt_id)
|
||||
titles = []
|
||||
for prompt_id in orphans[:MAX_ORPHAN_TITLES]:
|
||||
row = conn.execute(PROMPT_TITLE_SQL, (prompt_id,)).fetchone()
|
||||
if row is not None:
|
||||
titles.append(display_title((row[1],), row[0]) or "")
|
||||
return {"count": len(orphans), "titles": titles}
|
||||
|
||||
def remove_image_records(self, removal, *, mark=None) -> dict:
|
||||
"""Delete every record of one file (one transaction); see the module.
|
||||
|
||||
``mark``: ``(size, mtime_ns)`` to write the file's removal mark, or
|
||||
None (the file was moved to the trash). Returns the counts deleted.
|
||||
"""
|
||||
counts = dict.fromkeys(RECORD_KINDS, 0)
|
||||
locator = removal["locator"]
|
||||
with self._pair_transaction() as conn:
|
||||
links = _link_ids(conn, removal)
|
||||
if links:
|
||||
documents = documents_of_rows(conn, "id", list(links))
|
||||
for image_id in links:
|
||||
counts["pair_links"] += conn.execute(
|
||||
PAIR_LINK_COUNT_SQL, (image_id,)
|
||||
).fetchone()[0]
|
||||
conn.execute(DELETE_LINK_SQL, (image_id,))
|
||||
release_documents(conn, documents)
|
||||
counts["prompt_links"] = len(links)
|
||||
counts.update(self._remove_annotations(conn, locator))
|
||||
for key in removal["favorites"]:
|
||||
counts["favorites"] += conn.execute(DELETE_FAVORITE_SQL, key).rowcount
|
||||
counts["collections"] += conn.execute(
|
||||
DELETE_MEMBERSHIPS_SQL, key
|
||||
).rowcount
|
||||
if removal.get("category"):
|
||||
counts["categories"] += conn.execute(
|
||||
DELETE_CATEGORY_SQL, tuple(removal["category"])
|
||||
).rowcount
|
||||
if mark is not None:
|
||||
write_mark(conn, locator, *mark)
|
||||
if counts["review_items"]:
|
||||
notify_review_change(self.model.db_path)
|
||||
return counts
|
||||
|
||||
def _remove_annotations(self, conn, locator) -> dict:
|
||||
"""Tags, auto-tag marks, pending review items and, when free, the rows."""
|
||||
counts = {"image_tags": 0, "auto_tagged": 0, "review_items": 0}
|
||||
bytes_ = _byte_ids(conn, locator)
|
||||
for byte_id in bytes_:
|
||||
tags = conn.execute(CLEAR_TAGS_SQL, (byte_id,)).rowcount
|
||||
autotags = conn.execute(CLEAR_AUTOTAG_SQL, (byte_id,)).rowcount
|
||||
if tags:
|
||||
conn.execute(BUMP_TAG_REVISION_SQL, (byte_id,))
|
||||
counts["image_tags"] |= bool(tags)
|
||||
counts["auto_tagged"] |= bool(autotags)
|
||||
for (item_id,) in conn.execute(_PENDING_OF_LOCATOR, (locator,)).fetchall():
|
||||
counts["review_items"] += conn.execute(_SETTLE_ITEM, (item_id,)).rowcount
|
||||
if counts["review_items"]:
|
||||
conn.execute(_BUMP_REVIEW_INDEX)
|
||||
if conn.execute(FILE_EXISTS_SQL, (locator,)).fetchone():
|
||||
if not conn.execute(REVIEW_REFS_SQL, (locator,)).fetchone()[0]:
|
||||
conn.execute(DELETE_FILE_SQL, (locator,))
|
||||
conn.execute(_BUMP_INDEX)
|
||||
counts["image_tags"] = int(counts["image_tags"])
|
||||
counts["auto_tagged"] = int(counts["auto_tagged"])
|
||||
return counts
|
||||
|
||||
def removal_marks(self) -> dict:
|
||||
"""``{locator: (size, mtime_ns)}`` of every removal mark (one read)."""
|
||||
conn = self.model.get_connection()
|
||||
if not has_marks_table(conn):
|
||||
return {}
|
||||
return {row[0]: (row[1], row[2]) for row in conn.execute(MARKS_SQL)}
|
||||
|
||||
def removal_revision(self) -> int:
|
||||
conn = self.model.get_connection()
|
||||
if not has_marks_table(conn):
|
||||
return 0
|
||||
row = conn.execute(REVISION_SQL).fetchone()
|
||||
return row[0] if row else 0
|
||||
|
||||
def clear_removal_marks(self, locators) -> int:
|
||||
"""Show these files again (their marks go); the count removed."""
|
||||
locators = list(locators)
|
||||
if not locators:
|
||||
return 0
|
||||
with self._pair_transaction() as conn:
|
||||
removed = clear_marks(conn, locators)
|
||||
# #498: again after the commit, so a reader that cached the marks
|
||||
# between the in-transaction bump and the commit reloads them.
|
||||
bump_generation()
|
||||
return removed
|
||||
|
||||
def clear_hiding_marks(self, files) -> set:
|
||||
"""Delete the marks that still hide their file (#498, Return to library).
|
||||
|
||||
*files* are ``(locator, path)`` pairs. In one ``BEGIN IMMEDIATE``
|
||||
transaction each mark is read again and deleted only while the file at
|
||||
*path* still has its size and ``mtime_ns``. Returns the locators whose
|
||||
marks went.
|
||||
"""
|
||||
cleared = set()
|
||||
files = list(files)
|
||||
if not files:
|
||||
return cleared
|
||||
with self._pair_transaction() as conn:
|
||||
for locator, path in files:
|
||||
mark = mark_of(conn, locator)
|
||||
if mark is not None and mark_hides({locator: mark}, locator, path):
|
||||
cleared.add(locator)
|
||||
clear_marks(conn, sorted(cleared))
|
||||
bump_generation()
|
||||
return cleared
|
||||
@@ -1,213 +0,0 @@
|
||||
"""Removal marks (#480): files removed from the library that stay on disk.
|
||||
|
||||
``image_removals`` holds one row per hidden file: its private canonical
|
||||
locator (the ``media_annotation_files.locator`` form) and the ``size`` and
|
||||
``mtime_ns`` it had when it was removed. A file at that locator with other
|
||||
bytes is a new file and shows again. ``image_removals_meta`` is one row, the
|
||||
marks revision, bumped in the same transaction as every mark change so the
|
||||
workspace listing caches can tell marks changed.
|
||||
|
||||
The group installs like ``image_favorites``: optional in old backups,
|
||||
required once a database is current. Older builds ignore it.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
from .annotation_schema import MAX_LOCATOR_LENGTH
|
||||
|
||||
TABLE = "image_removals"
|
||||
META_TABLE = "image_removals_meta"
|
||||
TABLES = (TABLE, META_TABLE)
|
||||
_TABLE_DDL = {
|
||||
TABLE: (
|
||||
f"CREATE TABLE {TABLE} (locator TEXT PRIMARY KEY, "
|
||||
"size INTEGER NOT NULL CHECK(size>=0), mtime_ns INTEGER NOT NULL, "
|
||||
"removed_at TEXT NOT NULL) WITHOUT ROWID"
|
||||
),
|
||||
META_TABLE: (
|
||||
f"CREATE TABLE {META_TABLE} (singleton_id INTEGER PRIMARY KEY "
|
||||
"CHECK(singleton_id=1), revision INTEGER NOT NULL DEFAULT 0 "
|
||||
"CHECK(revision>=0))"
|
||||
),
|
||||
}
|
||||
_SEED_META = (
|
||||
f"INSERT OR IGNORE INTO {META_TABLE} (singleton_id, revision) VALUES (1, 0)"
|
||||
)
|
||||
_COLUMNS = {
|
||||
TABLE: (
|
||||
("locator", "TEXT", 1, 1),
|
||||
("size", "INTEGER", 1, 0),
|
||||
("mtime_ns", "INTEGER", 1, 0),
|
||||
("removed_at", "TEXT", 1, 0),
|
||||
),
|
||||
META_TABLE: (
|
||||
("singleton_id", "INTEGER", 0, 1),
|
||||
("revision", "INTEGER", 1, 0),
|
||||
),
|
||||
}
|
||||
_AUTOINDEX = f"sqlite_autoindex_{TABLE}_1"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
|
||||
rows = conn.execute(
|
||||
"SELECT type,name,tbl_name,sql FROM sqlite_master WHERE name IN (?,?) "
|
||||
"OR tbl_name IN (?,?)",
|
||||
TABLES + TABLES,
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return None if allow_missing else "Missing image removal marks schema"
|
||||
tables = {name: sql for kind, name, _tbl, sql in rows if kind == "table"}
|
||||
others = {(kind, name) for kind, name, _tbl, _sql in rows if kind != "table"}
|
||||
if set(tables) != set(TABLES) or others - {("index", _AUTOINDEX)}:
|
||||
return "Image removal marks name is reserved by an unsupported schema object"
|
||||
for table in TABLES:
|
||||
if _compact(tables[table]) != _compact(_TABLE_DDL[table]):
|
||||
return "Image removal marks have an invalid table definition"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({table})")
|
||||
)
|
||||
if columns != _COLUMNS[table]:
|
||||
return "Image removal marks have invalid columns"
|
||||
return None
|
||||
|
||||
|
||||
def _values_error(conn: sqlite3.Connection) -> str | None:
|
||||
limit = MAX_LOCATOR_LENGTH * 4
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(locator) != 'text' " # nosec B608
|
||||
f"OR length(CAST(locator AS BLOB)) > {limit} OR locator = '' "
|
||||
"OR typeof(size) != 'integer' OR typeof(mtime_ns) != 'integer' "
|
||||
"OR typeof(removed_at) != 'text' OR length(CAST(removed_at AS BLOB)) > 64 "
|
||||
"LIMIT 1"
|
||||
).fetchone():
|
||||
return "Image removal marks contain invalid values"
|
||||
meta = conn.execute(
|
||||
f"SELECT singleton_id, typeof(revision) FROM {META_TABLE}" # nosec B608
|
||||
).fetchall()
|
||||
if meta != [(1, "integer")]:
|
||||
return "Image removal marks have an invalid revision"
|
||||
return None
|
||||
|
||||
|
||||
def image_removals_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> str | None:
|
||||
"""Reject unsupported schemas and invalid persisted values (reads only)."""
|
||||
error = _structure_error(conn, allow_missing)
|
||||
if error:
|
||||
return error
|
||||
exists = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone()
|
||||
return _values_error(conn) if exists else None
|
||||
|
||||
|
||||
def migrate_image_removals(conn: sqlite3.Connection) -> None:
|
||||
"""Install both tables atomically; validate before and after."""
|
||||
conn.execute("SAVEPOINT image_removals_schema")
|
||||
try:
|
||||
error = image_removals_schema_error(conn)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
for sql in _TABLE_DDL.values():
|
||||
conn.execute(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
conn.execute(_SEED_META)
|
||||
error = image_removals_schema_error(conn, allow_missing=False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE image_removals_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO image_removals_schema")
|
||||
conn.execute("RELEASE image_removals_schema")
|
||||
raise
|
||||
|
||||
|
||||
_UNMARK_SQL = f"DELETE FROM {TABLE} WHERE locator=?" # nosec B608
|
||||
_MARK_SQL = (
|
||||
f"INSERT OR REPLACE INTO {TABLE} " # nosec B608
|
||||
"(locator, size, mtime_ns, removed_at) VALUES (?, ?, ?, ?)"
|
||||
)
|
||||
_MARK_OF_SQL = f"SELECT size, mtime_ns FROM {TABLE} WHERE locator=?" # nosec B608
|
||||
BUMP_MARKS_SQL = (
|
||||
f"UPDATE {META_TABLE} SET revision=revision+1 WHERE singleton_id=1" # nosec B608
|
||||
)
|
||||
|
||||
|
||||
# Bumped whenever this process writes or clears a mark (even in a transaction
|
||||
# that later rolls back): readers that cache the marks reload when it moves.
|
||||
_generation = 0
|
||||
|
||||
|
||||
def generation() -> int:
|
||||
"""The marks generation of this process (see ``_generation``)."""
|
||||
return _generation
|
||||
|
||||
|
||||
def _bump() -> None:
|
||||
global _generation
|
||||
_generation += 1
|
||||
|
||||
|
||||
def bump_generation() -> None:
|
||||
"""Move the generation after a commit (a cached read made before it reloads)."""
|
||||
_bump()
|
||||
|
||||
|
||||
def has_marks_table(conn: sqlite3.Connection) -> bool:
|
||||
return (
|
||||
conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def locator_of_key(root_key: str, relative_key: str) -> str:
|
||||
"""The locator of a favorite-style file key (root key + relative key)."""
|
||||
return os.path.normcase(os.path.join(root_key, *relative_key.split("/")))
|
||||
|
||||
|
||||
def mark_of(conn: sqlite3.Connection, locator: str):
|
||||
"""``(size, mtime_ns)`` of the file's removal mark, or None."""
|
||||
if not has_marks_table(conn):
|
||||
return None
|
||||
row = conn.execute(_MARK_OF_SQL, (locator,)).fetchone()
|
||||
return (row[0], row[1]) if row else None
|
||||
|
||||
|
||||
def write_mark(
|
||||
conn: sqlite3.Connection, locator: str, size: int, mtime_ns: int, now: str
|
||||
):
|
||||
conn.execute(_MARK_SQL, (locator, int(size), int(mtime_ns), now))
|
||||
conn.execute(BUMP_MARKS_SQL)
|
||||
_bump()
|
||||
|
||||
|
||||
def clear_marks(conn: sqlite3.Connection, locators) -> int:
|
||||
"""Delete the marks of *locators* on *conn* (no commit); the count removed.
|
||||
|
||||
Every record write for a file calls this (#480): a file with records is
|
||||
in the library, so it is never hidden.
|
||||
"""
|
||||
locators = list(dict.fromkeys(locators))
|
||||
if not locators or not has_marks_table(conn):
|
||||
return 0
|
||||
removed = 0
|
||||
for locator in locators:
|
||||
removed += conn.execute(_UNMARK_SQL, (locator,)).rowcount
|
||||
if removed:
|
||||
conn.execute(BUMP_MARKS_SQL)
|
||||
_bump()
|
||||
return removed
|
||||
|
||||
|
||||
def clear_marks_of_keys(conn: sqlite3.Connection, file_keys) -> int:
|
||||
"""``clear_marks`` for favorite-style ``(root key, relative key)`` pairs."""
|
||||
return clear_marks(conn, [locator_of_key(*key) for key in file_keys])
|
||||
@@ -5,9 +5,7 @@ import sqlite3
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
from .library_folders import folder_condition, list_library_folders
|
||||
from .library_tokens import library_edit_token
|
||||
from .prompt_archive import ARCHIVED, SHELVED
|
||||
|
||||
_ELIGIBLE = """(EXISTS(SELECT 1 FROM prompt_pairs ep WHERE ep.positive_prompt_id=p.id)
|
||||
OR EXISTS(SELECT 1 FROM generated_images eg WHERE eg.prompt_id=p.id)
|
||||
@@ -107,12 +105,6 @@ ARCHIVE_SCOPES = {
|
||||
}
|
||||
|
||||
|
||||
# Shelves (#443): the parents on the shelf, or the archived ones.
|
||||
PARENT_SCOPES = {"active": SHELVED, "archived": ARCHIVED}
|
||||
# A positive pair owner that is on the shelf, as a condition on alias pp.
|
||||
SHELVED_PAIR = SHELVED.replace("p.id", "pp.positive_prompt_id")
|
||||
|
||||
|
||||
def _escape(value: str) -> str:
|
||||
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
@@ -129,54 +121,7 @@ def _archive(value: str) -> str:
|
||||
return ARCHIVE_SCOPES[value]
|
||||
|
||||
|
||||
# How the Tags filter matches (#444): every tag (AND), any of them (OR), or
|
||||
# parents without any tag.
|
||||
TAG_MODES = ("all", "any", "untagged")
|
||||
# A parent that no tag's Browse returns (#444 "Untagged prompts"): no tag of its
|
||||
# own and no active pair carrying one. The vocabulary's untagged count uses it too.
|
||||
UNTAGGED = (
|
||||
"NOT EXISTS(SELECT 1 FROM prompt_tags ut WHERE ut.prompt_id=p.id) "
|
||||
"AND NOT EXISTS(SELECT 1 FROM prompt_pairs up JOIN prompt_pair_tags upt "
|
||||
"ON upt.pair_id=up.id WHERE up.positive_prompt_id=p.id AND up.archived=0)"
|
||||
)
|
||||
|
||||
|
||||
def _tag_mode(tag_mode: object, tags: list[str]) -> str:
|
||||
if not isinstance(tag_mode, str) or tag_mode not in TAG_MODES:
|
||||
raise ValueError("tag_mode must be one of " + ", ".join(TAG_MODES))
|
||||
if tag_mode == "untagged" and tags:
|
||||
raise ValueError("Untagged prompts cannot also filter by tags")
|
||||
return tag_mode
|
||||
|
||||
|
||||
def _tag_conditions(tags: list[str], any_tag: bool) -> tuple[list, list, list]:
|
||||
"""Parent-arm and pair-arm tag conditions and the tags' parameters."""
|
||||
parent_exists = (
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name{test})"
|
||||
)
|
||||
pair_exists = (
|
||||
"EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.pair_id=pp.id AND t.name{test})"
|
||||
)
|
||||
if any_tag and tags:
|
||||
# One condition per arm: the card, or one active pair, carries any tag.
|
||||
test = " IN (" + ",".join("?" for _ in tags) + ")"
|
||||
return [parent_exists.format(test=test)], [pair_exists.format(test=test)], tags
|
||||
return (
|
||||
[parent_exists.format(test="=?") for _ in tags],
|
||||
[pair_exists.format(test="=?") for _ in tags],
|
||||
tags,
|
||||
)
|
||||
|
||||
|
||||
def _filter(
|
||||
text: str,
|
||||
category: str | None,
|
||||
tags: list[str],
|
||||
tag_mode: str = "all",
|
||||
folder_prefixes: list[str] | None = None,
|
||||
) -> tuple[str, list]:
|
||||
def _filter(text: str, category: str | None, tags: list[str]) -> tuple[str, list]:
|
||||
if not isinstance(text, str) or len(text) > 500:
|
||||
raise ValueError("Search text must be a string of at most 500 characters")
|
||||
if category is not None and (not isinstance(category, str) or len(category) > 200):
|
||||
@@ -187,8 +132,6 @@ def _filter(
|
||||
or any(not isinstance(tag, str) or not tag or len(tag) > 200 for tag in tags)
|
||||
):
|
||||
raise ValueError("Invalid library tags")
|
||||
mode = _tag_mode(tag_mode, tags)
|
||||
untagged = mode == "untagged"
|
||||
parent, pair, parent_params, pair_params = [], ["pp.archived=0"], [], []
|
||||
if text:
|
||||
parent.append("(p.text LIKE ? ESCAPE '\\' OR ls.title LIKE ? ESCAPE '\\')")
|
||||
@@ -203,11 +146,17 @@ def _filter(
|
||||
pair.append("pp.category=?")
|
||||
parent_params.append(category)
|
||||
pair_params.append(category)
|
||||
parent_tags, pair_tags, tag_params = _tag_conditions(tags, mode == "any")
|
||||
parent.extend(parent_tags)
|
||||
pair.extend(pair_tags)
|
||||
parent_params.extend(tag_params)
|
||||
pair_params.extend(tag_params)
|
||||
for tag in tags:
|
||||
parent.append(
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name=?)"
|
||||
)
|
||||
pair.append(
|
||||
"EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.pair_id=pp.id AND t.name=?)"
|
||||
)
|
||||
parent_params.append(tag)
|
||||
pair_params.append(tag)
|
||||
condition = (
|
||||
"("
|
||||
+ (" AND ".join(parent) or "1=1")
|
||||
@@ -219,17 +168,7 @@ def _filter(
|
||||
+ " AND ".join(pair)
|
||||
+ ")"
|
||||
)
|
||||
if untagged:
|
||||
# A whole-card condition: inside the parent-or-pair OR an untagged pair
|
||||
# would let a tagged parent through.
|
||||
condition = f"({condition}) AND {UNTAGGED}"
|
||||
params = [*parent_params, *pair_params]
|
||||
if folder_prefixes:
|
||||
# A whole-card condition around both arms: images link to the parent.
|
||||
folder, folder_params = folder_condition(folder_prefixes)
|
||||
condition = f"({condition}) AND {folder}"
|
||||
params.extend(folder_params)
|
||||
return f"({_ELIGIBLE}) AND ({condition})", params
|
||||
return f"({_ELIGIBLE}) AND ({condition})", [*parent_params, *pair_params]
|
||||
|
||||
|
||||
class LibraryOperations:
|
||||
@@ -277,8 +216,6 @@ class LibraryOperations:
|
||||
rows = conn.execute(
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,p.created_at,"
|
||||
"p.run_count,p.last_used_at,"
|
||||
+ ARCHIVED
|
||||
+ " AS archived,"
|
||||
+ (f"{sort_expression} AS _sort," if sort_expression else "")
|
||||
+ "(SELECT COUNT(*) FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
|
||||
"AND v.archived=0) AS variant_count,"
|
||||
@@ -302,7 +239,6 @@ class LibraryOperations:
|
||||
# Usage is shown, never compared: the token hashes owner fields only.
|
||||
row["run_count"] = row["run_count"] if type(row["run_count"]) is int else 0
|
||||
row["last_used_at"] = utc_iso(row["last_used_at"])
|
||||
row["archived"] = bool(row["archived"])
|
||||
row["edit_token"] = library_edit_token(epoch, row)
|
||||
return result
|
||||
|
||||
@@ -315,21 +251,14 @@ class LibraryOperations:
|
||||
text: str = "",
|
||||
category: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
tag_mode: str = "all",
|
||||
folder_prefixes: list[str] | None = None,
|
||||
sort: str = "newest",
|
||||
limit: int = 40,
|
||||
after: dict | None = None,
|
||||
shelf: str = "active",
|
||||
) -> dict:
|
||||
_limit(limit)
|
||||
if sort not in _ORDERS:
|
||||
raise ValueError("Unknown library sort order")
|
||||
if not isinstance(shelf, str) or shelf not in PARENT_SCOPES:
|
||||
raise ValueError("shelf must be active or archived")
|
||||
where, params = _filter(
|
||||
text, category, [] if tags is None else tags, tag_mode, folder_prefixes
|
||||
)
|
||||
where, params = _filter(text, category, [] if tags is None else tags)
|
||||
if after is not None:
|
||||
if (
|
||||
not isinstance(after, dict)
|
||||
@@ -341,18 +270,15 @@ class LibraryOperations:
|
||||
ceiling = after["ceiling"]
|
||||
where += " AND p.id<=?"
|
||||
params.append(ceiling)
|
||||
# One SELECT counts the filtered parents of this scope, their active
|
||||
# variants, and the filtered archived parents (Show archived, #443).
|
||||
scope = PARENT_SCOPES[shelf]
|
||||
total, variant_total, archived_total = conn.execute(
|
||||
f"SELECT COALESCE(SUM({scope}),0),"
|
||||
f"COALESCE(SUM(CASE WHEN {scope} THEN (SELECT COUNT(*) "
|
||||
"FROM prompt_pairs v WHERE v.positive_prompt_id=p.id "
|
||||
"AND v.archived=0) ELSE 0 END),0),"
|
||||
f"COALESCE(SUM({ARCHIVED}),0) FROM " + _FROM + " WHERE " + where,
|
||||
# One SELECT counts the filtered parents and their active variants.
|
||||
total, variant_total = conn.execute(
|
||||
"SELECT COUNT(*),COALESCE(SUM((SELECT COUNT(*) FROM prompt_pairs v "
|
||||
"WHERE v.positive_prompt_id=p.id AND v.archived=0)),0) FROM "
|
||||
+ _FROM
|
||||
+ " WHERE "
|
||||
+ where,
|
||||
params,
|
||||
).fetchone()
|
||||
where += f" AND {scope}"
|
||||
expression, direction = _ORDERS[sort]
|
||||
if after is not None:
|
||||
operator = ">" if direction == "ASC" else "<"
|
||||
@@ -385,9 +311,6 @@ class LibraryOperations:
|
||||
"items": rows,
|
||||
"total": total,
|
||||
"variant_total": variant_total,
|
||||
"archived_total": archived_total,
|
||||
# Echoed so a client can tell this read applied its tag match.
|
||||
"tag_mode": tag_mode,
|
||||
"has_more": has_more,
|
||||
"next_key": next_key,
|
||||
}
|
||||
@@ -398,12 +321,9 @@ class LibraryOperations:
|
||||
text: str = "",
|
||||
category: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
tag_mode: str = "all",
|
||||
folder_prefixes: list[str] | None = None,
|
||||
sort: str = "newest",
|
||||
limit: int = 40,
|
||||
after: dict | None = None,
|
||||
shelf: str = "active",
|
||||
) -> dict:
|
||||
with self.library_snapshot(include_ceiling=True) as (conn, epoch, ceiling):
|
||||
return self._list_library(
|
||||
@@ -413,12 +333,9 @@ class LibraryOperations:
|
||||
text=text,
|
||||
category=category,
|
||||
tags=tags,
|
||||
tag_mode=tag_mode,
|
||||
folder_prefixes=folder_prefixes,
|
||||
sort=sort,
|
||||
limit=limit,
|
||||
after=after,
|
||||
shelf=shelf,
|
||||
)
|
||||
|
||||
def _is_library_positive(self, conn: sqlite3.Connection, positive_id: int) -> bool:
|
||||
@@ -546,13 +463,9 @@ class LibraryOperations:
|
||||
"categories": [row["category"] for row in rows],
|
||||
}
|
||||
|
||||
def get_library_options(self, folder_roots=None) -> dict:
|
||||
"""Categories; with *folder_roots* (root spellings) also ``folders``."""
|
||||
def get_library_options(self) -> dict:
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
result = self._get_library_options(conn, epoch)
|
||||
if folder_roots is not None:
|
||||
result["folders"] = list_library_folders(conn, folder_roots, _ELIGIBLE)
|
||||
return result
|
||||
return self._get_library_options(conn, epoch)
|
||||
|
||||
def get_library_previews(self, positive_ids: list[int]) -> dict[int, list[dict]]:
|
||||
"""Read at most two links per parent in one bounded page query."""
|
||||
@@ -663,9 +576,7 @@ class LibraryOperations:
|
||||
owners = {}
|
||||
order = []
|
||||
for row in conn.execute(
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes,"
|
||||
+ ARCHIVED
|
||||
+ " AS archived FROM "
|
||||
"SELECT p.id,p.text,ls.title,p.category,p.rating,p.notes FROM "
|
||||
+ _FROM
|
||||
+ " WHERE "
|
||||
+ _ELIGIBLE
|
||||
@@ -678,7 +589,6 @@ class LibraryOperations:
|
||||
"tags": [],
|
||||
"rating": row["rating"],
|
||||
"notes": row["notes"],
|
||||
"archived": bool(row["archived"]),
|
||||
"variants": [],
|
||||
}
|
||||
order.append(row["id"])
|
||||
@@ -723,9 +633,8 @@ class LibraryOperations:
|
||||
"""System › Overview counts from one snapshot, writing nothing.
|
||||
|
||||
``positive_prompts`` and ``negative_variants`` are the unfiltered
|
||||
Library's ``total`` and ``variant_total``: eligible parents on the
|
||||
shelf (not archived, #443) and their active pairs (every pair's
|
||||
positive is eligible). ``images`` counts indexed
|
||||
Library's ``total`` and ``variant_total``: eligible parents and active
|
||||
pairs (every pair's positive is eligible). ``images`` counts indexed
|
||||
``generated_images`` rows, not files on disk. An image has no rating
|
||||
of its own (it shows its linked prompt's), so ``rated_images`` counts
|
||||
the indexed images whose linked prompt is rated. ``review_pending`` is
|
||||
@@ -733,14 +642,8 @@ class LibraryOperations:
|
||||
"""
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
row = conn.execute(
|
||||
"SELECT (SELECT COUNT(*) FROM prompts p WHERE "
|
||||
+ _ELIGIBLE
|
||||
+ " AND "
|
||||
+ SHELVED
|
||||
+ "),"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs pp WHERE pp.archived=0 AND "
|
||||
+ SHELVED_PAIR
|
||||
+ "),"
|
||||
"SELECT (SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE + "),"
|
||||
"(SELECT COUNT(*) FROM prompt_pairs WHERE archived=0),"
|
||||
"(SELECT COUNT(*) FROM generated_images),"
|
||||
"(SELECT COUNT(*) FROM generated_images gi JOIN prompts r "
|
||||
"ON r.id=gi.prompt_id WHERE r.rating IS NOT NULL),"
|
||||
|
||||
@@ -1,473 +0,0 @@
|
||||
"""Delete selected Library prompts; archive the paired ones (#443, D152).
|
||||
|
||||
A selected Library parent is classified, in this order, as ``not_found``
|
||||
(missing, or not a Library parent), ``protected`` (tagged ``__protected__``;
|
||||
never changed), ``already_archived``, ``archive`` (the positive or negative of
|
||||
any saved pair, archived pair or not: archiving hides it from the shelf and
|
||||
removes nothing) or ``delete`` (in no pair: deleted exactly as the legacy
|
||||
delete does, through ``delete_prompt_rows``). Every command checks the
|
||||
storage epoch first and runs in one ``BEGIN IMMEDIATE`` transaction.
|
||||
|
||||
The write executes what the user reviewed: it is given the reviewed archive
|
||||
and delete lists, and deletes only reviewed deletes the route backed up
|
||||
(``candidates``) whose fingerprint is unchanged since that backup.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Iterable
|
||||
|
||||
from .library import _ELIGIBLE
|
||||
from .library_commands import _validate_bulk_ids, _validate_epoch
|
||||
from .models import utc_now_iso
|
||||
from .prompt_archive import ARCHIVED as _ARCHIVED
|
||||
from .prompt_archive import TABLE as ARCHIVE
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
try:
|
||||
from ..utils.hashing import generate_prompt_hash
|
||||
except ImportError:
|
||||
from utils.hashing import generate_prompt_hash
|
||||
|
||||
# Ids bound per statement: the pair check binds each id twice, which stays
|
||||
# under SQLite's historical 999-variable limit (maintenance.py ID_BATCH).
|
||||
ID_BATCH = 400
|
||||
_PROTECTED = (
|
||||
"EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id "
|
||||
"WHERE pt.prompt_id=p.id AND t.name='__protected__')"
|
||||
)
|
||||
_PAIRED = (
|
||||
"EXISTS(SELECT 1 FROM prompt_pairs pp "
|
||||
"WHERE pp.positive_prompt_id=p.id OR pp.negative_prompt_id=p.id)"
|
||||
)
|
||||
# #482: used as the negative of another positive's pair (archived or not):
|
||||
# never deleted, so that positive's variant stays (archive only).
|
||||
_NEGATIVE_OF_OTHER = (
|
||||
"EXISTS(SELECT 1 FROM prompt_pairs op "
|
||||
"WHERE op.negative_prompt_id=p.id AND op.positive_prompt_id!=p.id)"
|
||||
)
|
||||
# Classes in the order a prompt is tested; the first that holds wins.
|
||||
CLASSES = (
|
||||
"not_found",
|
||||
"protected",
|
||||
"already_archived",
|
||||
"not_archived",
|
||||
"archive",
|
||||
"delete",
|
||||
)
|
||||
# #492: the shelf a Delete… runs on. On the archived shelf an archived prompt is
|
||||
# classified by its pairs, and one restored since is ``not_archived``.
|
||||
SHELVES = ("active", "archived")
|
||||
|
||||
|
||||
def _check_shelf(shelf: str) -> bool:
|
||||
"""Whether ``shelf`` is the archived shelf (ValueError when unknown)."""
|
||||
if shelf not in SHELVES:
|
||||
raise ValueError("shelf must be active or archived")
|
||||
return shelf == "archived"
|
||||
|
||||
|
||||
_DIGEST = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def _batches(ids: list) -> Iterable[list]:
|
||||
for start in range(0, len(ids), ID_BATCH):
|
||||
yield ids[start : start + ID_BATCH]
|
||||
|
||||
|
||||
def delete_prompt_rows(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
|
||||
"""Delete prompts and their image link rows; return the prompts deleted.
|
||||
|
||||
The legacy delete's statements (``delete_prompt``, ``bulk_delete_prompts``)
|
||||
in the caller's transaction: image link rows first (``image_prompt_pairs``
|
||||
cascades, review decisions keep a NULL image), then the prompts (tags and
|
||||
Library state cascade), then the workflow documents no row references.
|
||||
Image files are never touched. The caller checks pairs first.
|
||||
"""
|
||||
documents = documents_of_rows(conn, "prompt_id", prompt_ids)
|
||||
deleted = 0
|
||||
for chunk in _batches(list(prompt_ids)):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
conn.execute(
|
||||
f"DELETE FROM generated_images WHERE prompt_id IN ({marks})", # nosec
|
||||
chunk,
|
||||
)
|
||||
deleted += conn.execute(
|
||||
f"DELETE FROM prompts WHERE id IN ({marks})", chunk # nosec B608
|
||||
).rowcount
|
||||
release_documents(conn, documents)
|
||||
return deleted
|
||||
|
||||
|
||||
class LibraryArchiveOperations:
|
||||
"""PromptDatabase mixin: classify, archive or delete, and restore parents."""
|
||||
|
||||
def _classify_library_delete(
|
||||
self, conn: sqlite3.Connection, prompt_ids: list[int], shelf: str = "active"
|
||||
) -> dict:
|
||||
on_archived_shelf = _check_shelf(shelf)
|
||||
found, deletable = {}, set()
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for row in conn.execute(
|
||||
f"SELECT p.id,{_PROTECTED},{_ARCHIVED},{_PAIRED}," # nosec B608
|
||||
f"{_NEGATIVE_OF_OTHER} "
|
||||
f"FROM prompts p WHERE p.id IN ({marks}) AND {_ELIGIBLE}",
|
||||
chunk,
|
||||
):
|
||||
protected, archived, paired, negative_of_other = row[1:5]
|
||||
if protected:
|
||||
found[row[0]] = "protected"
|
||||
elif on_archived_shelf and not archived:
|
||||
found[row[0]] = "not_archived"
|
||||
elif archived and not on_archived_shelf:
|
||||
found[row[0]] = "already_archived"
|
||||
elif paired:
|
||||
found[row[0]] = "archive"
|
||||
if not negative_of_other:
|
||||
deletable.add(row[0])
|
||||
else:
|
||||
found[row[0]] = "delete"
|
||||
classes = {name: [] for name in CLASSES}
|
||||
for pid in prompt_ids:
|
||||
classes[found.get(pid, "not_found")].append(pid)
|
||||
# #482: the archive ids Delete permanently may delete with their pairs.
|
||||
classes["deletable"] = [pid for pid in classes["archive"] if pid in deletable]
|
||||
return classes
|
||||
|
||||
@staticmethod
|
||||
def _image_links(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
|
||||
total = 0
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
total += conn.execute(
|
||||
"SELECT COUNT(*) FROM generated_images " # nosec B608
|
||||
f"WHERE prompt_id IN ({marks})",
|
||||
chunk,
|
||||
).fetchone()[0]
|
||||
return total
|
||||
|
||||
@staticmethod
|
||||
def _fingerprints(conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
|
||||
"""What a safety backup holds of each prompt a delete would remove.
|
||||
|
||||
The prompt row (usage included), its tag ids, its Library title and
|
||||
its image link rows (count and newest id), as a JSON-safe list.
|
||||
"""
|
||||
prints = {}
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for row in conn.execute(
|
||||
"SELECT p.id,p.text,p.hash,p.category,p.rating,p.notes," # nosec
|
||||
"p.updated_at,p.run_count,p.last_used_at,"
|
||||
"(SELECT group_concat(tag_id) FROM (SELECT tag_id FROM prompt_tags "
|
||||
"WHERE prompt_id=p.id ORDER BY tag_id)),"
|
||||
"(SELECT title FROM prompt_library_state WHERE prompt_id=p.id),"
|
||||
"(SELECT COUNT(*) FROM generated_images WHERE prompt_id=p.id),"
|
||||
"(SELECT MAX(id) FROM generated_images WHERE prompt_id=p.id) "
|
||||
f"FROM prompts p WHERE p.id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
prints[row[0]] = [
|
||||
value if isinstance(value, (int, str, type(None))) else repr(value)
|
||||
for value in tuple(row)[1:]
|
||||
]
|
||||
return prints
|
||||
|
||||
@staticmethod
|
||||
def _distinct_images(conn: sqlite3.Connection, prompt_ids: list[int]) -> int:
|
||||
"""Image rows of ``prompt_ids`` (each image once; #482 deletes them)."""
|
||||
return LibraryArchiveOperations._image_links(conn, prompt_ids)
|
||||
|
||||
@staticmethod
|
||||
def _pair_prints(conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
|
||||
"""#482: each prompt's own saved pairs, as the review saw them.
|
||||
|
||||
Per pair (by id): its negative, archived flag, metadata, usage, ordered
|
||||
tag ids and its image-pair links (count and newest id).
|
||||
"""
|
||||
prints = {pid: [] for pid in prompt_ids}
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for row in conn.execute(
|
||||
"SELECT pp.positive_prompt_id,pp.id,pp.negative_prompt_id," # nosec
|
||||
"pp.archived,pp.title,pp.category,pp.rating,pp.notes,pp.run_count,"
|
||||
"pp.last_used_at,pp.updated_at,"
|
||||
"(SELECT group_concat(tag_id) FROM (SELECT tag_id FROM "
|
||||
"prompt_pair_tags WHERE pair_id=pp.id ORDER BY tag_id)),"
|
||||
"(SELECT COUNT(*) FROM image_prompt_pairs WHERE pair_id=pp.id),"
|
||||
"(SELECT MAX(id) FROM image_prompt_pairs WHERE pair_id=pp.id) "
|
||||
f"FROM prompt_pairs pp WHERE pp.positive_prompt_id IN ({marks}) "
|
||||
"ORDER BY pp.id",
|
||||
chunk,
|
||||
):
|
||||
prints[row[0]].append(
|
||||
[
|
||||
v if isinstance(v, (int, str, type(None))) else repr(v)
|
||||
for v in tuple(row)[1:]
|
||||
]
|
||||
)
|
||||
return prints
|
||||
|
||||
@classmethod
|
||||
def _review_digests(cls, conn: sqlite3.Connection, prompt_ids: list[int]) -> dict:
|
||||
"""#482: one digest per deletable prompt of its row and its pairs."""
|
||||
prompts = cls._fingerprints(conn, prompt_ids)
|
||||
pairs = cls._pair_prints(conn, prompt_ids)
|
||||
return {
|
||||
pid: hashlib.sha256(
|
||||
json.dumps(
|
||||
[prompts.get(pid), pairs[pid]], separators=(",", ":")
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
for pid in prompt_ids
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _paired_counts(
|
||||
conn: sqlite3.Connection, prompt_ids: list[int], selected: list[int]
|
||||
) -> dict:
|
||||
"""#482: what Delete permanently would remove with ``prompt_ids``."""
|
||||
pairs = archived = variants = 0
|
||||
negatives = set()
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for archived_flag, negative, positive in conn.execute(
|
||||
"SELECT archived,negative_prompt_id,positive_prompt_id " # nosec
|
||||
f"FROM prompt_pairs WHERE positive_prompt_id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
pairs += 1
|
||||
archived += 1 if archived_flag else 0
|
||||
if negative is not None:
|
||||
variants += 1
|
||||
if negative != positive:
|
||||
negatives.add(negative)
|
||||
return {
|
||||
"pairs": pairs,
|
||||
"archived_pairs": archived,
|
||||
"variants": variants,
|
||||
"kept_negatives": len(negatives - set(selected)),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _checked_ids(prompt_ids: list[int], expected_epoch: str) -> list[int]:
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_bulk_ids(prompt_ids)
|
||||
return list(dict.fromkeys(prompt_ids))
|
||||
|
||||
def classify_library_delete(
|
||||
self, prompt_ids: list[int], *, expected_epoch: str, shelf: str = "active"
|
||||
) -> dict:
|
||||
"""What a delete of ``prompt_ids`` would do now (reads only).
|
||||
|
||||
Returns the id lists per class, in request order, ``image_links``
|
||||
(the image link rows the ``delete`` class would remove) and
|
||||
``fingerprints`` of the ``delete`` class (``_fingerprints``). On the
|
||||
archived ``shelf`` (#492) archived prompts are classified by their pairs.
|
||||
"""
|
||||
_check_shelf(shelf)
|
||||
prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
|
||||
with self.library_snapshot() as (conn, _epoch, _ceiling):
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
classes = self._classify_library_delete(conn, prompt_ids, shelf)
|
||||
deletable = classes["deletable"]
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
**classes,
|
||||
"image_links": self._image_links(conn, classes["delete"]),
|
||||
"fingerprints": self._fingerprints(conn, classes["delete"]),
|
||||
# #482: Delete permanently, with the deletable prompts' pairs.
|
||||
"review_digests": self._review_digests(conn, deletable),
|
||||
"paired_counts": self._paired_counts(conn, deletable, prompt_ids),
|
||||
"paired_image_links": self._distinct_images(conn, deletable),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _repair_hash(conn: sqlite3.Connection, prompt_id: int) -> None:
|
||||
"""Give an archived prompt its canonical hash, so ingestion finds it.
|
||||
|
||||
Scans, the monitor and node saves resolve prompts by hash; a stale or
|
||||
missing one would make them insert a visible copy (Q3). When another
|
||||
row already holds the canonical hash, lookups find that row and no new
|
||||
copy is made, so nothing is written.
|
||||
"""
|
||||
text, stored = conn.execute(
|
||||
"SELECT text,hash FROM prompts WHERE id=?", (prompt_id,)
|
||||
).fetchone()
|
||||
canonical = generate_prompt_hash(text) if isinstance(text, str) else None
|
||||
if canonical is None or stored == canonical:
|
||||
return
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM prompts WHERE hash=? AND id!=?", (canonical, prompt_id)
|
||||
).fetchone():
|
||||
return
|
||||
conn.execute("UPDATE prompts SET hash=? WHERE id=?", (canonical, prompt_id))
|
||||
|
||||
def archive_or_delete_library_prompts(
|
||||
self,
|
||||
archive_ids: list[int],
|
||||
delete_ids: list[int],
|
||||
*,
|
||||
expected_epoch: str,
|
||||
candidates: dict | None,
|
||||
delete_paired: dict | None = None,
|
||||
shelf: str = "active",
|
||||
) -> dict:
|
||||
"""Archive and delete what the user reviewed, all or nothing.
|
||||
|
||||
``archive_ids`` and ``delete_ids`` are the reviewed classes.
|
||||
``candidates`` maps each reviewed delete the route backed up to its
|
||||
fingerprint then (None: every reviewed delete, unchecked; tests and
|
||||
callers without a backup). A reviewed delete is deleted only when it
|
||||
still classifies ``delete`` and its fingerprint is unchanged; one that
|
||||
is now paired, and a reviewed archive now unpaired, are archived (the
|
||||
restorable direction); anything else is kept and counted.
|
||||
|
||||
``delete_paired`` (#482) maps reviewed archive ids the user chose to
|
||||
Delete permanently to their review digest. Each is deleted with its
|
||||
own saved pairs only while it is still deletable and its digest is
|
||||
unchanged; otherwise it is archived and counted ``changed``.
|
||||
|
||||
On the archived ``shelf`` (#492) the ids are archived already: nothing
|
||||
is archived or rewritten, every reviewed id not deleted (protected or
|
||||
changed) counts ``changed``, and one restored since ``not_archived``.
|
||||
"""
|
||||
on_archived_shelf = _check_shelf(shelf)
|
||||
_validate_epoch(expected_epoch)
|
||||
for ids in (archive_ids, delete_ids):
|
||||
if not isinstance(ids, list) or any(
|
||||
type(pid) is not int or not 1 <= pid <= 2**63 - 1 for pid in ids
|
||||
):
|
||||
raise ValueError("prompt ids must be lists of positive integers")
|
||||
reviewed_delete = set(delete_ids)
|
||||
delete_paired = {} if delete_paired is None else delete_paired
|
||||
if not isinstance(delete_paired, dict) or any(
|
||||
type(pid) is not int
|
||||
or not isinstance(digest, str)
|
||||
or not _DIGEST.match(digest)
|
||||
for pid, digest in delete_paired.items()
|
||||
):
|
||||
raise ValueError("delete_paired must map prompt ids to review digests")
|
||||
if not set(delete_paired) <= set(archive_ids):
|
||||
raise ValueError("A permanent delete must be a reviewed archive id")
|
||||
if not archive_ids and not delete_ids:
|
||||
raise ValueError("Nothing to delete or archive")
|
||||
if reviewed_delete & set(archive_ids):
|
||||
raise ValueError("A prompt cannot be both archived and deleted")
|
||||
prompt_ids = list(dict.fromkeys([*archive_ids, *delete_ids]))
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
classes = self._classify_library_delete(conn, prompt_ids, shelf)
|
||||
now_prints = self._fingerprints(conn, classes["delete"])
|
||||
deletes, archives, changed = [], [], 0
|
||||
paired_now = set(classes["deletable"])
|
||||
chosen = [pid for pid in classes["archive"] if pid in delete_paired]
|
||||
digests = self._review_digests(
|
||||
conn, [pid for pid in chosen if pid in paired_now]
|
||||
)
|
||||
paired_deletes = []
|
||||
for pid in classes["archive"]:
|
||||
if pid not in delete_paired:
|
||||
archives.append(pid)
|
||||
elif digests.get(pid) == delete_paired[pid]:
|
||||
paired_deletes.append(pid)
|
||||
else:
|
||||
archives.append(pid) # changed since the review: restorable
|
||||
changed += 1
|
||||
for pid in classes["delete"]:
|
||||
if pid not in reviewed_delete:
|
||||
archives.append(pid) # reviewed as archive, now unpaired
|
||||
changed += 1 if pid in delete_paired else 0
|
||||
elif candidates is None or candidates.get(pid) == now_prints[pid]:
|
||||
deletes.append(pid)
|
||||
else:
|
||||
changed += 1
|
||||
changed += len(classes["protected"])
|
||||
if on_archived_shelf:
|
||||
# Already archived: the rest stay as they are, each one counted.
|
||||
archives = []
|
||||
changed = len(prompt_ids) - len(deletes) - len(paired_deletes)
|
||||
changed -= len(classes["not_found"]) + len(classes["not_archived"])
|
||||
image_links = self._image_links(conn, [*deletes, *paired_deletes])
|
||||
now = utc_now_iso()
|
||||
for pid in archives:
|
||||
self._repair_hash(conn, pid)
|
||||
conn.executemany(
|
||||
f"INSERT INTO {ARCHIVE}(prompt_id, archived_at) VALUES (?, ?)",
|
||||
[(pid, now) for pid in archives],
|
||||
)
|
||||
# A changed row is the newest edit the restore fingerprint reads.
|
||||
conn.executemany(
|
||||
"UPDATE prompts SET updated_at=? WHERE id=?",
|
||||
[(now, pid) for pid in archives],
|
||||
)
|
||||
# #482: the image-pair links (RESTRICT) go first, then the pairs
|
||||
# (pair tags CASCADE, review decisions SET NULL), then the prompts.
|
||||
pairs_removed = 0
|
||||
for chunk in _batches(paired_deletes):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
pairs_of = (
|
||||
"SELECT id FROM prompt_pairs " # nosec B608
|
||||
f"WHERE positive_prompt_id IN ({marks})"
|
||||
)
|
||||
conn.execute(
|
||||
f"DELETE FROM image_prompt_pairs WHERE pair_id IN ({pairs_of})",
|
||||
chunk,
|
||||
)
|
||||
pairs_removed += conn.execute(
|
||||
"DELETE FROM prompt_pairs " # nosec B608
|
||||
f"WHERE positive_prompt_id IN ({marks})",
|
||||
chunk,
|
||||
).rowcount
|
||||
deleted = delete_prompt_rows(conn, [*deletes, *paired_deletes])
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"deleted": deleted,
|
||||
"archived": len(archives),
|
||||
"already_archived": len(classes["already_archived"]),
|
||||
"not_archived": len(classes["not_archived"]),
|
||||
"not_found": len(classes["not_found"]),
|
||||
"changed": changed,
|
||||
"image_links": image_links,
|
||||
"pairs_removed": pairs_removed,
|
||||
}
|
||||
|
||||
def restore_library_prompts(
|
||||
self, prompt_ids: list[int], *, expected_epoch: str
|
||||
) -> dict:
|
||||
"""Put archived prompts back on the shelf, all or nothing.
|
||||
|
||||
Every requested id with an archive row is restored, whatever its
|
||||
protection or Library parent state: restoring only un-hides it.
|
||||
"""
|
||||
prompt_ids = self._checked_ids(prompt_ids, expected_epoch)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
restored, existing = [], set()
|
||||
for chunk in _batches(prompt_ids):
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for pid, archived in conn.execute(
|
||||
f"SELECT p.id,{_ARCHIVED} FROM prompts p " # nosec B608
|
||||
f"WHERE p.id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
existing.add(pid)
|
||||
if archived:
|
||||
restored.append(pid)
|
||||
now = utc_now_iso()
|
||||
conn.executemany(
|
||||
f"DELETE FROM {ARCHIVE} WHERE prompt_id=?", [(pid,) for pid in restored]
|
||||
)
|
||||
conn.executemany(
|
||||
"UPDATE prompts SET updated_at=? WHERE id=?",
|
||||
[(now, pid) for pid in restored],
|
||||
)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"restored": len(restored),
|
||||
"unchanged": len(existing) - len(restored),
|
||||
"not_found": len(prompt_ids) - len(existing),
|
||||
}
|
||||
@@ -21,10 +21,6 @@ _UUID = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
|
||||
_TOKEN = re.compile(r"[0-9a-f]{64}")
|
||||
# Parent metadata stored on the prompt row; title lives in prompt_library_state.
|
||||
_PROMPT_COLUMNS = ("category", "rating", "notes")
|
||||
# Ids per IN (...) read: well under SQLite's oldest bound-parameter limit (999).
|
||||
_BULK_CHUNK = 500
|
||||
# The tags one prompt may carry: what the editor's save accepts (``validate_tags``).
|
||||
MAX_LIBRARY_TAGS = 20
|
||||
|
||||
|
||||
class LibraryCommandError(Exception):
|
||||
@@ -89,13 +85,6 @@ def _validate_epoch(expected_epoch: str) -> None:
|
||||
raise ValueError("expected_epoch must be a canonical UUID")
|
||||
|
||||
|
||||
def _validate_bulk_ids(prompt_ids: list[int]) -> None:
|
||||
if not isinstance(prompt_ids, list) or not prompt_ids:
|
||||
raise ValueError("prompt_ids must be a non-empty list")
|
||||
if any(type(pid) is not int or not 1 <= pid <= 2**63 - 1 for pid in prompt_ids):
|
||||
raise ValueError("prompt_ids must be positive integers")
|
||||
|
||||
|
||||
def _validate_guards(positive_id: int, expected_epoch: str, if_match: str) -> None:
|
||||
if type(positive_id) is not int or not 1 <= positive_id <= 2**63 - 1:
|
||||
raise ValueError("Invalid positive prompt ID")
|
||||
@@ -319,134 +308,3 @@ class LibraryCommandOperations:
|
||||
positive_created=False,
|
||||
pair_created=pair_created,
|
||||
)
|
||||
|
||||
def bulk_set_library_category(
|
||||
self, prompt_ids: list[int], category: str, *, expected_epoch: str
|
||||
) -> dict:
|
||||
"""Set one category on many Library parents, all or nothing (#437).
|
||||
|
||||
Guarded by the storage epoch the page read. Only rows whose category
|
||||
differs are written (their ``updated_at`` and edit token change) and
|
||||
registered as Library parents, as the editor's PATCH does; ids
|
||||
that are missing or are not Library parents count as not found.
|
||||
"""
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_bulk_ids(prompt_ids)
|
||||
prompt_ids = list(dict.fromkeys(prompt_ids))
|
||||
if not isinstance(category, str) or not category.strip():
|
||||
raise ValueError("Category is required")
|
||||
validate_category(category)
|
||||
category = category.strip()
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
current = {}
|
||||
for start in range(0, len(prompt_ids), _BULK_CHUNK):
|
||||
chunk = prompt_ids[start : start + _BULK_CHUNK]
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
current.update(
|
||||
conn.execute(
|
||||
f"SELECT p.id,p.category FROM prompts p " # nosec B608
|
||||
f"WHERE p.id IN ({marks}) AND {_ELIGIBLE}",
|
||||
chunk,
|
||||
).fetchall()
|
||||
)
|
||||
changed = [pid for pid, value in current.items() if value != category]
|
||||
for pid in changed:
|
||||
self._ensure_library_parent(conn, pid)
|
||||
now = utc_now_iso()
|
||||
conn.executemany(
|
||||
"UPDATE prompts SET category=?,updated_at=? WHERE id=?",
|
||||
[(category, now, pid) for pid in changed],
|
||||
)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"category": category,
|
||||
"updated": len(changed),
|
||||
"unchanged": len(current) - len(changed),
|
||||
"not_found": len(prompt_ids) - len(current),
|
||||
}
|
||||
|
||||
def bulk_add_library_tags(
|
||||
self, prompt_ids: list[int], tags: list[str], *, expected_epoch: str
|
||||
) -> dict:
|
||||
"""Add tags to many Library parents, all or nothing (#442).
|
||||
|
||||
Names are normalized once (trim, the vocabulary name rules, casefold
|
||||
repeats folded, at most 20). A parent is given only the names it lacks,
|
||||
compared with ``casefold`` against its own tags; registry rows are
|
||||
found and inserted by their exact name, and only for names some parent
|
||||
gets. A parent whose tags would pass 20 is skipped (``over_limit``) so
|
||||
the editor can still save it. Pair and image tags are not touched.
|
||||
"""
|
||||
from .annotations import ImageTagLimitError, normalize_image_tags
|
||||
|
||||
_validate_epoch(expected_epoch)
|
||||
_validate_bulk_ids(prompt_ids)
|
||||
prompt_ids = list(dict.fromkeys(prompt_ids))
|
||||
too_many = f"At most {MAX_LIBRARY_TAGS} tags can be added at once"
|
||||
try:
|
||||
tags = normalize_image_tags(tags)
|
||||
except ImageTagLimitError:
|
||||
# Its message names images; this is a prompt route.
|
||||
raise ValueError(too_many) from None
|
||||
if not tags:
|
||||
raise ValueError("tags must name at least one tag")
|
||||
if len(tags) > MAX_LIBRARY_TAGS:
|
||||
raise ValueError(too_many)
|
||||
with self._pair_transaction() as conn:
|
||||
epoch = self._library_epoch(conn, expected_epoch)
|
||||
held = self._library_parent_tags(conn, prompt_ids)
|
||||
additions, over_limit = {}, 0
|
||||
for pid, names in held.items():
|
||||
present = {name.casefold() for name in names}
|
||||
missing = [name for name in tags if name.casefold() not in present]
|
||||
if not missing:
|
||||
continue
|
||||
if len(names) + len(missing) > MAX_LIBRARY_TAGS:
|
||||
over_limit += 1
|
||||
continue
|
||||
additions[pid] = missing
|
||||
wanted = list(
|
||||
dict.fromkeys(n for names in additions.values() for n in names)
|
||||
)
|
||||
tag_ids = self._ensure_tags(conn, wanted)
|
||||
now = utc_now_iso()
|
||||
for pid, names in additions.items():
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO prompt_tags (prompt_id, tag_id) "
|
||||
"VALUES (?, ?)",
|
||||
[(pid, tag_ids[name]) for name in names],
|
||||
)
|
||||
self._ensure_library_parent(conn, pid)
|
||||
conn.execute("UPDATE prompts SET updated_at=? WHERE id=?", (now, pid))
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"tags": tags,
|
||||
"updated": len(additions),
|
||||
"unchanged": len(held) - len(additions) - over_limit,
|
||||
"not_found": len(prompt_ids) - len(held),
|
||||
"over_limit": over_limit,
|
||||
}
|
||||
|
||||
def _library_parent_tags(
|
||||
self, conn: sqlite3.Connection, prompt_ids: list[int]
|
||||
) -> dict[int, list[str]]:
|
||||
"""The Library parents among ``prompt_ids`` and their own tag names."""
|
||||
held: dict[int, list[str]] = {}
|
||||
for start in range(0, len(prompt_ids), _BULK_CHUNK):
|
||||
chunk = prompt_ids[start : start + _BULK_CHUNK]
|
||||
marks = ",".join("?" for _ in chunk)
|
||||
for (pid,) in conn.execute(
|
||||
f"SELECT p.id FROM prompts p WHERE p.id IN ({marks}) " # nosec B608
|
||||
f"AND {_ELIGIBLE}",
|
||||
chunk,
|
||||
):
|
||||
held[pid] = []
|
||||
for pid, name in conn.execute(
|
||||
"SELECT pt.prompt_id,t.name FROM prompt_tags pt " # nosec B608
|
||||
f"JOIN tags t ON t.id=pt.tag_id WHERE pt.prompt_id IN ({marks})",
|
||||
chunk,
|
||||
):
|
||||
if pid in held:
|
||||
held[pid].append(name)
|
||||
return held
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
"""Library output-folder filter (#444): folder names, root prefixes and listing.
|
||||
|
||||
A folder is a ``/``-joined path relative to an output root, as Images' folder
|
||||
filter and the legacy ``/prompt_manager/subfolders`` name them. A parent
|
||||
matches a folder when one of its linked images (``generated_images.prompt_id``,
|
||||
the link its previews and image count read) lies in that folder or below it,
|
||||
under any spelling of any output root.
|
||||
|
||||
Matching compares ``generated_images.file_path``, which is always
|
||||
``normcase(normpath(abspath(path)))`` (``models.normalize_image_path``), with
|
||||
prefixes built by the same normalization here, so case folding is the
|
||||
platform's own on both sides (Windows folds non-ASCII too, unlike SQLite's
|
||||
``lower()``) and the comparison is an exact ``substr`` with no wildcards.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
_MAX_FOLDER = 1024
|
||||
_MAX_PREFIXES = 64
|
||||
|
||||
|
||||
def normalize_folder(value: object, pathmod=os.path) -> str:
|
||||
"""The folder as ``a/b``, outer slashes dropped; empty means no filter.
|
||||
|
||||
``/`` always separates. ``\\`` separates only on Windows (``ntpath``): on
|
||||
POSIX it is an ordinary character of a folder's name, and the listing
|
||||
(``relative_folder``) names such a folder with it, so both agree.
|
||||
``.``/``..``/empty segments, control characters and (on Windows) a drive
|
||||
are refused: the value only ever names a folder below a root.
|
||||
"""
|
||||
if not isinstance(value, str) or len(value) > _MAX_FOLDER:
|
||||
raise ValueError(f"folder must be text of at most {_MAX_FOLDER} characters")
|
||||
windows = pathmod.sep == "\\"
|
||||
folder = (value.replace("\\", "/") if windows else value).strip("/")
|
||||
if not folder:
|
||||
return ""
|
||||
parts = folder.split("/")
|
||||
if (
|
||||
any(part in ("", ".", "..") for part in parts)
|
||||
or (windows and ":" in parts[0])
|
||||
or any(ord(char) < 32 or ord(char) == 127 for char in folder)
|
||||
):
|
||||
raise ValueError("folder must be a folder below an output root")
|
||||
return folder
|
||||
|
||||
|
||||
def _base(root: object, pathmod) -> str:
|
||||
return pathmod.normcase(pathmod.normpath(str(root)))
|
||||
|
||||
|
||||
def folder_prefixes(roots, folder: str, pathmod=os.path) -> list[str]:
|
||||
"""One ``file_path`` prefix per root spelling for a normalized folder."""
|
||||
prefixes = []
|
||||
for root in roots:
|
||||
joined = pathmod.join(_base(root, pathmod), *folder.split("/"))
|
||||
prefix = pathmod.normcase(pathmod.normpath(joined)) + pathmod.sep
|
||||
if prefix not in prefixes:
|
||||
prefixes.append(prefix)
|
||||
if not prefixes or len(prefixes) > _MAX_PREFIXES:
|
||||
raise ValueError("No output folder to filter by")
|
||||
return prefixes
|
||||
|
||||
|
||||
def folder_condition(prefixes: list[str]) -> tuple[str, list]:
|
||||
"""A condition on alias ``p``: one linked image under any prefix."""
|
||||
tests = " OR ".join("substr(fg.file_path,1,?)=?" for _ in prefixes)
|
||||
params = [value for prefix in prefixes for value in (len(prefix), prefix)]
|
||||
return (
|
||||
"EXISTS(SELECT 1 FROM generated_images fg WHERE fg.prompt_id=p.id "
|
||||
f"AND ({tests}))",
|
||||
params,
|
||||
)
|
||||
|
||||
|
||||
def relative_folder(image_path: str, roots, pathmod=os.path) -> str | None:
|
||||
"""The folder of *image_path* below the first root spelling holding it.
|
||||
|
||||
The spelling of the stored path is kept (its case on Windows); ``None``
|
||||
for an image directly in a root or outside every root.
|
||||
"""
|
||||
directory = pathmod.normpath(pathmod.dirname(image_path))
|
||||
folded = pathmod.normcase(directory)
|
||||
for root in roots:
|
||||
base = _base(root, pathmod)
|
||||
start = base if base.endswith(pathmod.sep) else base + pathmod.sep
|
||||
if not folded.startswith(start):
|
||||
continue
|
||||
# normcase keeps the length for ordinary names; else show the folded form.
|
||||
rest = (directory if len(directory) == len(folded) else folded)[len(start) :]
|
||||
parts = rest.split(pathmod.sep)
|
||||
if any(part in ("", ".", "..") for part in parts):
|
||||
return None
|
||||
return "/".join(parts)
|
||||
return None
|
||||
|
||||
|
||||
def list_library_folders(
|
||||
conn: sqlite3.Connection, roots, eligible: str, pathmod=os.path
|
||||
) -> list[str]:
|
||||
"""Every folder (and ancestor) holding an image of an eligible parent."""
|
||||
folders = set()
|
||||
for (image_path,) in conn.execute(
|
||||
"SELECT DISTINCT g.image_path FROM generated_images g "
|
||||
"JOIN prompts p ON p.id=g.prompt_id WHERE " + eligible
|
||||
):
|
||||
if not isinstance(image_path, str) or not image_path:
|
||||
continue
|
||||
folder = relative_folder(image_path, roots, pathmod)
|
||||
if folder is None:
|
||||
continue
|
||||
parts = folder.split("/")
|
||||
folders.update("/".join(parts[:index]) for index in range(1, len(parts) + 1))
|
||||
return sorted(folders, key=lambda value: (value.casefold(), value))
|
||||
@@ -78,10 +78,6 @@ MESSAGES = {
|
||||
),
|
||||
"same-folder": "The new folder is the node folder itself.",
|
||||
"target-invalid": "The new folder cannot hold the library.",
|
||||
"data-dir-served": (
|
||||
"The new folder is inside Prompt Manager's thumbnail folder. Nothing was "
|
||||
"moved."
|
||||
),
|
||||
"target-has-library": (
|
||||
"The new folder already holds a Prompt Manager library. Nothing was moved."
|
||||
),
|
||||
@@ -594,7 +590,6 @@ class _Move:
|
||||
try:
|
||||
os.rmdir(self.plan.target_dir)
|
||||
except OSError:
|
||||
# not empty or already gone: the undo leaves it alone
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -15,8 +15,6 @@ change:
|
||||
``cleanup_orphaned_images``), counted only under a source that can be read
|
||||
now, as the health check's ``missing_files`` judges them
|
||||
(``health.link_states``). It deletes database rows only, never a file.
|
||||
It also forgets the removal marks (#480) that hide nothing any more: the
|
||||
file is gone or changed, under a readable source (#501, ``_stale_marks``).
|
||||
|
||||
``preview_maintenance`` reads one snapshot on its own ``mode=ro`` connection
|
||||
(``health.readonly_uri``) and closes it in ``finally``; files are looked at
|
||||
@@ -43,16 +41,9 @@ from pathlib import Path
|
||||
|
||||
from .execution_contexts import LIVE_HINT_CTES
|
||||
from .health import BATCH_ROWS, link_states, readable_roots, readonly_uri
|
||||
from .image_removal_schema import TABLE as MARKS_TABLE
|
||||
from .image_removal_schema import clear_marks, has_marks_table
|
||||
from .pairs import PairDependencyError
|
||||
from .workflow_documents import documents_of_rows, release_documents
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
|
||||
MAINTENANCE_OPERATIONS = (
|
||||
"duplicate_prompts",
|
||||
"optimize",
|
||||
@@ -89,12 +80,6 @@ PROTECTIONS = (
|
||||
"with Library state (a title or an explicit parent)",
|
||||
"with Library state (a title or an explicit parent)",
|
||||
),
|
||||
(
|
||||
# #443: an archived prompt is restorable, never pruned.
|
||||
"EXISTS (SELECT 1 FROM prompt_archive pa WHERE pa.prompt_id = p.id)",
|
||||
"archived in the Library",
|
||||
"archived in the Library",
|
||||
),
|
||||
(
|
||||
"p.id IN (SELECT prompt_id FROM live_hint_prompts)",
|
||||
"used by recorded runs",
|
||||
@@ -103,8 +88,7 @@ PROTECTIONS = (
|
||||
)
|
||||
|
||||
# Exempt references: images, pairs, the __protected__ tag, Library parent
|
||||
# state, the Library archive, and live recorded-run hints (computed once,
|
||||
# see LIVE_HINT_CTES).
|
||||
# state, and live recorded-run hints (computed once, see LIVE_HINT_CTES).
|
||||
ORPHANED_PROMPTS_SQL = (
|
||||
"WITH "
|
||||
+ LIVE_HINT_CTES
|
||||
@@ -198,8 +182,6 @@ class MaintenanceDraft:
|
||||
protected: dict
|
||||
media: dict
|
||||
details: list = field(default_factory=list)
|
||||
# missing_links only: locators of removal marks whose file is gone (#501).
|
||||
marks: list = field(default_factory=list)
|
||||
|
||||
def public(self):
|
||||
"""The plan fields a response may carry (no fingerprint, no targets)."""
|
||||
@@ -302,96 +284,11 @@ def _delete_ids(conn, table, ids):
|
||||
return removed
|
||||
|
||||
|
||||
def _mark_rows(conn, only=None):
|
||||
"""``[(locator, size, mtime_ns)]`` of the removal marks (all, or those
|
||||
among *only*), sorted by locator."""
|
||||
if not has_marks_table(conn):
|
||||
return []
|
||||
sql = f"SELECT locator, size, mtime_ns FROM {MARKS_TABLE}" # nosec B608
|
||||
if only is None:
|
||||
return sorted(tuple(row) for row in conn.execute(sql))
|
||||
found = []
|
||||
for batch in _batches(sorted(only)):
|
||||
found.extend(
|
||||
tuple(row)
|
||||
for row in conn.execute(
|
||||
f"{sql} WHERE locator IN ({_marks(batch)})", batch # nosec B608
|
||||
)
|
||||
)
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def _lists_something(folder):
|
||||
"""Whether *folder* holds at least one entry (an unplugged drive's mount
|
||||
point, or an unbound volume, is an empty folder that reads fine)."""
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
return next(entries, None) is not None
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _name_listed(path):
|
||||
"""Whether *path*'s folder lists its name, compared under ``normcase``.
|
||||
|
||||
A locator is ``normcase``d; on a case-sensitive NTFS folder ``lstat`` of
|
||||
the lower-cased spelling fails for a file that is there.
|
||||
"""
|
||||
folder, name = os.path.split(path)
|
||||
wanted = os.path.normcase(name)
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
return any(os.path.normcase(entry.name) == wanted for entry in entries)
|
||||
except OSError:
|
||||
return True # cannot tell: keep the mark
|
||||
|
||||
|
||||
def _stale_mark(locator, size, mtime_ns):
|
||||
"""Whether the file of a mark is gone or has other bytes (blocking)."""
|
||||
try:
|
||||
info = os.lstat(locator)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return os.path.isdir(os.path.dirname(locator)) and not _name_listed(locator)
|
||||
except OSError:
|
||||
return False
|
||||
return (info.st_size, info.st_mtime_ns) != (size, mtime_ns)
|
||||
|
||||
|
||||
def _stale_marks(marks, roots):
|
||||
"""``[(locator, owner)]`` of the marks that hide nothing any more (#501).
|
||||
|
||||
*marks* are ``(locator, size, mtime_ns)`` rows. A mark is stale when its
|
||||
file was changed, or is gone from a folder that still exists. Only marks
|
||||
under a source that can be read now and lists something are judged (an
|
||||
unplugged drive's marks, or its empty mount point's, stay); any doubt
|
||||
keeps the mark. *owner* is the index of the longest source holding it.
|
||||
Locators are ``normcase(realpath)`` paths, so the sources are too.
|
||||
"""
|
||||
readable = readable_roots(roots)
|
||||
prefixes = [
|
||||
os.path.normcase(os.path.realpath(root)).rstrip("\\/") + os.sep
|
||||
for root in roots
|
||||
]
|
||||
judged = {}
|
||||
stale = []
|
||||
for locator, size, mtime_ns in marks:
|
||||
owners = [i for i, prefix in enumerate(prefixes) if locator.startswith(prefix)]
|
||||
if not owners:
|
||||
continue # under no source now (a source removed from the settings)
|
||||
owner = max(owners, key=lambda i: len(prefixes[i]))
|
||||
if owner not in judged:
|
||||
judged[owner] = readable[owner] and _lists_something(roots[owner])
|
||||
if judged[owner] and _stale_mark(locator, size, mtime_ns):
|
||||
stale.append((locator, owner))
|
||||
return stale
|
||||
|
||||
|
||||
def _read_snapshot(db_path, read):
|
||||
"""``read(conn)`` on one ``mode=ro`` snapshot; the connection always closes."""
|
||||
conn = sqlite3.connect(readonly_uri(db_path), uri=True)
|
||||
try:
|
||||
conn.row_factory = sqlite3.Row
|
||||
register_prompt_functions(conn) # LIVE_HINT_CTES calls them (#460)
|
||||
conn.execute("BEGIN") # one read snapshot for the whole plan
|
||||
return read(conn)
|
||||
finally:
|
||||
@@ -528,11 +425,8 @@ class MaintenanceOperations:
|
||||
if operation == "workflow_storage":
|
||||
return self._plan_workflow_storage()
|
||||
if operation == "missing_links":
|
||||
rows, marks = _read_snapshot(
|
||||
self.model.db_path,
|
||||
lambda conn: (self._link_rows(conn), _mark_rows(conn)),
|
||||
)
|
||||
return self._plan_missing_links(rows, roots, _stale_marks(marks, roots))
|
||||
rows = _read_snapshot(self.model.db_path, self._link_rows)
|
||||
return self._plan_missing_links(rows, roots)
|
||||
reader = (
|
||||
self._plan_duplicate_prompts
|
||||
if operation == "duplicate_prompts"
|
||||
@@ -663,15 +557,13 @@ class MaintenanceOperations:
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _links_fingerprint(candidates, stale=()):
|
||||
def _links_fingerprint(candidates):
|
||||
fingerprint = _Fingerprint("missing_links")
|
||||
for row, owner in candidates:
|
||||
fingerprint.add([row[0], row[1], row[3], owner])
|
||||
for locator, owner in stale:
|
||||
fingerprint.add(["mark", locator, owner])
|
||||
return fingerprint.hexdigest()
|
||||
|
||||
def _plan_missing_links(self, rows, roots, stale=()):
|
||||
def _plan_missing_links(self, rows, roots):
|
||||
readable = readable_roots(roots)
|
||||
candidates, unavailable, outside = [], 0, 0
|
||||
for row, owner, state in link_states(rows, roots, readable):
|
||||
@@ -699,47 +591,29 @@ class MaintenanceOperations:
|
||||
)
|
||||
)
|
||||
count = len(candidates)
|
||||
parts = []
|
||||
if count:
|
||||
parts.append(
|
||||
_plural(count, "link to a missing file", "links to missing files")
|
||||
)
|
||||
if stale:
|
||||
parts.append(
|
||||
_plural(
|
||||
len(stale),
|
||||
"removed image whose file is gone or changed",
|
||||
"removed images whose files are gone or changed",
|
||||
)
|
||||
)
|
||||
details = [
|
||||
{
|
||||
"label": _relative(row[1], roots[owner]),
|
||||
"detail": f"{_source_name(roots[owner], owner)} · the file is missing",
|
||||
}
|
||||
for row, owner in candidates
|
||||
] + [
|
||||
{
|
||||
"label": _relative(locator, os.path.realpath(roots[owner])),
|
||||
"detail": (
|
||||
f"{_source_name(roots[owner], owner)} · removed image, "
|
||||
"the file is gone or changed"
|
||||
),
|
||||
}
|
||||
for locator, owner in stale
|
||||
]
|
||||
return MaintenanceDraft(
|
||||
operation="missing_links",
|
||||
fingerprint=self._links_fingerprint(candidates, stale),
|
||||
fingerprint=self._links_fingerprint(candidates),
|
||||
targets=[row[0] for row, _owner in candidates],
|
||||
scope={
|
||||
"count": count + len(stale),
|
||||
"label": "; ".join(parts) or "No links to missing files",
|
||||
"count": count,
|
||||
"label": (
|
||||
_plural(count, "link to a missing file", "links to missing files")
|
||||
if count
|
||||
else "No links to missing files"
|
||||
),
|
||||
},
|
||||
protected={"count": unavailable + outside, "reasons": reasons},
|
||||
media={"deletes_files": False, "label": LINKS_ONLY},
|
||||
details=details[:MAX_DETAILS],
|
||||
marks=[locator for locator, _owner in stale],
|
||||
details=[
|
||||
{
|
||||
"label": _relative(row[1], roots[owner]),
|
||||
"detail": (
|
||||
f"{_source_name(roots[owner], owner)} · the file is missing"
|
||||
),
|
||||
}
|
||||
for row, owner in candidates[:MAX_DETAILS]
|
||||
],
|
||||
)
|
||||
|
||||
def _plan_optimize(self):
|
||||
@@ -789,18 +663,14 @@ class MaintenanceOperations:
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
def apply_maintenance(
|
||||
self, operation, fingerprint, targets, roots=(), marks=()
|
||||
) -> dict:
|
||||
def apply_maintenance(self, operation, fingerprint, targets, roots=()) -> dict:
|
||||
"""Apply a reviewed plan exactly, or raise PlanChanged (blocking).
|
||||
|
||||
Inside ``BEGIN IMMEDIATE`` the fingerprint is computed again on the
|
||||
write connection; when it differs nothing is written. ``targets`` and
|
||||
``roots`` are the plan's. Returns ``{changed, rows}``: how many
|
||||
records changed and up to ``MAX_RESULT_ROWS`` descriptions of them
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``). ``marks`` are a
|
||||
missing-links plan's stale removal marks; that apply also returns
|
||||
``marks``, how many it forgot.
|
||||
(``{prompt_id[, kept_id]}`` or ``{image_path}``).
|
||||
"""
|
||||
if operation not in MAINTENANCE_OPERATIONS or operation == "workflow_storage":
|
||||
raise ValueError(f"Unknown maintenance operation {operation!r}")
|
||||
@@ -813,12 +683,9 @@ class MaintenanceOperations:
|
||||
appliers = {
|
||||
"duplicate_prompts": self._apply_duplicate_prompts,
|
||||
"unused_components": self._apply_unused_components,
|
||||
"missing_links": self._apply_missing_links,
|
||||
}
|
||||
with self._pair_transaction() as conn:
|
||||
if operation == "missing_links":
|
||||
return self._apply_missing_links(
|
||||
conn, fingerprint, targets, roots, list(marks)
|
||||
)
|
||||
return appliers[operation](conn, fingerprint, targets, roots)
|
||||
|
||||
def _apply_duplicate_prompts(self, conn, fingerprint, targets, _roots):
|
||||
@@ -847,7 +714,7 @@ class MaintenanceOperations:
|
||||
]
|
||||
return {"changed": changed, "rows": rows}
|
||||
|
||||
def _apply_missing_links(self, conn, fingerprint, targets, roots, marks):
|
||||
def _apply_missing_links(self, conn, fingerprint, targets, roots):
|
||||
rows = []
|
||||
for batch in _batches(sorted(targets)):
|
||||
rows.extend(
|
||||
@@ -864,14 +731,10 @@ class MaintenanceOperations:
|
||||
if state != "missing": # back, or its source went away
|
||||
raise PlanChanged("missing_links")
|
||||
candidates.append((row, owner))
|
||||
stale = _stale_marks(_mark_rows(conn, marks) if marks else [], roots)
|
||||
if [locator for locator, _owner in stale] != marks: # a mark went or is valid
|
||||
raise PlanChanged("missing_links")
|
||||
if self._links_fingerprint(candidates, stale) != fingerprint:
|
||||
if self._links_fingerprint(candidates) != fingerprint:
|
||||
raise PlanChanged("missing_links")
|
||||
changed = _delete_ids(conn, "generated_images", [row[0] for row in rows])
|
||||
return {
|
||||
"changed": changed,
|
||||
"marks": clear_marks(conn, marks),
|
||||
"rows": [{"image_path": row[1]} for row in rows[:MAX_RESULT_ROWS]],
|
||||
}
|
||||
|
||||
@@ -35,51 +35,38 @@ from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import operator
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
from collections import deque
|
||||
from typing import Any, Callable, Iterable, Iterator, List, Optional
|
||||
from typing import Iterable, Iterator, List, Optional
|
||||
|
||||
import sqlite3
|
||||
|
||||
CLOSED_MESSAGE = "Cannot operate on a closed database."
|
||||
# Rows fetched per step when a cursor is iterated: the connection is marked
|
||||
# in use once per chunk instead of once per row (iteration costs about what
|
||||
# a plain cursor costs). Prefetched rows belong to the cursor so iteration
|
||||
# and explicit fetches always advance the same position.
|
||||
# a plain cursor costs). A loop that stops early leaves up to this many rows
|
||||
# fetched; ``fetchone``/``fetchmany`` after a partial loop skip them.
|
||||
ITER_CHUNK = 128
|
||||
|
||||
|
||||
class ManagedCursor(sqlite3.Cursor):
|
||||
"""A cursor whose calls into SQLite count as its connection being in use."""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._pending = deque()
|
||||
self._closed = False
|
||||
|
||||
def execute(self, sql, parameters=(), /):
|
||||
return self._run_statement(super().execute, sql, parameters)
|
||||
|
||||
def executemany(self, sql, seq_of_parameters, /):
|
||||
return self._run_statement(super().executemany, sql, seq_of_parameters)
|
||||
|
||||
def _run_statement(self, operation: Callable, *args: Any) -> ManagedCursor:
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
description = self.description
|
||||
completed = False
|
||||
try:
|
||||
result = operation(*args)
|
||||
completed = True
|
||||
return result
|
||||
return super().execute(sql, parameters)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def executemany(self, sql, seq_of_parameters, /):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
return super().executemany(sql, seq_of_parameters)
|
||||
finally:
|
||||
# Argument parsing can fail before SQLite replaces the old statement.
|
||||
if completed or self.description is not description:
|
||||
self._pending.clear()
|
||||
conn._leave()
|
||||
|
||||
def executescript(self, sql_script, /):
|
||||
@@ -94,8 +81,6 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
if self._pending and not self._closed:
|
||||
return self._pending.popleft()
|
||||
return super().fetchone()
|
||||
finally:
|
||||
conn._leave()
|
||||
@@ -104,20 +89,7 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
# Let SQLite validate the original arguments, including version-specific
|
||||
# handling of zero/negative sizes, before touching prefetched rows.
|
||||
rows = super().fetchmany(*args, **kwargs)
|
||||
if not self._pending:
|
||||
return rows
|
||||
size = operator.index(
|
||||
kwargs.get("size", args[0] if args else self.arraysize)
|
||||
)
|
||||
# Python 3.12 reads all rows for size=0; 3.13+ reads none.
|
||||
if size == 0 and sys.version_info >= (3, 13):
|
||||
return rows
|
||||
self._pending.extend(rows)
|
||||
count = len(self._pending) if size <= 0 else min(size, len(self._pending))
|
||||
return [self._pending.popleft() for _ in range(count)]
|
||||
return super().fetchmany(*args, **kwargs)
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
@@ -125,31 +97,25 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
conn = self.connection
|
||||
conn._enter()
|
||||
try:
|
||||
rows = super().fetchall()
|
||||
pending = list(self._pending)
|
||||
self._pending.clear()
|
||||
return pending + rows
|
||||
return super().fetchall()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
def __iter__(self) -> Iterator:
|
||||
return self
|
||||
return self._chunks()
|
||||
|
||||
def _chunks(self) -> Iterator:
|
||||
while True:
|
||||
rows = self.fetchmany(ITER_CHUNK)
|
||||
if not rows:
|
||||
return
|
||||
yield from rows
|
||||
|
||||
def __next__(self):
|
||||
conn = self.connection
|
||||
pending = self._pending
|
||||
# Buffered Python rows need no SQLite lock, but retirement and cursor
|
||||
# close must still stop an iterator obtained before either operation.
|
||||
if conn._retired or self._closed:
|
||||
raise sqlite3.ProgrammingError(CLOSED_MESSAGE)
|
||||
if pending:
|
||||
return pending.popleft()
|
||||
conn._enter()
|
||||
try:
|
||||
pending.extend(super().fetchmany(ITER_CHUNK))
|
||||
if not pending:
|
||||
raise StopIteration
|
||||
return pending.popleft()
|
||||
return super().__next__()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
@@ -160,10 +126,7 @@ class ManagedCursor(sqlite3.Cursor):
|
||||
except sqlite3.ProgrammingError:
|
||||
return super().close() # the connection is closed or closing
|
||||
try:
|
||||
result = super().close()
|
||||
self._pending = deque()
|
||||
self._closed = True
|
||||
return result
|
||||
return super().close()
|
||||
finally:
|
||||
conn._leave()
|
||||
|
||||
@@ -330,9 +293,6 @@ class ManagedConnection(sqlite3.Connection):
|
||||
cursors = list(self._cursors)
|
||||
try:
|
||||
for cursor in cursors:
|
||||
if isinstance(cursor, ManagedCursor):
|
||||
cursor._pending = deque()
|
||||
cursor._closed = True
|
||||
with contextlib.suppress(sqlite3.Error):
|
||||
sqlite3.Cursor.close(cursor)
|
||||
super().close()
|
||||
|
||||
@@ -38,14 +38,7 @@ from .review_schema import (
|
||||
review_without_annotations_error,
|
||||
)
|
||||
from .saved_views import migrate_saved_views, saved_views_schema_error
|
||||
|
||||
try:
|
||||
from ..utils.hashing import register_prompt_functions
|
||||
except ImportError:
|
||||
from utils.hashing import register_prompt_functions
|
||||
from .image_categories import image_categories_schema_error, migrate_image_categories
|
||||
from .image_removal_schema import image_removals_schema_error, migrate_image_removals
|
||||
from .prompt_archive import migrate_prompt_archive, prompt_archive_schema_error
|
||||
from .workflow_documents import (
|
||||
DOCUMENT_ID_COLUMNS,
|
||||
migrate_workflow_documents,
|
||||
@@ -160,15 +153,12 @@ def configure_connection(conn: sqlite3.Connection) -> None:
|
||||
(common on Windows) let other writers' polling busy handlers starve past
|
||||
BUSY_TIMEOUT_SECONDS and fail with "database is locked" (#242). Without
|
||||
WAL (for example an in-memory database) SQLite's FULL default is kept.
|
||||
It also registers the prompt-key SQL functions (register_prompt_functions).
|
||||
"""
|
||||
journal_mode = conn.execute("PRAGMA journal_mode = WAL").fetchone()[0]
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}")
|
||||
if str(journal_mode).lower() == "wal":
|
||||
conn.execute("PRAGMA synchronous = NORMAL")
|
||||
# History reads and the live-hint rule call Python's prompt key (#460)
|
||||
register_prompt_functions(conn)
|
||||
|
||||
|
||||
class PromptModel:
|
||||
@@ -368,10 +358,6 @@ class PromptModel:
|
||||
migrate_autotag_state(conn)
|
||||
# Image categories (#332): independent of every table above.
|
||||
migrate_image_categories(conn)
|
||||
# The Library parent archive (#443): independent of every table above.
|
||||
migrate_prompt_archive(conn)
|
||||
# Removal marks (#480): independent of every table above.
|
||||
migrate_image_removals(conn)
|
||||
# Last: the older generated_images rebuilds run only before file_path.
|
||||
migrate_workflow_documents(conn)
|
||||
|
||||
@@ -1248,16 +1234,6 @@ class PromptModel:
|
||||
)
|
||||
if categories_error:
|
||||
return False, categories_error
|
||||
archive_error = prompt_archive_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if archive_error:
|
||||
return False, archive_error
|
||||
removals_error = image_removals_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
if removals_error:
|
||||
return False, removals_error
|
||||
views_error = saved_views_schema_error(
|
||||
conn, allow_missing=not require_library_state
|
||||
)
|
||||
|
||||
@@ -18,14 +18,12 @@ from .models import PromptModel, normalize_image_path, utc_now_iso
|
||||
from .pairs import PairDependencyError, PairOperations # noqa: F401 (re-exported)
|
||||
from .library import LibraryOperations, titled
|
||||
from .image_favorites import ImageFavoriteOperations
|
||||
from .image_removal import ImageRemovalOperations, removed_from_library
|
||||
from .image_collections import ImageCollectionOperations
|
||||
from .annotations import AnnotationOperations
|
||||
from .review import ReviewOperations
|
||||
from .review_decisions import ReviewDecisionOperations
|
||||
from .saved_views import SavedViewOperations
|
||||
from .library_commands import LibraryCommandOperations
|
||||
from .library_archive import LibraryArchiveOperations, delete_prompt_rows
|
||||
from .vocabulary import VocabularyOperations
|
||||
from .execution_history import ExecutionHistoryOperations
|
||||
from .execution_contexts import ExecutionContextOperations
|
||||
@@ -34,7 +32,7 @@ from .history_reads import HistoryReadOperations
|
||||
from .health import HealthOperations, iter_consistency_issues
|
||||
from .duplicate_copies import DuplicateCopyOperations
|
||||
from .workflow_storage import WorkflowStorageOperations
|
||||
from .category_import import CategoryImportOperations
|
||||
from .category_import import CategoryImportOperations, EmptyReplacement # noqa: F401
|
||||
from .image_categories import ImageCategoryOperations
|
||||
from .workflow_documents import (
|
||||
DocumentReader,
|
||||
@@ -450,7 +448,6 @@ class PromptDatabase(
|
||||
PairOperations,
|
||||
LibraryOperations,
|
||||
LibraryCommandOperations,
|
||||
LibraryArchiveOperations,
|
||||
VocabularyOperations,
|
||||
ExecutionHistoryOperations,
|
||||
ExecutionContextOperations,
|
||||
@@ -468,7 +465,6 @@ class PromptDatabase(
|
||||
WorkflowStorageOperations,
|
||||
CategoryImportOperations,
|
||||
ImageCategoryOperations,
|
||||
ImageRemovalOperations,
|
||||
):
|
||||
"""Database operations class for managing prompts."""
|
||||
|
||||
@@ -1810,11 +1806,6 @@ class PromptDatabase(
|
||||
"""
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
if removed_from_library(conn, image_path):
|
||||
# #480: the user removed this exact file from the library; its
|
||||
# bytes are unchanged, so nothing links it again.
|
||||
self.logger.debug("Not linking a file removed from the library")
|
||||
return 0
|
||||
filename = os.path.basename(image_path)
|
||||
file_path = normalize_image_path(image_path)
|
||||
file_size, width, height, image_format = _coerce_file_info(metadata)
|
||||
@@ -2353,10 +2344,16 @@ class PromptDatabase(
|
||||
Returns:
|
||||
Number of prompts actually deleted.
|
||||
"""
|
||||
count = 0
|
||||
with self._pair_transaction() as conn:
|
||||
self._ensure_prompts_unpaired(conn, prompt_ids)
|
||||
# The workspace Library's delete runs the same statements (#443).
|
||||
count = delete_prompt_rows(conn, list(dict.fromkeys(prompt_ids)))
|
||||
documents = documents_of_rows(conn, "prompt_id", prompt_ids)
|
||||
for pid in prompt_ids:
|
||||
conn.execute("DELETE FROM generated_images WHERE prompt_id = ?", (pid,))
|
||||
cursor = conn.execute("DELETE FROM prompts WHERE id = ?", (pid,))
|
||||
if cursor.rowcount > 0:
|
||||
count += 1
|
||||
release_documents(conn, documents)
|
||||
conn.commit()
|
||||
return count
|
||||
|
||||
|
||||
@@ -112,14 +112,8 @@ class PairOperations:
|
||||
tags: Optional[List[str]] = None,
|
||||
rating: Optional[int] = None,
|
||||
notes: Optional[str] = None,
|
||||
fill_new_positive: bool = False,
|
||||
) -> int:
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata.
|
||||
|
||||
``fill_new_positive`` also gives the positive prompt the category and
|
||||
tags, only if this save inserted its row (#546). An existing prompt
|
||||
never changes, so a cleared category stays cleared.
|
||||
"""
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata."""
|
||||
with self._pair_transaction() as conn:
|
||||
return self._save_pair(
|
||||
conn,
|
||||
@@ -130,7 +124,6 @@ class PairOperations:
|
||||
tags=tags,
|
||||
rating=rating,
|
||||
notes=notes,
|
||||
fill_new_positive=fill_new_positive,
|
||||
)
|
||||
|
||||
def _save_pair(
|
||||
@@ -144,7 +137,6 @@ class PairOperations:
|
||||
tags: Optional[List[str]] = None,
|
||||
rating: Optional[int] = None,
|
||||
notes: Optional[str] = None,
|
||||
fill_new_positive: bool = False,
|
||||
) -> int:
|
||||
"""Save an ordered pair atomically; repeat saves preserve its metadata."""
|
||||
if not isinstance(positive_text, str) or not positive_text.strip():
|
||||
@@ -152,13 +144,8 @@ class PairOperations:
|
||||
if not isinstance(negative_text, str):
|
||||
raise ValueError("Negative prompt text must be a string; use '' for empty")
|
||||
# Validate metadata before any component insertion.
|
||||
tags = _validate_metadata(title, category, tags, rating, notes)
|
||||
positive, created = self._pair_component_outcome(conn, positive_text)
|
||||
if created and fill_new_positive:
|
||||
conn.execute(
|
||||
"UPDATE prompts SET category = ? WHERE id = ?", (category, positive)
|
||||
)
|
||||
self._sync_prompt_tags(conn, positive, tags or [])
|
||||
_validate_metadata(title, category, tags, rating, notes)
|
||||
positive = self._pair_component(conn, positive_text)
|
||||
negative = (
|
||||
self._pair_component(conn, negative_text) if negative_text.strip() else None
|
||||
)
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
"""Library parent archive (#443, D152): prompts hidden from the shelf, restorable.
|
||||
|
||||
A row means the Library parent is archived; no row means it is on the shelf.
|
||||
Nothing else about the prompt changes: its text, metadata, tags, run count,
|
||||
image links, pairs and recorded runs stay. Restore deletes the row. Deleting
|
||||
the prompt (any path) drops the row by cascade.
|
||||
|
||||
A table of its own, not a column: ``prompt_library_state`` is validated by its
|
||||
exact definition, so a new column there would make older builds refuse the
|
||||
upgraded database, while older builds ignore an unknown table (they show the
|
||||
archived prompts again and keep the rows). Optional in old backups, required
|
||||
once a database is current.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
TABLE = "prompt_archive"
|
||||
MAX_STAMP_BYTES = 64
|
||||
# What ``utc_now_iso`` writes starts with a date; anything else was not written here.
|
||||
_STAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T*"
|
||||
_DDL = (
|
||||
f"CREATE TABLE {TABLE} (prompt_id INTEGER PRIMARY KEY, "
|
||||
"archived_at TEXT NOT NULL, "
|
||||
"FOREIGN KEY(prompt_id) REFERENCES prompts(id) ON DELETE CASCADE)"
|
||||
)
|
||||
_COLUMNS = (("prompt_id", "INTEGER", 0, 1), ("archived_at", "TEXT", 1, 0))
|
||||
_FOREIGN_KEYS = {("prompt_id", "prompts", "id", "CASCADE")}
|
||||
# A predicate on ``p`` (the prompts alias): the parent is not archived.
|
||||
SHELVED = f"NOT EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
|
||||
ARCHIVED = f"EXISTS(SELECT 1 FROM {TABLE} pa WHERE pa.prompt_id=p.id)"
|
||||
|
||||
|
||||
def _compact(sql: str) -> str:
|
||||
return re.sub(r"\s+", "", sql or "").lower()
|
||||
|
||||
|
||||
def _structure_error(conn: sqlite3.Connection, allow_missing: bool) -> str | None:
|
||||
objects = conn.execute(
|
||||
"SELECT type,sql FROM sqlite_master WHERE name=? "
|
||||
"OR (type IN ('index','trigger') AND tbl_name=?)",
|
||||
(TABLE, TABLE),
|
||||
).fetchall()
|
||||
if not objects:
|
||||
return None if allow_missing else "Missing prompt archive schema"
|
||||
tables = [sql for kind, sql in objects if kind == "table"]
|
||||
if len(tables) != 1 or len(objects) != 1:
|
||||
return "Prompt archive name is reserved by an unsupported schema object"
|
||||
if _compact(tables[0]) != _compact(_DDL):
|
||||
return "Prompt archive has an invalid table definition"
|
||||
columns = tuple(
|
||||
(row[1], row[2].upper(), row[3], row[5])
|
||||
for row in conn.execute(f"PRAGMA table_info({TABLE})")
|
||||
)
|
||||
if columns != _COLUMNS:
|
||||
return "Prompt archive has invalid columns"
|
||||
foreign_keys = {
|
||||
(row[3], row[2], row[4], row[6])
|
||||
for row in conn.execute(f"PRAGMA foreign_key_list({TABLE})")
|
||||
}
|
||||
if foreign_keys != _FOREIGN_KEYS:
|
||||
return "Prompt archive has invalid foreign key definitions"
|
||||
return None
|
||||
|
||||
|
||||
def prompt_archive_schema_error(
|
||||
conn: sqlite3.Connection, *, allow_missing: bool = True
|
||||
) -> str | None:
|
||||
"""Reject an unsupported archive schema or values; reads only.
|
||||
|
||||
A missing table is acceptable for backups that predate it unless
|
||||
``allow_missing`` is false.
|
||||
"""
|
||||
error = _structure_error(conn, allow_missing)
|
||||
if error is not None:
|
||||
return error
|
||||
if not conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)
|
||||
).fetchone():
|
||||
return None
|
||||
if conn.execute(f"PRAGMA foreign_key_check({TABLE})").fetchone():
|
||||
return "Prompt archive contains foreign key violations"
|
||||
if conn.execute(
|
||||
f"SELECT 1 FROM {TABLE} WHERE typeof(archived_at)!='text' " # nosec B608
|
||||
f"OR length(CAST(archived_at AS BLOB))>{MAX_STAMP_BYTES} "
|
||||
f"OR archived_at NOT GLOB '{_STAMP_GLOB}' LIMIT 1"
|
||||
).fetchone():
|
||||
return "Prompt archive contains invalid values"
|
||||
return None
|
||||
|
||||
|
||||
def migrate_prompt_archive(conn: sqlite3.Connection) -> None:
|
||||
"""Install the table atomically; validate before and after (no backfill)."""
|
||||
conn.execute("SAVEPOINT prompt_archive_schema")
|
||||
try:
|
||||
error = _structure_error(conn, True)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute(_DDL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1))
|
||||
error = _structure_error(conn, False)
|
||||
if error:
|
||||
raise sqlite3.DatabaseError(error)
|
||||
conn.execute("RELEASE prompt_archive_schema")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK TO prompt_archive_schema")
|
||||
conn.execute("RELEASE prompt_archive_schema")
|
||||
raise
|
||||
@@ -149,7 +149,6 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
|
||||
collections (#331).
|
||||
"""
|
||||
from .library import _ELIGIBLE
|
||||
from .prompt_archive import SHELVED, TABLE as ARCHIVE
|
||||
|
||||
has_state = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND "
|
||||
@@ -158,15 +157,9 @@ def library_summary(conn: sqlite3.Connection) -> Dict[str, int]:
|
||||
has_pairs = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='prompt_pairs'"
|
||||
).fetchone()
|
||||
has_archive = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (ARCHIVE,)
|
||||
).fetchone()
|
||||
if has_state and has_pairs:
|
||||
# Archived parents (#443) are off the shelf, as in library_counts.
|
||||
positives = conn.execute(
|
||||
"SELECT COUNT(*) FROM prompts p WHERE "
|
||||
+ _ELIGIBLE
|
||||
+ (" AND " + SHELVED if has_archive else "")
|
||||
"SELECT COUNT(*) FROM prompts p WHERE " + _ELIGIBLE
|
||||
).fetchone()[0]
|
||||
else:
|
||||
positives = _count(conn, "prompts")
|
||||
|
||||
@@ -5,12 +5,8 @@ A decision is one row of ``media_review_decisions`` for one item revision.
|
||||
ingester's compatible-only rules (``_ingest_resolved_pair_in``) in the same
|
||||
``_pair_transaction`` as the decision row, the item's new state and revision
|
||||
and the review index revision; any conflict or later failure rolls all of it
|
||||
back. ``manual`` (#536) is the same but for a Library prompt the reviewer chose:
|
||||
its link's provenance names a manual choice and carries no prompt sides, so it
|
||||
cannot pose as file evidence, and the decision's ``candidate_hash`` (the table
|
||||
requires one) is the digest of that choice. ``unresolved`` records the decision
|
||||
and links nothing. No decision counts a run, touches usage or calls an execution
|
||||
writer.
|
||||
back. ``unresolved`` records the decision and links nothing. No decision
|
||||
counts a run, touches usage or calls an execution writer.
|
||||
|
||||
Callers verify the file first (the capture must be the item's bytes and the
|
||||
current file must still hash to them); nothing here reads the filesystem.
|
||||
@@ -21,7 +17,6 @@ import sqlite3
|
||||
|
||||
from .annotation_schema import BYTES_TABLE
|
||||
from .annotations import _new_id
|
||||
from .library import _ELIGIBLE, _FROM, SHELVED
|
||||
from .library_commands import LibraryCommandError
|
||||
from .models import utc_now_iso
|
||||
from .review import (
|
||||
@@ -35,28 +30,9 @@ from .review import (
|
||||
same_name_rows,
|
||||
)
|
||||
from .review_changes import notify_review_change
|
||||
from .review_schema import (
|
||||
DECISIONS_TABLE,
|
||||
ITEMS_TABLE,
|
||||
canonical_evidence,
|
||||
evidence_digest,
|
||||
)
|
||||
from .review_schema import DECISIONS_TABLE, ITEMS_TABLE, canonical_evidence
|
||||
|
||||
DECISION_KIND = "user-selection"
|
||||
MANUAL_KIND = "manual-link"
|
||||
_MANUAL_POSITIVE = (
|
||||
f"SELECT p.id,p.text FROM {_FROM} " # nosec B608
|
||||
f"WHERE p.id=? AND {_ELIGIBLE} AND {SHELVED}"
|
||||
)
|
||||
_MANUAL_PAIR = (
|
||||
"SELECT COALESCE(n.text,'') AS negative FROM prompt_pairs pp "
|
||||
"LEFT JOIN prompts n ON n.id=pp.negative_prompt_id "
|
||||
"WHERE pp.id=? AND pp.positive_prompt_id=? AND pp.archived=0"
|
||||
)
|
||||
_ARCHIVED_BARE_PAIR = (
|
||||
"SELECT 1 FROM prompt_pairs WHERE positive_prompt_id=? "
|
||||
"AND negative_prompt_id IS NULL AND archived=1"
|
||||
)
|
||||
_INSERT_DECISION = (
|
||||
f"INSERT INTO {DECISIONS_TABLE}(id,item_id,decision_revision," # nosec B608
|
||||
"created_at,outcome,candidate_hash,evidence,pair_id,image_id) "
|
||||
@@ -83,8 +59,6 @@ class ReviewCommandError(LibraryCommandError):
|
||||
``fingerprint`` the current dependency fingerprint, when known;
|
||||
``changed`` marks an association conflict caused by links that changed
|
||||
since the reviewer read them (rather than an incompatible choice).
|
||||
``prompt-missing``: the Library prompt or recipe of a manual link is gone,
|
||||
archived or not that prompt's.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -119,7 +93,6 @@ class ReviewDecisionOperations:
|
||||
sha256: str,
|
||||
outcome: str,
|
||||
candidate: dict | None = None,
|
||||
manual: dict | None = None,
|
||||
image_path: str | None = None,
|
||||
metadata: dict | None = None,
|
||||
expected_fingerprint: str | None = None,
|
||||
@@ -128,9 +101,7 @@ class ReviewDecisionOperations:
|
||||
) -> dict:
|
||||
"""Record the decision; return ``{item, total, pair}``.
|
||||
|
||||
``selected`` needs the evidence ``candidate`` (with its ``hash``) or
|
||||
``manual`` (``{positive_id, pair_id}``; a null ``pair_id`` is no
|
||||
negative), the
|
||||
``selected`` needs the evidence ``candidate`` (with its ``hash``), the
|
||||
``image_path`` the scan and the monitor link under, the captured
|
||||
``metadata`` and the ``expected_fingerprint`` the reviewer saw.
|
||||
``aliases`` are the other spellings legacy rows of the same physical
|
||||
@@ -144,12 +115,9 @@ class ReviewDecisionOperations:
|
||||
if outcome not in ("selected", "unresolved"):
|
||||
raise ValueError("outcome must be selected or unresolved")
|
||||
if outcome == "selected" and (
|
||||
(not isinstance(candidate, dict)) == (not isinstance(manual, dict))
|
||||
or not image_path
|
||||
not isinstance(candidate, dict) or not image_path
|
||||
):
|
||||
raise ValueError(
|
||||
"A selected decision needs one candidate or manual choice and a file"
|
||||
)
|
||||
raise ValueError("A selected decision needs a candidate and a file")
|
||||
try:
|
||||
with self._pair_transaction() as conn:
|
||||
decided = self._decide_in(
|
||||
@@ -160,7 +128,6 @@ class ReviewDecisionOperations:
|
||||
sha256=sha256,
|
||||
outcome=outcome,
|
||||
candidate=candidate,
|
||||
manual=manual,
|
||||
image_path=image_path,
|
||||
metadata=metadata or {},
|
||||
expected_fingerprint=expected_fingerprint,
|
||||
@@ -191,24 +158,13 @@ class ReviewDecisionOperations:
|
||||
raise _Rollback(ReviewCommandError("file-changed", epoch))
|
||||
decision_id = _new_id()
|
||||
pair = None
|
||||
selected = command["outcome"] == "selected"
|
||||
candidate = command["candidate"] if selected else None
|
||||
manual = command["manual"] if selected else None
|
||||
candidate_hash = candidate["hash"] if candidate is not None else None
|
||||
candidate = command["candidate"] if command["outcome"] == "selected" else None
|
||||
if candidate is not None:
|
||||
pair = self._apply_candidate(
|
||||
conn, epoch, current, decision_id, total, **command
|
||||
)
|
||||
elif manual is not None:
|
||||
pair = self._apply_manual(
|
||||
conn, epoch, current, decision_id, total, **command
|
||||
)
|
||||
manual = {"positive_id": manual["positive_id"], "pair_id": pair["id"]}
|
||||
candidate_hash = evidence_digest(
|
||||
canonical_evidence({"kind": MANUAL_KIND, **manual})
|
||||
)
|
||||
evidence = {
|
||||
"kind": MANUAL_KIND if manual is not None else DECISION_KIND,
|
||||
"kind": DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"outcome": command["outcome"],
|
||||
"reviewed_digest": command["sha256"],
|
||||
@@ -218,8 +174,6 @@ class ReviewDecisionOperations:
|
||||
"dependency_fingerprint": command["expected_fingerprint"],
|
||||
"pair": pair,
|
||||
}
|
||||
if manual is not None:
|
||||
evidence["manual"] = manual
|
||||
now = utc_now_iso()
|
||||
self._insert_review_decision(
|
||||
conn,
|
||||
@@ -229,7 +183,7 @@ class ReviewDecisionOperations:
|
||||
current["revision"] + 1,
|
||||
now,
|
||||
command["outcome"],
|
||||
candidate_hash,
|
||||
candidate["hash"] if candidate is not None else None,
|
||||
canonical_evidence(evidence),
|
||||
pair["id"] if pair else None,
|
||||
pair["image_id"] if pair else None,
|
||||
@@ -250,77 +204,6 @@ class ReviewDecisionOperations:
|
||||
self, conn, epoch, current, decision_id, total, **command
|
||||
) -> dict:
|
||||
"""Link the candidate's pair, compatible-only; raise on any conflict."""
|
||||
candidate = command["candidate"]
|
||||
positive, negative = candidate["positive"], candidate["negative"]
|
||||
provenance = {
|
||||
"kind": DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"reviewed_digest": command["sha256"],
|
||||
"candidate_hash": candidate["hash"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
}
|
||||
return self._link_compatible(
|
||||
conn, epoch, current, total, positive, negative, provenance, command
|
||||
)
|
||||
|
||||
def _apply_manual(self, conn, epoch, current, decision_id, total, **command):
|
||||
"""Link the chosen Library prompt, compatible-only; raise otherwise."""
|
||||
missing = _Rollback(
|
||||
ReviewCommandError("prompt-missing", epoch, item=current, total=total)
|
||||
)
|
||||
positive_id = command["manual"]["positive_id"]
|
||||
pair_id = command["manual"]["pair_id"]
|
||||
row = conn.execute(_MANUAL_POSITIVE, (positive_id,)).fetchone()
|
||||
if row is None:
|
||||
raise missing
|
||||
positive_text, negative_text = row[1], ""
|
||||
if pair_id is not None:
|
||||
chosen = conn.execute(_MANUAL_PAIR, (pair_id, positive_id)).fetchone()
|
||||
if chosen is None:
|
||||
raise missing
|
||||
negative_text = chosen[0]
|
||||
elif conn.execute(_ARCHIVED_BARE_PAIR, (positive_id,)).fetchone():
|
||||
raise missing
|
||||
provenance = {
|
||||
"kind": MANUAL_KIND,
|
||||
"decision_id": decision_id,
|
||||
"reviewed_digest": command["sha256"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"positive_id": positive_id,
|
||||
"pair_id": pair_id,
|
||||
}
|
||||
# The sides name no provenance: only the ingester reads them.
|
||||
pair = self._link_compatible(
|
||||
conn,
|
||||
epoch,
|
||||
current,
|
||||
total,
|
||||
{"text": positive_text},
|
||||
{"text": negative_text},
|
||||
provenance,
|
||||
command,
|
||||
)
|
||||
# A prompt whose stored hash is stale reaches another row: never link that.
|
||||
reached = conn.execute(
|
||||
"SELECT positive_prompt_id FROM prompt_pairs WHERE id=?", (pair["id"],)
|
||||
).fetchone()[0]
|
||||
if reached != positive_id or pair_id not in (None, pair["id"]):
|
||||
raise missing
|
||||
return pair
|
||||
|
||||
def _link_compatible(
|
||||
self,
|
||||
conn,
|
||||
epoch,
|
||||
current,
|
||||
total,
|
||||
positive,
|
||||
negative,
|
||||
provenance,
|
||||
command,
|
||||
) -> dict:
|
||||
# Imported here: pair_ingestion imports operations, which composes this.
|
||||
from .pair_ingestion import _ingest_resolved_pair_in
|
||||
|
||||
@@ -359,6 +242,17 @@ class ReviewDecisionOperations:
|
||||
# Enrich the one row of this file under the spelling it was
|
||||
# linked with, so the ingester's compatibility rules see it.
|
||||
image_path = rows[0][1]
|
||||
candidate = command["candidate"]
|
||||
positive, negative = candidate["positive"], candidate["negative"]
|
||||
provenance = {
|
||||
"kind": DECISION_KIND,
|
||||
"decision_id": decision_id,
|
||||
"reviewed_digest": command["sha256"],
|
||||
"candidate_hash": candidate["hash"],
|
||||
"resolver_version": current["resolver_version"],
|
||||
"positive": positive,
|
||||
"negative": negative,
|
||||
}
|
||||
result = _ingest_resolved_pair_in(
|
||||
conn,
|
||||
self,
|
||||
|
||||
@@ -46,7 +46,6 @@ def remove_quietly(*paths):
|
||||
if path and os.path.exists(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
# cleanup after a failure must not replace that failure
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -33,9 +33,6 @@ MAX_TAG_LENGTH = 50
|
||||
_IMAGE_SORTS = ("newest", "oldest", "name", "name-desc", "rating")
|
||||
_LIBRARY_SORTS = ("newest", "oldest", "name", "rating")
|
||||
_MEDIA = ("all", "image", "video")
|
||||
# How a Library view's Tags match (#444): every tag, any of them, or parents
|
||||
# without tags ("untagged" never goes with tags; the listing refuses both).
|
||||
_TAG_MODES = ("all", "any", "untagged")
|
||||
_CONTROL = re.compile(r"[\x00-\x1f\x7f]")
|
||||
_SOURCE = re.compile(r"all|0|[1-9][0-9]{0,8}")
|
||||
# A collection id as the listing's ``collection`` query takes it (#331): none
|
||||
@@ -70,9 +67,6 @@ _STATE_FIELDS = {
|
||||
"category": ("text", 100, ""),
|
||||
"sort": ("choice", _LIBRARY_SORTS, "newest"),
|
||||
"tags": ("tags", None, ()),
|
||||
# #444: the Tags match and the output folder (relative, "/"-joined).
|
||||
"tag_mode": ("choice", _TAG_MODES, "all"),
|
||||
"folder": ("text", 1024, ""),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -158,8 +152,6 @@ def encode_state(page: object, state: object) -> str:
|
||||
_validate_value(key, kind, limit, state[key])
|
||||
if state.get("favorites") is True and state.get("collection"):
|
||||
raise SavedViewInvalid("A view cannot filter by favorites and a collection")
|
||||
if state.get("tag_mode") == "untagged" and state.get("tags"):
|
||||
raise SavedViewInvalid("A view cannot list untagged prompts and filter tags")
|
||||
encoded = json.dumps(
|
||||
{key: state[key] for key in fields}, ensure_ascii=False, separators=(",", ":")
|
||||
)
|
||||
@@ -203,9 +195,6 @@ def normalize_state(page: str, raw: object) -> dict:
|
||||
# as the listing does for a URL with both (#331 D140).
|
||||
if state.get("favorites") is True and state.get("collection"):
|
||||
state["favorites"] = False
|
||||
# Never written by this build either: the tags win.
|
||||
if state.get("tag_mode") == "untagged" and state.get("tags"):
|
||||
state["tag_mode"] = "all"
|
||||
return state
|
||||
|
||||
|
||||
|
||||
@@ -20,8 +20,7 @@ counted, hashed into the token and renamed with the prompts and pairs.
|
||||
|
||||
import sqlite3
|
||||
|
||||
from .library import _ELIGIBLE, SHELVED_PAIR, UNTAGGED, _escape, _limit
|
||||
from .prompt_archive import SHELVED
|
||||
from .library import _ELIGIBLE, _limit
|
||||
from .library_commands import _TOKEN, LibraryCommandError, _validate_epoch
|
||||
from .library_tokens import canonical_token
|
||||
from .models import utc_now_iso
|
||||
@@ -39,75 +38,30 @@ _IMAGE_FILES = (
|
||||
)
|
||||
|
||||
|
||||
def _tag_rows(page: str, order: str = "page.id") -> str:
|
||||
def _tag_rows(page: str) -> str:
|
||||
"""Count rows for the registry rows selected by ``page`` (id, name)."""
|
||||
# Eligible parent components on the shelf carrying a tag in the page,
|
||||
# plus the positive parent of every active pair carrying it (always
|
||||
# eligible) when that parent is on the shelf: archived parents (#443)
|
||||
# are hidden with their pairs, so Browse lists what was counted.
|
||||
# Eligible parent components carrying a tag in the page, plus the
|
||||
# positive parent of every active pair carrying it (always eligible).
|
||||
return (
|
||||
"WITH page AS (" + page + "),"
|
||||
"owners(tag_id,pid) AS ("
|
||||
"SELECT pt.tag_id,p.id FROM prompts p "
|
||||
"JOIN prompt_tags pt ON pt.prompt_id=p.id "
|
||||
"WHERE pt.tag_id IN (SELECT id FROM page) AND " + _ELIGIBLE + " "
|
||||
"AND " + SHELVED + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id "
|
||||
"WHERE pp.archived=0 AND " + SHELVED_PAIR + " "
|
||||
"AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"WHERE pp.archived=0 AND ppt.tag_id IN (SELECT id FROM page)),"
|
||||
"images(tag_id,n) AS (" + _IMAGE_FILES + " "
|
||||
"WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
|
||||
"SELECT page.id,page.name,COUNT(DISTINCT owners.pid) AS prompt_count,"
|
||||
"COALESCE(MAX(images.n),0) AS image_count "
|
||||
"FROM page LEFT JOIN owners ON owners.tag_id=page.id "
|
||||
"LEFT JOIN images ON images.tag_id=page.id "
|
||||
"GROUP BY page.id ORDER BY " + order
|
||||
"GROUP BY page.id ORDER BY page.id"
|
||||
)
|
||||
|
||||
|
||||
# #444 tag sorts: sort -> (key expression on tags/counted, direction). Ties and
|
||||
# "created" follow the id, so every order is total and keyset pages are exact.
|
||||
TAG_SORTS = {
|
||||
"created": (None, "ASC"),
|
||||
"name": ("name COLLATE NOCASE", "ASC"),
|
||||
"name-desc": ("name COLLATE NOCASE", "DESC"),
|
||||
"prompts": ("prompt_count", "DESC"),
|
||||
"prompts-asc": ("prompt_count", "ASC"),
|
||||
}
|
||||
_MAX_SEARCH = 200
|
||||
_TAG_SEARCH = "name LIKE ? ESCAPE '\\'"
|
||||
# Every matching tag ranked by the prompt count its row shows (the owners rule
|
||||
# of ``_tag_rows``), then one keyset page of it with its image counts.
|
||||
_RANKED_TAG_PAGE = (
|
||||
"WITH owners(tag_id,pid) AS ("
|
||||
"SELECT pt.tag_id,p.id FROM prompts p "
|
||||
"JOIN prompt_tags pt ON pt.prompt_id=p.id WHERE " + _ELIGIBLE + " "
|
||||
"AND " + SHELVED + " "
|
||||
"UNION SELECT ppt.tag_id,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"JOIN prompt_pair_tags ppt ON ppt.pair_id=pp.id WHERE pp.archived=0 "
|
||||
"AND " + SHELVED_PAIR + "),"
|
||||
"counted AS (SELECT t.id,t.name,COUNT(DISTINCT o.pid) AS prompt_count "
|
||||
"FROM tags t LEFT JOIN owners o ON o.tag_id=t.id WHERE {search} "
|
||||
"GROUP BY t.id),"
|
||||
"page AS (SELECT id,name,prompt_count FROM counted WHERE {after} "
|
||||
"ORDER BY prompt_count {direction},id LIMIT ?),"
|
||||
"images(tag_id,n) AS (" + _IMAGE_FILES + " "
|
||||
"WHERE mit.tag_id IN (SELECT id FROM page) GROUP BY mit.tag_id) "
|
||||
"SELECT page.id,page.name,page.prompt_count,"
|
||||
"COALESCE(images.n,0) AS image_count FROM page "
|
||||
"LEFT JOIN images ON images.tag_id=page.id "
|
||||
"ORDER BY page.prompt_count {direction},page.id"
|
||||
)
|
||||
_UNTAGGED_COUNT = (
|
||||
"SELECT COUNT(*) FROM prompts p WHERE ("
|
||||
+ _ELIGIBLE
|
||||
+ ") AND "
|
||||
+ UNTAGGED
|
||||
# Untagged prompts opens the shelf, which hides archived parents (#443).
|
||||
+ " AND "
|
||||
+ SHELVED
|
||||
)
|
||||
_TAG_PAGE = _tag_rows("SELECT id,name FROM tags WHERE id>? ORDER BY id LIMIT ?")
|
||||
_TAG_ONE = _tag_rows("SELECT id,name FROM tags WHERE id=?")
|
||||
# Every owner kind a tag can sit on, for the delete confirmation.
|
||||
_TAG_IMPACT = (
|
||||
@@ -146,9 +100,7 @@ def _validate_tag_id(tag_id: object) -> None:
|
||||
|
||||
|
||||
# Category values in Tags & categories (#332 §7): the prompt-backed values
|
||||
# (nonblank values on eligible parents or on active pairs; ``prompt_count``
|
||||
# counts only parents on the shelf, so a value only archived parents (#443)
|
||||
# hold stays listed with 0, as a tag does; grouped exactly, so
|
||||
# (nonblank values on eligible parents or on active pairs, grouped exactly, so
|
||||
# whitespace and case variants stay distinct; /library/options lists the same
|
||||
# prompt-backed set with its own query) plus the values only image rows hold.
|
||||
# A prompt-backed value is ordered by its oldest owning positive; an image-only
|
||||
@@ -157,17 +109,15 @@ def _validate_tag_id(tag_id: object) -> None:
|
||||
# files missing on disk. No FULL OUTER JOIN, so it runs on SQLite 3.31+.
|
||||
_IMAGE_ONLY_FIRST_PID = 2**63 - 1
|
||||
_CATEGORY_ROWS = (
|
||||
"WITH owners(value,pid,shelved) AS ("
|
||||
"SELECT p.category,p.id," + SHELVED + " FROM prompts p "
|
||||
"WHERE p.category IS NOT NULL AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT pp.category,pp.positive_prompt_id," + SHELVED_PAIR + " "
|
||||
"FROM prompt_pairs pp "
|
||||
"WITH owners(value,pid) AS ("
|
||||
"SELECT p.category,p.id FROM prompts p WHERE p.category IS NOT NULL "
|
||||
"AND TRIM(p.category)!='' AND " + _ELIGIBLE + " "
|
||||
"UNION SELECT pp.category,pp.positive_prompt_id FROM prompt_pairs pp "
|
||||
"WHERE pp.archived=0 AND pp.category IS NOT NULL AND TRIM(pp.category)!=''),"
|
||||
"image_values(value,image_count) AS ("
|
||||
"SELECT category,COUNT(*) FROM image_categories GROUP BY category),"
|
||||
"grouped(value,first_pid,prompt_count) AS ("
|
||||
"SELECT value,MIN(pid),COUNT(DISTINCT CASE WHEN shelved THEN pid END) "
|
||||
"FROM owners GROUP BY value "
|
||||
"SELECT value,MIN(pid),COUNT(DISTINCT pid) FROM owners GROUP BY value "
|
||||
"UNION ALL SELECT value," + str(_IMAGE_ONLY_FIRST_PID) + ",0 FROM image_values "
|
||||
"WHERE value NOT IN (SELECT value FROM owners)),"
|
||||
"counted AS (SELECT g.value,g.first_pid,g.prompt_count,"
|
||||
@@ -260,28 +210,6 @@ def _tag_after(after: object) -> int:
|
||||
return after
|
||||
|
||||
|
||||
def _tag_search(search: object) -> str:
|
||||
if not isinstance(search, str) or len(search) > _MAX_SEARCH:
|
||||
raise ValueError(f"Tag search must be text of at most {_MAX_SEARCH} characters")
|
||||
return search.strip()
|
||||
|
||||
|
||||
def _sorted_tag_after(after: object, sort: str) -> tuple | None:
|
||||
"""The ``[key, id]`` cursor of a name or prompt-count sort."""
|
||||
if after is None:
|
||||
return None
|
||||
key_type = str if sort.startswith("name") else int
|
||||
if (
|
||||
not isinstance(after, list)
|
||||
or len(after) != 2
|
||||
or type(after[0]) is not key_type
|
||||
or type(after[1]) is not int
|
||||
or after[1] <= 0
|
||||
):
|
||||
raise ValueError("Invalid vocabulary cursor")
|
||||
return after[0], after[1]
|
||||
|
||||
|
||||
def _category_after(after: object) -> tuple[int, str] | None:
|
||||
if after is None:
|
||||
return None
|
||||
@@ -503,80 +431,22 @@ class VocabularyOperations:
|
||||
epoch: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
after: object = None,
|
||||
search: str = "",
|
||||
sort: str = "created",
|
||||
after: int | None = None,
|
||||
) -> dict:
|
||||
"""One page of tags matching ``search`` (a literal, case-insensitive
|
||||
substring) in ``sort`` order, with the untagged parents' count."""
|
||||
_limit(limit)
|
||||
search = _tag_search(search)
|
||||
if not isinstance(sort, str) or sort not in TAG_SORTS:
|
||||
raise ValueError("sort must be one of " + ", ".join(TAG_SORTS))
|
||||
where, params = "1=1", []
|
||||
if search:
|
||||
where, params = _TAG_SEARCH, [f"%{_escape(search)}%"]
|
||||
total, untagged = conn.execute(
|
||||
f"SELECT (SELECT COUNT(*) FROM tags WHERE {where}),({_UNTAGGED_COUNT})",
|
||||
params,
|
||||
).fetchone()
|
||||
rows = [
|
||||
dict(row) for row in self._tag_page(conn, where, params, sort, after, limit)
|
||||
]
|
||||
start = _tag_after(after)
|
||||
total = conn.execute("SELECT COUNT(*) FROM tags").fetchone()[0]
|
||||
rows = [dict(row) for row in conn.execute(_TAG_PAGE, (start, limit + 1))]
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
key, _direction = TAG_SORTS[sort]
|
||||
next_key = None
|
||||
if has_more:
|
||||
last = rows[-1]
|
||||
next_key = (
|
||||
last["id"]
|
||||
if key is None
|
||||
else [
|
||||
last["prompt_count" if key == "prompt_count" else "name"],
|
||||
last["id"],
|
||||
]
|
||||
)
|
||||
return {
|
||||
"library_epoch": epoch,
|
||||
"tags": rows,
|
||||
"total": total,
|
||||
"untagged_count": untagged,
|
||||
# Echoed so a client can tell this read applied its search and sort.
|
||||
"search": search,
|
||||
"sort": sort,
|
||||
"has_more": has_more,
|
||||
"next_key": next_key,
|
||||
"next_key": rows[-1]["id"] if has_more else None,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _tag_page(conn, where, params, sort, after, limit):
|
||||
key, direction = TAG_SORTS[sort]
|
||||
if key is None:
|
||||
page = (
|
||||
f"SELECT id,name FROM tags WHERE {where} AND id>? ORDER BY id LIMIT ?"
|
||||
)
|
||||
return conn.execute(
|
||||
_tag_rows(page), [*params, _tag_after(after), limit + 1]
|
||||
)
|
||||
cursor = _sorted_tag_after(after, sort)
|
||||
operator = ">" if direction == "ASC" else "<"
|
||||
keyset, keys = "1=1", []
|
||||
if cursor is not None:
|
||||
keyset = f"({key}{operator}? OR ({key}=? AND id>?))"
|
||||
keys = [cursor[0], cursor[0], cursor[1]]
|
||||
if key == "prompt_count":
|
||||
sql = _RANKED_TAG_PAGE.format(
|
||||
search=where, after=keyset, direction=direction
|
||||
)
|
||||
return conn.execute(sql, [*params, *keys, limit + 1])
|
||||
page = (
|
||||
f"SELECT id,name FROM tags WHERE {where} AND {keyset} "
|
||||
f"ORDER BY {key} {direction},id LIMIT ?"
|
||||
)
|
||||
order = f"page.{key} {direction},page.id"
|
||||
return conn.execute(_tag_rows(page, order), [*params, *keys, limit + 1])
|
||||
|
||||
def _vocabulary_categories(
|
||||
self,
|
||||
conn: sqlite3.Connection,
|
||||
@@ -620,18 +490,9 @@ class VocabularyOperations:
|
||||
),
|
||||
}
|
||||
|
||||
def list_vocabulary_tags(
|
||||
self,
|
||||
*,
|
||||
limit: int = 100,
|
||||
after: object = None,
|
||||
search: str = "",
|
||||
sort: str = "created",
|
||||
):
|
||||
def list_vocabulary_tags(self, *, limit: int = 100, after: int | None = None):
|
||||
with self.library_snapshot() as (conn, epoch, _ceiling):
|
||||
return self._vocabulary_tags(
|
||||
conn, epoch, limit=limit, after=after, search=search, sort=sort
|
||||
)
|
||||
return self._vocabulary_tags(conn, epoch, limit=limit, after=after)
|
||||
|
||||
def list_vocabulary_categories(
|
||||
self, *, limit: int = 100, after: list | None = None
|
||||
|
||||
@@ -12,8 +12,7 @@ code. This plan proposes implementation order and acceptance gates.
|
||||
- The native Prompt Library merged in [#210](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/210) for [#208](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/208): one positive parent, saved negative variants, full selected text and a shared filtered image gallery. It does not invent chronological runs from counters or scan dates.
|
||||
- [Durable history storage](prompt-history-storage.md) records job attempts and outcomes from [#212](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/212), frozen prompt contexts from [#213](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/213), and observed run-owned output locators from [#214](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/214). Those locators do not verify file bytes or establish gallery-image associations. That association contract and the chronological execution timeline remain pending under [#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209).
|
||||
- [#215](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/215) brings the shell, Images and natural-height Prompt Library presentation into the approved geometry and themes while retaining their existing operations. [#216](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/216) adds the shared current-file inspector with exact generation facts, scoped provenance, raw/download evidence and loaded-collection viewer controls, including a slideshow that pauses at native video. This delivery does not establish whole-application parity, change the default entry point or constitute a release. Recorded chronological run/context browsing continues under [#209](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/209); image tagging/review, System consolidation and remaining organization capabilities continue under #196.
|
||||
- Local Inspect (#249, PRs L1–L3) replaces the `#/inspect` handoff with the concept's native "Inspect a file" page: one local image or video previews inline and opens the shared viewer and full inspector on a record read entirely in the browser, with nothing uploaded, imported or stored. The legacy metadata page (`web/metadata.html`, served at `/prompt_manager/gallery.html`) stays reachable from Overview; its capabilities map to the shared inspector, including its one-click COPY ALL summary, which is the shared inspector's "Copy settings summary" (D117, LI-4). PR L3 ([#286](https://github.com/ComfyAssets/ComfyUI_PromptManager/pull/286)) shipped that button and local roles equal to the server's resolver (`web/js/workspace/local-resolver.mjs`, TESTING.md "Local roles (#249 PR L3, decision LI-1)"). Recorded deviations D60–D66 and D117 are in [TESTING.md](../TESTING.md).
|
||||
- Collections ([#331](https://github.com/ComfyAssets/ComfyUI_PromptManager/issues/331), three PRs) adds named, hand-picked groups of images, stored in the library database by file location as favorites are: a COLLECTIONS sidebar group whose rows open Images filtered to one collection (`#/collection/<id>`) with rename and delete, Add to collection and Remove from collection in the Images selection bar, a Collections… checklist in the shared inspector for the side panel and the viewer, and saved views that keep a collection. Membership survives restarts, source reordering and System › Maintenance duplicate cleanup; no file is copied, moved or changed.
|
||||
- Local Inspect (#249, PRs L1–L2) replaces the `#/inspect` handoff with the concept's native "Inspect a file" page: one local image or video previews inline and opens the shared viewer and full inspector on a record read entirely in the browser, with nothing uploaded, imported or stored. The legacy metadata page (`web/metadata.html`, served at `/prompt_manager/gallery.html`) stays reachable from Overview; its capabilities map to the shared inspector, except its one-click COPY ALL summary, whose ruled replacement ("Copy settings summary") is pending. Recorded deviations D60–D66 are in [TESTING.md](../TESTING.md). Local roles equal to the server's resolver follow in PR L3.
|
||||
|
||||
## Scope and compatibility
|
||||
|
||||
|
||||
@@ -18,13 +18,9 @@ protocol and peer checks reject unrelated messages. Same-origin scripts are
|
||||
trusted; this transport is not authentication against other scripts on that origin.
|
||||
Unsupported messaging is reported without breaking existing ComfyUI nodes.
|
||||
|
||||
Apply works in one click: without a preview, the sender requests the ticket itself
|
||||
and applies it. Preview remains optional, and shows the exact change first.
|
||||
|
||||
An apply timeout or abort after posting means the outcome is unknown. It cannot
|
||||
prove the host cancelled the write. The sender must inspect a fresh destination
|
||||
preview or choose the destination again before sending again; automatic retry must
|
||||
not overwrite edits. An error the host returned is a refusal: nothing was written.
|
||||
preview before deciding to send again; automatic retry must not overwrite edits.
|
||||
|
||||
## PromptManager adapter
|
||||
|
||||
@@ -35,9 +31,7 @@ field. Unknown/ambiguous metadata must not become an empty string.
|
||||
|
||||
Previews show original and requested text plus positive prepend/append composition.
|
||||
Connected composition is marked unknown rather than guessing upstream values.
|
||||
Connected destination fields reject writes. The sender compares the destination's revision
|
||||
(node object and graph version) from discovery with the one at the preview, and refuses if the workflow changed.
|
||||
Applying rechecks graph, node, widgets,
|
||||
Connected destination fields reject writes. Applying rechecks graph, node, widgets,
|
||||
values and connections. Ordinary serialization cache synchronization is accepted
|
||||
only when it preserves live values. Both requested roles are checked before writing.
|
||||
|
||||
|
||||
@@ -275,12 +275,7 @@ class PromptManager(PromptManagerBase, ComfyNodeABC):
|
||||
record_runtime_context(
|
||||
type(self).__name__, text, prepend_text, append_text, encoding_text
|
||||
)
|
||||
self._record_cached_item(
|
||||
unique_id,
|
||||
(encoding_text,),
|
||||
storage_text,
|
||||
fields=((text, prepend_text, append_text),),
|
||||
)
|
||||
self._record_cached_item(unique_id, (encoding_text,), storage_text)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -243,24 +243,14 @@ class PromptManagerBase:
|
||||
return False
|
||||
|
||||
def _record_cached_item(
|
||||
self,
|
||||
unique_id: Any,
|
||||
outputs: tuple,
|
||||
text: Any,
|
||||
negative_text: Any = None,
|
||||
fields: Optional[tuple] = None,
|
||||
self, unique_id: Any, outputs: tuple, text: Any, negative_text: Any = None
|
||||
) -> None:
|
||||
"""Remember this execution's outputs and stored text for cached jobs (#347).
|
||||
|
||||
``fields`` are the ``(raw, prepend, append)`` texts this execution gave
|
||||
``record_runtime_context`` for each output, so a cached job's history
|
||||
can record them too (#403).
|
||||
"""
|
||||
"""Remember this execution's outputs and stored text for cached jobs (#347)."""
|
||||
if unique_id is None:
|
||||
return
|
||||
try:
|
||||
self._cached_runs().record_item(
|
||||
unique_id, outputs, stored_facts(text, negative_text), fields
|
||||
unique_id, outputs, stored_facts(text, negative_text)
|
||||
)
|
||||
except Exception as error:
|
||||
self.logger.debug(f"Could not record the cached-run receipt: {error}")
|
||||
|
||||
@@ -198,11 +198,7 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
type(self).__name__, negative_text, "", "", negative_text, role="negative"
|
||||
)
|
||||
self._record_cached_item(
|
||||
unique_id,
|
||||
(positive, negative_text),
|
||||
text,
|
||||
negative_text,
|
||||
fields=((text, prepend_text, append_text), (negative_text, "", "")),
|
||||
unique_id, (positive, negative_text), text, negative_text
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -230,8 +226,6 @@ class PromptManagerPair(PromptManagerFocused):
|
||||
negative_text,
|
||||
category=category.strip() or None,
|
||||
tags=self._parse_tags(tags),
|
||||
# The Library shows the prompt's own category and tags, not the pair's.
|
||||
fill_new_positive=True,
|
||||
)
|
||||
pair = self.db.get_prompt_pair(pair_id)
|
||||
prompt_id = pair["positive_prompt_id"]
|
||||
|
||||
@@ -246,12 +246,7 @@ class PromptManagerText(PromptManagerBase, ComfyNodeABC):
|
||||
record_runtime_context(
|
||||
type(self).__name__, text, prepend_text, append_text, final_text
|
||||
)
|
||||
self._record_cached_item(
|
||||
unique_id,
|
||||
(final_text,),
|
||||
storage_text,
|
||||
fields=((text, prepend_text, append_text),),
|
||||
)
|
||||
self._record_cached_item(unique_id, (final_text,), storage_text)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -11,7 +11,6 @@ Split into domain-specific mixins for maintainability:
|
||||
- JobRoutesMixin: the session job registry behind Activity (py/api/jobs.py)
|
||||
- ImageTagJobsMixin: bulk Add tags / Auto-tag as image-tags jobs
|
||||
- ThumbnailJobsMixin: thumbnail plan, thumbnail job and the gallery stream on it
|
||||
- ThumbnailLocationMixin: the thumbnail location check and payloads (#320)
|
||||
- ThumbnailConvertMixin: Convert outdated thumbnails (#317), a thumbnail job mode
|
||||
- AutoThumbnailMixin: previews made on first view, on capture and after a scan
|
||||
- ReviewRoutesMixin: the Needs review queue, item detail and decisions
|
||||
@@ -20,7 +19,6 @@ Split into domain-specific mixins for maintainability:
|
||||
- SystemRoutesMixin: System health check, Overview and usage statistics
|
||||
- LibraryLocationRoutesMixin: where the library lives, Move library, Keep it here
|
||||
- MaintenanceRoutesMixin: reviewed maintenance plans (preview, apply)
|
||||
- LibraryDeleteRoutesMixin: the Library's Delete… (archive paired) and Restore
|
||||
- WorkflowStorageRoutesMixin: store each workflow once, then Reclaim space
|
||||
- DuplicateFilesMixin: duplicate image files as find and removal jobs
|
||||
- LoraImportRoutesMixin: the reviewed, staged LoRA reference import job
|
||||
@@ -42,10 +40,8 @@ from aiohttp import web
|
||||
from PIL import Image
|
||||
|
||||
from .prompts import PromptRoutesMixin
|
||||
from .revalidation import revalidated_response
|
||||
from .pairs import PairRoutesMixin
|
||||
from .library import LibraryRoutesMixin
|
||||
from .library_delete import LibraryDeleteRoutesMixin
|
||||
from .vocabulary import VocabularyRoutesMixin
|
||||
from .annotations import AnnotationRoutesMixin
|
||||
from .review import ReviewRoutesMixin
|
||||
@@ -61,7 +57,6 @@ from .settings_routes import SettingsRoutesMixin
|
||||
from .job_routes import JobRoutesMixin
|
||||
from .image_tag_jobs import ImageTagJobsMixin
|
||||
from .thumbnail_jobs import ThumbnailJobsMixin
|
||||
from .thumbnail_location import ThumbnailLocationMixin
|
||||
from .thumbnail_convert import ThumbnailConvertMixin
|
||||
from .thumbnail_auto import AutoThumbnailMixin
|
||||
from .system import SystemRoutesMixin
|
||||
@@ -79,16 +74,12 @@ from .autotag_routes import AutotagRoutesMixin
|
||||
from .lora_integration import LoraIntegrationMixin
|
||||
from .lora_import_routes import LoraImportRoutesMixin
|
||||
from .image_categories import ImageCategoryRoutesMixin
|
||||
from .image_delete import ImageDeleteMixin
|
||||
from .image_return import ImageReturnMixin
|
||||
from .source_rescope import SourceRescopeMixin
|
||||
|
||||
try:
|
||||
from ...database.operations import PromptDatabase
|
||||
from ...utils.file_prompt import parse_comfyui_prompt
|
||||
from ...utils.logging_config import get_logger
|
||||
from ...utils.native_memory import release_free_memory
|
||||
from ...utils.png_text import png_text
|
||||
from ...utils.video_metadata import is_video_path, read_video_metadata
|
||||
except ImportError:
|
||||
import sys
|
||||
@@ -99,8 +90,6 @@ except ImportError:
|
||||
from database.operations import PromptDatabase
|
||||
from utils.file_prompt import parse_comfyui_prompt
|
||||
from utils.logging_config import get_logger
|
||||
from utils.native_memory import release_free_memory
|
||||
from utils.png_text import png_text
|
||||
from utils.video_metadata import is_video_path, read_video_metadata
|
||||
|
||||
|
||||
@@ -324,7 +313,6 @@ async def _gzip_middleware(request, handler):
|
||||
class PromptManagerAPI(
|
||||
PairRoutesMixin,
|
||||
LibraryRoutesMixin,
|
||||
LibraryDeleteRoutesMixin,
|
||||
VocabularyRoutesMixin,
|
||||
AnnotationRoutesMixin,
|
||||
ReviewRoutesMixin,
|
||||
@@ -340,7 +328,6 @@ class PromptManagerAPI(
|
||||
JobRoutesMixin,
|
||||
ImageTagJobsMixin,
|
||||
ThumbnailJobsMixin,
|
||||
ThumbnailLocationMixin,
|
||||
ThumbnailConvertMixin,
|
||||
AutoThumbnailMixin,
|
||||
SystemRoutesMixin,
|
||||
@@ -356,8 +343,6 @@ class PromptManagerAPI(
|
||||
LoraIntegrationMixin,
|
||||
LoraImportRoutesMixin,
|
||||
ImageCategoryRoutesMixin,
|
||||
ImageDeleteMixin,
|
||||
ImageReturnMixin,
|
||||
SourceRescopeMixin,
|
||||
):
|
||||
"""REST API handler for PromptManager operations and web interface.
|
||||
@@ -438,12 +423,6 @@ class PromptManagerAPI(
|
||||
# Reviewed maintenance plans under single-use tokens; a restart
|
||||
# forgets them (py/api/maintenance.py).
|
||||
self._maintenance_plans = PlanStore()
|
||||
# #480: the removal marks the workspace listing hides, read once now so
|
||||
# a listing page reads no table until a mark or the database changes.
|
||||
try:
|
||||
self._removal_marks()
|
||||
except Exception:
|
||||
self.logger.warning("Removal marks could not be read", exc_info=True)
|
||||
|
||||
# A safety backup interrupted by a crash leaves its temp copy behind.
|
||||
try:
|
||||
@@ -533,14 +512,6 @@ class PromptManagerAPI(
|
||||
return await loop.run_in_executor(None, call)
|
||||
return await loop.run_in_executor(None, func, *args)
|
||||
|
||||
async def _release_job_memory(self):
|
||||
"""After a scan or thumbnail job: glibc's free heap pages go back to
|
||||
the OS once (``malloc_trim(0)``; nothing elsewhere, #523)."""
|
||||
try:
|
||||
await self._run_in_executor(release_free_memory)
|
||||
except Exception: # best effort: the job's result already stands
|
||||
self.logger.debug("Releasing free memory failed", exc_info=True)
|
||||
|
||||
def _root_listing(self, root) -> RootListing:
|
||||
"""The shared listing record of one output root (created empty)."""
|
||||
return self._root_listings.setdefault(str(root), RootListing())
|
||||
@@ -682,45 +653,40 @@ class PromptManagerAPI(
|
||||
|
||||
@routes.get("/prompt_manager/workspace")
|
||||
async def serve_workspace_ui(request: web.Request) -> web.Response:
|
||||
return self._serve_html_page(request, "workspace.html", "Workspace")
|
||||
return self._serve_html_page("workspace.html", "Workspace")
|
||||
|
||||
@routes.get("/prompt_manager/web")
|
||||
async def serve_web_ui(request):
|
||||
return self._serve_html_page(request, "index.html", "Web UI", cache=False)
|
||||
return self._serve_html_page("index.html", "Web UI", cache=False)
|
||||
|
||||
@routes.get("/prompt_manager/gallery.html")
|
||||
async def serve_gallery_ui(request):
|
||||
return self._serve_html_page(request, "metadata.html", "Gallery")
|
||||
return self._serve_html_page("metadata.html", "Gallery")
|
||||
|
||||
@routes.get("/prompt_manager/admin")
|
||||
async def serve_admin_ui(request):
|
||||
return self._serve_html_page(request, "admin.html", "Admin UI")
|
||||
return self._serve_html_page("admin.html", "Admin UI")
|
||||
|
||||
@routes.get("/prompt_manager/gallery")
|
||||
async def serve_gallery_admin_ui(request):
|
||||
return self._serve_html_page(request, "gallery.html", "Gallery")
|
||||
return self._serve_html_page("gallery.html", "Gallery")
|
||||
|
||||
# ── Static file serving ───────────────────────────────────────
|
||||
|
||||
@routes.get("/prompt_manager/lib/{filepath:.*}")
|
||||
async def serve_lib_static(request):
|
||||
"""Serve static library files (JS, CSS) from web/lib directory."""
|
||||
return self._serve_static_file(
|
||||
request, "lib", request.match_info.get("filepath")
|
||||
)
|
||||
return self._serve_static_file("lib", request.match_info.get("filepath"))
|
||||
|
||||
@routes.get("/prompt_manager/js/{filepath:.*}")
|
||||
async def serve_js_static(request):
|
||||
"""Serve static JavaScript files from web/js directory."""
|
||||
return self._serve_static_file(
|
||||
request, "js", request.match_info.get("filepath")
|
||||
)
|
||||
return self._serve_static_file("js", request.match_info.get("filepath"))
|
||||
|
||||
# ── Register domain-specific routes from mixins ───────────────
|
||||
|
||||
self._register_pair_routes(routes)
|
||||
self._register_library_routes(routes)
|
||||
self._register_library_delete_routes(routes)
|
||||
self._register_vocabulary_routes(routes)
|
||||
self._register_annotation_routes(routes)
|
||||
self._register_review_routes(routes)
|
||||
@@ -744,8 +710,6 @@ class PromptManagerAPI(
|
||||
self._register_lora_routes(routes)
|
||||
self._register_lora_import_routes(routes)
|
||||
self._register_image_category_routes(routes)
|
||||
self._register_image_delete_routes(routes)
|
||||
self._register_image_return_routes(routes)
|
||||
self._register_rescope_routes(routes)
|
||||
|
||||
# Register gzip compression middleware (once)
|
||||
@@ -762,11 +726,8 @@ class PromptManagerAPI(
|
||||
|
||||
self.logger.info("All routes registered with decorator pattern")
|
||||
|
||||
def _serve_static_file(self, request, subdir, filepath):
|
||||
"""Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing.
|
||||
|
||||
A served file is revalidated on every reuse (#478, ``revalidation``).
|
||||
"""
|
||||
def _serve_static_file(self, subdir, filepath):
|
||||
"""Serve ``web/<subdir>/<filepath>``; 403 outside it, 404 when missing."""
|
||||
root = os.path.join(_get_project_root(), "web", subdir)
|
||||
exact = STATIC_EXACT_FILES.get((subdir, filepath))
|
||||
# An exact file passes the same containment check, its extension only.
|
||||
@@ -781,13 +742,10 @@ class PromptManagerAPI(
|
||||
return web.Response(text="Not Found", status=404)
|
||||
with open(file_path, "rb") as f:
|
||||
content = f.read()
|
||||
return revalidated_response(request, content, content_type)
|
||||
return web.Response(body=content, content_type=content_type)
|
||||
|
||||
def _serve_html_page(self, request, filename, title, cache=True):
|
||||
"""Serve ``web/<filename>``; 404 when missing, 500 when unreadable.
|
||||
|
||||
A served page is revalidated on every reuse (#478, ``revalidation``).
|
||||
"""
|
||||
def _serve_html_page(self, filename, title, cache=True):
|
||||
"""Serve ``web/<filename>``; 404 when missing, 500 when unreadable."""
|
||||
html_path = os.path.join(_get_project_root(), "web", filename)
|
||||
try:
|
||||
if cache and html_path in self._html_cache:
|
||||
@@ -804,9 +762,7 @@ class PromptManagerAPI(
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
return revalidated_response(
|
||||
request, content.encode("utf-8"), "text/html", charset="utf-8"
|
||||
)
|
||||
return web.Response(text=content, content_type="text/html", charset="utf-8")
|
||||
except Exception:
|
||||
self.logger.exception(f"Failed to load {title}")
|
||||
return web.Response(
|
||||
@@ -855,11 +811,20 @@ class PromptManagerAPI(
|
||||
f"{url_quote(rel_path.as_posix(), safe='/')}"
|
||||
)
|
||||
|
||||
# Any preview the file owns, the current format first
|
||||
# (#317); no store I/O here in folder mode (#320).
|
||||
from .images import listing_thumbnail_url
|
||||
# Any preview the file owns, the current format first (#317)
|
||||
from . import thumbnail_settings
|
||||
from .prompts import thumbnail_url_for
|
||||
|
||||
thumbnail_url = listing_thumbnail_url(output_path, rel_path)
|
||||
existing = thumbnail_settings.existing_thumbnail(
|
||||
output_path, rel_path
|
||||
)
|
||||
thumbnail_url = None
|
||||
if existing is not None:
|
||||
thumb_rel = (
|
||||
thumbnail_settings.thumbnail_stem_rel(rel_path)
|
||||
+ existing.suffix
|
||||
)
|
||||
thumbnail_url = thumbnail_url_for(thumb_rel, existing)
|
||||
if thumbnail_url:
|
||||
image["thumbnail_url"] = thumbnail_url
|
||||
break # Found matching root, stop searching
|
||||
@@ -1054,8 +1019,6 @@ class PromptManagerAPI(
|
||||
def _extract_comfyui_metadata(self, image_path):
|
||||
"""Embedded ComfyUI metadata of an image (PNG text chunks) or a video.
|
||||
|
||||
PNG text is read without decoding the pixels (``utils.png_text``, #523).
|
||||
|
||||
Videos go through ffprobe (see utils/video_metadata). Unreadable files
|
||||
are a warning, not an error: a scan over a large output tree meets
|
||||
truncated and foreign files, and it carries on past them.
|
||||
@@ -1064,7 +1027,7 @@ class PromptManagerAPI(
|
||||
return read_video_metadata(image_path)
|
||||
try:
|
||||
with Image.open(image_path) as img:
|
||||
return png_text(img)
|
||||
return dict(getattr(img, "text", None) or {})
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f"Unreadable image {os.path.basename(str(image_path))}: {e}"
|
||||
|
||||
@@ -36,21 +36,10 @@ except ImportError:
|
||||
|
||||
try:
|
||||
from ...utils import exit_signal
|
||||
from ...utils.media_folders import (
|
||||
in_trash_below_roots,
|
||||
in_trash_folder,
|
||||
is_skipped_folder,
|
||||
)
|
||||
except ImportError:
|
||||
from utils import exit_signal
|
||||
from utils.media_folders import (
|
||||
in_trash_below_roots,
|
||||
in_trash_folder,
|
||||
is_skipped_folder,
|
||||
)
|
||||
|
||||
try:
|
||||
from ...database.image_removal import mark_hides
|
||||
from ...database.pair_ingestion import SCAN_MARKS, ingest_resolved_pair
|
||||
from ...database.run_attribution import (
|
||||
RecordedOutput,
|
||||
@@ -71,7 +60,6 @@ try:
|
||||
decide_file_prompt,
|
||||
)
|
||||
except ImportError:
|
||||
from database.image_removal import mark_hides
|
||||
from database.pair_ingestion import SCAN_MARKS, ingest_resolved_pair
|
||||
from database.run_attribution import (
|
||||
RecordedOutput,
|
||||
@@ -130,11 +118,10 @@ DUPLICATE_FIND_LEGACY_TITLE = "Find duplicate image files (gallery page)"
|
||||
def _collect_media_files(output_dirs, extensions):
|
||||
"""Media files under ``output_dirs`` (blocking), bounded and symlink-free.
|
||||
|
||||
Walks each root without following symlinks, skips ``thumbnails`` and
|
||||
trash folders (``utils.media_folders``) and symlinked files, never
|
||||
descends more than MAX_SCAN_DEPTH levels and stops after MAX_SCAN_FILES
|
||||
files in total. Extensions match case-insensitively and each path is
|
||||
listed once.
|
||||
Walks each root without following symlinks, skips ``thumbnails``
|
||||
directories and symlinked files, never descends more than MAX_SCAN_DEPTH
|
||||
levels and stops after MAX_SCAN_FILES files in total. Extensions match
|
||||
case-insensitively and each path is listed once.
|
||||
"""
|
||||
wanted = {ext.lower() for ext in extensions}
|
||||
found = []
|
||||
@@ -149,7 +136,7 @@ def _collect_media_files(output_dirs, extensions):
|
||||
dirnames[:] = sorted(
|
||||
d
|
||||
for d in dirnames
|
||||
if not is_skipped_folder(d)
|
||||
if d != "thumbnails"
|
||||
and depth < MAX_SCAN_DEPTH
|
||||
and not os.path.islink(os.path.join(current, d))
|
||||
)
|
||||
@@ -176,7 +163,7 @@ def _physical_file_key(path, stat):
|
||||
volumes on Windows) falls back to the resolved, case-folded path.
|
||||
"""
|
||||
if stat.st_ino:
|
||||
return ("inode", (stat.st_dev, stat.st_ino))
|
||||
return ("inode", stat.st_dev, stat.st_ino)
|
||||
return ("path", os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
@@ -249,12 +236,7 @@ SCAN_FINAL_STATES = {
|
||||
"error": "failed",
|
||||
}
|
||||
# The scope as the scan log's first line names it (the review dialog's words).
|
||||
SCAN_SCOPE_LOG = {
|
||||
"new": "new and unresolved files",
|
||||
"all": "all files",
|
||||
"returned": "images returned to the library",
|
||||
}
|
||||
RETURN_JOB_TITLE = "Return images to the library" # #498
|
||||
SCAN_SCOPE_LOG = {"new": "new and unresolved files", "all": "all files"}
|
||||
MIN_BATCHES_FOR_ESTIMATE = 2 # "about X min left" waits for this many batches
|
||||
# Log line prefix per outcome group (spec: + added, = known, ? review, ! error,
|
||||
# - no usable metadata).
|
||||
@@ -539,7 +521,6 @@ def _diagnose_output_dirs(configured_dirs, public_path=None):
|
||||
|
||||
candidates.append(folder_paths.get_output_directory())
|
||||
except ImportError:
|
||||
# outside ComfyUI there is no output folder to add
|
||||
pass
|
||||
|
||||
output_dirs = []
|
||||
@@ -902,8 +883,6 @@ class AdminRoutesMixin:
|
||||
extensions = IMAGE_EXTENSIONS + VIDEO_EXTENSIONS
|
||||
media_files = []
|
||||
seen = set()
|
||||
# #480: a file removed from the library is never a copy or a keeper.
|
||||
marks = self._removal_marks()
|
||||
for slot, root in enumerate(roots):
|
||||
for path in _collect_media_files([root], extensions):
|
||||
try:
|
||||
@@ -912,9 +891,6 @@ class AdminRoutesMixin:
|
||||
stat = os.stat(path)
|
||||
except OSError:
|
||||
continue
|
||||
locator = os.path.normcase(os.path.realpath(str(path)))
|
||||
if locator in marks and mark_hides(marks, locator, path):
|
||||
continue
|
||||
key = _physical_file_key(path, stat)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
@@ -1098,21 +1074,13 @@ class AdminRoutesMixin:
|
||||
return result
|
||||
|
||||
def _delete_one_duplicate(self, image_path, output_path):
|
||||
"""Delete one file and its thumbnail; return a failure reason or None.
|
||||
|
||||
What is removed is exactly the folder's real path plus the name: a
|
||||
link is removed itself, never its target, and the preview removed is
|
||||
that name's. Both the link's place and what it leads to are checked.
|
||||
"""
|
||||
"""Delete one file and its thumbnail; return a failure reason or None."""
|
||||
file_path = Path(image_path)
|
||||
if not file_path.is_absolute():
|
||||
file_path = output_path / file_path
|
||||
|
||||
try:
|
||||
real_root = output_path.resolve()
|
||||
rel_path = file_path.resolve().relative_to(real_root)
|
||||
exact = Path(os.path.realpath(file_path.parent)) / file_path.name
|
||||
exact_rel = exact.relative_to(real_root)
|
||||
rel_path = file_path.resolve().relative_to(output_path.resolve())
|
||||
except ValueError:
|
||||
self.logger.warning(
|
||||
f"Attempted to delete file outside output directory: {image_path}"
|
||||
@@ -1121,19 +1089,15 @@ class AdminRoutesMixin:
|
||||
|
||||
if file_path.suffix.lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||||
return "not a media file"
|
||||
if "thumbnails" in rel_path.parts or "thumbnails" in exact_rel.parts:
|
||||
if "thumbnails" in rel_path.parts:
|
||||
return "thumbnails are managed by the gallery"
|
||||
if in_trash_below_roots(file_path, [output_path]) or in_trash_folder(
|
||||
str(exact_rel)
|
||||
):
|
||||
return "it is in a trash folder" # never unlinked from a trash (#543)
|
||||
|
||||
if not file_path.is_file():
|
||||
return "file not found"
|
||||
|
||||
os.remove(exact)
|
||||
os.remove(file_path)
|
||||
self.logger.info(f"Deleted duplicate image: {image_path}")
|
||||
_remove_thumbnail(real_root, exact_rel, self.logger)
|
||||
_remove_thumbnail(output_path.resolve(), rel_path, self.logger)
|
||||
return None
|
||||
|
||||
async def get_statistics(self, request):
|
||||
@@ -1407,17 +1371,13 @@ class AdminRoutesMixin:
|
||||
}
|
||||
)
|
||||
|
||||
def _start_scan_job(
|
||||
self, files=DEFAULT_SCAN_SCOPE, roots=None, output_dirs=None, returned=None
|
||||
):
|
||||
def _start_scan_job(self, files=DEFAULT_SCAN_SCOPE, roots=None, output_dirs=None):
|
||||
"""Register the scan job synchronously; its runner starts on the loop.
|
||||
|
||||
``files`` is the scope (``SCAN_SCOPES``). ``roots`` selects sources by
|
||||
their index in ``output_dirs`` (the effective sources the caller
|
||||
resolved); None scans every source, resolved when the job starts.
|
||||
Raises JobBusy while another record-changing job runs. ``returned``
|
||||
(#498, ``[(path, locator)]``) makes it "Return images to the library":
|
||||
those files' removal marks go first, then only they are scanned.
|
||||
Raises JobBusy while another record-changing job runs.
|
||||
"""
|
||||
if files not in SCAN_SCOPES:
|
||||
raise ValueError(f"Unknown scan scope: {files!r}")
|
||||
@@ -1425,26 +1385,20 @@ class AdminRoutesMixin:
|
||||
output_dirs is None or not roots or max(roots) >= len(output_dirs)
|
||||
):
|
||||
raise ValueError("A scoped scan needs the sources its roots index")
|
||||
scope = {"files": files, "roots": None if roots is None else list(roots)}
|
||||
if returned is not None:
|
||||
scope = {"files": "returned", "roots": None, "images": len(returned)}
|
||||
job = self._jobs.start(
|
||||
SCAN_JOB_KIND,
|
||||
SCAN_JOB_TITLE if returned is None else RETURN_JOB_TITLE,
|
||||
SCAN_JOB_TITLE,
|
||||
record_changing=True,
|
||||
scope=scope,
|
||||
runner=functools.partial(
|
||||
self._run_scan_job, output_dirs=output_dirs, returned=returned
|
||||
),
|
||||
scope={"files": files, "roots": None if roots is None else list(roots)},
|
||||
runner=functools.partial(self._run_scan_job, output_dirs=output_dirs),
|
||||
groups=SCAN_GROUP_NAMES,
|
||||
)
|
||||
self._scan_job = job
|
||||
return job
|
||||
|
||||
async def _run_scan_job(self, job, output_dirs=None, returned=None):
|
||||
async def _run_scan_job(self, job, output_dirs=None):
|
||||
final = None
|
||||
extra = {} if returned is None else {"returned": returned}
|
||||
async for payload in self._scan_images_events(job, output_dirs, **extra):
|
||||
async for payload in self._scan_images_events(job, output_dirs):
|
||||
job.publish(payload)
|
||||
final = payload.get("type")
|
||||
state = SCAN_FINAL_STATES.get(final, "finished")
|
||||
@@ -1454,7 +1408,6 @@ class AdminRoutesMixin:
|
||||
verb = "cancelled" if state == "cancelled" else "finished"
|
||||
job.log_line(f"Scan {verb} · {_plural(job.processed, 'file')} processed")
|
||||
job.finish(state)
|
||||
await self._release_job_memory()
|
||||
if state == "finished":
|
||||
# After the scan, so the guarded thumbnail job is admitted (#249).
|
||||
await self._hand_off_scan_thumbnails(job, output_dirs)
|
||||
@@ -1483,7 +1436,7 @@ class AdminRoutesMixin:
|
||||
)
|
||||
return response
|
||||
|
||||
async def _scan_images_events(self, job, output_dirs=None, returned=None):
|
||||
async def _scan_images_events(self, job, output_dirs=None):
|
||||
"""Yield progress payloads while scanning the output directories.
|
||||
|
||||
Every effective source is walked, or only those ``job.scope["roots"]``
|
||||
@@ -1521,16 +1474,9 @@ class AdminRoutesMixin:
|
||||
0, f"Scanning {len(walked)} directory(ies) for media files..."
|
||||
)
|
||||
|
||||
if returned is None:
|
||||
media_files = await self._run_in_executor(
|
||||
self._collect_output_media_sync, walked
|
||||
)
|
||||
else:
|
||||
# #498: only the files whose marks this job deleted.
|
||||
media_files = await self._run_in_executor(
|
||||
self._clear_returned_marks, returned
|
||||
)
|
||||
job.log_line(f"Returned {_plural(len(media_files), 'image')}")
|
||||
media_files = await self._run_in_executor(
|
||||
self._collect_output_media_sync, walked
|
||||
)
|
||||
counts = dict.fromkeys(SCAN_COUNT_KEYS, 0)
|
||||
job.update(
|
||||
total=len(media_files),
|
||||
@@ -1605,13 +1551,8 @@ class AdminRoutesMixin:
|
||||
if job.cancelled:
|
||||
return
|
||||
batch = media_files[batch_start : batch_start + SCAN_BATCH_SIZE]
|
||||
# #480: a file removed from the library is not read or linked
|
||||
# again while its bytes are unchanged.
|
||||
removed = await self._run_in_executor(self._removed_paths, batch)
|
||||
skipped = [f for f in batch if str(f) in known or str(f) in removed]
|
||||
to_read = [
|
||||
f for f in batch if str(f) not in known and str(f) not in removed
|
||||
]
|
||||
skipped = [f for f in batch if str(f) in known]
|
||||
to_read = [f for f in batch if str(f) not in known]
|
||||
outcomes = [_scan_outcome(f, roots, "skipped") for f in skipped]
|
||||
counts["processed"] += len(skipped)
|
||||
counts["unchanged"] += len(skipped)
|
||||
@@ -1669,10 +1610,9 @@ class AdminRoutesMixin:
|
||||
def _extract_batch_metadata_sync(self, file_batch):
|
||||
"""Extract metadata from a batch of files in one executor call.
|
||||
|
||||
Files are read on the long-lived job pool of
|
||||
``PromptManagerConfig.WORKER_THREADS`` threads (#523).
|
||||
PNG text is read chunk by chunk without decoding pixels (#523); file
|
||||
reads and ffprobe release the GIL.
|
||||
Files are read on ``PromptManagerConfig.WORKER_THREADS`` threads:
|
||||
Pillow decodes the whole image to reach trailing text chunks and
|
||||
releases the GIL while doing so.
|
||||
"""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import math
|
||||
from ..tagging_defaults import ( # noqa: F401 - re-exported for the route
|
||||
LLM_MODEL_TYPES as LLM_MODELS,
|
||||
MAX_INSTRUCTION_BYTES,
|
||||
MAX_INSTRUCTION_CHARS,
|
||||
MODEL_TYPES,
|
||||
WD14_MODEL_TYPES as WD14_MODELS,
|
||||
validate_template,
|
||||
|
||||
@@ -19,15 +19,10 @@ from .jobs import (
|
||||
)
|
||||
|
||||
from .maintenance import disk_plan
|
||||
from .images import listing_thumbnail_url
|
||||
from . import preview_store
|
||||
from .prompts import thumbnail_url_for
|
||||
from . import thumbnail_settings
|
||||
from .. import model_gate
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import in_trash_below_roots, is_skipped_folder
|
||||
except ImportError:
|
||||
from utils.media_folders import in_trash_below_roots, is_skipped_folder
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
_INTERNAL_ERROR = "An internal error occurred. Check server logs for details."
|
||||
@@ -424,7 +419,6 @@ class AutotagRoutesMixin:
|
||||
title = f"Tagging model download · {label}"
|
||||
_download_titles.clear()
|
||||
_download_titles[owner] = title
|
||||
admitted = False
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
MODEL_DOWNLOAD_JOB_KIND,
|
||||
@@ -440,13 +434,13 @@ class AutotagRoutesMixin:
|
||||
self._run_model_download, service, model_type, owner
|
||||
),
|
||||
)
|
||||
admitted = True
|
||||
except JobBusy as refused:
|
||||
return None, busy_response(refused)
|
||||
finally:
|
||||
if not admitted: # JobBusy, or anything else: give the gate back.
|
||||
model_gate.release(owner)
|
||||
_download_titles.pop(owner, None)
|
||||
except BaseException as refused:
|
||||
# JobBusy, or anything else the registry raised: give the gate back.
|
||||
model_gate.release(owner)
|
||||
_download_titles.pop(owner, None)
|
||||
if isinstance(refused, JobBusy):
|
||||
return None, busy_response(refused)
|
||||
raise
|
||||
|
||||
def ended(_task):
|
||||
model_gate.release(owner)
|
||||
@@ -850,12 +844,9 @@ class AutotagRoutesMixin:
|
||||
"""Return the real path when it is a media file inside an output directory."""
|
||||
if Path(str(raw_path)).suffix.lower() not in _MEDIA_EXTENSIONS:
|
||||
return None
|
||||
roots = list(self._get_all_output_dirs())
|
||||
for output_dir in roots:
|
||||
for output_dir in self._get_all_output_dirs():
|
||||
if path_is_within(raw_path, output_dir):
|
||||
real = os.path.realpath(raw_path)
|
||||
if in_trash_below_roots(real, roots):
|
||||
return None # thrown away, not an output (#543)
|
||||
return real if os.path.isfile(real) else None
|
||||
return None
|
||||
|
||||
@@ -1042,12 +1033,12 @@ class AutotagRoutesMixin:
|
||||
return self._bad_request('existing must be "skip" or "merge"')
|
||||
try:
|
||||
options = self._single_options(data, model_type)
|
||||
except OptionError as error:
|
||||
return self._bad_request(str(error))
|
||||
except (ValueError, TypeError):
|
||||
return self._bad_request(
|
||||
"general_threshold and character_threshold must be numeric"
|
||||
)
|
||||
except OptionError as error:
|
||||
return self._bad_request(str(error))
|
||||
target, error = await self._select_autotag_target(data)
|
||||
if error is not None:
|
||||
return error
|
||||
@@ -1328,9 +1319,8 @@ class AutotagRoutesMixin:
|
||||
def _collect_scan_files(output_path: Path):
|
||||
"""Media files under ``output_path`` from a bounded, symlink-free walk.
|
||||
|
||||
Descends at most ``_SCAN_MAX_DEPTH`` levels, skips ``thumbnails`` and
|
||||
trash folders (``utils.media_folders``) and symlinked files, and stops
|
||||
after ``_SCAN_MAX_FILES`` files.
|
||||
Descends at most ``_SCAN_MAX_DEPTH`` levels, skips ``thumbnails``
|
||||
trees and symlinked files, and stops after ``_SCAN_MAX_FILES`` files.
|
||||
Returns ``(paths, truncated)``.
|
||||
"""
|
||||
found = []
|
||||
@@ -1340,7 +1330,7 @@ class AutotagRoutesMixin:
|
||||
dirnames[:] = sorted(
|
||||
name
|
||||
for name in dirnames
|
||||
if not is_skipped_folder(name) and depth + 1 <= _SCAN_MAX_DEPTH
|
||||
if name != "thumbnails" and depth + 1 <= _SCAN_MAX_DEPTH
|
||||
)
|
||||
for name in sorted(filenames):
|
||||
if Path(name).suffix.lower() not in _MEDIA_EXTENSIONS:
|
||||
@@ -1367,25 +1357,23 @@ class AutotagRoutesMixin:
|
||||
files, truncated = await self._run_in_executor(
|
||||
self._collect_scan_files, output_path
|
||||
)
|
||||
# #501: a file removed from the library is not offered for tagging.
|
||||
files = await self._run_in_executor(
|
||||
self._without_removed, output_path, files
|
||||
)
|
||||
location = preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE:
|
||||
has_thumbnails = location.available # no store I/O here (#320)
|
||||
else:
|
||||
store = preview_store.store_dir(output_path)
|
||||
has_thumbnails = store is not None and store.is_dir()
|
||||
has_thumbnails = (output_path / "thumbnails").is_dir()
|
||||
|
||||
images = []
|
||||
for image_path in files:
|
||||
rel_path = image_path.relative_to(output_path)
|
||||
thumbnail_url = None
|
||||
if has_thumbnails:
|
||||
# Any preview the file owns, the current format first
|
||||
# (#317); in folder mode without store I/O (#320).
|
||||
thumbnail_url = listing_thumbnail_url(output_path, rel_path)
|
||||
# Any preview the file owns, the current format first (#317)
|
||||
existing = thumbnail_settings.existing_thumbnail(
|
||||
output_path, rel_path
|
||||
)
|
||||
if existing is not None:
|
||||
thumb_rel = (
|
||||
thumbnail_settings.thumbnail_stem_rel(rel_path)
|
||||
+ existing.suffix
|
||||
)
|
||||
thumbnail_url = thumbnail_url_for(thumb_rel, existing)
|
||||
images.append(
|
||||
{
|
||||
"filename": image_path.name,
|
||||
|
||||
@@ -971,7 +971,6 @@ class BackupRestoreRoutesMixin:
|
||||
line = restored_line(staged.summary)
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
|
||||
async def _outlive_cancel(future):
|
||||
|
||||
@@ -213,8 +213,6 @@ def public_config():
|
||||
"max_image_age_days": GalleryConfig.MAX_IMAGE_AGE_DAYS,
|
||||
"max_concurrent_processing": GalleryConfig.MAX_CONCURRENT_PROCESSING,
|
||||
"metadata_extraction_timeout": GalleryConfig.METADATA_EXTRACTION_TIMEOUT,
|
||||
# config.json gallery.monitoring.save_foreign_runs (#247)
|
||||
"save_foreign_runs": GalleryConfig.SAVE_FOREIGN_RUNS,
|
||||
},
|
||||
"integrations": {
|
||||
"lora_manager_enabled": bool(IntegrationConfig.LORA_MANAGER_ENABLED),
|
||||
|
||||
@@ -574,13 +574,12 @@ class DuplicateFilesMixin:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return None
|
||||
return
|
||||
try:
|
||||
await self._remove_duplicates(job, db, plan, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
return None
|
||||
|
||||
async def _remove_duplicates(self, job, db, plan, backup):
|
||||
targets = plan.payload["targets"]
|
||||
|
||||
@@ -16,11 +16,6 @@ from pathlib import Path
|
||||
from .admin import DELETABLE_MEDIA_EXTENSIONS
|
||||
from .maintenance import DUPLICATE_FILES
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import in_trash_below_roots, is_trash_folder
|
||||
except ImportError:
|
||||
from utils.media_folders import in_trash_below_roots, is_trash_folder
|
||||
|
||||
# Keep entries one review takes; with paths of a few hundred characters the
|
||||
# list stays inside the preview body cap (DUPLICATE_PREVIEW_BODY_MAX, 1 MiB).
|
||||
MAX_KEEP = 2000
|
||||
@@ -202,8 +197,6 @@ def file_problem(
|
||||
parts = Path(os.path.relpath(lexical, base)).parts
|
||||
if "thumbnails" in parts or ".." in parts:
|
||||
return OUTSIDE_SOURCE
|
||||
if any(is_trash_folder(part) for part in parts[:-1]):
|
||||
return OUTSIDE_SOURCE # never removed from a trash folder (#543)
|
||||
if os.path.splitext(lexical)[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||||
return OUTSIDE_SOURCE
|
||||
try:
|
||||
@@ -219,8 +212,6 @@ def file_problem(
|
||||
try:
|
||||
if not _lexically_inside(_canonical_path(path), [_canonical_path(root)]):
|
||||
return OUTSIDE_SOURCE # a linked folder on the way
|
||||
if in_trash_below_roots(path, [root]):
|
||||
return OUTSIDE_SOURCE # a linked folder into a trash folder
|
||||
if keeper is not None:
|
||||
same = keeper_problem(path, root, info, keeper, keeper_root)
|
||||
if same:
|
||||
|
||||
@@ -27,7 +27,7 @@ def _identity(path):
|
||||
except OSError:
|
||||
return None
|
||||
if info.st_ino:
|
||||
return ("inode", (info.st_dev, info.st_ino))
|
||||
return ("inode", info.st_dev, info.st_ino)
|
||||
return ("path", os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
|
||||
@@ -27,11 +27,7 @@ from .history_current import (
|
||||
resolve_current_candidate,
|
||||
snapshot_current_output_scope,
|
||||
)
|
||||
from .inspection_service import (
|
||||
CurrentFileChanged,
|
||||
image_tags_reader,
|
||||
inspect_current_file,
|
||||
)
|
||||
from .inspection_service import CurrentFileChanged, inspect_current_file
|
||||
from .images import image_category_key
|
||||
|
||||
try:
|
||||
@@ -782,14 +778,6 @@ class HistoryRoutesMixin:
|
||||
),
|
||||
own,
|
||||
),
|
||||
# The file's own image tags, keyed as the Tag dialog
|
||||
# keys them: the normcased realpath (#457).
|
||||
image_tags=image_tags_reader(
|
||||
db,
|
||||
str(resolved.canonical_target),
|
||||
self.logger,
|
||||
close=True,
|
||||
),
|
||||
recorded=recorded_output_lookup(
|
||||
db, close=True, reference=receipt["reference"]
|
||||
),
|
||||
@@ -843,10 +831,7 @@ class HistoryRoutesMixin:
|
||||
raise
|
||||
|
||||
async def execute():
|
||||
# An inspection reads the file's header and text: the
|
||||
# interactive image pool (#523); media stays on the default.
|
||||
run = self._run_image if kind == "inspection" else self._run_in_executor
|
||||
pending = asyncio.create_task(run(project))
|
||||
pending = asyncio.create_task(self._run_in_executor(project))
|
||||
try:
|
||||
return await asyncio.shield(pending)
|
||||
except asyncio.CancelledError:
|
||||
|
||||
@@ -15,10 +15,9 @@ from typing import BinaryIO
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from . import preview_store, thumbnail_settings
|
||||
from . import thumbnail_settings
|
||||
from .images import _canonical, _public_path, SERVABLE_EXTENSIONS, VIDEO_EXTENSIONS
|
||||
from .images import in_trash_below_roots
|
||||
from .images import existing_media_thumbnail, media_preview_path
|
||||
from .images import existing_media_thumbnail, media_thumbnail_rel
|
||||
|
||||
try:
|
||||
from ...utils.execution_outputs import (
|
||||
@@ -199,9 +198,6 @@ def resolve_current_candidate(
|
||||
if namespace is None or not namespace.is_dir():
|
||||
return _unavailable("root_unavailable")
|
||||
target = _canonical(lexical)
|
||||
if in_trash_below_roots(target, [namespace]):
|
||||
# Every kind and the outputs listing come through here (#543).
|
||||
return _unavailable("missing")
|
||||
if not _authorized(target, namespace, scope):
|
||||
return _unavailable("not_authorized")
|
||||
if target.suffix.lower() not in SERVABLE_EXTENSIONS:
|
||||
@@ -244,36 +240,15 @@ def resolve_current_candidate(
|
||||
return _unavailable("temporary_unavailable", error=True)
|
||||
|
||||
|
||||
def _target_authorized(target, namespace, scope, store_root=None) -> bool:
|
||||
"""A preview in the thumbnail folder (#320, folder mode) is authorized by
|
||||
the store of its source (``preview_store.owns``: inside the key folder,
|
||||
no link); everything else by the output namespace and the roots."""
|
||||
if (
|
||||
store_root is not None
|
||||
and preview_store.current().mode == preview_store.FOLDER_MODE
|
||||
):
|
||||
return preview_store.owns(store_root, target)
|
||||
return _authorized(target, namespace, scope)
|
||||
|
||||
|
||||
def _open_current_target(
|
||||
target: Path,
|
||||
namespace: Path,
|
||||
scope: CurrentOutputScope,
|
||||
preview: bool = False,
|
||||
store_root: Path | None = None,
|
||||
target: Path, namespace: Path, scope: CurrentOutputScope, preview: bool = False
|
||||
) -> BinaryIO:
|
||||
"""Open only the canonical target and verify authority against the actual handle.
|
||||
|
||||
A *preview* (a file under ``thumbnails/``) may also be JPEG XL (#317); in
|
||||
folder mode it is authorized by the store of *store_root* (#320)."""
|
||||
A *preview* (a file under ``thumbnails/``) may also be JPEG XL (#317)."""
|
||||
if _canonical(target) != target:
|
||||
raise CurrentLocationUnavailable("not_authorized", stale=True)
|
||||
if in_trash_below_roots(target, [namespace]):
|
||||
# Also at the open: the cover chooses its preview after the candidate
|
||||
# check, and a preview can be a link into a trash folder (#543).
|
||||
raise CurrentLocationUnavailable("missing")
|
||||
if not _target_authorized(target, namespace, scope, store_root):
|
||||
if not _authorized(target, namespace, scope):
|
||||
raise CurrentLocationUnavailable("not_authorized")
|
||||
suffix = target.suffix.lower()
|
||||
if suffix not in SERVABLE_EXTENSIONS and not (
|
||||
@@ -306,7 +281,7 @@ def _open_current_target(
|
||||
if (
|
||||
_canonical(target) != target
|
||||
or (observed.st_dev, observed.st_ino) != (current.st_dev, current.st_ino)
|
||||
or not _target_authorized(target, namespace, scope, store_root)
|
||||
or not _authorized(target, namespace, scope)
|
||||
):
|
||||
raise CurrentLocationUnavailable("not_authorized", stale=True)
|
||||
return os.fdopen(fd, "rb")
|
||||
@@ -343,10 +318,12 @@ class _OpenedCurrentMedia:
|
||||
|
||||
|
||||
def _preview_writable(root: Path, preview: Path) -> bool:
|
||||
"""A preview may be written: the store of *root* and every link segment
|
||||
of the preview's folder resolve inside it (the serve route's rule, #249;
|
||||
``preview_store.owns``)."""
|
||||
return preview_store.owns(root, preview)
|
||||
"""A preview may be written: ``thumbnails`` and every link segment of the
|
||||
preview's folder resolve inside *root* (the serve route's rule, #249)."""
|
||||
folder = root / "thumbnails"
|
||||
return _canonical(folder).is_relative_to(_canonical(root)) and _canonical(
|
||||
preview
|
||||
).is_relative_to(_canonical(folder))
|
||||
|
||||
|
||||
def _is_jxl(path) -> bool:
|
||||
@@ -396,12 +373,8 @@ def _cover_target(resolved, ensure_preview, placeholder, accepts_jxl):
|
||||
return _canonical(preview)
|
||||
if preview is not None:
|
||||
return _canonical(preview)
|
||||
preview = media_preview_path(resolved.gallery_root, relative, create=True)
|
||||
if (
|
||||
ensure_preview is not None
|
||||
and preview is not None
|
||||
and _preview_writable(resolved.gallery_root, preview)
|
||||
):
|
||||
preview = resolved.gallery_root / media_thumbnail_rel(relative)
|
||||
if ensure_preview is not None and _preview_writable(resolved.gallery_root, preview):
|
||||
ensure_preview(target, preview, is_video)
|
||||
if is_video and placeholder is not None:
|
||||
return _placeholder_media(placeholder)
|
||||
@@ -437,13 +410,7 @@ def _prepare_current_media(
|
||||
return chosen
|
||||
if chosen is not None:
|
||||
target, is_preview = chosen, True
|
||||
handle = _open_current_target(
|
||||
target,
|
||||
resolved.namespace,
|
||||
scope,
|
||||
is_preview,
|
||||
resolved.gallery_root if is_preview else None,
|
||||
)
|
||||
handle = _open_current_target(target, resolved.namespace, scope, is_preview)
|
||||
try:
|
||||
return _OpenedCurrentMedia(
|
||||
handle,
|
||||
|
||||
@@ -232,17 +232,15 @@ class ImageCategoryRoutesMixin:
|
||||
return None, _coded(400, "too-many-images", TOO_MANY)
|
||||
return tuple(selection.targets), None
|
||||
|
||||
async def _category_request(self, request, fields: frozenset, data=None):
|
||||
async def _category_request(self, request, fields: frozenset):
|
||||
"""``(data, targets, roots, None)`` or ``(None, None, None, error)``.
|
||||
|
||||
The images are ``items`` (1 to 500 targets) or ``filter`` (all images
|
||||
an Images listing filter admits), besides *fields*. *data* is the body
|
||||
when the caller has already read it (a body is read once).
|
||||
an Images listing filter admits), besides *fields*.
|
||||
"""
|
||||
if data is None:
|
||||
data, error = await _json_body_or_error(request)
|
||||
if error is not None:
|
||||
return None, None, None, error
|
||||
data, error = await _json_body_or_error(request)
|
||||
if error is not None:
|
||||
return None, None, None, error
|
||||
names = set(data) - fields
|
||||
if set(data) & fields != fields or names not in ({"items"}, {"filter"}):
|
||||
return (
|
||||
|
||||
@@ -1,522 +0,0 @@
|
||||
"""Delete selected images: remove them from the library, or trash them (#480).
|
||||
|
||||
``POST /prompt_manager/images/delete/preview {items | filter}`` reviews the
|
||||
selection for both modes at once and answers a reviewed plan (the System ›
|
||||
Maintenance contract, ``py/api/maintenance.py``): the token, the counts of
|
||||
the records each file loses, the refused files with a coded reason (both
|
||||
modes, and the trash-only ones), the prompts that lose their last image and
|
||||
the safety backup. It writes nothing. ``items`` are Images listing identities
|
||||
(at most 500); ``filter`` is "Select all N matching". Each target is matched
|
||||
to its entry in the current listing (``not-listed`` otherwise), so every
|
||||
later step uses the walked spelling, never the request's text.
|
||||
|
||||
``POST /prompt_manager/maintenance/apply {token, mode}`` runs the plan as the
|
||||
``image_delete`` job (admission ``records``, not cancellable): the sources
|
||||
are checked, the promised safety backup is taken, then each file is checked
|
||||
again (containment and fingerprint) and
|
||||
|
||||
- ``mode="library"``: its records are deleted and its removal mark written
|
||||
in one transaction; the file stays on disk, hidden;
|
||||
- ``mode="trash"``: it is moved to the trash first (``utils/trash.py``); a
|
||||
refused move leaves the file and its records alone; after a move its
|
||||
records are deleted.
|
||||
|
||||
No path here unlinks a media file. A failed records write stops the job; the
|
||||
files after it are not attempted.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat as stat_module
|
||||
from pathlib import Path
|
||||
|
||||
from .admin import DELETABLE_MEDIA_EXTENSIONS, _remove_thumbnail
|
||||
from .duplicate_review import (
|
||||
CHANGED_CONTENT,
|
||||
GONE,
|
||||
LINK,
|
||||
OUTSIDE_SOURCE,
|
||||
UNREADABLE,
|
||||
file_problem,
|
||||
)
|
||||
from .images import _favorite_relative_key, _favorite_root_key
|
||||
from .jobs import IMAGE_DELETE_GROUP_NAMES, IMAGE_DELETE_JOB_KIND, JobBusy
|
||||
from .job_routes import busy_response
|
||||
from .maintenance import MAINTENANCE_PREVIEW_LEASE, _fail_job, _failure, backup_note
|
||||
|
||||
try:
|
||||
from ...database import image_removal_schema
|
||||
from ...database.image_removal import mark_hides
|
||||
from ...database.annotation_schema import MAX_LOCATOR_LENGTH
|
||||
from ...utils import trash
|
||||
from ...utils.media_folders import is_trash_folder
|
||||
except ImportError:
|
||||
from database import image_removal_schema
|
||||
from database.image_removal import mark_hides
|
||||
from database.annotation_schema import MAX_LOCATOR_LENGTH
|
||||
from utils import trash
|
||||
from utils.media_folders import is_trash_folder
|
||||
|
||||
IMAGE_DELETE = "delete_images"
|
||||
MODES = ("library", "trash")
|
||||
TITLES = {
|
||||
"library": "Delete images (remove from the library)",
|
||||
"trash": "Delete images (move to the trash)",
|
||||
}
|
||||
MAX_DETAIL_ROWS = 50
|
||||
# The system moved the file but its place in a trash could not be shown (#499).
|
||||
TRASH_UNCONFIRMED = "trash-unconfirmed"
|
||||
RECORDS_FAILED_CODE = "records-failed"
|
||||
|
||||
# Reason codes (design §4.4) and the sentence each shows.
|
||||
REASONS = {
|
||||
"not-listed": "No longer listed in Images.",
|
||||
"outside": "Outside the output folders.",
|
||||
"link": "A link, not a file.",
|
||||
"gone": "The file no longer exists.",
|
||||
"unreadable": "The file can't be read.",
|
||||
"thumbnail": "A generated preview, not an image.",
|
||||
"not-media": "Not a media file.",
|
||||
"trash-folder": "Inside a trash folder.",
|
||||
"path-too-long": "The path is too long.",
|
||||
"changed": "The file changed after the review.",
|
||||
trash.NO_TRASH: "This drive has no trash.",
|
||||
trash.NETWORK: "Network drives can't use the trash.",
|
||||
trash.TRASH_IN_ROOT: "This drive's trash is inside an output folder.",
|
||||
trash.LOCKED: "The file is open in another program. Close it and try again.",
|
||||
trash.FAILED: "The file could not be moved to the trash.",
|
||||
trash.NO_PERMISSION: (
|
||||
"PromptManager isn't allowed to move this file (no permission, or it is "
|
||||
"locked in Finder)."
|
||||
),
|
||||
trash.CLOUD_SYNC: (
|
||||
"Files in iCloud Drive or a cloud-synced folder can't be moved to the "
|
||||
"trash here."
|
||||
),
|
||||
TRASH_UNCONFIRMED: (
|
||||
"Moved by macOS, but its place in the trash could not be confirmed."
|
||||
),
|
||||
trash.WINDOWS_RECYCLE: (
|
||||
"Moving files to the Recycle Bin isn't available on Windows: files larger "
|
||||
"than the bin's capacity would be deleted permanently."
|
||||
),
|
||||
"not-attempted": "Not attempted: the job stopped after an error.",
|
||||
}
|
||||
_PROBLEM_CODES = {
|
||||
OUTSIDE_SOURCE: "outside",
|
||||
LINK: "link",
|
||||
GONE: "gone",
|
||||
UNREADABLE: "unreadable",
|
||||
CHANGED_CONTENT: "changed",
|
||||
}
|
||||
SOURCES_CHANGED = "The sources changed since the review; nothing was changed."
|
||||
RECORDS_FAILED = "The image records could not be removed; the job stopped."
|
||||
|
||||
|
||||
def lexical_problem(relative: str) -> str | None:
|
||||
"""The refusal code a root-relative POSIX path earns before any file access."""
|
||||
parts = relative.split("/")
|
||||
if "thumbnails" in parts[:-1]:
|
||||
return "thumbnail"
|
||||
if any(is_trash_folder(part) for part in parts[:-1]):
|
||||
return "trash-folder"
|
||||
if os.path.splitext(parts[-1])[1].lower() not in DELETABLE_MEDIA_EXTENSIONS:
|
||||
return "not-media"
|
||||
return None
|
||||
|
||||
|
||||
def file_check(path, root, relative, *, expected=None):
|
||||
"""``(code, lstat)``: why *path* may not be deleted, or ``(None, lstat)``.
|
||||
|
||||
Text first (inside the root, not a preview or trash folder, a media
|
||||
file), then ``duplicate_review.file_problem``: a regular file, not a
|
||||
link, whose resolved path is still inside the resolved root. With
|
||||
*expected* (``{size, mtime_ns, inode}`` from the review) the file must
|
||||
still be those bytes. Every handle is closed when this returns.
|
||||
"""
|
||||
code = lexical_problem(relative)
|
||||
if code:
|
||||
return code, None
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return "gone", None
|
||||
except OSError:
|
||||
return "unreadable", None
|
||||
if getattr(info, "st_file_attributes", 0) & getattr(
|
||||
stat_module, "FILE_ATTRIBUTE_REPARSE_POINT", 0
|
||||
):
|
||||
return "link", info
|
||||
if expected is None:
|
||||
problem = file_problem(path, root, None, info.st_size, hash_check=False)
|
||||
else:
|
||||
problem = file_problem(
|
||||
path,
|
||||
root,
|
||||
None,
|
||||
expected["size"],
|
||||
hash_check=False,
|
||||
mtime_ns=expected["mtime_ns"],
|
||||
inode=expected["inode"],
|
||||
)
|
||||
if problem:
|
||||
return _PROBLEM_CODES.get(problem, "outside"), info
|
||||
if len(os.path.normcase(os.path.realpath(path))) > MAX_LOCATOR_LENGTH:
|
||||
return "path-too-long", info
|
||||
return None, info
|
||||
|
||||
|
||||
def _row(target, code):
|
||||
return {
|
||||
"path": target["rel"],
|
||||
"root_label": target["root_label"],
|
||||
"code": code,
|
||||
"detail": REASONS[code],
|
||||
}
|
||||
|
||||
|
||||
def _summary(rows):
|
||||
counts = {}
|
||||
for row in rows:
|
||||
counts[row["code"]] = counts.get(row["code"], 0) + 1
|
||||
return {"count": len(rows), "counts": counts, "rows": rows[:MAX_DETAIL_ROWS]}
|
||||
|
||||
|
||||
def _fingerprint(targets) -> str:
|
||||
material = [
|
||||
(t["root_index"], t["rel"], t["size"], t["mtime_ns"], t["inode"])
|
||||
for t in targets
|
||||
]
|
||||
return hashlib.sha256(json.dumps(material).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class ImageDeleteMixin:
|
||||
"""The Images Delete… review and its apply job (#480)."""
|
||||
|
||||
def _register_image_delete_routes(self, routes):
|
||||
# A fixed path beside the dynamic /images/{image_id} routes: aiohttp
|
||||
# (3.10+) looks fixed paths up first (see image_categories.py).
|
||||
@routes.post("/prompt_manager/images/delete/preview")
|
||||
async def preview_image_delete_route(request):
|
||||
return await self.preview_image_delete(request)
|
||||
|
||||
# ── review ────────────────────────────────────────────────────────
|
||||
|
||||
async def preview_image_delete(self, request):
|
||||
"""Review the selection under the maintenance preview lease."""
|
||||
try:
|
||||
async with self._jobs.lease(MAINTENANCE_PREVIEW_LEASE):
|
||||
return await self._preview_image_delete(request)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _preview_image_delete(self, request):
|
||||
_data, targets, roots, error = await self._category_request(
|
||||
request, frozenset()
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
db = self.db
|
||||
identity = self._database_identity(db)
|
||||
try:
|
||||
files = {}
|
||||
for index in dict.fromkeys(t[0] for t in targets):
|
||||
files[index] = await self._get_gallery_files(roots[index])
|
||||
review = await self._run_in_executor(
|
||||
self._review_image_delete, db, roots, targets, files
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Image delete review failed")
|
||||
return _failure("The review failed. Check server logs for details.", 500)
|
||||
return await self._plan_response(
|
||||
db,
|
||||
identity,
|
||||
IMAGE_DELETE,
|
||||
_fingerprint(review["targets"]),
|
||||
review["public"],
|
||||
{"targets": review["targets"]},
|
||||
)
|
||||
|
||||
def _review_image_delete(self, db, roots, targets, files):
|
||||
"""Every target checked now (blocking); ``{targets, public}``."""
|
||||
listings, seen = {}, set()
|
||||
acted, refused, trash_refused = [], [], []
|
||||
spellings = self._source_spellings()
|
||||
root_texts = [str(root) for root in roots]
|
||||
for index, _scope, relative in targets:
|
||||
root = roots[index]
|
||||
if index not in listings:
|
||||
listings[index] = self._favorite_listing(root, files[index])
|
||||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||||
target = {
|
||||
"root_index": index,
|
||||
"root_scope": _scope,
|
||||
"root": str(root),
|
||||
"root_label": self._public_path(root),
|
||||
"rel": spelled or relative,
|
||||
}
|
||||
if spelled is None:
|
||||
refused.append(_row(target, "not-listed"))
|
||||
continue
|
||||
absolute = os.path.join(str(root), *spelled.split("/"))
|
||||
code, info = file_check(absolute, str(root), spelled)
|
||||
if code:
|
||||
refused.append(_row(target, code))
|
||||
continue
|
||||
identity = (info.st_dev, info.st_ino)
|
||||
if identity in seen:
|
||||
continue # one physical file named twice (nested roots, hardlinks)
|
||||
seen.add(identity)
|
||||
target.update(self._removal_of(absolute, info, spellings))
|
||||
target["trash"] = trash.trash_available(absolute, root_texts)
|
||||
if target["trash"]:
|
||||
trash_refused.append(_row(target, target["trash"]))
|
||||
acted.append(target)
|
||||
counted = db.count_removal_records([t["removal"] for t in acted])
|
||||
library_count = len(acted)
|
||||
public = {
|
||||
"operation": IMAGE_DELETE,
|
||||
"scope": {"count": library_count, "label": "images"},
|
||||
"library": {"count": library_count},
|
||||
"trash": {"count": library_count - len(trash_refused)},
|
||||
"protected": {"count": len(refused), "reasons": []},
|
||||
"refused": _summary(refused),
|
||||
"trash_refused": _summary(trash_refused),
|
||||
"records": counted["records"],
|
||||
"orphaned_prompts": counted["orphaned_prompts"],
|
||||
"media": {"deletes_files": False, "label": "Chosen in the review"},
|
||||
"details": [],
|
||||
}
|
||||
return {"targets": acted, "public": public}
|
||||
|
||||
@staticmethod
|
||||
def _removal_of(absolute, info, spellings):
|
||||
favorites = [
|
||||
(_favorite_root_key(root), _favorite_relative_key(relative))
|
||||
for root, relative in spellings.favorites(absolute).items()
|
||||
]
|
||||
return {
|
||||
"abs": absolute,
|
||||
"size": info.st_size,
|
||||
"mtime_ns": info.st_mtime_ns,
|
||||
"inode": [info.st_dev, info.st_ino],
|
||||
"removal": {
|
||||
"spellings": list(spellings.of(absolute).values()),
|
||||
"locator": os.path.normcase(os.path.realpath(absolute)),
|
||||
"favorites": favorites,
|
||||
"category": spellings.category_key(absolute),
|
||||
},
|
||||
}
|
||||
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
def _start_image_delete(self, db, plan, mode, backup_on):
|
||||
"""Register the delete job for a reviewed plan (on the loop)."""
|
||||
return self._jobs.start(
|
||||
IMAGE_DELETE_JOB_KIND,
|
||||
TITLES[mode],
|
||||
admission="records",
|
||||
cancellable=False,
|
||||
scope={"operation": IMAGE_DELETE, "mode": mode},
|
||||
groups=IMAGE_DELETE_GROUP_NAMES,
|
||||
runner=lambda job: self._run_image_delete(job, db, plan, mode, backup_on),
|
||||
)
|
||||
|
||||
async def _run_image_delete(self, job, db, plan, mode, backup_on):
|
||||
targets = plan.payload["targets"]
|
||||
unchanged = await self._run_in_executor(self._sources_unchanged, targets)
|
||||
if not unchanged:
|
||||
return _fail_job(job, "sources-changed", SOURCES_CHANGED)
|
||||
ok, backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return None
|
||||
try:
|
||||
await self._delete_images(job, db, targets, mode, backup)
|
||||
finally:
|
||||
if backup:
|
||||
await self._prune_backup_pool(job, db)
|
||||
return None
|
||||
|
||||
async def _delete_images(self, job, db, targets, mode, backup):
|
||||
job.update(total=len(targets), status_line=TITLES[mode])
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
done_group = "trashed" if mode == "trash" else "removed"
|
||||
changed, failed = 0, None
|
||||
for done, target in enumerate(targets, start=1):
|
||||
if failed:
|
||||
self._record_delete(job, "not_changed", target, "not-attempted")
|
||||
continue
|
||||
group, code, label = await self._run_in_executor(
|
||||
self._delete_one_image, db, target, mode, roots
|
||||
)
|
||||
if group == "errors":
|
||||
failed = code or RECORDS_FAILED_CODE
|
||||
text = REASONS[code] if code else RECORDS_FAILED
|
||||
job.record(
|
||||
"errors",
|
||||
{
|
||||
"path": target["rel"],
|
||||
"root": target["root_label"],
|
||||
"root_index": target["root_index"],
|
||||
"root_scope": target["root_scope"],
|
||||
"detail": text,
|
||||
},
|
||||
f"! {target['root_label']}/{target['rel']} · {text}",
|
||||
)
|
||||
elif group == done_group:
|
||||
changed += 1
|
||||
detail = (
|
||||
f"Moved to the {label}" if label else "Removed from the library"
|
||||
)
|
||||
self._record_delete(job, group, target, None, detail)
|
||||
else:
|
||||
self._record_delete(job, group, target, code)
|
||||
job.update(processed=done, progress=int(done / len(targets) * 100))
|
||||
if changed:
|
||||
self.invalidate_gallery_cache()
|
||||
verb = "Moved" if mode == "trash" else "Removed"
|
||||
where = "to the trash" if mode == "trash" else "from the library"
|
||||
line = f"{verb} {changed:,} of {len(targets):,} images {where}"
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
if failed:
|
||||
# "records-failed" names every stop at an errors row (the page's
|
||||
# "Stopped after N of M" toast); the row says which error it was.
|
||||
job.scope["failure"] = RECORDS_FAILED_CODE
|
||||
job.publish({"type": "error", "code": failed, "message": line})
|
||||
job.finish("failed")
|
||||
|
||||
@staticmethod
|
||||
def _record_delete(job, group, target, code, detail=None):
|
||||
text = detail or REASONS[code]
|
||||
row = {
|
||||
"path": target["rel"],
|
||||
"root": target["root_label"],
|
||||
"root_index": target["root_index"],
|
||||
"root_scope": target["root_scope"],
|
||||
"code": code,
|
||||
"detail": text,
|
||||
}
|
||||
mark = {"removed": "-", "trashed": "-"}.get(group, "=")
|
||||
job.record(
|
||||
group, row, f"{mark} {target['root_label']}/{target['rel']} · {text}"
|
||||
)
|
||||
|
||||
def _delete_one_image(self, db, target, mode, roots):
|
||||
"""``(group, code, trash label)`` for one reviewed file (blocking)."""
|
||||
code, _info = file_check(
|
||||
target["abs"],
|
||||
target["root"],
|
||||
target["rel"],
|
||||
expected={
|
||||
"size": target["size"],
|
||||
"mtime_ns": target["mtime_ns"],
|
||||
"inode": target["inode"],
|
||||
},
|
||||
)
|
||||
if code:
|
||||
return "not_changed", code, None
|
||||
label = None
|
||||
if mode == "trash":
|
||||
try:
|
||||
label = trash.move_to_trash(target["abs"], roots)
|
||||
except trash.TrashRefused as refused:
|
||||
code = refused.code if refused.code in REASONS else trash.FAILED
|
||||
self.logger.info(
|
||||
"Not moved to the trash (%s): %s %s",
|
||||
code,
|
||||
target["rel"],
|
||||
refused.detail,
|
||||
)
|
||||
return "not_changed", code, None
|
||||
except trash.TrashUnconfirmed as unconfirmed:
|
||||
# The file left its folder: keep its records, stop the job.
|
||||
self.logger.warning(
|
||||
"Trash move not confirmed: %s %s", target["rel"], unconfirmed.detail
|
||||
)
|
||||
return "errors", TRASH_UNCONFIRMED, None
|
||||
mark = None
|
||||
else:
|
||||
mark = (target["size"], target["mtime_ns"])
|
||||
try:
|
||||
db.remove_image_records(target["removal"], mark=mark)
|
||||
except Exception:
|
||||
self.logger.exception("Removing an image's records failed")
|
||||
return "errors", None, label
|
||||
_remove_thumbnail(Path(target["root"]), Path(target["rel"]), self.logger)
|
||||
return ("trashed" if mode == "trash" else "removed"), None, label
|
||||
|
||||
# ── the workspace listing ─────────────────────────────────────────
|
||||
|
||||
def _removal_marks(self) -> dict:
|
||||
"""The removal marks, cached until a mark changes or the database does.
|
||||
|
||||
A listing page reads no table while nothing changed: the cache follows
|
||||
this process's marks generation and the database identity (a restore).
|
||||
"""
|
||||
key = (self._database_identity(self.db), image_removal_schema.generation())
|
||||
cached = getattr(self, "_marks_cache", None)
|
||||
if cached is None or cached[0] != key:
|
||||
cached = (key, self.db.removal_marks())
|
||||
self._marks_cache = cached
|
||||
return cached[1]
|
||||
|
||||
def _hidden_entries(self, listings) -> set:
|
||||
"""Normalized paths of listed files a removal mark hides (blocking).
|
||||
|
||||
Only marked locators are looked at, and only those files are
|
||||
``lstat``'ed: a mark hides a file while its size and ``mtime_ns`` are
|
||||
still the ones it was removed with.
|
||||
"""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return set()
|
||||
located = self._workspace_locators(listings)
|
||||
return {
|
||||
entry.normalized
|
||||
for locator, entry in ((locator, located.get(locator)) for locator in marks)
|
||||
if entry is not None and mark_hides(marks, locator, entry.path)
|
||||
}
|
||||
|
||||
def _removed_paths(self, paths) -> set:
|
||||
"""The absolute paths among *paths* a removal mark hides (blocking)."""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return set()
|
||||
return {
|
||||
str(path)
|
||||
for path in paths
|
||||
if mark_hides(marks, os.path.normcase(os.path.realpath(str(path))), path)
|
||||
}
|
||||
|
||||
def _removal_check_under(self, root):
|
||||
"""``hidden(path) -> bool`` for files a walk of *root* found (blocking).
|
||||
|
||||
A walk never follows a link below *root*, so a file's locator is
|
||||
``normcase(join(realpath(root), relative))``: no ``realpath`` per
|
||||
file, and an ``lstat`` only for a marked one. Without marks it
|
||||
answers False at once.
|
||||
"""
|
||||
marks = self._removal_marks()
|
||||
if not marks:
|
||||
return lambda _path: False
|
||||
root = os.fspath(root)
|
||||
real = os.path.realpath(root)
|
||||
|
||||
def hidden(path):
|
||||
relative = os.path.relpath(os.fspath(path), root)
|
||||
locator = os.path.normcase(os.path.join(real, relative))
|
||||
return mark_hides(marks, locator, path)
|
||||
|
||||
return hidden
|
||||
|
||||
def _without_removed(self, root, paths) -> list:
|
||||
"""*paths* found by a walk of *root*, less the hidden ones (blocking)."""
|
||||
removed = self._removal_check_under(root)
|
||||
return [path for path in paths if not removed(path)]
|
||||
|
||||
|
||||
def parse_mode(value):
|
||||
"""The apply's ``mode``, or None when it is not one of MODES."""
|
||||
return value if isinstance(value, str) and value in MODES else None
|
||||
@@ -1,148 +0,0 @@
|
||||
"""Return removed images to the library (#498).
|
||||
|
||||
``POST /prompt_manager/images/removed/return {items | filter}`` names files a
|
||||
removal mark hides (#480, "Remove from the library only"): ``items`` are 1 to
|
||||
500 listing identities, ``filter`` the removed view's listing query
|
||||
(``removed: "1"`` plus the view's other parameters, at most 500 images). Only
|
||||
this route takes ``removed``: every other bulk route refuses it, because an
|
||||
explicit write there would show a file again without re-reading it.
|
||||
|
||||
The route answers 202 ``{job, returned, not_removed}`` after starting a
|
||||
``scan`` job titled "Return images to the library" (admission ``records``:
|
||||
409 ``busy`` beside a scan, restore or delete, nothing changed). The job
|
||||
first deletes the marks that still hide their file (same size and
|
||||
``mtime_ns``), in one transaction, then scans exactly those files with the
|
||||
scan's own pipeline: prompts, links, pairs and Needs review items come back
|
||||
from the files' embedded metadata. Image tags, auto-tag marks, the own
|
||||
category, collection memberships and the favorite were deleted by the remove
|
||||
and do not come back. No file is moved, written or deleted.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .image_categories import TOO_MANY, _coded
|
||||
from .images import (
|
||||
ListingSourceChanged,
|
||||
_favorite_relative_key,
|
||||
_json_body_or_error,
|
||||
parse_listing_filter,
|
||||
)
|
||||
from .image_tag_jobs import MAX_JOB_TARGETS
|
||||
from .jobs import JobBusy
|
||||
from .job_routes import busy_response
|
||||
|
||||
try:
|
||||
from ...database.image_removal import mark_hides
|
||||
except ImportError:
|
||||
from database.image_removal import mark_hides
|
||||
|
||||
NONE_REMOVED = "None of the selected images are removed from the library."
|
||||
SOURCE_CHANGED = "The image sources changed. Reload Images and try again."
|
||||
|
||||
|
||||
def _bad(message):
|
||||
return web.json_response({"success": False, "error": message}, status=400)
|
||||
|
||||
|
||||
class ImageReturnMixin:
|
||||
"""Return to library (#498): clear the marks, then scan those files."""
|
||||
|
||||
def _register_image_return_routes(self, routes):
|
||||
# A fixed path beside the dynamic /images/{image_id} routes.
|
||||
@routes.post("/prompt_manager/images/removed/return")
|
||||
async def return_removed_images_route(request):
|
||||
return await self.return_removed_images(request)
|
||||
|
||||
async def return_removed_images(self, request):
|
||||
data, error = await _json_body_or_error(request)
|
||||
if error is not None:
|
||||
return error
|
||||
if set(data) == {"filter"}:
|
||||
targets, roots, error = await self._removed_filter_targets(data["filter"])
|
||||
else:
|
||||
_data, targets, roots, error = await self._category_request(
|
||||
request, frozenset(), data=data
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
files = {}
|
||||
for index in dict.fromkeys(t[0] for t in targets):
|
||||
files[index] = await self._get_gallery_files(roots[index])
|
||||
returned, not_removed = await self._run_in_executor(
|
||||
self._hidden_targets, roots, targets, files
|
||||
)
|
||||
if not returned:
|
||||
return _coded(400, "no-images", NONE_REMOVED)
|
||||
try:
|
||||
job = self._start_scan_job(
|
||||
files="all", output_dirs=list(roots), returned=returned
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"job": job.summary(),
|
||||
"returned": len(returned),
|
||||
"not_removed": not_removed,
|
||||
},
|
||||
status=202,
|
||||
)
|
||||
|
||||
async def _removed_filter_targets(self, raw):
|
||||
"""``(targets, roots, None)`` for the removed view's filter, else an error."""
|
||||
if not isinstance(raw, dict) or raw.get("removed") != "1":
|
||||
return None, None, _bad('filter must hold removed: "1"')
|
||||
rest = {key: value for key, value in raw.items() if key != "removed"}
|
||||
try:
|
||||
parse_listing_filter(rest)
|
||||
selection = await self.resolve_listing_filter(
|
||||
{**rest, "removed": "1"}, MAX_JOB_TARGETS
|
||||
)
|
||||
except ListingSourceChanged:
|
||||
return None, None, _coded(409, "source-changed", SOURCE_CHANGED)
|
||||
except ValueError as exc:
|
||||
return None, None, _bad(str(exc))
|
||||
if selection.truncated:
|
||||
return None, None, _coded(400, "too-many-images", TOO_MANY)
|
||||
if not selection.targets:
|
||||
return None, None, _coded(400, "no-images", NONE_REMOVED)
|
||||
return tuple(selection.targets), list(self._get_all_output_dirs()), None
|
||||
|
||||
def _hidden_targets(self, roots, targets, files):
|
||||
"""``([(path, locator)], not_removed)``: the targets a mark hides now.
|
||||
|
||||
Each target is matched to its listed spelling (the walk's). One
|
||||
physical file named twice (nested roots) is returned once; a target
|
||||
that is not listed or not hidden counts in ``not_removed``.
|
||||
"""
|
||||
marks = self._removal_marks()
|
||||
listings, seen, returned, not_removed = {}, set(), [], 0
|
||||
for index, _scope, relative in targets:
|
||||
root = roots[index]
|
||||
if index not in listings:
|
||||
listings[index] = self._favorite_listing(root, files[index])
|
||||
spelled = listings[index].get(_favorite_relative_key(relative))
|
||||
if spelled is None:
|
||||
not_removed += 1
|
||||
continue
|
||||
path = os.path.join(str(root), *spelled.split("/"))
|
||||
locator = os.path.normcase(os.path.realpath(path))
|
||||
if locator in seen:
|
||||
continue
|
||||
if not mark_hides(marks, locator, path):
|
||||
not_removed += 1
|
||||
continue
|
||||
seen.add(locator)
|
||||
returned.append((Path(path), locator))
|
||||
return returned, not_removed
|
||||
|
||||
def _clear_returned_marks(self, returned) -> list:
|
||||
"""The paths whose marks this deleted (blocking; one transaction)."""
|
||||
cleared = self.db.clear_hiding_marks(
|
||||
[(locator, str(path)) for path, locator in returned]
|
||||
)
|
||||
return [path for path, locator in returned if locator in cleared]
|
||||
@@ -488,11 +488,7 @@ class ImageTagJobsMixin:
|
||||
_still_targets, self, index, root, files, seen
|
||||
)
|
||||
)
|
||||
# #480: a file removed from the library is never tagged by a batch.
|
||||
hidden = await self._run_in_executor(
|
||||
self._removed_paths, [target.path for target in found]
|
||||
)
|
||||
return [target for target in found if target.path not in hidden]
|
||||
return found
|
||||
|
||||
async def _start_autotag(
|
||||
self, *, model_type, existing, options, keep_in_memory, title, owner_kind
|
||||
@@ -531,7 +527,6 @@ class ImageTagJobsMixin:
|
||||
run = AutotagRun(
|
||||
model_type, existing, options, keep_in_memory, backup_on, owner
|
||||
)
|
||||
admitted = False
|
||||
try:
|
||||
job = self._jobs.start(
|
||||
AUTOTAG_BATCH_JOB_KIND,
|
||||
@@ -542,12 +537,11 @@ class ImageTagJobsMixin:
|
||||
groups=AUTOTAG_GROUP_NAMES,
|
||||
runner=functools.partial(self._run_autotag_batch, run),
|
||||
)
|
||||
admitted = True
|
||||
except JobBusy as refused:
|
||||
return None, _busy_refusal(refused)
|
||||
finally:
|
||||
if not admitted:
|
||||
model_gate.release(owner)
|
||||
except BaseException as refused:
|
||||
model_gate.release(owner)
|
||||
if isinstance(refused, JobBusy):
|
||||
return None, _busy_refusal(refused)
|
||||
raise
|
||||
job.task.add_done_callback(lambda _task: model_gate.release(owner))
|
||||
return job, None
|
||||
|
||||
|
||||
@@ -34,14 +34,15 @@ from .gallery_listing import (
|
||||
WalkOutcome,
|
||||
walk_root,
|
||||
)
|
||||
from . import preview_store, thumbnail_settings
|
||||
from .thumbnail_location import location_refusal
|
||||
from . import thumbnail_settings
|
||||
from .thumbnail_encode import (
|
||||
PREVIEW_HELD,
|
||||
EncoderClosed,
|
||||
PreviewWriteHeld,
|
||||
replace_unless_held,
|
||||
run_encoder,
|
||||
sweep_stray_temps,
|
||||
temp_sibling,
|
||||
thumbnail_frame,
|
||||
writer_settings,
|
||||
write_thumbnail,
|
||||
@@ -54,7 +55,6 @@ from .inspection_service import (
|
||||
IMAGE_EXTENSIONS as _IMAGE_EXTENSIONS,
|
||||
SERVABLE_EXTENSIONS,
|
||||
VIDEO_EXTENSIONS,
|
||||
image_tags_reader,
|
||||
inspect_current_file,
|
||||
)
|
||||
|
||||
@@ -65,12 +65,6 @@ try:
|
||||
from ...database.models import normalize_image_path
|
||||
from ...database.run_attribution import recorded_output_lookup
|
||||
from ...utils.card_seed import card_seed, header_params, parsed_positive
|
||||
from ...utils.media_folders import (
|
||||
in_skipped_folder,
|
||||
in_trash_below_roots,
|
||||
in_trash_folder,
|
||||
is_skipped_folder,
|
||||
)
|
||||
from ...utils.parallel import map_parallel
|
||||
from ...utils.json_strict import dumps_strict
|
||||
from ...utils.inspection_response import (
|
||||
@@ -84,7 +78,7 @@ try:
|
||||
OpenerUnavailable,
|
||||
RevealGate,
|
||||
RevealTargetChanged,
|
||||
is_same_origin_request,
|
||||
is_local_request,
|
||||
reveal_executor,
|
||||
reveal_in_file_manager,
|
||||
)
|
||||
@@ -96,12 +90,6 @@ except ImportError:
|
||||
from database.models import normalize_image_path
|
||||
from database.run_attribution import recorded_output_lookup
|
||||
from utils.card_seed import card_seed, header_params, parsed_positive
|
||||
from utils.media_folders import (
|
||||
in_skipped_folder,
|
||||
in_trash_below_roots,
|
||||
in_trash_folder,
|
||||
is_skipped_folder,
|
||||
)
|
||||
from utils.parallel import map_parallel
|
||||
from utils.json_strict import dumps_strict
|
||||
from utils.inspection_response import (
|
||||
@@ -115,7 +103,7 @@ except ImportError:
|
||||
OpenerUnavailable,
|
||||
RevealGate,
|
||||
RevealTargetChanged,
|
||||
is_same_origin_request,
|
||||
is_local_request,
|
||||
reveal_executor,
|
||||
reveal_in_file_manager,
|
||||
)
|
||||
@@ -286,20 +274,10 @@ def _forbidden(message="Access denied"):
|
||||
return web.json_response({"success": False, "error": message}, status=403)
|
||||
|
||||
|
||||
def _file_not_found():
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Image file not found"},
|
||||
status=404,
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
|
||||
def _media_access_error(path, allowed_dirs):
|
||||
"""The response refusing *path*, or None when it may be read.
|
||||
"""Return a 403 response when *path* must not be served, else None.
|
||||
|
||||
Fails closed: an empty *allowed_dirs* denies everything (403). A file in a
|
||||
trash folder below its root is thrown away, not an output: a 404, as if it
|
||||
were not there (#543), also through a link or another spelling.
|
||||
Fails closed: an empty *allowed_dirs* denies everything.
|
||||
"""
|
||||
if Path(path).suffix.lower() not in SERVABLE_EXTENSIONS:
|
||||
return _forbidden("Only media files can be served")
|
||||
@@ -307,8 +285,6 @@ def _media_access_error(path, allowed_dirs):
|
||||
return _forbidden("No allowed image directories configured")
|
||||
if not any(_is_within(path, root) for root in allowed_dirs):
|
||||
return _forbidden()
|
||||
if in_trash_below_roots(path, allowed_dirs):
|
||||
return _file_not_found()
|
||||
return None
|
||||
|
||||
|
||||
@@ -338,6 +314,11 @@ def _thumbnail_batch_size(workers):
|
||||
return max(4, workers * 2)
|
||||
|
||||
|
||||
# Preview files are written by thumbnail_encode in the configured format
|
||||
# (thumbnail_settings, #317); these names stay importable from here.
|
||||
_temp_sibling = temp_sibling
|
||||
_replace_unless_held = replace_unless_held
|
||||
|
||||
# A preview that could not be replaced (#249 S5 M6). Windows refuses to rename
|
||||
# over a file another process has open (WinError 5 or 32), such as a preview
|
||||
# the gallery is serving; the old preview stays and the file counts as an error.
|
||||
@@ -390,7 +371,6 @@ def _video_placeholder_image(thumbnail_size):
|
||||
font=font,
|
||||
)
|
||||
except (OSError, AttributeError):
|
||||
# a placeholder without its label still works
|
||||
pass
|
||||
return img
|
||||
|
||||
@@ -494,7 +474,6 @@ def _preview_names(folders, folder):
|
||||
except OSError:
|
||||
continue
|
||||
except OSError:
|
||||
# an unreadable folder lists as empty; nothing there to cache
|
||||
pass
|
||||
folders[folder] = names
|
||||
return names
|
||||
@@ -813,12 +792,12 @@ def _reveal_not_found():
|
||||
)
|
||||
|
||||
|
||||
def _not_same_origin():
|
||||
def _not_local():
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": "not-same-origin",
|
||||
"error": "This page's address does not match the server's.",
|
||||
"code": "not-local",
|
||||
"error": "Show in folder works only on the computer running ComfyUI.",
|
||||
},
|
||||
status=403,
|
||||
)
|
||||
@@ -935,8 +914,7 @@ def _name_suffix(name):
|
||||
def _walk_media(output_path, max_files):
|
||||
"""Yield the ``os.DirEntry`` of each media file under *output_path*.
|
||||
|
||||
Symlinks are never followed, ``thumbnails/`` and trash folders are
|
||||
skipped (``utils.media_folders``, #501), descent stops
|
||||
Symlinks are never followed, ``thumbnails/`` is skipped, descent stops
|
||||
at MAX_SCAN_DEPTH and at most *max_files* entries are produced. The order
|
||||
is ``os.walk``'s (top-down, subfolders in ``os.scandir`` order, files by
|
||||
name), which the gallery's ties in mtime keep. Each entry's type comes
|
||||
@@ -948,6 +926,8 @@ def _walk_media(output_path, max_files):
|
||||
produced = 0
|
||||
while stack:
|
||||
top, rel_parts = stack.pop()
|
||||
if "thumbnails" in rel_parts:
|
||||
continue
|
||||
dirs, files = [], []
|
||||
try:
|
||||
with os.scandir(top) as entries:
|
||||
@@ -971,7 +951,7 @@ def _walk_media(output_path, max_files):
|
||||
if len(rel_parts) >= MAX_SCAN_DEPTH:
|
||||
continue
|
||||
for entry in reversed(dirs):
|
||||
if not entry.is_symlink() and not is_skipped_folder(entry.name):
|
||||
if not entry.is_symlink():
|
||||
stack.append((entry.path, rel_parts + (entry.name,)))
|
||||
|
||||
|
||||
@@ -995,10 +975,8 @@ def _settled_entry(root, relative):
|
||||
*folders, name = relative.split(os.sep)
|
||||
if _name_suffix(name).lower() not in MEDIA_EXTENSIONS:
|
||||
return None
|
||||
if in_skipped_folder(folders) or len(folders) > MAX_SCAN_DEPTH:
|
||||
if "thumbnails" in folders or len(folders) > MAX_SCAN_DEPTH:
|
||||
return None
|
||||
if in_trash_below_roots(path, [root]):
|
||||
return None # a linked folder on the way leads into a trash folder
|
||||
folder = os.fspath(root)
|
||||
for part in ("", *folders):
|
||||
folder = os.path.join(folder, part) if part else folder
|
||||
@@ -1069,24 +1047,13 @@ def _iter_media_files(output_path, max_files):
|
||||
|
||||
|
||||
def media_thumbnail_rel(rel_path, settings=None):
|
||||
"""``thumbnails/<rel>_thumb<ext>`` (POSIX URL text) for the media file
|
||||
*rel_path*: the name the preview gets when it is written now (#317),
|
||||
images and videos alike (``writer_settings``' extension). Where it is on
|
||||
disk is ``media_preview_path``."""
|
||||
"""``thumbnails/<rel>_thumb<ext>`` (POSIX) for the media file *rel_path*:
|
||||
the name the preview gets when it is written now (#317), images and
|
||||
videos alike (``writer_settings``' extension)."""
|
||||
settings = settings or writer_settings()
|
||||
return thumbnail_settings.thumbnail_rel(rel_path, settings.extension)
|
||||
|
||||
|
||||
def media_preview_path(output_path, rel_path, settings=None, *, create=False):
|
||||
"""Where the preview of *rel_path* is written now (``preview_store``);
|
||||
*create* for a writer (a new source gets its key). None when nothing
|
||||
may be written (the thumbnail folder is unavailable)."""
|
||||
settings = settings or writer_settings()
|
||||
return preview_store.preview_path(
|
||||
output_path, rel_path, settings.extension, create=create
|
||||
)
|
||||
|
||||
|
||||
def existing_media_thumbnail(output_path, rel_path):
|
||||
"""The existing preview *rel_path* owns that a listing or cover shows
|
||||
(the format written now first, then older ones), or None. The Images
|
||||
@@ -1096,27 +1063,6 @@ def existing_media_thumbnail(output_path, rel_path):
|
||||
)
|
||||
|
||||
|
||||
def listing_thumbnail_url(output_path, rel_path):
|
||||
"""The thumbnail URL of an older listing (``_enrich_images``, the
|
||||
auto-tag scan), or None. These run on the event loop: in folder mode
|
||||
(#320) the store is never touched there, so the URL names the preview
|
||||
as it is written now and the serve route finds or makes it off the loop
|
||||
(None while the thumbnail folder is unavailable)."""
|
||||
from .prompts import thumbnail_url_for
|
||||
|
||||
location = preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE:
|
||||
if not location.available:
|
||||
return None
|
||||
quoted = urllib.parse.quote(media_thumbnail_rel(rel_path), safe="/")
|
||||
return f"/prompt_manager/images/serve/{quoted}"
|
||||
existing = thumbnail_settings.existing_thumbnail(output_path, rel_path)
|
||||
if existing is None:
|
||||
return None
|
||||
thumb_rel = thumbnail_settings.thumbnail_stem_rel(rel_path) + existing.suffix
|
||||
return thumbnail_url_for(thumb_rel, existing)
|
||||
|
||||
|
||||
def media_thumbnail(output_path, rel_path, root_index, *, root_scope=None):
|
||||
"""``(thumbnail_url, ready)`` for *rel_path* under *output_path* (a few
|
||||
blocking stats). The Images listing, the Needs review queue cards and the
|
||||
@@ -1212,10 +1158,7 @@ def _workspace_filters(query: Mapping) -> dict:
|
||||
the linked positive, and ``favorites=1`` keeps only favorites (the
|
||||
Favorites page, #249 D51). ``collection=<id>`` (1 to 9 digits) keeps only
|
||||
that collection's members (#331); it wins over ``favorites`` (a page never
|
||||
sets both). ``removed=1`` lists only the files a removal mark hides (#498,
|
||||
Show removed images); such a file has no records, so it never combines
|
||||
with ``favorites``, ``collection``, ``category``, ``tags`` or the rating
|
||||
sort. Raises ValueError for any unknown or oversized value.
|
||||
sets both). Raises ValueError for any unknown or oversized value.
|
||||
"""
|
||||
text = query.get("q", "").strip()
|
||||
folded = text.casefold()
|
||||
@@ -1230,12 +1173,6 @@ def _workspace_filters(query: Mapping) -> dict:
|
||||
)
|
||||
favorites = query.get("favorites", "0")
|
||||
collection = query.get("collection", "")
|
||||
removed = query.get("removed", "0")
|
||||
if removed not in ("0", "1") or (
|
||||
removed == "1"
|
||||
and (favorites == "1" or collection or category or tags or order == "rating")
|
||||
):
|
||||
raise ValueError("Invalid gallery filters")
|
||||
if (
|
||||
favorites not in ("0", "1")
|
||||
or (collection != "" and _COLLECTION_ID.fullmatch(collection) is None)
|
||||
@@ -1263,7 +1200,6 @@ def _workspace_filters(query: Mapping) -> dict:
|
||||
"tags": tags,
|
||||
"favorites": favorites == "1" and not collection,
|
||||
"collection": int(collection) if collection else None,
|
||||
"removed": removed == "1",
|
||||
}
|
||||
|
||||
|
||||
@@ -1566,7 +1502,7 @@ class ImageRoutesMixin:
|
||||
async def clear_image_favorite_route(request):
|
||||
return await self.update_image_favorite(request, favorite=False)
|
||||
|
||||
# POST, same-origin: opens the server computer's file manager (D38).
|
||||
# POST, local-only: opens this computer's file manager (D38).
|
||||
@routes.post("/prompt_manager/images/reveal")
|
||||
async def reveal_image_route(request):
|
||||
return await self.reveal_image(request)
|
||||
@@ -2135,7 +2071,6 @@ class ImageRoutesMixin:
|
||||
filters["tags"],
|
||||
filters["favorites"],
|
||||
filters["collection"],
|
||||
filters["removed"],
|
||||
]
|
||||
).encode()
|
||||
).hexdigest()
|
||||
@@ -2165,11 +2100,6 @@ class ImageRoutesMixin:
|
||||
self._select_workspace_page, listings, filters, after, limit, scope
|
||||
)
|
||||
)
|
||||
# #498: Show removed images (N), every removed file in the selected
|
||||
# source(s) whatever the filters (the hidden set is cached work).
|
||||
removed_total = len(
|
||||
await self._run_in_executor(self._hidden_entries, listings)
|
||||
)
|
||||
except CollectionMissing:
|
||||
return collection_not_found()
|
||||
except Exception:
|
||||
@@ -2196,7 +2126,6 @@ class ImageRoutesMixin:
|
||||
"limit": limit,
|
||||
"has_more": has_more,
|
||||
"next_cursor": next_cursor,
|
||||
"removed_total": removed_total,
|
||||
}
|
||||
if positive_count is _COUNT_OMITTED:
|
||||
del result["positive_count"]
|
||||
@@ -2520,13 +2449,6 @@ class ImageRoutesMixin:
|
||||
def key(entry):
|
||||
return _workspace_key(entry, order)
|
||||
|
||||
# #480: files removed from the library stay on disk, hidden here; the
|
||||
# removed view (#498) lists only them.
|
||||
hidden = self._hidden_entries(listings)
|
||||
if filters.get("removed"):
|
||||
entries = [entry for entry in entries if entry.normalized in hidden]
|
||||
elif hidden:
|
||||
entries = [entry for entry in entries if entry.normalized not in hidden]
|
||||
return (
|
||||
entries,
|
||||
key,
|
||||
@@ -2813,9 +2735,6 @@ class ImageRoutesMixin:
|
||||
if _has_traversal(filepath):
|
||||
return _forbidden()
|
||||
suffix = Path(filepath).suffix.lower()
|
||||
if in_trash_folder(filepath):
|
||||
# Before any file access, so no preview is made for it (#543).
|
||||
return _file_not_found()
|
||||
if suffix not in SERVABLE_EXTENSIONS and not (
|
||||
# A JPEG XL preview is served from thumbnails/ only (#317).
|
||||
suffix in thumbnail_settings.THUMBNAIL_EXTENSIONS
|
||||
@@ -2831,59 +2750,14 @@ class ImageRoutesMixin:
|
||||
output_dirs[0], filepath, int(root_param), len(outputs)
|
||||
)
|
||||
|
||||
# Preview text is mapped by the resolver (#320); any other text,
|
||||
# ``Thumbnails/...`` media on a case-sensitive disk included,
|
||||
# keeps the plain join.
|
||||
preview_parts = (
|
||||
tuple(filepath.split("/"))
|
||||
if preview_store.is_preview_text(filepath)
|
||||
else None
|
||||
)
|
||||
location = preview_store.current()
|
||||
if location.mode == preview_store.FOLDER_MODE:
|
||||
if preview_parts is None and await self._run_image(
|
||||
self._in_source_thumbnails, output_dirs, filepath
|
||||
):
|
||||
# Previews live in the thumbnail folder: anything else
|
||||
# under a source's thumbnails/ is a leftover (#320), in
|
||||
# any case on a case-insensitive disk.
|
||||
return _forbidden()
|
||||
if preview_parts is not None:
|
||||
# Store I/O never runs on the event loop (a share may
|
||||
# stall): the lookup runs on the image pool.
|
||||
found = await self._run_image(
|
||||
self._find_preview, output_dirs, preview_parts, location
|
||||
)
|
||||
if found == "forbidden":
|
||||
return _forbidden()
|
||||
if found is not None:
|
||||
return self._file_response(found)
|
||||
for output_path in output_dirs:
|
||||
if (
|
||||
preview_parts is not None
|
||||
and location.mode == preview_store.FOLDER_MODE
|
||||
):
|
||||
break # looked up above
|
||||
if preview_parts is None:
|
||||
image_path = output_path / filepath
|
||||
else:
|
||||
image_path = preview_store.request_path(output_path, preview_parts)
|
||||
if image_path is None:
|
||||
continue
|
||||
image_path = output_path / filepath
|
||||
if not image_path.exists():
|
||||
continue
|
||||
# A symlink that points outside the root (a preview: outside
|
||||
# the folder holding its store) is a hard deny, not a
|
||||
# fall-through to the next root.
|
||||
container = (
|
||||
output_path
|
||||
if preview_parts is None
|
||||
else preview_store.store_container(output_path)
|
||||
)
|
||||
if not _is_within(image_path, container):
|
||||
# A symlink that points outside the root is a hard deny, not
|
||||
# a fall-through to the next root.
|
||||
if not _is_within(image_path, output_path):
|
||||
return _forbidden()
|
||||
if in_trash_below_roots(image_path, output_dirs):
|
||||
return _file_not_found()
|
||||
if image_path.is_file():
|
||||
return self._file_response(image_path)
|
||||
|
||||
@@ -2906,48 +2780,6 @@ class ImageRoutesMixin:
|
||||
{"success": False, "error": _safe_error(e)}, status=500
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _in_source_thumbnails(output_dirs, filepath):
|
||||
"""Whether *filepath* lies in a source's ``thumbnails`` folder: its
|
||||
first part is ``thumbnails`` exactly, or another case of it that is
|
||||
the same folder (a case-insensitive disk). ``Thumbnails/`` media on
|
||||
a case-sensitive disk is another folder and stays served."""
|
||||
first = filepath.split("/", 1)[0]
|
||||
if first == "thumbnails":
|
||||
return True
|
||||
if first.casefold() != "thumbnails":
|
||||
return False
|
||||
for output_path in output_dirs:
|
||||
spelled = output_path / first
|
||||
cache = preview_store.source_folder(output_path)
|
||||
try:
|
||||
if spelled.is_dir() and cache.is_dir() and spelled.samefile(cache):
|
||||
return True
|
||||
except OSError:
|
||||
continue
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _find_preview(output_dirs, parts, location):
|
||||
"""The preview file *parts* names in folder mode, ``"forbidden"``
|
||||
(it resolves out of its store or through a link) or None (blocking;
|
||||
the first root whose store holds it, in ``_select_root`` order)."""
|
||||
for output_path in output_dirs:
|
||||
image_path = preview_store.request_path(output_path, parts, location)
|
||||
if image_path is None:
|
||||
continue
|
||||
try:
|
||||
with preview_store.store_io():
|
||||
if not os.path.lexists(image_path):
|
||||
continue
|
||||
if not preview_store.owns(output_path, image_path, location):
|
||||
return "forbidden"
|
||||
if image_path.is_file():
|
||||
return image_path
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
def _is_recorded_media(self, path) -> bool:
|
||||
"""Whether the database records ``path`` as a linked image.
|
||||
|
||||
@@ -2977,9 +2809,8 @@ class ImageRoutesMixin:
|
||||
if _is_thumbnail_cache_path(filepath):
|
||||
return _forbidden()
|
||||
image_path = root / filepath
|
||||
denied = _media_access_error(image_path, [root])
|
||||
if denied is not None:
|
||||
return denied
|
||||
if not _is_within(image_path, root):
|
||||
return _forbidden()
|
||||
if not image_path.is_file():
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Image file not found"}, status=404
|
||||
@@ -3074,9 +2905,6 @@ class ImageRoutesMixin:
|
||||
data, error = await _json_body_or_error(request)
|
||||
if error is not None:
|
||||
return error
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
return await self._run_held(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
THUMBNAIL_GENERATION_TITLE,
|
||||
@@ -3096,7 +2924,6 @@ class ImageRoutesMixin:
|
||||
result = await self._run_in_executor(
|
||||
self._generate_gallery_thumbnails_sync, output_dirs
|
||||
)
|
||||
await self._release_job_memory()
|
||||
return web.json_response(result)
|
||||
|
||||
except Exception as e:
|
||||
@@ -3109,7 +2936,6 @@ class ImageRoutesMixin:
|
||||
"""Blocking thumbnail generation loop (run in executor)."""
|
||||
settings = writer_settings(settings)
|
||||
start = _time.monotonic()
|
||||
thumbnails_dir = preview_store.given_store(output_path, thumbnails_dir)
|
||||
plan = self._plan_thumbnails(output_path, thumbnails_dir, settings=settings)
|
||||
payload = self._execute_thumbnail_plan(plan, settings, start)
|
||||
payload["thumbnails_path"] = _public_path(thumbnails_dir)
|
||||
@@ -3122,8 +2948,7 @@ class ImageRoutesMixin:
|
||||
start = _time.monotonic()
|
||||
plan = self._plan_gallery_thumbnails(output_dirs, settings=settings)
|
||||
payload = self._execute_thumbnail_plan(plan, settings, start)
|
||||
store = preview_store.store_dir(output_dirs[0])
|
||||
payload["thumbnails_path"] = None if store is None else _public_path(store)
|
||||
payload["thumbnails_path"] = _public_path(output_dirs[0] / "thumbnails")
|
||||
return payload
|
||||
|
||||
def _execute_thumbnail_plan(self, plan: dict, settings, start: float) -> dict:
|
||||
@@ -3166,12 +2991,9 @@ class ImageRoutesMixin:
|
||||
if stop is not None and stop():
|
||||
break
|
||||
root = Path(root)
|
||||
store = preview_store.ensure_store(root)
|
||||
if store is None:
|
||||
continue # the thumbnail folder is unavailable
|
||||
plan = self._plan_thumbnails(
|
||||
root,
|
||||
store,
|
||||
root / "thumbnails",
|
||||
mode=mode,
|
||||
stop=stop,
|
||||
settings=settings,
|
||||
@@ -3210,8 +3032,6 @@ class ImageRoutesMixin:
|
||||
"format": writer_settings().format,
|
||||
# Files whose automatic preview failed this session (#249).
|
||||
"failed": self._auto_thumbnail_failures(),
|
||||
# #320: where previews live (no path here, D6A).
|
||||
"location": self._thumbnail_location_payload(with_path=False),
|
||||
"collection_id": hashlib.sha256(collection.encode()).hexdigest()[
|
||||
:16
|
||||
],
|
||||
@@ -3250,27 +3070,20 @@ class ImageRoutesMixin:
|
||||
no symlinks), each preview folder is listed once, and at most
|
||||
MAX_THUMBNAIL_FILES targets are queued per batch (a job plans again
|
||||
after each batch); ``remaining`` counts the targets beyond that cap.
|
||||
Sources in *exclude* (worked on by an earlier batch) are left out, and
|
||||
so are files removed from the library (#501: a removal mark hides
|
||||
them, and their previews went with their records).
|
||||
Sources in *exclude* (worked on by an earlier batch) are left out.
|
||||
``stop()`` true ends the walk before the next file.
|
||||
"""
|
||||
settings = writer_settings(settings)
|
||||
output_path = Path(output_path)
|
||||
thumbnails_dir = preview_store.given_store(output_path, thumbnails_dir)
|
||||
thumbnails_dir = Path(thumbnails_dir)
|
||||
plan = {key: 0 for key in _PLAN_COUNTS}
|
||||
plan.update(targets=[], fresh_files=[])
|
||||
if thumbnails_dir is None:
|
||||
return plan # the thumbnail folder is unavailable
|
||||
folders = {}
|
||||
removed = self._removal_check_under(output_path)
|
||||
for src in _iter_media_files(output_path, MAX_GALLERY_FILES):
|
||||
if stop is not None and stop():
|
||||
break
|
||||
if exclude and src in exclude:
|
||||
continue
|
||||
if removed(src):
|
||||
continue
|
||||
rel_no_ext = src.relative_to(output_path).with_suffix("")
|
||||
stem = f"{rel_no_ext.name}{thumbnail_settings.THUMB_SUFFIX}"
|
||||
dst = thumbnails_dir / rel_no_ext.parent / (stem + settings.extension)
|
||||
@@ -3351,9 +3164,6 @@ class ImageRoutesMixin:
|
||||
"dir": src.parent.name,
|
||||
"type": "video" if is_video else "image",
|
||||
}
|
||||
# Taken when the write starts: a thumbnail location switched since
|
||||
# then drops the write at its rename (#320).
|
||||
guard = preview_store.write_guard(guard)
|
||||
dst = Path(dst)
|
||||
if dst.suffix != settings.extension:
|
||||
dst = dst.with_suffix(settings.extension)
|
||||
@@ -3364,7 +3174,7 @@ class ImageRoutesMixin:
|
||||
return {**result, "action": "error", "error": "File no longer exists"}
|
||||
if not rebuild and _thumbnail_is_fresh(src, dst):
|
||||
return {**result, "action": "skipped", "path": str(dst)}
|
||||
preview_store.make_parent(dst)
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if is_video:
|
||||
made = self._generate_video_thumbnail(
|
||||
src,
|
||||
@@ -3569,12 +3379,8 @@ class ImageRoutesMixin:
|
||||
|
||||
Invalid thumbnail directories return 400; unavailable gallery roots
|
||||
return 404, and unexpected filesystem failures return 500. Holds the
|
||||
registry as guarded work, like generation. Refused (409) while the
|
||||
thumbnail folder is unavailable (#320).
|
||||
registry as guarded work, like generation.
|
||||
"""
|
||||
refused = location_refusal()
|
||||
if refused is not None:
|
||||
return refused
|
||||
return await self._run_held(
|
||||
THUMBNAILS_JOB_KIND,
|
||||
THUMBNAIL_CLEAR_TITLE,
|
||||
@@ -3638,12 +3444,8 @@ class ImageRoutesMixin:
|
||||
self.logger.debug(f"No roots to sweep for preview temp files: {exc}")
|
||||
return 0
|
||||
for output_path in output_dirs:
|
||||
folder = preview_store.store_dir(output_path)
|
||||
if (
|
||||
folder is None
|
||||
or preview_store.store_is_link(folder)
|
||||
or not folder.is_dir()
|
||||
):
|
||||
folder = Path(output_path) / "thumbnails"
|
||||
if folder.is_symlink() or not folder.is_dir():
|
||||
continue
|
||||
removed += sweep_stray_temps(folder)[0]
|
||||
if removed:
|
||||
@@ -3661,16 +3463,13 @@ class ImageRoutesMixin:
|
||||
directories = []
|
||||
for output_path in output_dirs:
|
||||
output_path = Path(output_path)
|
||||
thumbnails_dir = preview_store.store_dir(output_path)
|
||||
if thumbnails_dir is None:
|
||||
continue
|
||||
thumbnails_dir = output_path / "thumbnails"
|
||||
if not thumbnails_dir.exists() and not thumbnails_dir.is_symlink():
|
||||
continue
|
||||
container = preview_store.store_container(output_path)
|
||||
if (
|
||||
preview_store.store_is_link(thumbnails_dir)
|
||||
or not _is_within(thumbnails_dir, container)
|
||||
or _canonical(thumbnails_dir) == _canonical(container)
|
||||
thumbnails_dir.is_symlink()
|
||||
or not _is_within(thumbnails_dir, output_path)
|
||||
or _canonical(thumbnails_dir) == _canonical(output_path)
|
||||
):
|
||||
raise ValueError(
|
||||
"Safety check failed: invalid thumbnails directory path"
|
||||
@@ -3685,15 +3484,7 @@ class ImageRoutesMixin:
|
||||
temps, temp_bytes = sweep_stray_temps(thumbnails_dir, min_age=0)
|
||||
cleared_count += temps
|
||||
cleared_size += temp_bytes
|
||||
folder_mode = preview_store.current().mode == preview_store.FOLDER_MODE
|
||||
for root, dirs, files in os.walk(thumbnails_dir):
|
||||
if folder_mode:
|
||||
# Never into a junction or reparse point (#320).
|
||||
dirs[:] = [
|
||||
d
|
||||
for d in dirs
|
||||
if not preview_store.store_is_link(Path(root) / d)
|
||||
]
|
||||
for root, _dirs, files in os.walk(thumbnails_dir):
|
||||
for name in files:
|
||||
path = Path(root) / name
|
||||
if (
|
||||
@@ -3701,7 +3492,6 @@ class ImageRoutesMixin:
|
||||
or path.suffix.lower()
|
||||
not in thumbnail_settings.THUMBNAIL_EXTENSIONS
|
||||
or path.is_symlink()
|
||||
or not preview_store.may_write(path)
|
||||
):
|
||||
continue
|
||||
try:
|
||||
@@ -3861,12 +3651,12 @@ class ImageRoutesMixin:
|
||||
async def reveal_image(self, request):
|
||||
"""POST ``{root_index, root_scope, path}``: show the file in its folder.
|
||||
|
||||
Same-origin and JSON-only, directly or through a proxy: the folder
|
||||
opens on the computer running ComfyUI, wherever the browser is
|
||||
(maintainer decision, #533). One reveal at a time (429 busy).
|
||||
Local-only (a same-origin browser request from a loopback client, no
|
||||
forwarding header) and JSON-only: the browser must run on the
|
||||
computer whose file manager opens. One reveal at a time (429 busy).
|
||||
"""
|
||||
if not is_same_origin_request(request.headers, request.scheme):
|
||||
return _not_same_origin()
|
||||
if not is_local_request(request.remote, request.headers, request.scheme):
|
||||
return _not_local()
|
||||
if request.content_type != "application/json":
|
||||
# A non-JSON POST is a CORS "simple" request with no preflight.
|
||||
return web.json_response(
|
||||
@@ -3944,9 +3734,7 @@ class ImageRoutesMixin:
|
||||
{
|
||||
"success": False,
|
||||
"code": "unavailable",
|
||||
"error": (
|
||||
"No file manager is available on the computer running ComfyUI."
|
||||
),
|
||||
"error": "No file manager is available on this computer.",
|
||||
},
|
||||
status=501,
|
||||
)
|
||||
@@ -4151,8 +3939,7 @@ class ImageRoutesMixin:
|
||||
**observed,
|
||||
)
|
||||
|
||||
# Header and text reads: the interactive image pool (#523).
|
||||
body = await self._run_image(inspect)
|
||||
body = await self._run_in_executor(inspect)
|
||||
if body is None:
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -4175,9 +3962,28 @@ class ImageRoutesMixin:
|
||||
)
|
||||
|
||||
def _inspection_image_tags(self, root: Path, parts: tuple) -> Callable:
|
||||
"""The inspector's ``image_tags`` reader for one rooted file (S1)."""
|
||||
"""The inspector's ``image_tags`` reader for one rooted file (S1).
|
||||
|
||||
The returned function takes the inspection's own stable stat and
|
||||
answers ``current`` only for a byte record verified at exactly that
|
||||
``(size, mtime_ns)``; a failed read is ``unavailable``, never an error.
|
||||
"""
|
||||
locator = os.path.normcase(os.path.join(os.path.realpath(root), *parts))
|
||||
return image_tags_reader(self.db, locator, self.logger)
|
||||
|
||||
def read(size: int, mtime_ns: int) -> dict:
|
||||
try:
|
||||
found = self.db.current_image_tags({locator: (size, mtime_ns)})
|
||||
except Exception:
|
||||
self.logger.warning(
|
||||
"Inspection image tags could not be read", exc_info=True
|
||||
)
|
||||
return {"state": "unavailable", "tags": []}
|
||||
tags = found.get(locator)
|
||||
if tags is None:
|
||||
return {"state": "none", "tags": []}
|
||||
return {"state": "current", "tags": list(tags)}
|
||||
|
||||
return read
|
||||
|
||||
def _read_own_category(self, sources, path):
|
||||
"""One file's own category, None, or OWN_CATEGORY_UNREAD (blocking).
|
||||
@@ -4365,22 +4171,6 @@ class ImageRoutesMixin:
|
||||
# (which Windows refuses on open files) is not held up by it.
|
||||
return _CardFacts(width, height, *self._card_parse(path, records))
|
||||
|
||||
def _cached_card_facts(self, entry: tuple) -> _CardFacts | None:
|
||||
path, mtime, root, index = entry[:4]
|
||||
return self._workspace_card_cache.get((index, str(root)), {}).get((path, mtime))
|
||||
|
||||
def _card_facts_of(self, entries: list, *, parse: bool) -> list[_CardFacts]:
|
||||
"""Card facts of *entries*, in order; files not cached yet are read on
|
||||
the interactive image pool (#523), cached ones answer here."""
|
||||
found = [self._cached_card_facts(entry) for entry in entries]
|
||||
missing = [i for i, facts in enumerate(found) if facts is None]
|
||||
read = self._map_image(
|
||||
lambda i: self._card_facts(entries[i], parse=parse), missing
|
||||
)
|
||||
for i, facts in zip(missing, read):
|
||||
found[i] = facts
|
||||
return found
|
||||
|
||||
def _card_facts(self, entry: tuple, *, parse: bool) -> _CardFacts:
|
||||
"""One listed image's card facts, read once per ``(path, mtime)``.
|
||||
|
||||
@@ -4389,11 +4179,11 @@ class ImageRoutesMixin:
|
||||
and drops the rest, and a roots change drops them all. Only files of
|
||||
the current scan are stored, so the cache is bounded by the listing.
|
||||
"""
|
||||
cached = self._cached_card_facts(entry)
|
||||
if cached is not None:
|
||||
return cached
|
||||
path, mtime, root, index = entry[:4]
|
||||
root_key, key = (index, str(root)), (path, mtime)
|
||||
cached = self._workspace_card_cache.get(root_key, {}).get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
facts = self._read_card_facts(path, parse=parse)
|
||||
if facts is None:
|
||||
# Not stored: a locked or half-written file may read next time.
|
||||
@@ -4419,17 +4209,15 @@ class ImageRoutesMixin:
|
||||
if linked
|
||||
else {}
|
||||
)
|
||||
# Videos: the card reads no video metadata. Library previews show
|
||||
# no card facts, so they skip the parse (``parse=linked``).
|
||||
stills = [image for image in images if not image["is_video"]]
|
||||
read = self._card_facts_of(
|
||||
[entries[(image["root_index"], image["path"])] for image in stills],
|
||||
parse=linked,
|
||||
)
|
||||
facts_of = {id(image): facts for image, facts in zip(stills, read)}
|
||||
for image in images:
|
||||
entry = entries[(image["root_index"], image["path"])]
|
||||
facts = facts_of.get(id(image), _NO_CARD_FACTS)
|
||||
# Videos: the card reads no video metadata. Library previews show
|
||||
# no card facts, so they skip the parse (``parse=linked``).
|
||||
facts = (
|
||||
_NO_CARD_FACTS
|
||||
if image["is_video"]
|
||||
else self._card_facts(entry, parse=linked)
|
||||
)
|
||||
image["width"], image["height"] = facts.width, facts.height
|
||||
if not linked:
|
||||
continue
|
||||
|
||||
@@ -93,35 +93,6 @@ def _in_host_temp(path: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def image_tags_reader(
|
||||
db: Any, locator: str, logger: Any, *, close: bool = False
|
||||
) -> Callable[[int, int], dict]:
|
||||
"""The inspector's ``image_tags`` reader for one file *locator* (S1).
|
||||
|
||||
The returned function takes the inspection's own stable stat and answers
|
||||
``current`` only for a byte record verified at exactly that
|
||||
``(size, mtime_ns)``; a failed read is ``unavailable``, never an error.
|
||||
``close`` closes the calling thread's connection afterwards, for routes
|
||||
that close theirs after reading (History).
|
||||
"""
|
||||
|
||||
def read(size: int, mtime_ns: int) -> dict:
|
||||
try:
|
||||
found = db.current_image_tags({locator: (size, mtime_ns)})
|
||||
except Exception:
|
||||
logger.warning("Inspection image tags could not be read", exc_info=True)
|
||||
return {"state": "unavailable", "tags": []}
|
||||
finally:
|
||||
if close:
|
||||
db.model.close()
|
||||
tags = found.get(locator)
|
||||
if tags is None:
|
||||
return {"state": "none", "tags": []}
|
||||
return {"state": "current", "tags": list(tags)}
|
||||
|
||||
return read
|
||||
|
||||
|
||||
def inspect_current_file(
|
||||
path: Path,
|
||||
*,
|
||||
|
||||
@@ -250,7 +250,7 @@ class JobRoutesMixin:
|
||||
try:
|
||||
await response.write(_summary_event(job))
|
||||
async with contextlib.aclosing(job.events()) as events:
|
||||
async for _ in events:
|
||||
async for _payload in events:
|
||||
await response.write(_summary_event(job))
|
||||
# The last event is published before the job takes its final state.
|
||||
await response.write(_summary_event(job))
|
||||
|
||||
@@ -123,11 +123,6 @@ MODEL_DOWNLOAD_JOB_KIND = "model_download"
|
||||
DUPLICATES_JOB_KIND = "duplicates"
|
||||
DUPLICATE_GROUP_NAMES = ("groups_found",)
|
||||
DUPLICATE_CLEANUP_GROUP_NAMES = ("removed", "kept", "errors")
|
||||
# The workspace Library's Delete… (#443, py/api/library_delete.py): a
|
||||
# ``records`` hold around its safety backup and write.
|
||||
PROMPT_DELETE_JOB_KIND = "prompt_delete"
|
||||
IMAGE_DELETE_JOB_KIND = "image_delete" # #480
|
||||
IMAGE_DELETE_GROUP_NAMES = ("trashed", "removed", "not_changed", "errors")
|
||||
# The System health check (py/api/system.py): a read-only ``read`` job with
|
||||
# no file outcomes, so it may run beside a scan.
|
||||
HEALTH_JOB_KIND = "health"
|
||||
|
||||
@@ -21,11 +21,9 @@ from .pairs import pair_response
|
||||
try:
|
||||
from ...database.library import ARCHIVE_SCOPES
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...database.library_folders import folder_prefixes, normalize_folder
|
||||
except ImportError:
|
||||
from database.library import ARCHIVE_SCOPES
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.library_folders import folder_prefixes, normalize_folder
|
||||
|
||||
_SECRET = os.urandom(32)
|
||||
|
||||
@@ -157,13 +155,6 @@ def _archive_scope(request: web.Request) -> str:
|
||||
return value
|
||||
|
||||
|
||||
def _shelf(request: web.Request) -> str:
|
||||
value = request.query.get("shelf", "active")
|
||||
if value not in ("active", "archived"):
|
||||
raise ValueError("shelf must be active or archived")
|
||||
return value
|
||||
|
||||
|
||||
def _missing() -> web.Response:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Positive prompt not found"}, status=404
|
||||
@@ -292,87 +283,36 @@ class LibraryRoutesMixin:
|
||||
outputs = list(self._get_all_output_dirs())
|
||||
return self._media_roots(outputs), len(outputs)
|
||||
|
||||
def _library_root_spellings(self) -> list[str]:
|
||||
"""Every spelling of each output root (#444 folder filter): the
|
||||
validated, resolved root and each configured or detected path that
|
||||
resolves to it, so images saved through a symlinked root match."""
|
||||
from ..config import GalleryConfig
|
||||
|
||||
roots = [str(root) for root in self._get_all_output_dirs()]
|
||||
candidates = [
|
||||
*GalleryConfig.MONITORING_DIRECTORIES,
|
||||
self._find_comfyui_output_dir(),
|
||||
*self._comfyui_output_dir_list(),
|
||||
]
|
||||
spellings = []
|
||||
for root in roots:
|
||||
for value in (root, *candidates):
|
||||
if not value or not isinstance(value, (str, os.PathLike)):
|
||||
continue
|
||||
value = str(value)
|
||||
same = value == root or os.path.normcase(
|
||||
os.path.realpath(value)
|
||||
) == os.path.normcase(root)
|
||||
if same and value not in spellings:
|
||||
spellings.append(value)
|
||||
return spellings
|
||||
|
||||
@pair_response
|
||||
async def get_library_options(self, request: web.Request) -> web.Response:
|
||||
include = request.query.get("include", "")
|
||||
if include not in ("", "folders"):
|
||||
raise ValueError("include must be empty or 'folders'")
|
||||
if include:
|
||||
# Folders are read only when Refine opens (a walk of linked paths).
|
||||
roots = await self._run_in_executor(self._library_root_spellings)
|
||||
result = await self._run_in_executor(self.db.get_library_options, roots)
|
||||
else:
|
||||
result = await self._run_in_executor(self.db.get_library_options)
|
||||
result = await self._run_in_executor(self.db.get_library_options)
|
||||
return web.json_response({"success": True, **result})
|
||||
|
||||
@pair_response
|
||||
async def list_library(self, request: web.Request) -> web.Response:
|
||||
folder = normalize_folder(request.query.get("folder", ""))
|
||||
options = {
|
||||
"text": request.query.get("text", ""),
|
||||
"category": request.query.get("category"),
|
||||
"tags": sorted(set(request.query.getall("tag", []))),
|
||||
"tag_mode": request.query.get("tag_mode", "all"),
|
||||
"sort": request.query.get("sort", "newest"),
|
||||
# #443: which shelf; part of the cursor scope, so a cursor never
|
||||
# crosses shelves. Not the variant ``archive`` scope of /{id}.
|
||||
"shelf": _shelf(request),
|
||||
}
|
||||
limit = _page_limit(request)
|
||||
cursor = request.query.get("cursor")
|
||||
roots = (
|
||||
await self._run_in_executor(self._library_root_spellings) if folder else []
|
||||
)
|
||||
prefixes = folder_prefixes(roots, folder) if folder else None
|
||||
|
||||
def read() -> tuple[dict, dict]:
|
||||
with self.db.library_snapshot(include_ceiling=True) as snapshot:
|
||||
conn, epoch, ceiling = snapshot
|
||||
# The folder and the roots it was resolved under join the scope.
|
||||
scope = _scope(
|
||||
{
|
||||
"kind": "parents",
|
||||
"projection": 2,
|
||||
"epoch": epoch,
|
||||
**options,
|
||||
**({"folder": folder, "roots": roots} if folder else {}),
|
||||
}
|
||||
{"kind": "parents", "projection": 2, "epoch": epoch, **options}
|
||||
)
|
||||
result = self.db._list_library(
|
||||
conn,
|
||||
epoch,
|
||||
ceiling,
|
||||
**options,
|
||||
folder_prefixes=prefixes,
|
||||
limit=limit,
|
||||
after=_decode(cursor, scope),
|
||||
)
|
||||
result["folder"] = folder
|
||||
records = self.db.get_library_previews(
|
||||
[parent["id"] for parent in result["items"]]
|
||||
)
|
||||
|
||||
@@ -1,321 +0,0 @@
|
||||
"""Delete or archive selected Library prompts, and restore archived ones (#443).
|
||||
|
||||
The workspace Prompt library's Delete… (D152). The legacy ``POST
|
||||
/prompt_manager/bulk/delete`` and ``DELETE /prompt_manager/delete/{id}`` are
|
||||
separate and unchanged.
|
||||
|
||||
- ``POST /prompt_manager/library/delete/preview {prompt_ids, expected_epoch}``
|
||||
classifies the selection (reads only, under a lease): the ids to ``delete``
|
||||
(in no saved pair), to ``archive`` (paired), the ``protected`` ones (tagged
|
||||
``__protected__``, kept), counts of ``already_archived`` and ``not_found``,
|
||||
the ``image_links`` a delete removes, the safety ``backup`` policy and,
|
||||
when a backup would be taken, the ``disk`` it needs.
|
||||
- ``POST /prompt_manager/library/delete {archive_ids, delete_ids,
|
||||
expected_epoch, backup}`` executes that review under a ``records`` hold
|
||||
(refused 409 ``busy`` beside a scan, restore, maintenance, move or
|
||||
Optimize). When a delete remains and a backup was promised (the reviewed
|
||||
``backup`` was "on" or the policy is on now) it checks the disk (507
|
||||
``insufficient-space``) and takes a verified safety backup (500
|
||||
``backup-failed``); then one transaction deletes the reviewed deletes whose
|
||||
fingerprint is unchanged since the backup and archives the paired ones.
|
||||
- ``POST /prompt_manager/library/restore {prompt_ids, expected_epoch}``
|
||||
removes the archive mark (under a lease).
|
||||
|
||||
A changed storage epoch answers 409 ``storage-changed``; nothing is written.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...database.safety_backup import SafetyBackupFailed
|
||||
except ImportError:
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.safety_backup import SafetyBackupFailed
|
||||
|
||||
from . import maintenance as _maintenance
|
||||
from .admin import _prune_safety_backups
|
||||
from .job_routes import busy_response
|
||||
from .jobs import PROMPT_DELETE_JOB_KIND, JobBusy
|
||||
from .prompts import parse_bulk_prompt_ids
|
||||
|
||||
PREVIEW_URL = "/prompt_manager/library/delete/preview"
|
||||
DELETE_URL = "/prompt_manager/library/delete"
|
||||
RESTORE_URL = "/prompt_manager/library/restore"
|
||||
DELETE_TITLE = "Delete prompts (Library)"
|
||||
_SELECTION_KEYS = frozenset({"prompt_ids", "expected_epoch"})
|
||||
_DELETE_KEYS = frozenset({"archive_ids", "delete_ids", "expected_epoch", "backup"})
|
||||
# #482: Delete permanently, with its digests; both or neither (a #443 body).
|
||||
_PAIRED_KEYS = frozenset({"delete_paired_ids", "paired_digests"})
|
||||
# #492: the shelf the Delete… runs on, optional on its own (default "active").
|
||||
_SHELF_KEYS = frozenset({"shelf"})
|
||||
STORAGE_CHANGED = "Prompt storage was restored; refresh the library"
|
||||
|
||||
|
||||
def _failure(status, code, error, **extra):
|
||||
return web.json_response(
|
||||
{"success": False, "code": code, "error": error, **extra}, status=status
|
||||
)
|
||||
|
||||
|
||||
def _invalid(message):
|
||||
return _failure(400, "invalid-request", message)
|
||||
|
||||
|
||||
def _storage_changed(exc):
|
||||
return _failure(409, exc.code, STORAGE_CHANGED, library_epoch=exc.epoch)
|
||||
|
||||
|
||||
async def _body(request, keys, *groups):
|
||||
"""The JSON object with exactly ``keys``, plus all or none of each group."""
|
||||
try:
|
||||
data = await request.json()
|
||||
except ValueError:
|
||||
raise ValueError("Expected a JSON object") from None
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("Expected a JSON object")
|
||||
extra = set(data) - keys
|
||||
partial = False
|
||||
for group in groups:
|
||||
if extra & group:
|
||||
partial = partial or not group <= extra
|
||||
extra -= group
|
||||
if not keys <= set(data) or extra or partial:
|
||||
raise ValueError("Request must contain exactly " + ", ".join(sorted(keys)))
|
||||
return data
|
||||
|
||||
|
||||
def _shelf(data):
|
||||
"""#492: the shelf a Delete… was reviewed on."""
|
||||
shelf = data.get("shelf", "active")
|
||||
if shelf not in ("active", "archived"):
|
||||
raise ValueError("shelf must be active or archived")
|
||||
return shelf
|
||||
|
||||
|
||||
def _paired_delete(data, archive_ids):
|
||||
"""#482: the reviewed archive ids to delete with their pairs -> digest."""
|
||||
if "delete_paired_ids" not in data:
|
||||
return {}
|
||||
ids = _reviewed_ids(data["delete_paired_ids"], "delete_paired_ids")
|
||||
digests = data["paired_digests"]
|
||||
if not isinstance(digests, dict):
|
||||
raise ValueError("paired_digests must map prompt ids to digests")
|
||||
if not set(ids) <= set(archive_ids):
|
||||
raise ValueError("delete_paired_ids must be reviewed archive ids")
|
||||
chosen = {}
|
||||
for pid in ids:
|
||||
digest = digests.get(str(pid))
|
||||
if not isinstance(digest, str) or len(digest) != 64:
|
||||
raise ValueError("Every delete_paired_ids entry needs its review digest")
|
||||
chosen[pid] = digest
|
||||
return chosen
|
||||
|
||||
|
||||
def _reviewed_ids(value, name):
|
||||
"""A reviewed id list: possibly empty, else as ``parse_bulk_prompt_ids``."""
|
||||
if value == []:
|
||||
return []
|
||||
try:
|
||||
return parse_bulk_prompt_ids(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError(str(exc).replace("prompt_ids", name)) from None
|
||||
|
||||
|
||||
class LibraryDeleteRoutesMixin:
|
||||
"""Mixin providing the workspace Library's delete, archive and restore."""
|
||||
|
||||
def _register_library_delete_routes(self, routes):
|
||||
routes.post(PREVIEW_URL)(self.preview_library_delete)
|
||||
routes.post(DELETE_URL)(self.delete_library_prompts)
|
||||
routes.post(RESTORE_URL)(self.restore_library_prompts)
|
||||
|
||||
async def preview_library_delete(self, request):
|
||||
try:
|
||||
data = await _body(request, _SELECTION_KEYS, _SHELF_KEYS)
|
||||
ids = parse_bulk_prompt_ids(data["prompt_ids"])
|
||||
shelf = _shelf(data)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
db = self.db
|
||||
policy = self._safety_backup_policy()
|
||||
try:
|
||||
async with self._jobs.lease("Library delete review"):
|
||||
review = await self._run_in_executor(
|
||||
db.classify_library_delete,
|
||||
ids,
|
||||
expected_epoch=data["expected_epoch"],
|
||||
shelf=shelf,
|
||||
)
|
||||
disk = None
|
||||
if (review["delete"] or review["deletable"]) and policy == "on":
|
||||
public, short, _size = await self._run_in_executor(
|
||||
_maintenance.disk_plan,
|
||||
db.model.db_path,
|
||||
backup=True,
|
||||
vacuum=False,
|
||||
)
|
||||
disk = {
|
||||
**public,
|
||||
"message": (
|
||||
None
|
||||
if short is None
|
||||
else _maintenance.short_space_message(short)
|
||||
),
|
||||
}
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
except LibraryCommandError as exc:
|
||||
return _storage_changed(exc)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Library delete review failed")
|
||||
return _failure(500, "review-failed", "The delete could not be reviewed")
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"library_epoch": review["library_epoch"],
|
||||
"delete": review["delete"],
|
||||
"archive": review["archive"],
|
||||
"protected": review["protected"],
|
||||
"already_archived": len(review["already_archived"]),
|
||||
"not_archived": len(review["not_archived"]),
|
||||
"not_found": len(review["not_found"]),
|
||||
"image_links": review["image_links"],
|
||||
# #482: the archive ids Delete permanently may delete with pairs.
|
||||
"deletable": review["deletable"],
|
||||
"review_digests": {
|
||||
str(pid): digest for pid, digest in review["review_digests"].items()
|
||||
},
|
||||
"paired_counts": review["paired_counts"],
|
||||
"paired_image_links": review["paired_image_links"],
|
||||
"backup": policy,
|
||||
"disk": disk,
|
||||
}
|
||||
)
|
||||
|
||||
async def delete_library_prompts(self, request):
|
||||
try:
|
||||
data = await _body(request, _DELETE_KEYS, _PAIRED_KEYS, _SHELF_KEYS)
|
||||
shelf = _shelf(data)
|
||||
archive_ids = _reviewed_ids(data["archive_ids"], "archive_ids")
|
||||
delete_ids = _reviewed_ids(data["delete_ids"], "delete_ids")
|
||||
if not archive_ids and not delete_ids:
|
||||
raise ValueError("Nothing to delete or archive")
|
||||
if len(archive_ids) + len(delete_ids) > 1000:
|
||||
raise ValueError("At most 1000 prompts can be changed at once")
|
||||
if set(archive_ids) & set(delete_ids):
|
||||
raise ValueError("A prompt cannot be both archived and deleted")
|
||||
if data["backup"] not in ("on", "off"):
|
||||
raise ValueError("backup must be on or off")
|
||||
paired = _paired_delete(data, archive_ids)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
busy = self._jobs.blocker(PROMPT_DELETE_JOB_KIND, admission="records")
|
||||
if busy is not None:
|
||||
return busy_response(JobBusy(busy))
|
||||
try:
|
||||
async with self._jobs.hold(
|
||||
PROMPT_DELETE_JOB_KIND, DELETE_TITLE, admission="records"
|
||||
) as job:
|
||||
response = await self._held_library_delete(
|
||||
self.db, data, archive_ids, delete_ids, paired, shelf
|
||||
)
|
||||
if response.status >= 400:
|
||||
job.finish("failed")
|
||||
return response
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _held_library_delete(
|
||||
self, db, data, archive_ids, delete_ids, paired, shelf="active"
|
||||
):
|
||||
epoch = data["expected_epoch"]
|
||||
promised = data["backup"] == "on" or self._safety_backup_policy() == "on"
|
||||
backup = None
|
||||
try:
|
||||
candidates = None
|
||||
if delete_ids or paired:
|
||||
review = await self._run_in_executor(
|
||||
db.classify_library_delete,
|
||||
[*delete_ids, *paired],
|
||||
expected_epoch=epoch,
|
||||
# #492: the same shelf, or archived ids would skip the backup.
|
||||
shelf=shelf,
|
||||
)
|
||||
candidates = review["fingerprints"]
|
||||
# #482: a permanent delete of a prompt in a saved pair is backed up too.
|
||||
if (candidates or review["deletable"]) and promised:
|
||||
_disk, short, _size = await self._run_in_executor(
|
||||
_maintenance.disk_plan,
|
||||
db.model.db_path,
|
||||
backup=True,
|
||||
vacuum=False,
|
||||
)
|
||||
if short is not None:
|
||||
return _failure(
|
||||
507,
|
||||
"insufficient-space",
|
||||
_maintenance.short_space_message(short),
|
||||
)
|
||||
backup = await self._run_in_executor(
|
||||
_maintenance.take_safety_backup, db.model
|
||||
)
|
||||
result = await self._run_in_executor(
|
||||
db.archive_or_delete_library_prompts,
|
||||
archive_ids,
|
||||
delete_ids,
|
||||
expected_epoch=epoch,
|
||||
candidates=candidates if candidates is not None else {},
|
||||
delete_paired=paired,
|
||||
shelf=shelf,
|
||||
)
|
||||
except LibraryCommandError as exc:
|
||||
return _storage_changed(exc)
|
||||
except SafetyBackupFailed as error:
|
||||
self.logger.error("Safety backup before a Library delete failed")
|
||||
return _failure(
|
||||
500, "backup-failed", _maintenance.backup_failed_message(error)
|
||||
)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Library delete failed")
|
||||
return _failure(500, "delete-failed", "The prompts could not be changed")
|
||||
finally:
|
||||
if backup is not None:
|
||||
await self._prune_after_delete(db)
|
||||
return web.json_response({"success": True, **result, "backup": backup})
|
||||
|
||||
async def _prune_after_delete(self, db):
|
||||
try:
|
||||
await self._run_in_executor(_prune_safety_backups, db.model.db_path)
|
||||
except Exception: # pruning old copies never fails the delete
|
||||
self.logger.exception("Could not prune old safety backups")
|
||||
|
||||
async def restore_library_prompts(self, request):
|
||||
try:
|
||||
data = await _body(request, _SELECTION_KEYS)
|
||||
ids = parse_bulk_prompt_ids(data["prompt_ids"])
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
try:
|
||||
async with self._jobs.lease("Library restore"):
|
||||
result = await self._run_in_executor(
|
||||
self.db.restore_library_prompts,
|
||||
ids,
|
||||
expected_epoch=data["expected_epoch"],
|
||||
)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
except LibraryCommandError as exc:
|
||||
return _storage_changed(exc)
|
||||
except ValueError as exc:
|
||||
return _invalid(str(exc))
|
||||
except (OSError, sqlite3.Error):
|
||||
self.logger.exception("Library restore failed")
|
||||
return _failure(500, "restore-failed", "The prompts could not be restored")
|
||||
return web.json_response({"success": True, **result})
|
||||
@@ -50,7 +50,6 @@ _REFUSAL_STATUS = {
|
||||
"not-movable": 400,
|
||||
"same-folder": 400,
|
||||
"target-invalid": 400,
|
||||
"data-dir-served": 400,
|
||||
"target-has-library": 409,
|
||||
"target-conflict": 409,
|
||||
"insufficient-space": 507,
|
||||
@@ -75,16 +74,6 @@ def _failure(message, status, code):
|
||||
)
|
||||
|
||||
|
||||
def _inside_thumbnail_folder(path):
|
||||
"""Whether *path* lies in the thumbnail folder in use (#320)."""
|
||||
from . import preview_store
|
||||
|
||||
setting = preview_store.setting()
|
||||
if setting.mode != preview_store.FOLDER_MODE or not setting.path:
|
||||
return False
|
||||
return ll._inside_any(path, [setting.path])
|
||||
|
||||
|
||||
def _plural(count, one, many=None):
|
||||
return f"{count:,} {one if count == 1 else (many or one + 's')}"
|
||||
|
||||
@@ -218,7 +207,6 @@ class LibraryLocationRoutesMixin:
|
||||
try:
|
||||
size = os.path.getsize(location.db_path)
|
||||
except OSError:
|
||||
# no library file yet: report size 0
|
||||
pass
|
||||
return {
|
||||
"path": self._library_public_path(location.db_path),
|
||||
@@ -283,8 +271,6 @@ class LibraryLocationRoutesMixin:
|
||||
def _plan_library_move(self):
|
||||
"""The checked plan (blocking); raises MoveFailed."""
|
||||
plan = lm.plan_move(self._live_location())
|
||||
if _inside_thumbnail_folder(plan.target_dir):
|
||||
raise lm.MoveFailed("data-dir-served")
|
||||
if lm.space_problem(plan) is not None:
|
||||
raise lm.MoveFailed("insufficient-space")
|
||||
return plan
|
||||
@@ -387,7 +373,6 @@ class LibraryLocationRoutesMixin:
|
||||
line += ". Logs move at the next restart."
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _fail_library_move(job, code, message):
|
||||
|
||||
@@ -205,4 +205,3 @@ class LogCleanupMixin:
|
||||
line += f", {failed} could not be deleted"
|
||||
job.log_line(line)
|
||||
job.update(processed=len(files), progress=100, status_line=line)
|
||||
return None
|
||||
|
||||
@@ -436,7 +436,7 @@ class LoraImportRoutesMixin:
|
||||
job.log_line(f"{CANCELLED} {CACHE_NOTE}")
|
||||
job.update(status_line=CANCELLED)
|
||||
job.finish("cancelled")
|
||||
return None
|
||||
return
|
||||
except Exception:
|
||||
self.logger.exception("LoRA import staging failed")
|
||||
return _fail_job(job, "import-failed", f"{FAILED} {CACHE_NOTE}")
|
||||
@@ -455,10 +455,10 @@ class LoraImportRoutesMixin:
|
||||
job.log_line(f"{CANCELLED} {CACHE_NOTE}")
|
||||
job.update(status_line=CANCELLED)
|
||||
job.finish("cancelled")
|
||||
return None
|
||||
return
|
||||
ok, _backup = await self._safety_backup_step(job, db, backup_on)
|
||||
if not ok:
|
||||
return None
|
||||
return
|
||||
# I1: the files are listed and fingerprinted again right before the
|
||||
# replace, so a file changed, removed or added while the downloads ran
|
||||
# (LoraManager refreshing its CivitAI info) aborts with nothing deleted.
|
||||
@@ -466,7 +466,6 @@ class LoraImportRoutesMixin:
|
||||
return _fail_job(job, "plan-changed", f"{SOURCE_CHANGED} {CACHE_NOTE}")
|
||||
job.update(status_line="Replacing the LoRA collection", progress=95)
|
||||
await self._replace_in_job(job, db, plan, staged)
|
||||
return None
|
||||
|
||||
def _count_downloads(self, job, staged):
|
||||
downloaded = staged.stats.get("downloaded", 0)
|
||||
@@ -519,4 +518,3 @@ class LoraImportRoutesMixin:
|
||||
staged.stats.get("download_errors", 0),
|
||||
)
|
||||
job.update(progress=100, processed=job.total, status_line=line)
|
||||
return None
|
||||
|
||||
@@ -44,7 +44,6 @@ _FILES_CHANGED = (
|
||||
"The LoRA metadata files changed during the scan; the previous imports "
|
||||
"were kept. Run the scan again."
|
||||
)
|
||||
_LORA_SCAN_FAILED = "The LoRA import failed. Check the server log for details."
|
||||
_IMPORTS_CHANGED = (
|
||||
"The imported LoRA prompts changed during the scan; the previous imports "
|
||||
"were kept. Run the scan again."
|
||||
@@ -408,21 +407,10 @@ class LoraIntegrationMixin:
|
||||
return
|
||||
if isinstance(error, PlanChanged):
|
||||
message = _IMPORTS_CHANGED
|
||||
elif isinstance(
|
||||
error,
|
||||
(
|
||||
LoraFilesChanged,
|
||||
LoraSettingsUnusable,
|
||||
EmptyReplacement,
|
||||
PairDependencyError,
|
||||
),
|
||||
):
|
||||
elif isinstance(error, (LoraFilesChanged, LoraSettingsUnusable)):
|
||||
message = str(error)
|
||||
elif isinstance(error, OSError):
|
||||
# Reduced to its reason and the file's basename.
|
||||
message = self._public_error(error)
|
||||
else:
|
||||
message = _LORA_SCAN_FAILED
|
||||
message = self._public_error(error)
|
||||
if isinstance(
|
||||
error,
|
||||
(
|
||||
@@ -548,7 +536,6 @@ class LoraIntegrationMixin:
|
||||
except asyncio.CancelledError:
|
||||
interrupted = True
|
||||
except Exception:
|
||||
# the await after this loop raises it for the caller
|
||||
pass
|
||||
if interrupted:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
@@ -281,7 +281,7 @@ def short_space_message(short):
|
||||
)
|
||||
|
||||
|
||||
def _changed_line(operation, count, marks=0):
|
||||
def _changed_line(operation, count):
|
||||
if operation == "optimize":
|
||||
return "Optimized the database"
|
||||
if operation == "duplicate_prompts":
|
||||
@@ -291,11 +291,7 @@ def _changed_line(operation, count, marks=0):
|
||||
noun = "unused prompt component" if count == 1 else "unused prompt components"
|
||||
return f"Removed {count:,} {noun}"
|
||||
noun = "link to a missing file" if count == 1 else "links to missing files"
|
||||
line = f"Removed {count:,} {noun}"
|
||||
if marks:
|
||||
gone = "removed image" if marks == 1 else "removed images"
|
||||
line += f"; forgot {marks:,} {gone} whose files are gone or changed"
|
||||
return line
|
||||
return f"Removed {count:,} {noun}"
|
||||
|
||||
|
||||
def take_safety_backup(model, label="backup"):
|
||||
@@ -423,7 +419,6 @@ class MaintenanceRoutesMixin:
|
||||
draft.public(),
|
||||
{
|
||||
"targets": draft.targets,
|
||||
"marks": draft.marks,
|
||||
"roots": roots,
|
||||
"protected": draft.protected["count"],
|
||||
"rows": draft.scope["count"],
|
||||
@@ -493,9 +488,7 @@ class MaintenanceRoutesMixin:
|
||||
# ── apply ─────────────────────────────────────────────────────────
|
||||
|
||||
async def apply_maintenance_plan(self, request):
|
||||
from .image_delete import IMAGE_DELETE, parse_mode
|
||||
|
||||
data, error = await self._read_maintenance_body(request, ("token", "mode"))
|
||||
data, error = await self._read_maintenance_body(request, ("token",))
|
||||
if error is not None:
|
||||
return error
|
||||
token = data.get("token")
|
||||
@@ -504,13 +497,6 @@ class MaintenanceRoutesMixin:
|
||||
plan, refused = self._peek_plan(token)
|
||||
if refused is not None:
|
||||
return refused
|
||||
mode = None
|
||||
if plan.operation == IMAGE_DELETE:
|
||||
mode = parse_mode(data.get("mode"))
|
||||
if mode is None:
|
||||
return _invalid('mode must be "library" or "trash"')
|
||||
elif "mode" in data:
|
||||
return _invalid("Unknown field: mode")
|
||||
store = self._maintenance_plans
|
||||
if plan.operation == ROTATED_LOGS:
|
||||
# Log files only: no database identity, backup or disk check.
|
||||
@@ -558,8 +544,6 @@ class MaintenanceRoutesMixin:
|
||||
try:
|
||||
if plan.operation == DUPLICATE_FILES:
|
||||
job = self._start_duplicate_cleanup(db, plan, backup)
|
||||
elif plan.operation == IMAGE_DELETE:
|
||||
job = self._start_image_delete(db, plan, mode, backup)
|
||||
elif plan.operation == WORKFLOW_STORAGE:
|
||||
job = self._start_workflow_storage(db, backup)
|
||||
else:
|
||||
@@ -658,7 +642,6 @@ class MaintenanceRoutesMixin:
|
||||
plan.fingerprint,
|
||||
payload["targets"],
|
||||
payload["roots"],
|
||||
payload.get("marks", ()),
|
||||
)
|
||||
except PlanChanged:
|
||||
return _fail_job(job, "plan-changed", PLAN_CHANGED)
|
||||
@@ -670,10 +653,9 @@ class MaintenanceRoutesMixin:
|
||||
)
|
||||
job.record_counted("changed", result["changed"], rows)
|
||||
job.record_counted("protected", payload["protected"])
|
||||
line = _changed_line(plan.operation, result["changed"], result.get("marks", 0))
|
||||
line = _changed_line(plan.operation, result["changed"])
|
||||
job.log_line(line)
|
||||
job.update(progress=100, status_line=f"{line} · {backup_note(backup)}")
|
||||
return None
|
||||
|
||||
def _maintenance_rows(self, operation, described):
|
||||
"""Outcome rows for the changed records (root-relative paths only)."""
|
||||
|
||||
@@ -9,15 +9,13 @@ by the caller.
|
||||
import os
|
||||
import re
|
||||
|
||||
from .images import _file_not_found, _forbidden, _has_traversal
|
||||
from .images import _forbidden, _has_traversal
|
||||
from .prompts import bad_request
|
||||
|
||||
try:
|
||||
from ...utils.media_capture import CAPTURE_EXTENSIONS
|
||||
from ...utils.media_folders import in_trash_folder
|
||||
except ImportError:
|
||||
from utils.media_capture import CAPTURE_EXTENSIONS
|
||||
from utils.media_folders import in_trash_folder
|
||||
|
||||
MAX_TARGET_PATH = 1024
|
||||
TARGET_FIELDS = frozenset({"root_index", "root_scope", "path"})
|
||||
@@ -31,9 +29,7 @@ def media_target(data: object, *, media_message: str = TAGGING_MEDIA_MESSAGE):
|
||||
Only the listing identity is accepted: an int root index, the root's
|
||||
64-hex scope and a bounded root-relative POSIX media path, never an
|
||||
absolute, drive-qualified, backslash or ``..`` path (a 400 for a malformed
|
||||
shape, a 403 for an unsafe or non-media path, carrying *media_message*, a
|
||||
404 for a path in a trash folder, #543). A link or a short-name spelling
|
||||
is refused later by the capture store's alias check.
|
||||
shape, a 403 for an unsafe or non-media path, carrying *media_message*).
|
||||
"""
|
||||
if not isinstance(data, dict) or set(data) != TARGET_FIELDS:
|
||||
return None, bad_request("target must name root_index, root_scope and path")
|
||||
@@ -55,6 +51,4 @@ def media_target(data: object, *, media_message: str = TAGGING_MEDIA_MESSAGE):
|
||||
return None, _forbidden()
|
||||
if os.path.splitext(parts[-1])[1].lower() not in CAPTURE_EXTENSIONS:
|
||||
return None, _forbidden(media_message)
|
||||
if in_trash_folder(relative):
|
||||
return None, _file_not_found()
|
||||
return (index, scope, relative), None
|
||||
|
||||
@@ -102,12 +102,7 @@ class PairRoutesMixin:
|
||||
validate_prompt_text(negative)
|
||||
fields = metadata_fields(data)
|
||||
pair_id = await self._run_in_executor(
|
||||
self.db.save_prompt_pair,
|
||||
positive,
|
||||
negative,
|
||||
# The node's Save pair button: as a run, a new prompt gets the metadata.
|
||||
fill_new_positive=True,
|
||||
**fields,
|
||||
self.db.save_prompt_pair, positive, negative, **fields
|
||||
)
|
||||
pair = await self._run_in_executor(self.db.get_prompt_pair, pair_id)
|
||||
return web.json_response({"success": True, "pair": pair})
|
||||
|
||||
@@ -1,643 +0,0 @@
|
||||
"""The thumbnail location setting (#320): parsing, validation, source keys
|
||||
and the thumbnail folder's marker.
|
||||
|
||||
``preview_store`` holds the runtime state and maps paths; this module holds
|
||||
the rules it applies. Nothing here keeps state.
|
||||
|
||||
Schema (config.json ``thumbnails.location``)::
|
||||
|
||||
{"mode": "folder", "path": "D:\\\\Thumbs", "id": "<uuid4>",
|
||||
"install": "<uuid4>", "cleanup": []}
|
||||
|
||||
The default (and a missing key) is ``{"mode": "source"}``: previews inside
|
||||
each source. An invalid value never falls back to the sources: it parses as
|
||||
a folder location with problem ``invalid``, so nothing is written anywhere.
|
||||
|
||||
Layout in folder mode: ``<folder>/<source key>/<rel dir>/<stem>_thumb<ext>``.
|
||||
A source key is a readable slug plus a short hash of the source's identity
|
||||
(``output-3f9a2c1b``), assigned once and stored in the marker
|
||||
``.promptmanager-thumbnails.json`` with the identity it belongs to.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import ntpath
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import stat as stat_module
|
||||
import threading
|
||||
import unicodedata
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
try:
|
||||
from ...utils import library_location
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils import library_location
|
||||
|
||||
logger = logging.getLogger("prompt_manager.thumbnails")
|
||||
|
||||
SOURCE_MODE = "source"
|
||||
FOLDER_MODE = "folder"
|
||||
MARKER_NAME = ".promptmanager-thumbnails.json"
|
||||
MARKER_VERSION = 1
|
||||
# A source key: one folder name, a slug and a 8-16 hex hash (``source_key``).
|
||||
KEY_PATTERN = re.compile(r"^[a-z0-9_-]{1,24}-[0-9a-f]{8,16}$")
|
||||
# A marker is a few keys; anything larger is not ours.
|
||||
MAX_MARKER_BYTES = 4 * 2**20
|
||||
# OS metadata files a folder may hold and still count as empty.
|
||||
OS_METADATA = frozenset({"thumbs.db", "desktop.ini", ".ds_store"})
|
||||
KEY_HASH_LENGTHS = (8, 12, 16)
|
||||
SLUG_LENGTH = 24
|
||||
# Folder + the longest key (24 + 1 + 16) + this margin must fit (Windows).
|
||||
WINDOWS_FOLDER_LIMIT = 247
|
||||
LONGEST_KEY = SLUG_LENGTH + 1 + KEY_HASH_LENGTHS[-1]
|
||||
PATH_MARGIN = 40
|
||||
# A validation stat on a share that does not answer in time is unavailable.
|
||||
VALIDATION_TIMEOUT = 5.0
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
|
||||
_RESERVED = re.compile(r"^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\..*)?$", re.IGNORECASE)
|
||||
_BAD_CHARS = re.compile(r'[<>:"|?*\x00-\x1f]')
|
||||
|
||||
# Problem codes, their HTTP status and the text the UI shows (no path).
|
||||
MESSAGES = {
|
||||
"location-not-absolute": (
|
||||
400,
|
||||
"Enter a full folder path, such as D:\\Thumbnails or /mnt/fast/thumbnails.",
|
||||
),
|
||||
"location-device-path": (
|
||||
400,
|
||||
"Enter the folder as a drive letter or \\\\server\\share path, "
|
||||
"without \\\\?\\.",
|
||||
),
|
||||
"location-invalid-name": (
|
||||
400,
|
||||
"This folder name can't be used on Windows. Choose another name.",
|
||||
),
|
||||
"location-too-broad": (
|
||||
400,
|
||||
"Choose a folder inside a drive or share, not the drive itself.",
|
||||
),
|
||||
"location-served": (
|
||||
400,
|
||||
"Choose a folder outside your image sources and ComfyUI's output, input "
|
||||
"and temp folders.",
|
||||
),
|
||||
"location-node-folder": (
|
||||
400,
|
||||
"Choose a folder outside the Prompt Manager node folder.",
|
||||
),
|
||||
"location-contains-source": (
|
||||
400,
|
||||
"This folder contains an image source. Choose a folder beside it instead.",
|
||||
),
|
||||
"location-same": (400, "Thumbnails are already stored here."),
|
||||
"location-parent-missing": (
|
||||
400,
|
||||
"The folder that would hold it doesn't exist. Check the path and that its "
|
||||
"drive is connected.",
|
||||
),
|
||||
"location-unavailable": (
|
||||
409,
|
||||
"This folder can't be reached right now. Check that its drive or share is "
|
||||
"connected.",
|
||||
),
|
||||
"location-not-empty": (
|
||||
409,
|
||||
"This folder already holds other files. Choose an empty folder or one "
|
||||
"Prompt Manager used before.",
|
||||
),
|
||||
"location-not-writable": (
|
||||
409,
|
||||
"Prompt Manager can't write to this folder. Check its permissions.",
|
||||
),
|
||||
"location-path-too-long": (
|
||||
400,
|
||||
"This path is too long for Windows. Choose a shorter folder path.",
|
||||
),
|
||||
"location-needs-move": (
|
||||
409,
|
||||
"Use Move thumbnails in System › Sources & scans.",
|
||||
),
|
||||
}
|
||||
HOLDS_THUMBNAILS_REASON = "This folder holds Prompt Manager's thumbnails."
|
||||
|
||||
|
||||
def message(code):
|
||||
"""``(status, text)`` of a problem code."""
|
||||
return MESSAGES[code]
|
||||
|
||||
|
||||
# ── parsing ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedLocation:
|
||||
"""A ``thumbnails.location`` value, checked as data (no filesystem)."""
|
||||
|
||||
mode: str = SOURCE_MODE
|
||||
path: str | None = None
|
||||
id: str | None = None
|
||||
install: str | None = None
|
||||
cleanup: tuple = field(default_factory=tuple)
|
||||
problem: str | None = None
|
||||
|
||||
def to_config(self):
|
||||
"""The config.json form (``invalid`` values are never written)."""
|
||||
if self.mode == SOURCE_MODE:
|
||||
value = {"mode": SOURCE_MODE}
|
||||
else:
|
||||
value = {"mode": FOLDER_MODE, "path": self.path, "id": self.id}
|
||||
if self.install:
|
||||
value["install"] = self.install
|
||||
value["cleanup"] = list(self.cleanup)
|
||||
return value
|
||||
|
||||
|
||||
def _cleanup(value):
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
return tuple(item for item in value if isinstance(item, dict))
|
||||
|
||||
|
||||
def parse_location(value, *, flavor=os.path):
|
||||
"""The location a ``thumbnails.location`` value names.
|
||||
|
||||
Missing: inside each source. Anything else that is not a well-formed
|
||||
location (an unknown mode, a relative path, a missing id) becomes a
|
||||
folder location with problem ``invalid``: never a silent fall-back.
|
||||
"""
|
||||
if value is None:
|
||||
return ParsedLocation()
|
||||
if not isinstance(value, dict):
|
||||
return ParsedLocation(mode=FOLDER_MODE, problem="invalid")
|
||||
mode = value.get("mode", SOURCE_MODE)
|
||||
install = value.get("install") if isinstance(value.get("install"), str) else None
|
||||
cleanup = _cleanup(value.get("cleanup"))
|
||||
if mode == SOURCE_MODE:
|
||||
return ParsedLocation(install=install, cleanup=cleanup)
|
||||
if mode != FOLDER_MODE:
|
||||
return ParsedLocation(mode=FOLDER_MODE, problem="invalid", cleanup=cleanup)
|
||||
path, marker_id = value.get("path"), value.get("id")
|
||||
if (
|
||||
not isinstance(path, str)
|
||||
or form_problem(path, flavor=flavor) is not None
|
||||
or not isinstance(marker_id, str)
|
||||
or not marker_id.strip()
|
||||
):
|
||||
return ParsedLocation(
|
||||
mode=FOLDER_MODE,
|
||||
path=path if isinstance(path, str) else None,
|
||||
problem="invalid",
|
||||
cleanup=cleanup,
|
||||
)
|
||||
return ParsedLocation(
|
||||
mode=FOLDER_MODE,
|
||||
path=flavor.abspath(path),
|
||||
id=marker_id,
|
||||
install=install,
|
||||
cleanup=cleanup,
|
||||
)
|
||||
|
||||
|
||||
# ── form (text only) ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _is_windows(flavor):
|
||||
return flavor is ntpath or getattr(flavor, "__name__", "") == "ntpath"
|
||||
|
||||
|
||||
def form_problem(path, *, flavor=os.path):
|
||||
"""The text-only problem of *path*, or None (no filesystem access).
|
||||
|
||||
``\\\\?\\`` and ``\\\\.\\`` prefixes, relative, drive-relative (``C:x``)
|
||||
and root-relative (``\\x``) spellings, and names Windows cannot hold
|
||||
(checked on every OS: the folder may be moved to Windows).
|
||||
"""
|
||||
if not isinstance(path, str) or not path.strip() or path != path.lstrip():
|
||||
return "location-not-absolute"
|
||||
if path.replace("/", "\\").startswith(("\\\\?\\", "\\\\.\\")):
|
||||
return "location-device-path"
|
||||
if _is_windows(flavor):
|
||||
drive, rest = ntpath.splitdrive(path)
|
||||
if not drive or not rest.startswith(("\\", "/")):
|
||||
return "location-not-absolute"
|
||||
elif not posixpath.isabs(path):
|
||||
return "location-not-absolute"
|
||||
drive, rest = flavor.splitdrive(path)
|
||||
for part in re.split(r"[\\/]+", rest):
|
||||
if not part:
|
||||
continue
|
||||
if (
|
||||
_RESERVED.match(part)
|
||||
or part.endswith((".", " "))
|
||||
or _BAD_CHARS.search(part)
|
||||
):
|
||||
return "location-invalid-name"
|
||||
return None
|
||||
|
||||
|
||||
def is_too_broad(path, *, home=None, flavor=os.path):
|
||||
"""A filesystem or share root, or the home folder itself."""
|
||||
normalized = flavor.normpath(path)
|
||||
drive, rest = flavor.splitdrive(normalized)
|
||||
if rest in ("", "\\", "/"):
|
||||
return True
|
||||
home = home if home is not None else os.path.expanduser("~")
|
||||
return bool(home) and flavor.normcase(normalized) == flavor.normcase(
|
||||
flavor.normpath(home)
|
||||
)
|
||||
|
||||
|
||||
def too_long_for_windows(path, long_paths=None):
|
||||
"""Folder + the longest key + a margin over the folder limit (Windows
|
||||
without long paths; *long_paths* injects the registry answer)."""
|
||||
if long_paths is None:
|
||||
long_paths = long_paths_enabled()
|
||||
if long_paths:
|
||||
return False
|
||||
return len(path) + 1 + LONGEST_KEY + PATH_MARGIN > WINDOWS_FOLDER_LIMIT
|
||||
|
||||
|
||||
def long_paths_enabled():
|
||||
"""``LongPathsEnabled`` on Windows (True on other systems)."""
|
||||
if os.name != "nt":
|
||||
return True
|
||||
try:
|
||||
import winreg
|
||||
|
||||
with winreg.OpenKey(
|
||||
winreg.HKEY_LOCAL_MACHINE,
|
||||
r"SYSTEM\CurrentControlSet\Control\FileSystem",
|
||||
) as key:
|
||||
return winreg.QueryValueEx(key, "LongPathsEnabled")[0] == 1
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
# ── links ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def is_link_like(path):
|
||||
"""A symlink, a junction or any reparse point (one lstat), or missing.
|
||||
|
||||
Raises OSError other than "not found" (an unreachable share).
|
||||
"""
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
if stat_module.S_ISLNK(info.st_mode):
|
||||
return True
|
||||
attributes = getattr(info, "st_file_attributes", 0)
|
||||
if attributes & FILE_ATTRIBUTE_REPARSE_POINT:
|
||||
return True
|
||||
is_junction = getattr(os.path, "isjunction", None)
|
||||
return bool(is_junction and is_junction(path))
|
||||
|
||||
|
||||
# ── source keys ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def slug(name):
|
||||
"""``NFKD`` to ASCII, lower case, runs outside ``[a-z0-9_]`` as ``-``,
|
||||
trimmed and cut to 24 characters; empty becomes ``source``."""
|
||||
text = unicodedata.normalize("NFKD", str(name))
|
||||
text = text.encode("ascii", "ignore").decode("ascii").lower()
|
||||
text = re.sub(r"[^a-z0-9_]+", "-", text).strip("-")
|
||||
text = text[:SLUG_LENGTH].strip("-")
|
||||
return text or "source"
|
||||
|
||||
|
||||
def source_key(spelling, identity, taken):
|
||||
"""The key of a source: ``slug(basename) + "-" + sha256(identity)[:n]``.
|
||||
|
||||
*taken* maps existing keys to their identity; a key taken by another
|
||||
identity grows its hash to 12, then 16 characters.
|
||||
"""
|
||||
base = slug(ntpath.basename(str(spelling).rstrip("\\/")))
|
||||
digest = hashlib.sha256(str(identity).encode("utf-8")).hexdigest()
|
||||
for length in KEY_HASH_LENGTHS:
|
||||
key = f"{base}-{digest[:length]}"
|
||||
owner = taken.get(key)
|
||||
if owner is None or owner == identity:
|
||||
return key
|
||||
raise ValueError("no free source key")
|
||||
|
||||
|
||||
def file_identity(path):
|
||||
"""``[st_dev, st_ino]`` of an existing folder, or None (no file ids)."""
|
||||
try:
|
||||
info = os.stat(path)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not info.st_ino:
|
||||
return None
|
||||
return [info.st_dev, info.st_ino]
|
||||
|
||||
|
||||
def find_key(keys, spelling, identity, dev_ino):
|
||||
"""The marker key of a source: the same configured spelling, then the
|
||||
same identity text, then the same file id; or None."""
|
||||
keys = {k: v for k, v in keys.items() if valid_key(k) and isinstance(v, dict)}
|
||||
for key, entry in keys.items():
|
||||
if entry.get("path") == spelling:
|
||||
return key
|
||||
for key, entry in keys.items():
|
||||
if entry.get("identity") == identity:
|
||||
return key
|
||||
if dev_ino:
|
||||
for key, entry in keys.items():
|
||||
if entry.get("dev_ino") == list(dev_ino):
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
# ── the marker ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def marker_path(folder):
|
||||
return os.path.join(folder, MARKER_NAME)
|
||||
|
||||
|
||||
def read_marker(folder):
|
||||
"""The marker in *folder* as a dict, or None (absent or unreadable as
|
||||
ours). OSError other than "not found" propagates (an unreachable share)."""
|
||||
try:
|
||||
with open(marker_path(folder), "r", encoding="utf-8") as handle:
|
||||
text = handle.read(MAX_MARKER_BYTES + 1)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return None
|
||||
if len(text) > MAX_MARKER_BYTES:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(data, dict) or not isinstance(data.get("id"), str):
|
||||
return None
|
||||
data["keys"] = valid_keys(data.get("keys"))
|
||||
return data
|
||||
|
||||
|
||||
def valid_key(key):
|
||||
"""Whether *key* can name a source's folder: one folder name of the
|
||||
``source_key`` form, never a separator, ``..`` or a drive."""
|
||||
return isinstance(key, str) and KEY_PATTERN.fullmatch(key) is not None
|
||||
|
||||
|
||||
def valid_keys(keys):
|
||||
"""The marker's keys that are safe to join onto the folder; a hand-edited
|
||||
or foreign entry that is not is dropped (and logged), never followed."""
|
||||
if not isinstance(keys, dict):
|
||||
return {}
|
||||
kept = {}
|
||||
for key, entry in keys.items():
|
||||
if valid_key(key) and isinstance(entry, dict):
|
||||
kept[key] = entry
|
||||
else:
|
||||
logger.warning(
|
||||
"Ignoring a thumbnail folder marker entry that is not a source " "key"
|
||||
)
|
||||
return kept
|
||||
|
||||
|
||||
def write_marker(folder, mutate):
|
||||
"""Re-read the marker, apply ``mutate(marker)`` and write it atomically
|
||||
(a temp sibling renamed over it). Callers hold ``preview_store.write_lock``.
|
||||
Returns the marker written."""
|
||||
current = read_marker(folder) or {}
|
||||
marker = {
|
||||
"version": MARKER_VERSION,
|
||||
"id": current.get("id"),
|
||||
"created_by_pm": bool(current.get("created_by_pm", False)),
|
||||
"keys": dict(current.get("keys") or {}),
|
||||
}
|
||||
mutate(marker)
|
||||
if not marker.get("id"):
|
||||
raise ValueError("a marker needs an id")
|
||||
target = marker_path(folder)
|
||||
temp = os.path.join(
|
||||
folder, f".{MARKER_NAME}.{os.getpid()}-{threading.get_ident()}.tmp"
|
||||
)
|
||||
try:
|
||||
with open(temp, "w", encoding="utf-8") as handle:
|
||||
json.dump(marker, handle, indent=2, sort_keys=True)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temp, target)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temp)
|
||||
except OSError:
|
||||
pass # gone with the rename; a leftover is swept with the temps
|
||||
return marker
|
||||
|
||||
|
||||
def new_id():
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def init_folder(folder, *, created_by_pm=False):
|
||||
"""The marker id of *folder*: an existing marker is adopted; a folder
|
||||
without one gets a new marker (new id, no keys)."""
|
||||
existing = read_marker(folder)
|
||||
if existing is not None:
|
||||
return existing["id"]
|
||||
|
||||
def create(marker):
|
||||
if not marker.get("id"):
|
||||
marker["id"] = new_id()
|
||||
marker["created_by_pm"] = created_by_pm
|
||||
|
||||
return write_marker(folder, create)["id"]
|
||||
|
||||
|
||||
def folder_state(path, expected_id):
|
||||
"""``(available, problem, marker)`` of a configured thumbnail folder.
|
||||
|
||||
Available: the folder exists, is a directory and not a link, junction or
|
||||
reparse point, and holds our marker with *expected_id*. Raises OSError
|
||||
when the folder cannot be reached (an I/O error other than not found).
|
||||
"""
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
return False, "missing", None
|
||||
except OSError as error:
|
||||
if error.errno in (errno.ENOENT, errno.ENOTDIR):
|
||||
return False, "missing", None
|
||||
raise
|
||||
if is_link_like(path):
|
||||
return False, "invalid", None
|
||||
if not stat_module.S_ISDIR(info.st_mode):
|
||||
return False, "invalid", None
|
||||
marker = read_marker(path)
|
||||
if marker is None:
|
||||
return False, "missing", None
|
||||
if marker["id"] != expected_id:
|
||||
return False, "marker-mismatch", marker
|
||||
return True, None, marker
|
||||
|
||||
|
||||
def holds_only_metadata(path):
|
||||
"""Whether an existing folder is empty but for OS metadata files."""
|
||||
with os.scandir(path) as entries:
|
||||
for entry in entries:
|
||||
name = entry.name.lower()
|
||||
if name in OS_METADATA or name.startswith("._"):
|
||||
continue
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ── validation ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Context:
|
||||
"""What a location is checked against (gathered by the caller)."""
|
||||
|
||||
roots: tuple = () # every image source of every root list
|
||||
contained: tuple = () # ComfyUI output/input/temp and the library folder
|
||||
node_folder: str | None = None
|
||||
default_path: str | None = None
|
||||
current_path: str | None = None # the folder in use (folder mode)
|
||||
home: str | None = None
|
||||
flavor: object = os.path
|
||||
long_paths: bool | None = None
|
||||
environ: object = None
|
||||
# ComfyUI's folder_paths module (None: none; unset: imported).
|
||||
folder_paths: object = library_location._UNSET
|
||||
|
||||
|
||||
def _timed(func, timeout):
|
||||
"""``func()`` on a daemon thread; raises TimeoutError past *timeout*.
|
||||
A stuck call is left to finish on its own thread."""
|
||||
result = {}
|
||||
done = threading.Event()
|
||||
|
||||
def run():
|
||||
try:
|
||||
result["value"] = func()
|
||||
except BaseException as error: # handed to the caller
|
||||
result["error"] = error
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
threading.Thread(target=run, name="pm-preview-stat", daemon=True).start()
|
||||
if not done.wait(timeout):
|
||||
raise TimeoutError("the folder did not answer in time")
|
||||
if "error" in result:
|
||||
raise result["error"]
|
||||
return result.get("value")
|
||||
|
||||
|
||||
def _inside(path, folders):
|
||||
return library_location._inside_any(path, folders)
|
||||
|
||||
|
||||
def _same(first, second):
|
||||
return library_location.same_folder(first, second) or os.path.normcase(
|
||||
os.path.realpath(first)
|
||||
) == os.path.normcase(os.path.realpath(second))
|
||||
|
||||
|
||||
def _served(path, context):
|
||||
environ = os.environ if context.environ is None else context.environ
|
||||
return (
|
||||
library_location.data_dir_problem(
|
||||
path,
|
||||
folder_paths=context.folder_paths,
|
||||
environ=environ,
|
||||
gallery_roots=list(context.roots),
|
||||
)
|
||||
== "data-dir-served"
|
||||
)
|
||||
|
||||
|
||||
def _probe_write(path):
|
||||
"""Create and delete a probe file in *path*; False when refused."""
|
||||
probe = os.path.join(path, f".pm-probe-{uuid.uuid4().hex[:8]}.tmp")
|
||||
try:
|
||||
with open(probe, "wb") as handle:
|
||||
handle.write(b"")
|
||||
except OSError:
|
||||
return False
|
||||
try:
|
||||
os.unlink(probe)
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def location_problem(path, *, purpose, context, timeout=VALIDATION_TIMEOUT):
|
||||
"""Why *path* cannot be the thumbnail folder, as a code, or None.
|
||||
|
||||
*purpose*: ``move`` (every rule), ``startup`` (no ``same``, ``empty``,
|
||||
``writable`` or parent rules: availability covers them) or ``recreate``
|
||||
(the folder is missing and is made again).
|
||||
"""
|
||||
flavor = context.flavor
|
||||
code = form_problem(path, flavor=flavor)
|
||||
if code is not None:
|
||||
return code
|
||||
path = flavor.abspath(path)
|
||||
default = bool(context.default_path) and flavor.normcase(path) == flavor.normcase(
|
||||
flavor.abspath(context.default_path)
|
||||
)
|
||||
if is_too_broad(path, home=context.home, flavor=flavor):
|
||||
return "location-too-broad"
|
||||
try:
|
||||
return _timed(
|
||||
lambda: _filesystem_problem(path, purpose, context, default), timeout
|
||||
)
|
||||
except (TimeoutError, OSError):
|
||||
return "location-unavailable"
|
||||
|
||||
|
||||
def _filesystem_problem(path, purpose, context, default):
|
||||
if _served(path, context):
|
||||
return "location-served"
|
||||
if context.node_folder and not default and _inside(path, [context.node_folder]):
|
||||
return "location-node-folder"
|
||||
inner = list(context.roots) + list(context.contained)
|
||||
if any(folder and _inside(folder, [path]) for folder in inner):
|
||||
if not default:
|
||||
return "location-contains-source"
|
||||
if (
|
||||
purpose == "move"
|
||||
and context.current_path
|
||||
and (
|
||||
os.path.normcase(path) == os.path.normcase(context.current_path)
|
||||
or (os.path.isdir(path) and _same(path, context.current_path))
|
||||
)
|
||||
):
|
||||
return "location-same"
|
||||
# Windows, or a test injecting its limit (``long_paths=False``).
|
||||
if os.name == "nt" or context.long_paths is False:
|
||||
if too_long_for_windows(path, context.long_paths):
|
||||
return "location-path-too-long"
|
||||
if purpose == "startup":
|
||||
return None
|
||||
if is_link_like(path):
|
||||
# Never through a link: the folder itself must be a real folder.
|
||||
return "location-not-empty"
|
||||
exists = os.path.isdir(path)
|
||||
if not exists:
|
||||
if os.path.lexists(path):
|
||||
return "location-not-empty"
|
||||
if not default and not os.path.isdir(os.path.dirname(path)):
|
||||
return "location-parent-missing"
|
||||
return None
|
||||
if read_marker(path) is None and not holds_only_metadata(path):
|
||||
return "location-not-empty"
|
||||
if not _probe_write(path):
|
||||
return "location-not-writable"
|
||||
return None
|
||||
@@ -1,671 +0,0 @@
|
||||
"""Where previews live on disk (#320): the one resolver.
|
||||
|
||||
Every mapping between an image source (a "root"), one of its media files and
|
||||
that file's preview on disk goes through this module. URL text stays in
|
||||
``thumbnail_settings`` (``thumbnail_rel``: ``thumbnails/<rel>_thumb<ext>``,
|
||||
the shape the serve route, ``inspection-state.mjs`` and ``admin.js`` share);
|
||||
this module answers where such a preview is on disk.
|
||||
|
||||
The store of a source is where its previews live. With the default location
|
||||
(mode ``source``, "Inside each source") it is ``<source>/thumbnails``, exactly
|
||||
the folder every caller joined by hand before, so nothing moves.
|
||||
|
||||
Writes and the switch: every preview writer takes ``write_guard`` when it
|
||||
starts (it records the location's ``generation``); the final rename runs
|
||||
under ``write_lock`` and only when the generation is unchanged and no hold
|
||||
applies (``thumbnail_encode.replace_unless_held``). Changing the location
|
||||
bumps the generation under the same lock, so no write that began before can
|
||||
land in the old store afterwards. Nothing reads or commits config while
|
||||
``write_lock`` is held. The hold check runs under it too, so hold checks take
|
||||
no lock (the job registry's ``running()`` is a lock-free snapshot) and
|
||||
``bump_generation`` is never called with a registry or config lock held.
|
||||
|
||||
All functions are blocking: call them off the event loop.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
|
||||
from . import preview_location, thumbnail_settings
|
||||
|
||||
try:
|
||||
from ...utils.media_folders import PREVIEW_FOLDER
|
||||
except ImportError: # tests import the package top-level
|
||||
from utils.media_folders import PREVIEW_FOLDER
|
||||
|
||||
logger = logging.getLogger("prompt_manager.thumbnails")
|
||||
|
||||
SOURCE_MODE = preview_location.SOURCE_MODE
|
||||
FOLDER_MODE = preview_location.FOLDER_MODE
|
||||
# The probe checks the folder this often, and a check (or a store call)
|
||||
# taking longer than the budget marks it unreachable.
|
||||
PROBE_INTERVAL = 5.0
|
||||
IO_BUDGET = 2.0
|
||||
PROBE_THREAD = "pm-preview-probe"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Location:
|
||||
"""One snapshot of the thumbnail location (take one per request or job)."""
|
||||
|
||||
mode: str = SOURCE_MODE
|
||||
path: str | None = None
|
||||
available: bool = True
|
||||
problem: str | None = None
|
||||
generation: int = 0
|
||||
|
||||
|
||||
write_lock = threading.Lock()
|
||||
_generation = 0
|
||||
# The runtime location: the parsed setting, availability and the marker keys
|
||||
# last read. Swapped as a whole; readers take one snapshot.
|
||||
_state_lock = threading.Lock()
|
||||
_parsed = preview_location.ParsedLocation()
|
||||
_available = True
|
||||
_problem = None
|
||||
_keys = {}
|
||||
_key_cache = {}
|
||||
_clock = time.monotonic
|
||||
|
||||
|
||||
def current():
|
||||
"""The thumbnail location now."""
|
||||
with _state_lock:
|
||||
parsed, available, problem, generation_now = (
|
||||
_parsed,
|
||||
_available,
|
||||
_problem,
|
||||
_generation,
|
||||
)
|
||||
if parsed.mode == SOURCE_MODE:
|
||||
return Location(generation=generation_now)
|
||||
return Location(FOLDER_MODE, parsed.path, available, problem, generation_now)
|
||||
|
||||
|
||||
def setting():
|
||||
"""The parsed setting in use (``preview_location.ParsedLocation``)."""
|
||||
return _parsed
|
||||
|
||||
|
||||
def generation():
|
||||
return _generation
|
||||
|
||||
|
||||
def bump_generation():
|
||||
"""Invalidate every write that started before (under ``write_lock``)."""
|
||||
global _generation
|
||||
with write_lock:
|
||||
_generation += 1
|
||||
return _generation
|
||||
|
||||
|
||||
def _canonical(path):
|
||||
"""Resolved and case-folded, so paths compare on every OS."""
|
||||
return Path(os.path.normcase(os.path.realpath(str(path))))
|
||||
|
||||
|
||||
def _within(path, folder):
|
||||
return _canonical(path).is_relative_to(_canonical(folder))
|
||||
|
||||
|
||||
# ── the setting and availability ─────────────────────────────────────
|
||||
|
||||
|
||||
def configure(value, *, context=None, check=True):
|
||||
"""Use the ``thumbnails.location`` *value* (config.json) from now on.
|
||||
|
||||
A folder location is validated against *context* (``startup`` rules: a
|
||||
problem other than an unreachable folder makes it ``invalid``), then
|
||||
checked once; the probe keeps checking it. Swapped under ``write_lock``
|
||||
with a generation bump, so no write that started before lands.
|
||||
"""
|
||||
global _parsed, _available, _problem, _generation
|
||||
parsed = preview_location.parse_location(value)
|
||||
problem = parsed.problem
|
||||
if parsed.mode == FOLDER_MODE and problem is None and context is not None:
|
||||
code = preview_location.location_problem(
|
||||
parsed.path, purpose="startup", context=context
|
||||
)
|
||||
if code == "location-unavailable":
|
||||
problem = "unreachable"
|
||||
elif code is not None:
|
||||
problem = "invalid"
|
||||
parsed = preview_location.ParsedLocation(
|
||||
mode=FOLDER_MODE,
|
||||
path=parsed.path,
|
||||
id=parsed.id,
|
||||
install=parsed.install,
|
||||
cleanup=parsed.cleanup,
|
||||
problem="invalid",
|
||||
)
|
||||
with write_lock:
|
||||
with _state_lock:
|
||||
_parsed = parsed
|
||||
_available = parsed.mode == SOURCE_MODE
|
||||
_problem = problem if parsed.mode == FOLDER_MODE else None
|
||||
_keys.clear()
|
||||
_key_cache.clear()
|
||||
_generation += 1
|
||||
if parsed.mode == FOLDER_MODE:
|
||||
if check and parsed.problem is None:
|
||||
refresh()
|
||||
_probe.start()
|
||||
else:
|
||||
_probe.stop()
|
||||
return current()
|
||||
|
||||
|
||||
def refresh():
|
||||
"""Check the folder now (bounded by IO_BUDGET); returns availability."""
|
||||
parsed = _parsed
|
||||
if parsed.mode != FOLDER_MODE or parsed.problem is not None:
|
||||
return parsed.mode == SOURCE_MODE
|
||||
try:
|
||||
available, problem, marker = _probe.check(parsed.path, parsed.id)
|
||||
except (OSError, TimeoutError, RuntimeError):
|
||||
available, problem, marker = False, "unreachable", None
|
||||
_set_state(parsed, available, problem, marker)
|
||||
return available
|
||||
|
||||
|
||||
def _set_keys(parsed, marker):
|
||||
"""Take the keys of a marker just written (availability unchanged)."""
|
||||
with _state_lock:
|
||||
if _parsed is not parsed:
|
||||
return
|
||||
keys = dict((marker or {}).get("keys") or {})
|
||||
if keys != _keys:
|
||||
_keys.clear()
|
||||
_keys.update(keys)
|
||||
_key_cache.clear()
|
||||
|
||||
|
||||
def mark_invalid():
|
||||
"""The setting cannot be used (a startup check that failed): folder
|
||||
mode stays in force, unavailable, so nothing is written anywhere."""
|
||||
global _parsed, _available, _problem
|
||||
with _state_lock:
|
||||
if _parsed.mode != FOLDER_MODE:
|
||||
_parsed = preview_location.ParsedLocation(
|
||||
mode=FOLDER_MODE, problem="invalid"
|
||||
)
|
||||
else:
|
||||
_parsed = replace(_parsed, problem="invalid")
|
||||
_available, _problem = False, "invalid"
|
||||
|
||||
|
||||
def _set_state(parsed, available, problem, marker):
|
||||
global _available, _problem
|
||||
with _state_lock:
|
||||
if _parsed is not parsed:
|
||||
return # the setting changed meanwhile
|
||||
_available, _problem = available, problem
|
||||
keys = dict((marker or {}).get("keys") or {})
|
||||
if keys != _keys:
|
||||
_keys.clear()
|
||||
_keys.update(keys)
|
||||
_key_cache.clear()
|
||||
|
||||
|
||||
def note_failure():
|
||||
"""A store call failed or stalled: unreachable until the probe finds the
|
||||
folder again (requests do no store I/O meanwhile)."""
|
||||
global _available, _problem
|
||||
if _parsed.mode != FOLDER_MODE:
|
||||
return
|
||||
with _state_lock:
|
||||
_available, _problem = False, "unreachable"
|
||||
_probe.wake()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def store_io():
|
||||
"""Around a store call in folder mode: an I/O error (other than a file
|
||||
that is not there) or a call over IO_BUDGET marks the folder
|
||||
unreachable."""
|
||||
if _parsed.mode != FOLDER_MODE:
|
||||
yield
|
||||
return
|
||||
started = _clock()
|
||||
try:
|
||||
yield
|
||||
except OSError as error:
|
||||
if not isinstance(error, (FileNotFoundError, NotADirectoryError)):
|
||||
note_failure()
|
||||
raise
|
||||
finally:
|
||||
if _clock() - started > IO_BUDGET:
|
||||
note_failure()
|
||||
|
||||
|
||||
class _Probe:
|
||||
"""The background check of the thumbnail folder (one thread, one check
|
||||
at a time; a stuck check is left to finish on its own thread)."""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._wake = threading.Event()
|
||||
self._stop = threading.Event()
|
||||
self._thread = None
|
||||
self._checking = None
|
||||
|
||||
def check(self, path, expected_id):
|
||||
"""``folder_state`` within IO_BUDGET; raises TimeoutError. A check
|
||||
already running (the probe's, or a stuck one) is joined, never run
|
||||
twice at once."""
|
||||
with self._lock:
|
||||
pending = self._checking
|
||||
if pending is None or pending[0] != (path, expected_id):
|
||||
pending = self._start_check(path, expected_id)
|
||||
key, done, result = pending
|
||||
if not done.wait(IO_BUDGET):
|
||||
raise TimeoutError("the thumbnail folder did not answer in time")
|
||||
with self._lock:
|
||||
if self._checking is pending:
|
||||
self._checking = None
|
||||
if "error" in result:
|
||||
raise result["error"]
|
||||
return result["value"]
|
||||
|
||||
def _start_check(self, path, expected_id):
|
||||
"""Start one check on its own thread (lock held)."""
|
||||
result = {}
|
||||
done = threading.Event()
|
||||
|
||||
def run():
|
||||
try:
|
||||
result["value"] = preview_location.folder_state(path, expected_id)
|
||||
except BaseException as error: # handed to the caller
|
||||
result["error"] = error
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
pending = ((path, expected_id), done, result)
|
||||
self._checking = pending
|
||||
threading.Thread(target=run, name=PROBE_THREAD + "-check", daemon=True).start()
|
||||
return pending
|
||||
|
||||
def start(self):
|
||||
with self._lock:
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return
|
||||
# Each thread has its own stop: a stopped one never resumes.
|
||||
stop = self._stop = threading.Event()
|
||||
thread = threading.Thread(
|
||||
target=self._run, args=(stop,), name=PROBE_THREAD, daemon=True
|
||||
)
|
||||
try:
|
||||
thread.start()
|
||||
except RuntimeError:
|
||||
return # refused at interpreter exit: no probe
|
||||
self._thread = thread
|
||||
|
||||
def _run(self, stop):
|
||||
while not stop.is_set():
|
||||
self._wake.wait(PROBE_INTERVAL)
|
||||
self._wake.clear()
|
||||
if stop.is_set():
|
||||
return
|
||||
try:
|
||||
refresh()
|
||||
except Exception: # one failed check never stops the probe
|
||||
logger.debug("Thumbnail folder check failed", exc_info=True)
|
||||
|
||||
def wake(self):
|
||||
self._wake.set()
|
||||
|
||||
def stop(self, timeout=2.0):
|
||||
"""Stop the probe; True when its thread ended."""
|
||||
with self._lock:
|
||||
thread, self._thread = self._thread, None
|
||||
self._stop.set()
|
||||
self._wake.set()
|
||||
if thread is None or thread is threading.current_thread():
|
||||
return True
|
||||
thread.join(timeout)
|
||||
return not thread.is_alive()
|
||||
|
||||
|
||||
_probe = _Probe()
|
||||
|
||||
|
||||
def stop_probe(timeout=2.0):
|
||||
"""Stop the folder probe (the package teardown, tests)."""
|
||||
return _probe.stop(timeout)
|
||||
|
||||
|
||||
# ── stores ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def source_folder(root):
|
||||
"""``<source>/thumbnails``: the store in source mode, a leftover in
|
||||
folder mode."""
|
||||
return Path(root) / PREVIEW_FOLDER
|
||||
|
||||
|
||||
def store_container(root, location=None):
|
||||
"""The folder a store of *root* must stay inside: the source itself, or
|
||||
the thumbnail folder in folder mode."""
|
||||
location = location or current()
|
||||
if location.mode == FOLDER_MODE:
|
||||
return Path(location.path) if location.path else None
|
||||
return Path(root)
|
||||
|
||||
|
||||
def _identity(root):
|
||||
return os.path.normcase(os.path.realpath(str(root)))
|
||||
|
||||
|
||||
def _lookup_key(root):
|
||||
spelling = str(root)
|
||||
with _state_lock:
|
||||
if spelling in _key_cache:
|
||||
return _key_cache[spelling]
|
||||
keys = dict(_keys)
|
||||
key = preview_location.find_key(
|
||||
keys,
|
||||
spelling,
|
||||
_identity(root),
|
||||
preview_location.file_identity(root),
|
||||
)
|
||||
if key is not None:
|
||||
with _state_lock:
|
||||
_key_cache[spelling] = key
|
||||
return key
|
||||
|
||||
|
||||
def store_dir(root, location=None):
|
||||
"""The store of *root*: ``<source>/thumbnails``, or in folder mode
|
||||
``<folder>/<key>`` (None while the folder is unavailable or before the
|
||||
source has a key: a reader then finds no preview)."""
|
||||
location = location or current()
|
||||
if location.mode == SOURCE_MODE:
|
||||
return Path(root) / PREVIEW_FOLDER
|
||||
if not location.available or not location.path:
|
||||
return None
|
||||
key = _lookup_key(root)
|
||||
return None if key is None else _key_folder(location.path, key)
|
||||
|
||||
|
||||
def _key_folder(folder, key):
|
||||
"""``<folder>/<key>``, only for a key that is one folder name directly
|
||||
below the thumbnail folder (a marker is data, never a path)."""
|
||||
if not preview_location.valid_key(key):
|
||||
return None
|
||||
store = Path(folder) / key
|
||||
if store.parent != Path(folder) or store.name != key:
|
||||
return None
|
||||
return store
|
||||
|
||||
|
||||
def ensure_store(root, location=None, *, create=False):
|
||||
"""``store_dir`` for a writer: in folder mode a source without a key gets
|
||||
one (recorded in the marker); with *create* its key folder is made (never
|
||||
the thumbnail folder itself). None while the folder is unavailable."""
|
||||
location = location or current()
|
||||
store = store_dir(root, location)
|
||||
if location.mode == SOURCE_MODE or (store is None and not location.available):
|
||||
return store
|
||||
if store is None:
|
||||
store = _assign_key(root, location)
|
||||
if store is None:
|
||||
return None
|
||||
if create:
|
||||
try:
|
||||
with store_io():
|
||||
os.mkdir(store)
|
||||
except FileExistsError:
|
||||
pass
|
||||
except OSError:
|
||||
return None
|
||||
return store
|
||||
|
||||
|
||||
def _assign_key(root, location):
|
||||
parsed = _parsed
|
||||
if location.path != parsed.path:
|
||||
return None
|
||||
spelling = str(root)
|
||||
identity = _identity(root)
|
||||
dev_ino = preview_location.file_identity(root)
|
||||
assigned = {}
|
||||
|
||||
def add(marker):
|
||||
if marker.get("id") != parsed.id:
|
||||
raise ValueError("another folder's marker") # marker-mismatch
|
||||
keys = marker["keys"]
|
||||
key = preview_location.find_key(keys, spelling, identity, dev_ino)
|
||||
if key is None:
|
||||
taken = {k: v.get("identity") for k, v in keys.items()}
|
||||
key = preview_location.source_key(spelling, identity, taken)
|
||||
keys[key] = {
|
||||
"path": spelling,
|
||||
"identity": identity,
|
||||
"dev_ino": dev_ino,
|
||||
"install": parsed.install,
|
||||
}
|
||||
assigned["key"] = key
|
||||
|
||||
try:
|
||||
with write_lock, store_io():
|
||||
marker = preview_location.write_marker(location.path, add)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
_set_keys(parsed, marker)
|
||||
return _key_folder(location.path, assigned["key"])
|
||||
|
||||
|
||||
def may_write(path, location=None):
|
||||
"""The one ownership check every preview write and delete passes.
|
||||
|
||||
Source mode: True (the callers' own root rules apply, as before). Folder
|
||||
mode: *path* lies below a source key folder of the thumbnail folder,
|
||||
lexically and resolved, and no link, junction or reparse point sits
|
||||
anywhere from the key folder down to it (the file itself included).
|
||||
"""
|
||||
location = location or current()
|
||||
if location.mode == SOURCE_MODE:
|
||||
return True
|
||||
if not location.path:
|
||||
return False
|
||||
folder = Path(os.path.abspath(location.path))
|
||||
path = Path(os.path.abspath(path))
|
||||
try:
|
||||
relative = path.relative_to(folder)
|
||||
except ValueError:
|
||||
return False
|
||||
if len(relative.parts) < 2 or ".." in relative.parts:
|
||||
return False
|
||||
store = _key_folder(folder, relative.parts[0])
|
||||
if store is None or not _within(path, folder):
|
||||
return False
|
||||
try:
|
||||
return _no_link_below(store, path)
|
||||
except OSError:
|
||||
note_failure()
|
||||
return False
|
||||
|
||||
|
||||
def make_parent(dst, location=None):
|
||||
"""Make the folders a preview *dst* is written into. In folder mode only
|
||||
folders below the thumbnail folder are made, never the folder itself,
|
||||
and only while it holds this install's marker (an unplugged drive or an
|
||||
unmounted mount point raises FileNotFoundError); a *dst* that does not
|
||||
pass ``may_write`` raises PermissionError."""
|
||||
location = location or current()
|
||||
dst = Path(dst)
|
||||
if location.mode == SOURCE_MODE or not location.path:
|
||||
if location.mode != SOURCE_MODE:
|
||||
raise FileNotFoundError(errno.ENOENT, "thumbnail folder unavailable")
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
return
|
||||
folder = Path(os.path.abspath(location.path))
|
||||
with store_io():
|
||||
if not location.available or not folder.is_dir():
|
||||
raise FileNotFoundError(errno.ENOENT, "thumbnail folder unavailable")
|
||||
marker = preview_location.read_marker(str(folder))
|
||||
if marker is None or marker.get("id") != _parsed.id:
|
||||
note_failure()
|
||||
raise FileNotFoundError(errno.ENOENT, "thumbnail folder unavailable")
|
||||
if not may_write(dst, location):
|
||||
raise PermissionError(errno.EACCES, "outside the thumbnail folder")
|
||||
current_folder = folder
|
||||
for part in Path(os.path.abspath(dst.parent)).relative_to(folder).parts:
|
||||
current_folder = current_folder / part
|
||||
try:
|
||||
os.mkdir(current_folder)
|
||||
except FileExistsError:
|
||||
pass
|
||||
if preview_location.is_link_like(current_folder):
|
||||
raise PermissionError(errno.EACCES, "a link in the thumbnail folder")
|
||||
|
||||
|
||||
def store_is_link(store, location=None):
|
||||
"""Whether a store folder is a link (a junction or reparse point too in
|
||||
folder mode): it is skipped by sweeps and refused by clears."""
|
||||
location = location or current()
|
||||
if location.mode == SOURCE_MODE:
|
||||
return Path(store).is_symlink()
|
||||
return preview_location.is_link_like(store)
|
||||
|
||||
|
||||
def given_store(root, store):
|
||||
"""A caller's explicit store argument: the default spelling
|
||||
(``<root>/thumbnails``) means the resolver's store; another folder (only
|
||||
tests pass one) is used as given."""
|
||||
store = Path(store)
|
||||
if store == Path(root) / PREVIEW_FOLDER:
|
||||
return store_dir(root)
|
||||
return store
|
||||
|
||||
|
||||
def preview_path(root, media_rel, ext=None, location=None, *, create=False):
|
||||
"""Where the preview of *media_rel* (relative to *root*) is, or is to be
|
||||
written (*create*: a writer, which gives a new source its key), in *ext*
|
||||
(the extension written now when None)."""
|
||||
if create:
|
||||
store = ensure_store(root, location)
|
||||
else:
|
||||
store = store_dir(root, location)
|
||||
if store is None:
|
||||
return None
|
||||
if ext is None:
|
||||
from .thumbnail_encode import writer_settings
|
||||
|
||||
ext = writer_settings().extension
|
||||
return store / (thumbnail_settings.preview_stem(media_rel) + ext)
|
||||
|
||||
|
||||
def request_path(root, parts, location=None):
|
||||
"""The disk path of the URL parts ``("thumbnails", ...)``, or None when
|
||||
*parts* is not under an exact ``thumbnails`` first part or the store has
|
||||
no place for it. The text is checked before (``is_preview_text``)."""
|
||||
parts = tuple(parts)
|
||||
if len(parts) < 2 or parts[0] != PREVIEW_FOLDER:
|
||||
return None
|
||||
store = store_dir(root, location)
|
||||
if store is None:
|
||||
return None
|
||||
return store.joinpath(*parts[1:])
|
||||
|
||||
|
||||
def _no_link_below(store, path):
|
||||
"""No folder from *store* down to *path* (and *store* itself) is a link,
|
||||
junction or reparse point (lexical walk, one lstat each)."""
|
||||
store = Path(os.path.abspath(store))
|
||||
path = Path(os.path.abspath(path))
|
||||
try:
|
||||
relative = path.relative_to(store)
|
||||
except ValueError:
|
||||
return False
|
||||
current = store
|
||||
for part in ("",) + relative.parts:
|
||||
current = current / part if part else current
|
||||
if preview_location.is_link_like(current):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def owns(root, path, location=None):
|
||||
"""Whether *path* lies inside the store of *root*, resolved: the store
|
||||
stays inside its container and no link takes *path* out of the store.
|
||||
In folder mode no link, junction or reparse point may sit anywhere from
|
||||
the store down to *path*."""
|
||||
location = location or current()
|
||||
store = store_dir(root, location)
|
||||
if store is None:
|
||||
return False
|
||||
container = store_container(root, location)
|
||||
if not (_within(store, container) and _within(path, store)):
|
||||
return False
|
||||
if location.mode == SOURCE_MODE:
|
||||
return True
|
||||
return may_write(path, location)
|
||||
|
||||
|
||||
# ── preview text ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def is_preview_text(filepath):
|
||||
"""URL text naming a preview: ``thumbnails/<...>/<name>_thumb<ext>``.
|
||||
|
||||
Text only, before any join: no backslash, drive or stream colon, and no
|
||||
empty, ``.`` or ``..`` segment; ``thumbnails`` exactly as the first part
|
||||
and nowhere else among the folders; the extension, in any case, one a
|
||||
preview may carry (``X_thumb.JPG`` of an older release included).
|
||||
"""
|
||||
if not filepath or "\\" in filepath or ":" in filepath:
|
||||
return False
|
||||
parts = filepath.split("/")
|
||||
if len(parts) < 2 or parts[0] != PREVIEW_FOLDER:
|
||||
return False
|
||||
if any(part in ("", ".", "..") for part in parts):
|
||||
return False
|
||||
if PREVIEW_FOLDER in parts[1:-1]:
|
||||
return False
|
||||
stem, extension = os.path.splitext(parts[-1])
|
||||
return (
|
||||
stem.endswith(thumbnail_settings.THUMB_SUFFIX)
|
||||
and len(stem) > len(thumbnail_settings.THUMB_SUFFIX)
|
||||
and extension.lower() in thumbnail_settings.THUMBNAIL_EXTENSIONS
|
||||
)
|
||||
|
||||
|
||||
# ── writes ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _WriteGuard:
|
||||
"""``guard()`` for one preview write: true while the location has not
|
||||
changed since the write started and *hold* (if any) allows it."""
|
||||
|
||||
__slots__ = ("generation", "_hold")
|
||||
|
||||
def __init__(self, hold=None):
|
||||
self.generation = _generation
|
||||
self._hold = hold
|
||||
|
||||
def __call__(self):
|
||||
if self.generation != _generation:
|
||||
return False
|
||||
return self._hold is None or bool(self._hold())
|
||||
|
||||
|
||||
def write_guard(hold=None):
|
||||
"""Take when a preview write starts; hand to its final rename."""
|
||||
if isinstance(hold, _WriteGuard):
|
||||
return hold
|
||||
return _WriteGuard(hold)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def committed(guard=None):
|
||||
"""Around a preview's final rename: holds ``write_lock`` and yields
|
||||
whether the write may land (``guard()``, re-checked under the lock)."""
|
||||
with write_lock:
|
||||
yield guard is None or bool(guard())
|
||||
@@ -8,10 +8,8 @@ import sqlite3
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
from ...database.library_commands import LibraryCommandError
|
||||
from ...database.pairs import PairDependencyError
|
||||
except ImportError:
|
||||
from database.library_commands import LibraryCommandError
|
||||
from database.pairs import PairDependencyError
|
||||
from urllib.parse import quote
|
||||
|
||||
@@ -1130,14 +1128,7 @@ class PromptRoutesMixin:
|
||||
async def bulk_add_tags(self, request):
|
||||
"""Bulk add tags to prompts."""
|
||||
try:
|
||||
try:
|
||||
data = await request.json()
|
||||
except ValueError:
|
||||
return bad_request("Invalid JSON body")
|
||||
if not isinstance(data, dict):
|
||||
return bad_request("Expected a JSON object")
|
||||
if "expected_epoch" in data:
|
||||
return await self._bulk_library_tags(data)
|
||||
data = await request.json()
|
||||
prompt_ids = data.get("prompt_ids", [])
|
||||
new_tags = data.get("tags", [])
|
||||
|
||||
@@ -1189,8 +1180,6 @@ class PromptRoutesMixin:
|
||||
)
|
||||
if "images_filter" in data:
|
||||
return await self._bulk_category_by_filter(data)
|
||||
if "expected_epoch" in data:
|
||||
return await self._bulk_library_category(data)
|
||||
if not data.get("prompt_ids"):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "No prompt IDs provided"}, status=400
|
||||
@@ -1227,54 +1216,6 @@ class PromptRoutesMixin:
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def _bulk_library_tags(self, data: dict) -> web.Response:
|
||||
"""The workspace Library's fenced form (#442).
|
||||
|
||||
``{prompt_ids, tags, expected_epoch}``: prompt tags only, never image tags.
|
||||
"""
|
||||
return await self._fenced_library_write(
|
||||
data, "tags", self.db.bulk_add_library_tags
|
||||
)
|
||||
|
||||
async def _fenced_library_write(
|
||||
self, data: dict, field: str, write
|
||||
) -> web.Response:
|
||||
"""One fenced Library bulk write: exact keys, checked ids, 409 on a restore."""
|
||||
fields = {"prompt_ids", field, "expected_epoch"}
|
||||
if set(data) != fields:
|
||||
return bad_request(
|
||||
"Request must contain exactly " + ", ".join(sorted(fields))
|
||||
)
|
||||
try:
|
||||
prompt_ids = parse_bulk_prompt_ids(data["prompt_ids"])
|
||||
result = await self._run_in_executor(
|
||||
write, prompt_ids, data[field], expected_epoch=data["expected_epoch"]
|
||||
)
|
||||
except LibraryCommandError as exc:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"code": exc.code,
|
||||
"library_epoch": exc.epoch,
|
||||
"error": "Prompt storage was restored; refresh the library",
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return bad_request(str(exc))
|
||||
return web.json_response({"success": True, **result})
|
||||
|
||||
async def _bulk_library_category(self, data: dict) -> web.Response:
|
||||
"""The workspace Library's fenced form (#437).
|
||||
|
||||
``{prompt_ids, category, expected_epoch}``: a non-blank category on
|
||||
the selected Library parents, refused with 409 ``storage-changed``
|
||||
when prompt storage was restored since the page read its epoch.
|
||||
"""
|
||||
return await self._fenced_library_write(
|
||||
data, "category", self.db.bulk_set_library_category
|
||||
)
|
||||
|
||||
async def _bulk_category_by_filter(self, data: dict) -> web.Response:
|
||||
"""Set the category on every linked positive a listing filter admits.
|
||||
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
"""Revalidated responses for the pages and static files PromptManager serves.
|
||||
|
||||
``Cache-Control: no-cache`` lets the browser keep a copy but makes it ask the
|
||||
server before every reuse, so a PromptManager update reaches the browser on a
|
||||
normal reload (#478). The ETag lets an unchanged file answer 304 without a
|
||||
body, which keeps that reload cheap.
|
||||
|
||||
The tag hashes the bytes rather than the file's mtime and size: an update can
|
||||
keep both (an archive extract, a same-length edit within one clock tick), and
|
||||
the handlers read the whole file anyway. It is weak because the gzip
|
||||
middleware may send the same content in a different encoding.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
REVALIDATE = "no-cache"
|
||||
|
||||
|
||||
def entity_tag(body: bytes) -> str:
|
||||
"""Weak ETag for ``body``."""
|
||||
return f'W/"{hashlib.sha256(body).hexdigest()[:32]}"'
|
||||
|
||||
|
||||
def etag_matches(if_none_match, etag: str) -> bool:
|
||||
"""True when an If-None-Match header names ``etag`` (weak comparison)."""
|
||||
if not if_none_match:
|
||||
return False
|
||||
if if_none_match.strip() == "*":
|
||||
return True
|
||||
opaque = etag.removeprefix("W/")
|
||||
return any(
|
||||
candidate.strip().removeprefix("W/") == opaque
|
||||
for candidate in if_none_match.split(",")
|
||||
)
|
||||
|
||||
|
||||
def revalidated_response(request, body: bytes, content_type: str, charset=None):
|
||||
"""200 with ``body``, or 304 when the request already holds this version."""
|
||||
etag = entity_tag(body)
|
||||
headers = {"Cache-Control": REVALIDATE, "ETag": etag}
|
||||
if etag_matches(request.headers.get("If-None-Match"), etag):
|
||||
return web.Response(status=304, headers=headers)
|
||||
return web.Response(
|
||||
body=body, content_type=content_type, charset=charset, headers=headers
|
||||
)
|
||||
@@ -9,10 +9,7 @@ candidates and the dependency fingerprint. ``POST .../items/{id}/decision``
|
||||
records one explicit decision: Apply links the chosen sampler's pair only
|
||||
when the capture is the item's bytes, the current file still hashes to them,
|
||||
the evidence they yield is unchanged and the ingester's compatible-only rules
|
||||
allow it; Leave unresolved links nothing. ``POST .../items/{id}/link`` (#536)
|
||||
links a Library prompt the reviewer chose, under the same capture and
|
||||
compatibility guards; it is their choice, so what the bytes resolve to is not
|
||||
asked.
|
||||
allow it; Leave unresolved links nothing.
|
||||
|
||||
Targets are formatted from the recorded spelling against the current roots
|
||||
(no file access beyond the roots' real paths); a file under no current root
|
||||
@@ -31,13 +28,12 @@ import urllib.parse
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .annotations import _capture_error, _error
|
||||
from .annotations import _HEX64, _TOKEN, _UUID, _capture_error, _error
|
||||
from .images import media_thumbnail_url
|
||||
from .inspection_service import VIDEO_EXTENSIONS
|
||||
from .job_routes import busy_response
|
||||
from .jobs import JobBusy
|
||||
from .prompts import bad_request
|
||||
from .review_guards import UNRESOLVED, _decision_guards, _link_guards
|
||||
|
||||
try:
|
||||
from ...database.library import display_title
|
||||
@@ -63,6 +59,17 @@ MAX_CURSOR = 4096
|
||||
MAX_BODY_BYTES = 64 * 1024
|
||||
# The serialized detail stays under this; past it the detail is ``limited``.
|
||||
MAX_DETAIL_BYTES = 8 * 1024 * 1024
|
||||
UNRESOLVED = "unresolved"
|
||||
_DECISION_FIELDS = frozenset(
|
||||
{
|
||||
"expected_epoch",
|
||||
"expected_revision",
|
||||
"capture_token",
|
||||
"sha256",
|
||||
"dependency_fingerprint",
|
||||
"choice",
|
||||
}
|
||||
)
|
||||
_CURSOR_SECRET = os.urandom(32)
|
||||
# The real path of a spelling; module-level so a test can fake a junction.
|
||||
_realpath = os.path.realpath
|
||||
@@ -77,7 +84,6 @@ _STATUS = {
|
||||
"storage-changed": 409,
|
||||
"source-changed": 409,
|
||||
"capture-expired": 409,
|
||||
"prompt-missing": 409,
|
||||
}
|
||||
_MESSAGES = {
|
||||
"review-missing": "This review item no longer exists. Reload the queue.",
|
||||
@@ -89,9 +95,6 @@ _MESSAGES = {
|
||||
"sampler path cannot be attached safely. The existing links were "
|
||||
"preserved."
|
||||
),
|
||||
"prompt-missing": (
|
||||
"That prompt or recipe is no longer in the Library. Search again."
|
||||
),
|
||||
"dependencies-changed": (
|
||||
"The image's saved links changed since this item was opened. Reopen it "
|
||||
"to see the current association."
|
||||
@@ -248,7 +251,6 @@ class ReviewRoutesMixin:
|
||||
routes.get(ITEMS_URL)(self.list_review_items)
|
||||
routes.get(ITEMS_URL + "/{item_id}")(self.get_review_item)
|
||||
routes.post(ITEMS_URL + "/{item_id}/decision")(self.decide_review_item)
|
||||
routes.post(ITEMS_URL + "/{item_id}/link")(self.link_review_item)
|
||||
|
||||
# -- shared ----------------------------------------------------------------
|
||||
|
||||
@@ -462,43 +464,6 @@ class ReviewRoutesMixin:
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def link_review_item(self, request: web.Request) -> web.Response:
|
||||
"""POST the Library prompt the reviewer chose for one pending item.
|
||||
|
||||
Under the same lease as a decision. The capture must be the item's
|
||||
current bytes; the choice and the file's links are checked in the
|
||||
decision's transaction.
|
||||
"""
|
||||
try:
|
||||
async with self._jobs.lease(REVIEW_DECISION_LEASE):
|
||||
return await self._link_review_item(request)
|
||||
except JobBusy as busy:
|
||||
return busy_response(busy)
|
||||
|
||||
async def _link_review_item(self, request: web.Request) -> web.Response:
|
||||
data, error = await self._read_json_body(request, max_bytes=MAX_BODY_BYTES)
|
||||
if error is not None:
|
||||
return error
|
||||
guards, error = _link_guards(data)
|
||||
if error is not None:
|
||||
return error
|
||||
try:
|
||||
item = await self._run_in_executor(
|
||||
self.db.review_item, request.match_info["item_id"]
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception("Review item read failed")
|
||||
return _failed()
|
||||
if item is None:
|
||||
return _coded("review-missing")
|
||||
verified, error = await self._verified_review_capture(
|
||||
item, guards, need_metadata=True
|
||||
)
|
||||
if error is not None:
|
||||
return error
|
||||
manual = {"positive_id": guards["positive_id"], "pair_id": guards["pair_id"]}
|
||||
return await self._record_decision(item, guards, None, verified, manual)
|
||||
|
||||
async def _decide_review_item(self, request: web.Request) -> web.Response:
|
||||
data, error = await self._read_json_body(request, max_bytes=MAX_BODY_BYTES)
|
||||
if error is not None:
|
||||
@@ -584,25 +549,22 @@ class ReviewRoutesMixin:
|
||||
return None, _coded("file-changed")
|
||||
metadata = None
|
||||
if need_metadata:
|
||||
# A manual link has no ``choice``: it rests on the bytes alone.
|
||||
if captured is None or (
|
||||
"choice" in guards
|
||||
and not _candidate_still_offered(captured[1], guards["choice"])
|
||||
if captured is None or not _candidate_still_offered(
|
||||
captured[1], guards["choice"]
|
||||
):
|
||||
return None, _coded("review-conflict", "evidence-changed")
|
||||
metadata = captured[0]
|
||||
image_path = os.path.join(str(root), *relative.split("/"))
|
||||
return (capture, facts, image_path, metadata), None
|
||||
|
||||
def _decide_sync(self, item, guards, candidate, verified, manual=None):
|
||||
def _decide_sync(self, item, guards, candidate, verified):
|
||||
"""Worker thread: resolve the file's spellings, decide, build the view.
|
||||
|
||||
Returns ``(result, view)``; a refusal raises with ``exc.view`` set.
|
||||
"""
|
||||
_capture, facts, image_path, metadata = verified or (None, None, None, None)
|
||||
aliases, known = (), None
|
||||
selected = candidate is not None or manual is not None
|
||||
if selected:
|
||||
if candidate is not None:
|
||||
aliases, known = physical_spellings(self.db, image_path, item["locator"])
|
||||
try:
|
||||
result = self.db.decide_review_item(
|
||||
@@ -611,9 +573,8 @@ class ReviewRoutesMixin:
|
||||
expected_revision=guards["expected_revision"],
|
||||
# The bytes the re-hash proved, or the body's for Leave unresolved.
|
||||
sha256=facts.sha256 if facts is not None else guards["sha256"],
|
||||
outcome="selected" if selected else UNRESOLVED,
|
||||
outcome="selected" if candidate is not None else UNRESOLVED,
|
||||
candidate=candidate,
|
||||
manual=manual,
|
||||
image_path=image_path,
|
||||
metadata=metadata,
|
||||
expected_fingerprint=guards["dependency_fingerprint"],
|
||||
@@ -628,11 +589,11 @@ class ReviewRoutesMixin:
|
||||
roots, real_roots = self._review_roots()
|
||||
return result, self._review_view(result["item"], roots, real_roots)
|
||||
|
||||
async def _record_decision(self, item, guards, candidate, verified, manual=None):
|
||||
async def _record_decision(self, item, guards, candidate, verified):
|
||||
capture = verified[0] if verified else None
|
||||
try:
|
||||
result, view = await self._run_in_executor(
|
||||
self._decide_sync, item, guards, candidate, verified, manual
|
||||
self._decide_sync, item, guards, candidate, verified
|
||||
)
|
||||
except ReviewCommandError as exc:
|
||||
return self._decision_refused(exc)
|
||||
@@ -668,6 +629,36 @@ class ReviewRoutesMixin:
|
||||
return _coded(exc.code, message, **extra)
|
||||
|
||||
|
||||
def _decision_guards(data: dict):
|
||||
"""The validated decision fields, or ``(None, error)``."""
|
||||
if set(data) != _DECISION_FIELDS:
|
||||
return None, bad_request(
|
||||
"Request must contain exactly " + ", ".join(sorted(_DECISION_FIELDS))
|
||||
)
|
||||
epoch, revision = data["expected_epoch"], data["expected_revision"]
|
||||
if not isinstance(epoch, str) or _UUID.fullmatch(epoch) is None:
|
||||
return None, bad_request("expected_epoch must be a canonical UUID")
|
||||
if type(revision) is not int or revision < 0:
|
||||
return None, bad_request("expected_revision must be a non-negative integer")
|
||||
token = data["capture_token"]
|
||||
if token is not None and (
|
||||
not isinstance(token, str) or _TOKEN.fullmatch(token) is None
|
||||
):
|
||||
return None, bad_request("Invalid capture token")
|
||||
for name in ("sha256", "dependency_fingerprint"):
|
||||
value = data[name]
|
||||
if not isinstance(value, str) or _HEX64.fullmatch(value) is None:
|
||||
return None, bad_request(
|
||||
f"{name} must be 64 lowercase hexadecimal characters"
|
||||
)
|
||||
choice = data["choice"]
|
||||
if not isinstance(choice, str) or (
|
||||
choice != UNRESOLVED and _HEX64.fullmatch(choice) is None
|
||||
):
|
||||
return None, bad_request("choice must be a candidate hash or unresolved")
|
||||
return {**data}, None
|
||||
|
||||
|
||||
def _verify_and_read(store, capture, item, root, need_metadata: bool):
|
||||
"""Worker thread: ``(relative, facts, captured)``, or None off the root.
|
||||
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
"""Body validation of Needs review decisions and manual links (#249 E3, #536)."""
|
||||
|
||||
from .annotations import _HEX64, _TOKEN, _UUID
|
||||
from .prompts import bad_request
|
||||
|
||||
UNRESOLVED = "unresolved"
|
||||
_GUARD_FIELDS = frozenset(
|
||||
{
|
||||
"expected_epoch",
|
||||
"expected_revision",
|
||||
"capture_token",
|
||||
"sha256",
|
||||
"dependency_fingerprint",
|
||||
}
|
||||
)
|
||||
_DECISION_FIELDS = _GUARD_FIELDS | {"choice"}
|
||||
_LINK_FIELDS = _GUARD_FIELDS | {"positive_id", "pair_id"}
|
||||
|
||||
|
||||
def _guard_error(data: dict, fields: frozenset):
|
||||
"""The first problem with a decision's shared fields, as a response."""
|
||||
if set(data) != fields:
|
||||
return bad_request("Request must contain exactly " + ", ".join(sorted(fields)))
|
||||
epoch, revision = data["expected_epoch"], data["expected_revision"]
|
||||
if not isinstance(epoch, str) or _UUID.fullmatch(epoch) is None:
|
||||
return bad_request("expected_epoch must be a canonical UUID")
|
||||
if type(revision) is not int or revision < 0:
|
||||
return bad_request("expected_revision must be a non-negative integer")
|
||||
token = data["capture_token"]
|
||||
if token is not None and (
|
||||
not isinstance(token, str) or _TOKEN.fullmatch(token) is None
|
||||
):
|
||||
return bad_request("Invalid capture token")
|
||||
for name in ("sha256", "dependency_fingerprint"):
|
||||
value = data[name]
|
||||
if not isinstance(value, str) or _HEX64.fullmatch(value) is None:
|
||||
return bad_request(f"{name} must be 64 lowercase hexadecimal characters")
|
||||
return None
|
||||
|
||||
|
||||
def _decision_guards(data: dict):
|
||||
"""The validated decision fields, or ``(None, error)``."""
|
||||
error = _guard_error(data, _DECISION_FIELDS)
|
||||
if error is not None:
|
||||
return None, error
|
||||
choice = data["choice"]
|
||||
if not isinstance(choice, str) or (
|
||||
choice != UNRESOLVED and _HEX64.fullmatch(choice) is None
|
||||
):
|
||||
return None, bad_request("choice must be a candidate hash or unresolved")
|
||||
return {**data}, None
|
||||
|
||||
|
||||
def _link_guards(data: dict):
|
||||
"""The validated manual link fields, or ``(None, error)``."""
|
||||
error = _guard_error(data, _LINK_FIELDS)
|
||||
if error is not None:
|
||||
return None, error
|
||||
if data["capture_token"] is None:
|
||||
return None, bad_request("A link needs a capture of the file")
|
||||
positive, pair = data["positive_id"], data["pair_id"]
|
||||
if type(positive) is not int or positive < 1:
|
||||
return None, bad_request("positive_id must be a positive integer")
|
||||
if pair is not None and (type(pair) is not int or pair < 1):
|
||||
return None, bad_request("pair_id must be a positive integer or null")
|
||||
return {**data}, None
|
||||
@@ -8,8 +8,7 @@ is shared with the LoRA integration's save.
|
||||
GET also reports the image sources and the monitor (``sources.py``). A save
|
||||
that changes the sources runs as a ``source_change`` registry hold, so it is
|
||||
refused while a record-changing job (a scan) runs. ``monitoring_enabled``
|
||||
takes effect at the next restart; ``save_foreign_runs`` (#247) at once, since
|
||||
the monitor reads it for every file. ``POST /prompt_manager/settings/validate-source``
|
||||
takes effect at the next restart. ``POST /prompt_manager/settings/validate-source``
|
||||
checks one path and writes nothing.
|
||||
"""
|
||||
|
||||
@@ -30,7 +29,6 @@ from . import thumbnail_settings
|
||||
from .admin import _current_image_monitor
|
||||
from .job_routes import busy_response
|
||||
from .jobs import THUMBNAILS_JOB_KIND, JobBusy
|
||||
from .thumbnail_location import needs_move_response
|
||||
|
||||
try:
|
||||
from ...utils.validators import validate_result_timeout, validate_worker_threads
|
||||
@@ -45,7 +43,6 @@ _SETTINGS_FILE_KEYS = {
|
||||
"INFINITE_SCROLL": ("web_ui", "infinite_scroll"),
|
||||
"WORKER_THREADS": ("performance", "worker_threads"),
|
||||
"MONITORING_ENABLED": ("gallery", "monitoring", "enabled"),
|
||||
"SAVE_FOREIGN_RUNS": ("gallery", "monitoring", "save_foreign_runs"),
|
||||
"SAFETY_BACKUP": ("maintenance", "safety_backup"),
|
||||
"THUMBNAIL_FORMAT": ("thumbnails", "format"),
|
||||
"THUMBNAIL_QUALITY": ("thumbnails", "quality"),
|
||||
@@ -61,7 +58,7 @@ _SETTINGS_FILE_KEYS = {
|
||||
# The thumbnail settings a running Convert outdated job fixes (#317).
|
||||
_THUMBNAIL_SETTINGS = ("THUMBNAIL_FORMAT", "THUMBNAIL_QUALITY", "THUMBNAIL_EDGE")
|
||||
# Attributes that live on GalleryConfig; the others on PromptManagerConfig.
|
||||
_GALLERY_SETTINGS = frozenset({"MONITORING_ENABLED", "SAVE_FOREIGN_RUNS"})
|
||||
_GALLERY_SETTINGS = frozenset({"MONITORING_ENABLED"})
|
||||
# Attributes that live on AutotagConfig, named without the AUTOTAG_ prefix.
|
||||
_AUTOTAG_SETTINGS = frozenset(
|
||||
name for name in _SETTINGS_FILE_KEYS if name.startswith("AUTOTAG_")
|
||||
@@ -265,15 +262,8 @@ class SettingsRoutesMixin:
|
||||
"sources": source_status,
|
||||
"monitor": monitor,
|
||||
"thumbnail_encoders": encoders,
|
||||
# #320: the full path only here (D6A).
|
||||
"thumbnail_location": await self._run_in_executor(
|
||||
functools.partial(
|
||||
self._thumbnail_location_payload, with_path=True
|
||||
)
|
||||
),
|
||||
"settings": {
|
||||
"monitoring_enabled": GalleryConfig.MONITORING_ENABLED,
|
||||
"save_foreign_runs": GalleryConfig.SAVE_FOREIGN_RUNS,
|
||||
"result_timeout": PromptManagerConfig.RESULT_TIMEOUT,
|
||||
"webui_display_mode": PromptManagerConfig.WEBUI_DISPLAY_MODE,
|
||||
"infinite_scroll": PromptManagerConfig.INFINITE_SCROLL,
|
||||
@@ -384,9 +374,6 @@ class SettingsRoutesMixin:
|
||||
if error_response is not None:
|
||||
return error_response
|
||||
|
||||
if "thumbnail_location" in data:
|
||||
# #320: only the Move thumbnails job changes the location.
|
||||
return needs_move_response()
|
||||
expected_revision = data.get("expected_revision")
|
||||
if expected_revision is not None and not isinstance(expected_revision, str):
|
||||
return web.json_response(
|
||||
@@ -639,10 +626,6 @@ class SettingsRoutesMixin:
|
||||
if not isinstance(data["monitoring_enabled"], bool):
|
||||
raise ValueError("monitoring_enabled must be true or false")
|
||||
updates["MONITORING_ENABLED"] = data["monitoring_enabled"]
|
||||
if "save_foreign_runs" in data:
|
||||
if not isinstance(data["save_foreign_runs"], bool):
|
||||
raise ValueError("save_foreign_runs must be true or false")
|
||||
updates["SAVE_FOREIGN_RUNS"] = data["save_foreign_runs"]
|
||||
if "safety_backup" in data:
|
||||
if not isinstance(data["safety_backup"], bool):
|
||||
raise ValueError("safety_backup must be true or false")
|
||||
|
||||
@@ -41,13 +41,6 @@ def _coded(status: int, code: str, message: str) -> web.Response:
|
||||
)
|
||||
|
||||
|
||||
def _missing_if_not_found(response: web.Response) -> web.Response:
|
||||
"""A file in a trash folder answers as a missing file does (#543)."""
|
||||
if response.status == 404:
|
||||
return _coded(404, "file-missing", "Image file not found")
|
||||
return response
|
||||
|
||||
|
||||
def _no_store(response: web.Response) -> web.Response:
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
@@ -78,7 +71,7 @@ class SourceRescopeMixin:
|
||||
path = root / relative
|
||||
denied = _media_access_error(path, [root])
|
||||
if denied is not None:
|
||||
return _missing_if_not_found(denied)
|
||||
return denied
|
||||
try:
|
||||
mtime = path.stat().st_mtime if path.is_file() else None
|
||||
except OSError:
|
||||
@@ -96,7 +89,7 @@ class SourceRescopeMixin:
|
||||
return bad_request("Send exactly one target")
|
||||
target, error = media_target(data["target"])
|
||||
if error is not None:
|
||||
return _missing_if_not_found(error)
|
||||
return error
|
||||
old_index, scope, relative = target
|
||||
if old_index > MAX_ROOT_INDEX:
|
||||
return bad_request("Invalid gallery root")
|
||||
|
||||