Vito Sansevero
2dbf79c9ae
The Images page, its sidebar counts, the subfolder list and the legacy gallery each rebuilt the output listing when the 30 s cache expired, all at once, and the subfolder list computed relative paths on the event loop. Thumbnail writes also threw the listing away. Keep one RootListing per output root (py/api/gallery_listing.py): - one walk per root at a time; concurrent readers wait for it (shielded) - a listing older than 30 s is served at once and refreshed in the background - the walk, the comparison with the previous listing and both folder sets run in the executor; an unchanged walk keeps the same list object - os.scandir probe: a missing root fails the walk and keeps the previous listing (one warning per failure streak) - Rescan waits for a walk started after the click and is a 500 when it fails, even with a previous listing - deletes keep the blocking invalidation; a generation counter that survives drops discards walks that raced it - thumbnail jobs, conversion and the legacy generate/clear routes no longer drop the listing - warm-up on the host's on_startup, cancelled on on_cleanup Adds tools/vendor/perf/images_load_probe.mjs (maintainer tool, not CI) and the TESTING.md section "Images listing cache (#348)".
445 lines
19 KiB
Python
445 lines
19 KiB
Python
"""Convert outdated thumbnails (#317): an explicit, cancellable Activity job.
|
|
|
|
``POST /prompt_manager/jobs/thumbnails {mode: "convert"}`` runs under the
|
|
thumbnail job's kind, admission (``guarded``) and worker threads. For every
|
|
media file whose preview is in another format than the one written now, it
|
|
writes the new preview from the SOURCE (verified, atomic), then removes that
|
|
file's old-format previews under ``thumbnails/``. A failure keeps the old
|
|
preview; previews whose source is gone (orphans) are never touched. The final
|
|
line reports converted / failed and bytes before → after.
|
|
|
|
Ordering is driven by events, never by timers.
|
|
"""
|
|
|
|
import asyncio
|
|
import os
|
|
import sys
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
try:
|
|
from tests.test_api_images import ThumbnailTestCase, make_png
|
|
except ImportError: # discovered with tests/ as the top-level directory
|
|
from test_api_images import ThumbnailTestCase, make_png
|
|
|
|
from PIL import Image # noqa: E402
|
|
|
|
from py.api import thumbnail_encode # noqa: E402
|
|
from py.config import PromptManagerConfig # noqa: E402
|
|
|
|
JOB = "/prompt_manager/jobs/thumbnails"
|
|
PLAN = "/prompt_manager/images/thumbnails/plan"
|
|
STATUS = "/prompt_manager/images/thumbnails/status"
|
|
WAIT = 10
|
|
|
|
|
|
def big_png(path, size=(512, 512)):
|
|
"""A noisy PNG: its PNG preview is large, a WebP one much smaller."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
Image.effect_noise(size, 40).convert("RGB").save(path, "PNG")
|
|
return path
|
|
|
|
|
|
class ConvertCase(ThumbnailTestCase):
|
|
async def asyncSetUp(self):
|
|
await super().asyncSetUp()
|
|
saved = PromptManagerConfig.get_config()["thumbnails"]
|
|
self.addCleanup(PromptManagerConfig.update_config, {"thumbnails": saved})
|
|
PromptManagerConfig.update_config(
|
|
{"thumbnails": {"format": "webp", "quality": 80, "edge": 512}}
|
|
)
|
|
thumbnail_encode.reset_encoders()
|
|
self.addCleanup(thumbnail_encode.reset_encoders)
|
|
self.addCleanup(self.api.close_auto_thumbnails)
|
|
|
|
def _legacy(self, name, suffix=".png", source=True):
|
|
"""A media file (unless ``source`` is False) and its old-format preview."""
|
|
if source:
|
|
big_png(self.output_dir / f"{name}.png")
|
|
preview = big_png(self._thumb_path(name, suffix), size=(256, 256))
|
|
if source:
|
|
later = (self.output_dir / f"{name}.png").stat().st_mtime + 5
|
|
os.utime(preview, (later, later))
|
|
return preview
|
|
|
|
async def _convert(self):
|
|
resp = await self.client.post(JOB, json={"mode": "convert"})
|
|
body = await resp.json()
|
|
self.assertEqual(resp.status, 202, body)
|
|
job = self.api._jobs.get(body["job"]["id"])
|
|
await asyncio.wait_for(job.task, WAIT)
|
|
return body["job"], job
|
|
|
|
|
|
class TestConvertOutdated(ConvertCase):
|
|
async def test_plan_counts_the_outdated_previews(self):
|
|
self._legacy("a")
|
|
self._legacy("b")
|
|
make_png(self.output_dir / "new.png") # missing: not this job's work
|
|
plan = await (await self.client.get(PLAN, params={"mode": "convert"})).json()
|
|
self.assertEqual(
|
|
(plan["mode"], plan["targets"], plan["outdated"]), ("convert", 2, 2)
|
|
)
|
|
self.assertEqual(self.api._jobs.list(), [])
|
|
|
|
async def test_conversion_keeps_the_listing(self):
|
|
self._legacy("a")
|
|
await self.client.get("/prompt_manager/images/output?mode=workspace")
|
|
files = self.api._root_listings[str(self.output_dir)].files
|
|
_summary, job = await self._convert()
|
|
self.assertEqual(job.state, "finished")
|
|
self.assertIs(self.api._root_listings[str(self.output_dir)].files, files)
|
|
|
|
async def test_converts_from_the_source_and_removes_the_old_preview(self):
|
|
old = self._legacy("a")
|
|
original = (self.output_dir / "a.png").read_bytes()
|
|
orphan = self._legacy("gone", source=False)
|
|
make_png(self.output_dir / "new.png")
|
|
before = old.stat().st_size
|
|
|
|
summary, job = await self._convert()
|
|
|
|
self.assertEqual(
|
|
(summary["kind"], summary["admission"], summary["cancellable"]),
|
|
("thumbnails", "guarded", True),
|
|
)
|
|
self.assertEqual(summary["title"], "Convert outdated thumbnails")
|
|
self.assertEqual(summary["scope"]["mode"], "convert")
|
|
self.assertEqual(job.state, "finished")
|
|
self.assertEqual(job.counters["generated"], 1)
|
|
self.assertEqual(job.counters["errors"], 0)
|
|
new = self._thumb_path("a")
|
|
with Image.open(new) as img:
|
|
self.assertEqual((img.format, max(img.size)), ("WEBP", 512))
|
|
self.assertFalse(old.exists(), "the old PNG preview is removed")
|
|
self.assertEqual((self.output_dir / "a.png").read_bytes(), original)
|
|
self.assertTrue(orphan.exists(), "an orphan preview is never touched")
|
|
self.assertFalse(self._thumb_path("new").exists(), "missing is not converted")
|
|
after = new.stat().st_size
|
|
self.assertRegex(job.status_line, r"^1 converted · 0 failed · .+ → .+$")
|
|
self.assertTrue(any("Conversion finished" in line for line in job.log))
|
|
self.assertLess(after, before)
|
|
status = await (await self.client.get(STATUS)).json()
|
|
self.assertEqual((status["outdated"], status["pending"]), (0, 1))
|
|
|
|
async def test_a_source_that_fails_keeps_its_old_preview(self):
|
|
old = self._legacy("bad")
|
|
(self.output_dir / "bad.png").write_bytes(b"\x89PNG broken")
|
|
later = old.stat().st_mtime + 10
|
|
os.utime(old, (later, later))
|
|
keep = old.read_bytes()
|
|
|
|
_, job = await self._convert()
|
|
|
|
self.assertEqual(job.counters["errors"], 1)
|
|
self.assertEqual(job.counters["generated"], 0)
|
|
self.assertEqual(old.read_bytes(), keep)
|
|
self.assertFalse(self._thumb_path("bad").exists())
|
|
self.assertRegex(job.status_line, r"^0 converted · 1 failed")
|
|
|
|
async def test_a_video_without_a_frame_keeps_its_old_preview(self):
|
|
(self.output_dir / "clip.mp4").write_bytes(b"no decoder")
|
|
old = big_png(self._thumb_path("clip", ".jpg"), size=(64, 64))
|
|
keep = old.read_bytes()
|
|
with (
|
|
patch.object(self.api, "_opencv_frame", return_value=None),
|
|
patch.object(self.api, "_ffmpeg_frame", return_value=None),
|
|
):
|
|
_, job = await self._convert()
|
|
self.assertEqual(job.counters["errors"], 1)
|
|
self.assertEqual(old.read_bytes(), keep)
|
|
self.assertFalse(self._thumb_path("clip").exists(), "never the placeholder")
|
|
|
|
async def test_an_old_preview_that_is_a_link_is_left(self):
|
|
big_png(self.output_dir / "l.png")
|
|
target = big_png(self.comfy_dir / "elsewhere.png", size=(32, 32))
|
|
link = self._thumb_path("l", ".png")
|
|
link.parent.mkdir(parents=True, exist_ok=True)
|
|
self._make_symlink(target, link)
|
|
_, job = await self._convert()
|
|
self.assertTrue(link.is_symlink())
|
|
self.assertTrue(target.exists())
|
|
self.assertEqual(job.counters["errors"], 0)
|
|
|
|
async def test_a_preview_already_current_only_loses_its_leftovers(self):
|
|
old = self._legacy("c")
|
|
current = big_png(self._thumb_path("c"), size=(64, 64))
|
|
later = old.stat().st_mtime + 5
|
|
os.utime(current, (later, later))
|
|
written = current.read_bytes()
|
|
_, job = await self._convert()
|
|
self.assertEqual(job.counters["generated"], 1)
|
|
self.assertEqual(current.read_bytes(), written, "not written again")
|
|
self.assertFalse(old.exists())
|
|
|
|
async def test_cancel_stops_at_a_file_boundary_keeping_what_converted(self):
|
|
original = PromptManagerConfig.WORKER_THREADS
|
|
PromptManagerConfig.WORKER_THREADS = 1
|
|
self.addCleanup(setattr, PromptManagerConfig, "WORKER_THREADS", original)
|
|
olds = [self._legacy(f"p{i}") for i in range(6)]
|
|
done = []
|
|
real = self.api._convert_one
|
|
job_ids = []
|
|
|
|
def counting(*args, **kwargs):
|
|
result = real(*args, **kwargs)
|
|
done.append(result)
|
|
if len(done) == 2:
|
|
self.assertTrue(self.api._jobs.cancel(job_ids[0]))
|
|
return result
|
|
|
|
self.api._convert_one = counting
|
|
resp = await self.client.post(JOB, json={"mode": "convert"})
|
|
job_ids.append((await resp.json())["job"]["id"])
|
|
job = self.api._jobs.get(job_ids[0])
|
|
await asyncio.wait_for(job.task, WAIT)
|
|
|
|
self.assertEqual(job.state, "cancelled")
|
|
self.assertEqual(job.counters["generated"], 2)
|
|
converted = [p for p in olds if not p.exists()]
|
|
self.assertEqual(len(converted), 2)
|
|
for old in olds:
|
|
webp = old.with_suffix(".webp")
|
|
self.assertEqual(webp.exists(), not old.exists(), old.name)
|
|
self.assertTrue(any("Conversion cancelled" in line for line in job.log))
|
|
|
|
async def test_refused_beside_a_running_thumbnail_job_or_scan(self):
|
|
self._legacy("a")
|
|
entered, release = threading.Event(), threading.Event()
|
|
self.addCleanup(release.set)
|
|
real = self.api._convert_one
|
|
|
|
def gated(*args, **kwargs):
|
|
entered.set()
|
|
release.wait(WAIT)
|
|
return real(*args, **kwargs)
|
|
|
|
self.api._convert_one = gated
|
|
resp = await self.client.post(JOB, json={"mode": "convert"})
|
|
first = (await resp.json())["job"]["id"]
|
|
loop = asyncio.get_running_loop()
|
|
self.assertTrue(await loop.run_in_executor(None, entered.wait, WAIT))
|
|
again = await self.client.post(JOB, json={"mode": "missing"})
|
|
self.assertEqual(again.status, 409)
|
|
scan = await self.client.post("/prompt_manager/jobs/scan")
|
|
self.assertEqual(scan.status, 409)
|
|
release.set()
|
|
await asyncio.wait_for(self.api._jobs.get(first).task, WAIT)
|
|
|
|
async def test_the_legacy_gallery_stream_never_attaches_to_a_conversion(self):
|
|
try:
|
|
from tests.test_api_images import parse_sse
|
|
except ImportError:
|
|
from test_api_images import parse_sse
|
|
|
|
self._legacy("a")
|
|
entered, release = threading.Event(), threading.Event()
|
|
self.addCleanup(release.set)
|
|
real = self.api._convert_one
|
|
|
|
def gated(*args, **kwargs):
|
|
entered.set()
|
|
release.wait(WAIT)
|
|
return real(*args, **kwargs)
|
|
|
|
self.api._convert_one = gated
|
|
resp = await self.client.post(JOB, json={"mode": "convert"})
|
|
first = (await resp.json())["job"]["id"]
|
|
loop = asyncio.get_running_loop()
|
|
self.assertTrue(await loop.run_in_executor(None, entered.wait, WAIT))
|
|
stream = await self.client.post(self.PROGRESS_URL)
|
|
events = parse_sse(await stream.text())
|
|
self.assertEqual(events[-1][0], "error")
|
|
self.assertIn("Convert outdated thumbnails", events[-1][1]["error"])
|
|
release.set()
|
|
await asyncio.wait_for(self.api._jobs.get(first).task, WAIT)
|
|
|
|
async def test_nothing_outdated_says_so(self):
|
|
big_png(self.output_dir / "a.png")
|
|
_, job = await self._convert()
|
|
self.assertEqual(job.state, "finished")
|
|
self.assertEqual(job.status_line, "No outdated previews to convert")
|
|
|
|
async def test_the_source_vanishing_keeps_the_old_preview(self):
|
|
old = self._legacy("v")
|
|
real = self.api._generate_one
|
|
|
|
def vanish(src, *args, **kwargs):
|
|
os.remove(src)
|
|
return real(src, *args, **kwargs)
|
|
|
|
self.api._generate_one = vanish
|
|
_, job = await self._convert()
|
|
self.assertEqual(job.counters["errors"], 1)
|
|
self.assertTrue(old.exists(), "a source gone mid-run makes an orphan")
|
|
|
|
async def test_same_stem_sources_convert_once_and_count_bytes_once(self):
|
|
# Review I2 / R2: x.png and x.jpg share x_thumb.webp. The pair is
|
|
# converted once, from the image first in the source order (.png),
|
|
# and every removed file's bytes count once.
|
|
original = PromptManagerConfig.WORKER_THREADS
|
|
PromptManagerConfig.WORKER_THREADS = 2
|
|
self.addCleanup(setattr, PromptManagerConfig, "WORKER_THREADS", original)
|
|
from py.api.thumbnail_convert import readable_bytes
|
|
|
|
png = self.output_dir / "x.png"
|
|
Image.new("RGB", (300, 300), (220, 20, 20)).save(png)
|
|
jpg = self.output_dir / "x.jpg"
|
|
Image.new("RGB", (300, 300), (20, 20, 220)).save(jpg)
|
|
old_png = big_png(self._thumb_path("x", ".png"), size=(200, 200))
|
|
old_jpg = self._thumb_path("x", ".jpg")
|
|
Image.effect_noise((200, 200), 40).convert("RGB").save(old_jpg, "JPEG")
|
|
later = max(png.stat().st_mtime, jpg.stat().st_mtime) + 5
|
|
for path in (old_png, old_jpg):
|
|
os.utime(path, (later, later))
|
|
before = old_png.stat().st_size + old_jpg.stat().st_size
|
|
|
|
_, job = await self._convert()
|
|
|
|
self.assertEqual(job.state, "finished")
|
|
self.assertEqual(
|
|
(job.counters["generated"], job.counters["errors"]), (2, 0), job.log
|
|
)
|
|
new = self._thumb_path("x")
|
|
self.assertFalse(old_png.exists() or old_jpg.exists())
|
|
with Image.open(new) as img:
|
|
red, _green, blue = img.convert("RGB").getpixel((5, 5))
|
|
self.assertGreater(red, blue, "the image first in the source order wins")
|
|
self.assertEqual(
|
|
job.status_line,
|
|
f"2 converted · 0 failed · {readable_bytes(before)} → "
|
|
f"{readable_bytes(new.stat().st_size)}",
|
|
)
|
|
|
|
async def test_an_old_preview_already_gone_counts_as_removed(self):
|
|
old = self._legacy("g")
|
|
real = self.api._generate_one
|
|
|
|
def remove_first(src, *args, **kwargs):
|
|
os.remove(old)
|
|
return real(src, *args, **kwargs)
|
|
|
|
self.api._generate_one = remove_first
|
|
_, job = await self._convert()
|
|
self.assertEqual((job.counters["generated"], job.counters["errors"]), (1, 0))
|
|
self.assertRegex(job.status_line, r"^1 converted · 0 failed · 0 B → ")
|
|
|
|
async def test_a_current_preview_that_does_not_decode_is_written_again(self):
|
|
# Review M4: a kept current preview is decoded before old ones go.
|
|
old = self._legacy("m")
|
|
current = self._thumb_path("m")
|
|
current.write_bytes(b"RIFF not a webp")
|
|
later = old.stat().st_mtime + 5
|
|
os.utime(current, (later, later))
|
|
_, job = await self._convert()
|
|
self.assertEqual((job.counters["generated"], job.counters["errors"]), (1, 0))
|
|
with Image.open(current) as img:
|
|
self.assertEqual(img.format, "WEBP")
|
|
self.assertFalse(old.exists())
|
|
|
|
async def test_a_format_change_underneath_stops_the_job(self):
|
|
# Review I4 (defense in depth): the job never removes previews in a
|
|
# format chosen after it started.
|
|
original = PromptManagerConfig.WORKER_THREADS
|
|
PromptManagerConfig.WORKER_THREADS = 1
|
|
self.addCleanup(setattr, PromptManagerConfig, "WORKER_THREADS", original)
|
|
olds = [self._legacy(f"f{i}") for i in range(4)]
|
|
real = self.api._convert_one
|
|
calls = []
|
|
|
|
def switching(*args, **kwargs):
|
|
result = real(*args, **kwargs)
|
|
calls.append(result)
|
|
if len(calls) == 1:
|
|
PromptManagerConfig.update_config(
|
|
{"thumbnails": {"format": "jpeg", "quality": 80, "edge": 512}}
|
|
)
|
|
jpeg = self._thumb_path("f3", ".jpg")
|
|
Image.new("RGB", (16, 16)).save(jpeg, "JPEG")
|
|
return result
|
|
|
|
self.api._convert_one = switching
|
|
_, job = await self._convert()
|
|
self.assertEqual(job.state, "cancelled")
|
|
self.assertIn("format changed", job.status_line)
|
|
self.assertEqual(job.counters["generated"], 1)
|
|
self.assertTrue(self._thumb_path("f3", ".jpg").exists(), "the new format stays")
|
|
self.assertEqual(sum(1 for p in olds if p.exists()), 3)
|
|
|
|
async def test_the_new_preview_is_never_removed_as_an_old_one(self):
|
|
# Re-review N1: on a case-insensitive disk x_thumb.JPG (an old
|
|
# spelling) can be the x_thumb.jpg just written. A hard link stands
|
|
# in for that second name of one file.
|
|
PromptManagerConfig.update_config(
|
|
{"thumbnails": {"format": "jpeg", "quality": 80, "edge": 512}}
|
|
)
|
|
src = self.output_dir / "x.jpg"
|
|
Image.new("RGB", (64, 64), (200, 10, 10)).save(src, "JPEG")
|
|
alias = self._thumb_path("x", ".JPG")
|
|
alias.parent.mkdir(parents=True, exist_ok=True)
|
|
Image.new("RGB", (16, 16)).save(alias, "JPEG")
|
|
later = src.stat().st_mtime + 5
|
|
os.utime(alias, (later, later))
|
|
real = self.api._generate_one
|
|
|
|
def same_file(source, dst, *args, **kwargs):
|
|
made = real(source, dst, *args, **kwargs)
|
|
os.remove(alias)
|
|
os.link(made["path"], alias)
|
|
return made
|
|
|
|
self.api._generate_one = same_file
|
|
_, job = await self._convert()
|
|
self.assertTrue(self._thumb_path("x", ".jpg").exists())
|
|
self.assertTrue(alias.exists(), "the new preview's other name is kept")
|
|
self.assertEqual(job.counters["errors"], 0)
|
|
|
|
def test_same_file_falls_back_to_the_path_without_inode_numbers(self):
|
|
from py.api import thumbnail_convert as tc
|
|
|
|
a = self.output_dir / "a_thumb.jpg"
|
|
b = self.output_dir / "b_thumb.jpg"
|
|
for path in (a, b):
|
|
path.write_bytes(b"x")
|
|
real_stat = os.stat
|
|
|
|
def no_inode(path, *args, **kwargs):
|
|
result = real_stat(path, *args, **kwargs)
|
|
return os.stat_result(
|
|
(result.st_mode, 0, result.st_dev) + tuple(result)[3:]
|
|
)
|
|
|
|
with patch.object(tc.os, "stat", side_effect=no_inode):
|
|
self.assertFalse(tc._same_file(a, b))
|
|
self.assertTrue(tc._same_file(a, a))
|
|
self.assertTrue(tc._same_file(a, self.output_dir / "." / "a_thumb.jpg"))
|
|
self.assertFalse(tc._same_file(a, self.output_dir / "gone.jpg"))
|
|
|
|
async def test_convert_goes_on_past_the_per_batch_cap(self):
|
|
import py.api.images as images_module
|
|
|
|
olds = [self._legacy(f"c{i}") for i in range(8)]
|
|
with patch.object(images_module, "MAX_THUMBNAIL_FILES", 3):
|
|
_, job = await self._convert()
|
|
self.assertEqual(job.state, "finished")
|
|
self.assertEqual(job.counters["generated"], 8)
|
|
self.assertFalse(any(old.exists() for old in olds))
|
|
self.assertEqual(job.summary()["total"], 8)
|
|
self.assertRegex(job.status_line, r"^8 converted · 0 failed · ")
|
|
self.assertNotIn("next run", job.status_line)
|
|
|
|
def test_byte_sizes_are_readable(self):
|
|
from py.api.thumbnail_convert import readable_bytes
|
|
|
|
self.assertEqual(readable_bytes(0), "0 B")
|
|
self.assertEqual(readable_bytes(1536), "1.5 KB")
|
|
self.assertEqual(readable_bytes(5 * 1024 * 1024), "5.0 MB")
|
|
self.assertEqual(readable_bytes(3 * 1024**3), "3.0 GB")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|