feat(thumbnails): keep previews in one folder, set from config.json (#320)
thumbnails.location in config.json can now name one folder for every source's previews: <folder>/<source key>/<rel>_thumb<ext>. A source key is a readable slug plus a short hash of the source's identity, stored in the folder's marker (.promptmanager-thumbnails.json) with the install that made it. Serving, on-demand and capture writes, listings, clear, delete, duplicates, the temp sweep and History covers all follow it; a leftover in <source>/thumbnails is never served and other text under thumbnails/ is refused. A missing, unplugged or foreign folder is unavailable (a probe checks it every 5 s within 2 s; a stalled or failing store call flips it at once): originals are served, nothing is written anywhere, the folder is never re-created, and generate, convert and clear answer 409 location-unavailable. A bad config value is unavailable too, never a fall-back to the sources. No UI yet: POST /settings refuses thumbnail_location (409 location-needs-move); GET /settings reports it with its path, the thumbnail status without one. Sources inside or around the folder and a library move into it are refused.
This commit is contained in:
@@ -22,6 +22,9 @@ from PIL.PngImagePlugin import PngInfo
|
||||
from tests.browser.folder_trash import close_trash, replace_folder
|
||||
from tests.browser.malloc_tuning import pin_mmap_threshold
|
||||
from tests.browser.request_hold import register_request_holds
|
||||
from tests.browser.thumbnail_location_fixture import (
|
||||
register_thumbnail_location_commands,
|
||||
)
|
||||
from tests.browser import slow_reset_watch
|
||||
from tests.browser.fast_reset import SEED_ON_LIVE_ENV, ScratchSeeder
|
||||
from tests.browser.workspace_activity_fixture import register_activity_commands
|
||||
@@ -240,6 +243,7 @@ def create_app(root: Path) -> web.Application:
|
||||
library_editor_commands = register_library_editor_commands(routes, api, root)
|
||||
register_vocabulary_commands(routes, api, root)
|
||||
restore_sources = register_source_change_commands(routes, api)
|
||||
reset_thumbnail_location = register_thumbnail_location_commands(routes, api, root)
|
||||
register_tagging_commands(routes, api, root, model, on_reset=restore_sources)
|
||||
register_review_commands(routes, api, root)
|
||||
register_collections_commands(routes, api, root)
|
||||
@@ -451,6 +455,8 @@ def create_app(root: Path) -> web.Application:
|
||||
restore_sources()
|
||||
# No preview queued by the last test may land in the new fixture.
|
||||
await api._run_in_executor(api.close_auto_thumbnails)
|
||||
# Previews inside each source again (#320).
|
||||
await api._run_in_executor(reset_thumbnail_location)
|
||||
library_editor_commands.reset()
|
||||
await api._run_in_executor(reset_logs)
|
||||
data = await request.json() if request.can_read_body else {}
|
||||
@@ -595,6 +601,7 @@ def create_app(root: Path) -> web.Application:
|
||||
history_commands.reset()
|
||||
library_editor_commands.reset()
|
||||
reveal_stub.reset()
|
||||
await api._run_in_executor(reset_thumbnail_location)
|
||||
history_fixture = None
|
||||
manifest = await api._run_in_executor(
|
||||
seeder.run,
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""The thumbnail location on the test server (#320), undone by every reset.
|
||||
|
||||
Folder mode is reachable from config.json and tests only until the Move
|
||||
thumbnails job lands, so page tests set it here, inside the server's root:
|
||||
|
||||
- ``POST /__test__/thumbnail-location`` ``{"mode": "folder"}`` (or
|
||||
``"source"``): previews live in ``<root>/thumbs/<key>`` from now on (a new
|
||||
folder with Prompt Manager's marker).
|
||||
- ``GET /__test__/thumbnail-files``: the preview files in the thumbnail
|
||||
folder and whether any source grew a ``thumbnails/`` folder.
|
||||
- ``POST /__test__/thumbnail-folder/remove``: the thumbnail folder goes, as
|
||||
when its drive is unplugged; the server notices it on its next check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Callable
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from py.api import PromptManagerAPI
|
||||
|
||||
FOLDER_NAME = "thumbs"
|
||||
|
||||
|
||||
def register_thumbnail_location_commands(
|
||||
routes: web.RouteTableDef, api: PromptManagerAPI, root: Path
|
||||
) -> Callable[[], None]:
|
||||
"""Register the commands; returns the reset every reset runs."""
|
||||
from py.api import preview_location, preview_store
|
||||
|
||||
folder = root / FOLDER_NAME
|
||||
|
||||
def reset() -> None:
|
||||
preview_store.configure(None)
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
|
||||
def use(mode: str) -> dict:
|
||||
reset()
|
||||
if mode == "source":
|
||||
return {"mode": "source"}
|
||||
folder.mkdir()
|
||||
marker_id = preview_location.init_folder(str(folder), created_by_pm=True)
|
||||
location = preview_store.configure(
|
||||
{
|
||||
"mode": "folder",
|
||||
"path": str(folder),
|
||||
"id": marker_id,
|
||||
"install": "browser-tests",
|
||||
}
|
||||
)
|
||||
return {"mode": location.mode, "available": location.available}
|
||||
|
||||
@routes.post("/__test__/thumbnail-location")
|
||||
async def set_location(request: web.Request) -> web.Response:
|
||||
data = await request.json()
|
||||
mode = data.get("mode") if isinstance(data, dict) else None
|
||||
if mode not in ("source", "folder"):
|
||||
raise web.HTTPBadRequest()
|
||||
return web.json_response(await api._run_in_executor(use, mode))
|
||||
|
||||
def files() -> dict:
|
||||
previews = sorted(
|
||||
path.relative_to(folder).as_posix()
|
||||
for path in folder.rglob("*_thumb.*")
|
||||
if path.is_file()
|
||||
)
|
||||
sources = [str(path) for path in api._get_all_output_dirs()]
|
||||
return {
|
||||
"folder": previews,
|
||||
"source_thumbnails": any(
|
||||
(Path(source) / "thumbnails").exists() for source in sources
|
||||
),
|
||||
}
|
||||
|
||||
@routes.get("/__test__/thumbnail-files")
|
||||
async def thumbnail_files(_request: web.Request) -> web.Response:
|
||||
return web.json_response(await api._run_in_executor(files))
|
||||
|
||||
def remove() -> bool:
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
return preview_store.refresh()
|
||||
|
||||
@routes.post("/__test__/thumbnail-folder/remove")
|
||||
async def remove_folder(_request: web.Request) -> web.Response:
|
||||
available = await api._run_in_executor(remove)
|
||||
return web.json_response({"available": available})
|
||||
|
||||
return reset
|
||||
@@ -261,3 +261,20 @@ test('prompt deletion requires confirmation and can retry after a network failur
|
||||
await expect(await request.get(url)).toBeOK();
|
||||
}
|
||||
});
|
||||
|
||||
// #320: with previews in one folder, the legacy gallery still shows them, made
|
||||
// and served from that folder (the fixture's own previews inside the sources
|
||||
// are never served then).
|
||||
test('the legacy gallery shows thumbnails kept in one folder', async ({ page, request }) => {
|
||||
await expect(await request.post('/__test__/thumbnail-location', { data: { mode: 'folder' } })).toBeOK();
|
||||
const served = page.waitForResponse(response =>
|
||||
response.url().includes('/prompt_manager/images/serve/thumbnails/') && response.status() === 200);
|
||||
await page.goto('/prompt_manager/gallery');
|
||||
await expect(page.locator('#galleryGrid .image-item')).toHaveCount(2);
|
||||
const response = await served; // made in the folder before it was served
|
||||
const name = decodeURIComponent(new URL(response.url()).pathname.split('/serve/thumbnails/')[1]);
|
||||
const files = await (await request.get('/__test__/thumbnail-files')).json();
|
||||
const stem = name.replace(/\.[^.]+$/, '');
|
||||
expect(files.folder.some(file => /^[a-z0-9_-]+-[0-9a-f]{8}\//.test(file)
|
||||
&& file.slice(file.indexOf('/') + 1).replace(/\.[^.]+$/, '') === stem)).toBe(true);
|
||||
});
|
||||
|
||||
+7
-1
@@ -212,6 +212,12 @@
|
||||
"thumbnail_encoders",
|
||||
"thumbnail_encoders.jpeg",
|
||||
"thumbnail_encoders.jxl",
|
||||
"thumbnail_encoders.webp"
|
||||
"thumbnail_encoders.webp",
|
||||
"thumbnail_location",
|
||||
"thumbnail_location.available",
|
||||
"thumbnail_location.default_path",
|
||||
"thumbnail_location.mode",
|
||||
"thumbnail_location.path",
|
||||
"thumbnail_location.problem"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
"""The thumbnail location setting (#320 PR 2): parsing, validation, source
|
||||
keys, the marker, the settings payloads and reverse containment.
|
||||
|
||||
Spelling rules are tested on both path flavors (``ntpath`` and
|
||||
``posixpath``) as text; filesystem rules run on the native one.
|
||||
"""
|
||||
|
||||
import json
|
||||
import ntpath
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.modules.setdefault("server", MagicMock())
|
||||
|
||||
from py.api import preview_location as pl # noqa: E402
|
||||
from py.api import preview_store # noqa: E402
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
class TestParse(unittest.TestCase):
|
||||
def test_missing_or_source_is_inside_each_source(self):
|
||||
for value in (None, {"mode": "source"}, {}):
|
||||
with self.subTest(value=value):
|
||||
parsed = pl.parse_location(value)
|
||||
self.assertEqual((parsed.mode, parsed.problem), ("source", None))
|
||||
|
||||
def test_a_folder_location_is_kept_as_spelled(self):
|
||||
parsed = pl.parse_location(
|
||||
{"mode": "folder", "path": "D:\\Thumbs", "id": "x", "install": "i"},
|
||||
flavor=ntpath,
|
||||
)
|
||||
self.assertEqual(
|
||||
(parsed.mode, parsed.path, parsed.id, parsed.problem),
|
||||
("folder", "D:\\Thumbs", "x", None),
|
||||
)
|
||||
|
||||
def test_invalid_values_never_fall_back_to_the_sources(self):
|
||||
for value in (
|
||||
"folder",
|
||||
{"mode": "elsewhere"},
|
||||
{"mode": "folder", "path": "relative", "id": "x"},
|
||||
{"mode": "folder", "path": "/abs"},
|
||||
{"mode": "folder", "path": "/abs", "id": ""},
|
||||
{"mode": "folder", "path": 7, "id": "x"},
|
||||
):
|
||||
with self.subTest(value=value):
|
||||
parsed = pl.parse_location(value, flavor=posixpath)
|
||||
self.assertEqual((parsed.mode, parsed.problem), ("folder", "invalid"))
|
||||
|
||||
|
||||
class TestForm(unittest.TestCase):
|
||||
def test_windows_spellings(self):
|
||||
cases = {
|
||||
"D:\\Thumbs": None,
|
||||
"\\\\server\\share\\thumbs": None,
|
||||
"C:thumbs": "location-not-absolute",
|
||||
"\\thumbs": "location-not-absolute",
|
||||
"thumbs": "location-not-absolute",
|
||||
"": "location-not-absolute",
|
||||
" D:\\Thumbs": "location-not-absolute",
|
||||
"\\\\?\\D:\\Thumbs": "location-device-path",
|
||||
"\\\\.\\D:\\Thumbs": "location-device-path",
|
||||
"D:\\CON\\x": "location-invalid-name",
|
||||
"D:\\com1.txt": "location-invalid-name",
|
||||
"D:\\thumbs.": "location-invalid-name",
|
||||
"D:\\thumbs ": "location-invalid-name",
|
||||
"D:\\a|b": "location-invalid-name",
|
||||
"D:\\a?b": "location-invalid-name",
|
||||
"D:\\a\x01b": "location-invalid-name",
|
||||
}
|
||||
for path, code in cases.items():
|
||||
with self.subTest(path=path):
|
||||
self.assertEqual(pl.form_problem(path, flavor=ntpath), code)
|
||||
|
||||
def test_posix_spellings(self):
|
||||
cases = {
|
||||
"/mnt/fast/thumbs": None,
|
||||
"mnt/thumbs": "location-not-absolute",
|
||||
"/mnt/nul": "location-invalid-name",
|
||||
"/mnt/a:b": "location-invalid-name",
|
||||
"//?/x": "location-device-path",
|
||||
}
|
||||
for path, code in cases.items():
|
||||
with self.subTest(path=path):
|
||||
self.assertEqual(pl.form_problem(path, flavor=posixpath), code)
|
||||
|
||||
def test_roots_and_home_are_too_broad(self):
|
||||
self.assertTrue(pl.is_too_broad("C:\\", home="C:\\Users\\a", flavor=ntpath))
|
||||
self.assertTrue(
|
||||
pl.is_too_broad("\\\\server\\share", home="C:\\Users\\a", flavor=ntpath)
|
||||
)
|
||||
self.assertTrue(
|
||||
pl.is_too_broad("C:\\users\\A", home="C:\\Users\\a", flavor=ntpath)
|
||||
)
|
||||
self.assertTrue(pl.is_too_broad("/", home="/home/a", flavor=posixpath))
|
||||
self.assertTrue(pl.is_too_broad("/home/a/", home="/home/a", flavor=posixpath))
|
||||
self.assertFalse(
|
||||
pl.is_too_broad("/home/a/thumbs", home="/home/a", flavor=posixpath)
|
||||
)
|
||||
|
||||
def test_long_windows_paths_are_refused_without_long_paths(self):
|
||||
short = "D:\\" + "a" * 100
|
||||
long = "D:\\" + "a" * 200
|
||||
self.assertFalse(pl.too_long_for_windows(short, long_paths=False))
|
||||
self.assertTrue(pl.too_long_for_windows(long, long_paths=False))
|
||||
self.assertFalse(pl.too_long_for_windows(long, long_paths=True))
|
||||
|
||||
|
||||
class TestSourceKeys(unittest.TestCase):
|
||||
def test_slug(self):
|
||||
self.assertEqual(pl.slug("ComfyUI Output"), "comfyui-output")
|
||||
self.assertEqual(pl.slug("Ünïcödé_Fôlder"), "unicode_folder")
|
||||
self.assertEqual(pl.slug(""), "source")
|
||||
self.assertEqual(pl.slug("!!!"), "source")
|
||||
self.assertEqual(len(pl.slug("x" * 80)), 24)
|
||||
|
||||
def test_keys_are_a_slug_plus_a_hash(self):
|
||||
key = pl.source_key("D:\\AI\\Output", "d:\\ai\\output", {})
|
||||
self.assertRegex(key, r"^output-[0-9a-f]{8}$")
|
||||
self.assertRegex(pl.source_key("D:\\", "d:\\", {}), r"^source-[0-9a-f]{8}$")
|
||||
self.assertRegex(
|
||||
pl.source_key("\\\\nas\\share\\Out", "\\\\nas\\share\\out", {}),
|
||||
r"^out-[0-9a-f]{8}$",
|
||||
)
|
||||
|
||||
def test_a_taken_key_grows_its_hash(self):
|
||||
first = pl.source_key("/a/out", "/a/out", {})
|
||||
taken = {first: "/someone/else"}
|
||||
second = pl.source_key("/a/out", "/a/out", taken)
|
||||
self.assertRegex(second, r"^out-[0-9a-f]{12}$")
|
||||
taken[second] = "/another"
|
||||
self.assertRegex(
|
||||
pl.source_key("/a/out", "/a/out", taken), r"^out-[0-9a-f]{16}$"
|
||||
)
|
||||
self.assertEqual(pl.source_key("/a/out", "/a/out", {first: "/a/out"}), first)
|
||||
|
||||
def test_lookup_by_spelling_identity_then_file_id(self):
|
||||
keys = {
|
||||
"out-1": {"path": "D:\\Out", "identity": "d:\\out", "dev_ino": [1, 2]},
|
||||
"lib-2": {"path": "E:\\Lib", "identity": "e:\\lib", "dev_ino": None},
|
||||
}
|
||||
self.assertEqual(pl.find_key(keys, "D:\\Out", "x", None), "out-1")
|
||||
self.assertEqual(pl.find_key(keys, "Z:\\Out", "d:\\out", None), "out-1")
|
||||
# A letter spelling and its UNC spelling: the same file id.
|
||||
self.assertEqual(
|
||||
pl.find_key(keys, "\\\\nas\\out", "\\\\nas\\out", (1, 2)), "out-1"
|
||||
)
|
||||
self.assertIsNone(pl.find_key(keys, "F:\\New", "f:\\new", (9, 9)))
|
||||
|
||||
|
||||
class TestMarker(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.folder = self._tmp.name
|
||||
|
||||
def test_init_creates_once_and_adopts_after(self):
|
||||
first = pl.init_folder(self.folder, created_by_pm=True)
|
||||
self.assertEqual(pl.init_folder(self.folder), first)
|
||||
marker = pl.read_marker(self.folder)
|
||||
self.assertEqual(marker["id"], first)
|
||||
self.assertTrue(marker["created_by_pm"])
|
||||
self.assertEqual(marker["version"], 1)
|
||||
|
||||
def test_writes_merge_what_another_writer_added(self):
|
||||
pl.init_folder(self.folder)
|
||||
pl.write_marker(self.folder, lambda m: m["keys"].update(a={"install": "1"}))
|
||||
pl.write_marker(self.folder, lambda m: m["keys"].update(b={"install": "2"}))
|
||||
self.assertEqual(sorted(pl.read_marker(self.folder)["keys"]), ["a", "b"])
|
||||
# No temp file is left beside it.
|
||||
self.assertEqual(os.listdir(self.folder), [pl.MARKER_NAME])
|
||||
|
||||
def test_a_folder_without_the_marker_is_missing(self):
|
||||
self.assertEqual(pl.folder_state(self.folder, "x")[:2], (False, "missing"))
|
||||
missing = os.path.join(self.folder, "gone")
|
||||
self.assertEqual(pl.folder_state(missing, "x")[:2], (False, "missing"))
|
||||
marker_id = pl.init_folder(self.folder)
|
||||
self.assertEqual(pl.folder_state(self.folder, marker_id)[:2], (True, None))
|
||||
self.assertEqual(
|
||||
pl.folder_state(self.folder, "other")[:2], (False, "marker-mismatch")
|
||||
)
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
def test_a_linked_folder_is_not_available(self):
|
||||
real = os.path.join(self.folder, "real")
|
||||
os.mkdir(real)
|
||||
marker_id = pl.init_folder(real)
|
||||
link = os.path.join(self.folder, "link")
|
||||
try:
|
||||
os.symlink(real, link, target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
self.assertEqual(pl.folder_state(link, marker_id)[:2], (False, "invalid"))
|
||||
|
||||
|
||||
class TestValidation(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.base = os.path.realpath(self._tmp.name)
|
||||
self.source = os.path.join(self.base, "output")
|
||||
self.node = os.path.join(self.base, "node")
|
||||
os.mkdir(self.source)
|
||||
os.mkdir(self.node)
|
||||
self.context = pl.Context(
|
||||
roots=(self.source,),
|
||||
node_folder=self.node,
|
||||
default_path=os.path.join(self.node, "data", "thumbnails"),
|
||||
home=os.path.join(self.base, "home"),
|
||||
environ={},
|
||||
long_paths=True,
|
||||
folder_paths=None,
|
||||
)
|
||||
|
||||
def problem(self, path, purpose="move", **changes):
|
||||
context = self.context
|
||||
if changes:
|
||||
context = pl.Context(**{**context.__dict__, **changes})
|
||||
return pl.location_problem(path, purpose=purpose, context=context)
|
||||
|
||||
def test_an_empty_new_folder_beside_the_sources_is_accepted(self):
|
||||
self.assertIsNone(self.problem(os.path.join(self.base, "thumbs")))
|
||||
os.mkdir(os.path.join(self.base, "empty"))
|
||||
Path(self.base, "empty", "Thumbs.db").write_bytes(b"")
|
||||
self.assertIsNone(self.problem(os.path.join(self.base, "empty")))
|
||||
|
||||
def test_codes(self):
|
||||
cases = {
|
||||
"relative": "location-not-absolute",
|
||||
os.path.join(self.source, "thumbs"): "location-served",
|
||||
os.path.join(self.node, "thumbs"): "location-node-folder",
|
||||
self.base: "location-contains-source",
|
||||
os.path.join(self.base, "no", "parent"): "location-parent-missing",
|
||||
}
|
||||
for path, code in cases.items():
|
||||
with self.subTest(path=path):
|
||||
self.assertEqual(self.problem(path), code)
|
||||
|
||||
def test_the_default_may_sit_in_the_node_folder_and_make_its_parents(self):
|
||||
self.assertIsNone(self.problem(self.context.default_path))
|
||||
|
||||
def test_the_folder_in_use_is_the_same(self):
|
||||
folder = os.path.join(self.base, "thumbs")
|
||||
os.mkdir(folder)
|
||||
self.assertEqual(self.problem(folder, current_path=folder), "location-same")
|
||||
|
||||
def test_a_folder_with_other_files_is_not_empty_unless_it_is_ours(self):
|
||||
folder = os.path.join(self.base, "full")
|
||||
os.mkdir(folder)
|
||||
Path(folder, "photo.jpg").write_bytes(b"x")
|
||||
self.assertEqual(self.problem(folder), "location-not-empty")
|
||||
pl.init_folder(folder)
|
||||
self.assertIsNone(self.problem(folder))
|
||||
|
||||
@unittest.skipIf(os.name == "nt" or os.geteuid() == 0, "needs POSIX modes")
|
||||
def test_a_folder_that_refuses_writes(self):
|
||||
folder = os.path.join(self.base, "ro")
|
||||
os.mkdir(folder)
|
||||
os.chmod(folder, 0o500)
|
||||
self.addCleanup(os.chmod, folder, 0o700)
|
||||
self.assertEqual(self.problem(folder), "location-not-writable")
|
||||
|
||||
def test_a_stat_that_does_not_answer_is_unavailable(self):
|
||||
release = threading.Event()
|
||||
self.addCleanup(release.set)
|
||||
|
||||
def stuck(*_args):
|
||||
release.wait(10)
|
||||
|
||||
with patch.object(pl, "_filesystem_problem", stuck):
|
||||
code = pl.location_problem(
|
||||
os.path.join(self.base, "thumbs"),
|
||||
purpose="move",
|
||||
context=self.context,
|
||||
timeout=0.1,
|
||||
)
|
||||
self.assertEqual(code, "location-unavailable")
|
||||
|
||||
def test_a_long_windows_path(self):
|
||||
path = os.path.join(self.base, "t" * 220)
|
||||
self.assertEqual(self.problem(path, long_paths=False), "location-path-too-long")
|
||||
|
||||
def test_every_code_has_a_message_without_a_path(self):
|
||||
for code, (status, text) in pl.MESSAGES.items():
|
||||
with self.subTest(code=code):
|
||||
self.assertIn(status, (400, 409))
|
||||
self.assertTrue(text.endswith("."))
|
||||
|
||||
|
||||
class TestNoLegacySaveCallers(unittest.TestCase):
|
||||
def test_save_to_file_has_no_callers(self):
|
||||
offenders = []
|
||||
for folder in ("py", "utils", "database"):
|
||||
for path in (REPO / folder).rglob("*.py"):
|
||||
for number, line in enumerate(
|
||||
path.read_text(encoding="utf-8").splitlines(), 1
|
||||
):
|
||||
code = line.split("#", 1)[0]
|
||||
if (
|
||||
re.search(r"\bsave_to_file\(", code)
|
||||
and "def save_to_file" not in code
|
||||
):
|
||||
if "PromptManagerConfig.save_to_file('" in line:
|
||||
continue # its own docstring example
|
||||
offenders.append(f"{path.relative_to(REPO)}:{number}")
|
||||
self.assertEqual(offenders, [])
|
||||
|
||||
|
||||
try:
|
||||
from tests.test_api_images import ImageAPITestCase
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_api_images import ImageAPITestCase
|
||||
|
||||
|
||||
class TestLocationRoutes(ImageAPITestCase):
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.addCleanup(preview_store.configure, None)
|
||||
|
||||
def use_folder(self):
|
||||
folder = self.comfy_dir / "thumbs"
|
||||
folder.mkdir()
|
||||
marker_id = pl.init_folder(str(folder), created_by_pm=True)
|
||||
preview_store.configure(
|
||||
{"mode": "folder", "path": str(folder), "id": marker_id}
|
||||
)
|
||||
return folder
|
||||
|
||||
async def test_settings_get_names_the_location_with_its_path(self):
|
||||
folder = self.use_folder()
|
||||
resp = await self.client.get("/prompt_manager/settings")
|
||||
location = (await resp.json())["thumbnail_location"]
|
||||
self.assertEqual(location["mode"], "folder")
|
||||
self.assertEqual(location["path"], str(folder))
|
||||
self.assertTrue(location["available"])
|
||||
self.assertIsNone(location["problem"])
|
||||
self.assertIn("default_path", location)
|
||||
|
||||
async def test_settings_post_never_changes_the_location(self):
|
||||
resp = await self.client.post(
|
||||
"/prompt_manager/settings",
|
||||
json={"thumbnail_location": {"mode": "folder", "path": "/x"}},
|
||||
)
|
||||
self.assertEqual(resp.status, 409)
|
||||
body = await resp.json()
|
||||
self.assertEqual(body["code"], "location-needs-move")
|
||||
self.assertNotIn("/x", body["error"])
|
||||
|
||||
async def test_status_has_the_location_but_no_path(self):
|
||||
folder = self.use_folder()
|
||||
resp = await self.client.get("/prompt_manager/images/thumbnails/status")
|
||||
body = await resp.text()
|
||||
self.assertNotIn(str(folder), body)
|
||||
self.assertEqual(
|
||||
json.loads(body)["location"],
|
||||
{"mode": "folder", "available": True, "problem": None},
|
||||
)
|
||||
|
||||
async def test_a_source_inside_or_around_the_folder_is_refused(self):
|
||||
from py.api import sources
|
||||
|
||||
folder = self.use_folder()
|
||||
inside = folder / "sub"
|
||||
inside.mkdir()
|
||||
with patch(
|
||||
"py.config.GalleryConfig.gallery_root_problem",
|
||||
classmethod(lambda cls, path: (None, "")),
|
||||
):
|
||||
for path in (inside, folder, self.comfy_dir):
|
||||
with self.subTest(path=path):
|
||||
code, reason = sources.source_problem(str(path))
|
||||
self.assertEqual(code, "holds-thumbnails")
|
||||
self.assertEqual(reason, pl.HOLDS_THUMBNAILS_REASON)
|
||||
self.assertEqual(sources.source_problem(str(self.output_dir))[0], None)
|
||||
|
||||
async def test_a_library_move_into_the_folder_is_refused(self):
|
||||
from py.api import library_location as routes
|
||||
|
||||
folder = self.use_folder()
|
||||
self.assertTrue(routes._inside_thumbnail_folder(str(folder / "lib")))
|
||||
self.assertFalse(routes._inside_thumbnail_folder(str(self.output_dir)))
|
||||
|
||||
def test_startup_makes_a_location_inside_a_source_invalid(self):
|
||||
inside = self.output_dir / "thumbs"
|
||||
inside.mkdir()
|
||||
marker_id = pl.init_folder(str(inside))
|
||||
from py.config import PromptManagerConfig
|
||||
|
||||
value = {"mode": "folder", "path": str(inside), "id": marker_id}
|
||||
with patch.object(PromptManagerConfig, "THUMBNAIL_LOCATION", value):
|
||||
location = self.api.start_preview_location()
|
||||
self.assertEqual((location.available, location.problem), (False, "invalid"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,362 @@
|
||||
"""One folder mode (#320 PR 2): previews in ``<folder>/<source key>``.
|
||||
|
||||
Folder mode is reachable from config.json and tests only (D4B). These tests
|
||||
set it on the resolver directly (``preview_store.configure``) and drive
|
||||
every route that reads, writes, lists or removes previews: they must touch
|
||||
only ``<folder>/<key>``, never ``<source>/thumbnails``, never through a link,
|
||||
and nothing at all while the folder is unavailable. Waits are events; no
|
||||
sleeps.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
try:
|
||||
from tests.test_thumbnail_auto import AutoThumbnailCase, _entry, WAIT
|
||||
from tests.test_api_images import make_png
|
||||
except ImportError: # discovered with tests/ as the top-level directory
|
||||
from test_thumbnail_auto import AutoThumbnailCase, _entry, WAIT
|
||||
from test_api_images import make_png
|
||||
|
||||
from py.api import preview_location, preview_store, thumbnail_settings # noqa: E402
|
||||
|
||||
SERVE = "/prompt_manager/images/serve/"
|
||||
|
||||
|
||||
class FolderModeCase(AutoThumbnailCase):
|
||||
"""The output root's previews live in ``<comfy>/thumbs/<key>``."""
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.folder = self.use_folder(self.comfy_dir / "thumbs")
|
||||
self.old_store = self.output_dir / "thumbnails"
|
||||
|
||||
def use_folder(self, folder, install="install-a"):
|
||||
folder.mkdir(exist_ok=True)
|
||||
marker_id = preview_location.init_folder(str(folder), created_by_pm=True)
|
||||
self.addCleanup(preview_store.configure, None)
|
||||
preview_store.configure(
|
||||
{"mode": "folder", "path": str(folder), "id": marker_id, "install": install}
|
||||
)
|
||||
self.assertTrue(preview_store.current().available)
|
||||
return folder
|
||||
|
||||
def key_folder(self):
|
||||
store = preview_store.ensure_store(self.output_dir, create=True)
|
||||
self.assertIsNotNone(store)
|
||||
return store
|
||||
|
||||
def preview(self, rel, ext=".webp"):
|
||||
return self.key_folder() / (thumbnail_settings.preview_stem(rel) + ext)
|
||||
|
||||
def marker(self):
|
||||
return json.loads(
|
||||
(self.folder / preview_location.MARKER_NAME).read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
|
||||
class TestFolderModeRoutes(FolderModeCase):
|
||||
async def test_the_key_is_a_readable_name_plus_a_hash_in_the_marker(self):
|
||||
key = self.key_folder().name
|
||||
self.assertRegex(key, r"^output-[0-9a-f]{8}$")
|
||||
entry = self.marker()["keys"][key]
|
||||
self.assertEqual(entry["path"], str(self.output_dir))
|
||||
self.assertEqual(entry["install"], "install-a")
|
||||
|
||||
async def test_another_case_of_the_source_finds_the_same_key(self):
|
||||
key = self.key_folder()
|
||||
other = Path(str(self.output_dir).swapcase())
|
||||
if not other.is_dir():
|
||||
self.skipTest("a case-sensitive volume")
|
||||
self.assertEqual(preview_store.store_dir(other), key)
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
async def test_a_linked_spelling_of_the_source_finds_the_same_key(self):
|
||||
key = self.key_folder()
|
||||
link = self.comfy_dir / "output-link"
|
||||
try:
|
||||
link.symlink_to(self.output_dir, target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
self.assertEqual(preview_store.store_dir(link), key)
|
||||
|
||||
async def test_an_existing_preview_is_listed_and_served_from_the_key_folder(self):
|
||||
make_png(self.output_dir / "sub" / "a.png", size=(64, 64))
|
||||
preview = self.preview("sub/a.png")
|
||||
make_png(preview, size=(32, 32))
|
||||
url = _entry(self.api, self.output_dir, "sub/a.png")["thumbnail_url"]
|
||||
self.assertIn("v=", url)
|
||||
self.assertTrue(url.startswith(SERVE + "thumbnails/sub/a_thumb.webp"))
|
||||
resp = await self.client.get(url)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(await resp.read(), preview.read_bytes())
|
||||
|
||||
async def test_a_missing_preview_is_made_in_the_key_folder(self):
|
||||
make_png(self.output_dir / "b.png", size=(640, 480))
|
||||
url = _entry(self.api, self.output_dir, "b.png")["thumbnail_url"]
|
||||
resp = await self.client.get(url)
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertTrue(self.preview("b.png").is_file())
|
||||
self.assertFalse(self.old_store.exists())
|
||||
|
||||
async def test_a_leftover_in_the_source_folder_is_never_served(self):
|
||||
make_png(self.output_dir / "c.png", size=(640, 480))
|
||||
self.old_store.mkdir()
|
||||
(self.old_store / "c_thumb.webp").write_bytes(b"leftover")
|
||||
resp = await self.client.get(SERVE + "thumbnails/c_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertNotEqual(await resp.read(), b"leftover")
|
||||
|
||||
async def test_other_text_under_thumbnails_is_refused(self):
|
||||
self.old_store.mkdir()
|
||||
make_png(self.old_store / "notes.png", size=(8, 8))
|
||||
resp = await self.client.get(SERVE + "thumbnails/notes.png?root=0")
|
||||
self.assertEqual(resp.status, 403)
|
||||
|
||||
async def test_traversal_through_thumbnails_is_refused(self):
|
||||
for text in (
|
||||
"thumbnails/../../thumbs/x_thumb.webp",
|
||||
"thumbnails/..%2F..%2Fx_thumb.webp",
|
||||
"thumbnails/%2e%2e/x_thumb.webp",
|
||||
):
|
||||
with self.subTest(text=text):
|
||||
resp = await self.client.get(SERVE + text + "?root=0")
|
||||
self.assertIn(resp.status, (403, 404))
|
||||
|
||||
@unittest.skipUnless(hasattr(os, "symlink"), "needs symlinks")
|
||||
async def test_a_link_out_of_the_key_folder_is_never_served_or_written(self):
|
||||
outside = self.comfy_dir / "outside"
|
||||
outside.mkdir()
|
||||
make_png(outside / "d_thumb.webp", size=(8, 8))
|
||||
make_png(self.output_dir / "sub" / "d.png", size=(64, 64))
|
||||
try:
|
||||
(self.key_folder() / "sub").symlink_to(outside, target_is_directory=True)
|
||||
except OSError:
|
||||
self.skipTest("symlinks not permitted")
|
||||
resp = await self.client.get(SERVE + "thumbnails/sub/d_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 403)
|
||||
self.assertFalse(preview_store.owns(self.output_dir, outside / "d_thumb.webp"))
|
||||
self.assertFalse(
|
||||
preview_store.owns(self.output_dir, self.key_folder() / "sub" / "x.webp")
|
||||
)
|
||||
|
||||
@unittest.skipUnless(os.name == "nt", "junctions are Windows only")
|
||||
async def test_a_junction_inside_a_store_is_refused(self):
|
||||
import _winapi
|
||||
|
||||
outside = self.comfy_dir / "outside"
|
||||
outside.mkdir()
|
||||
(outside / "e_thumb.webp").write_bytes(b"x")
|
||||
_winapi.CreateJunction(str(outside), str(self.key_folder() / "sub"))
|
||||
self.assertTrue(preview_location.is_link_like(self.key_folder() / "sub"))
|
||||
resp = await self.client.get(SERVE + "thumbnails/sub/e_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 403)
|
||||
|
||||
async def test_capture_writes_into_the_key_folder(self):
|
||||
path = make_png(self.output_dir / "run" / "f.png", size=(64, 64))
|
||||
self.assertEqual(self.api.queue_capture_thumbnail(str(path)), "queued")
|
||||
await self._idle()
|
||||
self.assertTrue(self.preview("run/f.png").is_file())
|
||||
self.assertFalse(self.old_store.exists())
|
||||
|
||||
async def test_clear_removes_previews_only(self):
|
||||
make_png(self.output_dir / "g.png", size=(64, 64))
|
||||
payload = await asyncio.get_running_loop().run_in_executor(
|
||||
None, self.api._generate_gallery_thumbnails_sync, [self.output_dir]
|
||||
)
|
||||
self.assertTrue(payload["success"])
|
||||
keep = self.key_folder() / "readme.txt"
|
||||
keep.write_text("not a preview", encoding="utf-8")
|
||||
resp = await self.client.post("/prompt_manager/images/clear-thumbnails")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertFalse(self.preview("g.png").exists())
|
||||
self.assertTrue(keep.exists())
|
||||
self.assertTrue((self.folder / preview_location.MARKER_NAME).is_file())
|
||||
self.assertTrue(self.output_dir.joinpath("g.png").is_file())
|
||||
|
||||
async def test_removing_a_file_removes_its_preview_in_the_key_folder(self):
|
||||
from py.api.admin import _remove_thumbnail
|
||||
|
||||
make_png(self.output_dir / "h.png", size=(64, 64))
|
||||
preview = self.preview("h.png")
|
||||
preview.write_bytes(b"x")
|
||||
self.assertTrue(_remove_thumbnail(self.output_dir, Path("h.png")))
|
||||
self.assertFalse(preview.exists())
|
||||
|
||||
async def test_history_cover_is_authorized_by_the_store(self):
|
||||
from py.api import history_current
|
||||
|
||||
preview = self.preview("i.png")
|
||||
preview.write_bytes(b"x")
|
||||
target = history_current._canonical(preview)
|
||||
self.assertTrue(
|
||||
history_current._target_authorized(
|
||||
target, self.output_dir, None, self.output_dir
|
||||
)
|
||||
)
|
||||
stray = self.comfy_dir / "stray_thumb.webp"
|
||||
stray.write_bytes(b"x")
|
||||
self.assertFalse(
|
||||
history_current._target_authorized(
|
||||
history_current._canonical(stray),
|
||||
self.output_dir,
|
||||
None,
|
||||
self.output_dir,
|
||||
)
|
||||
)
|
||||
|
||||
async def test_the_thumbnail_folder_is_never_recreated_by_a_write(self):
|
||||
make_png(self.output_dir / "j.png", size=(64, 64))
|
||||
self.key_folder()
|
||||
for child in sorted(self.folder.rglob("*"), reverse=True):
|
||||
child.rmdir() if child.is_dir() else child.unlink()
|
||||
self.folder.rmdir()
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
preview_store.make_parent(self.folder / "output-x" / "sub" / "a.webp")
|
||||
self.assertFalse(self.folder.exists())
|
||||
|
||||
|
||||
class TestUnavailableFolder(FolderModeCase):
|
||||
def go_missing(self):
|
||||
(self.folder / preview_location.MARKER_NAME).unlink()
|
||||
self.assertFalse(preview_store.refresh())
|
||||
location = preview_store.current()
|
||||
self.assertEqual((location.available, location.problem), (False, "missing"))
|
||||
|
||||
async def test_a_folder_without_its_marker_is_missing_and_writes_nothing(self):
|
||||
make_png(self.output_dir / "k.png", size=(640, 480))
|
||||
self.go_missing()
|
||||
entry = _entry(self.api, self.output_dir, "k.png")
|
||||
self.assertFalse(entry.get("thumbnail_ready", False))
|
||||
resp = await self.client.get(SERVE + "thumbnails/k_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertEqual(resp.headers["Cache-Control"], "no-store")
|
||||
self.assertEqual(
|
||||
await resp.read(), (self.output_dir / "k.png").read_bytes()
|
||||
) # the original
|
||||
self.assertEqual(
|
||||
sorted(p.name for p in self.folder.rglob("*") if p.is_file()), []
|
||||
)
|
||||
self.assertFalse(self.old_store.exists())
|
||||
|
||||
async def test_thumbnail_work_is_refused_while_unavailable(self):
|
||||
self.go_missing()
|
||||
for method, url, body in (
|
||||
("POST", "/prompt_manager/images/generate-thumbnails", {}),
|
||||
("POST", "/prompt_manager/images/generate-thumbnails/progress", None),
|
||||
("POST", "/prompt_manager/images/clear-thumbnails", None),
|
||||
("POST", "/prompt_manager/jobs/thumbnails", {"mode": "missing"}),
|
||||
):
|
||||
with self.subTest(url=url):
|
||||
resp = await self.client.request(method, url, json=body)
|
||||
self.assertEqual(resp.status, 409)
|
||||
self.assertEqual((await resp.json())["code"], "location-unavailable")
|
||||
|
||||
async def test_status_reports_the_location_without_a_path(self):
|
||||
self.go_missing()
|
||||
resp = await self.client.get("/prompt_manager/images/thumbnails/status")
|
||||
location = (await resp.json())["location"]
|
||||
self.assertEqual(
|
||||
location, {"mode": "folder", "available": False, "problem": "missing"}
|
||||
)
|
||||
|
||||
async def test_another_folders_marker_is_a_mismatch(self):
|
||||
marker = self.marker()
|
||||
marker["id"] = "someone-else"
|
||||
(self.folder / preview_location.MARKER_NAME).write_text(
|
||||
json.dumps(marker), encoding="utf-8"
|
||||
)
|
||||
self.assertFalse(preview_store.refresh())
|
||||
self.assertEqual(preview_store.current().problem, "marker-mismatch")
|
||||
|
||||
async def test_the_temp_sweep_skips_an_unavailable_folder(self):
|
||||
stray = self.key_folder() / ".l_thumb.part-1-1.webp"
|
||||
stray.write_bytes(b"x")
|
||||
old = os.path.getmtime(stray) - 10 * 24 * 3600
|
||||
os.utime(stray, (old, old))
|
||||
self.go_missing()
|
||||
self.assertEqual(self.api.sweep_thumbnail_temps(), 0)
|
||||
self.assertTrue(stray.exists())
|
||||
|
||||
async def test_a_stalled_store_call_marks_it_unreachable_and_stops_io(self):
|
||||
make_png(self.output_dir / "m.png", size=(64, 64))
|
||||
self.preview("m.png").write_bytes(b"x")
|
||||
ticks = iter([0.0, preview_store.IO_BUDGET + 1.0])
|
||||
with patch.object(preview_store, "_clock", lambda: next(ticks, 99.0)):
|
||||
_entry(self.api, self.output_dir, "m.png") # one slow lookup
|
||||
self.assertEqual(preview_store.current().problem, "unreachable")
|
||||
calls = []
|
||||
real = thumbnail_settings.owned_in_folder
|
||||
with patch.object(
|
||||
thumbnail_settings,
|
||||
"owned_in_folder",
|
||||
lambda *a, **k: calls.append(a) or real(*a, **k),
|
||||
):
|
||||
entry = _entry(self.api, self.output_dir, "m.png")
|
||||
self.assertEqual(calls, []) # no store I/O while unreachable
|
||||
self.assertNotIn("v=", entry["thumbnail_url"])
|
||||
self.assertTrue(preview_store.refresh()) # the probe finds it again
|
||||
|
||||
async def test_a_check_that_blocks_is_unreachable_within_the_budget(self):
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
self.addCleanup(release.set)
|
||||
|
||||
def blocking(path, expected_id):
|
||||
entered.set()
|
||||
release.wait(WAIT)
|
||||
return True, None, None
|
||||
|
||||
with (
|
||||
patch.object(preview_store, "IO_BUDGET", 0.2),
|
||||
patch.object(preview_location, "folder_state", blocking),
|
||||
):
|
||||
self.assertFalse(preview_store.refresh())
|
||||
self.assertTrue(entered.is_set())
|
||||
self.assertEqual(preview_store.current().problem, "unreachable")
|
||||
release.set()
|
||||
|
||||
async def test_an_invalid_config_value_writes_nothing_anywhere(self):
|
||||
preview_store.configure({"mode": "folder", "path": "relative/thumbs"})
|
||||
location = preview_store.current()
|
||||
self.assertEqual((location.available, location.problem), (False, "invalid"))
|
||||
make_png(self.output_dir / "n.png", size=(640, 480))
|
||||
resp = await self.client.get(SERVE + "thumbnails/n_thumb.webp?root=0")
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertFalse(self.old_store.exists())
|
||||
|
||||
|
||||
class TestSharedFolders(FolderModeCase):
|
||||
async def test_two_installs_keep_each_others_keys(self):
|
||||
self.key_folder()
|
||||
other_root = self.comfy_dir / "other"
|
||||
other_root.mkdir()
|
||||
setting = preview_store.setting()
|
||||
preview_store.configure(
|
||||
{
|
||||
"mode": "folder",
|
||||
"path": setting.path,
|
||||
"id": setting.id,
|
||||
"install": "install-b",
|
||||
}
|
||||
)
|
||||
self.assertIsNotNone(preview_store.ensure_store(other_root))
|
||||
keys = self.marker()["keys"]
|
||||
installs = sorted(entry["install"] for entry in keys.values())
|
||||
self.assertEqual(installs, ["install-a", "install-b"])
|
||||
|
||||
async def test_init_folder_adopts_an_existing_marker(self):
|
||||
marker_id = self.marker()["id"]
|
||||
self.assertEqual(preview_location.init_folder(str(self.folder)), marker_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user