fix(images): keep the 3.10-3.13 suffix rule in the scandir walk and harden the warm-up (#348)
Review of the walk speedup: - os.path.splitext drops names like ..png that Path.suffix listed on Python 3.10-3.13 (3.14 changed Path.suffix, which hid it from the parity test on a 3.14 venv). _name_suffix spells out the 3.10-3.13 rule; the test now asserts ..png and ...webp are listed and g. is not. - _warm_workspace reads the roots inside its try, so a failing _get_all_output_dirs warns instead of leaving an unretrieved task error. - The folder_sets fallback test now takes the fallback (a Windows case mismatch); the coalescing test pins the walk count.
This commit is contained in:
+15
-4
@@ -772,6 +772,17 @@ def _favorite_root_key(root) -> str:
|
||||
return _favorite_relative_key(os.path.realpath(str(root)))
|
||||
|
||||
|
||||
def _name_suffix(name):
|
||||
"""``Path(name).suffix`` as Python 3.10-3.13 compute it.
|
||||
|
||||
The last ``.``-part unless the dot is the first or last character, so
|
||||
``..png`` is ``.png`` and ``.png`` has none. Spelled out because 3.14
|
||||
changed ``Path.suffix`` and ``os.path.splitext`` differs for leading dots.
|
||||
"""
|
||||
dot = name.rfind(".")
|
||||
return name[dot:] if 0 < dot < len(name) - 1 else ""
|
||||
|
||||
|
||||
def _walk_media(output_path, max_files):
|
||||
"""Yield the ``os.DirEntry`` of each media file under *output_path*.
|
||||
|
||||
@@ -801,7 +812,7 @@ def _walk_media(output_path, max_files):
|
||||
except OSError:
|
||||
continue # as os.walk: an unreadable folder is skipped
|
||||
for entry in sorted(files, key=lambda item: item.name):
|
||||
if os.path.splitext(entry.name)[1].lower() not in MEDIA_EXTENSIONS:
|
||||
if _name_suffix(entry.name).lower() not in MEDIA_EXTENSIONS:
|
||||
continue
|
||||
if entry.is_symlink():
|
||||
continue # never read through a link out of the output tree
|
||||
@@ -1330,10 +1341,10 @@ class ImageRoutesMixin:
|
||||
async def _warm_workspace(self):
|
||||
while True:
|
||||
self._workspace_warm_again = False
|
||||
listings = self._stored_listings()
|
||||
if listings is None:
|
||||
return # a root has no listing yet; its walk asks again
|
||||
try:
|
||||
listings = self._stored_listings()
|
||||
if listings is None:
|
||||
return # a root has no listing yet; its walk asks again
|
||||
await self._run_in_executor(self._build_workspace_caches, listings)
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not prepare the Images page: {e}")
|
||||
|
||||
@@ -2052,6 +2052,9 @@ class TestOutputScanHelpers(ImageAPITestCase):
|
||||
"b/thumbnails/u.png",
|
||||
"e/.hidden.png",
|
||||
"e/f.jpeg",
|
||||
"e/..png",
|
||||
"e/...webp",
|
||||
"e/g.",
|
||||
):
|
||||
make_png(output / rel)
|
||||
try:
|
||||
@@ -2065,6 +2068,11 @@ class TestOutputScanHelpers(ImageAPITestCase):
|
||||
capped = list(images_module._iter_media_files(output, 3))
|
||||
self.assertEqual(listed, expected)
|
||||
self.assertEqual(capped, expected[:3])
|
||||
# Path.suffix of Python 3.10-3.13, which CI and hosts run (3.14 changed
|
||||
# it): "..png" has the suffix ".png", ".png" and "g." have none.
|
||||
names = {path.name for path in listed}
|
||||
self.assertTrue({"..png", "...webp", ".hidden.png"} <= names, names)
|
||||
self.assertFalse({"g.", "notes.txt"} & names, names)
|
||||
self.assertTrue(all(isinstance(path, Path) for path in listed))
|
||||
|
||||
def test_gallery_scan_reads_each_file_type_and_mtime_from_scandir(self):
|
||||
@@ -2436,7 +2444,10 @@ def _reference_iter_media_files(output_path, max_files):
|
||||
if len(rel_parts) >= images_module.MAX_SCAN_DEPTH:
|
||||
dirs[:] = []
|
||||
for name in sorted(files):
|
||||
if Path(name).suffix.lower() not in images_module.MEDIA_EXTENSIONS:
|
||||
# Path.suffix as Python 3.10-3.13 computed it (3.14 changed it).
|
||||
dot = name.rfind(".")
|
||||
suffix = name[dot:] if 0 < dot < len(name) - 1 else ""
|
||||
if suffix.lower() not in images_module.MEDIA_EXTENSIONS:
|
||||
continue
|
||||
if (root_path / name).is_symlink():
|
||||
continue
|
||||
@@ -3025,8 +3036,24 @@ class TestSharedListing(ImageAPITestCase):
|
||||
release.set()
|
||||
await self._settle()
|
||||
self.assertEqual(built, [1, 3])
|
||||
self.assertEqual(self.scan.calls, 3) # two stored walks, one more build
|
||||
self.assertTrue(self._workspace_ready())
|
||||
|
||||
async def test_a_build_that_cannot_list_the_roots_only_warns(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
await self._warmed()
|
||||
|
||||
def broken():
|
||||
raise RuntimeError("no roots")
|
||||
|
||||
self.api._get_all_output_dirs = broken
|
||||
with self.assertLogs(self.api.logger, "WARNING") as logs:
|
||||
self.api._schedule_workspace_warm()
|
||||
task = self.api._workspace_warm_task
|
||||
await asyncio.wait([task])
|
||||
self.assertIsNone(task.exception())
|
||||
self.assertTrue(any("Images page" in line for line in logs.output))
|
||||
|
||||
async def test_cleanup_stops_a_running_workspace_build(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
release, entered = threading.Event(), threading.Event()
|
||||
|
||||
@@ -80,13 +80,14 @@ class FolderSetsTests(unittest.TestCase):
|
||||
self.assertEqual(folders, {os.path.join("a", "b")})
|
||||
self.assertEqual(with_ancestors, {os.path.join("a", "b"), "a"})
|
||||
|
||||
def test_a_file_outside_the_root_string_still_gets_its_folder(self):
|
||||
root = Path("/out")
|
||||
odd = Path("/out/./x/y.png") # normalizes to /out/x/y.png
|
||||
files = [(odd, 1.0)]
|
||||
self.assertEqual(
|
||||
folder_sets(root, files)[0], _legacy_subfolders(root, files, False)
|
||||
)
|
||||
def test_a_path_not_spelled_like_the_root_falls_back_to_relative_to(self):
|
||||
# Windows folds case: the string prefix misses, relative_to matches.
|
||||
root = PureWindowsPath("C:/Out")
|
||||
files = [(PureWindowsPath("c:/out/x/y.png"), 1.0)]
|
||||
folders, with_ancestors = folder_sets(root, files)
|
||||
self.assertEqual(folders, {"x"})
|
||||
self.assertEqual(folders, _legacy_subfolders(root, files, False))
|
||||
self.assertEqual(with_ancestors, _legacy_subfolders(root, files, True))
|
||||
|
||||
|
||||
class SameListingTests(unittest.TestCase):
|
||||
|
||||
Reference in New Issue
Block a user