fix(images): keep the 3.10-3.13 suffix rule in the scandir walk and harden the warm-up (#348)

Review of the walk speedup:
- os.path.splitext drops names like ..png that Path.suffix listed on Python
  3.10-3.13 (3.14 changed Path.suffix, which hid it from the parity test on
  a 3.14 venv). _name_suffix spells out the 3.10-3.13 rule; the test now
  asserts ..png and ...webp are listed and g. is not.
- _warm_workspace reads the roots inside its try, so a failing
  _get_all_output_dirs warns instead of leaving an unretrieved task error.
- The folder_sets fallback test now takes the fallback (a Windows case
  mismatch); the coalescing test pins the walk count.
This commit is contained in:
Vito Sansevero
2026-10-26 11:23:41 -07:00
parent 3dd45c0efd
commit 5a9971784c
3 changed files with 51 additions and 12 deletions
+15 -4
View File
@@ -772,6 +772,17 @@ def _favorite_root_key(root) -> str:
return _favorite_relative_key(os.path.realpath(str(root)))
def _name_suffix(name):
"""``Path(name).suffix`` as Python 3.10-3.13 compute it.
The last ``.``-part unless the dot is the first or last character, so
``..png`` is ``.png`` and ``.png`` has none. Spelled out because 3.14
changed ``Path.suffix`` and ``os.path.splitext`` differs for leading dots.
"""
dot = name.rfind(".")
return name[dot:] if 0 < dot < len(name) - 1 else ""
def _walk_media(output_path, max_files):
"""Yield the ``os.DirEntry`` of each media file under *output_path*.
@@ -801,7 +812,7 @@ def _walk_media(output_path, max_files):
except OSError:
continue # as os.walk: an unreadable folder is skipped
for entry in sorted(files, key=lambda item: item.name):
if os.path.splitext(entry.name)[1].lower() not in MEDIA_EXTENSIONS:
if _name_suffix(entry.name).lower() not in MEDIA_EXTENSIONS:
continue
if entry.is_symlink():
continue # never read through a link out of the output tree
@@ -1330,10 +1341,10 @@ class ImageRoutesMixin:
async def _warm_workspace(self):
while True:
self._workspace_warm_again = False
listings = self._stored_listings()
if listings is None:
return # a root has no listing yet; its walk asks again
try:
listings = self._stored_listings()
if listings is None:
return # a root has no listing yet; its walk asks again
await self._run_in_executor(self._build_workspace_caches, listings)
except Exception as e:
self.logger.warning(f"Could not prepare the Images page: {e}")
+28 -1
View File
@@ -2052,6 +2052,9 @@ class TestOutputScanHelpers(ImageAPITestCase):
"b/thumbnails/u.png",
"e/.hidden.png",
"e/f.jpeg",
"e/..png",
"e/...webp",
"e/g.",
):
make_png(output / rel)
try:
@@ -2065,6 +2068,11 @@ class TestOutputScanHelpers(ImageAPITestCase):
capped = list(images_module._iter_media_files(output, 3))
self.assertEqual(listed, expected)
self.assertEqual(capped, expected[:3])
# Path.suffix of Python 3.10-3.13, which CI and hosts run (3.14 changed
# it): "..png" has the suffix ".png", ".png" and "g." have none.
names = {path.name for path in listed}
self.assertTrue({"..png", "...webp", ".hidden.png"} <= names, names)
self.assertFalse({"g.", "notes.txt"} & names, names)
self.assertTrue(all(isinstance(path, Path) for path in listed))
def test_gallery_scan_reads_each_file_type_and_mtime_from_scandir(self):
@@ -2436,7 +2444,10 @@ def _reference_iter_media_files(output_path, max_files):
if len(rel_parts) >= images_module.MAX_SCAN_DEPTH:
dirs[:] = []
for name in sorted(files):
if Path(name).suffix.lower() not in images_module.MEDIA_EXTENSIONS:
# Path.suffix as Python 3.10-3.13 computed it (3.14 changed it).
dot = name.rfind(".")
suffix = name[dot:] if 0 < dot < len(name) - 1 else ""
if suffix.lower() not in images_module.MEDIA_EXTENSIONS:
continue
if (root_path / name).is_symlink():
continue
@@ -3025,8 +3036,24 @@ class TestSharedListing(ImageAPITestCase):
release.set()
await self._settle()
self.assertEqual(built, [1, 3])
self.assertEqual(self.scan.calls, 3) # two stored walks, one more build
self.assertTrue(self._workspace_ready())
async def test_a_build_that_cannot_list_the_roots_only_warns(self):
make_png(self.output_dir / "a.png")
await self._warmed()
def broken():
raise RuntimeError("no roots")
self.api._get_all_output_dirs = broken
with self.assertLogs(self.api.logger, "WARNING") as logs:
self.api._schedule_workspace_warm()
task = self.api._workspace_warm_task
await asyncio.wait([task])
self.assertIsNone(task.exception())
self.assertTrue(any("Images page" in line for line in logs.output))
async def test_cleanup_stops_a_running_workspace_build(self):
make_png(self.output_dir / "a.png")
release, entered = threading.Event(), threading.Event()
+8 -7
View File
@@ -80,13 +80,14 @@ class FolderSetsTests(unittest.TestCase):
self.assertEqual(folders, {os.path.join("a", "b")})
self.assertEqual(with_ancestors, {os.path.join("a", "b"), "a"})
def test_a_file_outside_the_root_string_still_gets_its_folder(self):
root = Path("/out")
odd = Path("/out/./x/y.png") # normalizes to /out/x/y.png
files = [(odd, 1.0)]
self.assertEqual(
folder_sets(root, files)[0], _legacy_subfolders(root, files, False)
)
def test_a_path_not_spelled_like_the_root_falls_back_to_relative_to(self):
# Windows folds case: the string prefix misses, relative_to matches.
root = PureWindowsPath("C:/Out")
files = [(PureWindowsPath("c:/out/x/y.png"), 1.0)]
folders, with_ancestors = folder_sets(root, files)
self.assertEqual(folders, {"x"})
self.assertEqual(folders, _legacy_subfolders(root, files, False))
self.assertEqual(with_ancestors, _legacy_subfolders(root, files, True))
class SameListingTests(unittest.TestCase):