Files
Comfy-Org-ComfyUI-Manager/tests/test_cache_freshness.py
T
Perfect Johny bfd65e7173 Write cache files whole and age them by their last write (#3432)
The cache files of the channel lists and of the registry's node list
were written in place. A write that stopped part-way left a truncated
file, which get_data reads as an empty list.

is_file_created_within_one_day read os.path.getctime. On Windows that
is the creation time, which stays the same when the file is rewritten,
so a cache first created more than a day ago counted as expired however
recently it was written.

- Add write_cache_file, which writes a temporary file beside the cache
  and replaces the cache with it, keeping the mode of the existing
  file. Where the written file cannot be put in place, as on Windows
  while another reader has the cache open, the replace is tried three
  times, 0.1 s apart. A cache that stays open is left as it is, with a
  warning; the caller gets no error and goes on with its data. A
  failure while writing the temporary file leaves the cache as it is
  as well. Where the mode cannot be set, the cache is replaced all the
  same. A temporary file that cannot be removed, because a scanner
  holds it open, is left behind and does not fail the write
- Use it for the four writes of a cache file: two in manager_util, one
  in get_data_by_mode and one in default_cache_update
- Read the modification time for the age
- Add tests for a write that fails and one that runs out of space, the
  mode, a mode that cannot be set, a replace that is refused for a
  moment and one that stays refused, a temporary file that is held
  open, and the age
- Add a CHANGELOG entry

The same change for manager-v4 is #3532.
2026-11-15 14:30:12 +09:00

173 lines
6.7 KiB
Python

"""Cache freshness and durability:
- freshness must be mtime-based: on Windows getctime is the CREATION time,
and in-place rewrites never changed it, so every rewritten cache was
'expired' forever after day one;
- writes must be atomic: a kill mid-dump used to leave a truncated cache
that looked fresh but served {} for the whole TTL. A write that fails
part-way leaves the previous cache, the cache keeps its mode, and a cache
that stays open elsewhere is left as it is rather than written in place.
"""
import errno
import json
import os
import sys
import time
from unittest.mock import patch
import pytest
from manager_test_utils import load_manager_util
MANAGER_UTIL = load_manager_util()
def _fresh_cache(tmp_path, body):
cache = tmp_path / "cache.json"
cache.write_text(json.dumps(body), encoding="utf-8")
return cache
def test_rewritten_cache_stays_fresh(tmp_path):
cache = _fresh_cache(tmp_path, {"a": 1})
stale_creation = time.time() - 86401
with patch.object(MANAGER_UTIL, "get_cache_path", return_value=str(cache)), \
patch.object(os.path, "getctime", return_value=stale_creation):
MANAGER_UTIL.save_to_cache("https://example/list.json", {"b": 2}, silent=True)
# the rewrite moved mtime forward; freshness must hold even with a
# creation time older than a day (the Windows getctime regression)
assert MANAGER_UTIL.is_file_created_within_one_day(str(cache)) is True
# age the file explicitly: only then is it expired
old = time.time() - 86401
os.utime(cache, (old, old))
assert MANAGER_UTIL.is_file_created_within_one_day(str(cache)) is False
def test_save_to_cache_leaves_no_tmp_debris_and_valid_json(tmp_path):
cache = _fresh_cache(tmp_path, {"a": 1})
with patch.object(MANAGER_UTIL, "get_cache_path", return_value=str(cache)):
MANAGER_UTIL.save_to_cache("https://example/list.json", {"b": 2}, silent=True)
assert json.loads(cache.read_text(encoding="utf-8")) == {"b": 2}
assert [p.name for p in tmp_path.iterdir()] == ["cache.json"]
def _save(cache, body):
with patch.object(MANAGER_UTIL, "get_cache_path", return_value=str(cache)):
MANAGER_UTIL.save_to_cache("https://example/list.json", body, silent=True)
def test_write_that_fails_on_a_value_keeps_the_previous_cache(tmp_path):
cache = _fresh_cache(tmp_path, {"a": 1})
with pytest.raises(TypeError):
_save(cache, {"b": 2, "c": object()})
assert json.loads(cache.read_text(encoding="utf-8")) == {"a": 1}
assert [p.name for p in tmp_path.iterdir()] == ["cache.json"]
def test_write_that_runs_out_of_space_keeps_the_previous_cache(tmp_path):
cache = _fresh_cache(tmp_path, {"a": 1})
def disk_full(obj, file, **_):
file.write('{"b": ')
raise OSError(errno.ENOSPC, "No space left on device")
with patch.object(MANAGER_UTIL.json, "dump", disk_full), pytest.raises(OSError):
_save(cache, {"b": 2})
assert json.loads(cache.read_text(encoding="utf-8")) == {"a": 1}
assert [p.name for p in tmp_path.iterdir()] == ["cache.json"]
@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits")
def test_cache_keeps_its_mode(tmp_path):
plain = tmp_path / "plain"
plain.write_text("")
default_mode = plain.stat().st_mode & 0o777
plain.unlink()
cache = tmp_path / "cache.json"
_save(cache, {"a": 1})
assert cache.stat().st_mode & 0o777 == default_mode
other_mode = default_mode ^ 0o040 # never what a new file gets
os.chmod(cache, other_mode)
_save(cache, {"b": 2})
assert cache.stat().st_mode & 0o777 == other_mode
def test_cache_is_replaced_where_its_mode_cannot_be_set(tmp_path):
cache = _fresh_cache(tmp_path, {"a": 1})
replace = os.replace
replaced = []
def refuse(path, mode):
raise PermissionError(errno.EPERM, "Operation not permitted", path)
def record(src, dst):
replaced.append(dst)
replace(src, dst)
with patch.object(os, "chmod", refuse), patch.object(os, "replace", record):
_save(cache, {"b": 2})
assert replaced == [str(cache)]
assert json.loads(cache.read_text(encoding="utf-8")) == {"b": 2}
assert [p.name for p in tmp_path.iterdir()] == ["cache.json"]
def _refusing_replace(refusals):
"""An os.replace that refuses the given number of times, as Windows does
while another reader has the file open, and then replaces."""
replace = os.replace
attempts = []
def attempt(src, dst):
attempts.append(dst)
if len(attempts) <= refusals:
raise PermissionError(13, "The process cannot access the file", dst)
replace(src, dst)
return attempt, attempts
def test_cache_is_replaced_once_the_reader_has_closed_it(tmp_path):
cache = _fresh_cache(tmp_path, {"a": 1})
attempt, attempts = _refusing_replace(2)
with patch.object(os, "replace", attempt), patch.object(MANAGER_UTIL.time, "sleep"), \
patch.object(MANAGER_UTIL.json, "dump", wraps=json.dump) as dump:
_save(cache, {"b": 2})
assert len(attempts) == 3
assert dump.call_count == 1, "the cache must not be written in place"
assert json.loads(cache.read_text(encoding="utf-8")) == {"b": 2}
assert [p.name for p in tmp_path.iterdir()] == ["cache.json"]
def test_cache_that_stays_open_elsewhere_is_left_as_it_is(tmp_path, caplog):
cache = _fresh_cache(tmp_path, {"a": 1})
attempt, attempts = _refusing_replace(100)
with patch.object(os, "replace", attempt), patch.object(MANAGER_UTIL.time, "sleep"), \
patch.object(MANAGER_UTIL, "get_cache_path", return_value=str(cache)), caplog.at_level("INFO"):
# the caller sees no error: it goes on with the data it fetched
MANAGER_UTIL.save_to_cache("https://example/list.json", {"b": 2})
assert len(attempts) == 3
assert json.loads(cache.read_text(encoding="utf-8")) == {"a": 1}
assert [p.name for p in tmp_path.iterdir()] == ["cache.json"]
assert "The cache could not be replaced" in caplog.text
assert "default cache updated" not in caplog.text
def test_temporary_file_that_is_open_elsewhere_does_not_fail_the_write(tmp_path):
# a scanner that opens the new file keeps Windows from replacing with it and from removing it
cache = _fresh_cache(tmp_path, {"a": 1})
attempt, attempts = _refusing_replace(100)
def refuse(path):
raise PermissionError(13, "The process cannot access the file", path)
with patch.object(os, "replace", attempt), patch.object(os, "remove", refuse), \
patch.object(MANAGER_UTIL.time, "sleep"):
assert MANAGER_UTIL.write_cache_file(str(cache), {"b": 2}) is False
assert len(attempts) == 3
assert json.loads(cache.read_text(encoding="utf-8")) == {"a": 1}