perf(images): refresh the listing when the monitor sees a new output (#348, PR 2)
The image monitor now reports every settled output (linked or not, also when linking raised) with its resolved path through set_output_listener. The API's notify_output_settled hops from the monitor's worker thread onto the server's loop (call_soon_threadsafe) and does nothing before the loop is known or after cleanup. mark_root_dirty matches the path against each walk's resolved_root (normcase(realpath), computed in the executor by walk_root, now its fourth value) and marks the root dirty. A dirty root gets one background walk, no sooner than rewalk_interval = min(15, max(5, 5 x last walk's duration)) after the previous walk started (last_started_at / last_duration are recorded on every walk). _schedule_walk is shared with PR 1's delayed stale refresh: one deferred walk per root, never armed while a walk runs (the walk's end re-checks dirty: the trailing walk), at least 0.5 s away, the earlier deadline wins, and every walk start cancels it. A failed walk clears dirty. Rescan semantics are unchanged.
This commit is contained in:
Vito Sansevero
+422
-1
@@ -2540,7 +2540,10 @@ class TestSharedListing(ImageAPITestCase):
|
||||
from py.api import gallery_listing
|
||||
|
||||
record = self._record()
|
||||
record.walked_at -= (seconds or gallery_listing.REFRESH_AFTER) + 1
|
||||
shift = (seconds or gallery_listing.REFRESH_AFTER) + 1
|
||||
record.walked_at -= shift
|
||||
# An old listing's walk also started long ago (REWALK_MIN_INTERVAL).
|
||||
record.last_started_at -= shift
|
||||
|
||||
async def _settle(self):
|
||||
"""Wait for every background walk this API started."""
|
||||
@@ -3073,6 +3076,401 @@ class TestSharedListing(ImageAPITestCase):
|
||||
self.assertTrue(task.done())
|
||||
self.assertIsNone(self.api._workspace_warm_task)
|
||||
|
||||
# -- monitor-triggered walks (#348, PR 2) ---------------------------
|
||||
|
||||
def _interval(self, seconds):
|
||||
"""Patch REWALK_MIN_INTERVAL to *seconds*, whatever the walk took."""
|
||||
for name in ("REWALK_FLOOR", "REWALK_CAP"):
|
||||
patcher = patch.object(gallery_listing, name, seconds)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def _output(self, name):
|
||||
"""A new output on disk, spelled as the monitor reports it (resolved)."""
|
||||
return os.path.realpath(make_png(self.output_dir / name))
|
||||
|
||||
async def _listed(self):
|
||||
"""A stored listing of the root, built by a read."""
|
||||
await self._json("GET", self.URL)
|
||||
await self._settle()
|
||||
return self._record()
|
||||
|
||||
def _fire(self, record):
|
||||
"""Run the pending deferred walk now, as its timer would."""
|
||||
handle = record.rewalk_handle
|
||||
self.assertIsNotNone(handle, "no deferred walk is pending")
|
||||
handle.cancel()
|
||||
self.api._delayed_walk(self.output_dir, record)
|
||||
|
||||
def _remaining(self, handle):
|
||||
return handle.when() - asyncio.get_running_loop().time()
|
||||
|
||||
async def test_a_settled_output_is_listed_without_a_reload(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(0)
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
self.assertIsNotNone(record.flight) # the interval allows: walks now
|
||||
self.assertFalse(record.dirty)
|
||||
await self._settle()
|
||||
self.assertEqual(self.scan.calls, 2)
|
||||
self.assertEqual(len(record.files), 2)
|
||||
_, data = await self._json("GET", self.URL)
|
||||
self.assertEqual(data["total"], 2)
|
||||
self.assertEqual(self.scan.calls, 2)
|
||||
|
||||
async def test_the_monitor_thread_hops_onto_the_loop(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(0)
|
||||
path = self._output("b.png")
|
||||
calls = []
|
||||
real = self.api.mark_root_dirty
|
||||
|
||||
def mark(seen):
|
||||
calls.append(threading.get_ident())
|
||||
real(seen)
|
||||
|
||||
self.api.mark_root_dirty = mark
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, self.api.notify_output_settled, path)
|
||||
# The hop was queued before the executor's result: one loop turn runs it.
|
||||
await asyncio.sleep(0)
|
||||
self.assertEqual(calls, [threading.get_ident()])
|
||||
await self._settle()
|
||||
self.assertEqual(len(record.files), 2)
|
||||
|
||||
async def test_notify_before_the_loop_is_known_does_nothing(self):
|
||||
self.assertIsNone(self.api._listing_loop)
|
||||
with patch.object(self.api, "mark_root_dirty") as mark:
|
||||
self.api.notify_output_settled(self._output("b.png"))
|
||||
await asyncio.sleep(0)
|
||||
mark.assert_not_called()
|
||||
|
||||
async def test_notify_after_cleanup_does_nothing(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
await self.api.cleanup_gallery_listings(web.Application())
|
||||
self._interval(0)
|
||||
with patch.object(self.api, "mark_root_dirty") as mark:
|
||||
self.api.notify_output_settled(self._output("b.png"))
|
||||
await asyncio.sleep(0)
|
||||
mark.assert_not_called()
|
||||
# A hop queued before the cleanup is dropped when it runs.
|
||||
self.api.mark_root_dirty(self._output("c.png"))
|
||||
self.assertFalse(record.dirty)
|
||||
self.assertIsNone(record.flight)
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
|
||||
async def test_notify_on_a_closed_loop_does_nothing(self):
|
||||
loop = asyncio.new_event_loop()
|
||||
loop.close()
|
||||
self.api._listing_loop = loop
|
||||
self.api.notify_output_settled(self._output("b.png")) # no exception
|
||||
|
||||
async def test_notify_racing_the_loop_closing_does_nothing(self):
|
||||
class ClosingLoop:
|
||||
def is_running(self):
|
||||
return True
|
||||
|
||||
def call_soon_threadsafe(self, *_args):
|
||||
raise RuntimeError("Event loop is closed")
|
||||
|
||||
self.api._listing_loop = ClosingLoop()
|
||||
self.api.notify_output_settled(self._output("b.png")) # no exception
|
||||
|
||||
async def test_an_output_outside_every_root_is_ignored(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(0)
|
||||
elsewhere = make_png(self.comfy_dir / "output2" / "x.png")
|
||||
self.api.mark_root_dirty(os.path.realpath(elsewhere))
|
||||
self.assertFalse(record.dirty)
|
||||
self.assertIsNone(record.flight)
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
|
||||
async def test_a_root_never_listed_is_not_walked_by_a_mark(self):
|
||||
# No listing, no walk running: the first read walks anyway.
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
self.assertEqual(self.api._root_listings, {})
|
||||
self.assertEqual(self.scan.calls, 0)
|
||||
|
||||
async def test_a_dropped_listing_is_left_to_the_next_read(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self.api.invalidate_gallery_cache()
|
||||
self._interval(0)
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
self.assertIsNone(record.flight)
|
||||
self.assertEqual(self.scan.calls, 1)
|
||||
|
||||
async def test_the_walk_records_the_resolved_root_and_its_timing(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
before = images_module._time.monotonic()
|
||||
record = await self._listed()
|
||||
self.assertEqual(
|
||||
record.resolved_root, os.path.normcase(os.path.realpath(self.output_dir))
|
||||
)
|
||||
self.assertGreaterEqual(record.last_started_at, before)
|
||||
self.assertLessEqual(record.last_started_at, record.walked_at)
|
||||
# Measured when the walk ended, after its listing was stored.
|
||||
self.assertGreaterEqual(
|
||||
record.last_duration, record.walked_at - record.last_started_at
|
||||
)
|
||||
|
||||
async def test_a_failed_walk_records_its_timing_and_clears_dirty(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(30)
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
self.assertTrue(record.dirty)
|
||||
self.api._scan_gallery_files_sync = _raise
|
||||
started = record.last_started_at
|
||||
with self.assertLogs(self.api.logger, "WARNING"):
|
||||
self._fire(record)
|
||||
await self._settle()
|
||||
self.assertGreater(record.last_started_at, started)
|
||||
self.assertIsNotNone(record.last_duration)
|
||||
self.assertFalse(record.dirty)
|
||||
self.assertIsNone(record.rewalk_handle) # no trailing walk after a failure
|
||||
self.assertEqual(len(record.files), 1) # the previous listing is kept
|
||||
|
||||
async def test_a_burst_of_marks_coalesces_into_one_walk(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(30)
|
||||
for i in range(5):
|
||||
self.api.mark_root_dirty(self._output(f"b{i}.png"))
|
||||
handle = record.rewalk_handle
|
||||
self.assertIsNotNone(handle)
|
||||
self.assertIsNone(record.flight)
|
||||
self.assertEqual(self.scan.calls, 1)
|
||||
# Due one interval after the last walk started, not 30 s from now.
|
||||
due = record.last_started_at + 30 - images_module._time.monotonic()
|
||||
self.assertAlmostEqual(self._remaining(handle), due, delta=1)
|
||||
self._fire(record)
|
||||
await self._settle()
|
||||
self.assertEqual(self.scan.calls, 2)
|
||||
self.assertEqual(len(record.files), 6)
|
||||
self.assertFalse(record.dirty)
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
|
||||
async def test_a_mark_during_a_walk_gives_exactly_one_trailing_walk(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(30)
|
||||
gate = self.scan.hold_next()
|
||||
rescan = asyncio.ensure_future(
|
||||
self._json("POST", "/prompt_manager/gallery/rescan")
|
||||
)
|
||||
await self.scan.wait_entered() # listed before b.png exists
|
||||
for name in ("b.png", "c.png"):
|
||||
self.api.mark_root_dirty(self._output(name))
|
||||
# Never armed while a walk runs: the walk's end re-checks dirty.
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
self.assertTrue(record.dirty)
|
||||
gate.set()
|
||||
status, _ = await rescan
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(len(record.files), 1)
|
||||
handle = record.rewalk_handle
|
||||
self.assertIsNotNone(handle) # the trailing walk, deferred by the interval
|
||||
self.assertGreaterEqual(self._remaining(handle), 0.5)
|
||||
self._fire(record)
|
||||
await self._settle()
|
||||
self.assertEqual(self.scan.calls, 3)
|
||||
self.assertEqual(len(record.files), 3)
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
self.assertFalse(record.dirty)
|
||||
|
||||
async def test_a_walk_slower_than_the_interval_trails_at_once_and_stops(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(0) # the walk outlasts its interval
|
||||
gate = self.scan.hold_next()
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
await self.scan.wait_entered()
|
||||
self.api.mark_root_dirty(self._output("c.png"))
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
gate.set()
|
||||
await self._settle()
|
||||
# One walk for the first mark, one trailing walk for the second, then
|
||||
# nothing: no mark arrived during the trailing walk.
|
||||
self.assertEqual(self.scan.calls, 3)
|
||||
self.assertEqual(len(record.files), 3)
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
self.assertIsNone(record.flight)
|
||||
|
||||
async def test_a_read_triggered_walk_cancels_a_pending_rewalk(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(30)
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
handle = record.rewalk_handle
|
||||
status, data = await self._json("POST", "/prompt_manager/gallery/rescan")
|
||||
self.assertEqual((status, data["total"]), (200, 2))
|
||||
self.assertTrue(handle.cancelled())
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
self.assertFalse(record.dirty)
|
||||
await self._settle()
|
||||
self.assertEqual(self.scan.calls, 2)
|
||||
|
||||
async def test_after_the_trailing_walk_a_new_mark_arms_again(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(30)
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
self._fire(record)
|
||||
await self._settle()
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
self.api.mark_root_dirty(self._output("c.png"))
|
||||
self.assertIsNotNone(record.rewalk_handle)
|
||||
self._fire(record)
|
||||
await self._settle()
|
||||
self.assertEqual(self.scan.calls, 3)
|
||||
self.assertEqual(len(record.files), 3)
|
||||
|
||||
async def test_a_deferred_walk_waits_at_least_half_a_second(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(30)
|
||||
# The interval ends in a few milliseconds: still never a zero delay.
|
||||
record.last_started_at = images_module._time.monotonic() - 29.99
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
handle = record.rewalk_handle
|
||||
self.assertIsNotNone(handle)
|
||||
self.assertGreaterEqual(self._remaining(handle), 0.45)
|
||||
handle.cancel()
|
||||
|
||||
async def test_a_mark_moves_a_later_stale_refresh_earlier(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._age()
|
||||
self._interval(5)
|
||||
with patch.object(gallery_listing, "REFRESH_DELAY", 30):
|
||||
await self._json("GET", self.URL)
|
||||
stale = record.rewalk_handle
|
||||
self.assertGreater(self._remaining(stale), 25)
|
||||
# The interval has long passed: the mark's walk starts now, and that
|
||||
# walk is the refresh the stale read was waiting for.
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
self.assertTrue(stale.cancelled())
|
||||
self.assertIsNotNone(record.flight)
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
await self._settle()
|
||||
self.assertEqual(self.scan.calls, 2)
|
||||
self.assertEqual(len(record.files), 2)
|
||||
|
||||
async def test_a_stale_read_keeps_an_earlier_pending_rewalk(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(20)
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
pending = record.rewalk_handle
|
||||
when = pending.when()
|
||||
# Stale by its age, while the interval since the last walk still runs.
|
||||
record.walked_at -= gallery_listing.REFRESH_AFTER + 1
|
||||
with patch.object(gallery_listing, "REFRESH_DELAY", 30):
|
||||
await self._json("GET", self.URL)
|
||||
self.assertIs(record.rewalk_handle, pending)
|
||||
self.assertEqual(pending.when(), when)
|
||||
self.assertFalse(pending.cancelled())
|
||||
|
||||
async def test_a_stale_read_moves_a_later_pending_rewalk_earlier(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(15)
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
pending = record.rewalk_handle
|
||||
self.assertGreater(self._remaining(pending), 10)
|
||||
# A listing older than REFRESH_AFTER whose interval is over.
|
||||
self._age()
|
||||
with patch.object(gallery_listing, "REFRESH_DELAY", 2):
|
||||
await self._json("GET", self.URL)
|
||||
self.assertTrue(pending.cancelled())
|
||||
replaced = record.rewalk_handle
|
||||
self.assertIsNotNone(replaced)
|
||||
self.assertLess(self._remaining(replaced), 3)
|
||||
self.assertEqual(self.scan.calls, 1)
|
||||
replaced.cancel()
|
||||
|
||||
async def test_a_dirty_read_rearms_a_lost_rewalk(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(30)
|
||||
record.dirty = True # e.g. its timer was cancelled by hand
|
||||
await self._json("GET", self.URL)
|
||||
self.assertIsNotNone(record.rewalk_handle)
|
||||
self.assertEqual(self.scan.calls, 1)
|
||||
|
||||
async def test_cleanup_cancels_a_pending_rewalk(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(30)
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
handle = record.rewalk_handle
|
||||
await self.api.cleanup_gallery_listings(web.Application())
|
||||
self.assertTrue(handle.cancelled())
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
self.assertEqual(self.scan.calls, 1)
|
||||
|
||||
async def test_a_walk_cancelled_by_cleanup_starts_no_trailing_walk(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
record = await self._listed()
|
||||
self._interval(0)
|
||||
gate = self.scan.hold_next()
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
await self.scan.wait_entered()
|
||||
self.api.mark_root_dirty(self._output("c.png"))
|
||||
self.assertTrue(record.dirty)
|
||||
await self.api.cleanup_gallery_listings(web.Application())
|
||||
gate.set()
|
||||
self.assertIsNone(record.flight)
|
||||
self.assertIsNone(record.rewalk_handle)
|
||||
self.assertEqual(self.api._listing_tasks, set())
|
||||
self.assertEqual(self.scan.calls, 2)
|
||||
|
||||
async def test_a_warm_up_after_cleanup_listens_again(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
self._interval(0)
|
||||
await self._warmed()
|
||||
await self.api.cleanup_gallery_listings(web.Application())
|
||||
await self._warmed()
|
||||
record = self._record()
|
||||
self.api.mark_root_dirty(self._output("b.png"))
|
||||
await self._settle()
|
||||
self.assertEqual(len(record.files), 2)
|
||||
|
||||
async def test_monitor_walks_during_a_workspace_build_add_one_more_build(self):
|
||||
make_png(self.output_dir / "a.png")
|
||||
release, entered = threading.Event(), threading.Event()
|
||||
built = []
|
||||
real = self.api._build_workspace_caches
|
||||
|
||||
def build(listings):
|
||||
built.append(len(listings[0][2]))
|
||||
if len(built) == 1:
|
||||
entered.set()
|
||||
release.wait(_HANG_LIMIT)
|
||||
real(listings)
|
||||
|
||||
self.addCleanup(release.set)
|
||||
self._interval(0)
|
||||
with patch.object(self.api, "_build_workspace_caches", side_effect=build):
|
||||
await self.api.warm_gallery_listings(web.Application())
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, entered.wait, _HANG_LIMIT)
|
||||
for name in ("b.png", "c.png", "d.png"): # three monitor walks
|
||||
self.api.mark_root_dirty(self._output(name))
|
||||
while self.api._listing_tasks:
|
||||
await asyncio.gather(*list(self.api._listing_tasks))
|
||||
await asyncio.sleep(0)
|
||||
release.set()
|
||||
await self._settle()
|
||||
self.assertEqual(built, [1, 4])
|
||||
self.assertEqual(self.scan.calls, 4)
|
||||
self.assertTrue(self._workspace_ready())
|
||||
|
||||
|
||||
class TestGalleryListingHostHooks(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_root_registers_the_warm_up_and_its_cleanup(self):
|
||||
@@ -3092,3 +3490,26 @@ class TestGalleryListingHostHooks(unittest.IsolatedAsyncioTestCase):
|
||||
) as (package, host):
|
||||
self.assertIn(package.api.warm_gallery_listings, host.app.on_startup)
|
||||
self.assertIn(package.api.cleanup_gallery_listings, host.app.on_cleanup)
|
||||
|
||||
async def test_root_connects_the_monitor_to_the_listing(self):
|
||||
try:
|
||||
from test_init_package import fake_server_module, load_package
|
||||
except ImportError:
|
||||
from tests.test_init_package import fake_server_module, load_package
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
server = fake_server_module()
|
||||
server.PromptServer.instance.app = web.Application()
|
||||
with load_package(
|
||||
"gallery_listing_monitor_pkg",
|
||||
str(Path(directory) / "db.sqlite"),
|
||||
directory,
|
||||
server,
|
||||
) as (package, host):
|
||||
monitor = package._global_image_monitor
|
||||
self.assertEqual(
|
||||
monitor.output_listener, package.api.notify_output_settled
|
||||
)
|
||||
# The first cleanup hook stops the monitor's hand-offs.
|
||||
await host.app.on_cleanup[0](host.app)
|
||||
self.assertIsNone(monitor.output_listener)
|
||||
|
||||
@@ -188,6 +188,7 @@ class TestWorkspaceImages(ImageAPITestCase):
|
||||
newest = self.api._workspace_order_cache[key][1]["newest"]
|
||||
favorites = self.api._favorite_listing_cache[str(self.output_dir)]
|
||||
record.walked_at -= gallery_listing.REFRESH_AFTER + 1
|
||||
record.last_started_at -= gallery_listing.REFRESH_AFTER + 1
|
||||
with patch.object(
|
||||
self.api,
|
||||
"_scan_gallery_files_sync",
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Pure helpers of the shared Images listing (py/api/gallery_listing.py, #348)."""
|
||||
|
||||
import ntpath
|
||||
import os
|
||||
import posixpath
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -17,6 +19,8 @@ from py.api.gallery_listing import ( # noqa: E402
|
||||
RootListing,
|
||||
WalkOutcome,
|
||||
folder_sets,
|
||||
rewalk_interval,
|
||||
root_contains,
|
||||
same_listing,
|
||||
walk_root,
|
||||
)
|
||||
@@ -123,24 +127,36 @@ class WalkRootTests(unittest.TestCase):
|
||||
def test_returns_the_listing_and_both_folder_sets(self):
|
||||
files = [(self.root / "sub" / "a.png", 2.0), (self.root / "b.png", 1.0)]
|
||||
scan, calls = self._scan(files)
|
||||
listed, folders, with_ancestors = walk_root(scan, self.root, None)
|
||||
listed, folders, with_ancestors, resolved = walk_root(scan, self.root, None)
|
||||
self.assertEqual(calls, [self.root])
|
||||
self.assertEqual(listed, files)
|
||||
self.assertEqual(folders, frozenset({"sub"}))
|
||||
self.assertEqual(with_ancestors, frozenset({"sub"}))
|
||||
self.assertEqual(resolved, os.path.normcase(os.path.realpath(self.root)))
|
||||
|
||||
def test_the_resolved_root_follows_a_symlinked_root(self):
|
||||
link = Path(self._tmp.name) / "linked"
|
||||
try:
|
||||
os.symlink(self.root, link, target_is_directory=True)
|
||||
except (OSError, NotImplementedError) as error:
|
||||
self.skipTest(f"symlinks unavailable: {error}")
|
||||
scan, _calls = self._scan([])
|
||||
resolved = walk_root(scan, link, None)[3]
|
||||
self.assertEqual(resolved, os.path.normcase(os.path.realpath(self.root)))
|
||||
self.assertNotEqual(resolved, os.path.normcase(str(link)))
|
||||
|
||||
def test_an_unchanged_walk_returns_the_previous_list_object(self):
|
||||
files = [(self.root / "sub" / "a.png", 2.0)]
|
||||
previous = list(files)
|
||||
scan, _calls = self._scan(files)
|
||||
listed, _folders, _ancestors = walk_root(scan, self.root, previous)
|
||||
listed, _folders, _ancestors, _resolved = walk_root(scan, self.root, previous)
|
||||
self.assertIs(listed, previous)
|
||||
|
||||
def test_a_changed_walk_returns_the_new_list(self):
|
||||
previous = [(self.root / "sub" / "a.png", 2.0)]
|
||||
files = [(self.root / "sub" / "a.png", 3.0)]
|
||||
scan, _calls = self._scan(files)
|
||||
listed, _folders, _ancestors = walk_root(scan, self.root, previous)
|
||||
listed, _folders, _ancestors, _resolved = walk_root(scan, self.root, previous)
|
||||
self.assertIsNot(listed, previous)
|
||||
self.assertEqual(listed, files)
|
||||
|
||||
@@ -180,6 +196,96 @@ class RecordTests(unittest.TestCase):
|
||||
self.assertEqual(gallery_listing.REFRESH_DELAY, 1.0)
|
||||
self.assertIsNone(RootListing().rewalk_handle)
|
||||
|
||||
def test_a_new_record_is_clean_and_never_walked(self):
|
||||
record = RootListing()
|
||||
self.assertFalse(record.dirty)
|
||||
self.assertIsNone(record.resolved_root)
|
||||
self.assertIsNone(record.last_started_at)
|
||||
self.assertIsNone(record.last_duration)
|
||||
|
||||
def test_drop_keeps_the_resolved_root_and_the_walk_timing(self):
|
||||
# A drop clears the listing, not what is known about the root itself.
|
||||
record = RootListing(resolved_root="/o", last_started_at=1.0)
|
||||
record.last_duration = 0.5
|
||||
record.dirty = True
|
||||
record.drop()
|
||||
self.assertEqual(record.resolved_root, "/o")
|
||||
self.assertEqual((record.last_started_at, record.last_duration), (1.0, 0.5))
|
||||
self.assertTrue(record.dirty)
|
||||
|
||||
|
||||
class RewalkIntervalTests(unittest.TestCase):
|
||||
def test_constants(self):
|
||||
self.assertEqual(
|
||||
(
|
||||
gallery_listing.REWALK_FLOOR,
|
||||
gallery_listing.REWALK_FACTOR,
|
||||
gallery_listing.REWALK_CAP,
|
||||
gallery_listing.MIN_REWALK_DELAY,
|
||||
),
|
||||
(5, 5, 15, 0.5),
|
||||
)
|
||||
|
||||
def test_short_walks_wait_the_floor(self):
|
||||
self.assertEqual(rewalk_interval(0.2), 5)
|
||||
self.assertEqual(rewalk_interval(0), 5)
|
||||
self.assertEqual(rewalk_interval(None), 5) # never walked
|
||||
|
||||
def test_the_interval_scales_with_the_last_walk(self):
|
||||
self.assertEqual(rewalk_interval(1.0), 5)
|
||||
self.assertEqual(rewalk_interval(2.0), 10)
|
||||
self.assertAlmostEqual(rewalk_interval(2.5), 12.5)
|
||||
|
||||
def test_slow_walks_are_capped(self):
|
||||
self.assertEqual(rewalk_interval(3.0), 15)
|
||||
self.assertEqual(rewalk_interval(60.0), 15)
|
||||
|
||||
def test_the_constants_are_read_when_called(self):
|
||||
with (
|
||||
patch.object(gallery_listing, "REWALK_FLOOR", 0),
|
||||
patch.object(gallery_listing, "REWALK_CAP", 0),
|
||||
):
|
||||
self.assertEqual(rewalk_interval(2.0), 0)
|
||||
|
||||
|
||||
class RootContainsTests(unittest.TestCase):
|
||||
def test_posix_paths_under_the_root_match(self):
|
||||
root = posixpath.normcase("/srv/out")
|
||||
self.assertTrue(root_contains(root, "/srv/out/a.png", posixpath))
|
||||
self.assertTrue(root_contains(root, "/srv/out/sub/deep/a.png", posixpath))
|
||||
|
||||
def test_a_sibling_sharing_the_prefix_does_not_match(self):
|
||||
self.assertFalse(root_contains("/srv/out", "/srv/out2/a.png", posixpath))
|
||||
self.assertFalse(root_contains("/srv/out", "/srv/outside.png", posixpath))
|
||||
self.assertFalse(root_contains("/srv/out", "/srv/other/a.png", posixpath))
|
||||
|
||||
def test_posix_keeps_case(self):
|
||||
self.assertFalse(root_contains("/srv/out", "/srv/OUT/a.png", posixpath))
|
||||
|
||||
def test_the_root_itself_is_not_an_output(self):
|
||||
self.assertFalse(root_contains("/srv/out", "/srv/out", posixpath))
|
||||
|
||||
def test_windows_folds_case_and_separators(self):
|
||||
root = ntpath.normcase("C:\\ComfyUI\\Output")
|
||||
self.assertTrue(root_contains(root, "c:\\comfyui\\output\\a.png", ntpath))
|
||||
self.assertTrue(root_contains(root, "C:/ComfyUI/OUTPUT/sub/a.PNG", ntpath))
|
||||
self.assertFalse(root_contains(root, "C:\\ComfyUI\\Output2\\a.png", ntpath))
|
||||
self.assertFalse(root_contains(root, "D:\\ComfyUI\\Output\\a.png", ntpath))
|
||||
|
||||
def test_a_windows_drive_root_and_a_unc_share(self):
|
||||
drive = ntpath.normcase("E:\\")
|
||||
self.assertTrue(root_contains(drive, "e:\\x.png", ntpath))
|
||||
self.assertFalse(root_contains(drive, "F:\\x.png", ntpath))
|
||||
share = ntpath.normcase("\\\\Server\\Share\\Out")
|
||||
self.assertTrue(root_contains(share, "\\\\server\\share\\out\\a.png", ntpath))
|
||||
self.assertFalse(root_contains(share, "\\\\server\\share\\a.png", ntpath))
|
||||
|
||||
def test_the_default_is_this_platforms_rule(self):
|
||||
root = os.path.normcase(os.path.realpath(tempfile.gettempdir()))
|
||||
inside = os.path.join(root, "x.png")
|
||||
self.assertTrue(root_contains(root, inside))
|
||||
self.assertTrue(root_contains(root, inside.upper()) == (os.name == "nt"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -449,6 +449,109 @@ class TestHandlerScheduling(unittest.TestCase):
|
||||
self.assertEqual(self.processed, [first, second])
|
||||
|
||||
|
||||
class TestOutputListener(unittest.TestCase):
|
||||
"""Every settled output is reported to the Images listing (#348, PR 2)."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.handler = im_mod.ImageGenerationHandler(MagicMock(), MagicMock())
|
||||
self.addCleanup(self.handler.shutdown)
|
||||
self.handler.processing_delay = 0
|
||||
self.handler.settle_interval = 0.01
|
||||
self.handler.process_new_image = lambda path, prompt_snapshot=None: None
|
||||
self.reported = []
|
||||
self.handler.output_listener = self.reported.append
|
||||
|
||||
def _image(self, name, content=b"x" * 16):
|
||||
path = os.path.join(self.tmp.name, name)
|
||||
with open(path, "wb") as f:
|
||||
f.write(content)
|
||||
return path
|
||||
|
||||
def test_a_settled_output_is_reported_once_with_its_resolved_path(self):
|
||||
path = self._image("a.png")
|
||||
self.handler._process_pending(path)
|
||||
self.assertEqual(self.reported, [os.path.realpath(path)])
|
||||
|
||||
def test_a_symlinked_spelling_is_reported_resolved(self):
|
||||
real_dir = os.path.join(self.tmp.name, "real")
|
||||
os.makedirs(real_dir)
|
||||
link_dir = os.path.join(self.tmp.name, "link")
|
||||
try:
|
||||
os.symlink(real_dir, link_dir, target_is_directory=True)
|
||||
except (OSError, NotImplementedError) as error:
|
||||
self.skipTest(f"symlinks unavailable: {error}")
|
||||
with open(os.path.join(real_dir, "b.png"), "wb") as f:
|
||||
f.write(b"x" * 16)
|
||||
self.handler._process_pending(os.path.join(link_dir, "b.png"))
|
||||
self.assertEqual(
|
||||
self.reported, [os.path.realpath(os.path.join(real_dir, "b.png"))]
|
||||
)
|
||||
|
||||
def test_it_is_reported_even_when_linking_raises(self):
|
||||
def explode(path, prompt_snapshot=None):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
self.handler.process_new_image = explode
|
||||
path = self._image("c.png")
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.handler._process_pending(path)
|
||||
self.assertEqual(self.reported, [os.path.realpath(path)])
|
||||
self.assertEqual(self.handler.pending_count(), 0)
|
||||
|
||||
def test_an_output_that_never_settles_is_not_reported(self):
|
||||
self.handler.settle_attempts = 3
|
||||
self.handler._process_pending(os.path.join(self.tmp.name, "gone.png"))
|
||||
empty = self._image("empty.png", content=b"")
|
||||
self.handler._process_pending(empty)
|
||||
self.assertEqual(self.reported, [])
|
||||
|
||||
def test_a_video_is_reported_when_its_extension_is_monitored(self):
|
||||
self.handler.supported_extensions = (".png", ".mp4")
|
||||
path = self._image("clip.mp4")
|
||||
self.handler.on_created(FakeEvent(path))
|
||||
self.assertTrue(wait_for(lambda: self.reported != []))
|
||||
self.assertEqual(self.reported, [os.path.realpath(path)])
|
||||
|
||||
def test_a_failing_listener_is_logged_and_the_worker_carries_on(self):
|
||||
def broken(_path):
|
||||
raise RuntimeError("listing gone")
|
||||
|
||||
self.handler.output_listener = broken
|
||||
first = self._image("d.png")
|
||||
with self.assertLogs("prompt_manager.image_monitor", "WARNING") as logs:
|
||||
self.handler._process_pending(first) # no exception
|
||||
self.assertIn("Images listing", "\n".join(logs.output))
|
||||
self.handler.output_listener = self.reported.append
|
||||
second = self._image("e.png")
|
||||
self.handler.schedule(second)
|
||||
self.assertTrue(wait_for(lambda: self.reported != []))
|
||||
self.assertEqual(self.reported, [os.path.realpath(second)])
|
||||
|
||||
def test_no_listener_reports_nothing(self):
|
||||
self.handler.output_listener = None
|
||||
self.handler._process_pending(self._image("f.png")) # no exception
|
||||
self.assertEqual(self.reported, [])
|
||||
|
||||
def test_the_monitor_wires_running_and_later_handlers(self):
|
||||
monitor = im_mod.ImageMonitor(MagicMock(), MagicMock())
|
||||
listener = self.reported.append
|
||||
monitor.set_output_listener(listener)
|
||||
with (
|
||||
patch.object(im_mod, "Observer", FakeObserver),
|
||||
patch.object(im_mod.ImageMonitor, "_gallery_config", return_value=None),
|
||||
):
|
||||
monitor.start_monitoring([self.tmp.name])
|
||||
try:
|
||||
self.assertIs(monitor.handler.output_listener, listener)
|
||||
monitor.set_output_listener(None)
|
||||
self.assertIsNone(monitor.output_listener)
|
||||
self.assertIsNone(monitor.handler.output_listener)
|
||||
finally:
|
||||
monitor.stop_monitoring()
|
||||
|
||||
|
||||
class TestModuleIsolation(unittest.TestCase):
|
||||
"""This module must not leave mocks behind for later test modules."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user