ComfyAssets-ComfyUI_PromptManager/tests/test_browser_slow_reset_watch.py

"""A slow page-workflow reset reports where the server waited.

CI run 39904288262 had one ``/__test__/reset`` wait 16 s with nothing in the
server's stderr. tests/browser/slow_reset_watch.py reports any reset past
SLOW_RESET_SECONDS with every thread's stack, taken while it still waits.
These tests hold a real SQLite write lock behind a real test server's reset.
"""

import glob
import io
import os
import sqlite3
import subprocess
import sys
import tempfile
import threading
import time
import unittest
import urllib.request
from pathlib import Path

from tests.browser import slow_reset_watch
from tests.browser.slow_reset_watch import REPORT_PREFIX, ResetWatch, is_reset_path

REPO_ROOT = Path(__file__).resolve().parent.parent
SERVER_TIMEOUT_S = 60
HOLD_SECONDS = slow_reset_watch.SLOW_RESET_SECONDS + 4
# The reset may take a little longer than the hold, never the 30 s busy timeout.
RESET_MARGIN_SECONDS = 10
# While the reset waits, the server still answers other requests this fast.
# Reading the waiting connection from the report's thread froze it (the GIL is
# held while SQLite's connection mutex is awaited) until the lock was released.
RESPONSIVE_SECONDS = 2.0
PROBE_INTERVAL_SECONDS = 0.1


def blocked_here(gate: threading.Event) -> None:
    gate.wait()


class TestResetWatch(unittest.TestCase):
    def test_reset_paths_are_the_test_reset_routes(self):
        self.assertTrue(is_reset_path("/__test__/reset"))
        self.assertTrue(is_reset_path("/__test__/tagging/reset"))
        self.assertFalse(is_reset_path("/prompt_manager/reset"))
        self.assertFalse(is_reset_path("/__test__/reset/extra"))

    def test_a_stalled_reset_reports_state_and_stacks_while_it_waits(self):
        reported = threading.Event()

        class Stream(io.StringIO):
            def flush(self):
                reported.set()

        stream = Stream()
        gate = threading.Event()
        worker = threading.Thread(target=blocked_here, args=(gate,), name="held-up")
        worker.start()
        self.addCleanup(worker.join)
        self.addCleanup(gate.set)
        watch = ResetWatch(
            "POST /__test__/reset",
            lambda: ["database: 1 connections"],
            threshold=0.05,
            stream=stream,
        ).start()
        self.assertTrue(reported.wait(SERVER_TIMEOUT_S))
        report = stream.getvalue().splitlines()
        self.assertTrue(all(line.startswith(REPORT_PREFIX) for line in report))
        text = "\n".join(report)
        self.assertIn("POST /__test__/reset still running after", text)
        self.assertIn("database: 1 connections", text)
        self.assertIn(f"thread 0x{worker.ident:016x} is held-up", text)
        self.assertIn("in blocked_here", text)

        watch.record_call("seed", 0.25, 1.5)
        watch.finish("answered 200")
        self.assertIn("POST /__test__/reset answered 200 after", stream.getvalue())
        self.assertIn(
            "executor call seed: queued 0.250s, ran 1.500s", stream.getvalue()
        )

    def test_a_fast_reset_writes_nothing_and_stops_its_timer(self):
        stream = io.StringIO()
        watch = ResetWatch("reset", lambda: [], threshold=60, stream=stream).start()
        watch.finish("answered 200")
        self.assertTrue(watch._timer.finished.is_set())
        self.assertEqual(stream.getvalue(), "")

    def test_a_failing_state_probe_still_reports_the_stacks(self):
        def broken():
            raise RuntimeError("no state")

        reported = threading.Event()

        class Stream(io.StringIO):
            def flush(self):
                reported.set()

        stream = Stream()
        ResetWatch("reset", broken, threshold=0.01, stream=stream).start()
        self.assertTrue(reported.wait(SERVER_TIMEOUT_S))
        self.assertIn("state unavailable: RuntimeError('no state')", stream.getvalue())
        self.assertIn("most recent call first", stream.getvalue())


class TestSlowResetOnTheServer(unittest.TestCase):
    """A reset behind another process's write lock, on tests/browser/server.py."""

    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)
        env = {**os.environ, "TMPDIR": self.tmp.name}
        env.update(TEMP=self.tmp.name, TMP=self.tmp.name)
        self.stderr = tempfile.TemporaryFile()
        self.addCleanup(self.stderr.close)
        process = subprocess.Popen(
            [sys.executable, "-m", "tests.browser.server", "--port", "0"],
            cwd=REPO_ROOT,
            env=env,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=self.stderr,
        )

        def stop():
            if process.poll() is None:
                process.kill()
            process.wait(timeout=SERVER_TIMEOUT_S)
            for stream in (process.stdin, process.stdout):
                stream.close()

        self.addCleanup(stop)
        line = process.stdout.readline().decode()
        self.assertTrue(line.startswith("PM_TEST_SERVER_PORT="), line)
        self.base = f"http://127.0.0.1:{int(line.split('=', 1)[1])}"
        deadline = time.monotonic() + SERVER_TIMEOUT_S
        while True:
            try:
                with urllib.request.urlopen(  # nosec B310
                    self.base + "/prompt_manager/test", timeout=1
                ):
                    break
            except OSError:
                self.assertIsNone(process.poll(), "the server exited early")
                self.assertLess(time.monotonic(), deadline, "never ready")
                time.sleep(0.1)
        [self.db_path] = glob.glob(
            os.path.join(self.tmp.name, "pm-browser-*", "browser.db")
        )

    def reset(self) -> tuple[int, float]:
        request = urllib.request.Request(
            self.base + "/__test__/reset",
            data=b"{}",
            headers={"Content-Type": "application/json"},
            method="POST",
        )
        start = time.monotonic()
        with urllib.request.urlopen(  # nosec B310
            request, timeout=SERVER_TIMEOUT_S
        ) as response:
            return response.status, time.monotonic() - start

    def server_stderr(self) -> str:
        self.stderr.seek(0)
        return self.stderr.read().decode("utf-8", "replace")

    def test_a_reset_held_behind_a_write_lock_names_the_wait_and_ends_with_it(self):
        self.assertEqual(self.reset()[0], 200, self.server_stderr())
        self.assertNotIn(REPORT_PREFIX, self.server_stderr())

        held = threading.Event()

        def hold_write_lock():
            holder = sqlite3.connect(self.db_path, timeout=SERVER_TIMEOUT_S)
            try:
                holder.execute("BEGIN IMMEDIATE")
                held.set()
                time.sleep(HOLD_SECONDS)
                holder.rollback()
            finally:
                holder.close()

        lock = threading.Thread(target=hold_write_lock)
        lock.start()
        self.addCleanup(lock.join)
        self.assertTrue(held.wait(SERVER_TIMEOUT_S))
        outcome = []
        resetting = threading.Thread(target=lambda: outcome.append(self.reset()))
        resetting.start()
        self.addCleanup(resetting.join)
        slowest = 0.0
        while resetting.is_alive():
            start = time.monotonic()
            with urllib.request.urlopen(  # nosec B310
                self.base + "/prompt_manager/test", timeout=SERVER_TIMEOUT_S
            ):
                slowest = max(slowest, time.monotonic() - start)
            time.sleep(PROBE_INTERVAL_SECONDS)
        resetting.join()
        lock.join()
        self.assertEqual(len(outcome), 1, self.server_stderr())
        [(status, elapsed)] = outcome

        self.assertLess(slowest, RESPONSIVE_SECONDS)
        self.assertEqual(status, 200)
        self.assertGreaterEqual(elapsed, HOLD_SECONDS - 0.5)
        self.assertLess(elapsed, HOLD_SECONDS + RESET_MARGIN_SECONDS)
        report = [
            line[len(REPORT_PREFIX) :]
            for line in self.server_stderr().splitlines()
            if line.startswith(REPORT_PREFIX)
        ]
        text = "\n".join(report)
        self.assertIn("POST /__test__/reset still running after", text)
        self.assertIn("default executor:", text)
        if sys.platform.startswith("linux"):
            # SQLite's busy handler sleeps between its retries.
            self.assertRegex(
                text, r"kernel: asyncio_\d+ \(0x[0-9a-f]{16}\) state \w, waiting in "
            )
        # The plain reset seeds a scratch copy (#384), then waits to publish it.
        self.assertRegex(text, r"line \d+ in _publish\n")
        self.assertIn("POST /__test__/reset answered 200 after", text)
        self.assertRegex(
            text,
            r"executor call ScratchSeeder.run: queued \d+\.\d+s, ran \d+\.\d+s",
        )


if __name__ == "__main__":
    unittest.main()