Vito
a985af506b
* test(browser): seed resets on a scratch copy and publish them in one commit (#384) The concept and plain resets made hundreds of committed transactions on the live database; on a slow CI disk they queued behind WAL checkpoint syncs and took 30+ s. Each reset now copies the live database into a scratch database (backup API, no syncs), runs the unchanged seeder against it through an API proxy, and replaces the rows of the tables the seed changed in one live transaction. A commit by another live connection meanwhile fails the reset. tests/browser/reset_equivalence.py proves every reset kind leaves the same rows, files and response as the old path (PM_TEST_SEED_ON_LIVE=1). * test(browser): empty media folders by a rename, delete them in the background (#384) With the database commits gone, throttled resets still stalled in rmtree: unlinking the last fixture's files waits for their pending writeback. Each reseed now moves the old output and input folders into <root>/reset-trash and makes empty ones; a background thread deletes the trash, and the server's cleanup stops it and removes the rest before the temp root goes. * fix(test): recheck the live data version under the publish's write lock (#384) A writer whose transaction was open while the seed ran committed before BEGIN IMMEDIATE returned and its rows were overwritten. The publish now rechecks data_version under the lock, and restores foreign_keys and detaches the scratch database on every error path. * test(browser): keep the reset scratch database in /dev/shm when there is one (#384) Throttled runs still waited on the runner's disk while creating and rewriting the scratch database, whose pages were under writeback. * test(browser): keep the scratch on disk when /dev/shm is short, delete in place after cleanup (#384) The largest scratch copy is about 8 MB; Docker's default /dev/shm is 64 MB and shared with the browsers. A reset that empties a folder after the server's cleanup closed its trash deletes in place instead of starting a thread nobody stops. * fix(test): drop the gallery caches only after the reset's rows are published (#384) The seeders drop the Images listing at their end. Run on the scratch copy, that drop came before the publish, so a background walk in between could cache per-file values built from the live database's old rows: CI's inspector tag chip found 0 of 0 images. The proxy now records the drop and the seeder makes it after the publish, as the old reset did. * fix(test): remove /dev/shm reset scratch left by killed browser servers (#384) A server killed outright never runs its cleanup, and tmpfs keeps the scratch folder in memory until reboot: the Python suite's server tests left one per killed server. Scratch folders now carry the owner's pid, and each new one first removes those whose process is gone (POSIX only).
235 lines
8.7 KiB
Python
235 lines
8.7 KiB
Python
"""A slow page-workflow reset reports where the server waited.
|
|
|
|
CI run 39904288262 had one ``/__test__/reset`` wait 16 s with nothing in the
|
|
server's stderr. tests/browser/slow_reset_watch.py reports any reset past
|
|
SLOW_RESET_SECONDS with every thread's stack, taken while it still waits.
|
|
These tests hold a real SQLite write lock behind a real test server's reset.
|
|
"""
|
|
|
|
import glob
|
|
import io
|
|
import os
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from tests.browser import slow_reset_watch
|
|
from tests.browser.slow_reset_watch import REPORT_PREFIX, ResetWatch, is_reset_path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
SERVER_TIMEOUT_S = 60
|
|
HOLD_SECONDS = slow_reset_watch.SLOW_RESET_SECONDS + 4
|
|
# The reset may take a little longer than the hold, never the 30 s busy timeout.
|
|
RESET_MARGIN_SECONDS = 10
|
|
# While the reset waits, the server still answers other requests this fast.
|
|
# Reading the waiting connection from the report's thread froze it (the GIL is
|
|
# held while SQLite's connection mutex is awaited) until the lock was released.
|
|
RESPONSIVE_SECONDS = 2.0
|
|
PROBE_INTERVAL_SECONDS = 0.1
|
|
|
|
|
|
def blocked_here(gate: threading.Event) -> None:
|
|
gate.wait()
|
|
|
|
|
|
class TestResetWatch(unittest.TestCase):
|
|
def test_reset_paths_are_the_test_reset_routes(self):
|
|
self.assertTrue(is_reset_path("/__test__/reset"))
|
|
self.assertTrue(is_reset_path("/__test__/tagging/reset"))
|
|
self.assertFalse(is_reset_path("/prompt_manager/reset"))
|
|
self.assertFalse(is_reset_path("/__test__/reset/extra"))
|
|
|
|
def test_a_stalled_reset_reports_state_and_stacks_while_it_waits(self):
|
|
stream = io.StringIO()
|
|
reported = threading.Event()
|
|
|
|
class Stream(io.StringIO):
|
|
def flush(self):
|
|
reported.set()
|
|
|
|
stream = Stream()
|
|
gate = threading.Event()
|
|
worker = threading.Thread(target=blocked_here, args=(gate,), name="held-up")
|
|
worker.start()
|
|
self.addCleanup(worker.join)
|
|
self.addCleanup(gate.set)
|
|
watch = ResetWatch(
|
|
"POST /__test__/reset",
|
|
lambda: ["database: 1 connections"],
|
|
threshold=0.05,
|
|
stream=stream,
|
|
).start()
|
|
self.assertTrue(reported.wait(SERVER_TIMEOUT_S))
|
|
report = stream.getvalue().splitlines()
|
|
self.assertTrue(all(line.startswith(REPORT_PREFIX) for line in report))
|
|
text = "\n".join(report)
|
|
self.assertIn("POST /__test__/reset still running after", text)
|
|
self.assertIn("database: 1 connections", text)
|
|
self.assertIn(f"thread 0x{worker.ident:016x} is held-up", text)
|
|
self.assertIn("in blocked_here", text)
|
|
|
|
watch.record_call("seed", 0.25, 1.5)
|
|
watch.finish("answered 200")
|
|
self.assertIn("POST /__test__/reset answered 200 after", stream.getvalue())
|
|
self.assertIn(
|
|
"executor call seed: queued 0.250s, ran 1.500s", stream.getvalue()
|
|
)
|
|
|
|
def test_a_fast_reset_writes_nothing_and_stops_its_timer(self):
|
|
stream = io.StringIO()
|
|
watch = ResetWatch("reset", lambda: [], threshold=60, stream=stream).start()
|
|
watch.finish("answered 200")
|
|
self.assertTrue(watch._timer.finished.is_set())
|
|
self.assertEqual(stream.getvalue(), "")
|
|
|
|
def test_a_failing_state_probe_still_reports_the_stacks(self):
|
|
def broken():
|
|
raise RuntimeError("no state")
|
|
|
|
reported = threading.Event()
|
|
|
|
class Stream(io.StringIO):
|
|
def flush(self):
|
|
reported.set()
|
|
|
|
stream = Stream()
|
|
ResetWatch("reset", broken, threshold=0.01, stream=stream).start()
|
|
self.assertTrue(reported.wait(SERVER_TIMEOUT_S))
|
|
self.assertIn("state unavailable: RuntimeError('no state')", stream.getvalue())
|
|
self.assertIn("most recent call first", stream.getvalue())
|
|
|
|
|
|
class TestSlowResetOnTheServer(unittest.TestCase):
|
|
"""A reset behind another process's write lock, on tests/browser/server.py."""
|
|
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
env = {**os.environ, "TMPDIR": self.tmp.name}
|
|
env.update(TEMP=self.tmp.name, TMP=self.tmp.name)
|
|
self.stderr = tempfile.TemporaryFile()
|
|
self.addCleanup(self.stderr.close)
|
|
process = subprocess.Popen(
|
|
[sys.executable, "-m", "tests.browser.server", "--port", "0"],
|
|
cwd=REPO_ROOT,
|
|
env=env,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=self.stderr,
|
|
)
|
|
|
|
def stop():
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait(timeout=SERVER_TIMEOUT_S)
|
|
for stream in (process.stdin, process.stdout):
|
|
stream.close()
|
|
|
|
self.addCleanup(stop)
|
|
line = process.stdout.readline().decode()
|
|
self.assertTrue(line.startswith("PM_TEST_SERVER_PORT="), line)
|
|
self.base = f"http://127.0.0.1:{int(line.split('=', 1)[1])}"
|
|
deadline = time.monotonic() + SERVER_TIMEOUT_S
|
|
while True:
|
|
try:
|
|
with urllib.request.urlopen( # nosec B310
|
|
self.base + "/prompt_manager/test", timeout=1
|
|
):
|
|
break
|
|
except OSError:
|
|
self.assertIsNone(process.poll(), "the server exited early")
|
|
self.assertLess(time.monotonic(), deadline, "never ready")
|
|
time.sleep(0.1)
|
|
[self.db_path] = glob.glob(
|
|
os.path.join(self.tmp.name, "pm-browser-*", "browser.db")
|
|
)
|
|
|
|
def reset(self) -> tuple[int, float]:
|
|
request = urllib.request.Request(
|
|
self.base + "/__test__/reset",
|
|
data=b"{}",
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST",
|
|
)
|
|
start = time.monotonic()
|
|
with urllib.request.urlopen( # nosec B310
|
|
request, timeout=SERVER_TIMEOUT_S
|
|
) as response:
|
|
return response.status, time.monotonic() - start
|
|
|
|
def server_stderr(self) -> str:
|
|
self.stderr.seek(0)
|
|
return self.stderr.read().decode("utf-8", "replace")
|
|
|
|
def test_a_reset_held_behind_a_write_lock_names_the_wait_and_ends_with_it(self):
|
|
self.assertEqual(self.reset()[0], 200, self.server_stderr())
|
|
self.assertNotIn(REPORT_PREFIX, self.server_stderr())
|
|
|
|
held = threading.Event()
|
|
|
|
def hold_write_lock():
|
|
holder = sqlite3.connect(self.db_path, timeout=SERVER_TIMEOUT_S)
|
|
try:
|
|
holder.execute("BEGIN IMMEDIATE")
|
|
held.set()
|
|
time.sleep(HOLD_SECONDS)
|
|
holder.rollback()
|
|
finally:
|
|
holder.close()
|
|
|
|
lock = threading.Thread(target=hold_write_lock)
|
|
lock.start()
|
|
self.addCleanup(lock.join)
|
|
self.assertTrue(held.wait(SERVER_TIMEOUT_S))
|
|
outcome = []
|
|
resetting = threading.Thread(target=lambda: outcome.append(self.reset()))
|
|
resetting.start()
|
|
self.addCleanup(resetting.join)
|
|
slowest = 0.0
|
|
while resetting.is_alive():
|
|
start = time.monotonic()
|
|
with urllib.request.urlopen( # nosec B310
|
|
self.base + "/prompt_manager/test", timeout=SERVER_TIMEOUT_S
|
|
):
|
|
slowest = max(slowest, time.monotonic() - start)
|
|
time.sleep(PROBE_INTERVAL_SECONDS)
|
|
resetting.join()
|
|
lock.join()
|
|
self.assertEqual(len(outcome), 1, self.server_stderr())
|
|
[(status, elapsed)] = outcome
|
|
|
|
self.assertLess(slowest, RESPONSIVE_SECONDS)
|
|
self.assertEqual(status, 200)
|
|
self.assertGreaterEqual(elapsed, HOLD_SECONDS - 0.5)
|
|
self.assertLess(elapsed, HOLD_SECONDS + RESET_MARGIN_SECONDS)
|
|
report = [
|
|
line[len(REPORT_PREFIX) :]
|
|
for line in self.server_stderr().splitlines()
|
|
if line.startswith(REPORT_PREFIX)
|
|
]
|
|
text = "\n".join(report)
|
|
self.assertIn("POST /__test__/reset still running after", text)
|
|
self.assertIn("default executor:", text)
|
|
if sys.platform.startswith("linux"):
|
|
# SQLite's busy handler sleeps between its retries.
|
|
self.assertRegex(
|
|
text, r"kernel: asyncio_\d+ \(0x[0-9a-f]{16}\) state \w, waiting in "
|
|
)
|
|
# The plain reset seeds a scratch copy (#384), then waits to publish it.
|
|
self.assertRegex(text, r"line \d+ in _publish\n")
|
|
self.assertIn("POST /__test__/reset answered 200 after", text)
|
|
self.assertRegex(
|
|
text,
|
|
r"executor call ScratchSeeder.run: queued \d+\.\d+s, ran \d+\.\d+s",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|