From bb744550f4ac28178ad1ebb3285711c3ceb4dd5d Mon Sep 17 00:00:00 2001 From: Vito Date: Fri, 30 Oct 2026 01:43:33 +0000 Subject: [PATCH] fix(tests): pin glibc's mmap threshold in the browser test server (#516) (#519) Each tests/browser/server.py grew from about 13 MB to 0.6-1 GB RSS over a page-workflow shard while its Python heap stayed near 12 MB. glibc raises its mmap threshold (up to 32 MiB) after a large free, so the seeder's image-sized buffers came from per-thread malloc arenas, which fragmented and kept their pages. The server now fixes the threshold at glibc's 128 KiB start value (mallopt M_MMAP_THRESHOLD) before it starts, so large blocks go back to the OS on free. No memory limit is set; off glibc it does nothing. Closes #516 --- tests/browser/malloc_tuning.py | 31 ++++++++++ tests/browser/server.py | 3 + tests/test_browser_malloc_tuning.py | 92 +++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 tests/browser/malloc_tuning.py create mode 100644 tests/test_browser_malloc_tuning.py diff --git a/tests/browser/malloc_tuning.py b/tests/browser/malloc_tuning.py new file mode 100644 index 00000000..812efe8d --- /dev/null +++ b/tests/browser/malloc_tuning.py @@ -0,0 +1,31 @@ +"""Keep the browser test server's large frees going back to the OS (#516). + +glibc serves an allocation of at least the mmap threshold (128 KiB at start) with its +own mmap and unmaps it on free. Freeing such a block also raises the threshold, up to +32 MiB, so the server's later image-sized buffers (PIL pixel data, PNG encoding, +whole-file reads in the executor threads) come from per-thread malloc arenas instead. +Those arenas fragment and keep their pages: a server grew from 13 MB to 0.6-1 GB RSS +over one page-workflow shard while its Python heap stayed near 12 MB. Setting +M_MMAP_THRESHOLD fixes the threshold at its start value and turns the raising off. +""" + +import ctypes +import sys + +M_MMAP_THRESHOLD = -3 +MMAP_THRESHOLD_BYTES = 128 * 1024 + + +def pin_mmap_threshold(libc=None) -> bool: + """Fix glibc's mmap threshold; False where there is no glibc ``mallopt``.""" + if not sys.platform.startswith("linux"): + return False + if libc is None: + try: + libc = ctypes.CDLL(None) + except OSError: + return False + mallopt = getattr(libc, "mallopt", None) + if mallopt is None: + return False + return bool(mallopt(M_MMAP_THRESHOLD, MMAP_THRESHOLD_BYTES)) diff --git a/tests/browser/server.py b/tests/browser/server.py index bd05edc9..65616c67 100644 --- a/tests/browser/server.py +++ b/tests/browser/server.py @@ -20,6 +20,7 @@ from PIL import Image from PIL.PngImagePlugin import PngInfo from tests.browser.folder_trash import close_trash, replace_folder +from tests.browser.malloc_tuning import pin_mmap_threshold from tests.browser.request_hold import register_request_holds from tests.browser import slow_reset_watch from tests.browser.fast_reset import SEED_ON_LIVE_ENV, ScratchSeeder @@ -755,6 +756,8 @@ def main(argv: list[str] | None = None) -> None: help="shut down when stdin closes (the launcher's parent-death guard)", ) args = parser.parse_args(argv) + # Before any thread allocates (#516): large frees then return to the OS. + pin_mmap_threshold() listener = bind_listener(args.port) # One machine-readable line tells the launcher which port this process holds. print(f"{PORT_ANNOUNCEMENT}{listener.getsockname()[1]}", flush=True) diff --git a/tests/test_browser_malloc_tuning.py b/tests/test_browser_malloc_tuning.py new file mode 100644 index 00000000..d578fc9c --- /dev/null +++ b/tests/test_browser_malloc_tuning.py @@ -0,0 +1,92 @@ +"""#516: the browser test server pins glibc's mmap threshold. + +Large frees then go back to the OS instead of fragmenting per-thread arenas. +""" + +import ctypes +import sys +import unittest +from unittest import mock + +from tests.browser import malloc_tuning + + +class FakeLibc: + def __init__(self, result=1): + self.calls = [] + self.result = result + + def mallopt(self, option, value): + self.calls.append((option, value)) + return self.result + + +class PinMmapThresholdTests(unittest.TestCase): + def test_sets_a_fixed_threshold_through_mallopt(self): + libc = FakeLibc() + with mock.patch.object(sys, "platform", "linux"): + self.assertTrue(malloc_tuning.pin_mmap_threshold(libc)) + # M_MMAP_THRESHOLD is -3; setting it also turns off glibc's dynamic + # raising of the threshold. + self.assertEqual(libc.calls, [(-3, 128 * 1024)]) + + def test_reports_a_refused_setting(self): + with mock.patch.object(sys, "platform", "linux"): + self.assertFalse(malloc_tuning.pin_mmap_threshold(FakeLibc(result=0))) + + def test_skips_a_libc_without_mallopt(self): + with mock.patch.object(sys, "platform", "linux"): + self.assertFalse(malloc_tuning.pin_mmap_threshold(object())) + + def test_skips_other_platforms_without_loading_a_libc(self): + for name in ("win32", "darwin"): + with ( + self.subTest(name), + mock.patch.object(sys, "platform", name), + mock.patch.object( + ctypes, "CDLL", side_effect=AssertionError("no libc load") + ), + ): + self.assertFalse(malloc_tuning.pin_mmap_threshold()) + + def test_skips_when_the_process_libc_cannot_be_loaded(self): + with ( + mock.patch.object(sys, "platform", "linux"), + mock.patch.object(ctypes, "CDLL", side_effect=OSError("no libc")), + ): + self.assertFalse(malloc_tuning.pin_mmap_threshold()) + + @unittest.skipUnless( + sys.platform.startswith("linux") + and hasattr(ctypes.CDLL(None), "gnu_get_libc_version"), + "glibc only", + ) + def test_glibc_accepts_the_setting(self): + # This pins the threshold for the rest of this test process too; the + # test server does the same at start, so nothing here depends on it. + self.assertTrue(malloc_tuning.pin_mmap_threshold()) + + +class ServerMainTests(unittest.TestCase): + def test_main_pins_the_threshold_before_it_starts_serving(self): + from tests.browser import server + + order = [] + + def bind(port): + order.append("bind") + raise RuntimeError("stop before serving") + + with ( + mock.patch.object( + server, "pin_mmap_threshold", side_effect=lambda: order.append("pin") + ), + mock.patch.object(server, "bind_listener", side_effect=bind), + ): + with self.assertRaises(RuntimeError): + server.main(["--port", "0"]) + self.assertEqual(order, ["pin", "bind"]) + + +if __name__ == "__main__": + unittest.main()