From 1dfac0fabcbef94b46fb57443b470904628eebed Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 3 Aug 2026 16:18:35 +0300 Subject: [PATCH 1/3] gh-155109: Run tests exhausting the C stack with a limited C stack Add the @support.run_with_limited_c_stack() decorator which runs the test in a thread with a small C stack, so that the outcome does not depend on RLIMIT_STACK. Use it in tests which recurse to a fixed depth -- @support.skip_if_huge_c_stack() failed to skip them with a 16 MiB stack. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/test/list_tests.py | 4 +- Lib/test/mapping_tests.py | 2 +- Lib/test/support/__init__.py | 94 +++++++++++++++++++++++----- Lib/test/test_ast/test_ast.py | 3 +- Lib/test/test_compile.py | 3 +- Lib/test/test_dict.py | 2 +- Lib/test/test_dictviews.py | 4 +- Lib/test/test_exception_group.py | 6 +- Lib/test/test_json/test_recursion.py | 2 +- Lib/test/test_pyexpat.py | 2 +- Lib/test/test_typing.py | 6 +- Lib/test/test_xml_etree.py | 2 +- 12 files changed, 98 insertions(+), 32 deletions(-) diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index ec2aa59f2cb8728..ad9a9ea83035fb4 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, skip_if_huge_c_stack +from test.support import ALWAYS_EQ, NEVER_EQ, run_with_limited_c_stack from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow @@ -60,7 +60,7 @@ def test_repr(self): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") - @skip_if_huge_c_stack(200_000) + @run_with_limited_c_stack(200_000) @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_repr_deep(self): diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index ae2fb3f5f448e25..e3f348d272a03c6 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -629,7 +629,7 @@ def __repr__(self): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) - @support.skip_if_huge_c_stack() + @support.run_with_limited_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() @support.skip_if_sanitizer("requires deep stack", ub=True) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f4c4b4c1acfc182..163c05dacacf074 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -46,6 +46,7 @@ "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer", "requires_limited_api", "requires_specialization", "thread_unsafe", "skip_if_unlimited_stack_size", "skip_if_huge_c_stack", + "run_with_limited_c_stack", # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", @@ -2839,6 +2840,26 @@ def exceeds_recursion_limit(): return 150_000 +def _has_huge_c_stack(depth): + """Check that *depth* recursive calls cannot exhaust the C stack.""" + try: + from _testinternalcapi import get_c_recursion_remaining + except ImportError: + # Fall back to checking for an unlimited stack size. + if is_emscripten or is_wasi or os.name == "nt": + return False + import resource + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + return soft == hard and soft in (-1, 0xFFFF_FFFF_FFFF_FFFF) + else: + remaining = get_c_recursion_remaining() + # A negative value means integer overflow in the estimate + # (e.g. with an unlimited RLIMIT_STACK). The estimate is based on + # the size of the interpreter loop frame, so it is only a lower + # bound for recursion with smaller C frames. + return remaining >= depth or remaining < 0 + + def skip_if_huge_c_stack(depth=150_000): """Skip decorator for tests which cannot overflow the C stack. @@ -2846,23 +2867,66 @@ def skip_if_huge_c_stack(depth=150_000): trigger the recursion protection if the C stack is too large (e.g. with a large or unlimited RLIMIT_STACK), and either fail, or run for a very long time, or crash, or consume all memory. + + Prefer run_with_limited_c_stack() for tests recursing to a fixed depth. """ - try: - from _testinternalcapi import get_c_recursion_remaining - except ImportError: - # Fall back to checking for an unlimited stack size. - huge = False - if not (is_emscripten or is_wasi) and os.name != "nt": - import resource - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - huge = soft == hard and soft in (-1, 0xFFFF_FFFF_FFFF_FFFF) - else: - remaining = get_c_recursion_remaining() - # A negative value means integer overflow in the estimate - # (e.g. with an unlimited RLIMIT_STACK). - huge = remaining >= depth or remaining < 0 return unittest.skipIf( - huge, f"the C stack is large enough for {depth} recursive calls") + _has_huge_c_stack(depth), + f"the C stack is large enough for {depth} recursive calls") + + +# Large enough to run a test, but small enough to be exhausted by tens +# of thousands of recursive calls. +C_STACK_SIZE = 2 * 1024 * 1024 + + +def run_with_limited_c_stack(depth=150_000, size=C_STACK_SIZE): + """Decorator for tests exhausting the C stack with *depth* recursive calls. + + Run the test in a separate thread with the C stack of *size* bytes, so + that the outcome does not depend on the C stack size of the main thread + (which can be large or unlimited, see RLIMIT_STACK). + + If a thread with the limited C stack cannot be created, run the test in + the current thread, but skip it if the C stack is too large. + """ + reason = f"the C stack is large enough for {depth} recursive calls" + def decorator(test): + @functools.wraps(test) + def wrapper(*args, **kwargs): + def run_test(): + # The C stack can still be too large if limiting it failed. + if _has_huge_c_stack(depth): + raise unittest.SkipTest(reason) + test(*args, **kwargs) + + try: + import threading + old_size = threading.stack_size(size) + except (ImportError, ValueError, RuntimeError): + # Setting the thread stack size is not supported. + return run_test() + + exceptions = [] + def run(): + try: + run_test() + except BaseException as exc: + exceptions.append(exc) + + thread = threading.Thread(target=run) + try: + thread.start() + except RuntimeError: + # Threads are not supported. + return run_test() + finally: + threading.stack_size(old_size) + thread.join() + if exceptions: + raise exceptions[0] + return wrapper + return decorator # Windows doesn't have os.uname() but it doesn't support s390x. diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 87d63fcd8529336..28ac6c6fcbccc1f 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1025,7 +1025,8 @@ def next(self): enum._test_simple_enum(_Precedence, _ast_unparse._Precedence) @support.cpython_only - @support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000) + @support.run_with_limited_c_stack( + 100_000 if sys.platform == "android" else 500_000) @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_ast_recursion_limit(self): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 2c7b1181817cf55..df473d59fff3d8e 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -724,7 +724,8 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") - @support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000) + @support.run_with_limited_c_stack( + 100_000 if sys.platform == "android" else 500_000) @support.skip_emscripten_stack_overflow() def test_compiler_recursion_limit(self): # Compiler frames are small diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 1e665c86303078c..673987733fc8c4f 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -678,7 +678,7 @@ def __repr__(self): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) - @support.skip_if_huge_c_stack() + @support.run_with_limited_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() def test_repr_deep(self): diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 9816ae6c033ec74..3af0501765af29e 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -3,7 +3,7 @@ import pickle import unittest from test.support import (skip_emscripten_stack_overflow, - skip_wasi_stack_overflow, skip_if_huge_c_stack, + skip_wasi_stack_overflow, run_with_limited_c_stack, exceeds_recursion_limit) class DictSetTest(unittest.TestCase): @@ -279,7 +279,7 @@ def test_recursive_repr(self): # Again. self.assertIsInstance(r, str) - @skip_if_huge_c_stack() + @run_with_limited_c_stack() @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_deeply_nested_repr(self): diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 325b1c91fa5aeef..f79bfa4ae2d3220 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -2,7 +2,7 @@ import types import unittest from test.support import (skip_emscripten_stack_overflow, - skip_wasi_stack_overflow, skip_if_huge_c_stack, + skip_wasi_stack_overflow, run_with_limited_c_stack, exceeds_recursion_limit) class TestExceptionGroupTypeHierarchy(unittest.TestCase): @@ -549,7 +549,7 @@ def make_deep_eg(self): e = ExceptionGroup('eg', [e]) return e - @skip_if_huge_c_stack() + @run_with_limited_c_stack() @skip_emscripten_stack_overflow() @skip_wasi_stack_overflow() def test_deep_split(self): @@ -557,7 +557,7 @@ def test_deep_split(self): with self.assertRaises(RecursionError): e.split(TypeError) - @skip_if_huge_c_stack() + @run_with_limited_c_stack() @skip_emscripten_stack_overflow() @skip_wasi_stack_overflow() def test_deep_subgroup(self): diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index cbae9fbb4d624ba..17c7afe6417d887 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -69,7 +69,7 @@ def default(self, o): @support.skip_if_pgo_task # fails during PGO training w/ some stack sizes - @support.skip_if_huge_c_stack(500_000) + @support.run_with_limited_c_stack(500_000) @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_highly_nested_objects_decoding(self): diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 54dfa95ce8bff17..9869f0e88448cf1 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -904,7 +904,7 @@ def test_trigger_leak(self): parser.ElementDeclHandler = lambda _1, _2: None self.assertRaises(TypeError, parser.Parse, data, True) - @support.skip_if_huge_c_stack(800_000) + @support.run_with_limited_c_stack(800_000) @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_deeply_nested_content_model(self): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 106ffdede6fd4ef..51357127831f563 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -52,8 +52,8 @@ from test.support import ( captured_stderr, cpython_only, requires_docstrings, import_helper, run_code, subTests, EqualToForwardRef, - exceeds_recursion_limit, skip_if_huge_c_stack, skip_wasi_stack_overflow, - skip_emscripten_stack_overflow, + exceeds_recursion_limit, run_with_limited_c_stack, + skip_wasi_stack_overflow, skip_emscripten_stack_overflow, ) from test.typinganndata import ( ann_module695, mod_generics_cache, _typed_dict_helper, @@ -5097,7 +5097,7 @@ class MM2(collections.abc.MutableMapping, MutableMapping[str, str]): self.assertEqual(MM2.__bases__, (collections.abc.MutableMapping, Generic)) @cpython_only - @skip_if_huge_c_stack() + @run_with_limited_c_stack() @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_parameters_deep_recursion(self): diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 38bc681a267b951..0c944516ae115f1 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -3246,7 +3246,7 @@ def __deepcopy__(self, memo): self.assertEqual([c.tag for c in children[3:]], [a.tag, b.tag, a.tag, b.tag]) - @support.skip_if_huge_c_stack(500_000) + @support.run_with_limited_c_stack(500_000) @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_deeply_nested_deepcopy(self): From 74b489a659831c789199e86e4e5230233b46dc85 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 3 Aug 2026 20:10:52 +0300 Subject: [PATCH 2/3] Use a 8 MiB C stack in run_with_limited_c_stack() On platforms where the interpreter cannot query the thread stack size it assumes Py_C_STACK_SIZE (4 MiB), so a smaller thread stack crashes instead of raising RecursionError. Verified on illumos and OpenBSD: a 2 MiB stack dumps core, 8 MiB raises RecursionError. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/test/support/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 163c05dacacf074..7898ef5e15b2c40 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2875,9 +2875,10 @@ def skip_if_huge_c_stack(depth=150_000): f"the C stack is large enough for {depth} recursive calls") -# Large enough to run a test, but small enough to be exhausted by tens -# of thousands of recursive calls. -C_STACK_SIZE = 2 * 1024 * 1024 +# Small enough to be exhausted by tens of thousands of recursive calls, +# but not smaller than Py_C_STACK_SIZE (4 MiB) which the interpreter +# assumes if it cannot query the thread stack size. +C_STACK_SIZE = 8 * 1024 * 1024 def run_with_limited_c_stack(depth=150_000, size=C_STACK_SIZE): From 4856144bd086d5184f75e6a2d6837c7c14726550 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 3 Aug 2026 21:23:51 +0300 Subject: [PATCH 3/3] Add a NEWS entry Co-Authored-By: Claude Opus 5 (1M context) --- .../Tests/2026-08-03-20-15-00.gh-issue-155109.limstack.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2026-08-03-20-15-00.gh-issue-155109.limstack.rst diff --git a/Misc/NEWS.d/next/Tests/2026-08-03-20-15-00.gh-issue-155109.limstack.rst b/Misc/NEWS.d/next/Tests/2026-08-03-20-15-00.gh-issue-155109.limstack.rst new file mode 100644 index 000000000000000..cc6f057a364180c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-08-03-20-15-00.gh-issue-155109.limstack.rst @@ -0,0 +1,3 @@ +Add ``test.support.run_with_limited_c_stack()`` and use it in tests that +exhaust the C stack with a fixed number of recursive calls, so that their +outcome no longer depends on the C stack size.