From 47e21752b550f1715ec25547fd326fac8d675a96 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Wed, 12 Aug 2026 22:28:05 -0700 Subject: [PATCH 01/23] gh-85260: Extend the AST Validator to validate all identifiers (GH-21069) --- Lib/test/test_ast/test_ast.py | 28 +++++++++++++++++++ ...0-06-23-13-59-37.gh-issue-85260.o_LJ76.rst | 4 +++ Python/ast.c | 26 +++++++++++++++-- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2020-06-23-13-59-37.gh-issue-85260.o_LJ76.rst diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 28ac6c6fcbccc1f..7d35fc4ef7c3644 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -983,6 +983,34 @@ def test_constant_as_name(self): with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): compile(expr, "", "eval") + def test_constant_in_identifier_fields(self): + # gh-85260: an identifier field holding a constant name used to + # crash the compiler + for statement in [ + "def x(): pass", + "async def x(): pass", + "class x: pass", + "from a import x", + "from a import b as x", + "from a import b, c, d as x", + "import x", + "import a, b, x", + "try: pass\nexcept A as x: pass", + "try: pass\nexcept A as b: pass\nexcept B as x: pass\n", + ]: + for constant in "True", "False", "None": + with self.subTest(statement=statement, constant=constant): + tree = ast.parse(statement) + for node in ast.walk(tree): + for field, value in ast.iter_fields(node): + if value == "x": + setattr(node, field, constant) + with self.assertRaisesRegex( + ValueError, + f"identifier field can't represent " + f"'{constant}' constant"): + compile(tree, "", "exec") + def test_constant_as_unicode_name(self): constants = [ ("True", b"Tru\xe1\xb5\x89"), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2020-06-23-13-59-37.gh-issue-85260.o_LJ76.rst b/Misc/NEWS.d/next/Core_and_Builtins/2020-06-23-13-59-37.gh-issue-85260.o_LJ76.rst new file mode 100644 index 000000000000000..d156efe4ca24144 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2020-06-23-13-59-37.gh-issue-85260.o_LJ76.rst @@ -0,0 +1,4 @@ +:func:`compile` now raises :exc:`ValueError` instead of crashing on a debug +build if an identifier field of an AST node (such as the name of a function, +a class, an imported module or a caught exception) is ``"None"``, ``"True"`` +or ``"False"``. diff --git a/Python/ast.c b/Python/ast.c index 4cfa2ff559a5f7d..f625c59fe5bffc3 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -710,6 +710,23 @@ _validate_nonempty_seq(asdl_seq *seq, const char *what, const char *owner) } #define validate_nonempty_seq(seq, what, owner) _validate_nonempty_seq((asdl_seq*)seq, what, owner) +static int +validate_import_names(asdl_alias_seq *seq, const char *what, const char *owner) +{ + if (!validate_nonempty_seq(seq, what, owner)) { + return 0; + } + Py_ssize_t n = asdl_seq_LEN(seq); + for (Py_ssize_t i = 0; i < n; i++) { + alias_ty alias = asdl_seq_GET(seq, i); + if (!validate_name(alias->name) || + (alias->asname && !validate_name(alias->asname))) { + return 0; + } + } + return 1; +} + static int validate_assignlist(asdl_expr_seq *targets, expr_context_ty ctx) { @@ -735,6 +752,7 @@ validate_stmt(stmt_ty stmt) switch (stmt->kind) { case FunctionDef_kind: ret = validate_body(stmt->v.FunctionDef.body, "FunctionDef") && + validate_name(stmt->v.FunctionDef.name) && validate_type_params(stmt->v.FunctionDef.type_params) && validate_arguments(stmt->v.FunctionDef.args) && validate_exprs(stmt->v.FunctionDef.decorator_list, Load, 0) && @@ -743,6 +761,7 @@ validate_stmt(stmt_ty stmt) break; case ClassDef_kind: ret = validate_body(stmt->v.ClassDef.body, "ClassDef") && + validate_name(stmt->v.ClassDef.name) && validate_type_params(stmt->v.ClassDef.type_params) && validate_exprs(stmt->v.ClassDef.bases, Load, 0) && validate_keywords(stmt->v.ClassDef.keywords) && @@ -873,6 +892,8 @@ validate_stmt(stmt_ty stmt) VALIDATE_POSITIONS(handler); if ((handler->v.ExceptHandler.type && !validate_expr(handler->v.ExceptHandler.type, Load)) || + (handler->v.ExceptHandler.name && + !validate_name(handler->v.ExceptHandler.name)) || !validate_body(handler->v.ExceptHandler.body, "ExceptHandler")) return 0; } @@ -911,14 +932,14 @@ validate_stmt(stmt_ty stmt) (!stmt->v.Assert.msg || validate_expr(stmt->v.Assert.msg, Load)); break; case Import_kind: - ret = validate_nonempty_seq(stmt->v.Import.names, "names", "Import"); + ret = validate_import_names(stmt->v.Import.names, "names", "Import"); break; case ImportFrom_kind: if (stmt->v.ImportFrom.level < 0) { PyErr_SetString(PyExc_ValueError, "Negative ImportFrom level"); return 0; } - ret = validate_nonempty_seq(stmt->v.ImportFrom.names, "names", "ImportFrom"); + ret = validate_import_names(stmt->v.ImportFrom.names, "names", "ImportFrom"); break; case Global_kind: ret = validate_nonempty_seq(stmt->v.Global.names, "names", "Global"); @@ -931,6 +952,7 @@ validate_stmt(stmt_ty stmt) break; case AsyncFunctionDef_kind: ret = validate_body(stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef") && + validate_name(stmt->v.AsyncFunctionDef.name) && validate_type_params(stmt->v.AsyncFunctionDef.type_params) && validate_arguments(stmt->v.AsyncFunctionDef.args) && validate_exprs(stmt->v.AsyncFunctionDef.decorator_list, Load, 0) && From 8420052846e128f88f7097ae5e92ae37b59b4360 Mon Sep 17 00:00:00 2001 From: Petr Prikryl Date: Thu, 13 Aug 2026 07:50:34 +0200 Subject: [PATCH 02/23] gh-90756: Fix the ElementTree XML declaration for utf-8-sig (GH-31043) The BOM already determines the encoding, so the declaration is now omitted by default, and declares utf-8 if requested explicitly. Co-authored-by: Serhiy Storchaka Co-authored-by: Claude Opus 5 (1M context) --- Lib/test/test_xml_etree.py | 4 ++++ Lib/xml/etree/ElementTree.py | 2 ++ .../Library/2026-08-12-22-15-00.gh-issue-90756.Qn7dLm.rst | 2 ++ 3 files changed, 8 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-12-22-15-00.gh-issue-90756.Qn7dLm.rst diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 0c944516ae115f1..2af2d1fd64520b1 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -933,6 +933,7 @@ def test_tostring_xml_declaration_cases(self): (b"\n" b"\xf8", 'ISO-8859-1', None), ('ø', 'unicode', None), + (b"\xef\xbb\xbf\xc3\xb8", 'utf-8-sig', None), # ... xml_declaration = False (b"ø", None, False), @@ -940,6 +941,7 @@ def test_tostring_xml_declaration_cases(self): (b"ø", 'US-ASCII', False), (b"\xf8", 'ISO-8859-1', False), ("ø", 'unicode', False), + (b"\xef\xbb\xbf\xc3\xb8", 'utf-8-sig', False), # ... xml_declaration = True (b"\n" @@ -952,6 +954,8 @@ def test_tostring_xml_declaration_cases(self): b"\xf8", 'ISO-8859-1', True), ("\n" "ø", 'unicode', True), + (b"\xef\xbb\xbf\n" + b"\xc3\xb8", 'utf-8-sig', True), ] for expected_retval, encoding, xml_declaration in TESTCASES: diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 7708d38b7759cb1..951540eb9f45e90 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -732,6 +732,8 @@ def write(self, file_or_filename, if not encoding: encoding = "us-ascii" with _get_writer(file_or_filename, encoding) as (write, declared_encoding): + if declared_encoding.lower() == "utf-8-sig": + declared_encoding = "utf-8" if method == "xml" and (xml_declaration or (xml_declaration is None and encoding.lower() != "unicode" and diff --git a/Misc/NEWS.d/next/Library/2026-08-12-22-15-00.gh-issue-90756.Qn7dLm.rst b/Misc/NEWS.d/next/Library/2026-08-12-22-15-00.gh-issue-90756.Qn7dLm.rst new file mode 100644 index 000000000000000..dc15291d54a4ea4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-12-22-15-00.gh-issue-90756.Qn7dLm.rst @@ -0,0 +1,2 @@ +:meth:`xml.etree.ElementTree.ElementTree.write` now treats the ``utf-8-sig`` +encoding as ``utf-8`` in the XML declaration. From 033fc47f28fbac9312c2d2bc0df27969032296c3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 08:57:05 +0300 Subject: [PATCH 03/23] gh-90982: Use the standard wording in the site.getsitepackages() versionchanged (GH-155654) --- Doc/library/site.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 6f989e2b21b8bf7..fe8718280b2a015 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -460,7 +460,7 @@ Module contents .. versionadded:: 3.2 .. versionchanged:: 3.3 - Add the optional *prefixes* argument. + Added the optional *prefixes* parameter. .. function:: getuserbase() From a2104b7ce0a154a7c24ada33ff8c298f48156bd2 Mon Sep 17 00:00:00 2001 From: Oliver Giles Date: Thu, 13 Aug 2026 18:28:06 +1200 Subject: [PATCH 04/23] gh-88178: Fix exploded IPv6 addresses with a scope ID (GH-25824) IPv6Address.exploded raised AddressValueError, and IPv6Interface.exploded omitted the scope ID. Co-authored-by: Serhiy Storchaka Co-authored-by: Claude Opus 5 (1M context) --- Lib/ipaddress.py | 14 +++++++++----- Lib/test/test_ipaddress.py | 6 ++++++ .../2021-06-03-22-23-38.bpo-44012.BK3HfA.rst | 5 +++++ 3 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-06-03-22-23-38.bpo-44012.BK3HfA.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 9c7a0a46f4f0e1f..6978483544e621a 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1911,14 +1911,18 @@ def _explode_shorthand_ip_string(self): elif isinstance(self, IPv6Interface): ip_str = str(self.ip) else: - ip_str = str(self) + ip_str = self._string_from_ip_int(self._ip) ip_int = self._ip_int_from_string(ip_str) hex_str = '%032x' % ip_int - parts = [hex_str[x:x+4] for x in range(0, 32, 4)] - if isinstance(self, (_BaseNetwork, IPv6Interface)): - return '%s/%d' % (':'.join(parts), self._prefixlen) - return ':'.join(parts) + exploded = ':'.join([hex_str[x:x+4] for x in range(0, 32, 4)]) + if isinstance(self, _BaseNetwork): + return '%s/%d' % (exploded, self._prefixlen) + if self._scope_id: + exploded = '%s%%%s' % (exploded, self._scope_id) + if isinstance(self, IPv6Interface): + return '%s/%d' % (exploded, self._prefixlen) + return exploded def _reverse_pointer(self): """Return the reverse DNS pointer name for the IPv6 address. diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index a74b692784eb594..375d45172a9f4d1 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2718,6 +2718,12 @@ def testExplodeShortHandIpStr(self): addr1.exploded) self.assertEqual('0000:0000:0000:0000:0000:0000:0000:0001/128', ipaddress.IPv6Interface('::1/128').exploded) + self.assertEqual('fe80:0000:0000:0000:0000:0000:0000:0001%1', + ipaddress.IPv6Address('fe80::1%1').exploded) + self.assertEqual('fe80:0000:0000:0000:0000:0000:0000:0001%eth0', + ipaddress.IPv6Address('fe80::1%eth0').exploded) + self.assertEqual('fe80:0000:0000:0000:0000:0000:0000:0001%1/64', + ipaddress.IPv6Interface('fe80::1%1/64').exploded) # issue 77 self.assertEqual('2001:0000:5ef5:79fd:0000:059d:a0e5:0ba1', addr2.exploded) diff --git a/Misc/NEWS.d/next/Library/2021-06-03-22-23-38.bpo-44012.BK3HfA.rst b/Misc/NEWS.d/next/Library/2021-06-03-22-23-38.bpo-44012.BK3HfA.rst new file mode 100644 index 000000000000000..28c21c327504bde --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-06-03-22-23-38.bpo-44012.BK3HfA.rst @@ -0,0 +1,5 @@ +The ``exploded`` attribute of :class:`ipaddress.IPv6Address` and +:class:`ipaddress.IPv6Interface` now supports addresses with a scope ID +(link-local addresses). +Previously the former raised :exc:`~ipaddress.AddressValueError` +and the latter omitted the scope ID. From 025e7d278e29f3df2e95ed9daf9d9835760564c8 Mon Sep 17 00:00:00 2001 From: Sten Wessel Date: Thu, 13 Aug 2026 09:40:49 +0200 Subject: [PATCH 05/23] gh-113329: Catch OSError in doctest finder and document that inspect.getsourcefile/getfile can raise OSError (GH-113335) Fixes #113329, which occurs when running doctests in a class docstring in a REPL environment. Since Python 3.10, an OSError is raised when the class source code cannot be found. --- Doc/library/inspect.rst | 8 +++++--- Lib/doctest.py | 2 +- .../2023-12-20-22-05-40.gh-issue-113329.1CJy3o.rst | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-20-22-05-40.gh-issue-113329.1CJy3o.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 3b34445f019c36c..3ca63d1d026135c 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -747,6 +747,7 @@ Retrieving source code .. function:: getfile(object) Return the name of the (text or binary) file in which an object was defined. + An :exc:`OSError` is raised if the source code cannot be retrieved. This will fail with a :exc:`TypeError` if the object is a built-in module, class, or function. @@ -760,9 +761,10 @@ Retrieving source code .. function:: getsourcefile(object) Return the name of the Python source file in which an object was defined - or ``None`` if no way can be identified to get the source. This - will fail with a :exc:`TypeError` if the object is a built-in module, class, or - function. + or ``None`` if no way can be identified to get the source. An :exc:`OSError` is + raised if the source code cannot be retrieved. + This will fail with a :exc:`TypeError` if the object is a built-in module, + class, or function. .. function:: getsourcelines(object) diff --git a/Lib/doctest.py b/Lib/doctest.py index 8a55fe3ddd26154..cf31eae5365b6eb 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -922,7 +922,7 @@ def find(self, obj, name=None, module=None, globs=None, extraglobs=None): # given object's docstring. try: file = inspect.getsourcefile(obj) - except TypeError: + except (TypeError, OSError): source_lines = None else: if not file: diff --git a/Misc/NEWS.d/next/Library/2023-12-20-22-05-40.gh-issue-113329.1CJy3o.rst b/Misc/NEWS.d/next/Library/2023-12-20-22-05-40.gh-issue-113329.1CJy3o.rst new file mode 100644 index 000000000000000..d5abc0ec28dc9b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-20-22-05-40.gh-issue-113329.1CJy3o.rst @@ -0,0 +1 @@ +Fix :exc:`OSError` being raised when trying to run doctests on a class objects in the REPL. Patch by Sten Wessel. From 296f016e93002e91fdbca5732f9eb4a80bc1af39 Mon Sep 17 00:00:00 2001 From: Aniket <148300120+Aniketsy@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:44:51 +0530 Subject: [PATCH 06/23] gh-109817: Add --single-process-per-case option to libregrtest (GH-151689) --- Lib/test/libregrtest/cmdline.py | 17 ++ Lib/test/libregrtest/findtests.py | 56 ++++- Lib/test/libregrtest/main.py | 26 +- Lib/test/libregrtest/run_workers.py | 74 +++--- Lib/test/libregrtest/runtests.py | 16 ++ Lib/test/test_regrtest.py | 225 +++++++++++++++++- ...-08-12-20-52-16.gh-issue-109817.GZHUNg.rst | 4 + 7 files changed, 379 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-08-12-20-52-16.gh-issue-109817.GZHUNg.rst diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 64c035307e66542..720b073e460127f 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -194,6 +194,7 @@ def __init__(self, **kwargs) -> None: self._add_python_opts = True self.xmlpath = None self.single_process = False + self.single_process_per_case = False super().__init__(**kwargs) @@ -368,6 +369,11 @@ def _create_parser(): group.add_argument('--list-cases', action='store_true', help='only write the name of test cases that will be run, ' 'don\'t execute them') + group.add_argument('--single-process-per-case', action='store_true', + help='run each test case in its own process. ' + '(slow; for debugging order dependencies ' + "and environment leaks). Test cases from " + 'the same module run sequentially.') group.add_argument('-P', '--pgo', dest='pgo', action='store_true', help='enable Profile Guided Optimization (PGO) training') group.add_argument('--pgo-extended', action='store_true', @@ -492,6 +498,17 @@ def _parse_args(args, **kwargs): if ns.single_process: ns.use_mp = None + if ns.single_process_per_case: + if ns.rerun: + parser.error("--single-process-per-case and --rerun " + "options don't go together") + if ns.pgo: + parser.error("--single-process-per-case and --pgo " + "options don't go together") + if ns.single_process: + parser.error("--single-process-per-case and --single-process " + "options don't go together") + # When both --slow-ci and --fast-ci options are present, # --slow-ci has the priority if ns.slow_ci: diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index 6c0e50846a466bb..e7692c5156812e4 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -93,20 +93,58 @@ def list_cases(tests: TestTuple, *, match_tests: TestFilter | None = None, test_dir: StrPath | None = None) -> None: support.verbose = False - set_match_tests(match_tests) + cases_by_module, skipped = collect_cases(tests, match_tests=match_tests, + test_dir=test_dir) + for cases in cases_by_module.values(): + for case_id in cases: + print(case_id) + if skipped: + sys.stdout.flush() + stderr = sys.stderr + print(file=stderr) + print(count(len(skipped), "test"), "skipped:", file=stderr) + printlist(skipped, file=stderr) + +class _ModuleLoadFailed(Exception): + """The test module failed to load; its test cases are unknown.""" + - skipped = [] +def collect_cases(tests: TestTuple, *, + match_tests: TestFilter | None = None, + test_dir: StrPath | None = None + ) -> tuple[dict[TestName, list[str]], list[TestName]]: + # Install the filter unconditionally: passing None clears any + # previously installed global filter, so collection is not + # affected by unrelated state in this process. + set_match_tests(match_tests) + result: dict[TestName, list[str]] = {} + skipped: list[TestName] = [] for test_name in tests: module_name = abs_module_name(test_name, test_dir) + cases: list[str] = [] try: suite = unittest.defaultTestLoader.loadTestsFromName(module_name) - _list_cases(suite) + _collect_cases(suite, cases) except unittest.SkipTest: skipped.append(test_name) + continue + except _ModuleLoadFailed: + # The module failed to load. Run it as a whole, so that the + # error is reported as in the normal mode. + result[test_name] = [test_name] + continue + if cases: + result[test_name] = cases + return result, skipped - if skipped: - sys.stdout.flush() - stderr = sys.stderr - print(file=stderr) - print(count(len(skipped), "test"), "skipped:", file=stderr) - printlist(skipped, file=stderr) +def _collect_cases(suite: unittest.TestSuite, out: list[str]) -> None: + for test in suite: + if isinstance(test, unittest.TestSuite): + _collect_cases(test, out) + elif isinstance(test, unittest.loader._FailedTest): # type: ignore[attr-defined] + # The test module failed to load. Its test cases are + # unknown: let the caller run the whole module. + raise _ModuleLoadFailed + elif isinstance(test, unittest.TestCase): + if match_test(test): + out.append(test.id()) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 8773e9df73263b7..7391056627cd068 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -12,7 +12,7 @@ from test.support import os_helper, MS_WINDOWS, flush_std_streams from .cmdline import _parse_args, Namespace -from .findtests import findtests, split_test_packages, list_cases +from .findtests import findtests, split_test_packages, list_cases, collect_cases from .logger import Logger from .pgo import setup_pgo_tests from .result import TestResult @@ -73,6 +73,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.want_header: bool = ns.header self.want_list_tests: bool = ns.list_tests self.want_list_cases: bool = ns.list_cases + self.want_single_process_per_case: bool = ns.single_process_per_case self.want_wait: bool = ns.wait self.want_cleanup: bool = ns.cleanup self.want_rerun: bool = ns.rerun @@ -99,6 +100,10 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): else: num_workers = ns.use_mp # run in parallel self.num_workers: int = num_workers + if ns.single_process_per_case and ns.use_mp is None: + # Each test case runs in its own worker subprocess; + # default to one worker when -j was not given. + self.num_workers = 1 self.worker_json: StrJSON | None = ns.worker_json # Options to run tests @@ -521,6 +526,8 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: randomize=self.randomize, random_seed=self.random_seed, parallel_threads=self.parallel_threads, + single_process_per_case=self.want_single_process_per_case, + case_groups=None, ) def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: @@ -546,6 +553,23 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: print("Using random seed:", self.random_seed) runtests = self.create_run_tests(selected) + if self.want_single_process_per_case: + cases_by_module, _ = collect_cases( + selected, + match_tests=self.match_tests, + test_dir=self.test_dir) + groups = [] + for module_name in selected: + cases = cases_by_module.get(module_name) + if cases: + groups.append((module_name, tuple(cases))) + else: + groups.append((module_name, (module_name,))) + case_groups = tuple(groups) + case_ids = tuple( + case_id for _, cases in case_groups for case_id in cases + ) + runtests = runtests.copy(tests=case_ids, case_groups=case_groups) self.first_runtests = runtests self.logger.set_tests(runtests) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 7e6c7fa4cc5507a..a146a439d9d3eca 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -70,7 +70,6 @@ def stop(self): with self.lock: self.tests_iter = None - @dataclasses.dataclass(slots=True, frozen=True) class MultiprocessResult: result: TestResult @@ -269,16 +268,22 @@ def create_json_file(self, stack: contextlib.ExitStack) -> tuple[JsonFile, TextI json_file = JsonFile(json_fd, JsonFileType.UNIX_FD) return (json_file, json_tmpfile) - def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> WorkerRunTests: - tests = (test_name,) - if self.runtests.rerun: - match_tests = self.runtests.get_match_tests(test_name) + def create_worker_runtests(self, test_name: TestName, + json_file: JsonFile, + module_name: TestName | None = None, + ) -> WorkerRunTests: + kwargs: dict[str, Any] = {} + + if module_name is not None and test_name != module_name: + tests = (module_name,) + kwargs['match_tests'] = [(test_name, True)] else: - match_tests = None + tests = (test_name,) + if self.runtests.rerun: + match_tests = self.runtests.get_match_tests(test_name) + if match_tests: + kwargs['match_tests'] = [(test, True) for test in match_tests] - kwargs: dict[str, Any] = {} - if match_tests: - kwargs['match_tests'] = [(test, True) for test in match_tests] if self.runtests.output_on_failure: kwargs['verbose'] = True kwargs['output_on_failure'] = False @@ -356,11 +361,13 @@ def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, return (result, stdout) - def _runtest(self, test_name: TestName) -> MultiprocessResult: + def _runtest(self, test_name: TestName, + module_name: TestName | None = None) -> MultiprocessResult: with contextlib.ExitStack() as stack: stdout_file = self.create_stdout(stack) json_file, json_tmpfile = self.create_json_file(stack) - worker_runtests = self.create_worker_runtests(test_name, json_file) + worker_runtests = self.create_worker_runtests( + test_name, json_file, module_name=module_name) retcode: str | int | None retcode, tmp_files = self.run_tmp_files(worker_runtests, @@ -393,26 +400,38 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: def run(self) -> None: fail_fast = self.runtests.fail_fast fail_env_changed = self.runtests.fail_env_changed + single_process_per_case = self.runtests.single_process_per_case try: - while not self._stopped: + stop = False + while not self._stopped and not stop: try: - test_name = next(self.pending) + module_name, case_ids = next(self.pending) except StopIteration: break - self.start_time = time.monotonic() - self.test_name = test_name - try: - mp_result = self._runtest(test_name) - except WorkerError as exc: - mp_result = exc.mp_result - finally: - self.test_name = _NOT_RUNNING - mp_result.result.duration = time.monotonic() - self.start_time - self.output.put((False, mp_result)) - - if mp_result.result.must_stop(fail_fast, fail_env_changed): - break + # All cases of a group run sequentially on this thread + for test_name in case_ids: + if self._stopped: + break + self.start_time = time.monotonic() + self.test_name = test_name + try: + mp_result = self._runtest( + test_name, + module_name if single_process_per_case else None) + except WorkerError as exc: + mp_result = exc.mp_result + finally: + self.test_name = _NOT_RUNNING + mp_result.result.duration = time.monotonic() - self.start_time + if single_process_per_case: + # Report the test case, not the test module + mp_result.result.test_name = test_name + self.output.put((False, mp_result)) + + if mp_result.result.must_stop(fail_fast, fail_env_changed): + stop = True + break except ExitThread: pass except BaseException: @@ -489,8 +508,7 @@ def __init__(self, num_workers: int, runtests: RunTests, self.live_worker_count = 0 self.output: queue.Queue[QueueContent] = queue.Queue() - tests_iter = runtests.iter_tests() - self.pending = MultiprocessIterator(tests_iter) + self.pending = MultiprocessIterator(runtests.iter_case_groups()) self.timeout = runtests.timeout if self.timeout is not None: # Rely on faulthandler to kill a worker process. This timouet is diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 0a9edce1085be54..fbb04b5b705ecec 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -101,6 +101,8 @@ class RunTests: randomize: bool random_seed: int | str parallel_threads: int | None + single_process_per_case: bool + case_groups: tuple[tuple[TestName, tuple[TestName, ...]], ...] | None def copy(self, **override) -> 'RunTests': state = dataclasses.asdict(self) @@ -132,6 +134,20 @@ def iter_tests(self) -> Iterator[TestName]: else: yield from self.tests + def iter_case_groups(self) -> Iterator[tuple[TestName, tuple[TestName, ...]]]: + """ + Yield (module_name, case_ids) pairs. All case_ids in a group + must run sequentially on the same worker thread. + """ + if self.case_groups is None: + for name in self.iter_tests(): + yield (name, (name,)) + elif self.forever: + while True: + yield from self.case_groups + else: + yield from self.case_groups + def json_file_use_stdout(self) -> bool: # Use STDOUT in two cases: # diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 6ba440053089161..f05008bf8bc2c01 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -20,11 +20,15 @@ import sys import sysconfig import tempfile +import threading import textwrap import unittest import unittest.mock from xml.etree import ElementTree +from test.libregrtest.findtests import collect_cases +from test.libregrtest.filter import set_match_tests +from test.libregrtest.run_workers import MultiprocessIterator from test import support from test.support import import_helper from test.support import os_helper @@ -32,7 +36,7 @@ from test.libregrtest import main from test.libregrtest import setup from test.libregrtest import utils -from test.libregrtest.filter import get_match_tests, set_match_tests, match_test +from test.libregrtest.filter import get_match_tests, match_test from test.libregrtest.result import TestStats from test.libregrtest.utils import normalize_test_name @@ -574,6 +578,30 @@ def test_single_process(self): self.assertEqual(regrtest.num_workers, 0) self.assertTrue(regrtest.single_process) + def test_single_process_per_case(self): + ns = self.parse_args(['--single-process-per-case']) + self.assertTrue(ns.single_process_per_case) + + # No -j given: default to one worker + regrtest = self.create_regrtest(['--single-process-per-case']) + self.assertEqual(regrtest.num_workers, 1) + + # Explicit -j2 is respected + regrtest = self.create_regrtest(['-j2', '--single-process-per-case']) + self.assertEqual(regrtest.num_workers, 2) + + def test_single_process_per_case_conflicts(self): + # --single-process-per-case doesn't compose with options that + # re-run or restrict test selection after collection (gh-109817) + self.checkError(['--single-process-per-case', '--rerun'], + "don't go together") + self.checkError(['--single-process-per-case', '--single-process'], + "don't go together") + self.checkError(['--single-process-per-case', '--pgo'], + "don't go together") + self.checkError(['--single-process-per-case', '--fast-ci'], + "don't go together") + def test_pythoninfo(self): ns = self.parse_args([]) self.assertFalse(ns.pythoninfo) @@ -2332,6 +2360,94 @@ def test_crash(self): self.assertIn(f"Exit code {exitcode} (SIGSEGV)", output) self.check_line(output, "just before crash!", full=True, regex=False) + def test_single_process_per_case(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_one(self): + pass + def test_two(self): + pass + """) + testname = self.create_test(code=code) + + output = self.run_tests('--single-process-per-case', '-v', testname) + # Each case is reported individually (progress lines have a + # timestamp prefix, so match mid-line with assertIn) + self.assertIn(f'{testname}.Tests.test_one passed', output) + self.assertIn(f'{testname}.Tests.test_two passed', output) + self.check_line(output, 'All 2 tests OK.', regex=False) + + def test_single_process_per_case_order_independence(self): + # A module where test_b only passes if it runs in isolation. + # This simulates the real order-dependency bugs this feature + # is meant to catch (gh-109817): running each case in its own + # process means test_b can never observe state left behind + # by test_a. + code = textwrap.dedent(""" + import unittest + + counter = {'n': 0} + + class Tests(unittest.TestCase): + def test_a(self): + counter['n'] += 1 + def test_b(self): + self.assertEqual(counter['n'], 0) + """) + testname = self.create_test(code=code) + + output = self.run_tests('--single-process-per-case', '-v', testname) + self.assertIn(f'{testname}.Tests.test_a passed', output) + self.assertIn(f'{testname}.Tests.test_b passed', output) + self.check_line(output, 'All 2 tests OK.', regex=False) + + def test_single_process_per_case_failure_isolated(self): + # One failing case must not abort or contaminate sibling + # cases in the same module. + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_pass(self): + pass + def test_fail(self): + self.fail("expected failure") + """) + testname = self.create_test(code=code) + + output = self.run_tests('--single-process-per-case', testname, + exitcode=EXITCODE_BAD_TEST) + # The failing case is reported by its case ID... + self.assertIn(f'{testname}.Tests.test_fail failed', output) + self.check_line(output, '1 test failed:', regex=False) + self.assertIn(f' {testname}.Tests.test_fail', output) + # ...and the sibling case in the same module still ran and passed + self.assertIn(f'{testname}.Tests.test_pass passed', output) + self.check_line(output, '1 test OK.', regex=False) + + def test_single_process_per_case_import_error(self): + testname = self.create_test(code='raise ImportError("boom")') + + output = self.run_tests('--single-process-per-case', testname, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], failed=[testname], + stats=0) + self.assertNotIn('_FailedTest', output) + + def test_single_process_per_case_skipped_module(self): + code = textwrap.dedent(""" + import unittest + raise unittest.SkipTest("nope") + """) + testname = self.create_test(code=code) + + output = self.run_tests('--single-process-per-case', testname) + self.check_executed_tests(output, [testname], + skipped=[testname], stats=0) + + def test_verbose3(self): code = textwrap.dedent(r""" import unittest @@ -2442,6 +2558,113 @@ def test_pythoninfo(self): self.assertIn("Python build information", output) +class FindTestsTestCase(BaseTestCase): + def test_collect_cases_groups_by_module(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_one(self): + pass + def test_two(self): + pass + """) + testname = self.create_test(code=code) + sys.path.insert(0, self.tmptestdir) + self.addCleanup(sys.path.remove, self.tmptestdir) + self.addCleanup(sys.modules.pop, testname, None) + self.addCleanup(set_match_tests, None) + + cases_by_module, skipped = collect_cases((testname,), + test_dir=self.tmptestdir) + self.assertIn(testname, cases_by_module) + self.assertEqual(len(cases_by_module[testname]), 2) + self.assertEqual(skipped, []) + + def test_collect_cases_match_tests_filters(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_keep(self): + pass + def test_drop(self): + pass + """) + testname = self.create_test(code=code) + sys.path.insert(0, self.tmptestdir) + self.addCleanup(sys.path.remove, self.tmptestdir) + self.addCleanup(sys.modules.pop, testname, None) + self.addCleanup(set_match_tests, None) + + cases_by_module, _ = collect_cases( + (testname,), + match_tests=[('*test_keep*', True)], + test_dir=self.tmptestdir) + case_ids = cases_by_module.get(testname, []) + self.assertTrue(any('test_keep' in c for c in case_ids)) + self.assertFalse(any('test_drop' in c for c in case_ids)) + + def test_collect_cases_skiptest(self): + code = textwrap.dedent(""" + import unittest + raise unittest.SkipTest("module-level skip") + """) + testname = self.create_test(code=code) + sys.path.insert(0, self.tmptestdir) + self.addCleanup(sys.path.remove, self.tmptestdir) + self.addCleanup(sys.modules.pop, testname, None) + self.addCleanup(set_match_tests, None) + + cases_by_module, skipped = collect_cases((testname,), + test_dir=self.tmptestdir) + self.assertEqual(cases_by_module, {}) + self.assertIn(testname, skipped) + + +class MultiprocessIteratorTestCase(unittest.TestCase): + def test_yields_all_groups_once(self): + groups = [("mod_a", ("mod_a.A.t1", "mod_a.A.t2")), + ("mod_b", ("mod_b.B.t1",))] + it = MultiprocessIterator(iter(groups)) + seen = [] + while (g := next(it, None)) is not None: + seen.append(g) + self.assertEqual(seen, groups) + + def test_exhausted_returns_none(self): + it = MultiprocessIterator(iter([])) + self.assertIsNone(next(it, None)) + + def test_stop_halts_iteration(self): + groups = [("mod_a", ("mod_a.A.t1",)), ("mod_b", ("mod_b.B.t1",))] + it = MultiprocessIterator(iter(groups)) + next(it, None) + it.stop() + self.assertIsNone(next(it, None)) + + def test_thread_safety_no_duplicate_or_lost_groups(self): + n = 200 + groups = [(f"mod_{i}", (f"mod_{i}.T.t",)) for i in range(n)] + it = MultiprocessIterator(iter(groups)) + results = [] + results_lock = threading.Lock() + + def worker(): + while (g := next(it, None)) is not None: + with results_lock: + results.append(g) + + threads = [threading.Thread(target=worker) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + + self.assertEqual(sorted(results), sorted(groups)) + self.assertEqual(len(results), n) + + class TestUtils(unittest.TestCase): def test_format_duration(self): self.assertEqual(utils.format_duration(0), diff --git a/Misc/NEWS.d/next/Tests/2026-08-12-20-52-16.gh-issue-109817.GZHUNg.rst b/Misc/NEWS.d/next/Tests/2026-08-12-20-52-16.gh-issue-109817.GZHUNg.rst new file mode 100644 index 000000000000000..6064b331992d715 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-08-12-20-52-16.gh-issue-109817.GZHUNg.rst @@ -0,0 +1,4 @@ +Add the ``--single-process-per-case`` option to libregrtest to run +every test case in a separate process. Test cases from the same +test module are run sequentially. This helps to detect order +dependencies and environment leaks between test cases. From 7586115ce3adeb9133717f0a721fb05621dd6700 Mon Sep 17 00:00:00 2001 From: aaron-fl <43595516+aaron-fl@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:28:51 +0900 Subject: [PATCH 07/23] gh-102967: Don't pass None to seek(), simply truncate from the current position (GH-102968) --- Lib/doctest.py | 3 ++- Misc/ACKS | 1 + .../Library/2023-03-23-19-47-05.gh-issue-102967.gfIYkN.rst | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-23-19-47-05.gh-issue-102967.gfIYkN.rst diff --git a/Lib/doctest.py b/Lib/doctest.py index cf31eae5365b6eb..d5541ab41f38d4f 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -285,7 +285,8 @@ def getvalue(self): return result def truncate(self, size=None): - self.seek(size) + if size is not None: + self.seek(size) StringIO.truncate(self) # Worst-case linear-time ellipsis matching. diff --git a/Misc/ACKS b/Misc/ACKS index 9316e9359499581..4a3f6482294e087 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -426,6 +426,7 @@ Hauke Dämpfling Evan Dandrea Eric Daniel Scott David Daniels +Aaron Davidson Derzsi Dániel Lawrence D'Anna Steven D'Aprano diff --git a/Misc/NEWS.d/next/Library/2023-03-23-19-47-05.gh-issue-102967.gfIYkN.rst b/Misc/NEWS.d/next/Library/2023-03-23-19-47-05.gh-issue-102967.gfIYkN.rst new file mode 100644 index 000000000000000..677b8ee86e6c8f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-23-19-47-05.gh-issue-102967.gfIYkN.rst @@ -0,0 +1,2 @@ +A bug in :func:`!doctest._SpoofOut.truncate` was causing None to be passed to :func:`!StringIO.seek` when no size was given. +A simple fix skips the seek call when no size is given so the buffer can be truncated from the current position. From 726e48565670e56affd3e6d461871ec20ee12694 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 12:43:40 +0300 Subject: [PATCH 08/23] gh-114905: Test that ssl._create_stdlib_context() rejects check_hostname with CERT_NONE (GH-155509) With PROTOCOL_TLS_CLIENT, which became the default protocol in 3.10, this is an error. With an explicitly specified legacy protocol it used to succeed, silently raising verify_mode to CERT_REQUIRED and ignoring the requested CERT_NONE. No caller of ssl._create_stdlib_context() in the standard library passes check_hostname, so no public API reaches it. --- Lib/ssl.py | 2 ++ Lib/test/test_ssl.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/Lib/ssl.py b/Lib/ssl.py index db66c59c05cc542..dc957121728f283 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -754,6 +754,8 @@ def _create_unverified_context(protocol=None, *, cert_reqs=CERT_NONE, raise ValueError(purpose) context = SSLContext(protocol) + # Setting verify_mode to CERT_NONE fails while check_hostname is + # enabled, so assign check_hostname first (gh-114905). context.check_hostname = check_hostname if cert_reqs is not None: context.verify_mode = cert_reqs diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 6446f96eab42a43..2bba665d19343e6 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1817,6 +1817,35 @@ def test__create_stdlib_context(self): self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) self._assert_context_options(ctx) + def test__create_stdlib_context_check_hostname(self): + # gh-114905: check_hostname cannot be combined with CERT_NONE, + # the default for cert_reqs. + msg = "Cannot set verify_mode to CERT_NONE when check_hostname" + with self.assertRaisesRegex(ValueError, msg): + ssl._create_stdlib_context(check_hostname=True) + with self.assertRaisesRegex(ValueError, msg): + ssl._create_stdlib_context(cert_reqs=ssl.CERT_NONE, + check_hostname=True) + + # Accepted before 3.10 with a legacy protocol. + if has_tls_protocol('PROTOCOL_TLSv1_2'): + with warnings_helper.check_warnings(): + with self.assertRaisesRegex(ValueError, msg): + ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1_2, + cert_reqs=ssl.CERT_NONE, + check_hostname=True) + + # cert_reqs=None leaves PROTOCOL_TLS_CLIENT's CERT_REQUIRED. + ctx = ssl._create_stdlib_context(cert_reqs=None, check_hostname=True) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + + # CERT_REQUIRED is covered by test__create_stdlib_context(). + ctx = ssl._create_stdlib_context(cert_reqs=ssl.CERT_OPTIONAL, + check_hostname=True) + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + self.assertTrue(ctx.check_hostname) + def test_check_hostname(self): with warnings_helper.check_warnings(): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) From aca38ab4c1f0d805f1d411d58dd42493e56620cf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 12:44:27 +0300 Subject: [PATCH 09/23] gh-69619: Link to isspace() in the strip() and split() documentation (GH-155444) The methods that default to removing or splitting on whitespace now link to str.isspace() and bytes.isspace(), where whitespace is defined. --- Doc/library/stdtypes.rst | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 2c11d0edbc2cbfd..280a9f3b7d07518 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2590,7 +2590,8 @@ expression support in the :mod:`re` module). Return a list of the words in the string, using *sep* as the delimiter string. If *maxsplit* is given, at most *maxsplit* splits are done, the *rightmost* - ones. If *sep* is not specified or ``None``, any whitespace string is a + ones. If *sep* is not specified or ``None``, any + :meth:`whitespace ` string is a separator. Except for splitting from the right, :meth:`rsplit` behaves like :meth:`split` which is described in detail below. @@ -2651,7 +2652,8 @@ expression support in the :mod:`re` module). ['1', '2', '3<4'] If *sep* is not specified or is ``None``, a different splitting algorithm is - applied: runs of consecutive whitespace are regarded as a single separator, + applied: runs of consecutive :meth:`whitespace ` are regarded + as a single separator, and the result will contain no empty strings at the start or end if the string has leading or trailing whitespace. Consequently, splitting an empty string or a string consisting of just whitespace with a ``None`` separator @@ -3915,7 +3917,8 @@ produce new objects. Return a copy of the sequence with specified leading bytes removed. The *bytes* argument is a binary sequence specifying the set of byte values to be removed. If omitted or ``None``, the *bytes* argument defaults - to removing ASCII whitespace. The *bytes* argument is not a prefix; + to removing :meth:`ASCII whitespace `. + The *bytes* argument is not a prefix; rather, all combinations of its values are stripped:: >>> b' spacious '.lstrip() @@ -3959,7 +3962,8 @@ produce new objects. Split the binary sequence into subsequences of the same type, using *sep* as the delimiter string. If *maxsplit* is given, at most *maxsplit* splits are done, the *rightmost* ones. If *sep* is not specified or ``None``, - any subsequence consisting solely of ASCII whitespace is a separator. + any subsequence consisting solely of + :meth:`ASCII whitespace ` is a separator. Except for splitting from the right, :meth:`rsplit` behaves like :meth:`split` which is described in detail below. @@ -3970,7 +3974,8 @@ produce new objects. Return a copy of the sequence with specified trailing bytes removed. The *bytes* argument is a binary sequence specifying the set of byte values to be removed. If omitted or ``None``, the *bytes* argument defaults to - removing ASCII whitespace. The *bytes* argument is not a suffix; rather, + removing :meth:`ASCII whitespace `. + The *bytes* argument is not a suffix; rather, all combinations of its values are stripped:: >>> b' spacious '.rstrip() @@ -4023,7 +4028,8 @@ produce new objects. [b'1', b'2', b'3<4'] If *sep* is not specified or is ``None``, a different splitting algorithm - is applied: runs of consecutive ASCII whitespace are regarded as a single + is applied: runs of consecutive :meth:`ASCII whitespace ` + are regarded as a single separator, and the result will contain no empty strings at the start or end if the sequence has leading or trailing whitespace. Consequently, splitting an empty sequence or a sequence consisting solely of ASCII @@ -4046,7 +4052,8 @@ produce new objects. Return a copy of the sequence with specified leading and trailing bytes removed. The *bytes* argument is a binary sequence specifying the set of byte values to be removed. If omitted or ``None``, the *bytes* - argument defaults to removing ASCII whitespace. The *bytes* argument is + argument defaults to removing :meth:`ASCII whitespace `. + The *bytes* argument is not a prefix or suffix; rather, all combinations of its values are stripped:: From 582a2d3ebafa993234218470e5fbb490197d89e6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 12:53:29 +0300 Subject: [PATCH 10/23] gh-84548: Document Windows specific behavior of abspath() and realpath() (GH-155388) On Windows abspath() resolves a drive-relative path against the current directory of the specified drive, capitalizes the drive letter and strips trailing dots and spaces, so the result can differ from normpath(join(os.getcwd(), path)). realpath() returns the path in the case reported by the operating system. --- Doc/library/os.path.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 808187061733be0..f5ddb426c915f5a 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -59,6 +59,18 @@ the :mod:`glob` module.) Return a normalized absolutized version of the pathname *path*. On most platforms, this is equivalent to calling ``normpath(join(os.getcwd(), path))``. + On Windows the path is normalized by the operating system, + therefore the result can differ from ``normpath(join(os.getcwd(), path))``. + A drive-relative path is resolved against the current directory + of the specified drive, and the drive letter is capitalized. + Trailing dots and spaces are stripped. + For example:: + + >>> os.path.abspath('c:spam') + 'C:\\Temp\\spam' + >>> os.path.abspath('c:/temp/spam. . .') + 'c:\\temp\\spam' + .. seealso:: :func:`os.path.join` and :func:`os.path.normpath`. .. versionchanged:: 3.6 @@ -435,6 +447,9 @@ the :mod:`glob` module.) links encountered in the path (if they are supported by the operating system). On Windows, this function will also resolve MS-DOS (also called 8.3) style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. + The returned path uses the case reported by the operating system, + which can differ from the case of *path*, + in particular the drive letter is capitalized. By default, the path is evaluated up to the first component that does not exist, is a symlink loop, or whose evaluation raises :exc:`OSError`. From 116caab464af216d30f0b938703ab65fd2cf5edf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 12:53:48 +0300 Subject: [PATCH 11/23] gh-78526: Add tests for PEP 468 and PEP 520 (GH-155387) Test that the order of keyword arguments and the order of class attribute definitions are preserved. --- Lib/test/test_call.py | 48 +++++++++++++++++++++++++++++++++ Lib/test/test_class.py | 60 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 1e42356be21ddd6..76f1c351e159086 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -47,6 +47,54 @@ def fn(**kw): self.assertIsInstance(res, dict) self.assertEqual(list(res.items()), expected) + def test_kwargs_order_preserved(self): + # PEP 468: Preserving Keyword Argument Order + def fn(**kw): + return list(kw) + + self.assertEqual(fn(b=1, a=2, c=3), ['b', 'a', 'c']) + self.assertEqual(fn(c=3, a=2, b=1), ['c', 'a', 'b']) + # Unpacked mappings are merged in place, keeping their own order. + self.assertEqual(fn(z=0, **{'x': 1, 'a': 2}, y=3), + ['z', 'x', 'a', 'y']) + self.assertEqual(fn(**{'b': 1}, **{'a': 2}), ['b', 'a']) + # Named parameters are removed from **kwargs without reordering + # the rest. + def fn2(a, c=None, **kw): + return list(kw) + + self.assertEqual(fn2(d=1, a=2, b=3, c=4, e=5), ['d', 'b', 'e']) + + def test_kwargs_order_preserved_in_methods(self): + # PEP 468: Preserving Keyword Argument Order + class C: + def __init__(self, **kw): + self.init_kw = list(kw) + + def meth(self, **kw): + return list(kw) + + @classmethod + def cmeth(cls, **kw): + return list(kw) + + @staticmethod + def smeth(**kw): + return list(kw) + + c = C(b=1, a=2, c=3) + self.assertEqual(c.init_kw, ['b', 'a', 'c']) + self.assertEqual(c.meth(b=1, a=2, c=3), ['b', 'a', 'c']) + self.assertEqual(C.cmeth(b=1, a=2, c=3), ['b', 'a', 'c']) + self.assertEqual(C.smeth(b=1, a=2, c=3), ['b', 'a', 'c']) + + def test_kwargs_order_preserved_in_c_functions(self): + # PEP 468: Preserving Keyword Argument Order + self.assertEqual(list(dict(b=1, a=2, c=3)), ['b', 'a', 'c']) + self.assertEqual(list(dict(**{'b': 1}, a=2)), ['b', 'a']) + self.assertEqual(list(collections.OrderedDict(b=1, a=2, c=3)), + ['b', 'a', 'c']) + def test_frames_are_popped_after_failed_calls(self): # GH-93252: stuff blows up if we don't pop the new frame after # recovering from failed calls: diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index e07efd269669459..7bd6d966e8536b7 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1052,5 +1052,65 @@ def __init__(self): self.fail("MemoryError was not raised during deallocation") self.fail("the dictionary was not cleared") +class DefinitionOrderTests(unittest.TestCase): + # PEP 520: Preserving Class Attribute Definition Order + + @staticmethod + def defined_names(namespace): + # Skip the names added by the compiler, like __firstlineno__. + return [name for name in namespace if not name.startswith('__')] + + def test_definition_order(self): + class C: + b = 1 + a = 2 + def m(self): pass + @staticmethod + def s(): pass + z = 3 + + self.assertEqual(self.defined_names(C.__dict__), + ['b', 'a', 'm', 's', 'z']) + + def test_definition_order_redefinition(self): + class C: + b = 1 + a = 2 + b = 3 + + self.assertEqual(self.defined_names(C.__dict__), ['b', 'a']) + self.assertEqual(C.b, 3) + + def test_definition_order_after_deletion(self): + class C: + a = 1 + b = 2 + del a + a = 3 + + self.assertEqual(self.defined_names(C.__dict__), ['b', 'a']) + + def test_definition_order_in_namespace(self): + namespaces = [] + class Meta(type): + def __new__(mcls, name, bases, namespace, **kwds): + namespaces.append(list(namespace)) + return super().__new__(mcls, name, bases, namespace, **kwds) + + class C(metaclass=Meta): + b = 1 + a = 2 + def m(self): pass + + self.assertEqual(self.defined_names(namespaces[0]), ['b', 'a', 'm']) + + def test_prepare_preserves_order(self): + namespace = type.__prepare__('C', ()) + namespace['b'] = 1 + namespace['a'] = 2 + namespace['b'] = 3 + self.assertEqual(list(namespace), ['b', 'a']) + + if __name__ == '__main__': unittest.main() From 99b88473f6d73bd054bcd7d43239a03d583db81c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 12:57:48 +0300 Subject: [PATCH 12/23] gh-64660: Do not hardcode the name of the returned variable (GH-155268) Return converters had to hardcode "return_value", because declare() sets data.return_value to the variable which receives the value returned by the impl. The name of the variable returned by the parsing function is now available as data.parser_retval. --- .../2026-08-06-11-40-49.gh-issue-64660.MswRQB.rst | 3 +++ PC/msvcrtmodule.c | 7 ++++--- Tools/clinic/libclinic/clanguage.py | 1 + Tools/clinic/libclinic/codegen.py | 7 +++++-- Tools/clinic/libclinic/parse_args.py | 8 ++++---- Tools/clinic/libclinic/return_converters.py | 9 ++++++--- 6 files changed, 23 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2026-08-06-11-40-49.gh-issue-64660.MswRQB.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-08-06-11-40-49.gh-issue-64660.MswRQB.rst b/Misc/NEWS.d/next/Tools-Demos/2026-08-06-11-40-49.gh-issue-64660.MswRQB.rst new file mode 100644 index 000000000000000..aa0410b39b11272 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-08-06-11-40-49.gh-issue-64660.MswRQB.rst @@ -0,0 +1,3 @@ +Argument Clinic return converters no longer need to hardcode the name of the +variable returned by the parsing function. +It is now available as ``data.parser_retval``. diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 02f16d41b1457b1..26d7547c387f5f8 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -65,7 +65,7 @@ class byte_char_return_converter(CReturnConverter): data.declarations.append('char s[1];') data.return_value = 's[0]' data.return_conversion.append( - 'return_value = PyBytes_FromStringAndSize(s, 1);\n') + f'{data.parser_retval} = PyBytes_FromStringAndSize(s, 1);\n') class wchar_t_return_converter(CReturnConverter): type = 'wchar_t' @@ -73,9 +73,10 @@ class wchar_t_return_converter(CReturnConverter): def render(self, function, data): self.declare(data) data.return_conversion.append( - 'return_value = PyUnicode_FromOrdinal(_return_value);\n') + f'{data.parser_retval} = ' + f'PyUnicode_FromOrdinal({data.converter_retval});\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=ff031be44ab3250d]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=ed7a4a045a6d0496]*/ /*[clinic input] module msvcrt diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index 1581a19a4fd78ab..a8473dba0512460 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -525,6 +525,7 @@ def render_function( template_dict['cleanup'] = libclinic.format_escape("".join(data.cleanup)) template_dict['return_value'] = data.return_value + template_dict['parser_retval'] = data.parser_retval template_dict['lock'] = "\n".join(data.lock) template_dict['unlock'] = "\n".join(data.unlock) diff --git a/Tools/clinic/libclinic/codegen.py b/Tools/clinic/libclinic/codegen.py index b2f1db6f8ef8da7..3ca8c4a1b6859db 100644 --- a/Tools/clinic/libclinic/codegen.py +++ b/Tools/clinic/libclinic/codegen.py @@ -47,14 +47,17 @@ def __init__(self) -> None: # The arguments to the impl function at the time it's called. self.impl_arguments: list[str] = [] + # The name of the variable which is returned by the parser. + self.parser_retval = "return_value" + # For return converters: the name of the variable that # should receive the value returned by the impl. self.return_value = "return_value" # For return converters: the code to convert the return # value from the parse function. This is also where - # you should check the _return_value for errors, and - # "goto exit" if there are any. + # you should check the value returned by the impl for errors, + # and "goto exit" if there are any. self.return_conversion: list[str] = [] self.converter_retval = "_return_value" diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index 2ad1e94ea2b4c79..0e99a89d74d7241 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -320,7 +320,7 @@ def select_prototypes(self) -> None: self.docstring_prototype = '' self.docstring_definition = '' self.methoddef_define = METHODDEF_PROTOTYPE_DEFINE - self.return_value_declaration = "PyObject *return_value = NULL;" + self.return_value_declaration = "PyObject *{parser_retval} = NULL;" if self.is_new_or_init() and not self.func.docstring: pass @@ -331,7 +331,7 @@ def select_prototypes(self) -> None: elif self.func.kind is SETTER: if self.func.docstring: fail("docstrings are only supported for @getter, not @setter") - self.return_value_declaration = "int {return_value};" + self.return_value_declaration = "int {parser_retval};" self.methoddef_define = SETTERDEF_PROTOTYPE_DEFINE else: self.docstring_prototype = DOCSTRING_PROTOTYPE_VAR @@ -372,7 +372,7 @@ def parser_body( {exit_label} {cleanup} - return return_value; + return {parser_retval}; }} """) for field in preamble, *fields, finale: @@ -861,7 +861,7 @@ def handle_new_or_init(self) -> None: if self.func.kind is METHOD_NEW: self.parser_prototype = PARSER_PROTOTYPE_KEYWORD else: - self.return_value_declaration = "int return_value = -1;" + self.return_value_declaration = "int {parser_retval} = -1;" self.parser_prototype = PARSER_PROTOTYPE_KEYWORD___INIT__ fields: list[str] = list(self.parser_body_fields) diff --git a/Tools/clinic/libclinic/return_converters.py b/Tools/clinic/libclinic/return_converters.py index b41e053bae5f3a7..4134d8e065ec437 100644 --- a/Tools/clinic/libclinic/return_converters.py +++ b/Tools/clinic/libclinic/return_converters.py @@ -110,7 +110,8 @@ def render(self, function: Function, data: CRenderData) -> None: self.declare(data) self.err_occurred_if(f"{data.converter_retval} == -1", data) data.return_conversion.append( - f'return_value = PyBool_FromLong((long){data.converter_retval});\n' + f'{data.parser_retval} = ' + f'PyBool_FromLong((long){data.converter_retval});\n' ) @@ -124,7 +125,8 @@ def render(self, function: Function, data: CRenderData) -> None: self.declare(data) self.err_occurred_if(f"{data.converter_retval} == {self.unsigned_cast}-1", data) data.return_conversion.append( - f'return_value = {self.conversion_fn}({self.cast}{data.converter_retval});\n' + f'{data.parser_retval} = ' + f'{self.conversion_fn}({self.cast}{data.converter_retval});\n' ) @@ -164,7 +166,8 @@ def render(self, function: Function, data: CRenderData) -> None: self.declare(data) self.err_occurred_if(f"{data.converter_retval} == -1.0", data) data.return_conversion.append( - f'return_value = PyFloat_FromDouble({self.cast}{data.converter_retval});\n' + f'{data.parser_retval} = ' + f'PyFloat_FromDouble({self.cast}{data.converter_retval});\n' ) From 1370f8a8a19a1201b2cc27a301a3f8f1d4478764 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 13:00:30 +0300 Subject: [PATCH 13/23] gh-80678: Document the preferred attribute of csv.Sniffer (GH-155068) It can be modified to change the order in which the delimiters are preferred for breaking ties. --- Doc/library/csv.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 81949261c563d04..e78371f114e0a19 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -324,8 +324,8 @@ The :mod:`!csv` module defines the following classes: If several combinations fit the sample equally well --- for example if both ``','`` and ``';'`` split every row consistently --- - the delimiters ``','``, ``'\t'``, ``';'``, ``' '`` and ``':'`` - are preferred, in this order, + the delimiters listed in the :attr:`~Sniffer.preferred` attribute + are preferred, in that order, no matter how many times each of them occurs. .. versionchanged:: next @@ -354,6 +354,15 @@ The :mod:`!csv` module defines the following classes: This method is a rough heuristic and may produce both false positives and negatives. + The :class:`Sniffer` class has the following attribute: + + .. attribute:: preferred + + The list of the delimiters preferred for breaking ties, + in the order of preference. + It can be modified. + Its initial value is ``[',', '\t', ';', ' ', ':']``. + An example for :class:`Sniffer` use:: with open('example.csv', newline='') as csvfile: From 3cc891f7ae71ab1fed3e00ec83baadc42db1eb1e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 13:02:52 +0300 Subject: [PATCH 14/23] gh-105116: Document that an escaped csv quotechar does not cause quoting (GH-155073) Co-authored-by: Claude Opus 5 (1M context) --- Doc/library/csv.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index e78371f114e0a19..16ff9746f8f05a0 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -386,6 +386,8 @@ The :mod:`!csv` module defines the following constants: Instructs :class:`writer` objects to only quote those fields which contain special characters such as *delimiter*, *quotechar*, ``'\r'``, ``'\n'`` or any of the characters in *lineterminator*. + If *doublequote* is :const:`False` and *escapechar* is set, + the *quotechar* is escaped instead of causing the field to be quoted. .. data:: QUOTE_NONNUMERIC From a697faf45077f7fb24edf4f2f75b8aa09d873dcd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 13:05:33 +0300 Subject: [PATCH 15/23] gh-89024: Document the 3.10 change in escaping the csv escapechar (GH-155067) Add a versionchanged note for the escapechar itself being escaped. --- Doc/library/csv.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 16ff9746f8f05a0..710a5b11ab64055 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -492,6 +492,10 @@ Dialects support the following attributes: On reading, the *escapechar* removes any special meaning from the following character. It defaults to :const:`None`, which disables escaping. + .. versionchanged:: 3.10 + Previously the *escapechar* itself was not escaped, + which lost it on reading. + .. versionchanged:: 3.11 An empty *escapechar* is not allowed. From c0d0b286bc0b24512b56c9087a4bcf8ffca50ab6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 13:06:17 +0300 Subject: [PATCH 16/23] gh-155044: Support copy.replace() for optparse.Values (GH-155050) Co-Authored-By: Claude Opus 5 (1M context) --- Doc/library/optparse.rst | 6 ++++++ Lib/optparse.py | 6 ++++++ Lib/test/test_optparse.py | 7 +++++++ .../Library/2026-08-01-19-05-00.gh-issue-155044.Op1Val.rst | 1 + 4 files changed, 20 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-01-19-05-00.gh-issue-155044.Op1Val.rst diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 905212965bd70fe..09416f12ad45c2f 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -1073,6 +1073,12 @@ As you can see, most actions involve storing or updating a value somewhere. and can be overridden by a custom subclass passed to the *values* argument of :meth:`OptionParser.parse_args` (as described in :ref:`optparse-parsing-arguments`). + :class:`!Values` objects support :func:`copy.replace`, + which returns a copy of the object with the specified attributes replaced. + + .. versionchanged:: next + Added support for :func:`copy.replace`. + Option arguments (and various other values) are stored as attributes of this object, according to the :attr:`~Option.dest` (destination) option attribute. diff --git a/Lib/optparse.py b/Lib/optparse.py index de1082442ef7f2e..ae6737a3d139426 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -830,6 +830,12 @@ def __eq__(self, other): else: return NotImplemented + def __replace__(self, /, **changes): + new = self.__class__() + new.__dict__.update(self.__dict__) + new.__dict__.update(changes) + return new + def _update_careful(self, dict): """ Update the option values from an arbitrary dictionary, but only diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index fc8ef9520b3c0f3..9fca2dafd5c99e2 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -434,6 +434,13 @@ def test_basics(self): self.assertNotEqual(values, "") self.assertNotEqual(values, []) + def test_replace(self): + values = Values(defaults={"foo": "bar", "baz": 42}) + new = copy.replace(values, baz=43, spam="eggs") + self.assertIsInstance(new, Values) + self.assertEqual(vars(new), {"foo": "bar", "baz": 43, "spam": "eggs"}) + self.assertEqual(vars(values), {"foo": "bar", "baz": 42}) + class TestTypeAliases(BaseTest): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2026-08-01-19-05-00.gh-issue-155044.Op1Val.rst b/Misc/NEWS.d/next/Library/2026-08-01-19-05-00.gh-issue-155044.Op1Val.rst new file mode 100644 index 000000000000000..10665f913ef5e63 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-01-19-05-00.gh-issue-155044.Op1Val.rst @@ -0,0 +1 @@ +:class:`optparse.Values` objects now support :func:`copy.replace`. From b4af851f6b5e1cf4dcc2e0397321d7c761f227c6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 13:11:35 +0300 Subject: [PATCH 17/23] gh-155033: Support copy.replace() for csv dialects (GH-155035) Add __replace__() to the _csv.Dialect type and to the csv.Dialect base class. Formatting parameters that are not replaced are inherited from the original dialect, and the result is validated like a newly created dialect. Co-Authored-By: Claude Opus 5 (1M context) --- Doc/library/csv.rst | 7 +++ Lib/csv.py | 16 ++++++ Lib/test/test_csv.py | 55 +++++++++++++++++++ ...-08-01-12-00-00.gh-issue-155033.dR3pLc.rst | 2 + Modules/_csv.c | 25 +++++++++ 5 files changed, 105 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-01-12-00-00.gh-issue-155033.dR3pLc.rst diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 710a5b11ab64055..631b82ab9a335a8 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -543,6 +543,13 @@ Dialects support the following attributes: When ``True``, raise exception :exc:`Error` on bad CSV input. The default is ``False``. +Dialects support :func:`copy.replace`, +which returns a copy of the dialect +with the specified formatting parameters replaced. + +.. versionchanged:: next + Added support for :func:`copy.replace`. + .. _reader-objects: Reader Objects diff --git a/Lib/csv.py b/Lib/csv.py index c66717dc1ee59e7..0c5c40c759240ca 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -82,6 +82,11 @@ class excel: "unix_dialect"] +_dialect_attributes = frozenset({ + 'delimiter', 'quotechar', 'escapechar', 'doublequote', + 'skipinitialspace', 'lineterminator', 'quoting', 'strict', +}) + class Dialect: """Describe a CSV dialect. @@ -113,6 +118,17 @@ def _validate(self): # Re-raise to get a traceback showing more user code. raise Error(str(e)) from None + def __replace__(self, /, **changes): + unexpected = changes.keys() - _dialect_attributes + if unexpected: + raise TypeError(f'__replace__() got an unexpected keyword ' + f'argument {min(unexpected)!r}') + new = object.__new__(self.__class__) + new.__dict__.update(self.__dict__) + new.__dict__.update(changes) + new._validate() + return new + class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = ',' diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 91170cc16b3ac95..7a679d627ff8c08 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -707,6 +707,61 @@ def test_copy(self): dialect = csv.get_dialect(name) self.assertRaises(TypeError, copy.copy, dialect) + def test_replace(self): + dialect = csv.get_dialect('excel') + new = copy.replace(dialect, delimiter=';', strict=True) + self.assertIsInstance(new, type(dialect)) + self.assertEqual(new.delimiter, ';') + self.assertTrue(new.strict) + # Not replaced parameters are inherited from the original dialect. + self.assertEqual(new.quotechar, dialect.quotechar) + self.assertEqual(new.escapechar, dialect.escapechar) + self.assertEqual(new.lineterminator, dialect.lineterminator) + self.assertEqual(new.quoting, dialect.quoting) + self.assertEqual(new.doublequote, dialect.doublequote) + self.assertEqual(new.skipinitialspace, dialect.skipinitialspace) + # The original dialect is left unchanged. + self.assertEqual(dialect.delimiter, ',') + self.assertFalse(dialect.strict) + self.assertEqual(list(csv.reader(['a;b'], new)), [['a', 'b']]) + + self.assertIs(copy.replace(dialect), dialect) + self.assertRaises(TypeError, copy.replace, dialect, delimeter=';') + self.assertRaises(TypeError, copy.replace, dialect, delimiter=';;') + self.assertRaises(TypeError, dialect.__replace__, dialect) + + def test_replace_dialect_subclass(self): + class mydialect(csv.Dialect): + delimiter = ";" + quotechar = '"' + doublequote = False + skipinitialspace = True + lineterminator = '\r\n' + quoting = csv.QUOTE_ALL + + dialect = mydialect() + new = copy.replace(dialect, delimiter=':', quoting=csv.QUOTE_MINIMAL) + self.assertIsInstance(new, mydialect) + self.assertEqual(new.delimiter, ':') + self.assertEqual(new.quoting, csv.QUOTE_MINIMAL) + # Not replaced parameters are inherited from the original dialect. + self.assertEqual(new.quotechar, '"') + self.assertEqual(new.escapechar, None) + self.assertEqual(new.lineterminator, '\r\n') + self.assertFalse(new.doublequote) + self.assertTrue(new.skipinitialspace) + # The original dialect is left unchanged. + self.assertEqual(dialect.delimiter, ';') + self.assertEqual(dialect.quoting, csv.QUOTE_ALL) + self.assertEqual(list(csv.reader(['a:b'], new)), [['a', 'b']]) + # "strict" is supported even if it is not set on the class. + self.assertTrue(copy.replace(dialect, strict=True).strict) + + with self.assertRaises(csv.Error): + copy.replace(dialect, delimiter='::') + with self.assertRaisesRegex(TypeError, "'delimeter'"): + copy.replace(dialect, delimeter=':') + def test_pickle(self): for name in csv.list_dialects(): dialect = csv.get_dialect(name) diff --git a/Misc/NEWS.d/next/Library/2026-08-01-12-00-00.gh-issue-155033.dR3pLc.rst b/Misc/NEWS.d/next/Library/2026-08-01-12-00-00.gh-issue-155033.dR3pLc.rst new file mode 100644 index 000000000000000..9100d7be447e416 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-01-12-00-00.gh-issue-155033.dR3pLc.rst @@ -0,0 +1,2 @@ +CSV dialects (instances of :class:`csv.Dialect` subclasses and dialect +objects returned by :func:`csv.get_dialect`) now support :func:`copy.replace`. diff --git a/Modules/_csv.c b/Modules/_csv.c index a7fcc78e058f058..c640f2d36a84647 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -586,9 +586,34 @@ Dialect_reduce(PyObject *self, PyObject *args) { return NULL; } +PyDoc_STRVAR(dialect_replace_doc, +"__replace__($self, /, **changes)\n" +"--\n" +"\n" +"Return a copy of the dialect with the specified options replaced."); + +static PyObject * +Dialect_replace(PyObject *self, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) != 0) { + PyErr_SetString(PyExc_TypeError, + "__replace__() takes no positional arguments"); + return NULL; + } + PyObject *newargs = PyTuple_Pack(1, self); + if (newargs == NULL) { + return NULL; + } + PyObject *result = dialect_new(Py_TYPE(self), newargs, kwargs); + Py_DECREF(newargs); + return result; +} + static struct PyMethodDef dialect_methods[] = { {"__reduce__", Dialect_reduce, METH_VARARGS, dialect_reduce_doc}, {"__reduce_ex__", Dialect_reduce, METH_VARARGS, dialect_reduce_doc}, + {"__replace__", _PyCFunction_CAST(Dialect_replace), + METH_VARARGS | METH_KEYWORDS, dialect_replace_doc}, {NULL, NULL} }; From e75ef6a768a439ee86aa997cd3ecb1ff73e3c3da Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 13:12:24 +0300 Subject: [PATCH 18/23] gh-155040: Support copy.replace() for tarfile.TarInfo (GH-155046) Co-Authored-By: Claude Opus 5 (1M context) --- Doc/library/tarfile.rst | 5 +++++ Lib/tarfile.py | 2 ++ Lib/test/test_tarfile.py | 11 +++++++++++ .../2026-08-01-19-01-00.gh-issue-155040.Tr1nFo.rst | 1 + 4 files changed, 19 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-08-01-19-01-00.gh-issue-155040.Tr1nFo.rst diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index fc352e901f31dc7..f19038d837bb526 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -963,6 +963,11 @@ A ``TarInfo`` object has the following public data attributes: If *deep* is false, the copy is shallow, i.e. ``pax_headers`` and any custom attributes are shared with the original ``TarInfo`` object. + This method is also used by :func:`copy.replace`. + + .. versionchanged:: next + Added support for :func:`copy.replace`. + A :class:`TarInfo` object also provides some convenient query methods: diff --git a/Lib/tarfile.py b/Lib/tarfile.py index dc5c3a59744cbc4..592c4638c52c9b1 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1036,6 +1036,8 @@ def replace(self, *, result.gname = gname return result + __replace__ = replace + def get_info(self): """Return the TarInfo's attributes as a dictionary. """ diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 5fa97e2ac226c43..bd544dfea51da32 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1,3 +1,4 @@ +import copy import errno import sys import os @@ -3524,6 +3525,16 @@ def test_replace_internal(self): with self.assertRaises(TypeError): member.replace(offset=123456789) + def test_copy_replace(self): + member = self.tar.getmember('ustar/regtype') + replaced = copy.replace(member, name='misc/other', mode=0o644) + self.assertEqual(replaced.name, 'misc/other') + self.assertEqual(replaced.mode, 0o644) + self.assertEqual(replaced.size, member.size) + self.assertEqual(member.name, 'ustar/regtype') + with self.assertRaises(TypeError): + copy.replace(member, offset=123456789) + class NoneInfoExtractTests(ReadTest): # These mainly check that all kinds of members are extracted successfully diff --git a/Misc/NEWS.d/next/Library/2026-08-01-19-01-00.gh-issue-155040.Tr1nFo.rst b/Misc/NEWS.d/next/Library/2026-08-01-19-01-00.gh-issue-155040.Tr1nFo.rst new file mode 100644 index 000000000000000..d7875404202912d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-01-19-01-00.gh-issue-155040.Tr1nFo.rst @@ -0,0 +1 @@ +:class:`tarfile.TarInfo` objects now support :func:`copy.replace`. From 1ed6b78232a8739aa3b1ee1ce7ce2e1d998e3607 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 13:15:29 +0300 Subject: [PATCH 19/23] gh-154139: Document that curses is not thread-safe (GH-154173) Whether curses is thread-safe depends on the library and how it was built. The blocking and refresh methods release the GIL, so with a non-reentrant curses, unsynchronized use from several threads can crash. Co-authored-by: Claude Opus 4.8 --- Doc/library/curses.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index a833914a5d56d44..04cf2e17d538fcc 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -31,6 +31,19 @@ Linux and the BSD variants of Unix. Whenever the documentation mentions a *character string* it can be specified as a Unicode string or a byte string. +.. note:: + + Whether curses may be used from several threads + depends on the underlying library and how it was built. + In many implementations, including the default build of ncurses, + the screen state is shared and not thread-safe; + since the blocking and refresh methods + (such as :meth:`~window.getch` and :meth:`~window.refresh`) + release the :term:`GIL`, + unsynchronized use from several threads can then crash the interpreter. + Serialize the calls, + or wrap them in :meth:`window.use` and :meth:`screen.use`. + .. seealso:: Module :mod:`curses.ascii` From 31577863cb902717b735e5876020732ba52520b4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Aug 2026 13:21:03 +0300 Subject: [PATCH 20/23] gh-75008: Detect the lineterminator in csv.Sniffer.sniff() (GH-155061) It is guessed by a majority vote among the line endings of the sample, instead of always being '\r\n'. --- Doc/library/csv.rst | 7 +++- Doc/whatsnew/3.16.rst | 5 +++ Lib/csv.py | 20 ++++++++++- Lib/test/test_csv.py | 33 +++++++++++++++++++ ...6-08-01-23-50-47.gh-issue-75008.0Boa3r.rst | 2 ++ 5 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-01-23-50-47.gh-issue-75008.0Boa3r.rst diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 631b82ab9a335a8..53288e810bffcf6 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -328,10 +328,15 @@ The :mod:`!csv` module defines the following classes: are preferred, in that order, no matter how many times each of them occurs. + The *lineterminator* parameter is deduced separately, + by a majority vote among the line endings of the sample. + A tie is broken in the order ``'\r\n'``, ``'\n'``, ``'\r'``, + so a sample without a complete line gives ``'\r\n'``. + .. versionchanged:: next The dialect is now deduced by trial parsing and the results may differ from those of earlier Python versions. - The *escapechar* parameter can now be detected, + The *escapechar* and *lineterminator* parameters can now be detected, and the requested *delimiters* are not restricted to ASCII. diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index b017535b96979d9..fc09afc96b16763 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -135,6 +135,11 @@ csv The results may differ from those of earlier Python versions. (Contributed by Serhiy Storchaka in :gh:`83273`.) +* :meth:`csv.Sniffer.sniff` now detects the *lineterminator* parameter + by a majority vote among the line endings of the sample, + instead of always returning ``'\r\n'``. + (Contributed by Serhiy Storchaka in :gh:`75008`.) + curses ------ diff --git a/Lib/csv.py b/Lib/csv.py index 0c5c40c759240ca..b406d82efffd9f3 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -389,9 +389,9 @@ def sniff(self, sample, delimiters=None): class dialect(Dialect): _name = "sniffed" - lineterminator = '\r\n' quoting = QUOTE_MINIMAL + dialect.lineterminator = self._detect_lineterminator(lines) dialect.delimiter = delimiter # _csv.reader won't accept a quotechar of '' dialect.quotechar = quotechar or '"' @@ -614,6 +614,24 @@ def _detect_skipinitialspace(self, lines, delimiter, quotechar, for kept_row, skipped_row in zip(*results)] return all(first) or not any(first) + def _detect_lineterminator(self, lines): + """ + Detect the line terminator by majority vote among the line + endings. A line break inside a quoted field is counted too, + but it takes more of them than of the real ones to win the + vote. A tie is broken in the order '\\r\\n', '\\n', '\\r', + so a sample without a complete line gives '\\r\\n'. + """ + counts = dict.fromkeys(('\r\n', '\n', '\r'), 0) + for line in lines: + for lineterminator in counts: + if line.endswith(lineterminator): + counts[lineterminator] += 1 + break + # max() returns the first of equal candidates, and dict + # preserves the insertion order. + return max(counts, key=counts.get) + def has_header(self, sample): # Creates a dictionary of types of data in each column. If any # column is of a single type (say, integers), *except* for the first diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 7a679d627ff8c08..73e282d1abf7177 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1699,6 +1699,39 @@ def test_sniff_crlf_lineterminator(self): dialect = sniffer.sniff(sample) self.assertEqual(dialect.delimiter, ',') self.assertEqual(dialect.quotechar, '"') + self.assertEqual(dialect.lineterminator, '\r\n') + + def test_sniff_lineterminator(self): + sniffer = csv.Sniffer() + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + sample = lineterminator.join(['a,b,c', 'd,e,f', 'g,h,i', '']) + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.lineterminator, lineterminator) + self.assertEqual(dialect.delimiter, ',') + # The majority wins. + sample = 'a,b,c\nd,e,f\r\ng,h,i\n' + self.assertEqual(sniffer.sniff(sample).lineterminator, '\n') + sample = 'a,b,c\r\nd,e,f\ng,h,i\r\n' + self.assertEqual(sniffer.sniff(sample).lineterminator, '\r\n') + # A line break inside a quoted field is counted too, but it is + # outvoted by the real ones. + sample = 'a,"x\ny",c\r\nd,e,f\r\ng,h,i\r\n' + self.assertEqual(sniffer.sniff(sample).lineterminator, '\r\n') + + def test_sniff_lineterminator_tie(self): + # A tie is broken in the order '\r\n', '\n', '\r'. + sniffer = csv.Sniffer() + for sample, lineterminator in ( + ('a,b,c\nd,e,f\r\n', '\r\n'), + ('a,b,c\r\nd,e,f\ng,h,i\rj,k,l', '\r\n'), + ('a,b,c\nd,e,f\rg,h,i', '\n'), + # A sample without a complete line is a tie of zeros. + ('a,b,c', '\r\n'), + ): + with self.subTest(sample=sample): + self.assertEqual(sniffer.sniff(sample).lineterminator, + lineterminator) def test_sniff_excel_tab_with_quotes(self): # gh-62029: tab-delimited data with a quoted field containing diff --git a/Misc/NEWS.d/next/Library/2026-08-01-23-50-47.gh-issue-75008.0Boa3r.rst b/Misc/NEWS.d/next/Library/2026-08-01-23-50-47.gh-issue-75008.0Boa3r.rst new file mode 100644 index 000000000000000..19545f041093bd4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-01-23-50-47.gh-issue-75008.0Boa3r.rst @@ -0,0 +1,2 @@ +:meth:`csv.Sniffer.sniff` now detects the *lineterminator* parameter by a +majority vote among the line endings of the sample. From 76f2903865e39e85b6c152114d9522d99969c09d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Aug 2026 12:30:01 +0200 Subject: [PATCH 21/23] gh-155503: Check Py_TPFLAGS_IMMUTABLETYPE in check_immutable_type() (#155620) Check for the Py_TPFLAGS_IMMUTABLETYPE type flag in test.support.check_immutable_type(). --- Lib/test/support/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f98da49171dc3b5..3f2caebd21e336c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3484,3 +3484,11 @@ def check_immutable_type(testcase, type): regex = r'cannot set .* attribute of immutable type' with testcase.assertRaisesRegex(TypeError, regex): setattr(type, 'custom_attr', 123) + + try: + from _testlimitedcapi import type_getflags, Py_TPFLAGS_IMMUTABLETYPE + except ImportError: + pass + else: + flags = type_getflags(type) + testcase.assertTrue(flags & Py_TPFLAGS_IMMUTABLETYPE) From 1c9521f48511e6eb974057d4a1ddc49fa50736d1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Aug 2026 12:44:45 +0200 Subject: [PATCH 22/23] gh-153400: Use glibc functions instead of syscall() (#155518) Use glibc functions instead of syscall(): pidfd_open(), pidfd_getfd() and pidfd_send_signal() (glibc 2.36), gettid() and getdents64() (glibc 2.30), and getrandom() (glibc 2.25). Use unsigned int for os.getrandom() flags and signal.pidfd_send_signal() flags. --- ...-08-10-22-11-15.gh-issue-153400.Ds3GI3.rst | 4 +++ Modules/_posixsubprocess.c | 32 ++++++++++++----- Modules/clinic/posixmodule.c.h | 36 ++++++++++++------- Modules/clinic/signalmodule.c.h | 28 ++++++++++----- Modules/posixmodule.c | 36 ++++++++++++++----- Modules/signalmodule.c | 22 ++++++++---- Python/bootstrap_hash.c | 3 -- Python/perf_jit_trampoline.c | 2 ++ Python/thread_pthread.h | 7 +++- configure | 30 ++++++++++++++++ configure.ac | 7 ++-- pyconfig.h.in | 15 ++++++++ 12 files changed, 171 insertions(+), 51 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-10-22-11-15.gh-issue-153400.Ds3GI3.rst diff --git a/Misc/NEWS.d/next/Library/2026-08-10-22-11-15.gh-issue-153400.Ds3GI3.rst b/Misc/NEWS.d/next/Library/2026-08-10-22-11-15.gh-issue-153400.Ds3GI3.rst new file mode 100644 index 000000000000000..89792726cfe8314 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-10-22-11-15.gh-issue-153400.Ds3GI3.rst @@ -0,0 +1,4 @@ +:mod:`os` and :mod:`signal`: Use glibc functions instead of ``syscall()``: +``pidfd_open()``, ``pidfd_getfd()`` and ``pidfd_send_signal()`` (glibc 2.36), +``gettid()`` and ``getdents64()`` (glibc 2.30), and ``getrandom()`` (glibc +2.25). Patch by Victor Stinner. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 2aa3923f68e66ad..5d8ee661fa3b6de 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -388,20 +388,26 @@ _close_range_except(int start_fd, return 0; } -#if defined(__linux__) && defined(HAVE_SYS_SYSCALL_H) +#if defined(HAVE_GETDENTS64) \ + || (defined(__linux__) && defined(HAVE_SYS_SYSCALL_H)) + +#ifdef HAVE_GETDENTS64 +# define py_dirent64 dirent64 +#else /* It doesn't matter if d_name has room for NAME_MAX chars; we're using this * only to read a directory of short file descriptor number names. The kernel * will return an error if we didn't give it enough space. Highly Unlikely. * This structure is very old and stable: It will not change unless the kernel * chooses to break compatibility with all existing binaries. Highly Unlikely. */ -struct linux_dirent64 { +struct py_dirent64 { unsigned long long d_ino; long long d_off; unsigned short d_reclen; /* Length of this linux_dirent */ unsigned char d_type; char d_name[256]; /* Filename (null-terminated) */ }; +#endif // !HAVE_GETDENTS64 static int _brute_force_closer(int first, int last) @@ -441,19 +447,27 @@ _close_open_fds_safe(int start_fd, int *fds_to_keep, Py_ssize_t fds_to_keep_len) _brute_force_closer); return; } else { - char buffer[sizeof(struct linux_dirent64)]; - int bytes; - while ((bytes = syscall(SYS_getdents64, fd_dir_fd, - (struct linux_dirent64 *)buffer, - sizeof(buffer))) > 0) { - struct linux_dirent64 *entry; + char buffer[sizeof(struct py_dirent64)]; + Py_ssize_t bytes; + while (1) { +#ifdef HAVE_GETDENTS64 + bytes = getdents64(fd_dir_fd, buffer, sizeof(buffer)); +#else + bytes = syscall(SYS_getdents64, fd_dir_fd, + (struct py_dirent64 *)buffer, sizeof(buffer)); +#endif + if (bytes <= 0) { + break; + } + + struct py_dirent64 *entry; int offset; #ifdef _Py_MEMORY_SANITIZER __msan_unpoison(buffer, bytes); #endif for (offset = 0; offset < bytes; offset += entry->d_reclen) { int fd; - entry = (struct linux_dirent64 *)(buffer + offset); + entry = (struct py_dirent64 *)(buffer + offset); if ((fd = _pos_int_from_ascii(entry->d_name)) < 0) continue; /* Not a number. */ if (fd != fd_dir_fd && fd >= start_fd && diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index ac9b63dec9eb440..a6092dd9f5638f6 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -6330,7 +6330,7 @@ os_wait(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_WAIT) */ -#if (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) +#if (defined(HAVE_PIDFD_OPEN) || (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) PyDoc_STRVAR(os_pidfd_open__doc__, "pidfd_open($module, /, pid, flags=0)\n" @@ -6405,9 +6405,9 @@ os_pidfd_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ +#endif /* (defined(HAVE_PIDFD_OPEN) || (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) */ -#if (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) +#if (defined(HAVE_PIDFD_GETFD) || (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) PyDoc_STRVAR(os_pidfd_getfd__doc__, "pidfd_getfd($module, /, pidfd, targetfd, *, flags=0)\n" @@ -6492,7 +6492,7 @@ os_pidfd_getfd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ +#endif /* (defined(HAVE_PIDFD_GETFD) || (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) */ #if defined(HAVE_SETNS) @@ -12578,7 +12578,7 @@ os_fspath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k return return_value; } -#if defined(HAVE_GETRANDOM_SYSCALL) +#if (defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL)) PyDoc_STRVAR(os_getrandom__doc__, "getrandom($module, /, size, flags=0)\n" @@ -12590,7 +12590,7 @@ PyDoc_STRVAR(os_getrandom__doc__, {"getrandom", _PyCFunction_CAST(os_getrandom), METH_FASTCALL|METH_KEYWORDS, os_getrandom__doc__}, static PyObject * -os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags); +os_getrandom_impl(PyObject *module, Py_ssize_t size, unsigned int flags); static PyObject * os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -12626,7 +12626,7 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_ssize_t size; - int flags = 0; + unsigned int flags = 0; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -12648,9 +12648,21 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!noptargs) { goto skip_optional_pos; } - flags = PyLong_AsInt(args[1]); - if (flags == -1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &flags, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional_pos: return_value = os_getrandom_impl(module, size, flags); @@ -12659,7 +12671,7 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject return return_value; } -#endif /* defined(HAVE_GETRANDOM_SYSCALL) */ +#endif /* (defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL)) */ #if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) @@ -13734,4 +13746,4 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=d641f02a97057666 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6dc1e061bfd47375 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index ca47033446074cd..6125f253ac0380d 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -689,7 +689,7 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_PTHREAD_KILL) */ -#if (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) +#if (defined(HAVE_PIDFD_SEND_SIGNAL) || (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) PyDoc_STRVAR(signal_pidfd_send_signal__doc__, "pidfd_send_signal($module, pidfd, signalnum, siginfo=None, flags=0, /)\n" @@ -702,7 +702,7 @@ PyDoc_STRVAR(signal_pidfd_send_signal__doc__, static PyObject * signal_pidfd_send_signal_impl(PyObject *module, int pidfd, int signalnum, - PyObject *siginfo, int flags); + PyObject *siginfo, unsigned int flags); static PyObject * signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -711,7 +711,7 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar int pidfd; int signalnum; PyObject *siginfo = Py_None; - int flags = 0; + unsigned int flags = 0; if (!_PyArg_CheckPositional("pidfd_send_signal", nargs, 2, 4)) { goto exit; @@ -731,9 +731,21 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 4) { goto skip_optional; } - flags = PyLong_AsInt(args[3]); - if (flags == -1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[3], &flags, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: return_value = signal_pidfd_send_signal_impl(module, pidfd, signalnum, siginfo, flags); @@ -742,7 +754,7 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ +#endif /* (defined(HAVE_PIDFD_SEND_SIGNAL) || (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) */ #ifndef SIGNAL_ALARM_METHODDEF #define SIGNAL_ALARM_METHODDEF @@ -795,4 +807,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=0731d6f05c42c09a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2a04ec31f49b1c93 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9e84fd400527ea2..a7208aeef1871ee 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -68,6 +68,10 @@ # include "emscripten.h" // emscripten_debugger() #endif +#ifdef HAVE_SYS_RANDOM_H +# include // getrandom() +#endif + #ifdef HAVE_SYS_UIO_H # include #endif @@ -10810,8 +10814,9 @@ os_wait_impl(PyObject *module) // This system call always crashes on older Android versions. -#if defined(__linux__) && defined(__NR_pidfd_open) && \ - !(defined(__ANDROID__) && __ANDROID_API__ < 31) +#if defined(HAVE_PIDFD_OPEN) \ + || (defined(__linux__) && defined(__NR_pidfd_open) \ + && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) /*[clinic input] os.pidfd_open pid: pid_t @@ -10827,7 +10832,11 @@ static PyObject * os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) /*[clinic end generated code: output=5c7252698947dc41 input=03058b32c389f874]*/ { +#ifdef HAVE_PIDFD_OPEN + int fd = pidfd_open(pid, flags); +#else int fd = syscall(__NR_pidfd_open, pid, flags); +#endif if (fd < 0) { return posix_error(); } @@ -10836,8 +10845,9 @@ os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) #endif -#if defined(__linux__) && defined(__NR_pidfd_getfd) && \ - !(defined(__ANDROID__) && __ANDROID_API__ < 31) +#if defined(HAVE_PIDFD_GETFD) \ + || (defined(__linux__) && defined(__NR_pidfd_getfd) \ + && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) /*[clinic input] os.pidfd_getfd pidfd: int @@ -10856,7 +10866,11 @@ os_pidfd_getfd_impl(PyObject *module, int pidfd, int targetfd, unsigned int flags) /*[clinic end generated code: output=e1a1415a13c7137f input=ef6417fb10deb1cc]*/ { +#ifdef HAVE_PIDFD_GETFD + int fd = pidfd_getfd(pidfd, targetfd, flags); +#else int fd = syscall(__NR_pidfd_getfd, pidfd, targetfd, flags); +#endif if (fd < 0) { return posix_error(); } @@ -17376,19 +17390,19 @@ os_fspath_impl(PyObject *module, PyObject *path) return PyOS_FSPath(path); } -#ifdef HAVE_GETRANDOM_SYSCALL +#if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL) /*[clinic input] os.getrandom size: Py_ssize_t - flags: int=0 + flags: unsigned_int(bitwise=True) = 0 Obtain a series of random bytes. [clinic start generated code]*/ static PyObject * -os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) -/*[clinic end generated code: output=b3a618196a61409c input=59bafac39c594947]*/ +os_getrandom_impl(PyObject *module, Py_ssize_t size, unsigned int flags) +/*[clinic end generated code: output=c2163c05f0e1d0a1 input=e0174983f5703f82]*/ { if (size < 0) { errno = EINVAL; @@ -17403,7 +17417,11 @@ os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) Py_ssize_t n; while (1) { +#ifdef HAVE_GETRANDOM + n = getrandom(data, size, flags); +#else n = syscall(SYS_getrandom, data, size, flags); +#endif if (n < 0 && errno == EINTR) { if (PyErr_CheckSignals() < 0) { goto error; @@ -18511,7 +18529,7 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, RTLD_MEMBER)) return -1; #endif -#ifdef HAVE_GETRANDOM_SYSCALL +#if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL) if (PyModule_AddIntMacro(m, GRND_RANDOM)) return -1; if (PyModule_AddIntMacro(m, GRND_NONBLOCK)) return -1; #endif diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 8456239dee202d3..bc5aef55648e071 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -53,6 +53,10 @@ # include #endif +#ifdef HAVE_SYS_PIDFD_H +# include // pidfd_send_signal() +#endif + #ifndef SIG_ERR # define SIG_ERR ((PyOS_sighandler_t)(-1)) #endif @@ -1300,15 +1304,16 @@ signal_pthread_kill_impl(PyObject *module, unsigned long thread_id, // This system call always crashes on older Android versions. -#if defined(__linux__) && defined(__NR_pidfd_send_signal) && \ - !(defined(__ANDROID__) && __ANDROID_API__ < 31) +#if defined(HAVE_PIDFD_SEND_SIGNAL) \ + || (defined(__linux__) && defined(__NR_pidfd_send_signal) \ + && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) /*[clinic input] signal.pidfd_send_signal pidfd: int signalnum: int siginfo: object = None - flags: int = 0 + flags: unsigned_int(bitwise=True) = 0 / Send a signal to a process referred to by a pid file descriptor. @@ -1316,15 +1321,20 @@ Send a signal to a process referred to by a pid file descriptor. static PyObject * signal_pidfd_send_signal_impl(PyObject *module, int pidfd, int signalnum, - PyObject *siginfo, int flags) -/*[clinic end generated code: output=2d59f04a75d9cbdf input=2a6543a1f4ac2000]*/ + PyObject *siginfo, unsigned int flags) +/*[clinic end generated code: output=1804b5a19d269104 input=a6e82a3c264fa19d]*/ { if (siginfo != Py_None) { PyErr_SetString(PyExc_TypeError, "siginfo must be None"); return NULL; } - if (syscall(__NR_pidfd_send_signal, pidfd, signalnum, NULL, flags) < 0) { +#ifdef HAVE_PIDFD_SEND_SIGNAL + int res = pidfd_send_signal(pidfd, signalnum, NULL, flags); +#else + int res = syscall(__NR_pidfd_send_signal, pidfd, signalnum, NULL, flags); +#endif + if (res < 0) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index f0fb87c4a5d15e2..5bc45d503360376 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -126,9 +126,6 @@ py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise) n = getrandom(dest, n, flags); } #else - /* On Linux, use the syscall() function because the GNU libc doesn't - expose the Linux getrandom() syscall yet. See: - https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */ if (raise) { Py_BEGIN_ALLOW_THREADS n = syscall(SYS_getrandom, dest, n, flags); diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 32b147199544cfc..21beead742a3a25 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -672,6 +672,8 @@ static void perf_map_jit_write_entry_with_name( uint64_t thread_id = 0; pthread_threadid_np(NULL, &thread_id); ev.thread_id = (uint32_t)thread_id; +#elif defined(HAVE_GETTID) + ev.thread_id = gettid(); #else ev.thread_id = syscall(SYS_gettid); // Get thread ID via system call #endif diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index de93178576e6ec7..8d727326faeb70d 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -18,7 +18,9 @@ #include #include /* pause(), also getthrid() on OpenBSD */ -#if defined(__linux__) +#ifdef HAVE_GETTID +# include // gettid() +#elif defined(__linux__) # include /* syscall(SYS_gettid) */ #elif defined(__FreeBSD__) # include /* pthread_getthreadid_np() */ @@ -380,6 +382,9 @@ PyThread_get_thread_native_id(void) #ifdef __APPLE__ uint64_t native_id; (void) pthread_threadid_np(NULL, &native_id); +#elif defined(HAVE_GETTID) + pid_t native_id; + native_id = gettid(); #elif defined(__linux__) pid_t native_id; native_id = syscall(SYS_gettid); diff --git a/configure b/configure index e81b221faded675..598a21fb31f7501 100755 --- a/configure +++ b/configure @@ -20522,6 +20522,12 @@ if test "x$ac_cv_func_gai_strerror" = xyes then : printf "%s\n" "#define HAVE_GAI_STRERROR 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "getdents64" "ac_cv_func_getdents64" +if test "x$ac_cv_func_getdents64" = xyes +then : + printf "%s\n" "#define HAVE_GETDENTS64 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "getegid" "ac_cv_func_getegid" if test "x$ac_cv_func_getegid" = xyes @@ -20696,6 +20702,12 @@ if test "x$ac_cv_func_getspnam" = xyes then : printf "%s\n" "#define HAVE_GETSPNAM 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "gettid" "ac_cv_func_gettid" +if test "x$ac_cv_func_gettid" = xyes +then : + printf "%s\n" "#define HAVE_GETTID 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "getuid" "ac_cv_func_getuid" if test "x$ac_cv_func_getuid" = xyes @@ -20864,6 +20876,24 @@ if test "x$ac_cv_func_pause" = xyes then : printf "%s\n" "#define HAVE_PAUSE 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "pidfd_open" "ac_cv_func_pidfd_open" +if test "x$ac_cv_func_pidfd_open" = xyes +then : + printf "%s\n" "#define HAVE_PIDFD_OPEN 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "pidfd_getfd" "ac_cv_func_pidfd_getfd" +if test "x$ac_cv_func_pidfd_getfd" = xyes +then : + printf "%s\n" "#define HAVE_PIDFD_GETFD 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "pidfd_send_signal" "ac_cv_func_pidfd_send_signal" +if test "x$ac_cv_func_pidfd_send_signal" = xyes +then : + printf "%s\n" "#define HAVE_PIDFD_SEND_SIGNAL 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "pipe" "ac_cv_func_pipe" if test "x$ac_cv_func_pipe" = xyes diff --git a/configure.ac b/configure.ac index 36568288388555a..47c5f7e49422a5d 100644 --- a/configure.ac +++ b/configure.ac @@ -5497,13 +5497,14 @@ AC_CHECK_FUNCS([ \ copy_file_range ctermid dladdr dup execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ - gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ + gai_strerror getdents64 getegid geteuid getgid getgrent getgrgid getgrgid_r \ getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin getlogin_r \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ - getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ + getspnam gettid getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ - mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ + mknod mknodat mktime mmap mremap nice openat opendir pathconf pause \ + pidfd_open pidfd_getfd pidfd_send_signal pipe \ plock poll ppoll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ pread preadv preadv2 process_vm_readv \ diff --git a/pyconfig.h.in b/pyconfig.h.in index 691c6c0d9feb6d0..64b5c52790b4581 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -579,6 +579,9 @@ /* Define this if you have flockfile(), getc_unlocked(), and funlockfile() */ #undef HAVE_GETC_UNLOCKED +/* Define to 1 if you have the 'getdents64' function. */ +#undef HAVE_GETDENTS64 + /* Define to 1 if you have the 'getegid' function. */ #undef HAVE_GETEGID @@ -714,6 +717,9 @@ /* Define to 1 if you have the 'getspnam' function. */ #undef HAVE_GETSPNAM +/* Define to 1 if you have the 'gettid' function. */ +#undef HAVE_GETTID + /* Define to 1 if you have the 'getuid' function. */ #undef HAVE_GETUID @@ -1046,6 +1052,15 @@ /* Define to 1 if you have the 'pause' function. */ #undef HAVE_PAUSE +/* Define to 1 if you have the 'pidfd_getfd' function. */ +#undef HAVE_PIDFD_GETFD + +/* Define to 1 if you have the 'pidfd_open' function. */ +#undef HAVE_PIDFD_OPEN + +/* Define to 1 if you have the 'pidfd_send_signal' function. */ +#undef HAVE_PIDFD_SEND_SIGNAL + /* Define to 1 if you have the 'pipe' function. */ #undef HAVE_PIPE From 716cbae06c7d9d641626dfdb783f3959edf470a4 Mon Sep 17 00:00:00 2001 From: Hai Zhu Date: Thu, 13 Aug 2026 19:23:45 +0800 Subject: [PATCH 23/23] gh-154701: prevent executor self-links in JIT cold exits (GH-155323) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * prevent executor self-links in JIT cold exits * 📜🤖 Added by blurb_it. * fix windows ci --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Include/internal/pycore_optimizer.h | 2 +- Include/internal/pycore_uop_metadata.h | 2 +- Lib/test/test_capi/test_opt.py | 24 ++++++++++++++++++- ...-08-07-10-14-56.gh-issue-154701.zulh2S.rst | 1 + Python/bytecodes.c | 4 ++++ Python/executor_cases.c.h | 8 +++++++ 6 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-07-10-14-56.gh-issue-154701.zulh2S.rst diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 3d60638649dcb5a..9f4f8918a40d1ad 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -206,7 +206,7 @@ typedef struct _PyExecutorObject { PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset); int _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); -void _Py_ExecutorDetach(_PyExecutorObject *); +PyAPI_FUNC(void) _Py_ExecutorDetach(_PyExecutorObject *); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); /* We use a bloomfilter with k = 6, m = 256 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index e52233b21277591..02e755330d6c209 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -408,7 +408,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_SYNC_SP_FLAG, [_SPILL_OR_RELOAD] = 0, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, - [_COLD_EXIT] = HAS_SYNC_SP_FLAG, + [_COLD_EXIT] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_COLD_DYNAMIC_EXIT] = HAS_SYNC_SP_FLAG, [_GUARD_CODE_VERSION__PUSH_FRAME] = HAS_EXIT_FLAG, [_GUARD_CODE_VERSION_YIELD_VALUE] = HAS_EXIT_FLAG, diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 5806216d46e7eb6..d5a94ef69c14de0 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -12,7 +12,7 @@ from test.support import (script_helper, requires_specialization, import_helper, Py_GIL_DISABLED, requires_jit_enabled, - reset_code) + reset_code, SHORT_TIMEOUT, isolation) _testinternalcapi = import_helper.import_module("_testinternalcapi") @@ -6225,6 +6225,28 @@ def __exit__(self, e, v, t): ... f1() """), PYTHON_JIT="1") + @isolation.runInSubprocess(timeout=SHORT_TIMEOUT) + def test_for_iter_side_exit_does_not_self_link(self): + def exhaust(iterator): + for _ in iterator: + pass + + values = range(TIER2_THRESHOLD) + # After the initial trace, MAX_CHAIN_DEPTH side exits cause the final + # executor to be installed at FOR_ITER. + warmup_iterators = ( + iter(set(values)), + iter(dict.fromkeys(values)), + iter(values), + enumerate(values), + zip(values, values), + ) + for iterator in warmup_iterators: + exhaust(iterator) + + # A different iterator type must not link that executor to itself. + exhaust(map(bool, values)) + def global_identity(x): return x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-07-10-14-56.gh-issue-154701.zulh2S.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-07-10-14-56.gh-issue-154701.zulh2S.rst new file mode 100644 index 000000000000000..c32803e71cbf993 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-07-10-14-56.gh-issue-154701.zulh2S.rst @@ -0,0 +1 @@ +Fix an infinite loop in JIT when a ``FOR_ITER`` side exit links an executor back to itself. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4d7b338e2dbd4c3..d657ae579a8f986 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -6262,6 +6262,10 @@ dummy_func( if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; + if (executor == _PyExecutor_FromExit(exit)) { + _Py_ExecutorDetach(executor); + GOTO_TIER_ONE(target); + } Py_INCREF(executor); assert(tstate->jit_exit == exit); exit->executor = executor; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e45bbd7cceb295f..46e721ea34b6b6e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -23789,6 +23789,14 @@ if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; + if (executor == _PyExecutor_FromExit(exit)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_ExecutorDetach(executor); + _PyFrame_StackPointerInvalidate(frame); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(target); + } Py_INCREF(executor); assert(tstate->jit_exit == exit); exit->executor = executor;