Merge remote-tracking branch 'upstream/main' into fix-issue-37858-latest

Author: serhiy-storchaka

Doc/library/concurrent.futures.rst

@@ -61,11 +61,18 @@ Executor Objects
       The returned iterator raises a :exc:`TimeoutError`
       if :meth:`~iterator.__next__` is called and the result isn't available
       after *timeout* seconds from the original call to :meth:`Executor.map`.
-      *timeout* can be an int or a float.  If *timeout* is not specified or
+      *timeout* can be an int or a float.
+      It cancels all future calls of *fn* and closes the iterator.
+      If *timeout* is not specified or
       ``None``, there is no limit to the wait time.
 
       If a *fn* call raises an exception, then that exception will be
       raised when its value is retrieved from the iterator.
+      It does not cancel future calls of *fn*.
+
+      The returned iterator has method :meth:`!close` which cancels all
+      future calls of *fn* and discards the results of already finished calls
+      if they are available.
 
       When using :class:`ProcessPoolExecutor`, this method chops *iterables*
       into a number of chunks which it submits to the pool as separate
@@ -82,6 +89,10 @@ Executor Objects
       .. versionchanged:: 3.14
          Added the *buffersize* parameter.
 
+      .. versionchanged:: next
+         The returned iterator is no longer automatically closed if a *fn*
+         call raises an exception.
+
    .. method:: shutdown(wait=True, *, cancel_futures=False)
 
       Signal the executor that it should free any resources that it is using

Doc/library/turtle.rst

@@ -79,8 +79,8 @@ In a Python shell, import all the objects of the ``turtle`` module::
 
     from turtle import *
 
-If you run into a ``No module named '_tkinter'`` error, you'll have to
-install the :mod:`Tk interface package <tkinter>` on your system.
+If you run into a ``Standard library module '_tkinter' was not found`` error,
+you'll have to install the :mod:`Tk interface package <tkinter>` on your system.
 
 
 Basic drawing
@@ -167,14 +167,16 @@ filling can be turned on and off::
 
 Next we'll create a loop::
 
+    start = pos()
+
     while True:
         forward(200)
         left(170)
-        if abs(pos()) < 1:
+        if distance(start) < 1:
             break
 
-``abs(pos()) < 1`` is a good way to know when the turtle is back at its
-home position.
+``distance(start) < 1`` is a good way to know when the turtle is back at its
+start position.
 
 Finally, complete the filling::
 

Doc/using/cmdline.rst

@@ -519,7 +519,25 @@ Miscellaneous options
 .. option:: -x
 
    Skip the first line of the source, allowing use of non-Unix forms of
-   ``#!cmd``.  This is intended for a DOS specific hack only.
+   ``#!cmd``.
+
+   This can be used to turn a Python script into a Windows batch file.
+   Similarly to adding a shebang line and setting the executable bit on Unix,
+   the extension of the Python script can be changed to ``.bat`` and the
+   following line can be added at the start of the script:
+
+   .. code-block:: batch
+
+      @py -x "%~f0" %* & exit /b
+
+   Or, to specify the path to the Python interpreter explicitly:
+
+   .. code-block:: batch
+
+      @"C:\Path\to\python.exe" -x "%~f0" %* & exit /b
+
+   Unlike a shebang line which is a Python comment, this line is not valid
+   Python syntax, and the :option:`-x` option is needed to skip it.
 
 
 .. option:: -X

Doc/whatsnew/3.16.rst

@@ -276,6 +276,15 @@ ctypes
   (Contributed by Peter Bierma in :gh:`153903`.)
 
 
+concurrent.futures
+------------------
+
+* The iterator returned by :meth:`concurrent.futures.Executor.map` is no longer
+  automatically closed if a function call raises an exception.
+  Use method :meth:`!close` to explicitly close the iterator.
+  (Contributed by xzmeng and Serhiy Storchaka in :gh:`108518`.)
+
+
 encodings
 ---------
 
@@ -819,6 +828,13 @@ that may require changes to your code.
   :exc:`TypeError`.
   (Contributed by Serhiy Storchaka in :gh:`152587`.)
 
+* On Windows, seeking a pipe now fails instead of silently appearing to
+  succeed: :func:`os.lseek` and :meth:`~io.IOBase.seek` raise :exc:`OSError`,
+  and :meth:`~io.IOBase.seekable` returns ``False``.  As a consequence,
+  opening a pipe in a read-write binary mode (``'r+b'`` or ``'w+b'``) now
+  raises :exc:`io.UnsupportedOperation` unless buffering is disabled.
+  (Contributed by An Long in :gh:`86768`.)
+
 
 Build changes
 =============

Lib/concurrent/futures/_base.py

@@ -309,7 +309,11 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
 def _result_or_cancel(fut, timeout=None):
     try:
         try:
-            return fut.result(timeout)
+            return (fut.result(timeout), None)
+        except TimeoutError:
+            raise
+        except BaseException as exc:
+            return (None, exc)
         finally:
             fut.cancel()
     finally:
@@ -592,6 +596,7 @@ def _get_snapshot(self):
 
     __class_getitem__ = classmethod(types.GenericAlias)
 
+
 class Executor(object):
     """This is an abstract base class for concrete asynchronous executors."""
 
@@ -638,7 +643,10 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None):
             raise TypeError("buffersize must be an integer or None")
         if buffersize is not None and buffersize < 1:
             raise ValueError("buffersize must be None or > 0")
+        return _MapResultIterator(self._map(fn, *iterables, timeout=timeout,
+                                            buffersize=buffersize))
 
+    def _map(self, fn, *iterables, timeout=None, buffersize=None):
         if timeout is not None:
             end_time = timeout + time.monotonic()
 
@@ -701,6 +709,24 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         return False
 
 
+class _MapResultIterator:
+    """The iterator returned by map()."""
+    def __init__(self, gen):
+        self.gen = gen
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        value, exc = next(self.gen)
+        if exc is not None:
+            raise exc
+        return value
+
+    def close(self):
+        self.gen.close()
+
+
 class BrokenExecutor(RuntimeError):
     """
     Raised when an executor has become non-functional after a severe failure.

Lib/concurrent/futures/process.py

@@ -200,7 +200,14 @@ def _process_chunk(fn, chunk):
     This function is run in a separate process.
 
     """
-    return [fn(*args) for args in chunk]
+    results = []
+    for args in chunk:
+        try:
+            result = (fn(*args), None)
+        except BaseException as exc:
+            result = (None, exc)
+        results.append(result)
+    return results
 
 
 def _sendback_result(result_queue, work_id, result=None, exception=None,
@@ -963,7 +970,7 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None):
                               itertools.batched(zip(*iterables), chunksize),
                               timeout=timeout,
                               buffersize=buffersize)
-        return _chain_from_iterable_of_lists(results)
+        return _base._MapResultIterator(_chain_from_iterable_of_lists(results))
 
     def shutdown(self, wait=True, *, cancel_futures=False):
         with self._shutdown_lock:

Lib/json/encoder.py

@@ -223,29 +223,6 @@ def iterencode(self, o, _one_shot=False):
         else:
             _encoder = encode_basestring
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=float.__repr__, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
-
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
-
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
-
-            return text
-
-
         if self.indent is None or isinstance(self.indent, str):
             indent = self.indent
         else:
@@ -256,6 +233,28 @@ def floatstr(o, allow_nan=self.allow_nan,
                 self.key_separator, self.item_separator, self.sort_keys,
                 self.skipkeys, self.allow_nan)
         else:
+            def floatstr(o, allow_nan=self.allow_nan,
+                    _repr=float.__repr__, _inf=INFINITY, _neginf=-INFINITY):
+                # Check for specials.  Note that this type of test is processor
+                # and/or platform-specific, so do tests which don't depend on
+                # the internals.
+
+                if o != o:
+                    text = 'NaN'
+                elif o == _inf:
+                    text = 'Infinity'
+                elif o == _neginf:
+                    text = '-Infinity'
+                else:
+                    return _repr(o)
+
+                if not allow_nan:
+                    raise ValueError(
+                        "Out of range float values are not JSON compliant: " +
+                        repr(o))
+
+                return text
+
             _iterencode = _make_iterencode(
                 markers, self.default, _encoder, indent, floatstr,
                 self.key_separator, self.item_separator, self.sort_keys,

Lib/test/test_array.py

@@ -209,6 +209,10 @@ def test_numbers(self):
              [-1<<63, (1<<63)-1, 0]),
             (['l'], SIGNED_INT64_BE, '>qqq',
              [-1<<63, (1<<63)-1, 0]),
+            (['e'], IEEE_754_FLOAT16_LE, '<eeee',
+             [1.0, float('inf'), float('-inf'), -0.0]),
+            (['e'], IEEE_754_FLOAT16_BE, '>eeee',
+             [1.0, float('inf'), float('-inf'), -0.0]),
             (['f'], IEEE_754_FLOAT_LE, '<ffff',
              [16711938.0, float('inf'), float('-inf'), -0.0]),
             (['f'], IEEE_754_FLOAT_BE, '>ffff',
@@ -239,6 +243,16 @@ def test_numbers(self):
                 self.assertEqual(a, b,
                     msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase))
 
+    def test_float16_endianness(self):
+        # gh-154568: array_reconstructor() slow-path decoder for
+        # IEEE_754_FLOAT16_LE ignored the encoding.
+        le_bytes = struct.pack('<e', 1.5)
+        be_bytes = struct.pack('>e', 1.5)
+        b_le = array_reconstructor(array.array, 'd', IEEE_754_FLOAT16_LE, le_bytes)
+        b_be = array_reconstructor(array.array, 'd', IEEE_754_FLOAT16_BE, be_bytes)
+        self.assertEqual(b_le.tolist(), [1.5])
+        self.assertEqual(b_be.tolist(), [1.5])
+
     def test_unicode(self):
         teststr = "Bonne Journ\xe9e \U0002030a\U00020347"
         testcases = (

Lib/test/test_concurrent_futures/executor.py

@@ -71,21 +71,30 @@ def test_map(self):
 
     @warnings_helper.ignore_fork_in_thread_deprecation_warnings()
     def test_map_exception(self):
-        i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5])
-        self.assertEqual(i.__next__(), (0, 1))
-        self.assertEqual(i.__next__(), (0, 1))
-        with self.assertRaises(ZeroDivisionError):
-            i.__next__()
+        i = self.executor.map(divmod, [5, 5, 5, 5], [2, 3, 0, 5])
+        self.assertEqual(next(i), (2, 1))
+        self.assertEqual(next(i), (1, 2))
+        self.assertRaises(ZeroDivisionError, next, i)
+        self.assertEqual(next(i), (1, 0))
+        self.assertRaises(StopIteration, next, i)
+        self.assertRaises(StopIteration, next, i)
+
+        i = self.executor.map(divmod, [5, 5, 5, 5], [2, 0, 3, 5], chunksize=3)
+        self.assertEqual(next(i), (2, 1))
+        self.assertRaises(ZeroDivisionError, next, i)
+        self.assertEqual(next(i), (1, 2))
+        self.assertEqual(next(i), (1, 0))
+        self.assertRaises(StopIteration, next, i)
+        self.assertRaises(StopIteration, next, i)
 
     @warnings_helper.ignore_fork_in_thread_deprecation_warnings()
     @support.requires_resource('walltime')
     def test_map_timeout(self):
         results = []
+        i = self.executor.map(time.sleep, [0, 0, 6], timeout=5)
         try:
-            for i in self.executor.map(time.sleep,
-                                       [0, 0, 6],
-                                       timeout=5):
-                results.append(i)
+            for result in i:
+                results.append(result)
         except futures.TimeoutError:
             pass
         else:
@@ -95,6 +104,24 @@ def test_map_timeout(self):
         # take longer than the specified timeout.
         self.assertIn(results, ([None, None], [None], []))
 
+        # The remaining calls are cancelled, so the iterator is exhausted.
+        self.assertRaises(StopIteration, next, i)
+        self.assertRaises(StopIteration, next, i)
+
+    @warnings_helper.ignore_fork_in_thread_deprecation_warnings()
+    def test_map_close(self):
+        i = self.executor.map(divmod, [5, 5, 5, 5], [2, 0, 3, 5])
+        self.assertEqual(next(i), (2, 1))
+        i.close()
+        self.assertRaises(StopIteration, next, i)
+        self.assertRaises(StopIteration, next, i)
+
+        i = self.executor.map(divmod, [5, 5, 5, 5], [2, 0, 3, 5], chunksize=3)
+        self.assertEqual(next(i), (2, 1))
+        i.close()
+        self.assertRaises(StopIteration, next, i)
+        self.assertRaises(StopIteration, next, i)
+
     def test_map_buffersize_type_validation(self):
         for buffersize in ("foo", 2.0):
             with self.subTest(buffersize=buffersize):

Lib/test/test_os/test_os.py

@@ -2993,6 +2993,14 @@ def test_ftruncate(self):
     def test_lseek(self):
         self.check(os.lseek, 0, 0)
 
+    @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()')
+    @unittest.skipUnless(hasattr(os, 'pipe'), "need os.pipe()")
+    def test_lseek_on_pipe(self):
+        rfd, wfd = os.pipe()
+        self.addCleanup(os.close, rfd)
+        self.addCleanup(os.close, wfd)
+        self.assertRaises(OSError, os.lseek, rfd, 123, os.SEEK_END)
+
     @unittest.skipUnless(hasattr(os, 'read'), 'test needs os.read()')
     def test_read(self):
         self.check(os.read, 1)