Skip to content
2 changes: 1 addition & 1 deletion Doc/library/compression.zstd.rst
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ Compressing and decompressing data in memory
will be set to ``True``.

Attempting to decompress data after the end of a frame will raise a
:exc:`ZstdError`. Any data found after the end of the frame is ignored
:exc:`EOFError`. Any data found after the end of the frame is ignored
and saved in the :attr:`~.unused_data` attribute.

.. attribute:: eof
Expand Down
5 changes: 4 additions & 1 deletion Doc/library/io.rst
Original file line number Diff line number Diff line change
Expand Up @@ -763,10 +763,13 @@ than raw I/O does.

.. note::
As long as the view exists, the :class:`BytesIO` object cannot be
resized or closed.
resized. Closing it does not invalidate the view.

.. versionadded:: 3.2

.. versionchanged:: next
The :class:`BytesIO` object can now be closed while the view exists.

.. method:: getvalue()

Return :class:`bytes` containing the entire contents of the buffer.
Expand Down
1 change: 1 addition & 0 deletions Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -1595,6 +1595,7 @@ def test_repr_lock(self):
event.wait()
self.assertEqual(f'<Lock(owner=SomeOtherProcess)>', repr(lock))
p.terminate()
p.join()

def test_lock(self):
lock = self.Lock()
Expand Down
35 changes: 20 additions & 15 deletions Lib/test/test_io/test_memoryio.py
Original file line number Diff line number Diff line change
Expand Up @@ -457,9 +457,6 @@ def test_getbuffer(self):
# raises a BufferError.
self.assertRaises(BufferError, memio.write, b'x' * 100)
self.assertRaises(BufferError, memio.truncate)
# gh-111049: _io.BytesIO detach on close would lead to corruption.
if self.ioclass is io.BytesIO:
self.assertRaises(BufferError, memio.close)
self.assertFalse(memio.closed)
# Mutating the buffer updates the BytesIO
buf[3:6] = b"abc"
Expand All @@ -474,12 +471,7 @@ def test_getbuffer(self):
self.assertRaises(ValueError, memio.getbuffer)

def test_getbuffer_delete(self):
# gh-111330: _pyio .close() works and the buffer stays working
if self.ioclass is io.BytesIO:
# gh-111049: _io.BytesIO detach on close would lead to corruption.
# gh-111331: It would be nice to support this.
self.skipTest("io.BytesIO does not support, gh-111049")

# gh-111330, gh-111331: .close() works and the buffer stays working
memio = self.ioclass(b"1234567890")
buf = memio.getbuffer()
self.assertEqual(bytes(buf), b"1234567890")
Expand All @@ -489,6 +481,21 @@ def test_getbuffer_delete(self):
buf[3:6] = b"abc"
self.assertEqual(bytes(buf), b"123abc7890")
self.assertRaises(ValueError, memio.getbuffer)
self.assertRaises(ValueError, memio.getvalue)
del buf
support.gc_collect()
memio.close()

def test_getbuffer_del(self):
# gh-111330, gh-111331: deleting the BytesIO which has an exported
# buffer does not emit an unraisable exception.
memio = self.ioclass(b"1234567890")
buf = memio.getbuffer()
with support.catch_unraisable_exception() as cm:
del memio
support.gc_collect()
self.assertIsNone(cm.unraisable)
self.assertEqual(bytes(buf), b"1234567890")

def test_getbuffer_empty(self):
memio = self.ioclass()
Expand All @@ -513,15 +520,13 @@ def test_getbuffer_gc_collect(self):
a = [buf]
a.append(a)

# gh-111330: _pyio GC with exports should pass.
# gh-111330, gh-111331: no unraisable exception is emitted.
with support.catch_unraisable_exception() as cm:
del memio
self.assertIsNone(cm.unraisable)
del buf
del a
# The C implementation emits an unraisable exception.
with support.catch_unraisable_exception():
del buf
del a
gc.collect()
self.assertIsNone(cm.unraisable)
self.assertIsNone(memiowr())
self.assertIsNone(bufwr())

Expand Down
5 changes: 4 additions & 1 deletion Lib/test/test_pyrepl/test_pyrepl.py
Original file line number Diff line number Diff line change
Expand Up @@ -1467,7 +1467,10 @@ def test_attribute_completion(self):
reader = self.prepare_reader(events, namespace={})
output = reader.readline()
self.assertEqual(output, expected)
new_imports = sys.modules.keys() - _imported
# The reader imports its own helpers lazily.
new_imports = {name for name in
sys.modules.keys() - _imported
if not name.startswith('_pyrepl.')}
self.assertEqual(new_imports, expected_imports)

@patch.dict(sys.modules)
Expand Down
12 changes: 12 additions & 0 deletions Lib/test/test_sqlite3/test_dbapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -1396,6 +1396,18 @@ def test_blob_set_slice(self):
actual = self.cx.execute("select b from test").fetchone()[0]
self.assertEqual(actual, expected)

def test_blob_set_slice_with_step_keeps_bytes_intact(self):
# The buffer used for the read-patch-write cycle must not be the
# bytes object read from the blob: for a single byte it is an
# immortal singleton.
old_byte = self.data[5]
self.blob[5:6:2] = b"\xab"
self.assertEqual(bytes([old_byte])[0], old_byte)
self.assertEqual(self.blob[5:6], b"\xab")
expected = self.data[:5] + b"\xab" + self.data[6:]
actual = self.cx.execute("select b from test").fetchone()[0]
self.assertEqual(actual, expected)

def test_blob_set_empty_slice(self):
self.blob[0:0] = b""
self.assertEqual(self.blob[:], self.data)
Expand Down
7 changes: 7 additions & 0 deletions Lib/test/test_sysconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ def setUp(self):
self.isabs = os.path.isabs
self.splitdrive = os.path.splitdrive
self._config_vars = sysconfig._CONFIG_VARS, copy(sysconfig._CONFIG_VARS)
self._cached_prefixes = (sysconfig._config_vars_cached_prefix,
sysconfig._config_vars_cached_exec_prefix)
self._added_envvars = []
self._changed_envvars = []
for var in ('MACOSX_DEPLOYMENT_TARGET', 'PATH'):
Expand Down Expand Up @@ -92,6 +94,11 @@ def tearDown(self):
sysconfig._CONFIG_VARS = self._config_vars[0]
sysconfig._CONFIG_VARS.clear()
sysconfig._CONFIG_VARS.update(self._config_vars[1])
# Restoring _CONFIG_VARS is not enough: if the cached prefixes are
# left over from the test, the next get_config_vars() call finds
# the cache stale and replaces _CONFIG_VARS with a new dict.
(sysconfig._config_vars_cached_prefix,
sysconfig._config_vars_cached_exec_prefix) = self._cached_prefixes
for var, value in self._changed_envvars:
os.environ[var] = value
for var in self._added_envvars:
Expand Down
16 changes: 16 additions & 0 deletions Lib/test/test_typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -6054,6 +6054,22 @@ class A:
with self.assertRaises(TypeError):
a[int]

def test_parameter_added_after_parameters_cached(self):
# gh-155752: GenericAlias parameters are cached before substitution, so
# an argument can gain __typing_subst__ after the tuple is calculated.
class Parameter:
pass

first = Parameter()
first.__typing_subst__ = lambda value: value
late = Parameter()
alias = types.GenericAlias(dict, (first, late))
self.assertEqual(alias.__parameters__, (first,))
late.__typing_subst__ = lambda value: value

with self.assertRaisesRegex(TypeError, "not found in __parameters__"):
alias[0]

def test_return_non_tuple_while_unpacking(self):
# GH-138497: GenericAlias objects didn't ensure that __typing_subst__ actually
# returned a tuple
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix a crash when a :class:`types.GenericAlias` argument gains a
``__typing_subst__`` hook after the alias parameters have been cached.
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Closing a :class:`io.BytesIO` object which has exported buffers no longer
fails with :exc:`BufferError`. The exported buffers keep the data alive and
stay usable. As a result, destroying or garbage collecting such object no
longer emits an unraisable exception.
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fix :class:`sqlite3.Blob` slice assignment with a step.
It patched the bytes object read from the blob,
which for a single byte is an immortal singleton,
so that the value of that byte was changed in the whole process.
14 changes: 11 additions & 3 deletions Modules/_io/bytesio.c
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ typedef struct {
* Py_REFCNT(buf) == 1, exports == 0.
* Py_REFCNT(buf) > 1. exports == 0,
first modification or export causes the internal buffer copying.
* exports > 0. Py_REFCNT(buf) == 1, any modifications are forbidden.
* exports > 0. Any modifications are forbidden. Every exported buffer
keeps a reference to buf, so it outlives closing of the bytesio object.
*/

static int
Expand Down Expand Up @@ -925,7 +926,7 @@ static PyObject *
_io_BytesIO_close_impl(bytesio *self)
/*[clinic end generated code: output=1471bb9411af84a0 input=34ce76d8bd17a23b]*/
{
CHECK_EXPORTS(self);
/* The exported buffers keep the internal buffer alive. */
Py_CLEAR(self->buf);
Py_RETURN_NONE;
}
Expand Down Expand Up @@ -1281,6 +1282,9 @@ bytesiobuf_getbuffer_lock_held(PyObject *op, Py_buffer *view, int flags)

_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(b);

if (check_closed(b)) {
return -1;
}
if (FT_ATOMIC_LOAD_SSIZE_RELAXED(b->exports) == 0 && SHARED_BUF(b)) {
if (unshare_buffer_lock_held(b, b->string_size) < 0)
return -1;
Expand All @@ -1290,6 +1294,9 @@ bytesiobuf_getbuffer_lock_held(PyObject *op, Py_buffer *view, int flags)
(void)PyBuffer_FillInfo(view, op,
PyBytes_AS_STRING(b->buf), b->string_size,
0, flags);
/* Keep the internal buffer alive: the bytesio object can be closed
while the buffer is exported. */
view->internal = Py_NewRef(b->buf);
FT_ATOMIC_ADD_SSIZE(b->exports, 1);
return 0;
}
Expand All @@ -1311,11 +1318,12 @@ bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags)
}

static void
bytesiobuf_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view))
bytesiobuf_releasebuffer(PyObject *op, Py_buffer *view)
{
bytesiobuf *obj = bytesiobuf_CAST(op);
bytesio *b = bytesio_CAST(obj->source);
FT_ATOMIC_ADD_SSIZE(b->exports, -1);
Py_CLEAR(view->internal);
}

static int
Expand Down
57 changes: 40 additions & 17 deletions Modules/_sqlite/blob.c
Original file line number Diff line number Diff line change
Expand Up @@ -139,26 +139,35 @@ read_single(pysqlite_Blob *self, Py_ssize_t offset)
return PyLong_FromUnsignedLong((unsigned long)buf);
}

static PyObject *
read_multiple(pysqlite_Blob *self, Py_ssize_t length, Py_ssize_t offset)
static int
inner_read(pysqlite_Blob *self, char *buf, Py_ssize_t length,
Py_ssize_t offset)
{
assert(length <= sqlite3_blob_bytes(self->blob));
assert(offset < sqlite3_blob_bytes(self->blob));

PyBytesWriter *writer = PyBytesWriter_Create(length);
if (writer == NULL) {
return NULL;
}
char *raw_buffer = PyBytesWriter_GetData(writer);

int rc;
Py_BEGIN_ALLOW_THREADS
rc = sqlite3_blob_read(self->blob, raw_buffer, (int)length, (int)offset);
rc = sqlite3_blob_read(self->blob, buf, (int)length, (int)offset);
Py_END_ALLOW_THREADS

if (rc != SQLITE_OK) {
PyBytesWriter_Discard(writer);
blob_seterror(self, rc);
return -1;
}
return 0;
}

static PyObject *
read_multiple(pysqlite_Blob *self, Py_ssize_t length, Py_ssize_t offset)
{
PyBytesWriter *writer = PyBytesWriter_Create(length);
if (writer == NULL) {
return NULL;
}

if (inner_read(self, PyBytesWriter_GetData(writer), length, offset) < 0) {
PyBytesWriter_Discard(writer);
return NULL;
}
return PyBytesWriter_Finish(writer);
Expand Down Expand Up @@ -553,14 +562,28 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value)
rc = inner_write(self, vbuf.buf, len, start);
}
else {
PyObject *blob_bytes = read_multiple(self, stop - start, start);
if (blob_bytes != NULL) {
char *blob_buf = PyBytes_AS_STRING(blob_bytes);
for (Py_ssize_t i = 0, j = 0; i < len; i++, j += step) {
blob_buf[j] = ((char *)vbuf.buf)[i];
/* Read the affected region, patch it and write it back. The
object returned by read_multiple() cannot be used as the buffer,
because for a single byte it is an immortal singleton. */
Py_ssize_t length = stop - start;
if (length <= 0) {
/* start > stop for a negative step; see gh-150449. */
PyErr_SetString(PyExc_ValueError, "size must be >= 0");
}
else {
char *buf = PyMem_Malloc(length);
if (buf == NULL) {
PyErr_NoMemory();
}
else {
if (inner_read(self, buf, length, start) == 0) {
for (Py_ssize_t i = 0, j = 0; i < len; i++, j += step) {
buf[j] = ((char *)vbuf.buf)[i];
}
rc = inner_write(self, buf, length, start);
}
PyMem_Free(buf);
}
rc = inner_write(self, blob_buf, stop - start, start);
Py_DECREF(blob_bytes);
}
}
PyBuffer_Release(&vbuf);
Expand Down
14 changes: 12 additions & 2 deletions Objects/genericaliasobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -525,8 +525,18 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
}
if (subst) {
Py_ssize_t iparam = tuple_index(parameters, nparams, arg);
assert(iparam >= 0);
arg = PyObject_CallOneArg(subst, argitems[iparam]);
if (iparam < 0) {
// __parameters__ may be stale if an argument gained
// __typing_subst__ after the tuple was computed.
PyErr_Format(PyExc_TypeError,
"argument %R with __typing_subst__ was not found "
"in __parameters__",
arg);
arg = NULL;
}
else {
arg = PyObject_CallOneArg(subst, argitems[iparam]);
}
Py_DECREF(subst);
}
else {
Expand Down
Loading