Skip to content

Commit 9de3bc3

Browse files
gh-111331: Allow closing BytesIO with exported buffers
close() no longer fails with BufferError if there are exported buffers. Every exported buffer keeps a reference to the internal buffer, so it outlives closing of the BytesIO object. As a result, destroying or garbage collecting such object no longer emits an unraisable exception.
1 parent 998b890 commit 9de3bc3

4 files changed

Lines changed: 39 additions & 19 deletions

File tree

Doc/library/io.rst

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -763,10 +763,13 @@ than raw I/O does.
763763

764764
.. note::
765765
As long as the view exists, the :class:`BytesIO` object cannot be
766-
resized or closed.
766+
resized. Closing it does not invalidate the view.
767767

768768
.. versionadded:: 3.2
769769

770+
.. versionchanged:: next
771+
The :class:`BytesIO` object can now be closed while the view exists.
772+
770773
.. method:: getvalue()
771774

772775
Return :class:`bytes` containing the entire contents of the buffer.

Lib/test/test_io/test_memoryio.py

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -457,9 +457,6 @@ def test_getbuffer(self):
457457
# raises a BufferError.
458458
self.assertRaises(BufferError, memio.write, b'x' * 100)
459459
self.assertRaises(BufferError, memio.truncate)
460-
# gh-111049: _io.BytesIO detach on close would lead to corruption.
461-
if self.ioclass is io.BytesIO:
462-
self.assertRaises(BufferError, memio.close)
463460
self.assertFalse(memio.closed)
464461
# Mutating the buffer updates the BytesIO
465462
buf[3:6] = b"abc"
@@ -474,12 +471,7 @@ def test_getbuffer(self):
474471
self.assertRaises(ValueError, memio.getbuffer)
475472

476473
def test_getbuffer_delete(self):
477-
# gh-111330: _pyio .close() works and the buffer stays working
478-
if self.ioclass is io.BytesIO:
479-
# gh-111049: _io.BytesIO detach on close would lead to corruption.
480-
# gh-111331: It would be nice to support this.
481-
self.skipTest("io.BytesIO does not support, gh-111049")
482-
474+
# gh-111330, gh-111331: .close() works and the buffer stays working
483475
memio = self.ioclass(b"1234567890")
484476
buf = memio.getbuffer()
485477
self.assertEqual(bytes(buf), b"1234567890")
@@ -489,6 +481,21 @@ def test_getbuffer_delete(self):
489481
buf[3:6] = b"abc"
490482
self.assertEqual(bytes(buf), b"123abc7890")
491483
self.assertRaises(ValueError, memio.getbuffer)
484+
self.assertRaises(ValueError, memio.getvalue)
485+
del buf
486+
support.gc_collect()
487+
memio.close()
488+
489+
def test_getbuffer_del(self):
490+
# gh-111330, gh-111331: deleting the BytesIO which has an exported
491+
# buffer does not emit an unraisable exception.
492+
memio = self.ioclass(b"1234567890")
493+
buf = memio.getbuffer()
494+
with support.catch_unraisable_exception() as cm:
495+
del memio
496+
support.gc_collect()
497+
self.assertIsNone(cm.unraisable)
498+
self.assertEqual(bytes(buf), b"1234567890")
492499

493500
def test_getbuffer_empty(self):
494501
memio = self.ioclass()
@@ -513,15 +520,13 @@ def test_getbuffer_gc_collect(self):
513520
a = [buf]
514521
a.append(a)
515522

516-
# gh-111330: _pyio GC with exports should pass.
523+
# gh-111330, gh-111331: no unraisable exception is emitted.
517524
with support.catch_unraisable_exception() as cm:
518525
del memio
519-
self.assertIsNone(cm.unraisable)
520-
del buf
521-
del a
522-
# The C implementation emits an unraisable exception.
523-
with support.catch_unraisable_exception():
526+
del buf
527+
del a
524528
gc.collect()
529+
self.assertIsNone(cm.unraisable)
525530
self.assertIsNone(memiowr())
526531
self.assertIsNone(bufwr())
527532

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Closing a :class:`io.BytesIO` object which has exported buffers no longer
2+
fails with :exc:`BufferError`. The exported buffers keep the data alive and
3+
stay usable. As a result, destroying or garbage collecting such object no
4+
longer emits an unraisable exception.

Modules/_io/bytesio.c

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ typedef struct {
4040
* Py_REFCNT(buf) == 1, exports == 0.
4141
* Py_REFCNT(buf) > 1. exports == 0,
4242
first modification or export causes the internal buffer copying.
43-
* exports > 0. Py_REFCNT(buf) == 1, any modifications are forbidden.
43+
* exports > 0. Any modifications are forbidden. Every exported buffer
44+
keeps a reference to buf, so it outlives closing of the bytesio object.
4445
*/
4546

4647
static int
@@ -925,7 +926,7 @@ static PyObject *
925926
_io_BytesIO_close_impl(bytesio *self)
926927
/*[clinic end generated code: output=1471bb9411af84a0 input=34ce76d8bd17a23b]*/
927928
{
928-
CHECK_EXPORTS(self);
929+
/* The exported buffers keep the internal buffer alive. */
929930
Py_CLEAR(self->buf);
930931
Py_RETURN_NONE;
931932
}
@@ -1281,6 +1282,9 @@ bytesiobuf_getbuffer_lock_held(PyObject *op, Py_buffer *view, int flags)
12811282

12821283
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(b);
12831284

1285+
if (check_closed(b)) {
1286+
return -1;
1287+
}
12841288
if (FT_ATOMIC_LOAD_SSIZE_RELAXED(b->exports) == 0 && SHARED_BUF(b)) {
12851289
if (unshare_buffer_lock_held(b, b->string_size) < 0)
12861290
return -1;
@@ -1290,6 +1294,9 @@ bytesiobuf_getbuffer_lock_held(PyObject *op, Py_buffer *view, int flags)
12901294
(void)PyBuffer_FillInfo(view, op,
12911295
PyBytes_AS_STRING(b->buf), b->string_size,
12921296
0, flags);
1297+
/* Keep the internal buffer alive: the bytesio object can be closed
1298+
while the buffer is exported. */
1299+
view->internal = Py_NewRef(b->buf);
12931300
FT_ATOMIC_ADD_SSIZE(b->exports, 1);
12941301
return 0;
12951302
}
@@ -1311,11 +1318,12 @@ bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags)
13111318
}
13121319

13131320
static void
1314-
bytesiobuf_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view))
1321+
bytesiobuf_releasebuffer(PyObject *op, Py_buffer *view)
13151322
{
13161323
bytesiobuf *obj = bytesiobuf_CAST(op);
13171324
bytesio *b = bytesio_CAST(obj->source);
13181325
FT_ATOMIC_ADD_SSIZE(b->exports, -1);
1326+
Py_CLEAR(view->internal);
13191327
}
13201328

13211329
static int

0 commit comments

Comments
 (0)