gh-129813, PEP 782: Add PyBytesWriter C API (#138822)

Author: vstinner

Doc/c-api/bytes.rst

@@ -219,3 +219,153 @@ called with a non-bytes parameter.
    reallocation fails, the original bytes object at *\*bytes* is deallocated,
    *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is
    returned.
+
+PyBytesWriter
+-------------
+
+The :c:type:`PyBytesWriter` API can be used to create a Python :class:`bytes`
+object.
+
+.. versionadded:: next
+
+.. c:type:: PyBytesWriter
+
+   A bytes writer instance.
+
+   The API is **not thread safe**: a writer should only be used by a single
+   thread at the same time.
+
+   The instance must be destroyed by :c:func:`PyBytesWriter_Finish` on
+   success, or :c:func:`PyBytesWriter_Discard` on error.
+
+
+Create, Finish, Discard
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. c:function:: PyBytesWriter* PyBytesWriter_Create(Py_ssize_t size)
+
+   Create a :c:type:`PyBytesWriter` to write *size* bytes.
+
+   If *size* is greater than zero, allocate *size* bytes, and set the
+   writer size to *size*. The caller is responsible to write *size*
+   bytes using :c:func:`PyBytesWriter_GetData`.
+
+   On error, set an exception and return NULL.
+
+   *size* must be positive or zero.
+
+.. c:function:: PyObject* PyBytesWriter_Finish(PyBytesWriter *writer)
+
+   Finish a :c:type:`PyBytesWriter` created by
+   :c:func:`PyBytesWriter_Create`.
+
+   On success, return a Python :class:`bytes` object.
+   On error, set an exception and return ``NULL``.
+
+   The writer instance is invalid after the call in any case.
+   No API can be called on the writer after :c:func:`PyBytesWriter_Finish`.
+
+.. c:function:: PyObject* PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size)
+
+   Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer
+   to *size* bytes before creating the :class:`bytes` object.
+
+.. c:function:: PyObject* PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf)
+
+   Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer
+   using *buf* pointer before creating the :class:`bytes` object.
+
+   Set an exception and return ``NULL`` if *buf* pointer is outside the
+   internal buffer bounds.
+
+   Function pseudo-code::
+
+       Py_ssize_t size = (char*)buf - (char*)PyBytesWriter_GetData(writer);
+       return PyBytesWriter_FinishWithSize(writer, size);
+
+.. c:function:: void PyBytesWriter_Discard(PyBytesWriter *writer)
+
+   Discard a :c:type:`PyBytesWriter` created by :c:func:`PyBytesWriter_Create`.
+
+   Do nothing if *writer* is ``NULL``.
+
+   The writer instance is invalid after the call.
+   No API can be called on the writer after :c:func:`PyBytesWriter_Discard`.
+
+High-level API
+^^^^^^^^^^^^^^
+
+.. c:function:: int PyBytesWriter_WriteBytes(PyBytesWriter *writer, const void *bytes, Py_ssize_t size)
+
+   Grow the *writer* internal buffer by *size* bytes,
+   write *size* bytes of *bytes* at the *writer* end,
+   and add *size* to the *writer* size.
+
+   If *size* is equal to ``-1``, call ``strlen(bytes)`` to get the
+   string length.
+
+   On success, return ``0``.
+   On error, set an exception and return ``-1``.
+
+
+Getters
+^^^^^^^
+
+.. c:function:: Py_ssize_t PyBytesWriter_GetSize(PyBytesWriter *writer)
+
+   Get the writer size.
+
+.. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer)
+
+   Get the writer data: start of the internal buffer.
+
+   The pointer is valid until :c:func:`PyBytesWriter_Finish` or
+   :c:func:`PyBytesWriter_Discard` is called on *writer*.
+
+
+Low-level API
+^^^^^^^^^^^^^
+
+.. c:function:: int PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size)
+
+   Resize the writer to *size* bytes. It can be used to enlarge or to
+   shrink the writer.
+
+   Newly allocated bytes are left uninitialized.
+
+   On success, return ``0``.
+   On error, set an exception and return ``-1``.
+
+   *size* must be positive or zero.
+
+.. c:function:: int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow)
+
+   Resize the writer by adding *grow* bytes to the current writer size.
+
+   Newly allocated bytes are left uninitialized.
+
+   On success, return ``0``.
+   On error, set an exception and return ``-1``.
+
+   *size* can be negative to shrink the writer.
+
+.. c:function:: void* PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, void *buf)
+
+   Similar to :c:func:`PyBytesWriter_Grow`, but update also the *buf*
+   pointer.
+
+   The *buf* pointer is moved if the internal buffer is moved in memory.
+   The *buf* relative position within the internal buffer is left
+   unchanged.
+
+   On error, set an exception and return ``NULL``.
+
+   *buf* must not be ``NULL``.
+
+   Function pseudo-code::
+
+       Py_ssize_t pos = (char*)buf - (char*)PyBytesWriter_GetData(writer);
+       if (PyBytesWriter_Grow(writer, size) < 0) {
+           return NULL;
+       }
+       return (char*)PyBytesWriter_GetData(writer) + pos;

Doc/whatsnew/3.15.rst

@@ -707,6 +707,22 @@ New features
   and :c:data:`Py_mod_abi`.
   (Contributed by Petr Viktorin in :gh:`137210`.)
 
+* Implement :pep:`782`, the :c:type:`PyBytesWriter` API. Add functions:
+
+  * :c:func:`PyBytesWriter_Create`
+  * :c:func:`PyBytesWriter_Discard`
+  * :c:func:`PyBytesWriter_FinishWithPointer`
+  * :c:func:`PyBytesWriter_FinishWithSize`
+  * :c:func:`PyBytesWriter_Finish`
+  * :c:func:`PyBytesWriter_GetData`
+  * :c:func:`PyBytesWriter_GetSize`
+  * :c:func:`PyBytesWriter_GrowAndUpdatePointer`
+  * :c:func:`PyBytesWriter_Grow`
+  * :c:func:`PyBytesWriter_Resize`
+  * :c:func:`PyBytesWriter_WriteBytes`
+
+  (Contributed by Victor Stinner in :gh:`129813`.)
+
 
 Porting to Python 3.15
 ----------------------

Include/cpython/bytesobject.h

@@ -40,3 +40,42 @@ _PyBytes_Join(PyObject *sep, PyObject *iterable)
 {
     return PyBytes_Join(sep, iterable);
 }
+
+
+// --- PyBytesWriter API -----------------------------------------------------
+
+typedef struct PyBytesWriter PyBytesWriter;
+
+PyAPI_FUNC(PyBytesWriter *) PyBytesWriter_Create(
+    Py_ssize_t size);
+PyAPI_FUNC(void) PyBytesWriter_Discard(
+    PyBytesWriter *writer);
+PyAPI_FUNC(PyObject*) PyBytesWriter_Finish(
+    PyBytesWriter *writer);
+PyAPI_FUNC(PyObject*) PyBytesWriter_FinishWithSize(
+    PyBytesWriter *writer,
+    Py_ssize_t size);
+PyAPI_FUNC(PyObject*) PyBytesWriter_FinishWithPointer(
+    PyBytesWriter *writer,
+    void *buf);
+
+PyAPI_FUNC(void*) PyBytesWriter_GetData(
+    PyBytesWriter *writer);
+PyAPI_FUNC(Py_ssize_t) PyBytesWriter_GetSize(
+    PyBytesWriter *writer);
+
+PyAPI_FUNC(int) PyBytesWriter_WriteBytes(
+    PyBytesWriter *writer,
+    const void *bytes,
+    Py_ssize_t size);
+
+PyAPI_FUNC(int) PyBytesWriter_Resize(
+    PyBytesWriter *writer,
+    Py_ssize_t size);
+PyAPI_FUNC(int) PyBytesWriter_Grow(
+    PyBytesWriter *writer,
+    Py_ssize_t size);
+PyAPI_FUNC(void*) PyBytesWriter_GrowAndUpdatePointer(
+    PyBytesWriter *writer,
+    Py_ssize_t size,
+    void *buf);

Include/internal/pycore_bytesobject.h

@@ -143,6 +143,10 @@ PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer,
     const void *bytes,
     Py_ssize_t size);
 
+// Export for '_testcapi' shared extension.
+PyAPI_FUNC(PyBytesWriter*) _PyBytesWriter_CreateByteArray(
+    Py_ssize_t size);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_freelist_state.h

@@ -27,6 +27,7 @@ extern "C" {
 #  define Py_futureiters_MAXFREELIST 255
 #  define Py_object_stack_chunks_MAXFREELIST 4
 #  define Py_unicode_writers_MAXFREELIST 1
+#  define Py_bytes_writers_MAXFREELIST 1
 #  define Py_pycfunctionobject_MAXFREELIST 16
 #  define Py_pycmethodobject_MAXFREELIST 16
 #  define Py_pymethodobjects_MAXFREELIST 20
@@ -61,6 +62,7 @@ struct _Py_freelists {
     struct _Py_freelist futureiters;
     struct _Py_freelist object_stack_chunks;
     struct _Py_freelist unicode_writers;
+    struct _Py_freelist bytes_writers;
     struct _Py_freelist pycfunctionobject;
     struct _Py_freelist pycmethodobject;
     struct _Py_freelist pymethodobjects;

Lib/test/test_capi/test_bytes.py

@@ -299,5 +299,80 @@ def test_join(self):
             bytes_join(b'', NULL)
 
 
+class BytesWriterTest(unittest.TestCase):
+    result_type = bytes
+
+    def create_writer(self, alloc=0, string=b''):
+        return _testcapi.PyBytesWriter(alloc, string, 0)
+
+    def test_create(self):
+        # Test PyBytesWriter_Create()
+        writer = self.create_writer()
+        self.assertEqual(writer.get_size(), 0)
+        self.assertEqual(writer.finish(), self.result_type(b''))
+
+        writer = self.create_writer(3, b'abc')
+        self.assertEqual(writer.get_size(), 3)
+        self.assertEqual(writer.finish(), self.result_type(b'abc'))
+
+        writer = self.create_writer(10, b'abc')
+        self.assertEqual(writer.get_size(), 10)
+        self.assertEqual(writer.finish_with_size(3), self.result_type(b'abc'))
+
+    def test_write_bytes(self):
+         # Test PyBytesWriter_WriteBytes()
+         writer = self.create_writer()
+         writer.write_bytes(b'Hello World!', -1)
+         self.assertEqual(writer.finish(), self.result_type(b'Hello World!'))
+
+         writer = self.create_writer()
+         writer.write_bytes(b'Hello ', -1)
+         writer.write_bytes(b'World! <truncated>', 6)
+         self.assertEqual(writer.finish(), self.result_type(b'Hello World!'))
+
+    def test_resize(self):
+        # Test PyBytesWriter_Resize()
+        writer = self.create_writer()
+        writer.resize(len(b'number=123456'), b'number=123456')
+        writer.resize(len(b'number=123456'), b'')
+        self.assertEqual(writer.get_size(), len(b'number=123456'))
+        self.assertEqual(writer.finish(), self.result_type(b'number=123456'))
+
+        writer = self.create_writer()
+        writer.resize(0, b'')
+        writer.resize(len(b'number=123456'), b'number=123456')
+        self.assertEqual(writer.finish(), self.result_type(b'number=123456'))
+
+        writer = self.create_writer()
+        writer.resize(len(b'number='), b'number=')
+        writer.resize(len(b'number=123456'), b'123456')
+        self.assertEqual(writer.finish(), self.result_type(b'number=123456'))
+
+        writer = self.create_writer()
+        writer.resize(len(b'number='), b'number=')
+        writer.resize(len(b'number='), b'')
+        writer.resize(len(b'number=123456'), b'123456')
+        self.assertEqual(writer.finish(), self.result_type(b'number=123456'))
+
+        writer = self.create_writer()
+        writer.resize(len(b'number'), b'number')
+        writer.resize(len(b'number='), b'=')
+        writer.resize(len(b'number=123'), b'123')
+        writer.resize(len(b'number=123456'), b'456')
+        self.assertEqual(writer.finish(), self.result_type(b'number=123456'))
+
+    def test_example_abc(self):
+        self.assertEqual(_testcapi.byteswriter_abc(), b'abc')
+
+    def test_example_resize(self):
+        self.assertEqual(_testcapi.byteswriter_resize(), b'Hello World')
+
+
+class ByteArrayWriterTest(BytesWriterTest):
+    result_type = bytearray
+
+    def create_writer(self, alloc=0, string=b''):
+        return _testcapi.PyBytesWriter(alloc, string, 1)
+
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/C_API/2025-09-12-13-05-20.gh-issue-129813.dJZpME.rst

@@ -0,0 +1,15 @@
+Implement :pep:`782`, the :c:type:`PyBytesWriter` API. Add functions:
+
+* :c:func:`PyBytesWriter_Create`
+* :c:func:`PyBytesWriter_Discard`
+* :c:func:`PyBytesWriter_FinishWithPointer`
+* :c:func:`PyBytesWriter_FinishWithSize`
+* :c:func:`PyBytesWriter_Finish`
+* :c:func:`PyBytesWriter_GetData`
+* :c:func:`PyBytesWriter_GetSize`
+* :c:func:`PyBytesWriter_GrowAndUpdatePointer`
+* :c:func:`PyBytesWriter_Grow`
+* :c:func:`PyBytesWriter_Resize`
+* :c:func:`PyBytesWriter_WriteBytes`
+
+Patch by Victor Stinner.