From 0016507c168fa942d7856bdef371cd8d494b140b Mon Sep 17 00:00:00 2001
From: Victor Stinner <victor.stinner@gmail.com>
Date: Fri, 9 Oct 2015 01:53:21 +0200
Subject: [PATCH] Issue #25318: Move _PyBytesWriter to bytesobject.c

Declare also the private API in bytesobject.h.
---
 Include/bytesobject.h   |  52 ++++++++++
 Objects/bytesobject.c   | 193 ++++++++++++++++++++++++++++++++++++
 Objects/unicodeobject.c | 210 ----------------------------------------
 3 files changed, 245 insertions(+), 210 deletions(-)

diff --git a/Include/bytesobject.h b/Include/bytesobject.h
index e379bace37e..eafcdea4bb1 100644
--- a/Include/bytesobject.h
+++ b/Include/bytesobject.h
@@ -123,6 +123,58 @@ PyAPI_FUNC(Py_ssize_t) _PyBytes_InsertThousandsGrouping(char *buffer,
 #define F_ALT	(1<<3)
 #define F_ZERO	(1<<4)
 
+#ifndef Py_LIMITED_API
+/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer".
+   A _PyBytesWriter variable must be declared at the end of variables in a
+   function to optimize the memory allocation on the stack. */
+typedef struct {
+    /* bytes object */
+    PyObject *buffer;
+
+    /* Number of allocated size */
+    Py_ssize_t allocated;
+
+    /* Current size of the buffer (can be smaller than the allocated size) */
+    Py_ssize_t size;
+
+    /* If non-zero, overallocate the buffer (default: 0). */
+    int overallocate;
+
+    /* Stack buffer */
+    int use_stack_buffer;
+    char stack_buffer[512];
+} _PyBytesWriter;
+
+/* Initialize a bytes writer
+
+   By default, the overallocation is disabled. Set the overallocate attribute
+   to control the allocation of the buffer. */
+PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer);
+
+/* Get the buffer content and reset the writer.
+   Return a bytes object.
+   Raise an exception and return NULL on error. */
+PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer,
+    char *str);
+
+/* Deallocate memory of a writer (clear its internal buffer). */
+PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer);
+
+/* Allocate the buffer to write size bytes.
+   Return the pointer to the beginning of buffer data.
+   Raise an exception and return NULL on error. */
+PyAPI_FUNC(char*) _PyBytesWriter_Alloc(_PyBytesWriter *writer,
+    Py_ssize_t size);
+
+/* Add *size* bytes to the buffer.
+   str is the current pointer inside the buffer.
+   Return the updated current pointer inside the buffer.
+   Raise an exception and return NULL on error. */
+PyAPI_FUNC(char*) _PyBytesWriter_Prepare(_PyBytesWriter *writer,
+    char *str,
+    Py_ssize_t size);
+#endif   /* Py_LIMITED_API */
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index 08275ad5ada..46322aadf04 100644
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -3735,3 +3735,196 @@ bytes_iter(PyObject *seq)
     _PyObject_GC_TRACK(it);
     return (PyObject *)it;
 }
+
+
+/* _PyBytesWriter API */
+
+#ifdef MS_WINDOWS
+   /* On Windows, overallocate by 50% is the best factor */
+#  define OVERALLOCATE_FACTOR 2
+#else
+   /* On Linux, overallocate by 25% is the best factor */
+#  define OVERALLOCATE_FACTOR 4
+#endif
+
+void
+_PyBytesWriter_Init(_PyBytesWriter *writer)
+{
+    writer->buffer = NULL;
+    writer->allocated = 0;
+    writer->size = 0;
+    writer->overallocate = 0;
+    writer->use_stack_buffer = 0;
+#ifdef Py_DEBUG
+    memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer));
+#endif
+}
+
+void
+_PyBytesWriter_Dealloc(_PyBytesWriter *writer)
+{
+    Py_CLEAR(writer->buffer);
+}
+
+Py_LOCAL_INLINE(char*)
+_PyBytesWriter_AsString(_PyBytesWriter *writer)
+{
+    if (!writer->use_stack_buffer) {
+        assert(writer->buffer != NULL);
+        return PyBytes_AS_STRING(writer->buffer);
+    }
+    else {
+        assert(writer->buffer == NULL);
+        return writer->stack_buffer;
+    }
+}
+
+Py_LOCAL_INLINE(Py_ssize_t)
+_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str)
+{
+    char *start = _PyBytesWriter_AsString(writer);
+    assert(str != NULL);
+    assert(str >= start);
+    return str - start;
+}
+
+Py_LOCAL_INLINE(void)
+_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str)
+{
+#ifdef Py_DEBUG
+    char *start, *end;
+
+    if (!writer->use_stack_buffer) {
+        assert(writer->buffer != NULL);
+        assert(PyBytes_CheckExact(writer->buffer));
+        assert(Py_REFCNT(writer->buffer) == 1);
+    }
+    else {
+        assert(writer->buffer == NULL);
+    }
+
+    start = _PyBytesWriter_AsString(writer);
+    assert(0 <= writer->size && writer->size <= writer->allocated);
+    /* the last byte must always be null */
+    assert(start[writer->allocated] == 0);
+
+    end = start + writer->allocated;
+    assert(str != NULL);
+    assert(start <= str && str <= end);
+#endif
+}
+
+char*
+_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size)
+{
+    Py_ssize_t allocated, pos;
+
+    _PyBytesWriter_CheckConsistency(writer, str);
+    assert(size >= 0);
+
+    if (size == 0) {
+        /* nothing to do */
+        return str;
+    }
+
+    if (writer->size > PY_SSIZE_T_MAX - size) {
+        PyErr_NoMemory();
+        _PyBytesWriter_Dealloc(writer);
+        return NULL;
+    }
+    writer->size += size;
+
+    allocated = writer->allocated;
+    if (writer->size <= allocated)
+        return str;
+
+    allocated = writer->size;
+    if (writer->overallocate
+        && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) {
+        /* overallocate to limit the number of realloc() */
+        allocated += allocated / OVERALLOCATE_FACTOR;
+    }
+
+    pos = _PyBytesWriter_GetPos(writer, str);
+    if (!writer->use_stack_buffer) {
+        /* Note: Don't use a bytearray object because the conversion from
+           byterray to bytes requires to copy all bytes. */
+        if (_PyBytes_Resize(&writer->buffer, allocated)) {
+            assert(writer->buffer == NULL);
+            return NULL;
+        }
+    }
+    else {
+        /* convert from stack buffer to bytes object buffer */
+        assert(writer->buffer == NULL);
+
+        writer->buffer = PyBytes_FromStringAndSize(NULL, allocated);
+        if (writer->buffer == NULL)
+            return NULL;
+
+        if (pos != 0) {
+            Py_MEMCPY(PyBytes_AS_STRING(writer->buffer),
+                      writer->stack_buffer,
+                      pos);
+        }
+
+#ifdef Py_DEBUG
+        memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer));
+#endif
+
+        writer->use_stack_buffer = 0;
+    }
+    writer->allocated = allocated;
+
+    str = _PyBytesWriter_AsString(writer) + pos;
+    _PyBytesWriter_CheckConsistency(writer, str);
+    return str;
+}
+
+/* Allocate the buffer to write size bytes.
+   Return the pointer to the beginning of buffer data.
+   Raise an exception and return NULL on error. */
+char*
+_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size)
+{
+    /* ensure that _PyBytesWriter_Alloc() is only called once */
+    assert(writer->size == 0 && writer->buffer == NULL);
+    assert(size >= 0);
+
+    writer->use_stack_buffer = 1;
+#if Py_DEBUG
+    /* the last byte is reserved, it must be '\0' */
+    writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0;
+    writer->allocated = sizeof(writer->stack_buffer) - 1;
+#else
+    writer->allocated = sizeof(writer->stack_buffer);
+#endif
+    return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size);
+}
+
+PyObject *
+_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str)
+{
+    Py_ssize_t pos;
+    PyObject *result;
+
+    _PyBytesWriter_CheckConsistency(writer, str);
+
+    pos = _PyBytesWriter_GetPos(writer, str);
+    if (!writer->use_stack_buffer) {
+        if (pos != writer->allocated) {
+            if (_PyBytes_Resize(&writer->buffer, pos)) {
+                assert(writer->buffer == NULL);
+                return NULL;
+            }
+        }
+
+        result = writer->buffer;
+        writer->buffer = NULL;
+    }
+    else {
+        result = PyBytes_FromStringAndSize(writer->stack_buffer, pos);
+    }
+
+    return result;
+}
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index e0b3c688cdb..614d2147cb8 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -347,216 +347,6 @@ PyUnicode_GetMax(void)
 #endif
 }
 
-/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer".
-   A _PyBytesWriter variable must be declared at the end of variables in a
-   function to optimize the memory allocation on the stack. */
-typedef struct {
-    /* bytes object */
-    PyObject *buffer;
-
-    /* Number of allocated size */
-    Py_ssize_t allocated;
-
-    /* Current size of the buffer (can be smaller than the allocated size) */
-    Py_ssize_t size;
-
-    /* If non-zero, overallocate the buffer (default: 0). */
-    int overallocate;
-
-    /* Stack buffer */
-    int use_stack_buffer;
-    char stack_buffer[512];
-} _PyBytesWriter;
-
-static void
-_PyBytesWriter_Init(_PyBytesWriter *writer)
-{
-    writer->buffer = NULL;
-    writer->allocated = 0;
-    writer->size = 0;
-    writer->overallocate = 0;
-    writer->use_stack_buffer = 0;
-#ifdef Py_DEBUG
-    memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer));
-#endif
-}
-
-static void
-_PyBytesWriter_Dealloc(_PyBytesWriter *writer)
-{
-    Py_CLEAR(writer->buffer);
-}
-
-static char*
-_PyBytesWriter_AsString(_PyBytesWriter *writer)
-{
-    if (!writer->use_stack_buffer) {
-        assert(writer->buffer != NULL);
-        return PyBytes_AS_STRING(writer->buffer);
-    }
-    else {
-        assert(writer->buffer == NULL);
-        return writer->stack_buffer;
-    }
-}
-
-Py_LOCAL_INLINE(Py_ssize_t)
-_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str)
-{
-    char *start = _PyBytesWriter_AsString(writer);
-    assert(str != NULL);
-    assert(str >= start);
-    return str - start;
-}
-
-Py_LOCAL_INLINE(void)
-_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str)
-{
-#ifdef Py_DEBUG
-    char *start, *end;
-
-    if (!writer->use_stack_buffer) {
-        assert(writer->buffer != NULL);
-        assert(PyBytes_CheckExact(writer->buffer));
-        assert(Py_REFCNT(writer->buffer) == 1);
-    }
-    else {
-        assert(writer->buffer == NULL);
-    }
-
-    start = _PyBytesWriter_AsString(writer);
-    assert(0 <= writer->size && writer->size <= writer->allocated);
-    /* the last byte must always be null */
-    assert(start[writer->allocated] == 0);
-
-    end = start + writer->allocated;
-    assert(str != NULL);
-    assert(start <= str && str <= end);
-#endif
-}
-
-/* Add *size* bytes to the buffer.
-   str is the current pointer inside the buffer.
-   Return the updated current pointer inside the buffer.
-   Raise an exception and return NULL on error. */
-static char*
-_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size)
-{
-    Py_ssize_t allocated, pos;
-
-    _PyBytesWriter_CheckConsistency(writer, str);
-    assert(size >= 0);
-
-    if (size == 0) {
-        /* nothing to do */
-        return str;
-    }
-
-    if (writer->size > PY_SSIZE_T_MAX - size) {
-        PyErr_NoMemory();
-        _PyBytesWriter_Dealloc(writer);
-        return NULL;
-    }
-    writer->size += size;
-
-    allocated = writer->allocated;
-    if (writer->size <= allocated)
-        return str;
-
-    allocated = writer->size;
-    if (writer->overallocate
-        && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) {
-        /* overallocate to limit the number of realloc() */
-        allocated += allocated / OVERALLOCATE_FACTOR;
-    }
-
-    pos = _PyBytesWriter_GetPos(writer, str);
-    if (!writer->use_stack_buffer) {
-        /* Note: Don't use a bytearray object because the conversion from
-           byterray to bytes requires to copy all bytes. */
-        if (_PyBytes_Resize(&writer->buffer, allocated)) {
-            assert(writer->buffer == NULL);
-            return NULL;
-        }
-    }
-    else {
-        /* convert from stack buffer to bytes object buffer */
-        assert(writer->buffer == NULL);
-
-        writer->buffer = PyBytes_FromStringAndSize(NULL, allocated);
-        if (writer->buffer == NULL)
-            return NULL;
-
-        if (pos != 0) {
-            Py_MEMCPY(PyBytes_AS_STRING(writer->buffer),
-                      writer->stack_buffer,
-                      pos);
-        }
-
-#ifdef Py_DEBUG
-        memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer));
-#endif
-
-        writer->use_stack_buffer = 0;
-    }
-    writer->allocated = allocated;
-
-    str = _PyBytesWriter_AsString(writer) + pos;
-    _PyBytesWriter_CheckConsistency(writer, str);
-    return str;
-}
-
-/* Allocate the buffer to write size bytes.
-   Return the pointer to the beginning of buffer data.
-   Raise an exception and return NULL on error. */
-static char*
-_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size)
-{
-    /* ensure that _PyBytesWriter_Alloc() is only called once */
-    assert(writer->size == 0 && writer->buffer == NULL);
-    assert(size >= 0);
-
-    writer->use_stack_buffer = 1;
-#if Py_DEBUG
-    /* the last byte is reserved, it must be '\0' */
-    writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0;
-    writer->allocated = sizeof(writer->stack_buffer) - 1;
-#else
-    writer->allocated = sizeof(writer->stack_buffer);
-#endif
-    return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size);
-}
-
-/* Get the buffer content and reset the writer.
-   Return a bytes object.
-   Raise an exception and return NULL on error. */
-static PyObject *
-_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str)
-{
-    Py_ssize_t pos;
-    PyObject *result;
-
-    _PyBytesWriter_CheckConsistency(writer, str);
-
-    pos = _PyBytesWriter_GetPos(writer, str);
-    if (!writer->use_stack_buffer) {
-        if (pos != writer->allocated) {
-            if (_PyBytes_Resize(&writer->buffer, pos)) {
-                assert(writer->buffer == NULL);
-                return NULL;
-            }
-        }
-
-        result = writer->buffer;
-        writer->buffer = NULL;
-    }
-    else {
-        result = PyBytes_FromStringAndSize(writer->stack_buffer, pos);
-    }
-
-    return result;
-}
-
 #ifdef Py_DEBUG
 int
 _PyUnicode_CheckConsistency(PyObject *op, int check_content)