[Python-checkins] cpython: Issue #21233: Add new C functions: PyMem_RawCalloc(), PyMem_Calloc(),
victor.stinner
python-checkins at python.org
Fri May 2 22:31:30 CEST 2014
http://hg.python.org/cpython/rev/5b0fda8f5718
changeset: 90540:5b0fda8f5718
user: Victor Stinner <victor.stinner at gmail.com>
date: Fri May 02 22:31:14 2014 +0200
summary:
Issue #21233: Add new C functions: PyMem_RawCalloc(), PyMem_Calloc(),
PyObject_Calloc(), _PyObject_GC_Calloc(). bytes(int) and bytearray(int) are now
using ``calloc()`` instead of ``malloc()`` for large objects which is faster
and use less memory (until the bytearray buffer is filled with data).
files:
Doc/c-api/memory.rst | 36 ++++++-
Doc/whatsnew/3.5.rst | 20 +++-
Include/objimpl.h | 4 +-
Include/pymem.h | 5 +
Misc/NEWS | 6 +
Modules/_testcapimodule.c | 73 ++++++++++++++-
Modules/_tracemalloc.c | 57 +++++++++--
Modules/gcmodule.c | 24 ++++-
Objects/bytearrayobject.c | 16 ++-
Objects/bytesobject.c | 76 +++++++++-----
Objects/obmalloc.c | 126 ++++++++++++++++++++++---
11 files changed, 369 insertions(+), 74 deletions(-)
diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -92,8 +92,8 @@
need to be held.
The default raw memory block allocator uses the following functions:
-:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
-requesting zero bytes.
+:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call
+``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes.
.. versionadded:: 3.4
@@ -106,6 +106,17 @@
been initialized in any way.
+.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize)
+
+ Allocates *nelem* elements each whose size in bytes is *elsize* and returns
+ a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the
+ request fails. The memory is initialized to zeros. Requesting zero elements
+ or elements of size zero bytes returns a distinct non-*NULL* pointer if
+ possible, as if ``PyMem_RawCalloc(1, 1)`` had been called instead.
+
+ .. versionadded:: 3.5
+
+
.. c:function:: void* PyMem_RawRealloc(void *p, size_t n)
Resizes the memory block pointed to by *p* to *n* bytes. The contents will
@@ -136,8 +147,8 @@
memory from the Python heap.
The default memory block allocator uses the following functions:
-:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
-requesting zero bytes.
+:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call
+``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes.
.. warning::
@@ -152,6 +163,17 @@
been called instead. The memory will not have been initialized in any way.
+.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize)
+
+ Allocates *nelem* elements each whose size in bytes is *elsize* and returns
+ a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the
+ request fails. The memory is initialized to zeros. Requesting zero elements
+ or elements of size zero bytes returns a distinct non-*NULL* pointer if
+ possible, as if ``PyMem_Calloc(1, 1)`` had been called instead.
+
+ .. versionadded:: 3.5
+
+
.. c:function:: void* PyMem_Realloc(void *p, size_t n)
Resizes the memory block pointed to by *p* to *n* bytes. The contents will be
@@ -222,11 +244,17 @@
+----------------------------------------------------------+---------------------------------------+
| ``void* malloc(void *ctx, size_t size)`` | allocate a memory block |
+----------------------------------------------------------+---------------------------------------+
+ | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized |
+ | | with zeros |
+ +----------------------------------------------------------+---------------------------------------+
| ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block |
+----------------------------------------------------------+---------------------------------------+
| ``void free(void *ctx, void *ptr)`` | free a memory block |
+----------------------------------------------------------+---------------------------------------+
+ .. versionchanged:: 3.5
+ Add a new field ``calloc``.
+
.. c:type:: PyMemAllocatorDomain
Enum used to identify an allocator domain. Domains:
diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -164,7 +164,10 @@
Major performance enhancements have been added:
-* None yet.
+* Construction of ``bytes(int)`` and ``bytearray(int)`` (filled by zero bytes)
+ is faster and use less memory (until the bytearray buffer is filled with
+ data) for large objects. ``calloc()`` is used instead of ``malloc()`` to
+ allocate memory for these objects.
Build and C API Changes
@@ -172,7 +175,12 @@
Changes to Python's build process and to the C API include:
-* None yet.
+* New ``calloc`` functions:
+
+ * :c:func:`PyMem_RawCalloc`
+ * :c:func:`PyMem_Calloc`
+ * :c:func:`PyObject_Calloc`
+ * :c:func:`_PyObject_GC_Calloc`
Deprecated
@@ -209,6 +217,9 @@
This section lists previously described changes and other bugfixes
that may require changes to your code.
+Changes in the Python API
+-------------------------
+
* Before Python 3.5, a :class:`datetime.time` object was considered to be false
if it represented midnight in UTC. This behavior was considered obscure and
error-prone and has been removed in Python 3.5. See :issue:`13936` for full
@@ -217,3 +228,8 @@
* :meth:`ssl.SSLSocket.send()` now raises either :exc:`ssl.SSLWantReadError`
or :exc:`ssl.SSLWantWriteError` on a non-blocking socket if the operation
would block. Previously, it would return 0. See :issue:`20951`.
+
+Changes in the C API
+--------------------
+
+* The :c:type:`PyMemAllocator` structure has a new ``calloc`` field.
diff --git a/Include/objimpl.h b/Include/objimpl.h
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -95,6 +95,7 @@
the raw memory.
*/
PyAPI_FUNC(void *) PyObject_Malloc(size_t size);
+PyAPI_FUNC(void *) PyObject_Calloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyObject_Free(void *ptr);
@@ -321,7 +322,8 @@
(!PyTuple_CheckExact(obj) || _PyObject_GC_IS_TRACKED(obj)))
#endif /* Py_LIMITED_API */
-PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t);
+PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t size);
+PyAPI_FUNC(PyObject *) _PyObject_GC_Calloc(size_t size);
PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *);
PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t);
PyAPI_FUNC(void) PyObject_GC_Track(void *);
diff --git a/Include/pymem.h b/Include/pymem.h
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -13,6 +13,7 @@
#ifndef Py_LIMITED_API
PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
#endif
@@ -57,6 +58,7 @@
*/
PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_Free(void *ptr);
@@ -132,6 +134,9 @@
/* allocate a memory block */
void* (*malloc) (void *ctx, size_t size);
+ /* allocate a memory block initialized by zeros */
+ void* (*calloc) (void *ctx, size_t nelem, size_t elsize);
+
/* allocate or resize a memory block */
void* (*realloc) (void *ctx, void *ptr, size_t new_size);
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,12 @@
Core and Builtins
-----------------
+- Issue #21233: Add new C functions: PyMem_RawCalloc(), PyMem_Calloc(),
+ PyObject_Calloc(), _PyObject_GC_Calloc(). bytes(int) and bytearray(int)
+ are now using ``calloc()`` instead of ``malloc()`` for large objects which
+ is faster and use less memory (until the bytearray buffer is filled with
+ data).
+
- Issue #21377: PyBytes_Concat() now tries to concatenate in-place when the
first argument has a reference count of 1. Patch by Nikolaus Rath.
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -2710,6 +2710,20 @@
{
void *ptr;
+ ptr = PyMem_RawMalloc(0);
+ if (ptr == NULL) {
+ PyErr_SetString(PyExc_RuntimeError, "PyMem_RawMalloc(0) returns NULL");
+ return NULL;
+ }
+ PyMem_RawFree(ptr);
+
+ ptr = PyMem_RawCalloc(0, 0);
+ if (ptr == NULL) {
+ PyErr_SetString(PyExc_RuntimeError, "PyMem_RawCalloc(0, 0) returns NULL");
+ return NULL;
+ }
+ PyMem_RawFree(ptr);
+
ptr = PyMem_Malloc(0);
if (ptr == NULL) {
PyErr_SetString(PyExc_RuntimeError, "PyMem_Malloc(0) returns NULL");
@@ -2717,6 +2731,13 @@
}
PyMem_Free(ptr);
+ ptr = PyMem_Calloc(0, 0);
+ if (ptr == NULL) {
+ PyErr_SetString(PyExc_RuntimeError, "PyMem_Calloc(0, 0) returns NULL");
+ return NULL;
+ }
+ PyMem_Free(ptr);
+
ptr = PyObject_Malloc(0);
if (ptr == NULL) {
PyErr_SetString(PyExc_RuntimeError, "PyObject_Malloc(0) returns NULL");
@@ -2724,6 +2745,13 @@
}
PyObject_Free(ptr);
+ ptr = PyObject_Calloc(0, 0);
+ if (ptr == NULL) {
+ PyErr_SetString(PyExc_RuntimeError, "PyObject_Calloc(0, 0) returns NULL");
+ return NULL;
+ }
+ PyObject_Free(ptr);
+
Py_RETURN_NONE;
}
@@ -2731,6 +2759,8 @@
PyMemAllocator alloc;
size_t malloc_size;
+ size_t calloc_nelem;
+ size_t calloc_elsize;
void *realloc_ptr;
size_t realloc_new_size;
void *free_ptr;
@@ -2743,6 +2773,14 @@
return hook->alloc.malloc(hook->alloc.ctx, size);
}
+static void* hook_calloc (void* ctx, size_t nelem, size_t elsize)
+{
+ alloc_hook_t *hook = (alloc_hook_t *)ctx;
+ hook->calloc_nelem = nelem;
+ hook->calloc_elsize = elsize;
+ return hook->alloc.calloc(hook->alloc.ctx, nelem, elsize);
+}
+
static void* hook_realloc (void* ctx, void* ptr, size_t new_size)
{
alloc_hook_t *hook = (alloc_hook_t *)ctx;
@@ -2765,16 +2803,14 @@
const char *error_msg;
alloc_hook_t hook;
PyMemAllocator alloc;
- size_t size, size2;
+ size_t size, size2, nelem, elsize;
void *ptr, *ptr2;
- hook.malloc_size = 0;
- hook.realloc_ptr = NULL;
- hook.realloc_new_size = 0;
- hook.free_ptr = NULL;
+ memset(&hook, 0, sizeof(hook));
alloc.ctx = &hook;
alloc.malloc = &hook_malloc;
+ alloc.calloc = &hook_calloc;
alloc.realloc = &hook_realloc;
alloc.free = &hook_free;
PyMem_GetAllocator(domain, &hook.alloc);
@@ -2831,6 +2867,33 @@
goto fail;
}
+ nelem = 2;
+ elsize = 5;
+ switch(domain)
+ {
+ case PYMEM_DOMAIN_RAW: ptr = PyMem_RawCalloc(nelem, elsize); break;
+ case PYMEM_DOMAIN_MEM: ptr = PyMem_Calloc(nelem, elsize); break;
+ case PYMEM_DOMAIN_OBJ: ptr = PyObject_Calloc(nelem, elsize); break;
+ default: ptr = NULL; break;
+ }
+
+ if (ptr == NULL) {
+ error_msg = "calloc failed";
+ goto fail;
+ }
+
+ if (hook.calloc_nelem != nelem || hook.calloc_elsize != elsize) {
+ error_msg = "calloc invalid nelem or elsize";
+ goto fail;
+ }
+
+ switch(domain)
+ {
+ case PYMEM_DOMAIN_RAW: PyMem_RawFree(ptr); break;
+ case PYMEM_DOMAIN_MEM: PyMem_Free(ptr); break;
+ case PYMEM_DOMAIN_OBJ: PyObject_Free(ptr); break;
+ }
+
Py_INCREF(Py_None);
res = Py_None;
goto finally;
diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c
--- a/Modules/_tracemalloc.c
+++ b/Modules/_tracemalloc.c
@@ -476,17 +476,22 @@
}
static void*
-tracemalloc_malloc(void *ctx, size_t size)
+tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize)
{
PyMemAllocator *alloc = (PyMemAllocator *)ctx;
void *ptr;
- ptr = alloc->malloc(alloc->ctx, size);
+ assert(nelem <= PY_SIZE_MAX / elsize);
+
+ if (use_calloc)
+ ptr = alloc->calloc(alloc->ctx, nelem, elsize);
+ else
+ ptr = alloc->malloc(alloc->ctx, nelem * elsize);
if (ptr == NULL)
return NULL;
TABLES_LOCK();
- if (tracemalloc_add_trace(ptr, size) < 0) {
+ if (tracemalloc_add_trace(ptr, nelem * elsize) < 0) {
/* Failed to allocate a trace for the new memory block */
TABLES_UNLOCK();
alloc->free(alloc->ctx, ptr);
@@ -560,13 +565,16 @@
}
static void*
-tracemalloc_malloc_gil(void *ctx, size_t size)
+tracemalloc_alloc_gil(int use_calloc, void *ctx, size_t nelem, size_t elsize)
{
void *ptr;
if (get_reentrant()) {
PyMemAllocator *alloc = (PyMemAllocator *)ctx;
- return alloc->malloc(alloc->ctx, size);
+ if (use_calloc)
+ return alloc->calloc(alloc->ctx, nelem, elsize);
+ else
+ return alloc->malloc(alloc->ctx, nelem * elsize);
}
/* Ignore reentrant call. PyObjet_Malloc() calls PyMem_Malloc() for
@@ -574,13 +582,25 @@
allocation twice. */
set_reentrant(1);
- ptr = tracemalloc_malloc(ctx, size);
+ ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize);
set_reentrant(0);
return ptr;
}
static void*
+tracemalloc_malloc_gil(void *ctx, size_t size)
+{
+ return tracemalloc_alloc_gil(0, ctx, 1, size);
+}
+
+static void*
+tracemalloc_calloc_gil(void *ctx, size_t nelem, size_t elsize)
+{
+ return tracemalloc_alloc_gil(1, ctx, nelem, elsize);
+}
+
+static void*
tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size)
{
void *ptr2;
@@ -614,7 +634,7 @@
#ifdef TRACE_RAW_MALLOC
static void*
-tracemalloc_raw_malloc(void *ctx, size_t size)
+tracemalloc_raw_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize)
{
#ifdef WITH_THREAD
PyGILState_STATE gil_state;
@@ -623,7 +643,10 @@
if (get_reentrant()) {
PyMemAllocator *alloc = (PyMemAllocator *)ctx;
- return alloc->malloc(alloc->ctx, size);
+ if (use_calloc)
+ return alloc->calloc(alloc->ctx, nelem, elsize);
+ else
+ return alloc->malloc(alloc->ctx, nelem * elsize);
}
/* Ignore reentrant call. PyGILState_Ensure() may call PyMem_RawMalloc()
@@ -633,10 +656,10 @@
#ifdef WITH_THREAD
gil_state = PyGILState_Ensure();
- ptr = tracemalloc_malloc(ctx, size);
+ ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize);
PyGILState_Release(gil_state);
#else
- ptr = tracemalloc_malloc(ctx, size);
+ ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize);
#endif
set_reentrant(0);
@@ -644,6 +667,18 @@
}
static void*
+tracemalloc_raw_malloc(void *ctx, size_t size)
+{
+ return tracemalloc_raw_alloc(0, ctx, 1, size);
+}
+
+static void*
+tracemalloc_raw_calloc(void *ctx, size_t nelem, size_t elsize)
+{
+ return tracemalloc_raw_alloc(1, ctx, nelem, elsize);
+}
+
+static void*
tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size)
{
#ifdef WITH_THREAD
@@ -856,6 +891,7 @@
#ifdef TRACE_RAW_MALLOC
alloc.malloc = tracemalloc_raw_malloc;
+ alloc.calloc = tracemalloc_raw_calloc;
alloc.realloc = tracemalloc_raw_realloc;
alloc.free = tracemalloc_free;
@@ -865,6 +901,7 @@
#endif
alloc.malloc = tracemalloc_malloc_gil;
+ alloc.calloc = tracemalloc_calloc_gil;
alloc.realloc = tracemalloc_realloc_gil;
alloc.free = tracemalloc_free;
diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -1703,15 +1703,19 @@
_PyObject_GC_UNTRACK(op);
}
-PyObject *
-_PyObject_GC_Malloc(size_t basicsize)
+static PyObject *
+_PyObject_GC_Alloc(int use_calloc, size_t basicsize)
{
PyObject *op;
PyGC_Head *g;
+ size_t size;
if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
return PyErr_NoMemory();
- g = (PyGC_Head *)PyObject_MALLOC(
- sizeof(PyGC_Head) + basicsize);
+ size = sizeof(PyGC_Head) + basicsize;
+ if (use_calloc)
+ g = (PyGC_Head *)PyObject_Calloc(1, size);
+ else
+ g = (PyGC_Head *)PyObject_Malloc(size);
if (g == NULL)
return PyErr_NoMemory();
g->gc.gc_refs = 0;
@@ -1731,6 +1735,18 @@
}
PyObject *
+_PyObject_GC_Malloc(size_t basicsize)
+{
+ return _PyObject_GC_Alloc(0, basicsize);
+}
+
+PyObject *
+_PyObject_GC_Calloc(size_t basicsize)
+{
+ return _PyObject_GC_Alloc(1, basicsize);
+}
+
+PyObject *
_PyObject_GC_New(PyTypeObject *tp)
{
PyObject *op = _PyObject_GC_Malloc(_PyObject_SIZE(tp));
diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
--- a/Objects/bytearrayobject.c
+++ b/Objects/bytearrayobject.c
@@ -813,9 +813,21 @@
}
else {
if (count > 0) {
- if (PyByteArray_Resize((PyObject *)self, count))
+ void *sval;
+ Py_ssize_t alloc;
+
+ assert (Py_SIZE(self) == 0);
+
+ alloc = count + 1;
+ sval = PyObject_Calloc(1, alloc);
+ if (sval == NULL)
return -1;
- memset(PyByteArray_AS_STRING(self), 0, count);
+
+ PyObject_Free(self->ob_bytes);
+
+ self->ob_bytes = self->ob_start = sval;
+ Py_SIZE(self) = count;
+ self->ob_alloc = alloc;
}
return 0;
}
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -71,6 +71,44 @@
PyBytes_FromStringAndSize()) or the length of the string in the `str'
parameter (for PyBytes_FromString()).
*/
+static PyObject *
+_PyBytes_FromSize(Py_ssize_t size, int use_calloc)
+{
+ PyBytesObject *op;
+ assert(size >= 0);
+ if (size == 0 && (op = nullstring) != NULL) {
+#ifdef COUNT_ALLOCS
+ null_strings++;
+#endif
+ Py_INCREF(op);
+ return (PyObject *)op;
+ }
+
+ if (size > PY_SSIZE_T_MAX - PyBytesObject_SIZE) {
+ PyErr_SetString(PyExc_OverflowError,
+ "byte string is too large");
+ return NULL;
+ }
+
+ /* Inline PyObject_NewVar */
+ if (use_calloc)
+ op = (PyBytesObject *)PyObject_Calloc(1, PyBytesObject_SIZE + size);
+ else
+ op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + size);
+ if (op == NULL)
+ return PyErr_NoMemory();
+ (void)PyObject_INIT_VAR(op, &PyBytes_Type, size);
+ op->ob_shash = -1;
+ if (!use_calloc)
+ op->ob_sval[size] = '\0';
+ /* empty byte string singleton */
+ if (size == 0) {
+ nullstring = op;
+ Py_INCREF(op);
+ }
+ return (PyObject *) op;
+}
+
PyObject *
PyBytes_FromStringAndSize(const char *str, Py_ssize_t size)
{
@@ -80,13 +118,6 @@
"Negative size passed to PyBytes_FromStringAndSize");
return NULL;
}
- if (size == 0 && (op = nullstring) != NULL) {
-#ifdef COUNT_ALLOCS
- null_strings++;
-#endif
- Py_INCREF(op);
- return (PyObject *)op;
- }
if (size == 1 && str != NULL &&
(op = characters[*str & UCHAR_MAX]) != NULL)
{
@@ -97,26 +128,15 @@
return (PyObject *)op;
}
- if (size > PY_SSIZE_T_MAX - PyBytesObject_SIZE) {
- PyErr_SetString(PyExc_OverflowError,
- "byte string is too large");
+ op = (PyBytesObject *)_PyBytes_FromSize(size, 0);
+ if (op == NULL)
return NULL;
- }
-
- /* Inline PyObject_NewVar */
- op = (PyBytesObject *)PyObject_MALLOC(PyBytesObject_SIZE + size);
- if (op == NULL)
- return PyErr_NoMemory();
- (void)PyObject_INIT_VAR(op, &PyBytes_Type, size);
- op->ob_shash = -1;
- if (str != NULL)
- Py_MEMCPY(op->ob_sval, str, size);
- op->ob_sval[size] = '\0';
+ if (str == NULL)
+ return (PyObject *) op;
+
+ Py_MEMCPY(op->ob_sval, str, size);
/* share short strings */
- if (size == 0) {
- nullstring = op;
- Py_INCREF(op);
- } else if (size == 1 && str != NULL) {
+ if (size == 1) {
characters[*str & UCHAR_MAX] = op;
Py_INCREF(op);
}
@@ -2482,7 +2502,7 @@
"argument");
return NULL;
}
- return PyBytes_FromString("");
+ return PyBytes_FromStringAndSize(NULL, 0);
}
if (PyUnicode_Check(x)) {
@@ -2532,11 +2552,9 @@
return NULL;
}
else {
- new = PyBytes_FromStringAndSize(NULL, size);
+ new = _PyBytes_FromSize(size, 1);
if (new == NULL)
return NULL;
- if (size > 0)
- memset(((PyBytesObject*)new)->ob_sval, 0, size);
return new;
}
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -5,6 +5,7 @@
#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */
/* Forward declaration */
static void* _PyMem_DebugMalloc(void *ctx, size_t size);
+static void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize);
static void _PyMem_DebugFree(void *ctx, void *p);
static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size);
@@ -43,6 +44,7 @@
/* Forward declaration */
static void* _PyObject_Malloc(void *ctx, size_t size);
+static void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize);
static void _PyObject_Free(void *ctx, void *p);
static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
#endif
@@ -51,7 +53,7 @@
static void *
_PyMem_RawMalloc(void *ctx, size_t size)
{
- /* PyMem_Malloc(0) means malloc(1). Some systems would return NULL
+ /* PyMem_RawMalloc(0) means malloc(1). Some systems would return NULL
for malloc(0), which would be treated as an error. Some platforms would
return a pointer with no memory behind it, which would break pymalloc.
To solve these problems, allocate an extra byte. */
@@ -61,6 +63,20 @@
}
static void *
+_PyMem_RawCalloc(void *ctx, size_t nelem, size_t elsize)
+{
+ /* PyMem_RawCalloc(0, 0) means calloc(1, 1). Some systems would return NULL
+ for calloc(0, 0), which would be treated as an error. Some platforms
+ would return a pointer with no memory behind it, which would break
+ pymalloc. To solve these problems, allocate an extra byte. */
+ if (nelem == 0 || elsize == 0) {
+ nelem = 1;
+ elsize = 1;
+ }
+ return calloc(nelem, elsize);
+}
+
+static void *
_PyMem_RawRealloc(void *ctx, void *ptr, size_t size)
{
if (size == 0)
@@ -123,9 +139,9 @@
#endif
-#define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawRealloc, _PyMem_RawFree
+#define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree
#ifdef WITH_PYMALLOC
-# define PYOBJ_FUNCS _PyObject_Malloc, _PyObject_Realloc, _PyObject_Free
+# define PYOBJ_FUNCS _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free
#else
# define PYOBJ_FUNCS PYRAW_FUNCS
#endif
@@ -147,7 +163,7 @@
{'o', {NULL, PYOBJ_FUNCS}}
};
-#define PYDBG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugRealloc, _PyMem_DebugFree
+#define PYDBG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree
#endif
static PyMemAllocator _PyMem_Raw = {
@@ -196,6 +212,7 @@
PyMemAllocator alloc;
alloc.malloc = _PyMem_DebugMalloc;
+ alloc.calloc = _PyMem_DebugCalloc;
alloc.realloc = _PyMem_DebugRealloc;
alloc.free = _PyMem_DebugFree;
@@ -228,9 +245,10 @@
case PYMEM_DOMAIN_MEM: *allocator = _PyMem; break;
case PYMEM_DOMAIN_OBJ: *allocator = _PyObject; break;
default:
- /* unknown domain */
+ /* unknown domain: set all attributes to NULL */
allocator->ctx = NULL;
allocator->malloc = NULL;
+ allocator->calloc = NULL;
allocator->realloc = NULL;
allocator->free = NULL;
}
@@ -272,8 +290,16 @@
*/
if (size > (size_t)PY_SSIZE_T_MAX)
return NULL;
+ return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);
+}
- return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);
+void *
+PyMem_RawCalloc(size_t nelem, size_t elsize)
+{
+ /* see PyMem_RawMalloc() */
+ if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+ return NULL;
+ return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize);
}
void*
@@ -300,6 +326,15 @@
}
void *
+PyMem_Calloc(size_t nelem, size_t elsize)
+{
+ /* see PyMem_RawMalloc() */
+ if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+ return NULL;
+ return _PyMem.calloc(_PyMem.ctx, nelem, elsize);
+}
+
+void *
PyMem_Realloc(void *ptr, size_t new_size)
{
/* see PyMem_RawMalloc() */
@@ -352,6 +387,15 @@
}
void *
+PyObject_Calloc(size_t nelem, size_t elsize)
+{
+ /* see PyMem_RawMalloc() */
+ if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
+ return NULL;
+ return _PyObject.calloc(_PyObject.ctx, nelem, elsize);
+}
+
+void *
PyObject_Realloc(void *ptr, size_t new_size)
{
/* see PyMem_RawMalloc() */
@@ -1122,8 +1166,9 @@
*/
static void *
-_PyObject_Malloc(void *ctx, size_t nbytes)
+_PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize)
{
+ size_t nbytes;
block *bp;
poolp pool;
poolp next;
@@ -1138,9 +1183,12 @@
goto redirect;
#endif
- /*
- * This implicitly redirects malloc(0).
- */
+ if (nelem == 0 || elsize == 0)
+ goto redirect;
+
+ assert(nelem <= PY_SSIZE_T_MAX / elsize);
+ nbytes = nelem * elsize;
+
if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) {
LOCK();
/*
@@ -1158,6 +1206,8 @@
assert(bp != NULL);
if ((pool->freeblock = *(block **)bp) != NULL) {
UNLOCK();
+ if (use_calloc)
+ memset(bp, 0, nbytes);
return (void *)bp;
}
/*
@@ -1170,6 +1220,8 @@
pool->nextoffset += INDEX2SIZE(size);
*(block **)(pool->freeblock) = NULL;
UNLOCK();
+ if (use_calloc)
+ memset(bp, 0, nbytes);
return (void *)bp;
}
/* Pool is full, unlink from used pools. */
@@ -1178,6 +1230,8 @@
next->prevpool = pool;
pool->nextpool = next;
UNLOCK();
+ if (use_calloc)
+ memset(bp, 0, nbytes);
return (void *)bp;
}
@@ -1257,6 +1311,8 @@
assert(bp != NULL);
pool->freeblock = *(block **)bp;
UNLOCK();
+ if (use_calloc)
+ memset(bp, 0, nbytes);
return (void *)bp;
}
/*
@@ -1272,6 +1328,8 @@
pool->freeblock = bp + size;
*(block **)(pool->freeblock) = NULL;
UNLOCK();
+ if (use_calloc)
+ memset(bp, 0, nbytes);
return (void *)bp;
}
@@ -1311,13 +1369,29 @@
* has been reached.
*/
{
- void *result = PyMem_RawMalloc(nbytes);
+ void *result;
+ if (use_calloc)
+ result = PyMem_RawCalloc(nelem, elsize);
+ else
+ result = PyMem_RawMalloc(nbytes);
if (!result)
_Py_AllocatedBlocks--;
return result;
}
}
+static void *
+_PyObject_Malloc(void *ctx, size_t nbytes)
+{
+ return _PyObject_Alloc(0, ctx, 1, nbytes);
+}
+
+static void *
+_PyObject_Calloc(void *ctx, size_t nelem, size_t elsize)
+{
+ return _PyObject_Alloc(1, ctx, nelem, elsize);
+}
+
/* free */
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS
@@ -1561,7 +1635,7 @@
#endif
if (p == NULL)
- return _PyObject_Malloc(ctx, nbytes);
+ return _PyObject_Alloc(0, ctx, 1, nbytes);
#ifdef WITH_VALGRIND
/* Treat running_on_valgrind == -1 the same as 0 */
@@ -1589,7 +1663,7 @@
}
size = nbytes;
}
- bp = _PyObject_Malloc(ctx, nbytes);
+ bp = _PyObject_Alloc(0, ctx, 1, nbytes);
if (bp != NULL) {
memcpy(bp, p, size);
_PyObject_Free(ctx, p);
@@ -1745,7 +1819,7 @@
*/
static void *
-_PyMem_DebugMalloc(void *ctx, size_t nbytes)
+_PyMem_DebugAlloc(int use_calloc, void *ctx, size_t nbytes)
{
debug_alloc_api_t *api = (debug_alloc_api_t *)ctx;
uchar *p; /* base address of malloc'ed block */
@@ -1758,7 +1832,10 @@
/* overflow: can't represent total as a size_t */
return NULL;
- p = (uchar *)api->alloc.malloc(api->alloc.ctx, total);
+ if (use_calloc)
+ p = (uchar *)api->alloc.calloc(api->alloc.ctx, 1, total);
+ else
+ p = (uchar *)api->alloc.malloc(api->alloc.ctx, total);
if (p == NULL)
return NULL;
@@ -1767,7 +1844,7 @@
p[SST] = (uchar)api->api_id;
memset(p + SST + 1, FORBIDDENBYTE, SST-1);
- if (nbytes > 0)
+ if (nbytes > 0 && !use_calloc)
memset(p + 2*SST, CLEANBYTE, nbytes);
/* at tail, write pad (SST bytes) and serialno (SST bytes) */
@@ -1778,6 +1855,21 @@
return p + 2*SST;
}
+static void *
+_PyMem_DebugMalloc(void *ctx, size_t nbytes)
+{
+ return _PyMem_DebugAlloc(0, ctx, nbytes);
+}
+
+static void *
+_PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize)
+{
+ size_t nbytes;
+ assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize);
+ nbytes = nelem * elsize;
+ return _PyMem_DebugAlloc(1, ctx, nbytes);
+}
+
/* The debug free first checks the 2*SST bytes on each end for sanity (in
particular, that the FORBIDDENBYTEs with the api ID are still intact).
Then fills the original bytes with DEADBYTE.
@@ -1811,7 +1903,7 @@
int i;
if (p == NULL)
- return _PyMem_DebugMalloc(ctx, nbytes);
+ return _PyMem_DebugAlloc(0, ctx, nbytes);
_PyMem_DebugCheckAddress(api->api_id, p);
bumpserialno();
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list