cpython/Include/pymem.h

232 lines
8.2 KiB
C
Raw Normal View History

/* The PyMem_ family: low-level memory allocation interfaces.
See objimpl.h for the PyObject_ memory family.
*/
#ifndef Py_PYMEM_H
#define Py_PYMEM_H
#include "pyport.h"
#ifdef __cplusplus
extern "C" {
#endif
#ifndef Py_LIMITED_API
PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
/* Configure the Python memory allocators. Pass NULL to use default
allocators. */
PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt);
#ifdef WITH_PYMALLOC
PyAPI_FUNC(int) _PyMem_PymallocEnabled(void);
#endif
/* Identifier of an address space (domain) in tracemalloc */
typedef unsigned int _PyTraceMalloc_domain_t;
/* Track an allocated memory block in the tracemalloc module.
Return 0 on success, return -1 on error (failed to allocate memory to store
the trace).
Return -2 if tracemalloc is disabled.
If memory block is already tracked, update the existing trace. */
PyAPI_FUNC(int) _PyTraceMalloc_Track(
_PyTraceMalloc_domain_t domain,
uintptr_t ptr,
size_t size);
/* Untrack an allocated memory block in the tracemalloc module.
Do nothing if the block was not tracked.
Return -2 if tracemalloc is disabled, otherwise return 0. */
PyAPI_FUNC(int) _PyTraceMalloc_Untrack(
_PyTraceMalloc_domain_t domain,
uintptr_t ptr);
/* Get the traceback where a memory block was allocated.
Return a tuple of (filename: str, lineno: int) tuples.
Return None if the tracemalloc module is disabled or if the memory block
is not tracked by tracemalloc.
Raise an exception and return NULL on error. */
PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback(
_PyTraceMalloc_domain_t domain,
uintptr_t ptr);
#endif /* !Py_LIMITED_API */
/* BEWARE:
Each interface exports both functions and macros. Extension modules should
use the functions, to ensure binary compatibility across Python versions.
Because the Python implementation is free to change internal details, and
the macros may (or may not) expose details for speed, if you do use the
macros you must recompile your extensions with each Python release.
Never mix calls to PyMem_ with calls to the platform malloc/realloc/
calloc/free. For example, on Windows different DLLs may end up using
different heaps, and if you use PyMem_Malloc you'll get the memory from the
heap used by the Python DLL; it could be a disaster if you free()'ed that
directly in your own extension. Using PyMem_Free instead ensures Python
can return the memory to the proper heap. As another example, in
PYMALLOC_DEBUG mode, Python wraps all calls to all PyMem_ and PyObject_
memory functions in special debugging wrappers that add additional
debugging info to dynamic memory blocks. The system routines have no idea
what to do with that stuff, and the Python wrappers have no idea what to do
with raw blocks obtained directly by the system routines then.
Merged revisions 55325-55327 via svnmerge from svn+ssh://pythondev@svn.python.org/python/branches/p3yk ................ r55326 | guido.van.rossum | 2007-05-14 15:07:35 -0700 (Mon, 14 May 2007) | 2 lines Don't use err.message, use err.args[0]. ................ r55327 | guido.van.rossum | 2007-05-14 15:11:37 -0700 (Mon, 14 May 2007) | 259 lines Merged revisions 54988-55226,55228-55323 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r54995 | neal.norwitz | 2007-04-26 23:45:32 -0700 (Thu, 26 Apr 2007) | 3 lines This gets the test working on Solaris. It seems a little hokey to me, but the test passed on Linux and Solaris, hopefully other platforms too. ........ r55002 | georg.brandl | 2007-04-27 12:20:00 -0700 (Fri, 27 Apr 2007) | 2 lines Version fix (bug #1708710) ........ r55021 | neal.norwitz | 2007-04-29 16:53:24 -0700 (Sun, 29 Apr 2007) | 1 line There really are some tests that are problematic. ........ r55024 | kristjan.jonsson | 2007-04-30 08:17:46 -0700 (Mon, 30 Apr 2007) | 1 line Complete revamp of PCBuild8 directory. Use subdirectories for each project under the main pcbuild solution. Now make extensive use of property sheets to simplify project configuration. x64 build fully supported, and the process for building PGO version (Profiler Guided Optimization) simplified. All projects are now present, except _ssl, which needs to be reimplemented. Also, some of the projects that require external libraries need extra work to fully compile on x64. ........ r55025 | thomas.heller | 2007-04-30 08:44:17 -0700 (Mon, 30 Apr 2007) | 4 lines Make sure to call PyErr_NoMemory() in several places where PyMem_Malloc() could potentially fail. Will backport to the release25-maint branch. ........ r55027 | thomas.heller | 2007-04-30 09:04:57 -0700 (Mon, 30 Apr 2007) | 8 lines When accessing the .value attribute of a c_wchar_p instance, and the instance does not point to a valid wchar_t zero-terminated string, raise a ValueError. c_char_p does this already. The ValueError message now contains the correct pointer address. Will backport to release25-maint. ........ r55036 | georg.brandl | 2007-04-30 23:04:11 -0700 (Mon, 30 Apr 2007) | 2 lines Bug #1710295: exceptions are now new-style classes. ........ r55038 | georg.brandl | 2007-04-30 23:08:15 -0700 (Mon, 30 Apr 2007) | 2 lines Patch #1710352: add missing public functions to locale.__all__. ........ r55041 | vinay.sajip | 2007-05-01 03:20:03 -0700 (Tue, 01 May 2007) | 1 line Added new optional credentials argument to SMTPHandler.__init__, and smtp.login() is now called in SMTPHandler.emit() if credentials are specified. ........ r55042 | vinay.sajip | 2007-05-01 03:21:45 -0700 (Tue, 01 May 2007) | 1 line Added documentation for new optional credentials argument to SMTPHandler.__init__(). ........ r55070 | neal.norwitz | 2007-05-01 21:47:55 -0700 (Tue, 01 May 2007) | 3 lines Stop using PyMem_FREE while the GIL is not held. For details see: http://mail.python.org/pipermail/python-dev/2007-May/072896.html ........ r55080 | armin.rigo | 2007-05-02 12:23:31 -0700 (Wed, 02 May 2007) | 6 lines Fix for #1303614 and #1174712: - __dict__ descriptor abuse for subclasses of built-in types - subclassing from both ModuleType and another built-in types Thanks zseil for the patch. ........ r55083 | georg.brandl | 2007-05-02 13:02:29 -0700 (Wed, 02 May 2007) | 3 lines Actually raise an exception before calling ast_error_finish. Triggers an assertion otherwise. ........ r55087 | neal.norwitz | 2007-05-02 23:47:18 -0700 (Wed, 02 May 2007) | 1 line Handle a couple of uncaught errors. This should be backported ........ r55090 | neal.norwitz | 2007-05-03 00:20:57 -0700 (Thu, 03 May 2007) | 4 lines Remove dead code. This code couldn't be reached because earlier in the function there is another check for z != Py_None. ........ r55092 | thomas.heller | 2007-05-03 05:02:08 -0700 (Thu, 03 May 2007) | 1 line Fix building _ctypes.pyd for x64 / Windows. ........ r55093 | thomas.heller | 2007-05-03 05:05:20 -0700 (Thu, 03 May 2007) | 1 line Don't truncate pointers to integers (on win64 platform). ........ r55094 | walter.doerwald | 2007-05-03 08:13:55 -0700 (Thu, 03 May 2007) | 3 lines Clarify the behaviour of PyUnicode_DecodeUTF16(): A BOM is only skipped in native order mode, and only if it's the first two bytes. ........ r55101 | kristjan.jonsson | 2007-05-03 13:04:53 -0700 (Thu, 03 May 2007) | 2 lines Fix pcbuild8 after recent overhaul: Added the version resource to python26.dll. Adjust stacksize to 2Mb and made large address aware for 32 bits, and set stacksize to 3Mb for 64 bits. Todo: Set .dll optimized load addresses, and side-by-side packaging of the python26.dll. ........ r55102 | kristjan.jonsson | 2007-05-03 13:09:56 -0700 (Thu, 03 May 2007) | 1 line Fix those parts in the testsuite that assumed that sys.maxint would cause overflow on x64. Now the testsuite is well behaved on that platform. ........ r55103 | kristjan.jonsson | 2007-05-03 13:27:03 -0700 (Thu, 03 May 2007) | 11 lines Fix problems in x64 build that were discovered by the testsuite: - Reenable modules on x64 that had been disabled aeons ago for Itanium. - Cleared up confusion about compilers for 64 bit windows. There is only Itanium and x64. Added macros MS_WINI64 and MS_WINX64 for those rare cases where it matters, such as the disabling of modules above. - Set target platform (_WIN32_WINNT and WINVER) to 0x0501 (XP) for x64, and 0x0400 (NT 4.0) otherwise, which are the targeted minimum platforms. - Fixed thread_nt.h. The emulated InterlockedCompareExchange function didn?\194?\180t work on x64, probaby due to the lack of a "volatile" specifier. Anyway, win95 is no longer a target platform. - Itertools module used wrong constant to check for overflow in count() - PyInt_AsSsize_t couldn't deal with attribute error when accessing the __long__ member. - PyLong_FromSsize_t() incorrectly specified that the operand were unsigned. With these changes, the x64 passes the testsuite, for those modules present. ........ r55107 | kristjan.jonsson | 2007-05-03 17:25:08 -0700 (Thu, 03 May 2007) | 1 line Revert compiler comment to AMD64 for x64/AMD64 builds. ........ r55115 | thomas.heller | 2007-05-04 00:14:39 -0700 (Fri, 04 May 2007) | 4 lines Fix some ctypes test crashes, when running with a debug Python version on win64 by using proper argtypes and restype function attributes. ........ r55117 | thomas.heller | 2007-05-04 01:20:41 -0700 (Fri, 04 May 2007) | 4 lines On 64-bit Windows, ffi_arg must be 8 bytes long. This fixes the remaining crashes in the ctypes tests, when functions return float or double types. ........ r55120 | kristjan.jonsson | 2007-05-04 08:48:15 -0700 (Fri, 04 May 2007) | 1 line Update the pcbuild8 solution. Straightened out the _ctypes project by using a .vsproj file and a masm64.rules file to avoid redundancy ........ r55121 | kristjan.jonsson | 2007-05-04 10:28:06 -0700 (Fri, 04 May 2007) | 1 line Minor fix of PCBuild8/_ctypes vcproj, moving include dir into the .vsprops file. ........ r55129 | thomas.heller | 2007-05-04 12:54:22 -0700 (Fri, 04 May 2007) | 3 lines Do not truncate 64-bit pointers to 32-bit integers. Fixes SF #1703286, will backport to release25-maint. ........ r55131 | thomas.heller | 2007-05-04 12:56:32 -0700 (Fri, 04 May 2007) | 1 line Oops, these tests do not run on Windows CE. ........ r55140 | brett.cannon | 2007-05-04 18:34:02 -0700 (Fri, 04 May 2007) | 2 lines Deprecate BaseException.message as per PEP 352. ........ r55154 | georg.brandl | 2007-05-05 11:55:37 -0700 (Sat, 05 May 2007) | 2 lines Bug #1713535: typo in logging example. ........ r55158 | vinay.sajip | 2007-05-06 10:53:37 -0700 (Sun, 06 May 2007) | 1 line Updates of recent changes to logging. ........ r55165 | neal.norwitz | 2007-05-07 00:02:26 -0700 (Mon, 07 May 2007) | 1 line Verify changes to the trunk go to the normal checkins list ........ r55169 | kristjan.jonsson | 2007-05-07 09:46:54 -0700 (Mon, 07 May 2007) | 1 line As per Armin Rigo's suggestion, remove special handing from intobject.c to deal with the peculiarities of classobject's implementation of the number protocol. The nb_long method of classobject now falls back to nb_int if there is no __long__ attribute present. ........ r55197 | collin.winter | 2007-05-08 21:14:36 -0700 (Tue, 08 May 2007) | 9 lines Fix a bug in test.test_support.open_urlresource(). If the call to requires() doesn't precede the filesystem check, we get the following situation: 1. ./python Lib/test/regrtest.py test_foo # test needs urlfetch, not enabled, so skipped 2. ./python Lib/test/regrtest.py -u urlfetch test_foo # test runs 3. ./python Lib/test/regrtest.py test_foo # test runs (!) By moving the call to requires() *before* the filesystem check, the fact that fetched files are cached on the local disk becomes an implementation detail, rather than a semantics-changing point of note. ........ r55198 | neal.norwitz | 2007-05-08 23:43:15 -0700 (Tue, 08 May 2007) | 1 line Add markup for True/False. Will backport ........ r55205 | walter.doerwald | 2007-05-09 11:10:47 -0700 (Wed, 09 May 2007) | 4 lines Backport checkin: Fix a segfault when b"" was passed to b2a_qp() -- it was using strchr() instead of memchr(). ........ r55241 | neal.norwitz | 2007-05-10 22:55:15 -0700 (Thu, 10 May 2007) | 6 lines Don't ever report a failure when the sum of the reference count differences are zero. This should help reduce the false positives. The message about references leaking is maintained to provide as much info as possible rather than simply suppressing the message at the source. ........ r55242 | neal.norwitz | 2007-05-10 23:23:01 -0700 (Thu, 10 May 2007) | 1 line Fix typo in docstring (the module is popen2, not 3). ........ r55244 | neal.norwitz | 2007-05-10 23:56:52 -0700 (Thu, 10 May 2007) | 1 line Remove trailing whitespace in docstring ........ r55245 | neal.norwitz | 2007-05-10 23:57:33 -0700 (Thu, 10 May 2007) | 1 line Deprecate os.popen* and popen2 module in favor of the subprocess module. ........ r55247 | neal.norwitz | 2007-05-11 00:13:30 -0700 (Fri, 11 May 2007) | 1 line Deprecate os.popen* and popen2 module in favor of the subprocess module. (forgot the doc) ........ r55253 | georg.brandl | 2007-05-11 02:41:37 -0700 (Fri, 11 May 2007) | 3 lines Remove an XXX that is unnecessary. ........ r55258 | georg.brandl | 2007-05-11 04:04:26 -0700 (Fri, 11 May 2007) | 2 lines Patch #1714700: clarify os.linesep vs. tfiles opened in text mode. (backport) ........ r55259 | georg.brandl | 2007-05-11 04:43:56 -0700 (Fri, 11 May 2007) | 2 lines Update DDJ link. ........ r55273 | raymond.hettinger | 2007-05-11 10:59:59 -0700 (Fri, 11 May 2007) | 1 line Better tests for posixpath.commonprefix ........ r55287 | georg.brandl | 2007-05-12 14:06:41 -0700 (Sat, 12 May 2007) | 2 lines Bug #1046945: document SWIG options of distutils. ........ r55290 | georg.brandl | 2007-05-13 01:04:07 -0700 (Sun, 13 May 2007) | 2 lines Add bz2 to content encodings. ........ r55297 | neal.norwitz | 2007-05-13 13:45:05 -0700 (Sun, 13 May 2007) | 3 lines Remove Amoeba doc which was removed in version 1.0! according to Misc/HISTORY. Hopefully Guido won't shed a tear. :-) ........ r55298 | neal.norwitz | 2007-05-13 13:54:19 -0700 (Sun, 13 May 2007) | 1 line Remove references to stdwin which was removed long ago. ........ r55299 | neal.norwitz | 2007-05-13 14:13:42 -0700 (Sun, 13 May 2007) | 3 lines Remove support for freebsd[23] which haven't been released since 2000 or earlier. http://www.freebsd.org/releases/index.html ........ r55320 | raymond.hettinger | 2007-05-14 13:52:31 -0700 (Mon, 14 May 2007) | 1 line Small speedup. ........ ................
2007-05-14 19:51:27 -03:00
The GIL must be held when using these APIs.
*/
/*
* Raw memory interface
* ====================
*/
/* Functions
Functions supplying platform-independent semantics for malloc/realloc/
free. These functions make sure that allocating 0 bytes returns a distinct
non-NULL pointer (whenever possible -- if we're flat out of memory, NULL
may be returned), even if the platform malloc and realloc don't.
Returned pointers must be checked for NULL explicitly. No action is
performed on failure (no exception is set, no warning is printed, etc).
*/
PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_Free(void *ptr);
#ifndef Py_LIMITED_API
PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str);
PyAPI_FUNC(char *) _PyMem_Strdup(const char *str);
#endif
/* Macros. */
2002-11-23 05:13:40 -04:00
/* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL
for malloc(0), which would be treated as an error. Some platforms
would return a pointer with no memory behind it, which would break
pymalloc. To solve these problems, allocate an extra byte. */
/* Returns NULL to indicate error if a negative size or size larger than
Py_ssize_t can represent is supplied. Helps prevents security holes. */
#define PyMem_MALLOC(n) PyMem_Malloc(n)
#define PyMem_REALLOC(p, n) PyMem_Realloc(p, n)
#define PyMem_FREE(p) PyMem_Free(p)
/*
* Type-oriented memory interface
* ==============================
*
* Allocate memory for n objects of the given type. Returns a new pointer
* or NULL if the request was too large or memory allocation failed. Use
* these macros rather than doing the multiplication yourself so that proper
* overflow checking is always done.
*/
#define PyMem_New(type, n) \
( ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \
( (type *) PyMem_Malloc((n) * sizeof(type)) ) )
#define PyMem_NEW(type, n) \
( ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \
( (type *) PyMem_MALLOC((n) * sizeof(type)) ) )
/*
* The value of (p) is always clobbered by this macro regardless of success.
* The caller MUST check if (p) is NULL afterwards and deal with the memory
* error if so. This means the original value of (p) MUST be saved for the
* caller's memory error handler to not lose track of it.
*/
#define PyMem_Resize(p, type, n) \
( (p) = ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \
(type *) PyMem_Realloc((p), (n) * sizeof(type)) )
#define PyMem_RESIZE(p, type, n) \
( (p) = ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \
(type *) PyMem_REALLOC((p), (n) * sizeof(type)) )
/* PyMem{Del,DEL} are left over from ancient days, and shouldn't be used
* anymore. They're just confusing aliases for PyMem_{Free,FREE} now.
*/
#define PyMem_Del PyMem_Free
#define PyMem_DEL PyMem_FREE
Give Python a debug-mode pymalloc, much as sketched on Python-Dev. When WITH_PYMALLOC is defined, define PYMALLOC_DEBUG to enable the debug allocator. This can be done independent of build type (release or debug). A debug build automatically defines PYMALLOC_DEBUG when pymalloc is enabled. It's a detected error to define PYMALLOC_DEBUG when pymalloc isn't enabled. Two debugging entry points defined only under PYMALLOC_DEBUG: + _PyMalloc_DebugCheckAddress(const void *p) can be used (e.g., from gdb) to sanity-check a memory block obtained from pymalloc. It sprays info to stderr (see next) and dies via Py_FatalError if the block is detectably damaged. + _PyMalloc_DebugDumpAddress(const void *p) can be used to spray info about a debug memory block to stderr. A tiny start at implementing "API family" checks isn't good for anything yet. _PyMalloc_DebugRealloc() has been optimized to do little when the new size is <= old size. However, if the new size is larger, it really can't call the underlying realloc() routine without either violating its contract, or knowing something non-trivial about how the underlying realloc() works. A memcpy is always done in this case. This was a disaster for (and only) one of the std tests: test_bufio creates single text file lines up to a million characters long. On Windows, fileobject.c's get_line() uses the horridly funky getline_via_fgets(), which keeps growing and growing a string object hoping to find a newline. It grew the string object 1000 bytes each time, so for a million-character string it took approximately forever (I gave up after a few minutes). So, also: fileobject.c, getline_via_fgets(): When a single line is outrageously long, grow the string object at a mildly exponential rate, instead of just 1000 bytes at a time. That's enough so that a debug-build test_bufio finishes in about 5 seconds on my Win98SE box. I'm curious to try this on Win2K, because it has very different memory behavior than Win9X, and test_bufio always took a factor of 10 longer to complete on Win2K. It *could* be that the endless reallocs were simply killing it on Win2K even in the release build.
2002-03-23 06:03:50 -04:00
#ifndef Py_LIMITED_API
typedef enum {
/* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */
PYMEM_DOMAIN_RAW,
/* PyMem_Malloc(), PyMem_Realloc() and PyMem_Free() */
PYMEM_DOMAIN_MEM,
/* PyObject_Malloc(), PyObject_Realloc() and PyObject_Free() */
PYMEM_DOMAIN_OBJ
} PyMemAllocatorDomain;
typedef struct {
/* user context passed as the first argument to the 4 functions */
void *ctx;
/* allocate a memory block */
void* (*malloc) (void *ctx, size_t size);
/* allocate a memory block initialized by zeros */
void* (*calloc) (void *ctx, size_t nelem, size_t elsize);
/* allocate or resize a memory block */
void* (*realloc) (void *ctx, void *ptr, size_t new_size);
/* release a memory block */
void (*free) (void *ctx, void *ptr);
} PyMemAllocatorEx;
/* Get the memory block allocator of the specified domain. */
PyAPI_FUNC(void) PyMem_GetAllocator(PyMemAllocatorDomain domain,
PyMemAllocatorEx *allocator);
/* Set the memory block allocator of the specified domain.
The new allocator must return a distinct non-NULL pointer when requesting
zero bytes.
For the PYMEM_DOMAIN_RAW domain, the allocator must be thread-safe: the GIL
is not held when the allocator is called.
If the new allocator is not a hook (don't call the previous allocator), the
PyMem_SetupDebugHooks() function must be called to reinstall the debug hooks
on top on the new allocator. */
PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain,
PyMemAllocatorEx *allocator);
/* Setup hooks to detect bugs in the following Python memory allocator
functions:
- PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
- PyMem_Malloc(), PyMem_Realloc(), PyMem_Free()
- PyObject_Malloc(), PyObject_Realloc() and PyObject_Free()
Newly allocated memory is filled with the byte 0xCB, freed memory is filled
with the byte 0xDB. Additionnal checks:
- detect API violations, ex: PyObject_Free() called on a buffer allocated
by PyMem_Malloc()
- detect write before the start of the buffer (buffer underflow)
- detect write after the end of the buffer (buffer overflow)
The function does nothing if Python is not compiled is debug mode. */
PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);
#endif
#ifdef __cplusplus
}
#endif
#endif /* !Py_PYMEM_H */