Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-112532: Require mimalloc in `--disable-gil` builds (gh-112883)

This commit is contained in:
Sam Gross 2023-12-11 19:04:48 -05:00 committed by GitHub
parent fed294c645
commit fdee7b7b3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 76 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Include/internal/pycore_pymem_init.h
View File

@ -18,7 +18,19 @@ extern void * _PyMem_RawRealloc(void *, void *, size_t);
extern void _PyMem_RawFree(void *, void *);
#define PYRAW_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree}
#if defined(WITH_PYMALLOC)
#ifdef Py_GIL_DISABLED
// Py_GIL_DISABLED requires mimalloc
extern void* _PyObject_MiMalloc(void *, size_t);
extern void* _PyObject_MiCalloc(void *, size_t, size_t);
extern void _PyObject_MiFree(void *, void *);
extern void* _PyObject_MiRealloc(void *, void *, size_t);
# define PYOBJ_ALLOC {NULL, _PyObject_MiMalloc, _PyObject_MiCalloc, _PyObject_MiRealloc, _PyObject_MiFree}
extern void* _PyMem_MiMalloc(void *, size_t);
extern void* _PyMem_MiCalloc(void *, size_t, size_t);
extern void _PyMem_MiFree(void *, void *);
extern void* _PyMem_MiRealloc(void *, void *, size_t);
# define PYMEM_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree}
#elif defined(WITH_PYMALLOC)
extern void* _PyObject_Malloc(void *, size_t);
extern void* _PyObject_Calloc(void *, size_t, size_t);
extern void _PyObject_Free(void *, void *);

2
Lib/test/support/__init__.py
View File

@ -1844,7 +1844,7 @@ class SaveSignals:
def with_pymalloc():
import _testcapi
return _testcapi.WITH_PYMALLOC
return _testcapi.WITH_PYMALLOC and not Py_GIL_DISABLED
def with_mimalloc():

7
Lib/test/test_capi/test_mem.py
View File

@ -152,6 +152,8 @@ class PyMemDebugTests(unittest.TestCase):
self.assertGreaterEqual(count, i*5-2)
# Py_GIL_DISABLED requires mimalloc (not malloc)
@unittest.skipIf(support.Py_GIL_DISABLED, 'need malloc')
class PyMemMallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'malloc_debug'
@ -161,6 +163,11 @@ class PyMemPymallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'pymalloc_debug'
@unittest.skipUnless(support.with_mimalloc(), 'need mimaloc')
class PyMemMimallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'mimalloc_debug'
@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
class PyMemDefaultTests(PyMemDebugTests):
# test default allocator of Python compiled in debug mode

13
Lib/test/test_cmd_line.py
View File

@ -738,6 +738,8 @@ class CmdLineTest(unittest.TestCase):
out = self.run_xdev("-c", code, check_exitcode=False)
if support.with_pymalloc():
alloc_name = "pymalloc_debug"
elif support.Py_GIL_DISABLED:
alloc_name = "mimalloc_debug"
else:
alloc_name = "malloc_debug"
self.assertEqual(out, alloc_name)
@ -814,9 +816,13 @@ class CmdLineTest(unittest.TestCase):
@support.cpython_only
def test_pythonmalloc(self):
# Test the PYTHONMALLOC environment variable
malloc = not support.Py_GIL_DISABLED
pymalloc = support.with_pymalloc()
mimalloc = support.with_mimalloc()
if pymalloc:
if support.Py_GIL_DISABLED:
default_name = 'mimalloc_debug' if support.Py_DEBUG else 'mimalloc'
default_name_debug = 'mimalloc_debug'
elif pymalloc:
default_name = 'pymalloc_debug' if support.Py_DEBUG else 'pymalloc'
default_name_debug = 'pymalloc_debug'
else:
@ -826,9 +832,12 @@ class CmdLineTest(unittest.TestCase):
tests = [
(None, default_name),
('debug', default_name_debug),
]
if malloc:
tests.extend([
('malloc', 'malloc'),
('malloc_debug', 'malloc_debug'),
]
])
if pymalloc:
tests.extend((
('pymalloc', 'pymalloc'),

14
Lib/test/test_embed.py
View File

@ -23,6 +23,12 @@ MACOS = (sys.platform == 'darwin')
PYMEM_ALLOCATOR_NOT_SET = 0
PYMEM_ALLOCATOR_DEBUG = 2
PYMEM_ALLOCATOR_MALLOC = 3
PYMEM_ALLOCATOR_MIMALLOC = 7
if support.Py_GIL_DISABLED:
ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MIMALLOC
else:
ALLOCATOR_FOR_CONFIG = PYMEM_ALLOCATOR_MALLOC
Py_STATS = hasattr(sys, '_stats_on')
# _PyCoreConfig_InitCompatConfig()
@ -841,7 +847,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
def test_init_from_config(self):
preconfig = {
'allocator': PYMEM_ALLOCATOR_MALLOC,
'allocator': ALLOCATOR_FOR_CONFIG,
'utf8_mode': 1,
}
config = {
@ -908,7 +914,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
def test_init_compat_env(self):
preconfig = {
'allocator': PYMEM_ALLOCATOR_MALLOC,
'allocator': ALLOCATOR_FOR_CONFIG,
}
config = {
'use_hash_seed': 1,
@ -942,7 +948,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
def test_init_python_env(self):
preconfig = {
'allocator': PYMEM_ALLOCATOR_MALLOC,
'allocator': ALLOCATOR_FOR_CONFIG,
'utf8_mode': 1,
}
config = {
@ -984,7 +990,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
api=API_COMPAT)
def test_init_env_dev_mode_alloc(self):
preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC)
preconfig = dict(allocator=ALLOCATOR_FOR_CONFIG)
config = dict(dev_mode=1,
faulthandler=1,
warnoptions=['default'])

3
Lib/test/test_os.py
View File

@ -5080,6 +5080,9 @@ class ForkTests(unittest.TestCase):
support.wait_process(pid, exitcode=0)
"""
assert_python_ok("-c", code)
if support.Py_GIL_DISABLED:
assert_python_ok("-c", code, PYTHONMALLOC="mimalloc_debug")
else:
assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug")
@unittest.skipUnless(sys.platform in ("linux", "darwin"),

15
Objects/obmalloc.c
View File

@ -16,6 +16,10 @@
# include "mimalloc/internal.h" // for stats
#endif
#if defined(Py_GIL_DISABLED) && !defined(WITH_MIMALLOC)
# error "Py_GIL_DISABLED requires WITH_MIMALLOC"
#endif
#undef uint
#define uint pymem_uint
@ -153,7 +157,12 @@ void* _PyObject_Realloc(void *ctx, void *ptr, size_t size);
# define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free}
#endif // WITH_PYMALLOC
#if defined(WITH_PYMALLOC)
#if defined(Py_GIL_DISABLED)
// Py_GIL_DISABLED requires using mimalloc for "mem" and "obj" domains.
# define PYRAW_ALLOC MALLOC_ALLOC
# define PYMEM_ALLOC MIMALLOC_ALLOC
# define PYOBJ_ALLOC MIMALLOC_OBJALLOC
#elif defined(WITH_PYMALLOC)
# define PYRAW_ALLOC MALLOC_ALLOC
# define PYMEM_ALLOC PYMALLOC_ALLOC
# define PYOBJ_ALLOC PYMALLOC_ALLOC
@ -350,7 +359,7 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)
else if (strcmp(name, "debug") == 0) {
*allocator = PYMEM_ALLOCATOR_DEBUG;
}
#ifdef WITH_PYMALLOC
#if defined(WITH_PYMALLOC) && !defined(Py_GIL_DISABLED)
else if (strcmp(name, "pymalloc") == 0) {
*allocator = PYMEM_ALLOCATOR_PYMALLOC;
}
@ -366,12 +375,14 @@ _PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator)
*allocator = PYMEM_ALLOCATOR_MIMALLOC_DEBUG;
}
#endif
#ifndef Py_GIL_DISABLED
else if (strcmp(name, "malloc") == 0) {
*allocator = PYMEM_ALLOCATOR_MALLOC;
}
else if (strcmp(name, "malloc_debug") == 0) {
*allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG;
}
#endif
else {
/* unknown allocator */
return -1;

12
Programs/_testembed.c
View File

@ -576,7 +576,11 @@ static int test_init_from_config(void)
_PyPreConfig_InitCompatConfig(&preconfig);
putenv("PYTHONMALLOC=malloc_debug");
#ifndef Py_GIL_DISABLED
preconfig.allocator = PYMEM_ALLOCATOR_MALLOC;
#else
preconfig.allocator = PYMEM_ALLOCATOR_MIMALLOC;
#endif
putenv("PYTHONUTF8=0");
Py_UTF8Mode = 0;
@ -765,7 +769,11 @@ static int test_init_dont_parse_argv(void)
static void set_most_env_vars(void)
{
putenv("PYTHONHASHSEED=42");
#ifndef Py_GIL_DISABLED
putenv("PYTHONMALLOC=malloc");
#else
putenv("PYTHONMALLOC=mimalloc");
#endif
putenv("PYTHONTRACEMALLOC=2");
putenv("PYTHONPROFILEIMPORTTIME=1");
putenv("PYTHONNODEBUGRANGES=1");
@ -851,7 +859,11 @@ static int test_init_env_dev_mode_alloc(void)
/* Test initialization from environment variables */
Py_IgnoreEnvironmentFlag = 0;
set_all_env_vars_dev_mode();
#ifndef Py_GIL_DISABLED
putenv("PYTHONMALLOC=malloc");
#else
putenv("PYTHONMALLOC=mimalloc");
#endif
_testembed_Py_InitializeFromConfig();
dump_config();
Py_Finalize();

2
configure generated vendored
View File

@ -16891,6 +16891,8 @@ printf "%s\n" "#define WITH_MIMALLOC 1" >>confdefs.h
MIMALLOC_HEADERS='$(MIMALLOC_HEADERS)'
elif test "$disable_gil" = "yes"; then
as_fn_error $? "--disable-gil requires mimalloc memory allocator (--with-mimalloc)." "$LINENO" 5
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_mimalloc" >&5

2
configure.ac
View File

@ -4558,6 +4558,8 @@ if test "$with_mimalloc" != no; then
with_mimalloc=yes
AC_DEFINE([WITH_MIMALLOC], [1], [Define if you want to compile in mimalloc memory allocator.])
AC_SUBST([MIMALLOC_HEADERS], ['$(MIMALLOC_HEADERS)'])
elif test "$disable_gil" = "yes"; then
AC_MSG_ERROR([--disable-gil requires mimalloc memory allocator (--with-mimalloc).])
fi
AC_MSG_RESULT([$with_mimalloc])