From 43fdbd2729cb7cdbb5afb5d16352f6604859e564 Mon Sep 17 00:00:00 2001 From: Zackery Spytz Date: Wed, 29 May 2019 13:57:07 -0600 Subject: [PATCH] bpo-26836: Add os.memfd_create() (#13567) * bpo-26836: Add os.memfd_create() * Use the glibc wrapper for memfd_create() Co-Authored-By: Christian Heimes * Fix deletions caused by autoreconf. * Use MFD_CLOEXEC as the default value for *flags*. * Add memset_s to configure.ac. * Revert memset_s changes. * Apply the requested changes. * Tweak the docs. --- Doc/library/os.rst | 38 +++++++++++ Doc/whatsnew/3.8.rst | 4 ++ Lib/test/test_os.py | 16 +++++ .../2019-05-25-08-18-01.bpo-26836.rplYWW.rst | 1 + Modules/clinic/posixmodule.c.h | 61 +++++++++++++++++- Modules/posixmodule.c | 64 +++++++++++++++++++ aclocal.m4 | 10 +-- configure | 49 +++++++++++++- configure.ac | 16 ++++- pyconfig.h.in | 12 +++- 10 files changed, 259 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-05-25-08-18-01.bpo-26836.rplYWW.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 6df2b49c532..b53fd71e65b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2982,6 +2982,44 @@ features: Added support for :class:`bytes` paths. +.. function:: memfd_create(name[, flags=os.MFD_CLOEXEC]) + + Create an anonymous file and return a file descriptor that refers to it. + *flags* must be one of the ``os.MFD_*`` constants available on the system + (or a bitwise ORed combination of them). By default, the new file + descriptor is :ref:`non-inheritable `. + + .. availability:: Linux 3.17 or newer with glibc 2.27 or newer. + + .. versionadded:: 3.8 + + +.. data:: MFD_CLOEXEC + MFD_ALLOW_SEALING + MFD_HUGETLB + MFD_HUGE_SHIFT + MFD_HUGE_MASK + MFD_HUGE_64KB + MFD_HUGE_512KB + MFD_HUGE_1MB + MFD_HUGE_2MB + MFD_HUGE_8MB + MFD_HUGE_16MB + MFD_HUGE_32MB + MFD_HUGE_256MB + MFD_HUGE_512MB + MFD_HUGE_1GB + MFD_HUGE_2GB + MFD_HUGE_16GB + + These flags can be passed to :func:`memfd_create`. + + .. availability:: Linux 3.17 or newer with glibc 2.27 or newer. The + ``MFD_HUGE*`` flags are only available since Linux 4.14. + + .. versionadded:: 3.8 + + Linux extended attributes ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index f704b47098e..b4a80309170 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -486,6 +486,10 @@ Added new function :func:`~os.add_dll_directory` on Windows for providing additional search paths for native dependencies when importing extension modules or loading DLLs using :mod:`ctypes`. +A new :func:`os.memfd_create` function was added to wrap the +``memfd_create()`` syscall. +(Contributed by Zackery Spytz and Christian Heimes in :issue:`26836`.) + os.path ------- diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 353b9a50a2b..820c99c7a07 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3122,6 +3122,22 @@ class TermsizeTests(unittest.TestCase): self.assertEqual(expected, actual) +@unittest.skipUnless(hasattr(os, 'memfd_create'), 'requires os.memfd_create') +class MemfdCreateTests(unittest.TestCase): + def test_memfd_create(self): + fd = os.memfd_create("Hi", os.MFD_CLOEXEC) + self.assertNotEqual(fd, -1) + self.addCleanup(os.close, fd) + self.assertFalse(os.get_inheritable(fd)) + with open(fd, "wb", closefd=False) as f: + f.write(b'memfd_create') + self.assertEqual(f.tell(), 12) + + fd2 = os.memfd_create("Hi") + self.addCleanup(os.close, fd2) + self.assertFalse(os.get_inheritable(fd2)) + + class OSErrorTests(unittest.TestCase): def setUp(self): class Str(str): diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-25-08-18-01.bpo-26836.rplYWW.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-25-08-18-01.bpo-26836.rplYWW.rst new file mode 100644 index 00000000000..bbf63ec82ca --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-25-08-18-01.bpo-26836.rplYWW.rst @@ -0,0 +1 @@ +Add :func:`os.memfd_create`. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index f2745591b23..13f25460b4f 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -7343,6 +7343,61 @@ exit: return return_value; } +#if defined(HAVE_MEMFD_CREATE) + +PyDoc_STRVAR(os_memfd_create__doc__, +"memfd_create($module, /, name, flags=MFD_CLOEXEC)\n" +"--\n" +"\n"); + +#define OS_MEMFD_CREATE_METHODDEF \ + {"memfd_create", (PyCFunction)(void(*)(void))os_memfd_create, METH_FASTCALL|METH_KEYWORDS, os_memfd_create__doc__}, + +static PyObject * +os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags); + +static PyObject * +os_memfd_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"name", "flags", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "memfd_create", 0}; + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *name = NULL; + unsigned int flags = MFD_CLOEXEC; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_FSConverter(args[0], &name)) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (PyFloat_Check(args[1])) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float" ); + goto exit; + } + flags = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); + if (flags == (unsigned int)-1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = os_memfd_create_impl(module, name, flags); + +exit: + /* Cleanup for name */ + Py_XDECREF(name); + + return return_value; +} + +#endif /* defined(HAVE_MEMFD_CREATE) */ + PyDoc_STRVAR(os_cpu_count__doc__, "cpu_count($module, /)\n" "--\n" @@ -8549,6 +8604,10 @@ exit: #define OS_LISTXATTR_METHODDEF #endif /* !defined(OS_LISTXATTR_METHODDEF) */ +#ifndef OS_MEMFD_CREATE_METHODDEF + #define OS_MEMFD_CREATE_METHODDEF +#endif /* !defined(OS_MEMFD_CREATE_METHODDEF) */ + #ifndef OS_GET_HANDLE_INHERITABLE_METHODDEF #define OS_GET_HANDLE_INHERITABLE_METHODDEF #endif /* !defined(OS_GET_HANDLE_INHERITABLE_METHODDEF) */ @@ -8576,4 +8635,4 @@ exit: #ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF #define OS__REMOVE_DLL_DIRECTORY_METHODDEF #endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */ -/*[clinic end generated code: output=5ee9420fb2e7aa2c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=855b81aafd05beed input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index cd5b5ce082e..a3d979c3bcd 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -389,6 +389,19 @@ extern char *ctermid_r(char *); #define HAVE_STRUCT_STAT_ST_FSTYPE 1 #endif +/* memfd_create is either defined in sys/mman.h or sys/memfd.h + * linux/memfd.h defines additional flags + */ +#ifdef HAVE_SYS_MMAN_H +#include +#endif +#ifdef HAVE_SYS_MEMFD_H +#include +#endif +#ifdef HAVE_LINUX_MEMFD_H +#include +#endif + #ifdef _Py_MEMORY_SANITIZER # include #endif @@ -11897,6 +11910,31 @@ os_urandom_impl(PyObject *module, Py_ssize_t size) return bytes; } +#ifdef HAVE_MEMFD_CREATE +/*[clinic input] +os.memfd_create + + name: FSConverter + flags: unsigned_int(bitwise=True, c_default="MFD_CLOEXEC") = MFD_CLOEXEC + +[clinic start generated code]*/ + +static PyObject * +os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags) +/*[clinic end generated code: output=6681ede983bdb9a6 input=a42cfc199bcd56e9]*/ +{ + int fd; + const char *bytes = PyBytes_AS_STRING(name); + Py_BEGIN_ALLOW_THREADS + fd = memfd_create(bytes, flags); + Py_END_ALLOW_THREADS + if (fd == -1) { + return PyErr_SetFromErrno(PyExc_OSError); + } + return PyLong_FromLong(fd); +} +#endif + /* Terminal size querying */ static PyTypeObject* TerminalSizeType; @@ -13554,6 +13592,7 @@ static PyMethodDef posix_methods[] = { OS_SCANDIR_METHODDEF OS_FSPATH_METHODDEF OS_GETRANDOM_METHODDEF + OS_MEMFD_CREATE_METHODDEF #ifdef MS_WINDOWS OS__ADD_DLL_DIRECTORY_METHODDEF OS__REMOVE_DLL_DIRECTORY_METHODDEF @@ -14003,6 +14042,27 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, GRND_RANDOM)) return -1; if (PyModule_AddIntMacro(m, GRND_NONBLOCK)) return -1; #endif +#ifdef HAVE_MEMFD_CREATE + if (PyModule_AddIntMacro(m, MFD_CLOEXEC)) return -1; + if (PyModule_AddIntMacro(m, MFD_ALLOW_SEALING)) return -1; +#ifdef MFD_HUGETLB + if (PyModule_AddIntMacro(m, MFD_HUGETLB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_SHIFT)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_MASK)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_64KB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_512KB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_1MB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_2MB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_8MB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_16MB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_32MB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_256MB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_512MB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_1GB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_2GB)) return -1; + if (PyModule_AddIntMacro(m, MFD_HUGE_16GB)) return -1; +#endif +#endif #if defined(__APPLE__) if (PyModule_AddIntConstant(m, "_COPYFILE_DATA", COPYFILE_DATA)) return -1; @@ -14119,6 +14179,10 @@ static const char * const have_functions[] = { "HAVE_LUTIMES", #endif +#ifdef HAVE_MEMFD_CREATE + "HAVE_MEMFD_CREATE", +#endif + #ifdef HAVE_MKDIRAT "HAVE_MKDIRAT", #endif diff --git a/aclocal.m4 b/aclocal.m4 index 038bd4e2cea..85f00dd5fac 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -55,7 +55,7 @@ dnl dnl See the "Since" comment for each macro you use to see what version dnl of the macros you require. m4_defun([PKG_PREREQ], -[m4_define([PKG_MACROS_VERSION], [0.29.2]) +[m4_define([PKG_MACROS_VERSION], [0.29.1]) m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) ])dnl PKG_PREREQ @@ -156,7 +156,7 @@ AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl pkg_failed=no -AC_MSG_CHECKING([for $2]) +AC_MSG_CHECKING([for $1]) _PKG_CONFIG([$1][_CFLAGS], [cflags], [$2]) _PKG_CONFIG([$1][_LIBS], [libs], [$2]) @@ -166,11 +166,11 @@ and $1[]_LIBS to avoid the need to call pkg-config. See the pkg-config man page for more details.]) if test $pkg_failed = yes; then - AC_MSG_RESULT([no]) + AC_MSG_RESULT([no]) _PKG_SHORT_ERRORS_SUPPORTED if test $_pkg_short_errors_supported = yes; then $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` - else + else $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` fi # Put the nasty error message in config.log where it belongs @@ -187,7 +187,7 @@ installed software in a non-standard prefix. _PKG_TEXT])[]dnl ]) elif test $pkg_failed = untried; then - AC_MSG_RESULT([no]) + AC_MSG_RESULT([no]) m4_default([$4], [AC_MSG_FAILURE( [The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full diff --git a/configure b/configure index e8cbfeb154a..cc18322d751 100755 --- a/configure +++ b/configure @@ -785,6 +785,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -897,6 +898,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1149,6 +1151,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1286,7 +1297,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1439,6 +1450,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -7892,7 +7904,7 @@ sys/stat.h sys/syscall.h sys/sys_domain.h sys/termio.h sys/time.h \ sys/times.h sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h pty.h \ libutil.h sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \ linux/tipc.h linux/random.h spawn.h util.h alloca.h endian.h \ -sys/endian.h sys/sysmacros.h +sys/endian.h sys/sysmacros.h linux/memfd.h sys/memfd.h sys/mman.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" @@ -11785,6 +11797,39 @@ $as_echo "no" >&6; } fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for memfd_create" >&5 +$as_echo_n "checking for memfd_create... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#ifdef HAVE_SYS_MMAN_H +#include +#endif +#ifdef HAVE_SYS_MEMFD_H +#include +#endif + +int +main () +{ +void *x=memfd_create + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +$as_echo "#define HAVE_MEMFD_CREATE 1" >>confdefs.h + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + # On some systems (eg. FreeBSD 5), we would find a definition of the # functions ctermid_r, setgroups in the library, but no prototype # (e.g. because we use _XOPEN_SOURCE). See whether we can take their diff --git a/configure.ac b/configure.ac index 864c0abcf93..1190b37e9f9 100644 --- a/configure.ac +++ b/configure.ac @@ -2133,7 +2133,7 @@ sys/stat.h sys/syscall.h sys/sys_domain.h sys/termio.h sys/time.h \ sys/times.h sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h pty.h \ libutil.h sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \ linux/tipc.h linux/random.h spawn.h util.h alloca.h endian.h \ -sys/endian.h sys/sysmacros.h) +sys/endian.h sys/sysmacros.h linux/memfd.h sys/memfd.h sys/mman.h) AC_HEADER_DIRENT AC_HEADER_MAJOR @@ -3626,6 +3626,20 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ [AC_MSG_RESULT(no) ]) +AC_MSG_CHECKING(for memfd_create) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ +#ifdef HAVE_SYS_MMAN_H +#include +#endif +#ifdef HAVE_SYS_MEMFD_H +#include +#endif +]], [[void *x=memfd_create]])], + [AC_DEFINE(HAVE_MEMFD_CREATE, 1, Define if you have the 'memfd_create' function.) + AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no) +]) + # On some systems (eg. FreeBSD 5), we would find a definition of the # functions ctermid_r, setgroups in the library, but no prototype # (e.g. because we use _XOPEN_SOURCE). See whether we can take their diff --git a/pyconfig.h.in b/pyconfig.h.in index cc643554164..b9bb3ffa6f6 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -625,6 +625,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_CAN_RAW_H +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_MEMFD_H + /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_NETLINK_H @@ -667,15 +670,15 @@ /* Define to 1 if you have the `mbrtowc' function. */ #undef HAVE_MBRTOWC +/* Define if you have the 'memfd_create' function. */ +#undef HAVE_MEMFD_CREATE + /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H /* Define to 1 if you have the `memrchr' function. */ #undef HAVE_MEMRCHR -/* Define to 1 if you have the `memset_s' function. */ -#undef HAVE_MEMSET_S - /* Define to 1 if you have the `mkdirat' function. */ #undef HAVE_MKDIRAT @@ -1119,6 +1122,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_LOCK_H +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_MEMFD_H + /* Define to 1 if you have the header file. */ #undef HAVE_SYS_MKDEV_H