bpo-33625: Release GIL for grp.getgr{nam,gid} and pwd.getpw{nam,uid} (GH-7081)

Release GIL on grp.getgrnam(), grp.getgrgid(), pwd.getpwnam() and
pwd.getpwuid() if reentrant variants of these functions are available.

Patch by William Grzybowski.

Misc/NEWS.d/next/Library/2018-05-23-17-07-54.bpo-33625.nzQgD8.rst

@@ -0,0 +1,3 @@
+Release GIL on `grp.getgrnam`, `grp.getgrgid`, `pwd.getpwnam` and
+`pwd.getpwuid` if reentrant variants of these functions are available.
+Patch by William Grzybowski.

Modules/grpmodule.c

@@ -37,6 +37,8 @@ static PyStructSequence_Desc struct_group_type_desc = {
 static int initialized;
 static PyTypeObject StructGrpType;
 
+#define DEFAULT_BUFFER_SIZE 1024
+
 static PyObject *
 mkgrent(struct group *p)
 {
@@ -96,7 +98,9 @@ static PyObject *
 grp_getgrgid_impl(PyObject *module, PyObject *id)
 /*[clinic end generated code: output=30797c289504a1ba input=15fa0e2ccf5cda25]*/
 {
-    PyObject *py_int_id;
+    PyObject *py_int_id, *retval = NULL;
+    int nomem = 0;
+    char *buf = NULL, *buf2 = NULL;
     gid_t gid;
     struct group *p;
 
@@ -119,8 +123,48 @@ grp_getgrgid_impl(PyObject *module, PyObject *id)
         }
         Py_DECREF(py_int_id);
     }
+#ifdef HAVE_GETGRGID_R
+    Py_BEGIN_ALLOW_THREADS
+    int status;
+    Py_ssize_t bufsize;
+    struct group grp;
 
-    if ((p = getgrgid(gid)) == NULL) {
+    bufsize = sysconf(_SC_GETGR_R_SIZE_MAX);
+    if (bufsize == -1) {
+        bufsize = DEFAULT_BUFFER_SIZE;
+    }
+
+    while (1) {
+        buf2 = PyMem_RawRealloc(buf, bufsize);
+        if (buf2 == NULL) {
+            p = NULL;
+            nomem = 1;
+            break;
+        }
+        buf = buf2;
+        status = getgrgid_r(gid, &grp, buf, bufsize, &p);
+        if (status != 0) {
+            p = NULL;
+        }
+        if (p != NULL || status != ERANGE) {
+            break;
+        }
+        if (bufsize > (PY_SSIZE_T_MAX >> 1)) {
+            nomem = 1;
+            break;
+        }
+        bufsize <<= 1;
+    }
+
+    Py_END_ALLOW_THREADS
+#else
+    p = getgrgid(gid);
+#endif
+    if (p == NULL) {
+        PyMem_RawFree(buf);
+        if (nomem == 1) {
+            return PyErr_NoMemory();
+        }
         PyObject *gid_obj = _PyLong_FromGid(gid);
         if (gid_obj == NULL)
             return NULL;
@@ -128,7 +172,11 @@ grp_getgrgid_impl(PyObject *module, PyObject *id)
         Py_DECREF(gid_obj);
         return NULL;
     }
-    return mkgrent(p);
+    retval = mkgrent(p);
+#ifdef HAVE_GETGRGID_R
+    PyMem_RawFree(buf);
+#endif
+    return retval;
 }
 
 /*[clinic input]
@@ -145,7 +193,8 @@ static PyObject *
 grp_getgrnam_impl(PyObject *module, PyObject *name)
 /*[clinic end generated code: output=67905086f403c21c input=08ded29affa3c863]*/
 {
-    char *name_chars;
+    char *buf = NULL, *buf2 = NULL, *name_chars;
+    int nomem = 0;
     struct group *p;
     PyObject *bytes, *retval = NULL;
 
@@ -154,13 +203,55 @@ grp_getgrnam_impl(PyObject *module, PyObject *name)
     /* check for embedded null bytes */
     if (PyBytes_AsStringAndSize(bytes, &name_chars, NULL) == -1)
         goto out;
+#ifdef HAVE_GETGRNAM_R
+    Py_BEGIN_ALLOW_THREADS
+    int status;
+    Py_ssize_t bufsize;
+    struct group grp;
 
-    if ((p = getgrnam(name_chars)) == NULL) {
+    bufsize = sysconf(_SC_GETGR_R_SIZE_MAX);
-        PyErr_Format(PyExc_KeyError, "getgrnam(): name not found: %s", name_chars);
+    if (bufsize == -1) {
+        bufsize = DEFAULT_BUFFER_SIZE;
+    }
+
+    while(1) {
+        buf2 = PyMem_RawRealloc(buf, bufsize);
+        if (buf2 == NULL) {
+            p = NULL;
+            nomem = 1;
+            break;
+        }
+        buf = buf2;
+        status = getgrnam_r(name_chars, &grp, buf, bufsize, &p);
+        if (status != 0) {
+            p = NULL;
+        }
+        if (p != NULL || status != ERANGE) {
+            break;
+        }
+        if (bufsize > (PY_SSIZE_T_MAX >> 1)) {
+            nomem = 1;
+            break;
+        }
+        bufsize <<= 1;
+    }
+
+    Py_END_ALLOW_THREADS
+#else
+    p = getgrnam(name_chars);
+#endif
+    if (p == NULL) {
+        if (nomem == 1) {
+            PyErr_NoMemory();
+        }
+        else {
+            PyErr_Format(PyExc_KeyError, "getgrnam(): name not found: %s", name_chars);
+        }
         goto out;
     }
     retval = mkgrent(p);
 out:
+    PyMem_RawFree(buf);
     Py_DECREF(bytes);
     return retval;
 }

Modules/pwdmodule.c

@@ -50,6 +50,8 @@ exception is raised if the entry asked for cannot be found.");
 static int initialized;
 static PyTypeObject StructPwdType;
 
+#define DEFAULT_BUFFER_SIZE 1024
+
 static void
 sets(PyObject *v, int i, const char* val)
 {
@@ -116,8 +118,11 @@ static PyObject *
 pwd_getpwuid(PyObject *module, PyObject *uidobj)
 /*[clinic end generated code: output=c4ee1d4d429b86c4 input=ae64d507a1c6d3e8]*/
 {
+    PyObject *retval = NULL;
     uid_t uid;
+    int nomem = 0;
     struct passwd *p;
+    char *buf = NULL, *buf2 = NULL;
 
     if (!_Py_Uid_Converter(uidobj, &uid)) {
         if (PyErr_ExceptionMatches(PyExc_OverflowError))
@@ -125,7 +130,47 @@ pwd_getpwuid(PyObject *module, PyObject *uidobj)
                          "getpwuid(): uid not found");
         return NULL;
     }
-    if ((p = getpwuid(uid)) == NULL) {
+#ifdef HAVE_GETPWUID_R
+    Py_BEGIN_ALLOW_THREADS
+    int status;
+    Py_ssize_t bufsize;
+    struct passwd pwd;
+
+    bufsize = sysconf(_SC_GETPW_R_SIZE_MAX);
+    if (bufsize == -1) {
+        bufsize = DEFAULT_BUFFER_SIZE;
+    }
+
+    while(1) {
+        buf2 = PyMem_RawRealloc(buf, bufsize);
+        if (buf2 == NULL) {
+            nomem = 1;
+            break;
+        }
+        buf = buf2;
+        status = getpwuid_r(uid, &pwd, buf, bufsize, &p);
+        if (status != 0) {
+            p = NULL;
+        }
+        if (p != NULL || status != ERANGE) {
+            break;
+        }
+        if (bufsize > (PY_SSIZE_T_MAX >> 1)) {
+            nomem = 1;
+            break;
+        }
+        bufsize <<= 1;
+    }
+
+    Py_END_ALLOW_THREADS
+#else
+    p = getpwuid(uid);
+#endif
+    if (p == NULL) {
+        PyMem_RawFree(buf);
+        if (nomem == 1) {
+            return PyErr_NoMemory();
+        }
         PyObject *uid_obj = _PyLong_FromUid(uid);
         if (uid_obj == NULL)
             return NULL;
@@ -134,7 +179,11 @@ pwd_getpwuid(PyObject *module, PyObject *uidobj)
         Py_DECREF(uid_obj);
         return NULL;
     }
-    return mkpwent(p);
+    retval = mkpwent(p);
+#ifdef HAVE_GETPWUID_R
+    PyMem_RawFree(buf);
+#endif
+    return retval;
 }
 
 /*[clinic input]
@@ -152,7 +201,8 @@ static PyObject *
 pwd_getpwnam_impl(PyObject *module, PyObject *arg)
 /*[clinic end generated code: output=6abeee92430e43d2 input=d5f7e700919b02d3]*/
 {
-    char *name;
+    char *buf = NULL, *buf2 = NULL, *name;
+    int nomem = 0;
     struct passwd *p;
     PyObject *bytes, *retval = NULL;
 
@@ -161,13 +211,55 @@ pwd_getpwnam_impl(PyObject *module, PyObject *arg)
     /* check for embedded null bytes */
     if (PyBytes_AsStringAndSize(bytes, &name, NULL) == -1)
         goto out;
-    if ((p = getpwnam(name)) == NULL) {
+#ifdef HAVE_GETPWNAM_R
-        PyErr_Format(PyExc_KeyError,
+    Py_BEGIN_ALLOW_THREADS
-                     "getpwnam(): name not found: %s", name);
+    int status;
+    Py_ssize_t bufsize;
+    struct passwd pwd;
+
+    bufsize = sysconf(_SC_GETPW_R_SIZE_MAX);
+    if (bufsize == -1) {
+        bufsize = DEFAULT_BUFFER_SIZE;
+    }
+
+    while(1) {
+        buf2 = PyMem_RawRealloc(buf, bufsize);
+        if (buf2 == NULL) {
+            nomem = 1;
+            break;
+        }
+        buf = buf2;
+        status = getpwnam_r(name, &pwd, buf, bufsize, &p);
+        if (status != 0) {
+            p = NULL;
+        }
+        if (p != NULL || status != ERANGE) {
+            break;
+        }
+        if (bufsize > (PY_SSIZE_T_MAX >> 1)) {
+            nomem = 1;
+            break;
+        }
+        bufsize <<= 1;
+    }
+
+    Py_END_ALLOW_THREADS
+#else
+    p = getpwnam(name);
+#endif
+    if (p == NULL) {
+        if (nomem == 1) {
+            PyErr_NoMemory();
+        }
+        else {
+            PyErr_Format(PyExc_KeyError,
+                         "getpwnam(): name not found: %s", name);
+        }
         goto out;
     }
     retval = mkpwent(p);
 out:
+    PyMem_RawFree(buf);
     Py_DECREF(bytes);
     return retval;
 }

configure

@@ -781,6 +781,7 @@ infodir
 docdir
 oldincludedir
 includedir
+runstatedir
 localstatedir
 sharedstatedir
 sysconfdir
@@ -890,6 +891,7 @@ datadir='${datarootdir}'
 sysconfdir='${prefix}/etc'
 sharedstatedir='${prefix}/com'
 localstatedir='${prefix}/var'
+runstatedir='${localstatedir}/run'
 includedir='${prefix}/include'
 oldincludedir='/usr/include'
 docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1142,6 +1144,15 @@ do
   | -silent | --silent | --silen | --sile | --sil)
     silent=yes ;;
 
+  -runstatedir | --runstatedir | --runstatedi | --runstated \
+  | --runstate | --runstat | --runsta | --runst | --runs \
+  | --run | --ru | --r)
+    ac_prev=runstatedir ;;
+  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
+  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
+  | --run=* | --ru=* | --r=*)
+    runstatedir=$ac_optarg ;;
+
   -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
     ac_prev=sbindir ;;
   -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1279,7 +1290,7 @@ fi
 for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
 		datadir sysconfdir sharedstatedir localstatedir includedir \
 		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
-		libdir localedir mandir
+		libdir localedir mandir runstatedir
 do
   eval ac_val=\$$ac_var
   # Remove trailing slashes.
@@ -1432,6 +1443,7 @@ Fine tuning of the installation directories:
   --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
   --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
   --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
+  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
   --libdir=DIR            object code libraries [EPREFIX/lib]
   --includedir=DIR        C header files [PREFIX/include]
   --oldincludedir=DIR     C header files for non-gcc [/usr/include]
@@ -11242,8 +11254,9 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
  clock confstr ctermid dup3 execv faccessat fchmod fchmodat fchown fchownat \
  fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
  futimens futimes gai_strerror getentropy \
+ getgrgid_r getgrnam_r \
  getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
- getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \
+ getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \
  if_nameindex \
  initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
  memrchr mbrtowc mkdirat mkfifo \
@@ -13675,6 +13688,7 @@ fi
 
 
 
+
 # checks for system services
 # (none yet)
 

configure.ac

@@ -3434,8 +3434,9 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
  clock confstr ctermid dup3 execv faccessat fchmod fchmodat fchown fchownat \
  fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
  futimens futimes gai_strerror getentropy \
+ getgrgid_r getgrnam_r \
  getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
- getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \
+ getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \
  if_nameindex \
  initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
  memrchr mbrtowc mkdirat mkfifo \

pyconfig.h.in

@@ -403,6 +403,12 @@
 /* Define to 1 if you have the `getentropy' function. */
 #undef HAVE_GETENTROPY
 
+/* Define to 1 if you have the `getgrgid_r' function. */
+#undef HAVE_GETGRGID_R
+
+/* Define to 1 if you have the `getgrnam_r' function. */
+#undef HAVE_GETGRNAM_R
+
 /* Define to 1 if you have the `getgrouplist' function. */
 #undef HAVE_GETGROUPLIST
 
@@ -457,6 +463,12 @@
 /* Define to 1 if you have the `getpwent' function. */
 #undef HAVE_GETPWENT
 
+/* Define to 1 if you have the `getpwnam_r' function. */
+#undef HAVE_GETPWNAM_R
+
+/* Define to 1 if you have the `getpwuid_r' function. */
+#undef HAVE_GETPWUID_R
+
 /* Define to 1 if the getrandom() function is available */
 #undef HAVE_GETRANDOM