bpo-40423: Optimization: use close_range(2) if available (GH-22651)

close_range(2) should be preferred at all times if it's available, otherwise we'll use closefrom(2) if available with a fallback to fdwalk(3) or plain old loop over fd range in order of most efficient to least.

[note that this version does check for ENOSYS, but currently ignores all other errors]

Automerge-Triggered-By: @pablogsal

Misc/NEWS.d/next/C API/2020-10-11-19-17-44.bpo-40423.GsmgEj.rst

@@ -0,0 +1,3 @@
+The :mod:`subprocess` module and ``os.closerange`` will now use the
+``close_range(low, high, flags)`` syscall when it is available for more
+efficient closing of ranges of descriptors.

Modules/posixmodule.c

@@ -8741,12 +8741,15 @@ os_close_impl(PyObject *module, int fd)
 }
 
 /* Our selection logic for which function to use is as follows:
- * 1. If closefrom(2) is available, we'll attempt to use that next if we're
+ * 1. If close_range(2) is available, always prefer that; it's better for
+ *    contiguous ranges like this than fdwalk(3) which entails iterating over
+ *    the entire fd space and simply doing nothing for those outside the range.
+ * 2. If closefrom(2) is available, we'll attempt to use that next if we're
  *    closing up to sysconf(_SC_OPEN_MAX).
- * 1a. Fallback to fdwalk(3) if we're not closing up to sysconf(_SC_OPEN_MAX),
+ * 2a. Fallback to fdwalk(3) if we're not closing up to sysconf(_SC_OPEN_MAX),
  *    as that will be more performant if the range happens to have any chunk of
  *    non-opened fd in the middle.
- * 1b. If fdwalk(3) isn't available, just do a plain close(2) loop.
+ * 2b. If fdwalk(3) isn't available, just do a plain close(2) loop.
  */
 #ifdef __FreeBSD__
 #define USE_CLOSEFROM
@@ -8779,6 +8782,14 @@ void
 _Py_closerange(int first, int last)
 {
     first = Py_MAX(first, 0);
+#ifdef HAVE_CLOSE_RANGE
+    if (close_range(first, last, 0) == 0 || errno != ENOSYS) {
+        /* Any errors encountered while closing file descriptors are ignored;
+         * ENOSYS means no kernel support, though,
+         * so we'll fallback to the other methods. */
+    }
+    else
+#endif /* HAVE_CLOSE_RANGE */
 #ifdef USE_CLOSEFROM
     if (last >= sysconf(_SC_OPEN_MAX)) {
         /* Any errors encountered while closing file descriptors are ignored */

configure

@@ -11672,8 +11672,8 @@ fi
 
 # checks for library functions
 for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
- clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \
+ clock confstr close_range copy_file_range ctermid dup3 execv explicit_bzero \
- faccessat fchmod fchmodat fchown fchownat \
+ explicit_memset faccessat fchmod fchmodat fchown fchownat \
  fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
  futimens futimes gai_strerror getentropy \
  getgrgid_r getgrnam_r \

configure.ac

@@ -3664,8 +3664,8 @@ fi
 
 # checks for library functions
 AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
- clock confstr copy_file_range ctermid dup3 execv explicit_bzero explicit_memset \
+ clock confstr close_range copy_file_range ctermid dup3 execv explicit_bzero \
- faccessat fchmod fchmodat fchown fchownat \
+ explicit_memset faccessat fchmod fchmodat fchown fchownat \
  fdwalk fexecve fdopendir fork fpathconf fstatat ftime ftruncate futimesat \
  futimens futimes gai_strerror getentropy \
  getgrgid_r getgrnam_r \

pyconfig.h.in

@@ -139,6 +139,9 @@
 /* Define to 1 if you have the `clock_settime' function. */
 #undef HAVE_CLOCK_SETTIME
 
+/* Define to 1 if you have the `close_range' function. */
+#undef HAVE_CLOSE_RANGE
+
 /* Define if the C compiler supports computed gotos. */
 #undef HAVE_COMPUTED_GOTOS