gh-108765: Python.h no longer includes <ctype.h> (#108831)

Remove <ctype.h> in C files which don't use it; only sre.c and
_decimal.c still use it.

Remove _PY_PORT_CTYPE_UTF8_ISSUE code from pyport.h:

* Code added by commit b5047fd019
  in 2004 for MacOSX and FreeBSD.
* Test removed by commit 52ddaefb6b
  in 2007, since Python str type now uses locale independent
  functions like Py_ISALPHA() and Py_TOLOWER() and the Unicode
  database.

Modules/_sre/sre.c replaces _PY_PORT_CTYPE_UTF8_ISSUE with new
functions: sre_isalnum(), sre_tolower(), sre_toupper().

Remove unused includes:

* _localemodule.c: remove <stdio.h>.
* getargs.c: remove <float.h>.
* dynload_win.c: remove <direct.h>, it no longer calls _getcwd()
  since commit fb1f68ed7c (in 2001).

Doc/whatsnew/3.13.rst

@@ -942,6 +942,13 @@ Porting to Python 3.13
   and ``setitimer()`` functions.
   (Contributed by Victor Stinner in :gh:`108765`.)
 
+* ``Python.h`` no longer includes the ``<ctype.h>`` standard header file. If
+  needed, it should now be included explicitly. For example, it provides
+  ``isalpha()`` and ``tolower()`` functions which are locale dependent. Python
+  provides locale independent functions, like :c:func:`!Py_ISALPHA` and
+  :c:func:`!Py_TOLOWER`.
+  (Contributed by Victor Stinner in :gh:`108765`.)
+
 Deprecated
 ----------
 

Include/Python.h

@@ -17,7 +17,6 @@
 
 // Include standard header files
 #include <assert.h>               // assert()
-#include <ctype.h>                // tolower()
 #include <inttypes.h>             // uintptr_t
 #include <limits.h>               // INT_MAX
 #include <math.h>                 // HUGE_VAL

Include/pyport.h

@@ -392,44 +392,6 @@ extern "C" {
 #  define Py_NO_INLINE
 #endif
 
-/* On 4.4BSD-descendants, ctype functions serves the whole range of
- * wchar_t character set rather than single byte code points only.
- * This characteristic can break some operations of string object
- * including str.upper() and str.split() on UTF-8 locales.  This
- * workaround was provided by Tim Robbins of FreeBSD project.
- */
-
-#if defined(__APPLE__)
-#  define _PY_PORT_CTYPE_UTF8_ISSUE
-#endif
-
-#ifdef _PY_PORT_CTYPE_UTF8_ISSUE
-#ifndef __cplusplus
-   /* The workaround below is unsafe in C++ because
-    * the <locale> defines these symbols as real functions,
-    * with a slightly different signature.
-    * See issue #10910
-    */
-#include <ctype.h>
-#include <wctype.h>
-#undef isalnum
-#define isalnum(c) iswalnum(btowc(c))
-#undef isalpha
-#define isalpha(c) iswalpha(btowc(c))
-#undef islower
-#define islower(c) iswlower(btowc(c))
-#undef isspace
-#define isspace(c) iswspace(btowc(c))
-#undef isupper
-#define isupper(c) iswupper(btowc(c))
-#undef tolower
-#define tolower(c) towlower(btowc(c))
-#undef toupper
-#define toupper(c) towupper(btowc(c))
-#endif
-#endif
-
-
 /* Declarations for symbol visibility.
 
   PyAPI_FUNC(type): Declares a public Python API function and return type

Misc/NEWS.d/next/C API/2023-09-02-22-35-55.gh-issue-108765.4TOdBT.rst

@@ -0,0 +1,5 @@
+``Python.h`` no longer includes the ``<ctype.h>`` standard header file. If
+needed, it should now be included explicitly. For example, it provides
+``isalpha()`` and ``tolower()`` functions which are locale dependent. Python
+provides locale independent functions, like :c:func:`!Py_ISALPHA` and
+:c:func:`!Py_TOLOWER`. Patch by Victor Stinner.

Modules/_localemodule.c

@@ -10,35 +10,25 @@ This software comes with no warranty. Use at your own risk.
 ******************************************************************/
 
 #include "Python.h"
-#include "pycore_fileutils.h"
+#include "pycore_fileutils.h"     // _Py_GetLocaleconvNumeric()
-#include "pycore_pymem.h"  // _PyMem_Strdup
+#include "pycore_pymem.h"         // _PyMem_Strdup()
-
-#include <stdio.h>
-#include <locale.h>
-#include <string.h>
-#include <ctype.h>
 
+#include <locale.h>               // setlocale()
+#include <string.h>               // strlen()
 #ifdef HAVE_ERRNO_H
-#include <errno.h>
+#  include <errno.h>              // errno
 #endif
-
 #ifdef HAVE_LANGINFO_H
-#include <langinfo.h>
+#  include <langinfo.h>           // nl_langinfo()
 #endif
-
 #ifdef HAVE_LIBINTL_H
-#include <libintl.h>
+#  include <libintl.h>
 #endif
-
+#ifdef MS_WINDOWS
-#ifdef HAVE_WCHAR_H
+#  ifndef WIN32_LEAN_AND_MEAN
-#include <wchar.h>
+#    define WIN32_LEAN_AND_MEAN
-#endif
+#  endif
-
+#  include <windows.h>
-#if defined(MS_WINDOWS)
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#endif
-#include <windows.h>
 #endif
 
 PyDoc_STRVAR(locale__doc__, "Support for POSIX locales.");

Modules/_sre/sre.c

@@ -43,12 +43,40 @@ static const char copyright[] =
 #include "pycore_long.h"          // _PyLong_GetZero()
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
 
+#include "sre.h"                  // SRE_CODE
 
-#include "sre.h"
+#include <ctype.h>                // tolower(), toupper(), isalnum()
 
 #define SRE_CODE_BITS (8 * sizeof(SRE_CODE))
 
-#include <ctype.h>
+// On macOS, use the wide character ctype API using btowc()
+#if defined(__APPLE__)
+#  define USE_CTYPE_WINT_T
+#endif
+
+static int sre_isalnum(unsigned int ch) {
+#ifdef USE_CTYPE_WINT_T
+    return (unsigned int)iswalnum(btowc((int)ch));
+#else
+    return (unsigned int)isalnum((int)ch);
+#endif
+}
+
+static unsigned int sre_tolower(unsigned int ch) {
+#ifdef USE_CTYPE_WINT_T
+    return (unsigned int)towlower(btowc((int)ch));
+#else
+    return (unsigned int)tolower((int)ch);
+#endif
+}
+
+static unsigned int sre_toupper(unsigned int ch) {
+#ifdef USE_CTYPE_WINT_T
+    return (unsigned int)towupper(btowc((int)ch));
+#else
+    return (unsigned int)toupper((int)ch);
+#endif
+}
 
 /* Defining this one controls tracing:
  * 0 -- disabled
@@ -114,17 +142,17 @@ static unsigned int sre_lower_ascii(unsigned int ch)
 /* locale-specific character predicates */
 /* !(c & ~N) == (c < N+1) for any unsigned c, this avoids
  * warnings when c's type supports only numbers < N+1 */
-#define SRE_LOC_IS_ALNUM(ch) (!((ch) & ~255) ? isalnum((ch)) : 0)
+#define SRE_LOC_IS_ALNUM(ch) (!((ch) & ~255) ? sre_isalnum((ch)) : 0)
 #define SRE_LOC_IS_WORD(ch) (SRE_LOC_IS_ALNUM((ch)) || (ch) == '_')
 
 static unsigned int sre_lower_locale(unsigned int ch)
 {
-    return ((ch) < 256 ? (unsigned int)tolower((ch)) : ch);
+    return ((ch) < 256 ? (unsigned int)sre_tolower((ch)) : ch);
 }
 
 static unsigned int sre_upper_locale(unsigned int ch)
 {
-    return ((ch) < 256 ? (unsigned int)toupper((ch)) : ch);
+    return ((ch) < 256 ? (unsigned int)sre_toupper((ch)) : ch);
 }
 
 /* unicode-specific character predicates */

Modules/_struct.c

@@ -12,7 +12,6 @@
 #include "pycore_long.h"          // _PyLong_AsByteArray()
 #include "pycore_moduleobject.h"  // _PyModule_GetState()
 
-#include <ctype.h>
 #include <stddef.h>               // offsetof()
 
 /*[clinic input]

Modules/_tkinter.c

@@ -26,15 +26,14 @@ Copyright (C) 1994 Steen Lumholt.
 #endif
 
 #include "Python.h"
-#include <ctype.h>
 #ifdef MS_WINDOWS
 #  include "pycore_fileutils.h"   // _Py_stat()
 #endif
 
-#include "pycore_long.h"
+#include "pycore_long.h"          // _PyLong_IsNegative()
 
 #ifdef MS_WINDOWS
-#include <windows.h>
+#  include <windows.h>
 #endif
 
 #define CHECK_SIZE(size, elemsize) \
@@ -46,11 +45,11 @@ Copyright (C) 1994 Steen Lumholt.
 #define TCL_THREADS
 
 #ifdef TK_FRAMEWORK
-#include <Tcl/tcl.h>
+#  include <Tcl/tcl.h>
-#include <Tk/tk.h>
+#  include <Tk/tk.h>
 #else
-#include <tcl.h>
+#  include <tcl.h>
-#include <tk.h>
+#  include <tk.h>
 #endif
 
 #include "tkinter.h"

Modules/_zoneinfo.c

@@ -6,11 +6,10 @@
 #include "pycore_long.h"          // _PyLong_GetOne()
 #include "pycore_pyerrors.h"      // _PyErr_ChainExceptions1()
 
-#include <ctype.h>
+#include "datetime.h"             // PyDateTime_TZInfo
-#include <stddef.h>
-#include <stdint.h>
 
-#include "datetime.h"
+#include <stddef.h>               // offsetof()
+#include <stdint.h>
 
 #include "clinic/_zoneinfo.c.h"
 /*[clinic input]

Modules/pyexpat.c

@@ -7,7 +7,6 @@
 #include "pycore_pyhash.h"        // _Py_HashSecret
 #include "pycore_traceback.h"     // _PyTraceback_Add()
 
-#include <ctype.h>
 #include <stddef.h>               // offsetof()
 #include "expat.h"
 #include "pyexpat.h"

Modules/timemodule.c

@@ -6,22 +6,21 @@
 #include "pycore_namespace.h"     // _PyNamespace_New()
 #include "pycore_runtime.h"       // _Py_ID()
 
-#include <ctype.h>
 #include <time.h>                 // clock()
 #ifdef HAVE_SYS_TIMES_H
-#  include <sys/times.h>
+#  include <sys/times.h>          // times()
 #endif
 #ifdef HAVE_SYS_TYPES_H
 #  include <sys/types.h>
 #endif
 #if defined(HAVE_SYS_RESOURCE_H)
-#  include <sys/resource.h>
+#  include <sys/resource.h>       // getrusage(RUSAGE_SELF)
 #endif
 #ifdef QUICKWIN
 # include <io.h>
 #endif
 #if defined(HAVE_PTHREAD_H)
-#  include <pthread.h>
+#  include <pthread.h>            // pthread_getcpuclockid()
 #endif
 #if defined(_AIX)
 #   include <sys/thread.h>

Objects/abstract.c

@@ -9,9 +9,8 @@
 #include "pycore_pyerrors.h"      // _PyErr_Occurred()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_unionobject.h"   // _PyUnion_Check()
-#include <ctype.h>
-#include <stddef.h>               // offsetof()
 
+#include <stddef.h>               // offsetof()
 
 
 /* Shorthands to return certain errors */

Objects/floatobject.c

@@ -16,8 +16,7 @@
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_structseq.h"     // _PyStructSequence_FiniBuiltin()
 
-#include <ctype.h>
+#include <float.h>                // DBL_MAX
-#include <float.h>
 #include <stdlib.h>               // strtol()
 
 /*[clinic input]

Objects/longobject.c

@@ -10,10 +10,8 @@
 #include "pycore_runtime.h"       // _PY_NSMALLPOSINTS
 #include "pycore_structseq.h"     // _PyStructSequence_FiniBuiltin()
 
-#include <ctype.h>
+#include <float.h>                // DBL_MANT_DIG
-#include <float.h>
+#include <stddef.h>               // offsetof
-#include <stddef.h>
-#include <stdlib.h>               // abs()
 
 #include "clinic/longobject.c.h"
 /*[clinic input]

Objects/typeobject.c

@@ -19,8 +19,6 @@
 #include "pycore_weakref.h"       // _PyWeakref_GET_REF()
 #include "opcode.h"               // MAKE_CELL
 
-
-#include <ctype.h>
 #include <stddef.h>               // ptrdiff_t
 
 /*[clinic input]

Python/bltinmodule.c

@@ -1,7 +1,6 @@
 /* Built-in functions */
 
 #include "Python.h"
-#include <ctype.h>
 #include "pycore_ast.h"           // _PyAST_Validate()
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "pycore_ceval.h"         // _PyEval_Vector()

Python/ceval.c

@@ -35,9 +35,7 @@
 #include "pydtrace.h"
 #include "setobject.h"
 
-
+#include <stdbool.h>              // bool
-#include <ctype.h>
-#include <stdbool.h>
 
 #ifdef Py_DEBUG
    /* For debugging the interpreter: */

Python/codecs.c

@@ -14,7 +14,6 @@ Copyright (c) Corporation for National Research Initiatives.
 #include "pycore_pyerrors.h"      // _PyErr_FormatNote()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI
-#include <ctype.h>
 
 const char *Py_hexdigits = "0123456789abcdef";
 

Python/dynload_win.c

@@ -5,13 +5,8 @@
 #include "pycore_fileutils.h"     // _Py_add_relfile()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 
-#ifdef HAVE_DIRECT_H
+#include "importdl.h"             // dl_funcptr
-#include <direct.h>
+#include "patchlevel.h"           // PY_MAJOR_VERSION
-#endif
-#include <ctype.h>
-
-#include "importdl.h"
-#include "patchlevel.h"
 #include <windows.h>
 
 #ifdef _DEBUG

Python/errors.c

@@ -10,7 +10,6 @@
 #include "pycore_sysmodule.h"     // _PySys_Audit()
 #include "pycore_traceback.h"     // _PyTraceBack_FromFrame()
 
-#include <ctype.h>
 #ifdef MS_WINDOWS
 #  include <windows.h>
 #  include <winbase.h>

Python/getargs.c

@@ -7,10 +7,6 @@
 #include "pycore_pylifecycle.h"   // _PyArg_Fini
 #include "pycore_tuple.h"         // _PyTuple_ITEMS()
 
-#include <ctype.h>
-#include <float.h>
-
-
 #ifdef __cplusplus
 extern "C" {
 #endif

Python/mystrtoul.c

@@ -1,16 +1,22 @@
+// strtol() and strtoul(), renamed to avoid conflicts.
+//
+// API:
+//
+// - PyOS_strtol(): convert string to C long integer.
+// - PyOS_strtoul(): convert string to C unsigned long integer.
+
 #include "Python.h"
 #include "pycore_long.h"          // _PyLong_DigitValue
 
 #if defined(__sgi) && !defined(_SGI_MP_SOURCE)
-#define _SGI_MP_SOURCE
+#  define _SGI_MP_SOURCE
 #endif
 
 /* strtol and strtoul, renamed to avoid conflicts */
 
 
-#include <ctype.h>
 #ifdef HAVE_ERRNO_H
-#include <errno.h>
+#  include <errno.h>              // errno
 #endif
 
 /* Static overflow check values for bases 2 through 36.
@@ -75,7 +81,7 @@ static const int digitlimit[] = {
     14,  14, 14, 14, 13, 13, 13, 13, 13, 13,  /* 20 - 29 */
     13,  12, 12, 12, 12, 12, 12};             /* 30 - 36 */
 #else
-#error "Need table for SIZEOF_LONG"
+#  error "Need table for SIZEOF_LONG"
 #endif
 
 /*