bpo-36635: Change pyport.h for Py_BUILD_CORE_MODULE define (GH-12853)

Change PyAPI_FUNC(type), PyAPI_DATA(type) and PyMODINIT_FUNC macros
of pyport.h when Py_BUILD_CORE_MODULE is defined.

The Py_BUILD_CORE_MODULE define must be now be used to build a C
extension as a dynamic library accessing Python internals: export the
PyInit_xxx() function in DLL exports on Windows.

Changes:

* Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE now imply
  Py_BUILD_CORE directy in pyport.h.
* ceval.c compilation now fails with an error if Py_BUILD_CORE is not
  defined, just to ensure that Python is build with the correct
  defines.
* setup.py now compiles _pickle.c with Py_BUILD_CORE_MODULE define.
* setup.py compiles _json.c with Py_BUILD_CORE_MODULE define, rather
  than Py_BUILD_CORE_BUILTIN define
* PCbuild/pythoncore.vcxproj: Add Py_BUILD_CORE_BUILTIN define.

Include/internal/pycore_accu.h

@@ -9,8 +9,8 @@ extern "C" {
  *** Its definition may be changed or removed at any moment.
  ***/
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 /*

Include/internal/pycore_atomic.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #include "dynamic_annotations.h"

Include/internal/pycore_ceval.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #include "pycore_atomic.h"

Include/internal/pycore_condvar.h

@@ -1,8 +1,8 @@
 #ifndef Py_INTERNAL_CONDVAR_H
 #define Py_INTERNAL_CONDVAR_H
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #ifndef _POSIX_THREADS

Include/internal/pycore_context.h

@@ -1,8 +1,8 @@
 #ifndef Py_INTERNAL_CONTEXT_H
 #define Py_INTERNAL_CONTEXT_H
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #include "pycore_hamt.h"

Include/internal/pycore_coreconfig.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN defined"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 

Include/internal/pycore_getopt.h

@@ -1,8 +1,8 @@
 #ifndef Py_INTERNAL_PYGETOPT_H
 #define Py_INTERNAL_PYGETOPT_H
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 extern int _PyOS_opterr;

Include/internal/pycore_gil.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #include "pycore_condvar.h"

Include/internal/pycore_hamt.h

@@ -1,8 +1,8 @@
 #ifndef Py_INTERNAL_HAMT_H
 #define Py_INTERNAL_HAMT_H
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #define _Py_HAMT_MAX_TREE_DEPTH 7

Include/internal/pycore_object.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN defined"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #include "pycore_pystate.h"   /* _PyRuntime */

Include/internal/pycore_pathconfig.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 typedef struct _PyPathConfig {

Include/internal/pycore_pyhash.h

@@ -1,8 +1,8 @@
 #ifndef Py_INTERNAL_HASH_H
 #define Py_INTERNAL_HASH_H
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 uint64_t _Py_KeyedHash(uint64_t, const char *, Py_ssize_t);

Include/internal/pycore_pylifecycle.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 /* True if the main interpreter thread exited due to an unhandled

Include/internal/pycore_pymem.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN defined"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #include "objimpl.h"

Include/internal/pycore_pystate.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #include "cpython/coreconfig.h"

Include/internal/pycore_tupleobject.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #include "tupleobject.h"

Include/internal/pycore_warnings.h

@@ -4,8 +4,8 @@
 extern "C" {
 #endif
 
-#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN)
+#ifndef Py_BUILD_CORE
-#  error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define"
+#  error "this header requires Py_BUILD_CORE define"
 #endif
 
 #include "object.h"

Include/pyport.h

@@ -5,6 +5,27 @@
 
 #include <inttypes.h>
 
+
+/* Defines to build Python and its standard library:
+ *
+ * - Py_BUILD_CORE: Build Python core. Give access to Python internals, but
+ *   should not be used by third-party modules.
+ * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module.
+ * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library.
+ *
+ * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE.
+ *
+ * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas
+ * Py_BUILD_CORE_BUILTIN does not.
+ */
+#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE)
+#  define Py_BUILD_CORE
+#endif
+#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE)
+#  define Py_BUILD_CORE
+#endif
+
+
 /**************************************************************************
 Symbols and macros to supply platform-independent interfaces to basic
 C language & library operations whose spellings vary across platforms.
@@ -623,7 +644,7 @@ extern char * _getpty(int *, int, mode_t, int);
 /* only get special linkage if built as shared or platform is Cygwin */
 #if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__)
 #       if defined(HAVE_DECLSPEC_DLL)
-#               if defined(Py_BUILD_CORE) || defined(Py_BUILD_CORE_BUILTIN)
+#               if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
 #                       define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE
 #                       define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE
         /* module init functions inside the core need no external linkage */
@@ -755,7 +776,7 @@ extern char * _getpty(int *, int, mode_t, int);
 #define PY_LITTLE_ENDIAN 1
 #endif
 
-#if defined(Py_BUILD_CORE) || defined(Py_BUILD_CORE_BUILTIN)
+#ifdef Py_BUILD_CORE
 /*
  * Macros to protect CRT calls against instant termination when passed an
  * invalid parameter (issue23524).

Misc/NEWS.d/next/Build/2019-04-16-13-58-52.bpo-36635.JKlzkf.rst

@@ -0,0 +1,5 @@
+Change ``PyAPI_FUNC(type)``, ``PyAPI_DATA(type)`` and ``PyMODINIT_FUNC``
+macros of ``pyport.h`` when ``Py_BUILD_CORE_MODULE`` is defined. The
+``Py_BUILD_CORE_MODULE`` define must be now be used to build a C extension
+as a dynamic library accessing Python internals: export the PyInit_xxx()
+function in DLL exports on Windows.

Modules/Setup

@@ -101,29 +101,29 @@ PYTHONPATH=$(COREPYTHONPATH)
 # This only contains the minimal set of modules required to run the
 # setup.py script in the root of the Python source tree.
 
-posix -DPy_BUILD_CORE -I$(srcdir)/Include/internal posixmodule.c # posix (UNIX) system calls
+posix -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal posixmodule.c # posix (UNIX) system calls
 errno errnomodule.c			# posix (UNIX) errno values
 pwd pwdmodule.c				# this is needed to find out the user's home dir
 					# if $HOME is not set
 _sre _sre.c				# Fredrik Lundh's new regular expressions
 _codecs _codecsmodule.c			# access to the builtin codecs and codec registry
 _weakref _weakref.c			# weak references
-_functools -DPy_BUILD_CORE -I$(srcdir)/Include/internal _functoolsmodule.c   # Tools for working with functions and callable objects
+_functools -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal _functoolsmodule.c   # Tools for working with functions and callable objects
 _operator _operator.c	        	# operator.add() and similar goodies
 _collections _collectionsmodule.c	# Container types
 _abc _abc.c				# Abstract base classes
 itertools itertoolsmodule.c		# Functions creating iterators for efficient looping
 atexit atexitmodule.c			# Register functions to be run at interpreter-shutdown
-_signal -DPy_BUILD_CORE -I$(srcdir)/Include/internal signalmodule.c
+_signal -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal signalmodule.c
 _stat _stat.c				# stat.h interface
-time -DPy_BUILD_CORE -I$(srcdir)/Include/internal timemodule.c	# -lm # time operations and variables
+time -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal timemodule.c	# -lm # time operations and variables
-_thread -DPy_BUILD_CORE -I$(srcdir)/Include/internal _threadmodule.c	# low-level threading interface
+_thread -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal _threadmodule.c	# low-level threading interface
 
 # access to ISO C locale support
-_locale -DPy_BUILD_CORE _localemodule.c  # -lintl
+_locale -DPy_BUILD_CORE_BUILTIN _localemodule.c  # -lintl
 
 # Standard I/O baseline
-_io -DPy_BUILD_CORE -I$(srcdir)/Include/internal -I$(srcdir)/Modules/_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c
+_io -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal -I$(srcdir)/Modules/_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c
 
 # faulthandler module
 faulthandler faulthandler.c

Modules/_json.c

@@ -1,8 +1,11 @@
+/* JSON accelerator C extensor: _json module.
+ *
+ * It is built as a built-in module (Py_BUILD_CORE_BUILTIN define) on Windows
+ * and as an extension module (Py_BUILD_CORE_MODULE define) on other
+ * platforms. */
 
-/* Core extension modules are built-in on some platforms (e.g. Windows). */
+#if !defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE_MODULE)
-#ifdef Py_BUILD_CORE
+#  error "Py_BUILD_CORE_BUILTIN or Py_BUILD_CORE_MODULE must be defined"
-#define Py_BUILD_CORE_BUILTIN
-#undef Py_BUILD_CORE
 #endif
 
 #include "Python.h"

Modules/_pickle.c

@@ -1,8 +1,11 @@
+/* pickle accelerator C extensor: _pickle module.
+ *
+ * It is built as a built-in module (Py_BUILD_CORE_BUILTIN define) on Windows
+ * and as an extension module (Py_BUILD_CORE_MODULE define) on other
+ * platforms. */
 
-/* Core extension modules are built-in on some platforms (e.g. Windows). */
+#if !defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE_MODULE)
-#ifdef Py_BUILD_CORE
+#  error "Py_BUILD_CORE_BUILTIN or Py_BUILD_CORE_MODULE must be defined"
-#define Py_BUILD_CORE_BUILTIN
-#undef Py_BUILD_CORE
 #endif
 
 #include "Python.h"

Modules/_testcapimodule.c

@@ -5,6 +5,11 @@
  * standard Python regression test, via Lib/test/test_capi.py.
  */
 
+/* The Visual Studio projects builds _testcapi with Py_BUILD_CORE_MODULE
+   define, but we only want to test the public C API, not the internal
+   C API. */
+#undef Py_BUILD_CORE_MODULE
+
 #define PY_SSIZE_T_CLEAN
 
 #include "Python.h"

PCbuild/pythoncore.vcxproj

@@ -86,7 +86,7 @@
       <AdditionalOptions>/Zm200  %(AdditionalOptions)</AdditionalOptions>
       <AdditionalIncludeDirectories>$(PySourcePath)Python;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories Condition="$(IncludeExternals)">$(zlibDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="$(IncludeExternals)">_Py_HAVE_ZLIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>

Python/ceval.c

@@ -30,6 +30,10 @@
 #define CHECKEXC 1      /* Double-check exception checking */
 #endif
 
+#if !defined(Py_BUILD_CORE)
+#  error "ceval.c must be build with Py_BUILD_CORE define for best performance"
+#endif
+
 /* Private API for the LOAD_METHOD opcode. */
 extern int _PyObject_GetMethod(PyObject *, PyObject *, PyObject **);
 

setup.py

@@ -725,13 +725,13 @@ class PyBuildExt(build_ext):
         # heapq
         self.add(Extension("_heapq", ["_heapqmodule.c"]))
         # C-optimized pickle replacement
-        self.add(Extension("_pickle", ["_pickle.c"]))
+        self.add(Extension("_pickle", ["_pickle.c"],
+                           extra_compile_args=['-D Py_BUILD_CORE_MODULE']))
         # atexit
         self.add(Extension("atexit", ["atexitmodule.c"]))
         # _json speedups
         self.add(Extension("_json", ["_json.c"],
-                           # pycore_accu.h requires Py_BUILD_CORE_BUILTIN
+                           extra_compile_args=['-D Py_BUILD_CORE_MODULE']))
-                           extra_compile_args=['-DPy_BUILD_CORE_BUILTIN']))
 
         # profiler (_lsprof is for cProfile.py)
         self.add(Extension('_lsprof', ['_lsprof.c', 'rotatingtree.c']))