From d13007fa114ffd9524eb9ea5f0ad2cad3cda0074 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 28 Jun 2011 19:43:15 -0700 Subject: [PATCH 1/6] Issue #9516: Correct and expand OS X deployment target tests in distutils test_build_ext. --- Lib/distutils/tests/test_build_ext.py | 47 +++++++++++++++++++++------ 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index d924f585ba5..0ce7f0f81c8 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -11,7 +11,8 @@ from distutils.tests.support import TempdirManager from distutils.tests.support import LoggingSilencer from distutils.extension import Extension from distutils.errors import ( - CompileError, DistutilsSetupError, UnknownFileError) + CompileError, DistutilsPlatformError, DistutilsSetupError, + UnknownFileError) import unittest from test import support @@ -431,18 +432,43 @@ class BuildExtTestCase(TempdirManager, @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for MacOSX') - def test_deployment_target(self): - self._try_compile_deployment_target() + def test_deployment_target_default(self): + # Issue 9516: Test that, in the absence of the environment variable, + # an extension module is compiled with the same deployment target as + # the interpreter. + self._try_compile_deployment_target('==', None) + @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for MacOSX') + def test_deployment_target_too_low(self): + # Issue 9516: Test that an extension module is not allowed to be + # compiled with a deployment target less than that of the interpreter. + self.assertRaises(DistutilsPlatformError, + self._try_compile_deployment_target, '>', '10.1') + + @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for MacOSX') + def test_deployment_target_higher_ok(self): + # Issue 9516: Test that an extension module can be compiled with a + # deployment target higher than that of the interpreter: the ext + # module may depend on some newer OS feature. + deptarget = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if deptarget: + # increment the minor version number (i.e. 10.6 -> 10.7) + deptarget = [int(x) for x in deptarget.split('.')] + deptarget[-1] += 1 + deptarget = '.'.join(str(i) for i in deptarget) + self._try_compile_deployment_target('<', deptarget) + + def _try_compile_deployment_target(self, operator, target): orig_environ = os.environ os.environ = orig_environ.copy() self.addCleanup(setattr, os, 'environ', orig_environ) - os.environ['MACOSX_DEPLOYMENT_TARGET']='10.1' - self._try_compile_deployment_target() + if target is None: + if os.environ.get('MACOSX_DEPLOYMENT_TARGET'): + del os.environ['MACOSX_DEPLOYMENT_TARGET'] + else: + os.environ['MACOSX_DEPLOYMENT_TARGET'] = target - - def _try_compile_deployment_target(self): deptarget_c = os.path.join(self.tmp_dir, 'deptargetmodule.c') with open(deptarget_c, 'w') as fp: @@ -451,16 +477,17 @@ class BuildExtTestCase(TempdirManager, int dummy; - #if TARGET != MAC_OS_X_VERSION_MIN_REQUIRED + #if TARGET %s MAC_OS_X_VERSION_MIN_REQUIRED + #else #error "Unexpected target" #endif - ''')) + ''' % operator)) + # get the deployment target that the interpreter was built with target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') target = tuple(map(int, target.split('.'))) target = '%02d%01d0' % target - deptarget_ext = Extension( 'deptarget', [deptarget_c], From a8f8b50bd7c8d36935178a2f9f386d6b250d1eed Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 28 Jun 2011 19:44:24 -0700 Subject: [PATCH 2/6] Issue #9516: Change distutils to no longer globally attempt to check and set the MACOSX_DEPLOYMENT_TARGET env variable for the interpreter process on OS X. This could cause failures in non-distutils subprocesses and was unreliable since tests or user programs could modify the interpreter environment after distutils set it. Instead, have distutils set the the deployment target only in the environment of each build subprocess. Continue to use the previous algorithm for deriving the deployment target value: if MACOSX_DEPLOYMENT_TARGET is not set in the interpreter's env: use the interpreter build configure MACOSX_DEPLOYMENT_TARGET elif the MACOSX_DEPLOYMENT_TARGET env value >= configure value: use the env MACOSX_DEPLOYMENT_TARGET else: # env value less than interpreter build configure value raise exception This allows building extensions that can only run on newer versions of the OS than the version python was built for, for example with a python built for 10.3 or later and an extension that needs to be built for 10.5. --- Lib/distutils/spawn.py | 29 ++++++++++++++++++++++++++++- Lib/distutils/sysconfig.py | 15 --------------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py index 8c476dc23f5..2b62c968a42 100644 --- a/Lib/distutils/spawn.py +++ b/Lib/distutils/spawn.py @@ -96,15 +96,42 @@ def _spawn_os2(cmd, search_path=1, verbose=0, dry_run=0): raise DistutilsExecError( "command '%s' failed with exit status %d" % (cmd[0], rc)) +if sys.platform == 'darwin': + from distutils import sysconfig + _cfg_target = None + _cfg_target_split = None + def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0): log.info(' '.join(cmd)) if dry_run: return exec_fn = search_path and os.execvp or os.execv + exec_args = [cmd[0], cmd] + if sys.platform == 'darwin': + global _cfg_target, _cfg_target_split + if _cfg_target is None: + _cfg_target = sysconfig.get_config_var( + 'MACOSX_DEPLOYMENT_TARGET') or '' + if _cfg_target: + _cfg_target_split = [int(x) for x in _cfg_target.split('.')] + if _cfg_target: + # ensure that the deployment target of build process is not less + # than that used when the interpreter was built. This ensures + # extension modules are built with correct compatibility values + cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) + if _cfg_target_split > [int(x) for x in cur_target.split('.')]: + my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' + 'now "%s" but "%s" during configure' + % (cur_target, _cfg_target)) + raise DistutilsPlatformError(my_msg) + env = dict(os.environ, + MACOSX_DEPLOYMENT_TARGET=cur_target) + exec_fn = search_path and os.execvpe or os.execve + exec_args.append(env) pid = os.fork() if pid == 0: # in the child try: - exec_fn(cmd[0], cmd) + exec_fn(*exec_args) except OSError as e: sys.stderr.write("unable to execute %s: %s\n" % (cmd[0], e.strerror)) diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 06bbc01cd6a..9d7d1902aa6 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -419,21 +419,6 @@ def _init_posix(): raise DistutilsPlatformError(my_msg) - # On MacOSX we need to check the setting of the environment variable - # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so - # it needs to be compatible. - # If it isn't set we set it to the configure-time value - if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in g: - cfg_target = g['MACOSX_DEPLOYMENT_TARGET'] - cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '') - if cur_target == '': - cur_target = cfg_target - os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target - elif [int(x) for x in cfg_target.split('.')] > [int(x) for x in cur_target.split('.')]: - my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure' - % (cur_target, cfg_target)) - raise DistutilsPlatformError(my_msg) - # On AIX, there are wrong paths to the linker scripts in the Makefile # -- these paths are relative to the Python source, but when installed # the scripts are in another directory. From 657b2de893a206dbcc3d34baa65f6cfda11726e5 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 28 Jun 2011 19:51:30 -0700 Subject: [PATCH 3/6] Issue #9516: Update Misc/NEWS. --- Misc/NEWS | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index a8c607936cb..834dea6487f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,17 @@ Core and Builtins Library ------- +- Issue #9516: On Mac OS X, change Distutils to no longer globally attempt to + check or set the MACOSX_DEPLOYMENT_TARGET environment variable for the + interpreter process. This could cause failures in non-Distutils subprocesses + and was unreliable since tests or user programs could modify the interpreter + environment after Distutils set it. Instead, have Distutils set the the + deployment target only in the environment of each build subprocess. It is + still possible to globally override the default by setting + MACOSX_DEPLOYMENT_TARGET before launching the interpreter; its value must be + greater or equal to the default value, the value with which the interpreter + was built. + - Issue #12404: Remove C89 incompatible code from mmap module. Patch by Akira Kitada. From 5c727cb978d5d27005fb9f7d29025fb26f1aa949 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 28 Jun 2011 20:03:17 -0700 Subject: [PATCH 4/6] Issue #9516: Port OS X deployment target tests from distutils to packaging test_command_build_ext. --- Lib/packaging/tests/test_command_build_ext.py | 93 ++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/Lib/packaging/tests/test_command_build_ext.py b/Lib/packaging/tests/test_command_build_ext.py index ddf168feeb6..e144fd0738b 100644 --- a/Lib/packaging/tests/test_command_build_ext.py +++ b/Lib/packaging/tests/test_command_build_ext.py @@ -3,9 +3,11 @@ import sys import site import shutil import sysconfig +import textwrap from io import StringIO from packaging.dist import Distribution -from packaging.errors import UnknownFileError, CompileError +from packaging.errors import (UnknownFileError, CompileError, + PackagingPlatformError) from packaging.command.build_ext import build_ext from packaging.compiler.extension import Extension from test.script_helper import assert_python_ok @@ -362,6 +364,95 @@ class BuildExtTestCase(support.TempdirManager, wanted = os.path.join(curdir, 'twisted', 'runner', 'portmap' + ext) self.assertEqual(wanted, path) + @unittest.skipUnless(sys.platform == 'darwin', + 'test only relevant for Mac OS X') + def test_deployment_target_default(self): + # Issue 9516: Test that, in the absence of the environment variable, + # an extension module is compiled with the same deployment target as + # the interpreter. + self._try_compile_deployment_target('==', None) + + @unittest.skipUnless(sys.platform == 'darwin', + 'test only relevant for Mac OS X') + def test_deployment_target_too_low(self): + # Issue 9516: Test that an extension module is not allowed to be + # compiled with a deployment target less than that of the interpreter. + self.assertRaises(PackagingPlatformError, + self._try_compile_deployment_target, '>', '10.1') + + @unittest.skipUnless(sys.platform == 'darwin', + 'test only relevant for Mac OS X') + def test_deployment_target_higher_ok(self): + # Issue 9516: Test that an extension module can be compiled with a + # deployment target higher than that of the interpreter: the ext + # module may depend on some newer OS feature. + deptarget = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if deptarget: + # increment the minor version number (i.e. 10.6 -> 10.7) + deptarget = [int(x) for x in deptarget.split('.')] + deptarget[-1] += 1 + deptarget = '.'.join(str(i) for i in deptarget) + self._try_compile_deployment_target('<', deptarget) + + def _try_compile_deployment_target(self, operator, target): + orig_environ = os.environ + os.environ = orig_environ.copy() + self.addCleanup(setattr, os, 'environ', orig_environ) + + if target is None: + if os.environ.get('MACOSX_DEPLOYMENT_TARGET'): + del os.environ['MACOSX_DEPLOYMENT_TARGET'] + else: + os.environ['MACOSX_DEPLOYMENT_TARGET'] = target + + deptarget_c = os.path.join(self.tmp_dir, 'deptargetmodule.c') + + with open(deptarget_c, 'w') as fp: + fp.write(textwrap.dedent('''\ + #include + + int dummy; + + #if TARGET %s MAC_OS_X_VERSION_MIN_REQUIRED + #else + #error "Unexpected target" + #endif + + ''' % operator)) + + # get the deployment target that the interpreter was built with + target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + target = tuple(map(int, target.split('.'))) + target = '%02d%01d0' % target + + deptarget_ext = Extension( + 'deptarget', + [deptarget_c], + extra_compile_args=['-DTARGET=%s' % (target,)], + ) + dist = Distribution({ + 'name': 'deptarget', + 'ext_modules': [deptarget_ext], + }) + dist.package_dir = self.tmp_dir + cmd = build_ext(dist) + cmd.build_lib = self.tmp_dir + cmd.build_temp = self.tmp_dir + + try: + old_stdout = sys.stdout + if not verbose: + # silence compiler output + sys.stdout = StringIO() + try: + cmd.ensure_finalized() + cmd.run() + finally: + sys.stdout = old_stdout + + except CompileError: + self.fail("Wrong deployment target during compilation") + def test_suite(): src = _get_source_filename() From fceb4120fc4dc4872255206c82f33fd39deb2615 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 28 Jun 2011 20:04:24 -0700 Subject: [PATCH 5/6] Issue #9516: Port the revised deployment target processing for OSX from distutils to packaging. --- Lib/packaging/util.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py index 4dff547a835..76db89a9a69 100644 --- a/Lib/packaging/util.py +++ b/Lib/packaging/util.py @@ -757,6 +757,9 @@ def split_leading_dir(path): else: return path, '' +if sys.platform == 'darwin': + _cfg_target = None + _cfg_target_split = None def spawn(cmd, search_path=True, verbose=0, dry_run=False, env=None): """Run another program specified as a command list 'cmd' in a new process. @@ -781,6 +784,26 @@ def spawn(cmd, search_path=True, verbose=0, dry_run=False, env=None): if dry_run: logging.debug('dry run, no process actually spawned') return + if sys.platform == 'darwin': + global _cfg_target, _cfg_target_split + if _cfg_target is None: + _cfg_target = sysconfig.get_config_var( + 'MACOSX_DEPLOYMENT_TARGET') or '' + if _cfg_target: + _cfg_target_split = [int(x) for x in _cfg_target.split('.')] + if _cfg_target: + # ensure that the deployment target of build process is not less + # than that used when the interpreter was built. This ensures + # extension modules are built with correct compatibility values + env = env or os.environ + cur_target = env.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) + if _cfg_target_split > [int(x) for x in cur_target.split('.')]: + my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' + 'now "%s" but "%s" during configure' + % (cur_target, _cfg_target)) + raise PackagingPlatformError(my_msg) + env = dict(env, MACOSX_DEPLOYMENT_TARGET=cur_target) + exit_status = subprocess.call(cmd, env=env) if exit_status != 0: msg = "command %r failed with exit status %d" From 044369fd52209eb04e34a3ebf4628dec5500142d Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 28 Jun 2011 20:08:34 -0700 Subject: [PATCH 6/6] Issue #9516: Update Misc/NEWS to include packaging. --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index de3026bc6ba..282dc5ac0c6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -209,7 +209,7 @@ Library still possible to globally override the default by setting MACOSX_DEPLOYMENT_TARGET before launching the interpreter; its value must be greater or equal to the default value, the value with which the interpreter - was built. + was built. Also, implement the same handling in packaging. - Issue #12422: In the copy module, don't store objects that are their own copy in the memo dict.