diff --git a/Lib/compileall.py b/Lib/compileall.py index 7be23a67c05..aa65c6b904e 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -16,10 +16,6 @@ import importlib.util import py_compile import struct -try: - from concurrent.futures import ProcessPoolExecutor -except ImportError: - ProcessPoolExecutor = None from functools import partial __all__ = ["compile_dir","compile_file","compile_path"] @@ -70,9 +66,17 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, workers: maximum number of parallel workers invalidation_mode: how the up-to-dateness of the pyc will be checked """ - if workers is not None and workers < 0: - raise ValueError('workers must be greater or equal to 0') - + ProcessPoolExecutor = None + if workers is not None: + if workers < 0: + raise ValueError('workers must be greater or equal to 0') + elif workers != 1: + try: + # Only import when needed, as low resource platforms may + # fail to import it + from concurrent.futures import ProcessPoolExecutor + except ImportError: + workers = 1 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, ddir=ddir) success = True diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index bf2a2c4f0ab..2e2552303f8 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -167,7 +167,7 @@ class CompileallTestsBase: self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertTrue(os.path.isfile(self.bc_path)) - @mock.patch('compileall.ProcessPoolExecutor') + @mock.patch('concurrent.futures.ProcessPoolExecutor') def test_compile_pool_called(self, pool_mock): compileall.compile_dir(self.directory, quiet=True, workers=5) self.assertTrue(pool_mock.called) @@ -177,19 +177,19 @@ class CompileallTestsBase: "workers must be greater or equal to 0"): compileall.compile_dir(self.directory, workers=-1) - @mock.patch('compileall.ProcessPoolExecutor') + @mock.patch('concurrent.futures.ProcessPoolExecutor') def test_compile_workers_cpu_count(self, pool_mock): compileall.compile_dir(self.directory, quiet=True, workers=0) self.assertEqual(pool_mock.call_args[1]['max_workers'], None) - @mock.patch('compileall.ProcessPoolExecutor') + @mock.patch('concurrent.futures.ProcessPoolExecutor') @mock.patch('compileall.compile_file') def test_compile_one_worker(self, compile_file_mock, pool_mock): compileall.compile_dir(self.directory, quiet=True) self.assertFalse(pool_mock.called) self.assertTrue(compile_file_mock.called) - @mock.patch('compileall.ProcessPoolExecutor', new=None) + @mock.patch('concurrent.futures.ProcessPoolExecutor', new=None) @mock.patch('compileall.compile_file') def test_compile_missing_multiprocessing(self, compile_file_mock): compileall.compile_dir(self.directory, quiet=True, workers=5) diff --git a/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst b/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst new file mode 100644 index 00000000000..cc09533b712 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst @@ -0,0 +1,2 @@ +compileall: import ProcessPoolExecutor only when needed, preventing hangs on +low resource platforms