traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-29877: compileall: import ProcessPoolExecutor only when needed (GH-4856) 

Importing ProcessPoolExecutor may hang or cause an error when the import
accesses urandom on a low resource platform


https://bugs.python.org/issue29877
This commit is contained in:
Dustin Spicuzza 2018-11-23 12:06:55 -05:00 committed by Miss Islington (bot)
parent 9de3632715
commit 1d817e4c82
3 changed files with 17 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Lib/compileall.py
View File

@ -16,10 +16,6 @@ import importlib.util
import py_compile import py_compile
import struct import struct
try:
from concurrent.futures import ProcessPoolExecutor
except ImportError:
ProcessPoolExecutor = None
from functools import partial from functools import partial
__all__ = ["compile_dir","compile_file","compile_path"] __all__ = ["compile_dir","compile_file","compile_path"]
@ -70,9 +66,17 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
workers: maximum number of parallel workers workers: maximum number of parallel workers
invalidation_mode: how the up-to-dateness of the pyc will be checked invalidation_mode: how the up-to-dateness of the pyc will be checked
""" """
if workers is not None and workers < 0: ProcessPoolExecutor = None
raise ValueError('workers must be greater or equal to 0') if workers is not None:
if workers < 0:
raise ValueError('workers must be greater or equal to 0')
elif workers != 1:
try:
# Only import when needed, as low resource platforms may
# fail to import it
from concurrent.futures import ProcessPoolExecutor
except ImportError:
workers = 1
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
ddir=ddir) ddir=ddir)
success = True success = True

8
Lib/test/test_compileall.py
View File

@ -167,7 +167,7 @@ class CompileallTestsBase:
self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)')
self.assertTrue(os.path.isfile(self.bc_path)) self.assertTrue(os.path.isfile(self.bc_path))
@mock.patch('compileall.ProcessPoolExecutor') @mock.patch('concurrent.futures.ProcessPoolExecutor')
def test_compile_pool_called(self, pool_mock): def test_compile_pool_called(self, pool_mock):
compileall.compile_dir(self.directory, quiet=True, workers=5) compileall.compile_dir(self.directory, quiet=True, workers=5)
self.assertTrue(pool_mock.called) self.assertTrue(pool_mock.called)
@ -177,19 +177,19 @@ class CompileallTestsBase:
"workers must be greater or equal to 0"): "workers must be greater or equal to 0"):
compileall.compile_dir(self.directory, workers=-1) compileall.compile_dir(self.directory, workers=-1)
@mock.patch('compileall.ProcessPoolExecutor') @mock.patch('concurrent.futures.ProcessPoolExecutor')
def test_compile_workers_cpu_count(self, pool_mock): def test_compile_workers_cpu_count(self, pool_mock):
compileall.compile_dir(self.directory, quiet=True, workers=0) compileall.compile_dir(self.directory, quiet=True, workers=0)
self.assertEqual(pool_mock.call_args[1]['max_workers'], None) self.assertEqual(pool_mock.call_args[1]['max_workers'], None)
@mock.patch('compileall.ProcessPoolExecutor') @mock.patch('concurrent.futures.ProcessPoolExecutor')
@mock.patch('compileall.compile_file') @mock.patch('compileall.compile_file')
def test_compile_one_worker(self, compile_file_mock, pool_mock): def test_compile_one_worker(self, compile_file_mock, pool_mock):
compileall.compile_dir(self.directory, quiet=True) compileall.compile_dir(self.directory, quiet=True)
self.assertFalse(pool_mock.called) self.assertFalse(pool_mock.called)
self.assertTrue(compile_file_mock.called) self.assertTrue(compile_file_mock.called)
@mock.patch('compileall.ProcessPoolExecutor', new=None) @mock.patch('concurrent.futures.ProcessPoolExecutor', new=None)
@mock.patch('compileall.compile_file') @mock.patch('compileall.compile_file')
def test_compile_missing_multiprocessing(self, compile_file_mock): def test_compile_missing_multiprocessing(self, compile_file_mock):
compileall.compile_dir(self.directory, quiet=True, workers=5) compileall.compile_dir(self.directory, quiet=True, workers=5)

2
Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst Normal file
View File

@ -0,0 +1,2 @@
compileall: import ProcessPoolExecutor only when needed, preventing hangs on
low resource platforms