Simplify concurrent.futures.process code by using itertools.batched() (GH-114221)

This commit is contained in:
NewUserHa 2024-01-27 16:29:38 +08:00 committed by GitHub
parent 926881dc10
commit 547c135d70
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 11 deletions

View File

@ -190,16 +190,6 @@ class _SafeQueue(Queue):
super()._on_queue_feeder_error(e, obj)
def _get_chunks(*iterables, chunksize):
""" Iterates over zip()ed iterables in chunks. """
it = zip(*iterables)
while True:
chunk = tuple(itertools.islice(it, chunksize))
if not chunk:
return
yield chunk
def _process_chunk(fn, chunk):
""" Processes a chunk of an iterable passed to map.
@ -847,7 +837,7 @@ class ProcessPoolExecutor(_base.Executor):
raise ValueError("chunksize must be >= 1.")
results = super().map(partial(_process_chunk, fn),
_get_chunks(*iterables, chunksize=chunksize),
itertools.batched(zip(*iterables), chunksize),
timeout=timeout)
return _chain_from_iterable_of_lists(results)