cpython/Doc/includes/mp_pool.py

315 lines
6.9 KiB
Python
Raw Normal View History

#
# A test of `multiprocessing.Pool` class
#
Merged revisions 67398,67423-67424,67432,67440-67441,67444-67445,67454,67457,67463 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r67398 | benjamin.peterson | 2008-11-26 18:39:17 +0100 (Wed, 26 Nov 2008) | 1 line fix typo in sqlite3 docs ........ r67423 | jesse.noller | 2008-11-28 19:59:35 +0100 (Fri, 28 Nov 2008) | 2 lines issue4238: bsd support for cpu_count ........ r67424 | christian.heimes | 2008-11-28 20:33:33 +0100 (Fri, 28 Nov 2008) | 1 line Retain copyright of processing examples. This was requested by a Debian maintainer during packaging of the multiprocessing package for 2.4/2.5 ........ r67432 | benjamin.peterson | 2008-11-29 00:18:46 +0100 (Sat, 29 Nov 2008) | 1 line SVN format 9 is the same it seems ........ r67440 | jeremy.hylton | 2008-11-29 00:42:59 +0100 (Sat, 29 Nov 2008) | 4 lines Move definition int sval into branch of ifdef where it is used. Otherwise, you get a warning about an undefined variable. ........ r67441 | jeremy.hylton | 2008-11-29 01:09:16 +0100 (Sat, 29 Nov 2008) | 2 lines Reflow long lines. ........ r67444 | amaury.forgeotdarc | 2008-11-29 03:03:32 +0100 (Sat, 29 Nov 2008) | 2 lines Fix a small typo in docstring ........ r67445 | benjamin.peterson | 2008-11-30 04:07:33 +0100 (Sun, 30 Nov 2008) | 1 line StringIO.close() stops you from using the buffer, too ........ r67454 | benjamin.peterson | 2008-11-30 15:43:23 +0100 (Sun, 30 Nov 2008) | 1 line note the version that works ........ r67457 | christian.heimes | 2008-11-30 22:16:28 +0100 (Sun, 30 Nov 2008) | 1 line w# requires Py_ssize_t ........ r67463 | skip.montanaro | 2008-12-01 02:55:22 +0100 (Mon, 01 Dec 2008) | 1 line typo in comment ........
2008-12-05 05:00:55 -04:00
# Copyright (c) 2006-2008, R Oudkerk
# All rights reserved.
#
import multiprocessing
import time
import random
import sys
#
# Functions used by test code
#
def calculate(func, args):
result = func(*args)
return '%s says that %s%s = %s' % (
multiprocessing.current_process().name,
func.__name__, args, result
)
def calculatestar(args):
return calculate(*args)
def mul(a, b):
time.sleep(0.5*random.random())
return a * b
def plus(a, b):
time.sleep(0.5*random.random())
return a + b
def f(x):
return 1.0 / (x-5.0)
def pow3(x):
return x**3
def noop(x):
pass
#
# Test code
#
def test():
print 'cpu_count() = %d\n' % multiprocessing.cpu_count()
#
# Create pool
#
PROCESSES = 4
print 'Creating pool with %d processes\n' % PROCESSES
pool = multiprocessing.Pool(PROCESSES)
print 'pool = %s' % pool
print
#
# Tests
#
TASKS = [(mul, (i, 7)) for i in range(10)] + \
[(plus, (i, 8)) for i in range(10)]
results = [pool.apply_async(calculate, t) for t in TASKS]
imap_it = pool.imap(calculatestar, TASKS)
imap_unordered_it = pool.imap_unordered(calculatestar, TASKS)
print 'Ordered results using pool.apply_async():'
for r in results:
print '\t', r.get()
print
print 'Ordered results using pool.imap():'
for x in imap_it:
print '\t', x
print
print 'Unordered results using pool.imap_unordered():'
for x in imap_unordered_it:
print '\t', x
print
print 'Ordered results using pool.map() --- will block till complete:'
for x in pool.map(calculatestar, TASKS):
print '\t', x
print
#
# Simple benchmarks
#
N = 100000
print 'def pow3(x): return x**3'
t = time.time()
A = map(pow3, xrange(N))
print '\tmap(pow3, xrange(%d)):\n\t\t%s seconds' % \
(N, time.time() - t)
t = time.time()
B = pool.map(pow3, xrange(N))
print '\tpool.map(pow3, xrange(%d)):\n\t\t%s seconds' % \
(N, time.time() - t)
t = time.time()
C = list(pool.imap(pow3, xrange(N), chunksize=N//8))
print '\tlist(pool.imap(pow3, xrange(%d), chunksize=%d)):\n\t\t%s' \
' seconds' % (N, N//8, time.time() - t)
assert A == B == C, (len(A), len(B), len(C))
print
L = [None] * 1000000
print 'def noop(x): pass'
print 'L = [None] * 1000000'
t = time.time()
A = map(noop, L)
print '\tmap(noop, L):\n\t\t%s seconds' % \
(time.time() - t)
t = time.time()
B = pool.map(noop, L)
print '\tpool.map(noop, L):\n\t\t%s seconds' % \
(time.time() - t)
t = time.time()
C = list(pool.imap(noop, L, chunksize=len(L)//8))
print '\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \
(len(L)//8, time.time() - t)
assert A == B == C, (len(A), len(B), len(C))
print
del A, B, C, L
#
# Test error handling
#
print 'Testing error handling:'
try:
print pool.apply(f, (5,))
except ZeroDivisionError:
print '\tGot ZeroDivisionError as expected from pool.apply()'
else:
Merged revisions 72558,72745,72750,72876,73042,73045-73048,73069,73089,73163,73186,73213,73215,73217,73257-73258,73260 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r72558 | benjamin.peterson | 2009-05-11 01:52:09 +0200 (Mo, 11 Mai 2009) | 1 line sys.setdefaultencoding() strikes me as a bad example ........ r72745 | benjamin.peterson | 2009-05-17 16:16:29 +0200 (So, 17 Mai 2009) | 1 line ignore .rst files in sphinx its self ........ r72750 | benjamin.peterson | 2009-05-17 18:59:27 +0200 (So, 17 Mai 2009) | 1 line chop off slash ........ r72876 | benjamin.peterson | 2009-05-23 22:59:09 +0200 (Sa, 23 Mai 2009) | 1 line remove mention of old ctypes version ........ r73042 | benjamin.peterson | 2009-05-30 05:10:52 +0200 (Sa, 30 Mai 2009) | 1 line no fdatasync on macos ........ r73045 | georg.brandl | 2009-05-30 09:26:04 +0200 (Sa, 30 Mai 2009) | 1 line #6146: fix markup bug. ........ r73046 | georg.brandl | 2009-05-30 09:31:25 +0200 (Sa, 30 Mai 2009) | 1 line Use preferred form of raising exceptions. ........ r73047 | georg.brandl | 2009-05-30 12:33:23 +0200 (Sa, 30 Mai 2009) | 1 line Fix some more small markup problems. ........ r73048 | georg.brandl | 2009-05-30 12:34:25 +0200 (Sa, 30 Mai 2009) | 1 line Fix markup problem. ........ r73069 | benjamin.peterson | 2009-05-31 02:42:42 +0200 (So, 31 Mai 2009) | 1 line fix signature ........ r73089 | andrew.kuchling | 2009-06-01 02:14:19 +0200 (Mo, 01 Jun 2009) | 1 line The class for regexes isn't called RegexObject any more; correct the text ........ r73163 | georg.brandl | 2009-06-03 09:25:35 +0200 (Mi, 03 Jun 2009) | 1 line Use the preferred form of raise statements in the docs. ........ r73186 | georg.brandl | 2009-06-03 23:21:09 +0200 (Mi, 03 Jun 2009) | 1 line #6174: fix indentation in code example. ........ r73213 | georg.brandl | 2009-06-04 12:15:57 +0200 (Do, 04 Jun 2009) | 1 line #5967: note that the C slicing APIs do not support negative indices. ........ r73215 | georg.brandl | 2009-06-04 12:22:31 +0200 (Do, 04 Jun 2009) | 1 line #6176: fix man page section for flock(2). ........ r73217 | georg.brandl | 2009-06-04 12:27:21 +0200 (Do, 04 Jun 2009) | 1 line #6175: document that inet_aton supports alternate input formats with less than three dots. ........ r73257 | georg.brandl | 2009-06-06 19:50:05 +0200 (Sa, 06 Jun 2009) | 1 line #6211: elaborate a bit on ways to call the function. ........ r73258 | georg.brandl | 2009-06-06 19:51:31 +0200 (Sa, 06 Jun 2009) | 1 line #6204: use a real reference instead of "see later". ........ r73260 | georg.brandl | 2009-06-06 20:21:58 +0200 (Sa, 06 Jun 2009) | 1 line #6224: s/JPython/Jython/, and remove one link to a module nine years old. ........
2009-10-27 11:19:50 -03:00
raise AssertionError('expected ZeroDivisionError')
try:
print pool.map(f, range(10))
except ZeroDivisionError:
print '\tGot ZeroDivisionError as expected from pool.map()'
else:
Merged revisions 72558,72745,72750,72876,73042,73045-73048,73069,73089,73163,73186,73213,73215,73217,73257-73258,73260 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r72558 | benjamin.peterson | 2009-05-11 01:52:09 +0200 (Mo, 11 Mai 2009) | 1 line sys.setdefaultencoding() strikes me as a bad example ........ r72745 | benjamin.peterson | 2009-05-17 16:16:29 +0200 (So, 17 Mai 2009) | 1 line ignore .rst files in sphinx its self ........ r72750 | benjamin.peterson | 2009-05-17 18:59:27 +0200 (So, 17 Mai 2009) | 1 line chop off slash ........ r72876 | benjamin.peterson | 2009-05-23 22:59:09 +0200 (Sa, 23 Mai 2009) | 1 line remove mention of old ctypes version ........ r73042 | benjamin.peterson | 2009-05-30 05:10:52 +0200 (Sa, 30 Mai 2009) | 1 line no fdatasync on macos ........ r73045 | georg.brandl | 2009-05-30 09:26:04 +0200 (Sa, 30 Mai 2009) | 1 line #6146: fix markup bug. ........ r73046 | georg.brandl | 2009-05-30 09:31:25 +0200 (Sa, 30 Mai 2009) | 1 line Use preferred form of raising exceptions. ........ r73047 | georg.brandl | 2009-05-30 12:33:23 +0200 (Sa, 30 Mai 2009) | 1 line Fix some more small markup problems. ........ r73048 | georg.brandl | 2009-05-30 12:34:25 +0200 (Sa, 30 Mai 2009) | 1 line Fix markup problem. ........ r73069 | benjamin.peterson | 2009-05-31 02:42:42 +0200 (So, 31 Mai 2009) | 1 line fix signature ........ r73089 | andrew.kuchling | 2009-06-01 02:14:19 +0200 (Mo, 01 Jun 2009) | 1 line The class for regexes isn't called RegexObject any more; correct the text ........ r73163 | georg.brandl | 2009-06-03 09:25:35 +0200 (Mi, 03 Jun 2009) | 1 line Use the preferred form of raise statements in the docs. ........ r73186 | georg.brandl | 2009-06-03 23:21:09 +0200 (Mi, 03 Jun 2009) | 1 line #6174: fix indentation in code example. ........ r73213 | georg.brandl | 2009-06-04 12:15:57 +0200 (Do, 04 Jun 2009) | 1 line #5967: note that the C slicing APIs do not support negative indices. ........ r73215 | georg.brandl | 2009-06-04 12:22:31 +0200 (Do, 04 Jun 2009) | 1 line #6176: fix man page section for flock(2). ........ r73217 | georg.brandl | 2009-06-04 12:27:21 +0200 (Do, 04 Jun 2009) | 1 line #6175: document that inet_aton supports alternate input formats with less than three dots. ........ r73257 | georg.brandl | 2009-06-06 19:50:05 +0200 (Sa, 06 Jun 2009) | 1 line #6211: elaborate a bit on ways to call the function. ........ r73258 | georg.brandl | 2009-06-06 19:51:31 +0200 (Sa, 06 Jun 2009) | 1 line #6204: use a real reference instead of "see later". ........ r73260 | georg.brandl | 2009-06-06 20:21:58 +0200 (Sa, 06 Jun 2009) | 1 line #6224: s/JPython/Jython/, and remove one link to a module nine years old. ........
2009-10-27 11:19:50 -03:00
raise AssertionError('expected ZeroDivisionError')
try:
print list(pool.imap(f, range(10)))
except ZeroDivisionError:
print '\tGot ZeroDivisionError as expected from list(pool.imap())'
else:
Merged revisions 72558,72745,72750,72876,73042,73045-73048,73069,73089,73163,73186,73213,73215,73217,73257-73258,73260 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r72558 | benjamin.peterson | 2009-05-11 01:52:09 +0200 (Mo, 11 Mai 2009) | 1 line sys.setdefaultencoding() strikes me as a bad example ........ r72745 | benjamin.peterson | 2009-05-17 16:16:29 +0200 (So, 17 Mai 2009) | 1 line ignore .rst files in sphinx its self ........ r72750 | benjamin.peterson | 2009-05-17 18:59:27 +0200 (So, 17 Mai 2009) | 1 line chop off slash ........ r72876 | benjamin.peterson | 2009-05-23 22:59:09 +0200 (Sa, 23 Mai 2009) | 1 line remove mention of old ctypes version ........ r73042 | benjamin.peterson | 2009-05-30 05:10:52 +0200 (Sa, 30 Mai 2009) | 1 line no fdatasync on macos ........ r73045 | georg.brandl | 2009-05-30 09:26:04 +0200 (Sa, 30 Mai 2009) | 1 line #6146: fix markup bug. ........ r73046 | georg.brandl | 2009-05-30 09:31:25 +0200 (Sa, 30 Mai 2009) | 1 line Use preferred form of raising exceptions. ........ r73047 | georg.brandl | 2009-05-30 12:33:23 +0200 (Sa, 30 Mai 2009) | 1 line Fix some more small markup problems. ........ r73048 | georg.brandl | 2009-05-30 12:34:25 +0200 (Sa, 30 Mai 2009) | 1 line Fix markup problem. ........ r73069 | benjamin.peterson | 2009-05-31 02:42:42 +0200 (So, 31 Mai 2009) | 1 line fix signature ........ r73089 | andrew.kuchling | 2009-06-01 02:14:19 +0200 (Mo, 01 Jun 2009) | 1 line The class for regexes isn't called RegexObject any more; correct the text ........ r73163 | georg.brandl | 2009-06-03 09:25:35 +0200 (Mi, 03 Jun 2009) | 1 line Use the preferred form of raise statements in the docs. ........ r73186 | georg.brandl | 2009-06-03 23:21:09 +0200 (Mi, 03 Jun 2009) | 1 line #6174: fix indentation in code example. ........ r73213 | georg.brandl | 2009-06-04 12:15:57 +0200 (Do, 04 Jun 2009) | 1 line #5967: note that the C slicing APIs do not support negative indices. ........ r73215 | georg.brandl | 2009-06-04 12:22:31 +0200 (Do, 04 Jun 2009) | 1 line #6176: fix man page section for flock(2). ........ r73217 | georg.brandl | 2009-06-04 12:27:21 +0200 (Do, 04 Jun 2009) | 1 line #6175: document that inet_aton supports alternate input formats with less than three dots. ........ r73257 | georg.brandl | 2009-06-06 19:50:05 +0200 (Sa, 06 Jun 2009) | 1 line #6211: elaborate a bit on ways to call the function. ........ r73258 | georg.brandl | 2009-06-06 19:51:31 +0200 (Sa, 06 Jun 2009) | 1 line #6204: use a real reference instead of "see later". ........ r73260 | georg.brandl | 2009-06-06 20:21:58 +0200 (Sa, 06 Jun 2009) | 1 line #6224: s/JPython/Jython/, and remove one link to a module nine years old. ........
2009-10-27 11:19:50 -03:00
raise AssertionError('expected ZeroDivisionError')
it = pool.imap(f, range(10))
for i in range(10):
try:
x = it.next()
except ZeroDivisionError:
if i == 5:
pass
except StopIteration:
break
else:
if i == 5:
Merged revisions 72558,72745,72750,72876,73042,73045-73048,73069,73089,73163,73186,73213,73215,73217,73257-73258,73260 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r72558 | benjamin.peterson | 2009-05-11 01:52:09 +0200 (Mo, 11 Mai 2009) | 1 line sys.setdefaultencoding() strikes me as a bad example ........ r72745 | benjamin.peterson | 2009-05-17 16:16:29 +0200 (So, 17 Mai 2009) | 1 line ignore .rst files in sphinx its self ........ r72750 | benjamin.peterson | 2009-05-17 18:59:27 +0200 (So, 17 Mai 2009) | 1 line chop off slash ........ r72876 | benjamin.peterson | 2009-05-23 22:59:09 +0200 (Sa, 23 Mai 2009) | 1 line remove mention of old ctypes version ........ r73042 | benjamin.peterson | 2009-05-30 05:10:52 +0200 (Sa, 30 Mai 2009) | 1 line no fdatasync on macos ........ r73045 | georg.brandl | 2009-05-30 09:26:04 +0200 (Sa, 30 Mai 2009) | 1 line #6146: fix markup bug. ........ r73046 | georg.brandl | 2009-05-30 09:31:25 +0200 (Sa, 30 Mai 2009) | 1 line Use preferred form of raising exceptions. ........ r73047 | georg.brandl | 2009-05-30 12:33:23 +0200 (Sa, 30 Mai 2009) | 1 line Fix some more small markup problems. ........ r73048 | georg.brandl | 2009-05-30 12:34:25 +0200 (Sa, 30 Mai 2009) | 1 line Fix markup problem. ........ r73069 | benjamin.peterson | 2009-05-31 02:42:42 +0200 (So, 31 Mai 2009) | 1 line fix signature ........ r73089 | andrew.kuchling | 2009-06-01 02:14:19 +0200 (Mo, 01 Jun 2009) | 1 line The class for regexes isn't called RegexObject any more; correct the text ........ r73163 | georg.brandl | 2009-06-03 09:25:35 +0200 (Mi, 03 Jun 2009) | 1 line Use the preferred form of raise statements in the docs. ........ r73186 | georg.brandl | 2009-06-03 23:21:09 +0200 (Mi, 03 Jun 2009) | 1 line #6174: fix indentation in code example. ........ r73213 | georg.brandl | 2009-06-04 12:15:57 +0200 (Do, 04 Jun 2009) | 1 line #5967: note that the C slicing APIs do not support negative indices. ........ r73215 | georg.brandl | 2009-06-04 12:22:31 +0200 (Do, 04 Jun 2009) | 1 line #6176: fix man page section for flock(2). ........ r73217 | georg.brandl | 2009-06-04 12:27:21 +0200 (Do, 04 Jun 2009) | 1 line #6175: document that inet_aton supports alternate input formats with less than three dots. ........ r73257 | georg.brandl | 2009-06-06 19:50:05 +0200 (Sa, 06 Jun 2009) | 1 line #6211: elaborate a bit on ways to call the function. ........ r73258 | georg.brandl | 2009-06-06 19:51:31 +0200 (Sa, 06 Jun 2009) | 1 line #6204: use a real reference instead of "see later". ........ r73260 | georg.brandl | 2009-06-06 20:21:58 +0200 (Sa, 06 Jun 2009) | 1 line #6224: s/JPython/Jython/, and remove one link to a module nine years old. ........
2009-10-27 11:19:50 -03:00
raise AssertionError('expected ZeroDivisionError')
assert i == 9
print '\tGot ZeroDivisionError as expected from IMapIterator.next()'
print
#
# Testing timeouts
#
print 'Testing ApplyResult.get() with timeout:',
res = pool.apply_async(calculate, TASKS[0])
while 1:
sys.stdout.flush()
try:
sys.stdout.write('\n\t%s' % res.get(0.02))
break
except multiprocessing.TimeoutError:
sys.stdout.write('.')
print
print
print 'Testing IMapIterator.next() with timeout:',
it = pool.imap(calculatestar, TASKS)
while 1:
sys.stdout.flush()
try:
sys.stdout.write('\n\t%s' % it.next(0.02))
except StopIteration:
break
except multiprocessing.TimeoutError:
sys.stdout.write('.')
print
print
#
# Testing callback
#
print 'Testing callback:'
A = []
B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729]
r = pool.apply_async(mul, (7, 8), callback=A.append)
r.wait()
r = pool.map_async(pow3, range(10), callback=A.extend)
r.wait()
if A == B:
print '\tcallbacks succeeded\n'
else:
print '\t*** callbacks failed\n\t\t%s != %s\n' % (A, B)
#
# Check there are no outstanding tasks
#
assert not pool._cache, 'cache = %r' % pool._cache
#
# Check close() methods
#
print 'Testing close():'
for worker in pool._pool:
assert worker.is_alive()
result = pool.apply_async(time.sleep, [0.5])
pool.close()
pool.join()
assert result.get() is None
for worker in pool._pool:
assert not worker.is_alive()
print '\tclose() succeeded\n'
#
# Check terminate() method
#
print 'Testing terminate():'
pool = multiprocessing.Pool(2)
DELTA = 0.1
ignore = pool.apply(pow3, [2])
results = [pool.apply_async(time.sleep, [DELTA]) for i in range(100)]
pool.terminate()
pool.join()
for worker in pool._pool:
assert not worker.is_alive()
print '\tterminate() succeeded\n'
#
# Check garbage collection
#
print 'Testing garbage collection:'
pool = multiprocessing.Pool(2)
DELTA = 0.1
processes = pool._pool
ignore = pool.apply(pow3, [2])
results = [pool.apply_async(time.sleep, [DELTA]) for i in range(100)]
results = pool = None
time.sleep(DELTA * 2)
for worker in processes:
assert not worker.is_alive()
print '\tgarbage collection succeeded\n'
if __name__ == '__main__':
multiprocessing.freeze_support()
assert len(sys.argv) in (1, 2)
if len(sys.argv) == 1 or sys.argv[1] == 'processes':
print ' Using processes '.center(79, '-')
elif sys.argv[1] == 'threads':
print ' Using threads '.center(79, '-')
import multiprocessing.dummy as multiprocessing
else:
print 'Usage:\n\t%s [processes | threads]' % sys.argv[0]
raise SystemExit(2)
test()