mirror of
https://github.com/python/cpython.git
synced 2025-11-17 09:30:10 +00:00
In multiprocessing, a pool worker process would die if the result/error could not be pickled. This could cause pool methods to hang. In 3.x this was fixed by 0aa8af79359d (which also added an error_callback argument to some methods), but the fix was not back ported.
This commit is contained in:
parent
a9e18cdd7f
commit
0c200c282b
2 changed files with 42 additions and 1 deletions
|
|
@ -68,6 +68,23 @@ def mapstar(args):
|
|||
# Code run by worker processes
|
||||
#
|
||||
|
||||
class MaybeEncodingError(Exception):
|
||||
"""Wraps possible unpickleable errors, so they can be
|
||||
safely sent through the socket."""
|
||||
|
||||
def __init__(self, exc, value):
|
||||
self.exc = repr(exc)
|
||||
self.value = repr(value)
|
||||
super(MaybeEncodingError, self).__init__(self.exc, self.value)
|
||||
|
||||
def __str__(self):
|
||||
return "Error sending result: '%s'. Reason: '%s'" % (self.value,
|
||||
self.exc)
|
||||
|
||||
def __repr__(self):
|
||||
return "<MaybeEncodingError: %s>" % str(self)
|
||||
|
||||
|
||||
def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None):
|
||||
assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0)
|
||||
put = outqueue.put
|
||||
|
|
@ -96,7 +113,13 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None):
|
|||
result = (True, func(*args, **kwds))
|
||||
except Exception, e:
|
||||
result = (False, e)
|
||||
put((job, i, result))
|
||||
try:
|
||||
put((job, i, result))
|
||||
except Exception as e:
|
||||
wrapped = MaybeEncodingError(e, result[1])
|
||||
debug("Possible encoding error while sending result: %s" % (
|
||||
wrapped))
|
||||
put((job, i, (False, wrapped)))
|
||||
completed += 1
|
||||
debug('worker exiting after %d tasks' % completed)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue