mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-03 16:39:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-31699 Deadlocks in concurrent.futures.ProcessPoolExecutor with pickling error (#3895)

Browse source 
Fix deadlocks in :class:`concurrent.futures.ProcessPoolExecutor` when task arguments or results cause pickling or unpickling errors.
This should make sure that calls to the :class:`ProcessPoolExecutor` API always eventually return.
This commit is contained in:
Thomas Moreau 2018-01-05 11:15:54 +01:00 • committed by Antoine Pitrou
parent 65f2a6dcc2
commit 94459fd7dc
5 changed files with 387 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

21
Lib/multiprocessing/queues.py
View file

@ -160,9 +160,10 @@ class Queue(object):
self._thread = threading.Thread(
target=Queue._feed,
args=(self._buffer, self._notempty, self._send_bytes,
self._wlock, self._writer.close, self._ignore_epipe),
self._wlock, self._writer.close, self._ignore_epipe,
self._on_queue_feeder_error),
name='QueueFeederThread'
)
)
self._thread.daemon = True
debug('doing self._thread.start()')
@ -201,7 +202,8 @@ class Queue(object):
notempty.notify()
@staticmethod
def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe):
def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe,
onerror):
debug('starting thread to feed data to pipe')
nacquire = notempty.acquire
nrelease = notempty.release
@ -253,8 +255,17 @@ class Queue(object):
info('error in queue thread: %s', e)
return
else:
import traceback
traceback.print_exc()
onerror(e, obj)
@staticmethod
def _on_queue_feeder_error(e, obj):
"""
Private API hook called when feeding data in the background thread
raises an exception. For overriding by concurrent.futures.
"""
import traceback
traceback.print_exc()
_sentinel = object()