bpo-26903: Limit ProcessPoolExecutor to 61 workers on Windows (GH-13132)

Co-Authored-By: brianquinlan <brian@sweetapp.com>
This commit is contained in:
Brian Quinlan 2019-05-08 14:04:53 -04:00 committed by Steve Dower
parent b9b08cd948
commit 39889864c0
4 changed files with 26 additions and 0 deletions

View file

@ -57,6 +57,7 @@ import threading
import weakref
from functools import partial
import itertools
import sys
import traceback
# Workers are created as daemon threads and processes. This is done to allow the
@ -109,6 +110,12 @@ def _python_exit():
EXTRA_QUEUED_CALLS = 1
# On Windows, WaitForMultipleObjects is used to wait for processes to finish.
# It can wait on, at most, 63 objects. There is an overhead of two objects:
# - the result queue reader
# - the thread wakeup reader
_MAX_WINDOWS_WORKERS = 63 - 2
# Hack to embed stringification of remote traceback in local traceback
class _RemoteTraceback(Exception):
@ -505,9 +512,16 @@ class ProcessPoolExecutor(_base.Executor):
if max_workers is None:
self._max_workers = os.cpu_count() or 1
if sys.platform == 'win32':
self._max_workers = min(_MAX_WINDOWS_WORKERS,
self._max_workers)
else:
if max_workers <= 0:
raise ValueError("max_workers must be greater than 0")
elif (sys.platform == 'win32' and
max_workers > _MAX_WINDOWS_WORKERS):
raise ValueError(
f"max_workers must be <= {_MAX_WINDOWS_WORKERS}")
self._max_workers = max_workers