mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-03 16:39:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-30919: shared memory allocation performance regression in multiprocessing (#2708)

Browse source 
* Fix #30919: shared memory allocation performance regression in multiprocessing

* Change strategy for Arena directory choice

* Add blurb
This commit is contained in:
Antoine Pitrou 2017-07-23 13:05:26 +02:00 • committed by GitHub
parent 2b1e6e9696
commit 3051f0b78e
2 changed files with 20 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

26
Lib/multiprocessing/heap.py
View file

@ -60,26 +60,32 @@ if sys.platform == 'win32':
else:
class Arena(object):
if sys.platform == 'linux':
_dir_candidates = ['/dev/shm']
else:
_dir_candidates = []
def __init__(self, size, fd=-1):
self.size = size
self.fd = fd
if fd == -1:
self.fd, name = tempfile.mkstemp(
prefix='pym-%d-'%os.getpid(), dir=util.get_temp_dir())
prefix='pym-%d-'%os.getpid(),
dir=self._choose_dir(size))
os.unlink(name)
util.Finalize(self, os.close, (self.fd,))
with open(self.fd, 'wb', closefd=False) as f:
bs = 1024 * 1024
if size >= bs:
zeros = b'\0' * bs
for _ in range(size // bs):
f.write(zeros)
del zeros
f.write(b'\0' * (size % bs))
assert f.tell() == size
os.ftruncate(self.fd, size)
self.buffer = mmap.mmap(self.fd, self.size)
def _choose_dir(self, size):
# Choose a non-storage backed directory if possible,
# to improve performance
for d in self._dir_candidates:
st = os.statvfs(d)
if st.f_bavail * st.f_frsize >= size: # enough free space?
return d
return util.get_temp_dir()
def reduce_arena(a):
if a.fd == -1:
raise ValueError('Arena is unpicklable because '