mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 18:29:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

gh-121313: multiprocessing: simplify by increasing the connection buffer size to 64KiB (GH-123559)

Browse source 
Increases the multiprocessing connection buffer size from 8k to 64k for efficiency, without overallocating.

Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
Co-authored-by: Victor Stinner <vstinner@python.org>
This commit is contained in:
Inada Naoki 2024-09-03 12:32:38 +09:00 • committed by GitHub
parent 1f4a49ea53
commit 13f61bf7f1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 5 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

20
Lib/multiprocessing/connection.py
View file

@ -13,7 +13,6 @@ import errno
import io import io
import itertools import itertools
import os import os
import stat
import sys import sys
import socket import socket
import struct import struct
@ -40,7 +39,9 @@ except ImportError:
# #
# #
BUFSIZE = 8192 # 64 KiB is the default PIPE buffer size of most POSIX platforms.
BUFSIZE = 64 * 1024
# A very generous timeout when it comes to local connections... # A very generous timeout when it comes to local connections...
CONNECTION_TIMEOUT = 20. CONNECTION_TIMEOUT = 20.
@ -361,11 +362,6 @@ if _winapi:
f.write(ov.getbuffer()) f.write(ov.getbuffer())
return f return f
"""
The default size of a pipe on Linux systems is 16 times the base page size:
https://man7.org/linux/man-pages/man7/pipe.7.html
"""
PAGES_PER_PIPE = 16
class Connection(_ConnectionBase): class Connection(_ConnectionBase):
""" """
@ -378,14 +374,11 @@ class Connection(_ConnectionBase):
_close(self._handle) _close(self._handle)
_write = _multiprocessing.send _write = _multiprocessing.send
_read = _multiprocessing.recv _read = _multiprocessing.recv
_default_pipe_size = 0
else: else:
def _close(self, _close=os.close): def _close(self, _close=os.close):
_close(self._handle) _close(self._handle)
_write = os.write _write = os.write
_read = os.read _read = os.read
_base_page_size = os.sysconf(os.sysconf_names['SC_PAGESIZE'])
_default_pipe_size = _base_page_size * PAGES_PER_PIPE
def _send(self, buf, write=_write): def _send(self, buf, write=_write):
remaining = len(buf) remaining = len(buf)
@ -400,13 +393,8 @@ class Connection(_ConnectionBase):
buf = io.BytesIO() buf = io.BytesIO()
handle = self._handle handle = self._handle
remaining = size remaining = size
is_pipe = False
if size > self._default_pipe_size > 0:
mode = os.fstat(handle).st_mode
is_pipe = stat.S_ISFIFO(mode)
limit = self._default_pipe_size if is_pipe else remaining
while remaining > 0: while remaining > 0:
to_read = min(limit, remaining) to_read = min(BUFSIZE, remaining)
chunk = read(handle, to_read) chunk = read(handle, to_read)
n = len(chunk) n = len(chunk)
if n == 0: if n == 0:

1
Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst
View file

@ -1 +0,0 @@
Limit reading size in multiprocessing connection._recv for pipes to default pipe size of 16 times base page size, in order to avoid memory overallocation and unnecessary memory management system calls.

1
Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst Normal file
View file

@ -0,0 +1 @@
Limit the reading size in the :class:`multiprocessing.connection.Connection` class to 64 KiB to prevent memory overallocation and unnecessary memory management system calls.