mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 08:59:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

bpo-31945: Configurable blocksize in HTTP(S)Connection (#4279)

Browse source 
blocksize was hardcoded to 8192, preventing efficient upload when using
file-like body. Add blocksize argument to __init__, so users can
configure the blocksize to fit their needs.

I tested this uploading data from /dev/zero to a web server dropping the
received data, to test the overhead of the HTTPConnection.send() with a
file-like object.

Here is an example 10g upload with the default buffer size (8192):

$ time ~/src/cpython/release/python upload-httplib.py 10 https://localhost:8000/
Uploaded 10.00g in 17.53 seconds (584.00m/s)

real	0m17.574s
user	0m8.887s
sys	0m5.971s

Same with 512k blocksize:

$ time ~/src/cpython/release/python upload-httplib.py 10 https://localhost:8000/
Uploaded 10.00g in 6.60 seconds (1551.15m/s)

real	0m6.641s
user	0m3.426s
sys	0m2.162s

In real world usage the difference will be smaller, depending on the
local and remote storage and the network.

See https://github.com/nirs/http-bench for more info.
This commit is contained in:
Nir Soffer 2017-11-06 23:16:37 +02:00 • committed by Victor Stinner
parent 30f4fa456e
commit ad455cd924
5 changed files with 56 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

14
Lib/http/client.py
View file

@ -825,9 +825,10 @@ class HTTPConnection:
return None
def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None):
source_address=None, blocksize=8192):
self.timeout = timeout
self.source_address = source_address
self.blocksize = blocksize
self.sock = None
self._buffer = []
self.__response = None
@ -958,7 +959,6 @@ class HTTPConnection:
if self.debuglevel > 0:
print("send:", repr(data))
blocksize = 8192
if hasattr(data, "read") :
if self.debuglevel > 0:
print("sendIng a read()able")
@ -966,7 +966,7 @@ class HTTPConnection:
if encode and self.debuglevel > 0:
print("encoding file using iso-8859-1")
while 1:
datablock = data.read(blocksize)
datablock = data.read(self.blocksize)
if not datablock:
break
if encode:
@ -991,14 +991,13 @@ class HTTPConnection:
self._buffer.append(s)
def _read_readable(self, readable):
blocksize = 8192
if self.debuglevel > 0:
print("sendIng a read()able")
encode = self._is_textIO(readable)
if encode and self.debuglevel > 0:
print("encoding file using iso-8859-1")
while True:
datablock = readable.read(blocksize)
datablock = readable.read(self.blocksize)
if not datablock:
break
if encode:
@ -1353,9 +1352,10 @@ else:
def __init__(self, host, port=None, key_file=None, cert_file=None,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None, *, context=None,
check_hostname=None):
check_hostname=None, blocksize=8192):
super(HTTPSConnection, self).__init__(host, port, timeout,
source_address)
source_address,
blocksize=blocksize)
if (key_file is not None or cert_file is not None or
check_hostname is not None):
import warnings