gh-99813: Start using SSL_sendfile when available (#99907)

Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
Co-authored-by: Peter Bierma <zintensitydev@gmail.com>

Doc/library/ssl.rst

@@ -1078,8 +1078,9 @@ SSL Sockets
      (but passing a non-zero ``flags`` argument is not allowed)
    - :meth:`~socket.socket.send`, :meth:`~socket.socket.sendall` (with
      the same limitation)
-   - :meth:`~socket.socket.sendfile` (but :mod:`os.sendfile` will be used
-     for plain-text sockets only, else :meth:`~socket.socket.send` will be used)
+   - :meth:`~socket.socket.sendfile` (it may be high-performant only when
+     the kernel TLS is enabled by setting :data:`~ssl.OP_ENABLE_KTLS` or when a
+     socket is plain-text, else :meth:`~socket.socket.send` will be used)
    - :meth:`~socket.socket.shutdown`
 
    However, since the SSL (and TLS) protocol has its own framing atop
@@ -1113,6 +1114,11 @@ SSL Sockets
       functions support reading and writing of data larger than 2 GB. Writing
       zero-length data no longer fails with a protocol violation error.
 
+   .. versionchanged:: next
+      Python now uses ``SSL_sendfile`` internally when possible. The
+      function sends a file more efficiently because it performs TLS encryption
+      in the kernel to avoid additional context switches.
+
 SSL sockets also have the following additional methods and attributes:
 
 .. method:: SSLSocket.read(len=1024, buffer=None)

Lib/socket.py

@@ -56,6 +56,7 @@ import io
 import os
 import sys
 from enum import IntEnum, IntFlag
+from functools import partial
 
 try:
     import errno
@@ -348,10 +349,11 @@ class socket(_socket.socket):
         text.mode = mode
         return text
 
-    if hasattr(os, 'sendfile'):
-
-        def _sendfile_use_sendfile(self, file, offset=0, count=None):
-            # Lazy import to improve module import time
+    def _sendfile_zerocopy(self, zerocopy_func, giveup_exc_type, file,
+                           offset=0, count=None):
+        """
+        Send a file using a zero-copy function.
+        """
         import selectors
 
         self._check_sendfile_params(file, offset, count)
@@ -359,11 +361,11 @@ class socket(_socket.socket):
         try:
             fileno = file.fileno()
         except (AttributeError, io.UnsupportedOperation) as err:
-                raise _GiveupOnSendfile(err)  # not a regular file
+            raise giveup_exc_type(err)  # not a regular file
         try:
             fsize = os.fstat(fileno).st_size
         except OSError as err:
-                raise _GiveupOnSendfile(err)  # not a regular file
+            raise giveup_exc_type(err)  # not a regular file
         if not fsize:
             return 0  # empty file
         # Truncate to 1GiB to avoid OverflowError, see bpo-38319.
@@ -383,7 +385,6 @@ class socket(_socket.socket):
         total_sent = 0
         # localize variable access to minimize overhead
         selector_select = selector.select
-            os_sendfile = os.sendfile
         try:
             while True:
                 if timeout and not selector_select(timeout):
@@ -393,7 +394,7 @@ class socket(_socket.socket):
                     if blocksize <= 0:
                         break
                 try:
-                        sent = os_sendfile(sockno, fileno, offset, blocksize)
+                    sent = zerocopy_func(fileno, offset, blocksize)
                 except BlockingIOError:
                     if not timeout:
                         # Block until the socket is ready to send some
@@ -406,7 +407,7 @@ class socket(_socket.socket):
                         # one being 'file' is not a regular mmap(2)-like
                         # file, in which case we'll fall back on using
                         # plain send().
-                            raise _GiveupOnSendfile(err)
+                        raise giveup_exc_type(err)
                     raise err from None
                 else:
                     if sent == 0:
@@ -417,6 +418,14 @@ class socket(_socket.socket):
         finally:
             if total_sent > 0 and hasattr(file, 'seek'):
                 file.seek(offset)
+
+    if hasattr(os, 'sendfile'):
+        def _sendfile_use_sendfile(self, file, offset=0, count=None):
+            return self._sendfile_zerocopy(
+                partial(os.sendfile, self.fileno()),
+                _GiveupOnSendfile,
+                file, offset, count,
+            )
     else:
         def _sendfile_use_sendfile(self, file, offset=0, count=None):
             raise _GiveupOnSendfile(

Lib/ssl.py

@@ -975,6 +975,10 @@ def _sslcopydoc(func):
     return func
 
 
+class _GiveupOnSSLSendfile(Exception):
+    pass
+
+
 class SSLSocket(socket):
     """This class implements a subtype of socket.socket that wraps
     the underlying OS socket in an SSL context when necessary, and
@@ -1266,15 +1270,26 @@ class SSLSocket(socket):
             return super().sendall(data, flags)
 
     def sendfile(self, file, offset=0, count=None):
-        """Send a file, possibly by using os.sendfile() if this is a
-        clear-text socket.  Return the total number of bytes sent.
+        """Send a file, possibly by using an efficient sendfile() call if
+        the system supports it.  Return the total number of bytes sent.
         """
-        if self._sslobj is not None:
-            return self._sendfile_use_send(file, offset, count)
-        else:
-            # os.sendfile() works with plain sockets only
+        if self._sslobj is None:
             return super().sendfile(file, offset, count)
 
+        if not self._sslobj.uses_ktls_for_send():
+            return self._sendfile_use_send(file, offset, count)
+
+        sendfile = getattr(self._sslobj, "sendfile", None)
+        if sendfile is None:
+            return self._sendfile_use_send(file, offset, count)
+
+        try:
+            return self._sendfile_zerocopy(
+                sendfile, _GiveupOnSSLSendfile, file, offset, count,
+            )
+        except _GiveupOnSSLSendfile:
+            return self._sendfile_use_send(file, offset, count)
+
     def recv(self, buflen=1024, flags=0):
         self._checkClosed()
         if self._sslobj is not None:

Lib/test/test_ssl.py

@@ -4316,19 +4316,30 @@ class ThreadedTests(unittest.TestCase):
             self.assertRaises(ValueError, s.write, b'hello')
 
     def test_sendfile(self):
+        """Try to send a file using kTLS if possible."""
         TEST_DATA = b"x" * 512
         with open(os_helper.TESTFN, 'wb') as f:
             f.write(TEST_DATA)
         self.addCleanup(os_helper.unlink, os_helper.TESTFN)
         client_context, server_context, hostname = testing_context()
+        client_context.options |= getattr(ssl, 'OP_ENABLE_KTLS', 0)
         server = ThreadedEchoServer(context=server_context, chatty=False)
-        with server:
-            with client_context.wrap_socket(socket.socket(),
-                                            server_hostname=hostname) as s:
-                s.connect((HOST, server.port))
+        # kTLS seems to work only with a connection created before
+        # wrapping `sock` by the SSL context in contrast to calling
+        # `sock.connect()` after the wrapping.
+        with server, socket.create_connection((HOST, server.port)) as sock:
+            with client_context.wrap_socket(
+                sock, server_hostname=hostname
+            ) as ssock:
+                if support.verbose:
+                    ktls_used = ssock._sslobj.uses_ktls_for_send()
+                    print(
+                        'kTLS is',
+                        'available' if ktls_used else 'unavailable',
+                    )
                 with open(os_helper.TESTFN, 'rb') as file:
-                    s.sendfile(file)
-                    self.assertEqual(s.recv(1024), TEST_DATA)
+                    ssock.sendfile(file)
+                self.assertEqual(ssock.recv(1024), TEST_DATA)
 
     def test_session(self):
         client_context, server_context, hostname = testing_context()

Misc/NEWS.d/next/Library/2023-03-13-22-51-40.gh-issue-99813.40TV02.rst

@@ -0,0 +1,4 @@
+:mod:`ssl` now uses ``SSL_sendfile`` internally when it is possible (see
+:data:`~ssl.OP_ENABLE_KTLS`). The function sends a file more efficiently
+because it performs TLS encryption in the kernel to avoid additional context
+switches. Patch by Illia Volochii.

Modules/_ssl.c

@@ -75,6 +75,33 @@
 #endif
 
 
+#ifdef BIO_get_ktls_send
+#  ifdef MS_WINDOWS
+typedef long long Py_off_t;
+#  else
+typedef off_t Py_off_t;
+#  endif
+
+static int
+Py_off_t_converter(PyObject *arg, void *addr)
+{
+#ifdef HAVE_LARGEFILE_SUPPORT
+    *((Py_off_t *)addr) = PyLong_AsLongLong(arg);
+#else
+    *((Py_off_t *)addr) = PyLong_AsLong(arg);
+#endif
+    return PyErr_Occurred() ? 0 : 1;
+}
+
+/*[python input]
+
+class Py_off_t_converter(CConverter):
+    type = 'Py_off_t'
+    converter = 'Py_off_t_converter'
+
+[python start generated code]*/
+/*[python end generated code: output=da39a3ee5e6b4b0d input=3fd9ca8ca6f0cbb8]*/
+#endif /* BIO_get_ktls_send */
 
 struct py_ssl_error_code {
     const char *mnemonic;
@@ -2442,6 +2469,184 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout)
     return rc == 0 ? SOCKET_HAS_TIMED_OUT : SOCKET_OPERATION_OK;
 }
 
+/*[clinic input]
+@critical_section
+_ssl._SSLSocket.uses_ktls_for_send
+
+Check if the Kernel TLS data-path is used for sending.
+[clinic start generated code]*/
+
+static PyObject *
+_ssl__SSLSocket_uses_ktls_for_send_impl(PySSLSocket *self)
+/*[clinic end generated code: output=f9d95fbefceb5068 input=8d1ce4a131190e6b]*/
+{
+#ifdef BIO_get_ktls_send
+    int uses = BIO_get_ktls_send(SSL_get_wbio(self->ssl));
+    // BIO_get_ktls_send() returns 1 if kTLS is used and 0 if not.
+    // Also, it returns -1 for failure before OpenSSL 3.0.4.
+    return Py_NewRef(uses == 1 ? Py_True : Py_False);
+#else
+    Py_RETURN_FALSE;
+#endif
+}
+
+/*[clinic input]
+@critical_section
+_ssl._SSLSocket.uses_ktls_for_recv
+
+Check if the Kernel TLS data-path is used for receiving.
+[clinic start generated code]*/
+
+static PyObject *
+_ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self)
+/*[clinic end generated code: output=ce38b00317a1f681 input=a13778a924fc7d44]*/
+{
+#ifdef BIO_get_ktls_recv
+    int uses = BIO_get_ktls_recv(SSL_get_rbio(self->ssl));
+    // BIO_get_ktls_recv() returns 1 if kTLS is used and 0 if not.
+    // Also, it returns -1 for failure before OpenSSL 3.0.4.
+    return Py_NewRef(uses == 1 ? Py_True : Py_False);
+#else
+    Py_RETURN_FALSE;
+#endif
+}
+
+#ifdef BIO_get_ktls_send
+/*[clinic input]
+@critical_section
+_ssl._SSLSocket.sendfile
+    fd: int
+    offset: Py_off_t
+    size: size_t
+    flags: int = 0
+    /
+
+Write size bytes from offset in the file descriptor fd to the SSL connection.
+
+This method uses the zero-copy technique and returns the number of bytes
+written. It should be called only when Kernel TLS is used for sending data in
+the connection.
+
+The meaning of flags is platform dependent.
+[clinic start generated code]*/
+
+static PyObject *
+_ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset,
+                              size_t size, int flags)
+/*[clinic end generated code: output=0c6815b0719ca8d5 input=dfc1b162bb020de1]*/
+{
+    Py_ssize_t retval;
+    int sockstate;
+    _PySSLError err;
+    PySocketSockObject *sock = GET_SOCKET(self);
+    PyTime_t timeout, deadline = 0;
+    int has_timeout;
+
+    if (sock != NULL) {
+        if ((PyObject *)sock == Py_None) {
+            _setSSLError(get_state_sock(self),
+                         "Underlying socket connection gone",
+                         PY_SSL_ERROR_NO_SOCKET, __FILE__, __LINE__);
+            return NULL;
+        }
+        Py_INCREF(sock);
+        /* just in case the blocking state of the socket has been changed */
+        int nonblocking = (sock->sock_timeout >= 0);
+        BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking);
+        BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking);
+    }
+
+    timeout = GET_SOCKET_TIMEOUT(sock);
+    has_timeout = (timeout > 0);
+    if (has_timeout) {
+        deadline = _PyDeadline_Init(timeout);
+    }
+
+    sockstate = PySSL_select(sock, 1, timeout);
+    switch (sockstate) {
+        case SOCKET_HAS_TIMED_OUT:
+            PyErr_SetString(PyExc_TimeoutError,
+                            "The write operation timed out");
+            goto error;
+        case SOCKET_HAS_BEEN_CLOSED:
+            PyErr_SetString(get_state_sock(self)->PySSLErrorObject,
+                            "Underlying socket has been closed.");
+            goto error;
+        case SOCKET_TOO_LARGE_FOR_SELECT:
+            PyErr_SetString(get_state_sock(self)->PySSLErrorObject,
+                            "Underlying socket too large for select().");
+            goto error;
+    }
+
+    do {
+        PySSL_BEGIN_ALLOW_THREADS
+        retval = SSL_sendfile(self->ssl, fd, (off_t)offset, size, flags);
+        err = _PySSL_errno(retval < 0, self->ssl, (int)retval);
+        PySSL_END_ALLOW_THREADS
+        self->err = err;
+
+        if (PyErr_CheckSignals()) {
+            goto error;
+        }
+
+        if (has_timeout) {
+            timeout = _PyDeadline_Get(deadline);
+        }
+
+        switch (err.ssl) {
+            case SSL_ERROR_WANT_READ:
+                sockstate = PySSL_select(sock, 0, timeout);
+                break;
+            case SSL_ERROR_WANT_WRITE:
+                sockstate = PySSL_select(sock, 1, timeout);
+                break;
+            default:
+                sockstate = SOCKET_OPERATION_OK;
+                break;
+        }
+
+        if (sockstate == SOCKET_HAS_TIMED_OUT) {
+            PyErr_SetString(PyExc_TimeoutError,
+                            "The sendfile operation timed out");
+            goto error;
+        }
+        else if (sockstate == SOCKET_HAS_BEEN_CLOSED) {
+            PyErr_SetString(get_state_sock(self)->PySSLErrorObject,
+                            "Underlying socket has been closed.");
+            goto error;
+        }
+        else if (sockstate == SOCKET_IS_NONBLOCKING) {
+            break;
+        }
+    } while (err.ssl == SSL_ERROR_WANT_READ
+             || err.ssl == SSL_ERROR_WANT_WRITE);
+
+    if (err.ssl == SSL_ERROR_SSL
+        && ERR_GET_REASON(ERR_peek_error()) == SSL_R_UNINITIALIZED)
+    {
+        /* OpenSSL fails to return SSL_ERROR_SYSCALL if an error
+         * happens in sendfile(), and returns SSL_ERROR_SSL with
+         * SSL_R_UNINITIALIZED reason instead. */
+        _setSSLError(get_state_sock(self),
+                     "Some I/O error occurred in sendfile()",
+                     PY_SSL_ERROR_SYSCALL, __FILE__, __LINE__);
+        goto error;
+    }
+    Py_XDECREF(sock);
+    if (retval < 0) {
+        return PySSL_SetError(self, __FILE__, __LINE__);
+    }
+    if (PySSL_ChainExceptions(self) < 0) {
+        return NULL;
+    }
+    return PyLong_FromSize_t(retval);
+error:
+    Py_XDECREF(sock);
+    (void)PySSL_ChainExceptions(self);
+    return NULL;
+}
+#endif /* BIO_get_ktls_send */
+
 /*[clinic input]
 @critical_section
 _ssl._SSLSocket.write
@@ -3017,6 +3222,9 @@ static PyGetSetDef ssl_getsetlist[] = {
 
 static PyMethodDef PySSLMethods[] = {
     _SSL__SSLSOCKET_DO_HANDSHAKE_METHODDEF
+    _SSL__SSLSOCKET_USES_KTLS_FOR_SEND_METHODDEF
+    _SSL__SSLSOCKET_USES_KTLS_FOR_RECV_METHODDEF
+    _SSL__SSLSOCKET_SENDFILE_METHODDEF
     _SSL__SSLSOCKET_WRITE_METHODDEF
     _SSL__SSLSOCKET_READ_METHODDEF
     _SSL__SSLSOCKET_PENDING_METHODDEF

Modules/clinic/_ssl.c.h

@@ -7,6 +7,7 @@ preserve
 #  include "pycore_runtime.h"     // _Py_ID()
 #endif
 #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION()
+#include "pycore_long.h"          // _PyLong_Size_t_Converter()
 #include "pycore_modsupport.h"    // _PyArg_CheckPositional()
 
 PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__,
@@ -442,6 +443,115 @@ _ssl__SSLSocket_owner_set(PyObject *self, PyObject *value, void *Py_UNUSED(conte
     return return_value;
 }
 
+PyDoc_STRVAR(_ssl__SSLSocket_uses_ktls_for_send__doc__,
+"uses_ktls_for_send($self, /)\n"
+"--\n"
+"\n"
+"Check if the Kernel TLS data-path is used for sending.");
+
+#define _SSL__SSLSOCKET_USES_KTLS_FOR_SEND_METHODDEF    \
+    {"uses_ktls_for_send", (PyCFunction)_ssl__SSLSocket_uses_ktls_for_send, METH_NOARGS, _ssl__SSLSocket_uses_ktls_for_send__doc__},
+
+static PyObject *
+_ssl__SSLSocket_uses_ktls_for_send_impl(PySSLSocket *self);
+
+static PyObject *
+_ssl__SSLSocket_uses_ktls_for_send(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    PyObject *return_value = NULL;
+
+    Py_BEGIN_CRITICAL_SECTION(self);
+    return_value = _ssl__SSLSocket_uses_ktls_for_send_impl((PySSLSocket *)self);
+    Py_END_CRITICAL_SECTION();
+
+    return return_value;
+}
+
+PyDoc_STRVAR(_ssl__SSLSocket_uses_ktls_for_recv__doc__,
+"uses_ktls_for_recv($self, /)\n"
+"--\n"
+"\n"
+"Check if the Kernel TLS data-path is used for receiving.");
+
+#define _SSL__SSLSOCKET_USES_KTLS_FOR_RECV_METHODDEF    \
+    {"uses_ktls_for_recv", (PyCFunction)_ssl__SSLSocket_uses_ktls_for_recv, METH_NOARGS, _ssl__SSLSocket_uses_ktls_for_recv__doc__},
+
+static PyObject *
+_ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self);
+
+static PyObject *
+_ssl__SSLSocket_uses_ktls_for_recv(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    PyObject *return_value = NULL;
+
+    Py_BEGIN_CRITICAL_SECTION(self);
+    return_value = _ssl__SSLSocket_uses_ktls_for_recv_impl((PySSLSocket *)self);
+    Py_END_CRITICAL_SECTION();
+
+    return return_value;
+}
+
+#if defined(BIO_get_ktls_send)
+
+PyDoc_STRVAR(_ssl__SSLSocket_sendfile__doc__,
+"sendfile($self, fd, offset, size, flags=0, /)\n"
+"--\n"
+"\n"
+"Write size bytes from offset in the file descriptor fd to the SSL connection.\n"
+"\n"
+"This method uses the zero-copy technique and returns the number of bytes\n"
+"written. It should be called only when Kernel TLS is used for sending data in\n"
+"the connection.\n"
+"\n"
+"The meaning of flags is platform dependent.");
+
+#define _SSL__SSLSOCKET_SENDFILE_METHODDEF    \
+    {"sendfile", _PyCFunction_CAST(_ssl__SSLSocket_sendfile), METH_FASTCALL, _ssl__SSLSocket_sendfile__doc__},
+
+static PyObject *
+_ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset,
+                              size_t size, int flags);
+
+static PyObject *
+_ssl__SSLSocket_sendfile(PyObject *self, PyObject *const *args, Py_ssize_t nargs)
+{
+    PyObject *return_value = NULL;
+    int fd;
+    Py_off_t offset;
+    size_t size;
+    int flags = 0;
+
+    if (!_PyArg_CheckPositional("sendfile", nargs, 3, 4)) {
+        goto exit;
+    }
+    fd = PyLong_AsInt(args[0]);
+    if (fd == -1 && PyErr_Occurred()) {
+        goto exit;
+    }
+    if (!Py_off_t_converter(args[1], &offset)) {
+        goto exit;
+    }
+    if (!_PyLong_Size_t_Converter(args[2], &size)) {
+        goto exit;
+    }
+    if (nargs < 4) {
+        goto skip_optional;
+    }
+    flags = PyLong_AsInt(args[3]);
+    if (flags == -1 && PyErr_Occurred()) {
+        goto exit;
+    }
+skip_optional:
+    Py_BEGIN_CRITICAL_SECTION(self);
+    return_value = _ssl__SSLSocket_sendfile_impl((PySSLSocket *)self, fd, offset, size, flags);
+    Py_END_CRITICAL_SECTION();
+
+exit:
+    return return_value;
+}
+
+#endif /* defined(BIO_get_ktls_send) */
+
 PyDoc_STRVAR(_ssl__SSLSocket_write__doc__,
 "write($self, b, /)\n"
 "--\n"
@@ -2893,6 +3003,10 @@ exit:
 
 #endif /* defined(_MSC_VER) */
 
+#ifndef _SSL__SSLSOCKET_SENDFILE_METHODDEF
+    #define _SSL__SSLSOCKET_SENDFILE_METHODDEF
+#endif /* !defined(_SSL__SSLSOCKET_SENDFILE_METHODDEF) */
+
 #ifndef _SSL_ENUM_CERTIFICATES_METHODDEF
     #define _SSL_ENUM_CERTIFICATES_METHODDEF
 #endif /* !defined(_SSL_ENUM_CERTIFICATES_METHODDEF) */
@@ -2900,4 +3014,4 @@ exit:
 #ifndef _SSL_ENUM_CRLS_METHODDEF
     #define _SSL_ENUM_CRLS_METHODDEF
 #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */
-/*[clinic end generated code: output=748650909fec8906 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=1adc3780d8ca682a input=a9049054013a1b77]*/