Issue #1926: Add support for NNTP over SSL on port 563, as well as

STARTTLS.  Patch by Andrew Vant.

Doc/library/nntplib.rst

@@ -69,6 +69,22 @@ The module itself defines the following classes:
    *readermode* defaults to ``None``. *usenetrc* defaults to ``True``.
 
 
+.. class:: NNTP_SSL(host, port=563, user=None, password=None, ssl_context=None, readermode=None, usenetrc=True, [timeout])
+
+   Return a new :class:`NNTP_SSL` object, representing an encrypted
+   connection to the NNTP server running on host *host*, listening at
+   port *port*.  :class:`NNTP_SSL` objects have the same methods as
+   :class:`NNTP` objects.  If *port* is omitted, port 563 (NNTPS) is used.
+   *ssl_context* is also optional, and is a :class:`~ssl.SSLContext` object.
+   All other parameters behave the same as for :class:`NNTP`.
+
+   Note that SSL-on-563 is discouraged per :rfc:`4642`, in favor of
+   STARTTLS as described below.  However, some servers only support the
+   former.
+
+   .. versionadded:: 3.2
+
+
 .. exception:: NNTPError
 
    Derived from the standard exception :exc:`Exception`, this is the base
@@ -111,8 +127,8 @@ The module itself defines the following classes:
 NNTP Objects
 ------------
 
-When connected, :class:`NNTP` objects support the following methods and
+When connected, :class:`NNTP` and :class:`NNTP_SSL` objects support the
-attributes.
+following methods and attributes.
 
 Attributes
 ^^^^^^^^^^
@@ -179,6 +195,35 @@ tuples or objects that the method normally returns will be empty.
    .. versionadded:: 3.2
 
 
+.. method:: NNTP.login(user=None, password=None, usenetrc=True)
+
+   Send ``AUTHINFO`` commands with the user name and password.  If *user*
+   and *password* are None and *usenetrc* is True, credentials from
+   ``~/.netrc`` will be used if possible.
+
+   Unless intentionally delayed, login is normally performed during the
+   :class:`NNTP` object initialization and separately calling this function
+   is unnecessary.  To force authentication to be delayed, you must not set
+   *user* or *password* when creating the object, and must set *usenetrc* to
+   False.
+
+   .. versionadded:: 3.2
+
+
+.. method:: NNTP.starttls(ssl_context=None)
+
+   Send a ``STARTTLS`` command.  The *ssl_context* argument is optional
+   and should be a :class:`ssl.SSLContext` object.  This will enable
+   encryption on the NNTP connection.
+
+   Note that this may not be done after authentication information has
+   been transmitted, and authentication occurs by default if possible during a
+   :class:`NNTP` object initialization.  See :meth:`NNTP.login` for information
+   on suppressing this behavior.
+
+   .. versionadded:: 3.2
+
+
 .. method:: NNTP.newgroups(date, *, file=None)
 
    Send a ``NEWGROUPS`` command.  The *date* argument should be a

Lib/nntplib.py

@@ -69,6 +69,13 @@ import collections
 import datetime
 import warnings
 
+try:
+    import ssl
+except ImportError:
+    _have_ssl = False
+else:
+    _have_ssl = True
+
 from email.header import decode_header as _email_decode_header
 from socket import _GLOBAL_DEFAULT_TIMEOUT
 
@@ -111,7 +118,7 @@ class NNTPDataError(NNTPError):
 
 # Standard port used by NNTP servers
 NNTP_PORT = 119
-
+NNTP_SSL_PORT = 563
 
 # Response numbers that are followed by additional text (e.g. article)
 _LONGRESP = {
@@ -263,6 +270,23 @@ def _unparse_datetime(dt, legacy=False):
     return date_str, time_str
 
 
+if _have_ssl:
+
+    def _encrypt_on(sock, context):
+        """Wrap a socket in SSL/TLS. Arguments:
+        - sock: Socket to wrap
+        - context: SSL context to use for the encrypted connection
+        Returns:
+        - sock: New, encrypted socket.
+        """
+        # Generate a default SSL context if none was passed.
+        if context is None:
+            context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+            # SSLv2 considered harmful.
+            context.options |= ssl.OP_NO_SSLv2
+        return context.wrap_socket(sock)
+
+
 # The classes themselves
 class _NNTPBase:
     # UTF-8 is the character set for all NNTP commands and responses: they
@@ -280,18 +304,13 @@ class _NNTPBase:
     encoding = 'utf-8'
     errors = 'surrogateescape'
 
-    def __init__(self, file, host, user=None, password=None,
+    def __init__(self, file, host,
-                 readermode=None, usenetrc=True,
+                 readermode=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
-                 timeout=_GLOBAL_DEFAULT_TIMEOUT):
         """Initialize an instance.  Arguments:
         - file: file-like object (open for read/write in binary mode)
         - host: hostname of the server (used if `usenetrc` is True)
-        - user: username to authenticate with
-        - password: password to use with username
         - readermode: if true, send 'mode reader' command after
                       connecting.
-        - usenetrc: allow loading username and password from ~/.netrc file
-                    if not specified explicitly
         - timeout: timeout (in seconds) used for socket connections
 
         readermode is sometimes necessary if you are connecting to an
@@ -300,74 +319,32 @@ class _NNTPBase:
         unexpected NNTPPermanentErrors, you might need to set
         readermode.
         """
+        self.host = host
         self.file = file
         self.debugging = 0
         self.welcome = self._getresp()
 
-        # 'mode reader' is sometimes necessary to enable 'reader' mode.
+        # 'MODE READER' is sometimes necessary to enable 'reader' mode.
-        # However, the order in which 'mode reader' and 'authinfo' need to
+        # However, the order in which 'MODE READER' and 'AUTHINFO' need to
-        # arrive differs between some NNTP servers. Try to send
+        # arrive differs between some NNTP servers. If _setreadermode() fails
-        # 'mode reader', and if it fails with an authorization failed
+        # with an authorization failed error, it will set this to True;
-        # error, try again after sending authinfo.
+        # the login() routine will interpret that as a request to try again
-        readermode_afterauth = 0
+        # after performing its normal function.
+        self.readermode_afterauth = False
         if readermode:
-            try:
+            self._setreadermode()
-                self.welcome = self._shortcmd('mode reader')
-            except NNTPPermanentError:
-                # error 500, probably 'not implemented'
-                pass
-            except NNTPTemporaryError as e:
-                if user and e.response.startswith('480'):
-                    # Need authorization before 'mode reader'
-                    readermode_afterauth = 1
-                else:
-                    raise
-        # If no login/password was specified, try to get them from ~/.netrc
-        # Presume that if .netc has an entry, NNRP authentication is required.
-        try:
-            if usenetrc and not user:
-                import netrc
-                credentials = netrc.netrc()
-                auth = credentials.authenticators(host)
-                if auth:
-                    user = auth[0]
-                    password = auth[2]
-        except IOError:
-            pass
-        # Perform NNTP authentication if needed.
-        if user:
-            resp = self._shortcmd('authinfo user '+user)
-            if resp.startswith('381'):
-                if not password:
-                    raise NNTPReplyError(resp)
-                else:
-                    resp = self._shortcmd(
-                            'authinfo pass '+password)
-                    if not resp.startswith('281'):
-                        raise NNTPPermanentError(resp)
-            if readermode_afterauth:
-                try:
-                    self.welcome = self._shortcmd('mode reader')
-                except NNTPPermanentError:
-                    # error 500, probably 'not implemented'
-                    pass
 
-        # Inquire about capabilities (RFC 3977)
+        # RFC 4642 2.2.2: Both the client and the server MUST know if there is
-        self.nntp_version = 1
+        # a TLS session active.  A client MUST NOT attempt to start a TLS
-        self.nntp_implementation = None
+        # session if a TLS session is already active.
-        try:
+        self.tls_on = False
-            resp, caps = self.capabilities()
+
-        except NNTPPermanentError:
+        # Inquire about capabilities (RFC 3977).
-            # Server doesn't support capabilities
+        self._caps = None
-            self._caps = {}
+        self.getcapabilities()
-        else:
+
-            self._caps = caps
+        # Log in and encryption setup order is left to subclasses.
-            if 'VERSION' in caps:
+        self.authenticated = False
-                # The server can advertise several supported versions,
-                # choose the highest.
-                self.nntp_version = max(map(int, caps['VERSION']))
-            if 'IMPLEMENTATION' in caps:
-                self.nntp_implementation = ' '.join(caps['IMPLEMENTATION'])
 
     def getwelcome(self):
         """Get the welcome message from the server
@@ -382,6 +359,22 @@ class _NNTPBase:
         """Get the server capabilities, as read by __init__().
         If the CAPABILITIES command is not supported, an empty dict is
         returned."""
+        if self._caps is None:
+            self.nntp_version = 1
+            self.nntp_implementation = None
+            try:
+                resp, caps = self.capabilities()
+            except NNTPPermanentError:
+                # Server doesn't support capabilities
+                self._caps = {}
+            else:
+                self._caps = caps
+                if 'VERSION' in caps:
+                    # The server can advertise several supported versions,
+                    # choose the highest.
+                    self.nntp_version = max(map(int, caps['VERSION']))
+                if 'IMPLEMENTATION' in caps:
+                    self.nntp_implementation = ' '.join(caps['IMPLEMENTATION'])
         return self._caps
 
     def set_debuglevel(self, level):
@@ -918,6 +911,77 @@ class _NNTPBase:
             self._close()
         return resp
 
+    def login(self, user=None, password=None, usenetrc=True):
+        if self.authenticated:
+            raise ValueError("Already logged in.")
+        if not user and not usenetrc:
+            raise ValueError(
+                "At least one of `user` and `usenetrc` must be specified")
+        # If no login/password was specified but netrc was requested,
+        # try to get them from ~/.netrc
+        # Presume that if .netrc has an entry, NNRP authentication is required.
+        try:
+            if usenetrc and not user:
+                import netrc
+                credentials = netrc.netrc()
+                auth = credentials.authenticators(self.host)
+                if auth:
+                    user = auth[0]
+                    password = auth[2]
+        except IOError:
+            pass
+        # Perform NNTP authentication if needed.
+        if not user:
+            return
+        resp = self._shortcmd('authinfo user ' + user)
+        if resp.startswith('381'):
+            if not password:
+                raise NNTPReplyError(resp)
+            else:
+                resp = self._shortcmd('authinfo pass ' + password)
+                if not resp.startswith('281'):
+                    raise NNTPPermanentError(resp)
+        # Attempt to send mode reader if it was requested after login.
+        if self.readermode_afterauth:
+            self._setreadermode()
+
+    def _setreadermode(self):
+        try:
+            self.welcome = self._shortcmd('mode reader')
+        except NNTPPermanentError:
+            # Error 5xx, probably 'not implemented'
+            pass
+        except NNTPTemporaryError as e:
+            if e.response.startswith('480'):
+                # Need authorization before 'mode reader'
+                self.readermode_afterauth = True
+            else:
+                raise
+
+    if _have_ssl:
+        def starttls(self, context=None):
+            """Process a STARTTLS command. Arguments:
+            - context: SSL context to use for the encrypted connection
+            """
+            # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if
+            # a TLS session already exists.
+            if self.tls_on:
+                raise ValueError("TLS is already enabled.")
+            if self.authenticated:
+                raise ValueError("TLS cannot be started after authentication.")
+            resp = self._shortcmd('STARTTLS')
+            if resp.startswith('382'):
+                self.file.close()
+                self.sock = _encrypt_on(self.sock, context)
+                self.file = self.sock.makefile("rwb")
+                self.tls_on = True
+                # Capabilities may change after TLS starts up, so ask for them
+                # again.
+                self._caps = None
+                self.getcapabilities()
+            else:
+                raise NNTPError("TLS failed to start.")
+
 
 class NNTP(_NNTPBase):
 
@@ -945,8 +1009,10 @@ class NNTP(_NNTPBase):
         self.port = port
         self.sock = socket.create_connection((host, port), timeout)
         file = self.sock.makefile("rwb")
-        _NNTPBase.__init__(self, file, host, user, password,
+        _NNTPBase.__init__(self, file, host,
-                           readermode, usenetrc, timeout)
+                           readermode, timeout)
+        if user or usenetrc:
+            self.login(user, password, usenetrc)
 
     def _close(self):
         try:
@@ -955,6 +1021,33 @@ class NNTP(_NNTPBase):
             self.sock.close()
 
 
+if _have_ssl:
+    class NNTP_SSL(_NNTPBase):
+
+        def __init__(self, host, port=NNTP_SSL_PORT,
+                    user=None, password=None, ssl_context=None,
+                    readermode=None, usenetrc=True,
+                    timeout=_GLOBAL_DEFAULT_TIMEOUT):
+            """This works identically to NNTP.__init__, except for the change
+            in default port and the `ssl_context` argument for SSL connections.
+            """
+            self.sock = socket.create_connection((host, port), timeout)
+            self.sock = _encrypt_on(self.sock, ssl_context)
+            file = self.sock.makefile("rwb")
+            _NNTPBase.__init__(self, file, host,
+                               readermode=readermode, timeout=timeout)
+            if user or usenetrc:
+                self.login(user, password, usenetrc)
+
+        def _close(self):
+            try:
+                _NNTPBase._close(self)
+            finally:
+                self.sock.close()
+
+    __all__.append("NNTP_SSL")
+
+
 # Test retrieval when run as a script.
 if __name__ == '__main__':
     import argparse
@@ -966,13 +1059,27 @@ if __name__ == '__main__':
                         help='group to fetch messages from (default: %(default)s)')
     parser.add_argument('-s', '--server', default='news.gmane.org',
                         help='NNTP server hostname (default: %(default)s)')
-    parser.add_argument('-p', '--port', default=NNTP_PORT, type=int,
+    parser.add_argument('-p', '--port', default=-1, type=int,
-                        help='NNTP port number (default: %(default)s)')
+                        help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT))
     parser.add_argument('-n', '--nb-articles', default=10, type=int,
                         help='number of articles to fetch (default: %(default)s)')
+    parser.add_argument('-S', '--ssl', action='store_true', default=False,
+                        help='use NNTP over SSL')
     args = parser.parse_args()
 
-    s = NNTP(host=args.server, port=args.port)
+    port = args.port
+    if not args.ssl:
+        if port == -1:
+            port = NNTP_PORT
+        s = NNTP(host=args.server, port=port)
+    else:
+        if port == -1:
+            port = NNTP_SSL_PORT
+        s = NNTP_SSL(host=args.server, port=port)
+
+    caps = s.getcapabilities()
+    if 'STARTTLS' in caps:
+        s.starttls()
     resp, count, first, last, name = s.group(args.group)
     print('Group', name, 'has', count, 'articles, range', first, 'to', last)
 

Lib/test/test_nntplib.py

@@ -4,8 +4,10 @@ import textwrap
 import unittest
 import contextlib
 from test import support
-from nntplib import NNTP, GroupInfo
+from nntplib import NNTP, GroupInfo, _have_ssl
 import nntplib
+if _have_ssl:
+    import ssl
 
 TIMEOUT = 30
 
@@ -106,7 +108,7 @@ class NetworkedNNTPTestsMixin:
              "references", ":bytes", ":lines"}
             )
         for v in art_dict.values():
-            self.assertIsInstance(v, str)
+            self.assertIsInstance(v, (str, type(None)))
 
     def test_xover(self):
         resp, count, first, last, name = self.server.group(self.GROUP_NAME)
@@ -162,26 +164,19 @@ class NetworkedNNTPTestsMixin:
         self.server.quit()
         self.server = None
 
-
+    def test_login(self):
-class NetworkedNNTPTests(NetworkedNNTPTestsMixin, unittest.TestCase):
+        baduser = "notarealuser"
-    NNTP_HOST = 'news.gmane.org'
+        badpw = "notarealpassword"
-    GROUP_NAME = 'gmane.comp.python.devel'
+        # Check that bogus credentials cause failure
-    GROUP_PAT = 'gmane.comp.python.d*'
+        self.assertRaises(nntplib.NNTPError, self.server.login,
-
+                     user=baduser, password=badpw, usenetrc=False)
-    def setUp(self):
+        # FIXME: We should check that correct credentials succeed, but that
-        support.requires("network")
+        # would require valid details for some server somewhere to be in the
-        with support.transient_internet(self.NNTP_HOST):
+        # test suite, I think. Gmane is anonymous, at least as used for the
-            self.server = NNTP(self.NNTP_HOST, timeout=TIMEOUT, usenetrc=False)
+        # other tests.
-
-    def tearDown(self):
-        if self.server is not None:
-            self.server.quit()
-
-    # Disabled with gmane as it produces too much data
-    test_list = None
 
     def test_capabilities(self):
-        # As of this writing, gmane implements NNTP version 2 and has a
+        # The server under test implements NNTP version 2 and has a
         # couple of well-known capabilities. Just sanity check that we
         # got them.
         def _check_caps(caps):
@@ -194,6 +189,63 @@ class NetworkedNNTPTests(NetworkedNNTPTestsMixin, unittest.TestCase):
         resp, caps = self.server.capabilities()
         _check_caps(caps)
 
+    if _have_ssl:
+        def test_starttls(self):
+            file = self.server.file
+            sock = self.server.sock
+            try:
+                self.server.starttls()
+            except nntplib.NNTPPermanentError:
+                self.skipTest("STARTTLS not supported by server.")
+            else:
+                # Check that the socket and internal pseudo-file really were
+                # changed.
+                self.assertNotEqual(file, self.server.file)
+                self.assertNotEqual(sock, self.server.sock)
+                # Check that the new socket really is an SSL one
+                self.assertIsInstance(self.server.sock, ssl.SSLSocket)
+                # Check that trying starttls when it's already active fails.
+                self.assertRaises(ValueError, self.server.starttls)
+
+
+class NetworkedNNTPTests(NetworkedNNTPTestsMixin, unittest.TestCase):
+    # This server supports STARTTLS (gmane doesn't)
+    NNTP_HOST = 'news.trigofacile.com'
+    GROUP_NAME = 'fr.comp.lang.python'
+    GROUP_PAT = 'fr.comp.lang.*'
+
+    def setUp(self):
+        support.requires("network")
+        with support.transient_internet(self.NNTP_HOST):
+            self.server = NNTP(self.NNTP_HOST, timeout=TIMEOUT, usenetrc=False)
+
+    def tearDown(self):
+        if self.server is not None:
+            self.server.quit()
+
+
+if _have_ssl:
+    class NetworkedNNTP_SSLTests(NetworkedNNTPTestsMixin, unittest.TestCase):
+        NNTP_HOST = 'snews.gmane.org'
+        GROUP_NAME = 'gmane.comp.python.devel'
+        GROUP_PAT = 'gmane.comp.python.d*'
+
+        def setUp(self):
+            support.requires("network")
+            with support.transient_internet(self.NNTP_HOST):
+                self.server = nntplib.NNTP_SSL(self.NNTP_HOST, timeout=TIMEOUT,
+                                               usenetrc=False)
+
+        def tearDown(self):
+            if self.server is not None:
+                self.server.quit()
+
+        # Disabled with gmane as it produces too much data
+        test_list = None
+
+        # Disabled as the connection will already be encrypted.
+        test_starttls = None
+
 
 #
 # Non-networked tests using a local server (or something mocking it).
@@ -261,7 +313,6 @@ class MockedNNTPTestsMixin:
         # Using BufferedRWPair instead of BufferedRandom ensures the file
         # isn't seekable.
         file = io.BufferedRWPair(self.sio, self.sio)
-        kwargs.setdefault('usenetrc', False)
         self.server = nntplib._NNTPBase(file, 'test.server', *args, **kwargs)
         return self.server
 
@@ -1134,9 +1185,10 @@ class MiscTests(unittest.TestCase):
 
 
 def test_main():
-    support.run_unittest(MiscTests, NNTPv1Tests, NNTPv2Tests,
+    tests = [MiscTests, NNTPv1Tests, NNTPv2Tests, NetworkedNNTPTests]
-                         NetworkedNNTPTests
+    if _have_ssl:
-                         )
+        tests.append(NetworkedNNTP_SSLTests)
+    support.run_unittest(*tests)
 
 
 if __name__ == "__main__":

Misc/ACKS

@@ -843,6 +843,7 @@ Hector Urtubia
 Andi Vajda
 Case Van Horsen
 Kyle VanderBeek
+Andrew Vant
 Atul Varma
 Dmitry Vasiliev
 Alexandre Vassalotti

Misc/NEWS

@@ -60,6 +60,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #1926: Add support for NNTP over SSL on port 563, as well as
+  STARTTLS.  Patch by Andrew Vant.
+
 - Issue #10335: Add tokenize.open(), detect the file encoding using
   tokenize.detect_encoding() and open it in read only mode.