Patch #624325: urlparse.urlparse() and urlparse.urlsplit() results

now sport attributes that provide access to the parts of the result.

Doc/lib/liburlparse.tex

@@ -25,48 +25,74 @@ draft!). It supports the following URL schemes:
 \code{nntp}, \code{prospero}, \code{rsync}, \code{rtsp}, \code{rtspu}, 
 \code{sftp}, \code{shttp}, \code{sip}, \code{sips}, \code{snews}, \code{svn}, 
 \code{svn+ssh}, \code{telnet}, \code{wais}.
+
 \versionadded[Support for the \code{sftp} and \code{sips} schemes]{2.5}
 
 The \module{urlparse} module defines the following functions:
 
-\begin{funcdesc}{urlparse}{urlstring\optional{, default_scheme\optional{, allow_fragments}}}
+\begin{funcdesc}{urlparse}{urlstring\optional{,
-Parse a URL into 6 components, returning a 6-tuple: (addressing
+                           default_scheme\optional{, allow_fragments}}}
-scheme, network location, path, parameters, query, fragment
+Parse a URL into six components, returning a 6-tuple.  This
-identifier).  This corresponds to the general structure of a URL:
+corresponds to the general structure of a URL:
 \code{\var{scheme}://\var{netloc}/\var{path};\var{parameters}?\var{query}\#\var{fragment}}.
 Each tuple item is a string, possibly empty.
-The components are not broken up in smaller parts (e.g. the network
+The components are not broken up in smaller parts (for example, the network
 location is a single string), and \% escapes are not expanded.
-The delimiters as shown above are not part of the tuple items,
+The delimiters as shown above are not part of the result,
 except for a leading slash in the \var{path} component, which is
-retained if present.
+retained if present.  For example:
-
-Example:
-
-\begin{verbatim}
-urlparse('http://www.cwi.nl:80/%7Eguido/Python.html')
-\end{verbatim}
-
-yields the tuple
 
 \begin{verbatim}
+>>> from urlparse import urlparse
+>>> o = urlparse('http://www.cwi.nl:80/%7Eguido/Python.html')
+>>> o
 ('http', 'www.cwi.nl:80', '/%7Eguido/Python.html', '', '', '')
+>>> o.scheme
+'http'
+>>> o.port
+80
+>>> o.geturl()
+'http://www.cwi.nl:80/%7Eguido/Python.html'
 \end{verbatim}
 
 If the \var{default_scheme} argument is specified, it gives the
-default addressing scheme, to be used only if the URL string does not
+default addressing scheme, to be used only if the URL does not
 specify one.  The default value for this argument is the empty string.
 
-If the \var{allow_fragments} argument is zero, fragment identifiers
+If the \var{allow_fragments} argument is false, fragment identifiers
 are not allowed, even if the URL's addressing scheme normally does
-support them.  The default value for this argument is \code{1}.
+support them.  The default value for this argument is \constant{True}.
+
+The return value is actually an instance of a subclass of
+\pytype{tuple}.  This class has the following additional read-only
+convenience attributes:
+
+\begin{tableiv}{l|c|l|c}{member}{Attribute}{Index}{Value}{Value if not present}
+  \lineiv{scheme}  {0} {URL scheme specifier}             {empty string}
+  \lineiv{netloc}  {1} {Network location part}            {empty string}
+  \lineiv{path}    {2} {Hierarchical path}                {empty string}
+  \lineiv{params}  {3} {Parameters for last path element} {empty string}
+  \lineiv{query}   {4} {Query component}                  {empty string}
+  \lineiv{fragment}{5} {Fragment identifier}              {empty string}
+  \lineiv{username}{ } {User name}                        {\constant{None}}
+  \lineiv{password}{ } {Password}                         {\constant{None}}
+  \lineiv{hostname}{ } {Host name (lower case)}           {\constant{None}}
+  \lineiv{port}    { } {Port number as integer, if present} {\constant{None}}
+\end{tableiv}
+
+See section~\ref{urlparse-result-object}, ``Results of
+\function{urlparse()} and \function{urlsplit()},'' for more
+information on the result object.
+
+\versionchanged[Added attributes to return value]{2.5}
 \end{funcdesc}
 
-\begin{funcdesc}{urlunparse}{tuple}
+\begin{funcdesc}{urlunparse}{parts}
-Construct a URL string from a tuple as returned by \code{urlparse()}.
+Construct a URL from a tuple as returned by \code{urlparse()}.
+The \var{parts} argument be any six-item iterable.
 This may result in a slightly different, but equivalent URL, if the
-URL that was parsed originally had redundant delimiters, e.g. a ? with
+URL that was parsed originally had unnecessary delimiters (for example,
-an empty query (the draft states that these are equivalent).
+a ? with an empty query; the RFC states that these are equivalent).
 \end{funcdesc}
 
 \begin{funcdesc}{urlsplit}{urlstring\optional{,
@@ -79,12 +105,38 @@ the URL (see \rfc{2396}) is wanted.  A separate function is needed to
 separate the path segments and parameters.  This function returns a
 5-tuple: (addressing scheme, network location, path, query, fragment
 identifier).
+
+The return value is actually an instance of a subclass of
+\pytype{tuple}.  This class has the following additional read-only
+convenience attributes:
+
+\begin{tableiv}{l|c|l|c}{member}{Attribute}{Index}{Value}{Value if not present}
+  \lineiv{scheme}   {0} {URL scheme specifier}   {empty string}
+  \lineiv{netloc}   {1} {Network location part}  {empty string}
+  \lineiv{path}     {2} {Hierarchical path}      {empty string}
+  \lineiv{query}    {3} {Query component}        {empty string}
+  \lineiv{fragment} {4} {Fragment identifier}    {empty string}
+  \lineiv{username} { } {User name}              {\constant{None}}
+  \lineiv{password} { } {Password}               {\constant{None}}
+  \lineiv{hostname} { } {Host name (lower case)} {\constant{None}}
+  \lineiv{port}     { } {Port number as integer, if present} {\constant{None}}
+\end{tableiv}
+
+See section~\ref{urlparse-result-object}, ``Results of
+\function{urlparse()} and \function{urlsplit()},'' for more
+information on the result object.
+
 \versionadded{2.2}
+\versionchanged[Added attributes to return value]{2.5}
 \end{funcdesc}
 
-\begin{funcdesc}{urlunsplit}{tuple}
+\begin{funcdesc}{urlunsplit}{parts}
 Combine the elements of a tuple as returned by \function{urlsplit()}
 into a complete URL as a string.
+The \var{parts} argument be any five-item iterable.
+This may result in a slightly different, but equivalent URL, if the
+URL that was parsed originally had unnecessary delimiters (for example,
+a ? with an empty query; the RFC states that these are equivalent).
 \versionadded{2.2}
 \end{funcdesc}
 
@@ -93,22 +145,16 @@ Construct a full (``absolute'') URL by combining a ``base URL''
 (\var{base}) with a ``relative URL'' (\var{url}).  Informally, this
 uses components of the base URL, in particular the addressing scheme,
 the network location and (part of) the path, to provide missing
-components in the relative URL.
+components in the relative URL.  For example:
-
-Example:
-
-\begin{verbatim}
-urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html')
-\end{verbatim}
-
-yields the string
 
 \begin{verbatim}
+>>> from urlparse import urljoin
+>>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html')
 'http://www.cwi.nl/%7Eguido/FAQ.html'
 \end{verbatim}
 
-The \var{allow_fragments} argument has the same meaning as for
+The \var{allow_fragments} argument has the same meaning and default as
-\code{urlparse()}.
+for \function{urlparse()}.
 \end{funcdesc}
 
 \begin{funcdesc}{urldefrag}{url}
@@ -133,3 +179,61 @@ in \var{url}, returns \var{url} unmodified and an empty string.
         both Uniform Resource Names (URNs) and Uniform Resource
         Locators (URLs).}
 \end{seealso}
+
+
+\subsection{Results of \function{urlparse()} and \function{urlsplit()}
+            \label{urlparse-result-object}}
+
+The result objects from the \function{urlparse()} and
+\function{urlsplit()} functions are subclasses of the \pytype{tuple}
+type.  These subclasses add the attributes described in those
+functions, as well as provide an additional method:
+
+\begin{methoddesc}[ParseResult]{geturl}{}
+  Return the re-combined version of the original URL as a string.
+  This may differ from the original URL in that the scheme will always
+  be normalized to lower case and empty components may be dropped.
+  Specifically, empty parameters, queries, and fragment identifiers
+  will be removed.
+
+  The result of this method is a fixpoint if passed back through the
+  original parsing function:
+
+\begin{verbatim}
+>>> import urlparse
+>>> url = 'HTTP://www.Python.org/doc/#'
+
+>>> r1 = urlparse.urlsplit(url)
+>>> r1.geturl()
+'http://www.Python.org/doc/'
+
+>>> r2 = urlparse.urlsplit(r1.geturl())
+>>> r2.geturl()
+'http://www.Python.org/doc/'
+\end{verbatim}
+
+\versionadded{2.5}
+\end{methoddesc}
+
+The following classes provide the implementations of the parse results::
+
+\begin{classdesc*}{BaseResult}
+  Base class for the concrete result classes.  This provides most of
+  the attribute definitions.  It does not provide a \method{geturl()}
+  method.  It is derived from \class{tuple}, but does not override the
+  \method{__init__()} or \method{__new__()} methods.
+\end{classdesc*}
+
+
+\begin{classdesc}{ParseResult}{scheme, netloc, path, params, query, fragment}
+  Concrete class for \function{urlparse()} results.  The
+  \method{__new__()} method is overridden to support checking that the
+  right number of arguments are passed.
+\end{classdesc}
+
+
+\begin{classdesc}{SplitResult}{scheme, netloc, path, query, fragment}
+  Concrete class for \function{urlsplit()} results.  The
+  \method{__new__()} method is overridden to support checking that the
+  right number of arguments are passed.
+\end{classdesc}

Lib/test/test_urlparse.py

@@ -12,15 +12,53 @@ class UrlParseTestCase(unittest.TestCase):
     def checkRoundtrips(self, url, parsed, split):
         result = urlparse.urlparse(url)
         self.assertEqual(result, parsed)
+        t = (result.scheme, result.netloc, result.path,
+             result.params, result.query, result.fragment)
+        self.assertEqual(t, parsed)
         # put it back together and it should be the same
         result2 = urlparse.urlunparse(result)
         self.assertEqual(result2, url)
+        self.assertEqual(result2, result.geturl())
+
+        # the result of geturl() is a fixpoint; we can always parse it
+        # again to get the same result:
+        result3 = urlparse.urlparse(result.geturl())
+        self.assertEqual(result3.geturl(), result.geturl())
+        self.assertEqual(result3,          result)
+        self.assertEqual(result3.scheme,   result.scheme)
+        self.assertEqual(result3.netloc,   result.netloc)
+        self.assertEqual(result3.path,     result.path)
+        self.assertEqual(result3.params,   result.params)
+        self.assertEqual(result3.query,    result.query)
+        self.assertEqual(result3.fragment, result.fragment)
+        self.assertEqual(result3.username, result.username)
+        self.assertEqual(result3.password, result.password)
+        self.assertEqual(result3.hostname, result.hostname)
+        self.assertEqual(result3.port,     result.port)
 
         # check the roundtrip using urlsplit() as well
         result = urlparse.urlsplit(url)
         self.assertEqual(result, split)
+        t = (result.scheme, result.netloc, result.path,
+             result.query, result.fragment)
+        self.assertEqual(t, split)
         result2 = urlparse.urlunsplit(result)
         self.assertEqual(result2, url)
+        self.assertEqual(result2, result.geturl())
+
+        # check the fixpoint property of re-parsing the result of geturl()
+        result3 = urlparse.urlsplit(result.geturl())
+        self.assertEqual(result3.geturl(), result.geturl())
+        self.assertEqual(result3,          result)
+        self.assertEqual(result3.scheme,   result.scheme)
+        self.assertEqual(result3.netloc,   result.netloc)
+        self.assertEqual(result3.path,     result.path)
+        self.assertEqual(result3.query,    result.query)
+        self.assertEqual(result3.fragment, result.fragment)
+        self.assertEqual(result3.username, result.username)
+        self.assertEqual(result3.password, result.password)
+        self.assertEqual(result3.hostname, result.hostname)
+        self.assertEqual(result3.port,     result.port)
 
     def test_roundtrips(self):
         testcases = [
@@ -187,6 +225,69 @@ class UrlParseTestCase(unittest.TestCase):
             ]:
             self.assertEqual(urlparse.urldefrag(url), (defrag, frag))
 
+    def test_urlsplit_attributes(self):
+        url = "HTTP://WWW.PYTHON.ORG/doc/#frag"
+        p = urlparse.urlsplit(url)
+        self.assertEqual(p.scheme, "http")
+        self.assertEqual(p.netloc, "WWW.PYTHON.ORG")
+        self.assertEqual(p.path, "/doc/")
+        self.assertEqual(p.query, "")
+        self.assertEqual(p.fragment, "frag")
+        self.assertEqual(p.username, None)
+        self.assertEqual(p.password, None)
+        self.assertEqual(p.hostname, "www.python.org")
+        self.assertEqual(p.port, None)
+        # geturl() won't return exactly the original URL in this case
+        # since the scheme is always case-normalized
+        #self.assertEqual(p.geturl(), url)
+
+        url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag"
+        p = urlparse.urlsplit(url)
+        self.assertEqual(p.scheme, "http")
+        self.assertEqual(p.netloc, "User:Pass@www.python.org:080")
+        self.assertEqual(p.path, "/doc/")
+        self.assertEqual(p.query, "query=yes")
+        self.assertEqual(p.fragment, "frag")
+        self.assertEqual(p.username, "User")
+        self.assertEqual(p.password, "Pass")
+        self.assertEqual(p.hostname, "www.python.org")
+        self.assertEqual(p.port, 80)
+        self.assertEqual(p.geturl(), url)
+
+    def test_attributes_bad_port(self):
+        """Check handling of non-integer ports."""
+        p = urlparse.urlsplit("http://www.example.net:foo")
+        self.assertEqual(p.netloc, "www.example.net:foo")
+        self.assertRaises(ValueError, lambda: p.port)
+
+        p = urlparse.urlparse("http://www.example.net:foo")
+        self.assertEqual(p.netloc, "www.example.net:foo")
+        self.assertRaises(ValueError, lambda: p.port)
+
+    def test_attributes_without_netloc(self):
+        # This example is straight from RFC 3261.  It looks like it
+        # should allow the username, hostname, and port to be filled
+        # in, but doesn't.  Since it's a URI and doesn't use the
+        # scheme://netloc syntax, the netloc and related attributes
+        # should be left empty.
+        uri = "sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15"
+        p = urlparse.urlsplit(uri)
+        self.assertEqual(p.netloc, "")
+        self.assertEqual(p.username, None)
+        self.assertEqual(p.password, None)
+        self.assertEqual(p.hostname, None)
+        self.assertEqual(p.port, None)
+        self.assertEqual(p.geturl(), uri)
+
+        p = urlparse.urlparse(uri)
+        self.assertEqual(p.netloc, "")
+        self.assertEqual(p.username, None)
+        self.assertEqual(p.password, None)
+        self.assertEqual(p.hostname, None)
+        self.assertEqual(p.port, None)
+        self.assertEqual(p.geturl(), uri)
+
+
 def test_main():
     test_support.run_unittest(UrlParseTestCase)
 

Lib/urlparse.py

@@ -41,7 +41,111 @@ def clear_cache():
     _parse_cache = {}
 
 
-def urlparse(url, scheme='', allow_fragments=1):
+class BaseResult(tuple):
+    """Base class for the parsed result objects.
+
+    This provides the attributes shared by the two derived result
+    objects as read-only properties.  The derived classes are
+    responsible for checking the right number of arguments were
+    supplied to the constructor.
+
+    """
+
+    __slots__ = ()
+
+    # Attributes that access the basic components of the URL:
+
+    @property
+    def scheme(self):
+        return self[0]
+
+    @property
+    def netloc(self):
+        return self[1]
+
+    @property
+    def path(self):
+        return self[2]
+
+    @property
+    def query(self):
+        return self[-2]
+
+    @property
+    def fragment(self):
+        return self[-1]
+
+    # Additional attributes that provide access to parsed-out portions
+    # of the netloc:
+
+    @property
+    def username(self):
+        netloc = self.netloc
+        if "@" in netloc:
+            userinfo = netloc.split("@", 1)[0]
+            if ":" in userinfo:
+                userinfo = userinfo.split(":", 1)[0]
+            return userinfo
+        return None
+
+    @property
+    def password(self):
+        netloc = self.netloc
+        if "@" in netloc:
+            userinfo = netloc.split("@", 1)[0]
+            if ":" in userinfo:
+                return userinfo.split(":", 1)[1]
+        return None
+
+    @property
+    def hostname(self):
+        netloc = self.netloc
+        if "@" in netloc:
+            netloc = netloc.split("@", 1)[1]
+        if ":" in netloc:
+            netloc = netloc.split(":", 1)[0]
+        return netloc.lower() or None
+
+    @property
+    def port(self):
+        netloc = self.netloc
+        if "@" in netloc:
+            netloc = netloc.split("@", 1)[1]
+        if ":" in netloc:
+            port = netloc.split(":", 1)[1]
+            return int(port, 10)
+        return None
+
+
+class SplitResult(BaseResult):
+
+    __slots__ = ()
+
+    def __new__(cls, scheme, netloc, path, query, fragment):
+        return BaseResult.__new__(
+            cls, (scheme, netloc, path, query, fragment))
+
+    def geturl(self):
+        return urlunsplit(self)
+
+
+class ParseResult(BaseResult):
+
+    __slots__ = ()
+
+    def __new__(cls, scheme, netloc, path, params, query, fragment):
+        return BaseResult.__new__(
+            cls, (scheme, netloc, path, params, query, fragment))
+
+    @property
+    def params(self):
+        return self[3]
+
+    def geturl(self):
+        return urlunparse(self)
+
+
+def urlparse(url, scheme='', allow_fragments=True):
     """Parse a URL into 6 components:
     <scheme>://<netloc>/<path>;<params>?<query>#<fragment>
     Return a 6-tuple: (scheme, netloc, path, params, query, fragment).
@@ -53,7 +157,7 @@ def urlparse(url, scheme='', allow_fragments=1):
         url, params = _splitparams(url)
     else:
         params = ''
-    return scheme, netloc, url, params, query, fragment
+    return ParseResult(scheme, netloc, url, params, query, fragment)
 
 def _splitparams(url):
     if '/'  in url:
@@ -73,12 +177,13 @@ def _splitnetloc(url, start=0):
         delim = len(url)
     return url[start:delim], url[delim:]
 
-def urlsplit(url, scheme='', allow_fragments=1):
+def urlsplit(url, scheme='', allow_fragments=True):
     """Parse a URL into 5 components:
     <scheme>://<netloc>/<path>?<query>#<fragment>
     Return a 5-tuple: (scheme, netloc, path, query, fragment).
     Note that we don't break the components up in smaller bits
     (e.g. netloc is a single string) and we don't expand % escapes."""
+    allow_fragments = bool(allow_fragments)
     key = url, scheme, allow_fragments
     cached = _parse_cache.get(key, None)
     if cached:
@@ -97,9 +202,9 @@ def urlsplit(url, scheme='', allow_fragments=1):
                 url, fragment = url.split('#', 1)
             if '?' in url:
                 url, query = url.split('?', 1)
-            tuple = scheme, netloc, url, query, fragment
+            v = SplitResult(scheme, netloc, url, query, fragment)
-            _parse_cache[key] = tuple
+            _parse_cache[key] = v
-            return tuple
+            return v
         for c in url[:i]:
             if c not in scheme_chars:
                 break
@@ -111,9 +216,9 @@ def urlsplit(url, scheme='', allow_fragments=1):
         url, fragment = url.split('#', 1)
     if scheme in uses_query and '?' in url:
         url, query = url.split('?', 1)
-    tuple = scheme, netloc, url, query, fragment
+    v = SplitResult(scheme, netloc, url, query, fragment)
-    _parse_cache[key] = tuple
+    _parse_cache[key] = v
-    return tuple
+    return v
 
 def urlunparse((scheme, netloc, url, params, query, fragment)):
     """Put a parsed URL back together again.  This may result in a
@@ -136,7 +241,7 @@ def urlunsplit((scheme, netloc, url, query, fragment)):
         url = url + '#' + fragment
     return url
 
-def urljoin(base, url, allow_fragments = 1):
+def urljoin(base, url, allow_fragments=True):
     """Join a base URL and a possibly relative URL to form an absolute
     interpretation of the latter."""
     if not base:

Misc/NEWS

@@ -489,6 +489,9 @@ Extension Modules
 Library
 -------
 
+- Patch #624325: urlparse.urlparse() and urlparse.urlsplit() results
+  now sport attributes that provide access to the parts of the result.
+
 - Patch #1462498: sgmllib now handles entity and character references
   in attribute values.