mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-24 03:35:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Patch #624325: urlparse.urlparse() and urlparse.urlsplit() results

Browse source 
now sport attributes that provide access to the parts of the result.
This commit is contained in:
Fred Drake 2006-04-01 22:14:43 +00:00
parent f878b8120c
commit ad5177cf8d
4 changed files with 357 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

172
Doc/lib/liburlparse.tex
View file

@ -25,48 +25,74 @@ draft!). It supports the following URL schemes:
\code{nntp}, \code{prospero}, \code{rsync}, \code{rtsp}, \code{rtspu},
\code{sftp}, \code{shttp}, \code{sip}, \code{sips}, \code{snews}, \code{svn},
\code{svn+ssh}, \code{telnet}, \code{wais}.
\versionadded[Support for the \code{sftp} and \code{sips} schemes]{2.5}
The \module{urlparse} module defines the following functions:
\begin{funcdesc}{urlparse}{urlstring\optional{, default_scheme\optional{, allow_fragments}}}
Parse a URL into 6 components, returning a 6-tuple: (addressing
scheme, network location, path, parameters, query, fragment
identifier). This corresponds to the general structure of a URL:
\begin{funcdesc}{urlparse}{urlstring\optional{,
default_scheme\optional{, allow_fragments}}}
Parse a URL into six components, returning a 6-tuple. This
corresponds to the general structure of a URL:
\code{\var{scheme}://\var{netloc}/\var{path};\var{parameters}?\var{query}\#\var{fragment}}.
Each tuple item is a string, possibly empty.
The components are not broken up in smaller parts (e.g. the network
The components are not broken up in smaller parts (for example, the network
location is a single string), and \% escapes are not expanded.
The delimiters as shown above are not part of the tuple items,
The delimiters as shown above are not part of the result,
except for a leading slash in the \var{path} component, which is
retained if present.
Example:
\begin{verbatim}
urlparse('http://www.cwi.nl:80/%7Eguido/Python.html')
\end{verbatim}
yields the tuple
retained if present. For example:
\begin{verbatim}
>>> from urlparse import urlparse
>>> o = urlparse('http://www.cwi.nl:80/%7Eguido/Python.html')
>>> o
('http', 'www.cwi.nl:80', '/%7Eguido/Python.html', '', '', '')
>>> o.scheme
'http'
>>> o.port
80
>>> o.geturl()
'http://www.cwi.nl:80/%7Eguido/Python.html'
\end{verbatim}
If the \var{default_scheme} argument is specified, it gives the
default addressing scheme, to be used only if the URL string does not
default addressing scheme, to be used only if the URL does not
specify one. The default value for this argument is the empty string.
If the \var{allow_fragments} argument is zero, fragment identifiers
If the \var{allow_fragments} argument is false, fragment identifiers
are not allowed, even if the URL's addressing scheme normally does
support them. The default value for this argument is \code{1}.
support them. The default value for this argument is \constant{True}.
The return value is actually an instance of a subclass of
\pytype{tuple}. This class has the following additional read-only
convenience attributes:
\begin{tableiv}{l|c|l|c}{member}{Attribute}{Index}{Value}{Value if not present}
\lineiv{scheme} {0} {URL scheme specifier} {empty string}
\lineiv{netloc} {1} {Network location part} {empty string}
\lineiv{path} {2} {Hierarchical path} {empty string}
\lineiv{params} {3} {Parameters for last path element} {empty string}
\lineiv{query} {4} {Query component} {empty string}
\lineiv{fragment}{5} {Fragment identifier} {empty string}
\lineiv{username}{ } {User name} {\constant{None}}
\lineiv{password}{ } {Password} {\constant{None}}
\lineiv{hostname}{ } {Host name (lower case)} {\constant{None}}
\lineiv{port} { } {Port number as integer, if present} {\constant{None}}
\end{tableiv}
See section~\ref{urlparse-result-object}, ``Results of
\function{urlparse()} and \function{urlsplit()},'' for more
information on the result object.
\versionchanged[Added attributes to return value]{2.5}
\end{funcdesc}
\begin{funcdesc}{urlunparse}{tuple}
Construct a URL string from a tuple as returned by \code{urlparse()}.
\begin{funcdesc}{urlunparse}{parts}
Construct a URL from a tuple as returned by \code{urlparse()}.
The \var{parts} argument be any six-item iterable.
This may result in a slightly different, but equivalent URL, if the
URL that was parsed originally had redundant delimiters, e.g. a ? with
an empty query (the draft states that these are equivalent).
URL that was parsed originally had unnecessary delimiters (for example,
a ? with an empty query; the RFC states that these are equivalent).
\end{funcdesc}
\begin{funcdesc}{urlsplit}{urlstring\optional{,
@ -79,12 +105,38 @@ the URL (see \rfc{2396}) is wanted. A separate function is needed to
separate the path segments and parameters. This function returns a
5-tuple: (addressing scheme, network location, path, query, fragment
identifier).
The return value is actually an instance of a subclass of
\pytype{tuple}. This class has the following additional read-only
convenience attributes:
\begin{tableiv}{l|c|l|c}{member}{Attribute}{Index}{Value}{Value if not present}
\lineiv{scheme} {0} {URL scheme specifier} {empty string}
\lineiv{netloc} {1} {Network location part} {empty string}
\lineiv{path} {2} {Hierarchical path} {empty string}
\lineiv{query} {3} {Query component} {empty string}
\lineiv{fragment} {4} {Fragment identifier} {empty string}
\lineiv{username} { } {User name} {\constant{None}}
\lineiv{password} { } {Password} {\constant{None}}
\lineiv{hostname} { } {Host name (lower case)} {\constant{None}}
\lineiv{port} { } {Port number as integer, if present} {\constant{None}}
\end{tableiv}
See section~\ref{urlparse-result-object}, ``Results of
\function{urlparse()} and \function{urlsplit()},'' for more
information on the result object.
\versionadded{2.2}
\versionchanged[Added attributes to return value]{2.5}
\end{funcdesc}
\begin{funcdesc}{urlunsplit}{tuple}
\begin{funcdesc}{urlunsplit}{parts}
Combine the elements of a tuple as returned by \function{urlsplit()}
into a complete URL as a string.
The \var{parts} argument be any five-item iterable.
This may result in a slightly different, but equivalent URL, if the
URL that was parsed originally had unnecessary delimiters (for example,
a ? with an empty query; the RFC states that these are equivalent).
\versionadded{2.2}
\end{funcdesc}
@ -93,22 +145,16 @@ Construct a full (``absolute'') URL by combining a ``base URL''
(\var{base}) with a ``relative URL'' (\var{url}). Informally, this
uses components of the base URL, in particular the addressing scheme,
the network location and (part of) the path, to provide missing
components in the relative URL.
Example:
\begin{verbatim}
urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html')
\end{verbatim}
yields the string
components in the relative URL. For example:
\begin{verbatim}
>>> from urlparse import urljoin
>>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html')
'http://www.cwi.nl/%7Eguido/FAQ.html'
\end{verbatim}
The \var{allow_fragments} argument has the same meaning as for
\code{urlparse()}.
The \var{allow_fragments} argument has the same meaning and default as
for \function{urlparse()}.
\end{funcdesc}
\begin{funcdesc}{urldefrag}{url}
@ -133,3 +179,61 @@ in \var{url}, returns \var{url} unmodified and an empty string.
both Uniform Resource Names (URNs) and Uniform Resource
Locators (URLs).}
\end{seealso}
\subsection{Results of \function{urlparse()} and \function{urlsplit()}
\label{urlparse-result-object}}
The result objects from the \function{urlparse()} and
\function{urlsplit()} functions are subclasses of the \pytype{tuple}
type. These subclasses add the attributes described in those
functions, as well as provide an additional method:
\begin{methoddesc}[ParseResult]{geturl}{}
Return the re-combined version of the original URL as a string.
This may differ from the original URL in that the scheme will always
be normalized to lower case and empty components may be dropped.
Specifically, empty parameters, queries, and fragment identifiers
will be removed.
The result of this method is a fixpoint if passed back through the
original parsing function:
\begin{verbatim}
>>> import urlparse
>>> url = 'HTTP://www.Python.org/doc/#'
>>> r1 = urlparse.urlsplit(url)
>>> r1.geturl()
'http://www.Python.org/doc/'
>>> r2 = urlparse.urlsplit(r1.geturl())
>>> r2.geturl()
'http://www.Python.org/doc/'
\end{verbatim}
\versionadded{2.5}
\end{methoddesc}
The following classes provide the implementations of the parse results::
\begin{classdesc*}{BaseResult}
Base class for the concrete result classes. This provides most of
the attribute definitions. It does not provide a \method{geturl()}
method. It is derived from \class{tuple}, but does not override the
\method{__init__()} or \method{__new__()} methods.
\end{classdesc*}
\begin{classdesc}{ParseResult}{scheme, netloc, path, params, query, fragment}
Concrete class for \function{urlparse()} results. The
\method{__new__()} method is overridden to support checking that the
right number of arguments are passed.
\end{classdesc}
\begin{classdesc}{SplitResult}{scheme, netloc, path, query, fragment}
Concrete class for \function{urlsplit()} results. The
\method{__new__()} method is overridden to support checking that the
right number of arguments are passed.
\end{classdesc}

101
Lib/test/test_urlparse.py
View file

@ -12,15 +12,53 @@ class UrlParseTestCase(unittest.TestCase):
def checkRoundtrips(self, url, parsed, split):
result = urlparse.urlparse(url)
self.assertEqual(result, parsed)
t = (result.scheme, result.netloc, result.path,
result.params, result.query, result.fragment)
self.assertEqual(t, parsed)
# put it back together and it should be the same
result2 = urlparse.urlunparse(result)
self.assertEqual(result2, url)
self.assertEqual(result2, result.geturl())
# the result of geturl() is a fixpoint; we can always parse it
# again to get the same result:
result3 = urlparse.urlparse(result.geturl())
self.assertEqual(result3.geturl(), result.geturl())
self.assertEqual(result3, result)
self.assertEqual(result3.scheme, result.scheme)
self.assertEqual(result3.netloc, result.netloc)
self.assertEqual(result3.path, result.path)
self.assertEqual(result3.params, result.params)
self.assertEqual(result3.query, result.query)
self.assertEqual(result3.fragment, result.fragment)
self.assertEqual(result3.username, result.username)
self.assertEqual(result3.password, result.password)
self.assertEqual(result3.hostname, result.hostname)
self.assertEqual(result3.port, result.port)
# check the roundtrip using urlsplit() as well
result = urlparse.urlsplit(url)
self.assertEqual(result, split)
t = (result.scheme, result.netloc, result.path,
result.query, result.fragment)
self.assertEqual(t, split)
result2 = urlparse.urlunsplit(result)
self.assertEqual(result2, url)
self.assertEqual(result2, result.geturl())
# check the fixpoint property of re-parsing the result of geturl()
result3 = urlparse.urlsplit(result.geturl())
self.assertEqual(result3.geturl(), result.geturl())
self.assertEqual(result3, result)
self.assertEqual(result3.scheme, result.scheme)
self.assertEqual(result3.netloc, result.netloc)
self.assertEqual(result3.path, result.path)
self.assertEqual(result3.query, result.query)
self.assertEqual(result3.fragment, result.fragment)
self.assertEqual(result3.username, result.username)
self.assertEqual(result3.password, result.password)
self.assertEqual(result3.hostname, result.hostname)
self.assertEqual(result3.port, result.port)
def test_roundtrips(self):
testcases = [
@ -187,6 +225,69 @@ class UrlParseTestCase(unittest.TestCase):
]:
self.assertEqual(urlparse.urldefrag(url), (defrag, frag))
def test_urlsplit_attributes(self):
url = "HTTP://WWW.PYTHON.ORG/doc/#frag"
p = urlparse.urlsplit(url)
self.assertEqual(p.scheme, "http")
self.assertEqual(p.netloc, "WWW.PYTHON.ORG")
self.assertEqual(p.path, "/doc/")
self.assertEqual(p.query, "")
self.assertEqual(p.fragment, "frag")
self.assertEqual(p.username, None)
self.assertEqual(p.password, None)
self.assertEqual(p.hostname, "www.python.org")
self.assertEqual(p.port, None)
# geturl() won't return exactly the original URL in this case
# since the scheme is always case-normalized
#self.assertEqual(p.geturl(), url)
url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag"
p = urlparse.urlsplit(url)
self.assertEqual(p.scheme, "http")
self.assertEqual(p.netloc, "User:Pass@www.python.org:080")
self.assertEqual(p.path, "/doc/")
self.assertEqual(p.query, "query=yes")
self.assertEqual(p.fragment, "frag")
self.assertEqual(p.username, "User")
self.assertEqual(p.password, "Pass")
self.assertEqual(p.hostname, "www.python.org")
self.assertEqual(p.port, 80)
self.assertEqual(p.geturl(), url)
def test_attributes_bad_port(self):
"""Check handling of non-integer ports."""
p = urlparse.urlsplit("http://www.example.net:foo")
self.assertEqual(p.netloc, "www.example.net:foo")
self.assertRaises(ValueError, lambda: p.port)
p = urlparse.urlparse("http://www.example.net:foo")
self.assertEqual(p.netloc, "www.example.net:foo")
self.assertRaises(ValueError, lambda: p.port)
def test_attributes_without_netloc(self):
# This example is straight from RFC 3261. It looks like it
# should allow the username, hostname, and port to be filled
# in, but doesn't. Since it's a URI and doesn't use the
# scheme://netloc syntax, the netloc and related attributes
# should be left empty.
uri = "sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15"
p = urlparse.urlsplit(uri)
self.assertEqual(p.netloc, "")
self.assertEqual(p.username, None)
self.assertEqual(p.password, None)
self.assertEqual(p.hostname, None)
self.assertEqual(p.port, None)
self.assertEqual(p.geturl(), uri)
p = urlparse.urlparse(uri)
self.assertEqual(p.netloc, "")
self.assertEqual(p.username, None)
self.assertEqual(p.password, None)
self.assertEqual(p.hostname, None)
self.assertEqual(p.port, None)
self.assertEqual(p.geturl(), uri)
def test_main():
test_support.run_unittest(UrlParseTestCase)

125
Lib/urlparse.py
View file

@ -41,7 +41,111 @@ def clear_cache():
_parse_cache = {}
def urlparse(url, scheme='', allow_fragments=1):
class BaseResult(tuple):
"""Base class for the parsed result objects.
This provides the attributes shared by the two derived result
objects as read-only properties. The derived classes are
responsible for checking the right number of arguments were
supplied to the constructor.
"""
__slots__ = ()
# Attributes that access the basic components of the URL:
@property
def scheme(self):
return self[0]
@property
def netloc(self):
return self[1]
@property
def path(self):
return self[2]
@property
def query(self):
return self[-2]
@property
def fragment(self):
return self[-1]
# Additional attributes that provide access to parsed-out portions
# of the netloc:
@property
def username(self):
netloc = self.netloc
if "@" in netloc:
userinfo = netloc.split("@", 1)[0]
if ":" in userinfo:
userinfo = userinfo.split(":", 1)[0]
return userinfo
return None
@property
def password(self):
netloc = self.netloc
if "@" in netloc:
userinfo = netloc.split("@", 1)[0]
if ":" in userinfo:
return userinfo.split(":", 1)[1]
return None
@property
def hostname(self):
netloc = self.netloc
if "@" in netloc:
netloc = netloc.split("@", 1)[1]
if ":" in netloc:
netloc = netloc.split(":", 1)[0]
return netloc.lower() or None
@property
def port(self):
netloc = self.netloc
if "@" in netloc:
netloc = netloc.split("@", 1)[1]
if ":" in netloc:
port = netloc.split(":", 1)[1]
return int(port, 10)
return None
class SplitResult(BaseResult):
__slots__ = ()
def __new__(cls, scheme, netloc, path, query, fragment):
return BaseResult.__new__(
cls, (scheme, netloc, path, query, fragment))
def geturl(self):
return urlunsplit(self)
class ParseResult(BaseResult):
__slots__ = ()
def __new__(cls, scheme, netloc, path, params, query, fragment):
return BaseResult.__new__(
cls, (scheme, netloc, path, params, query, fragment))
@property
def params(self):
return self[3]
def geturl(self):
return urlunparse(self)
def urlparse(url, scheme='', allow_fragments=True):
"""Parse a URL into 6 components:
<scheme>://<netloc>/<path>;<params>?<query>#<fragment>
Return a 6-tuple: (scheme, netloc, path, params, query, fragment).
@ -53,7 +157,7 @@ def urlparse(url, scheme='', allow_fragments=1):
url, params = _splitparams(url)
else:
params = ''
return scheme, netloc, url, params, query, fragment
return ParseResult(scheme, netloc, url, params, query, fragment)
def _splitparams(url):
if '/' in url:
@ -73,12 +177,13 @@ def _splitnetloc(url, start=0):
delim = len(url)
return url[start:delim], url[delim:]
def urlsplit(url, scheme='', allow_fragments=1):
def urlsplit(url, scheme='', allow_fragments=True):
"""Parse a URL into 5 components:
<scheme>://<netloc>/<path>?<query>#<fragment>
Return a 5-tuple: (scheme, netloc, path, query, fragment).
Note that we don't break the components up in smaller bits
(e.g. netloc is a single string) and we don't expand % escapes."""
allow_fragments = bool(allow_fragments)
key = url, scheme, allow_fragments
cached = _parse_cache.get(key, None)
if cached:
@ -97,9 +202,9 @@ def urlsplit(url, scheme='', allow_fragments=1):
url, fragment = url.split('#', 1)
if '?' in url:
url, query = url.split('?', 1)
tuple = scheme, netloc, url, query, fragment
_parse_cache[key] = tuple
return tuple
v = SplitResult(scheme, netloc, url, query, fragment)
_parse_cache[key] = v
return v
for c in url[:i]:
if c not in scheme_chars:
break
@ -111,9 +216,9 @@ def urlsplit(url, scheme='', allow_fragments=1):
url, fragment = url.split('#', 1)
if scheme in uses_query and '?' in url:
url, query = url.split('?', 1)
tuple = scheme, netloc, url, query, fragment
_parse_cache[key] = tuple
return tuple
v = SplitResult(scheme, netloc, url, query, fragment)
_parse_cache[key] = v
return v
def urlunparse((scheme, netloc, url, params, query, fragment)):
"""Put a parsed URL back together again. This may result in a
@ -136,7 +241,7 @@ def urlunsplit((scheme, netloc, url, query, fragment)):
url = url + '#' + fragment
return url
def urljoin(base, url, allow_fragments = 1):
def urljoin(base, url, allow_fragments=True):
"""Join a base URL and a possibly relative URL to form an absolute
interpretation of the latter."""
if not base:

3
Misc/NEWS
View file

@ -489,6 +489,9 @@ Extension Modules
Library
-------
- Patch #624325: urlparse.urlparse() and urlparse.urlsplit() results
now sport attributes that provide access to the parts of the result.
- Patch #1462498: sgmllib now handles entity and character references
in attribute values.