From 8e08ac9f32d89bf387c75bb6d0710a7b59026b5b Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Mon, 5 May 2025 18:03:42 +0100 Subject: [PATCH] GH-123599: `url2pathname()`: don't call `gethostbyname()` by default (#132610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to 66cdb2bd8ab93a4691bead7f5d1e54e5ca6895b4. Add *resolve_host* keyword-only argument to `url2pathname()`, defaulting to false. When set to true, we call `socket.gethostbyname()` to resolve the URL hostname. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Steve Dower --- Doc/library/pathlib.rst | 8 ++++---- Doc/library/urllib.request.rst | 18 ++++++++++++------ Doc/whatsnew/3.14.rst | 8 +++++--- Lib/test/test_pathlib/test_pathlib.py | 1 - Lib/test/test_urllib.py | 16 ++++++++++++---- Lib/urllib/request.py | 19 ++++++++++++------- ...-11-14-21-17-48.gh-issue-126838.Yr5vKF.rst | 8 +++----- 7 files changed, 48 insertions(+), 30 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index b8298690286..acb91812818 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -872,10 +872,10 @@ conforming to :rfc:`8089`. .. versionadded:: 3.13 .. versionchanged:: next - If a URL authority (e.g. a hostname) is present and resolves to a local - address, it is discarded. If an authority is present and *doesn't* - resolve to a local address, then on Windows a UNC path is returned (as - before), and on other platforms a :exc:`ValueError` is raised. + The URL authority is discarded if it matches the local hostname. + Otherwise, if the authority isn't empty or ``localhost``, then on + Windows a UNC path is returned (as before), and on other platforms a + :exc:`ValueError` is raised. .. method:: Path.as_uri() diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index b7c0c7d5099..234827132bf 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -172,10 +172,10 @@ The :mod:`urllib.request` module defines the following functions: the URL ``///etc/hosts``. .. versionchanged:: next - The *add_scheme* argument was added. + The *add_scheme* parameter was added. -.. function:: url2pathname(url, *, require_scheme=False) +.. function:: url2pathname(url, *, require_scheme=False, resolve_host=False) Convert the given ``file:`` URL to a local path. This function uses :func:`~urllib.parse.unquote` to decode the URL. @@ -185,6 +185,13 @@ The :mod:`urllib.request` module defines the following functions: value should include the prefix; a :exc:`~urllib.error.URLError` is raised if it doesn't. + The URL authority is discarded if it is empty, ``localhost``, or the local + hostname. Otherwise, if *resolve_host* is set to true, the authority is + resolved using :func:`socket.gethostbyname` and discarded if it matches a + local IP address (as per :rfc:`RFC 8089 §3 <8089#section-3>`). If the + authority is still unhandled, then on Windows a UNC path is returned, and + on other platforms a :exc:`~urllib.error.URLError` is raised. + This example shows the function being used on Windows:: >>> from urllib.request import url2pathname @@ -198,14 +205,13 @@ The :mod:`urllib.request` module defines the following functions: :exc:`OSError` exception to be raised on Windows. .. versionchanged:: next - This function calls :func:`socket.gethostbyname` if the URL authority - isn't empty, ``localhost``, or the machine hostname. If the authority - resolves to a local IP address then it is discarded; otherwise, on + The URL authority is discarded if it matches the local hostname. + Otherwise, if the authority isn't empty or ``localhost``, then on Windows a UNC path is returned (as before), and on other platforms a :exc:`~urllib.error.URLError` is raised. .. versionchanged:: next - The *require_scheme* argument was added. + The *require_scheme* and *resolve_host* parameters were added. .. function:: getproxies() diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 506a820175b..68ffedf4729 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1703,9 +1703,11 @@ urllib - Accept a complete URL when the new *require_scheme* argument is set to true. - - Discard URL authorities that resolve to a local IP address. - - Raise :exc:`~urllib.error.URLError` if a URL authority doesn't resolve - to a local IP address, except on Windows where we return a UNC path. + - Discard URL authority if it matches the local hostname. + - Discard URL authority if it resolves to a local IP address when the new + *resolve_host* argument is set to true. + - Raise :exc:`~urllib.error.URLError` if a URL authority isn't local, + except on Windows where we return a UNC path as before. In :func:`urllib.request.pathname2url`: diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 41a79d0dceb..e23dac3c25f 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -3290,7 +3290,6 @@ class PathTest(PurePathTest): self.assertEqual(P.from_uri('file:////foo/bar'), P('//foo/bar')) self.assertEqual(P.from_uri('file://localhost/foo/bar'), P('/foo/bar')) if not is_wasi: - self.assertEqual(P.from_uri('file://127.0.0.1/foo/bar'), P('/foo/bar')) self.assertEqual(P.from_uri(f'file://{socket.gethostname()}/foo/bar'), P('/foo/bar')) self.assertRaises(ValueError, P.from_uri, 'foo/bar') diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 90de828cc71..c965860fbb1 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1551,7 +1551,8 @@ class Pathname_Tests(unittest.TestCase): urllib.request.url2pathname(url, require_scheme=True), expected_path) - error_subtests = [ + def test_url2pathname_require_scheme_errors(self): + subtests = [ '', ':', 'foo', @@ -1561,13 +1562,20 @@ class Pathname_Tests(unittest.TestCase): 'data:file:foo', 'data:file://foo', ] - for url in error_subtests: + for url in subtests: with self.subTest(url=url): self.assertRaises( urllib.error.URLError, urllib.request.url2pathname, url, require_scheme=True) + def test_url2pathname_resolve_host(self): + fn = urllib.request.url2pathname + sep = os.path.sep + self.assertEqual(fn('//127.0.0.1/foo/bar', resolve_host=True), f'{sep}foo{sep}bar') + self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar'), f'{sep}foo{sep}bar') + self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar', resolve_host=True), f'{sep}foo{sep}bar') + @unittest.skipUnless(sys.platform == 'win32', 'test specific to Windows pathnames.') def test_url2pathname_win(self): @@ -1598,6 +1606,7 @@ class Pathname_Tests(unittest.TestCase): self.assertEqual(fn('//server/path/to/file'), '\\\\server\\path\\to\\file') self.assertEqual(fn('////server/path/to/file'), '\\\\server\\path\\to\\file') self.assertEqual(fn('/////server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('//127.0.0.1/path/to/file'), '\\\\127.0.0.1\\path\\to\\file') # Localhost paths self.assertEqual(fn('//localhost/C:/path/to/file'), 'C:\\path\\to\\file') self.assertEqual(fn('//localhost/C|/path/to/file'), 'C:\\path\\to\\file') @@ -1622,8 +1631,7 @@ class Pathname_Tests(unittest.TestCase): self.assertRaises(urllib.error.URLError, fn, '//:80/foo/bar') self.assertRaises(urllib.error.URLError, fn, '//:/foo/bar') self.assertRaises(urllib.error.URLError, fn, '//c:80/foo/bar') - self.assertEqual(fn('//127.0.0.1/foo/bar'), '/foo/bar') - self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar'), '/foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//127.0.0.1/foo/bar') @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') def test_url2pathname_nonascii(self): diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 9a6b29a90a2..41dc5d7b35d 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1466,7 +1466,7 @@ class FileHandler(BaseHandler): def open_local_file(self, req): import email.utils import mimetypes - localfile = url2pathname(req.full_url, require_scheme=True) + localfile = url2pathname(req.full_url, require_scheme=True, resolve_host=True) try: stats = os.stat(localfile) size = stats.st_size @@ -1482,7 +1482,7 @@ class FileHandler(BaseHandler): file_open = open_local_file -def _is_local_authority(authority): +def _is_local_authority(authority, resolve): # Compare hostnames if not authority or authority == 'localhost': return True @@ -1494,9 +1494,11 @@ def _is_local_authority(authority): if authority == hostname: return True # Compare IP addresses + if not resolve: + return False try: address = socket.gethostbyname(authority) - except (socket.gaierror, AttributeError): + except (socket.gaierror, AttributeError, UnicodeEncodeError): return False return address in FileHandler().get_names() @@ -1641,13 +1643,16 @@ class DataHandler(BaseHandler): return addinfourl(io.BytesIO(data), headers, url) -# Code move from the old urllib module +# Code moved from the old urllib module -def url2pathname(url, *, require_scheme=False): +def url2pathname(url, *, require_scheme=False, resolve_host=False): """Convert the given file URL to a local file system path. The 'file:' scheme prefix must be omitted unless *require_scheme* is set to true. + + The URL authority may be resolved with gethostbyname() if + *resolve_host* is set to true. """ if require_scheme: scheme, url = _splittype(url) @@ -1655,7 +1660,7 @@ def url2pathname(url, *, require_scheme=False): raise URLError("URL is missing a 'file:' scheme") authority, url = _splithost(url) if os.name == 'nt': - if not _is_local_authority(authority): + if not _is_local_authority(authority, resolve_host): # e.g. file://server/share/file.txt url = '//' + authority + url elif url[:3] == '///': @@ -1669,7 +1674,7 @@ def url2pathname(url, *, require_scheme=False): # Older URLs use a pipe after a drive letter url = url[:1] + ':' + url[2:] url = url.replace('/', '\\') - elif not _is_local_authority(authority): + elif not _is_local_authority(authority, resolve_host): raise URLError("file:// scheme is supported only on localhost") encoding = sys.getfilesystemencoding() errors = sys.getfilesystemencodeerrors() diff --git a/Misc/NEWS.d/next/Library/2024-11-14-21-17-48.gh-issue-126838.Yr5vKF.rst b/Misc/NEWS.d/next/Library/2024-11-14-21-17-48.gh-issue-126838.Yr5vKF.rst index 857cc359229..c45c22e7344 100644 --- a/Misc/NEWS.d/next/Library/2024-11-14-21-17-48.gh-issue-126838.Yr5vKF.rst +++ b/Misc/NEWS.d/next/Library/2024-11-14-21-17-48.gh-issue-126838.Yr5vKF.rst @@ -1,5 +1,3 @@ -Fix issue where :func:`urllib.request.url2pathname` mishandled file URLs with -authorities. If an authority is present and resolves to ``localhost``, it is -now discarded. If an authority is present but *doesn't* resolve to -``localhost``, then on Windows a UNC path is returned (as before), and on -other platforms a :exc:`urllib.error.URLError` is now raised. +Add *resolve_host* keyword-only parameter to +:func:`urllib.request.url2pathname`, and fix handling of file URLs with +authorities.