mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-07 07:31:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

#1745761, #755670, #13357, #12629, #1200313: improve attribute handling in HTMLParser.

Browse source
This commit is contained in:
Ezio Melotti 2011-11-14 18:53:33 +02:00
parent b245ed1cdf
commit c2fe57762b
3 changed files with 106 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

19
Lib/html/parser.py
View file

@ -30,8 +30,8 @@ attrfind = re.compile(
r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*'
r'(\'[^\']*\'|"[^"]*"|[^\s"\'=<>`]*))?') r'(\'[^\']*\'|"[^"]*"|[^\s"\'=<>`]*))?')
attrfind_tolerant = re.compile( attrfind_tolerant = re.compile(
r',?\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
r'(\'[^\']*\'|"[^"]*"|[^>\s]*))?') r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
locatestarttagend = re.compile(r""" locatestarttagend = re.compile(r"""
<[a-zA-Z][-.a-zA-Z0-9:_]* # tag name <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
(?:\s+ # whitespace before attribute name (?:\s+ # whitespace before attribute name
@ -49,16 +49,16 @@ locatestarttagend = re.compile(r"""
locatestarttagend_tolerant = re.compile(r""" locatestarttagend_tolerant = re.compile(r"""
<[a-zA-Z][-.a-zA-Z0-9:_]* # tag name <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
(?:\s* # optional whitespace before attribute name (?:\s* # optional whitespace before attribute name
(?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name (?:(?<=['"\s])[^\s/>][^\s/=>]* # attribute name
(?:\s*=\s* # value indicator (?:\s*=+\s* # value indicator
(?:'[^']*' # LITA-enclosed value (?:'[^']*' # LITA-enclosed value
|\"[^\"]*\" # LIT-enclosed value |"[^"]*" # LIT-enclosed value
|[^'\">\s]+ # bare value |(?!['"])[^>\s]* # bare value
) )
(?:\s*,)* # possibly followed by a comma (?:\s*,)* # possibly followed by a comma
)? )?\s*
) )*
)* )?
\s* # trailing whitespace \s* # trailing whitespace
""", re.VERBOSE) """, re.VERBOSE)
endendtag = re.compile('>') endendtag = re.compile('>')
@ -295,6 +295,7 @@ class HTMLParser(_markupbase.ParserBase):
elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
attrvalue[:1] == '"' == attrvalue[-1:]: attrvalue[:1] == '"' == attrvalue[-1:]:
attrvalue = attrvalue[1:-1] attrvalue = attrvalue[1:-1]
if attrvalue:
attrvalue = self.unescape(attrvalue) attrvalue = self.unescape(attrvalue)
attrs.append((attrname.lower(), attrvalue)) attrs.append((attrname.lower(), attrvalue))
k = m.end() k = m.end()

107
Lib/test/test_htmlparser.py
View file

@ -241,13 +241,11 @@ DOCTYPE html [
self._parse_error("<a<a>") self._parse_error("<a<a>")
self._parse_error("</a<a>") self._parse_error("</a<a>")
self._parse_error("<!") self._parse_error("<!")
self._parse_error("<a $>")
self._parse_error("<a") self._parse_error("<a")
self._parse_error("<a foo='bar'") self._parse_error("<a foo='bar'")
self._parse_error("<a foo='bar") self._parse_error("<a foo='bar")
self._parse_error("<a foo='>'") self._parse_error("<a foo='>'")
self._parse_error("<a foo='>") self._parse_error("<a foo='>")
self._parse_error("<a foo=>")
def test_declaration_junk_chars(self): def test_declaration_junk_chars(self):
self._parse_error("<!DOCTYPE foo $ >") self._parse_error("<!DOCTYPE foo $ >")
@ -313,15 +311,14 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
def test_tolerant_parsing(self): def test_tolerant_parsing(self):
self._run_check('<html <html>te>>xt&a<<bc</a></html>\n' self._run_check('<html <html>te>>xt&a<<bc</a></html>\n'
'<img src="URL><//img></html</html>', [ '<img src="URL><//img></html</html>', [
('data', '<html '), ('starttag', 'html', [('<html', None)]),
('starttag', 'html', []), ('data', 'te>>xt'),
('data', 'te>>xt'), ('entityref', 'a'),
('entityref', 'a'), ('data', '<<bc'),
('data', '<<bc'), ('endtag', 'a'),
('endtag', 'a'), ('endtag', 'html'),
('endtag', 'html'), ('data', '\n<img src="URL><//img></html'),
('data', '\n<img src="URL><//img></html'), ('endtag', 'html')])
('endtag', 'html')])
def test_with_unquoted_attributes(self): def test_with_unquoted_attributes(self):
# see #12008 # see #12008
@ -352,7 +349,7 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
'method="post">', [ 'method="post">', [
('starttag', 'form', ('starttag', 'form',
[('action', '/xxx.php?a=1&b=2&amp'), [('action', '/xxx.php?a=1&b=2&amp'),
('method', 'post')])]) (',', None), ('method', 'post')])])
def test_weird_chars_in_unquoted_attribute_values(self): def test_weird_chars_in_unquoted_attribute_values(self):
self._run_check('<form action=bogus|&#()value>', [ self._run_check('<form action=bogus|&#()value>', [
@ -383,7 +380,7 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
html = '<div style="", foo = "bar" ><b>The <a href="some_url">rain</a>' html = '<div style="", foo = "bar" ><b>The <a href="some_url">rain</a>'
expected = [ expected = [
('starttag', 'div', [('style', ''), ('foo', 'bar')]), ('starttag', 'div', [('style', ''), (',', None), ('foo', 'bar')]),
('starttag', 'b', []), ('starttag', 'b', []),
('data', 'The '), ('data', 'The '),
('starttag', 'a', [('href', 'some_url')]), ('starttag', 'a', [('href', 'some_url')]),
@ -460,9 +457,91 @@ class AttributesStrictTestCase(TestCaseBase):
[("starttag", "html", [("foo", "\u20AC&aa&unsupported;")])]) [("starttag", "html", [("foo", "\u20AC&aa&unsupported;")])])
class AttributesTolerantTestCase(AttributesStrictTestCase):
def get_collector(self):
return EventCollector(strict=False)
def test_attr_funky_names2(self):
self._run_check(
"<a $><b $=%><c \=/>",
[("starttag", "a", [("$", None)]),
("starttag", "b", [("$", "%")]),
("starttag", "c", [("\\", "/")])])
def test_entities_in_attribute_value(self):
# see #1200313
for entity in ['&', '&amp;', '&#38;', '&#x26;']:
self._run_check('<a href="%s">' % entity,
[("starttag", "a", [("href", "&")])])
self._run_check("<a href='%s'>" % entity,
[("starttag", "a", [("href", "&")])])
self._run_check("<a href=%s>" % entity,
[("starttag", "a", [("href", "&")])])
def test_malformed_attributes(self):
# see #13357
html = (
"<a href=test'style='color:red;bad1'>test - bad1</a>"
"<a href=test'+style='color:red;ba2'>test - bad2</a>"
"<a href=test'&nbsp;style='color:red;bad3'>test - bad3</a>"
"<a href = test'&nbsp;style='color:red;bad4' >test - bad4</a>"
)
expected = [
('starttag', 'a', [('href', "test'style='color:red;bad1'")]),
('data', 'test - bad1'), ('endtag', 'a'),
('starttag', 'a', [('href', "test'+style='color:red;ba2'")]),
('data', 'test - bad2'), ('endtag', 'a'),
('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]),
('data', 'test - bad3'), ('endtag', 'a'),
('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]),
('data', 'test - bad4'), ('endtag', 'a')
]
self._run_check(html, expected)
def test_malformed_adjacent_attributes(self):
# see #12629
self._run_check('<x><y z=""o"" /></x>',
[('starttag', 'x', []),
('startendtag', 'y', [('z', ''), ('o""', None)]),
('endtag', 'x')])
self._run_check('<x><y z="""" /></x>',
[('starttag', 'x', []),
('startendtag', 'y', [('z', ''), ('""', None)]),
('endtag', 'x')])
# see #755670 for the following 3 tests
def test_adjacent_attributes(self):
self._run_check('<a width="100%"cellspacing=0>',
[("starttag", "a",
[("width", "100%"), ("cellspacing","0")])])
self._run_check('<a id="foo"class="bar">',
[("starttag", "a",
[("id", "foo"), ("class","bar")])])
def test_missing_attribute_value(self):
self._run_check('<a v=>',
[("starttag", "a", [("v", "")])])
def test_javascript_attribute_value(self):
self._run_check("<a href=javascript:popup('/popup/help.html')>",
[("starttag", "a",
[("href", "javascript:popup('/popup/help.html')")])])
def test_end_tag_in_attribute_value(self):
# see #1745761
self._run_check("<a href='http://www.example.org/\">;'>spam</a>",
[("starttag", "a",
[("href", "http://www.example.org/\">;")]),
("data", "spam"), ("endtag", "a")])
def test_main(): def test_main():
support.run_unittest(HTMLParserStrictTestCase, HTMLParserTolerantTestCase, support.run_unittest(HTMLParserStrictTestCase, HTMLParserTolerantTestCase,
AttributesStrictTestCase) AttributesStrictTestCase, AttributesTolerantTestCase)
if __name__ == "__main__": if __name__ == "__main__":

3
Misc/NEWS
View file

@ -73,6 +73,9 @@ Core and Builtins
Library Library
------- -------
- Issues #1745761, #755670, #13357, #12629, #1200313: HTMLParser now correctly
handles non-valid attributes, including adjacent and unquoted attributes.
- Issue #13193: Fix distutils.filelist.FileList under Windows. The - Issue #13193: Fix distutils.filelist.FileList under Windows. The
"recursive-include" directive now recognizes both legal path separators. "recursive-include" directive now recognizes both legal path separators.