mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 18:29:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

Added reasonable parsing of the DOCTYPE declaration, fixed edge cases

Browse source 
regarding bare ampersands in content.
This commit is contained in:
Fred Drake 2001-09-04 15:10:16 +00:00
parent 212a2e1f9f
commit 68eac2b574
1 changed files with 260 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

264
Lib/HTMLParser.py
View file

@ -15,8 +15,7 @@ import string
interesting_normal = re.compile('[&<]') interesting_normal = re.compile('[&<]')
interesting_cdata = re.compile(r'<(/|\Z)') interesting_cdata = re.compile(r'<(/|\Z)')
incomplete = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*' incomplete = re.compile('&[a-zA-Z#]')
'|#([0-9]*|[xX][0-9a-fA-F]*))?')
entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
@ -185,16 +184,18 @@ class HTMLParser:
k = self.parse_pi(i) k = self.parse_pi(i)
elif declopen.match(rawdata, i): # <! elif declopen.match(rawdata, i): # <!
k = self.parse_declaration(i) k = self.parse_declaration(i)
else: elif (i + 1) < n:
self.handle_data("<") self.handle_data("<")
k = i + 1 k = i + 1
else:
break
if k < 0: if k < 0:
if end: if end:
raise HTMLParseError("EOF in middle of construct", raise HTMLParseError("EOF in middle of construct",
self.getpos()) self.getpos())
break break
i = self.updatepos(i, k) i = self.updatepos(i, k)
elif rawdata[i] == '&': elif rawdata[i:i+2] == "&#":
match = charref.match(rawdata, i) match = charref.match(rawdata, i)
if match: if match:
name = match.group()[2:-1] name = match.group()[2:-1]
@ -204,6 +205,9 @@ class HTMLParser:
k = k - 1 k = k - 1
i = self.updatepos(i, k) i = self.updatepos(i, k)
continue continue
else:
break
elif rawdata[i] == '&':
match = entityref.match(rawdata, i) match = entityref.match(rawdata, i)
if match: if match:
name = match.group(1) name = match.group(1)
@ -215,14 +219,21 @@ class HTMLParser:
continue continue
match = incomplete.match(rawdata, i) match = incomplete.match(rawdata, i)
if match: if match:
# match.group() will contain at least 2 chars
rest = rawdata[i:] rest = rawdata[i:]
if end and rest != "&" and match.group() == rest: if end and match.group() == rest:
raise HTMLParseError( raise HTMLParseError(
"EOF in middle of entity or char ref", "EOF in middle of entity or char ref",
self.getpos()) self.getpos())
return -1 # incomplete # incomplete
break
elif (i + 1) < n:
# not the end of the buffer, and can't be confused
# with some other construct
self.handle_data("&") self.handle_data("&")
i = self.updatepos(i, i + 1) i = self.updatepos(i, i + 1)
else:
break
else: else:
assert 0, "interesting.search() lied" assert 0, "interesting.search() lied"
# end while # end while
@ -232,12 +243,13 @@ class HTMLParser:
self.rawdata = rawdata[i:] self.rawdata = rawdata[i:]
# Internal -- parse comment, return end or -1 if not terminated # Internal -- parse comment, return end or -1 if not terminated
def parse_comment(self, i): def parse_comment(self, i, report=1):
rawdata = self.rawdata rawdata = self.rawdata
assert rawdata[i:i+4] == '<!--', 'unexpected call to parse_comment()' assert rawdata[i:i+4] == '<!--', 'unexpected call to parse_comment()'
match = commentclose.search(rawdata, i+4) match = commentclose.search(rawdata, i+4)
if not match: if not match:
return -1 return -1
if report:
j = match.start() j = match.start()
self.handle_comment(rawdata[i+4: j]) self.handle_comment(rawdata[i+4: j])
j = match.end() j = match.end()
@ -257,11 +269,17 @@ class HTMLParser:
return -1 return -1
# in practice, this should look like: ((name|stringlit) S*)+ '>' # in practice, this should look like: ((name|stringlit) S*)+ '>'
n = len(rawdata) n = len(rawdata)
decltype = None
extrachars = ""
while j < n: while j < n:
c = rawdata[j] c = rawdata[j]
if c == ">": if c == ">":
# end of declaration syntax # end of declaration syntax
self.handle_decl(rawdata[i+2:j]) data = rawdata[i+2:j]
if decltype == "doctype":
self.handle_decl(data)
else:
self.unknown_decl(data)
return j + 1 return j + 1
if c in "\"'": if c in "\"'":
m = declstringlit.match(rawdata, j) m = declstringlit.match(rawdata, j)
@ -273,12 +291,242 @@ class HTMLParser:
if not m: if not m:
return -1 # incomplete return -1 # incomplete
j = m.end() j = m.end()
if decltype is None:
decltype = m.group(0).rstrip().lower()
if decltype != "doctype":
extrachars = "="
elif c == "[" and decltype == "doctype":
j = self.parse_doctype_subset(j + 1, i)
if j < 0:
return j
elif c in extrachars:
j = j + 1
while j < n and rawdata[j] in string.whitespace:
j = j + 1
if j == n:
# end of buffer while in declaration
return -1
else: else:
raise HTMLParseError( raise HTMLParseError(
"unexpected char in declaration: %s" % `rawdata[j]`, "unexpected char in declaration: %s" % `rawdata[j]`,
self.getpos()) self.getpos())
decltype = decltype or ''
return -1 # incomplete return -1 # incomplete
# Internal -- scan past the internal subset in a <!DOCTYPE declaration,
# returning the index just past any whitespace following the trailing ']'.
def parse_doctype_subset(self, i, declstartpos):
rawdata = self.rawdata
n = len(rawdata)
j = i
while j < n:
c = rawdata[j]
if c == "<":
s = rawdata[j:j+2]
if s == "<":
# end of buffer; incomplete
return -1
if s != "<!":
self.updatepos(declstartpos, j + 1)
raise HTMLParseError("unexpect char in internal subset",
self.getpos())
if (j + 2) == n:
# end of buffer; incomplete
return -1
if (j + 4) > n:
# end of buffer; incomplete
return -1
if rawdata[j:j+4] == "<!--":
j = self.parse_comment(j, report=0)
if j < 0:
return j
continue
name, j = self.scan_name(j + 2, declstartpos)
if j == -1:
return -1
if name not in ("attlist", "element", "entity", "notation"):
self.updatepos(declstartpos, j + 2)
raise HTMLParseError(
"unknown declaration %s in internal subset" % `name`,
self.getpos())
# handle the individual names
meth = getattr(self, "parse_doctype_" + name)
j = meth(j, declstartpos)
if j < 0:
return j
elif c == "%":
# parameter entity reference
if (j + 1) == n:
# end of buffer; incomplete
return -1
m = declname.match(rawdata, j + 1)
s = m.group()
if s == rawdata[j+1:]:
return -1
j = j + 1 + len(s.rstrip())
if rawdata[j] == ";":
j = j + 1
elif c == "]":
j = j + 1
while j < n and rawdata[j] in string.whitespace:
j = j + 1
if j < n:
if rawdata[j] == ">":
return j
self.updatepos(declstartpos, j)
raise HTMLParseError(
"unexpected char after internal subset",
self.getpos())
else:
return -1
elif c in string.whitespace:
j = j + 1
else:
self.updatepos(declstartpos, j)
raise HTMLParseError("unexpected char in internal subset",
self.getpos())
# end of buffer reached
return -1
def parse_doctype_element(self, i, declstartpos):
rawdata = self.rawdata
n = len(rawdata)
name, j = self.scan_name(i, declstartpos)
if j == -1:
return -1
# style content model; just skip until '>'
if '>' in rawdata[j:]:
return string.find(rawdata, ">", j) + 1
return -1
def parse_doctype_attlist(self, i, declstartpos):
rawdata = self.rawdata
name, j = self.scan_name(i, declstartpos)
c = rawdata[j:j+1]
if c == "":
return -1
if c == ">":
return j + 1
while 1:
# scan a series of attribute descriptions; simplified:
# name type [value] [#constraint]
name, j = self.scan_name(j, declstartpos)
if j < 0:
return j
c = rawdata[j:j+1]
if c == "":
return -1
if c == "(":
# an enumerated type; look for ')'
if ")" in rawdata[j:]:
j = string.find(rawdata, ")", j) + 1
else:
return -1
while rawdata[j:j+1] in string.whitespace:
j = j + 1
if not rawdata[j:]:
# end of buffer, incomplete
return -1
else:
name, j = self.scan_name(j, declstartpos)
c = rawdata[j:j+1]
if not c:
return -1
if c in "'\"":
m = declstringlit.match(rawdata, j)
if m:
j = m.end()
else:
return -1
c = rawdata[j:j+1]
if not c:
return -1
if c == "#":
if rawdata[j:] == "#":
# end of buffer
return -1
name, j = self.scan_name(j + 1, declstartpos)
if j < 0:
return j
c = rawdata[j:j+1]
if not c:
return -1
if c == '>':
# all done
return j + 1
def parse_doctype_notation(self, i, declstartpos):
name, j = self.scan_name(i, declstartpos)
if j < 0:
return j
rawdata = self.rawdata
while 1:
c = rawdata[j:j+1]
if not c:
# end of buffer; incomplete
return -1
if c == '>':
return j + 1
if c in "'\"":
m = declstringlit.match(rawdata, j)
if not m:
return -1
j = m.end()
else:
name, j = self.scan_name(j, declstartpos)
if j < 0:
return j
def parse_doctype_entity(self, i, declstartpos):
rawdata = self.rawdata
if rawdata[i:i+1] == "%":
j = i + 1
while 1:
c = rawdata[j:j+1]
if not c:
return -1
if c in string.whitespace:
j = j + 1
else:
break
else:
j = i
name, j = self.scan_name(j, declstartpos)
if j < 0:
return j
while 1:
c = self.rawdata[j:j+1]
if not c:
return -1
if c in "'\"":
m = declstringlit.match(rawdata, j)
if m:
j = m.end()
else:
return -1 # incomplete
elif c == ">":
return j + 1
else:
name, j = self.scan_name(j, declstartpos)
if j < 0:
return j
def scan_name(self, i, declstartpos):
rawdata = self.rawdata
n = len(rawdata)
if i == n:
return None, -1
m = declname.match(rawdata, i)
if m:
s = m.group()
name = s.strip()
if (i + len(s)) == n:
return None, -1 # end of buffer
return name.lower(), m.end()
else:
self.updatepos(declstartpos, i)
raise HTMLParseError("expected name token", self.getpos())
# Internal -- parse processing instr, return end or -1 if not terminated # Internal -- parse processing instr, return end or -1 if not terminated
def parse_pi(self, i): def parse_pi(self, i):
rawdata = self.rawdata rawdata = self.rawdata