[Python-checkins] cpython (3.2): Improve handling of declarations in HTMLParser.
ezio.melotti
python-checkins at python.org
Mon Feb 13 14:52:36 CET 2012
http://hg.python.org/cpython/rev/4c4ff9fd19b6
changeset: 74906:4c4ff9fd19b6
branch: 3.2
user: Ezio Melotti <ezio.melotti at gmail.com>
date: Mon Feb 13 15:50:37 2012 +0200
summary:
Improve handling of declarations in HTMLParser.
files:
Lib/html/parser.py | 30 +++++++++++----
Lib/test/test_htmlparser.py | 50 ++++++++++++++----------
2 files changed, 51 insertions(+), 29 deletions(-)
diff --git a/Lib/html/parser.py b/Lib/html/parser.py
--- a/Lib/html/parser.py
+++ b/Lib/html/parser.py
@@ -187,17 +187,10 @@
elif startswith("<?", i):
k = self.parse_pi(i)
elif startswith("<!", i):
- # this might fail with things like <! not a comment > or
- # <! -- space before '--' -->. When strict is True an
- # error is raised, when it's False they will be considered
- # as bogus comments and parsed (see parse_bogus_comment).
if self.strict:
k = self.parse_declaration(i)
else:
- try:
- k = self.parse_declaration(i)
- except HTMLParseError:
- k = self.parse_bogus_comment(i)
+ k = self.parse_html_declaration(i)
elif (i + 1) < n:
self.handle_data("<")
k = i + 1
@@ -269,6 +262,27 @@
i = self.updatepos(i, n)
self.rawdata = rawdata[i:]
+ # Internal -- parse html declarations, return length or -1 if not terminated
+ # See w3.org/TR/html5/tokenization.html#markup-declaration-open-state
+ # See also parse_declaration in _markupbase
+ def parse_html_declaration(self, i):
+ rawdata = self.rawdata
+ if rawdata[i:i+2] != '<!':
+ self.error('unexpected call to parse_html_declaration()')
+ if rawdata[i:i+4] == '<!--':
+ return self.parse_comment(i)
+ elif rawdata[i:i+3] == '<![':
+ return self.parse_marked_section(i)
+ elif rawdata[i:i+9].lower() == '<!doctype':
+ # find the closing >
+ gtpos = rawdata.find('>', 9)
+ if gtpos == -1:
+ return -1
+ self.handle_decl(rawdata[i+2:gtpos])
+ return gtpos+1
+ else:
+ return self.parse_bogus_comment(i)
+
# Internal -- parse bogus comment, return length or -1 if not terminated
# see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state
def parse_bogus_comment(self, i, report=1):
diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py
--- a/Lib/test/test_htmlparser.py
+++ b/Lib/test/test_htmlparser.py
@@ -122,7 +122,7 @@
<Img sRc='Bar' isMAP>sample
text
“
-<!--comment2a-- --comment2b--><!>
+<!--comment2a-- --comment2b-->
</Html>
""", [
("data", "\n"),
@@ -157,24 +157,6 @@
("data", " foo"),
])
- def test_doctype_decl(self):
- inside = """\
-DOCTYPE html [
- <!ELEMENT html - O EMPTY>
- <!ATTLIST html
- version CDATA #IMPLIED
- profile CDATA 'DublinCore'>
- <!NOTATION datatype SYSTEM 'http://xml.python.org/notations/python-module'>
- <!ENTITY myEntity 'internal parsed entity'>
- <!ENTITY anEntity SYSTEM 'http://xml.python.org/entities/something.xml'>
- <!ENTITY % paramEntity 'name|name|name'>
- %paramEntity;
- <!-- comment -->
-]"""
- self._run_check("<!%s>" % inside, [
- ("decl", inside),
- ])
-
def test_bad_nesting(self):
# Strangely, this *is* supposed to test that overlapping
# elements are allowed. HTMLParser is more geared toward
@@ -247,6 +229,30 @@
self._parse_error("<a foo='>'")
self._parse_error("<a foo='>")
+ def test_valid_doctypes(self):
+ # from http://www.w3.org/QA/2002/04/valid-dtd-list.html
+ dtds = ['HTML', # HTML5 doctype
+ ('HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
+ '"http://www.w3.org/TR/html4/strict.dtd"'),
+ ('HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" '
+ '"http://www.w3.org/TR/html4/loose.dtd"'),
+ ('html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" '
+ '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"'),
+ ('html PUBLIC "-//W3C//DTD XHTML 1.0 Frameset//EN" '
+ '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd"'),
+ ('math PUBLIC "-//W3C//DTD MathML 2.0//EN" '
+ '"http://www.w3.org/Math/DTD/mathml2/mathml2.dtd"'),
+ ('html PUBLIC "-//W3C//DTD '
+ 'XHTML 1.1 plus MathML 2.0 plus SVG 1.1//EN" '
+ '"http://www.w3.org/2002/04/xhtml-math-svg/xhtml-math-svg.dtd"'),
+ ('svg PUBLIC "-//W3C//DTD SVG 1.1//EN" '
+ '"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"'),
+ 'html PUBLIC "-//IETF//DTD HTML 2.0//EN"',
+ 'html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"']
+ for dtd in dtds:
+ self._run_check("<!DOCTYPE %s>" % dtd,
+ [('decl', 'DOCTYPE ' + dtd)])
+
def test_declaration_junk_chars(self):
self._parse_error("<!DOCTYPE foo $ >")
@@ -384,8 +390,7 @@
self._run_check("<a foo='>", [('data', "<a foo='>")])
def test_declaration_junk_chars(self):
- # XXX this is wrong
- self._run_check("<!DOCTYPE foo $ >", [('comment', 'DOCTYPE foo $ ')])
+ self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')])
def test_illegal_declarations(self):
# XXX this might be wrong
@@ -510,11 +515,14 @@
html = ('<! not really a comment >'
'<! not a comment either -->'
'<! -- close enough -->'
+ '<!><!<-- this was an empty comment>'
'<!!! another bogus comment !!!>')
expected = [
('comment', ' not really a comment '),
('comment', ' not a comment either --'),
('comment', ' -- close enough --'),
+ ('comment', ''),
+ ('comment', '<-- this was an empty comment'),
('comment', '!! another bogus comment !!!'),
]
self._run_check(html, expected)
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list