mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Add support for line numbers to the HTML 5 parser
This commit is contained in:
Kovid Goyal
parent
0d1c917281
commit
9503652a4b
@ -256,8 +256,9 @@ class TreeBuilder(BaseTreeBuilder):
|
||||
documentClass = Document
|
||||
doctypeClass = DocType
|
||||
|
||||
def __init__(self, namespaceHTMLElements=True):
|
||||
def __init__(self, namespaceHTMLElements=True, linenumber_attribute=None):
|
||||
BaseTreeBuilder.__init__(self, namespaceHTMLElements)
|
||||
self.linenumber_attribute = linenumber_attribute
|
||||
self.lxml_context = create_lxml_context()
|
||||
self.elementClass = partial(ElementFactory, context=self.lxml_context)
|
||||
self.proxy_cache = []
|
||||
@ -304,6 +305,20 @@ class TreeBuilder(BaseTreeBuilder):
|
||||
elem.name = token_name
|
||||
elem.namespace = elem.nsmap[elem.prefix]
|
||||
elem.nameTuple = (elem.nsmap[elem.prefix], elem.name)
|
||||
position = token.get('position', None)
|
||||
if position is not None:
|
||||
# Unfortunately, libxml2 can only store line numbers upto 65535
|
||||
# (unsigned short). If you really need to workaround this, use the
|
||||
# patch here:
|
||||
# https://bug325533.bugzilla-attachments.gnome.org/attachment.cgi?id=56951
|
||||
# (replacing int with size_t) and patching lxml correspondingly to
|
||||
# get rid of the OverflowError
|
||||
try:
|
||||
elem.sourceline = position[0][0]
|
||||
except OverflowError:
|
||||
elem.sourceline = 65535
|
||||
if self.linenumber_attribute is not None:
|
||||
elem.set(self.linenumber_attribute, str(position[0][0]))
|
||||
return elem
|
||||
|
||||
def insertElementNormal(self, token):
|
||||
@ -367,8 +382,9 @@ def process_namespace_free_attribs(attrs):
|
||||
|
||||
class NoNamespaceTreeBuilder(TreeBuilder):
|
||||
|
||||
def __init__(self, namespaceHTMLElements=False):
|
||||
def __init__(self, namespaceHTMLElements=False, linenumber_attribute=None):
|
||||
BaseTreeBuilder.__init__(self, namespaceHTMLElements)
|
||||
self.linenumber_attribute = linenumber_attribute
|
||||
self.lxml_context = create_lxml_context()
|
||||
self.elementClass = partial(ElementFactory, context=self.lxml_context)
|
||||
self.proxy_cache = []
|
||||
@ -387,6 +403,14 @@ class NoNamespaceTreeBuilder(TreeBuilder):
|
||||
elem.name = elem.tag
|
||||
elem.namespace = token.get('namespace', self.defaultNamespace)
|
||||
elem.nameTuple = (elem.namespace or html_ns, elem.name)
|
||||
position = token.get('position', None)
|
||||
if position is not None:
|
||||
try:
|
||||
elem.sourceline = position[0][0]
|
||||
except OverflowError:
|
||||
elem.sourceline = 65535
|
||||
if self.linenumber_attribute is not None:
|
||||
elem.set(self.linenumber_attribute, str(position[0][0]))
|
||||
return elem
|
||||
|
||||
def apply_html_attributes(self, attrs):
|
||||
@ -401,6 +425,7 @@ class NoNamespaceTreeBuilder(TreeBuilder):
|
||||
except ValueError:
|
||||
html.set(to_xml_name(k), v)
|
||||
|
||||
# Input Stream {{{
|
||||
_regex_cache = {}
|
||||
|
||||
class FastStream(object):
|
||||
@ -414,7 +439,7 @@ class FastStream(object):
|
||||
self.charEncoding = ("utf-8", "certain")
|
||||
self.track_position = track_position
|
||||
if track_position:
|
||||
self.new_lines = tuple(m.start() for m in re.finditer(r'\n', raw))
|
||||
self.new_lines = tuple(m.start() + 1 for m in re.finditer(r'\n', raw))
|
||||
|
||||
def reset(self):
|
||||
self.pos = 0
|
||||
@ -451,17 +476,24 @@ class FastStream(object):
|
||||
def position(self):
|
||||
if not self.track_position:
|
||||
return (-1, -1)
|
||||
lnum = bisect(self.new_lines, self.pos)
|
||||
if lnum == 0:
|
||||
return (1, self.pos)
|
||||
return (lnum, self.pos - self.new_lines[lnum - 1])
|
||||
pos = self.pos
|
||||
lnum = bisect(self.new_lines, pos)
|
||||
# lnum is the line from which the next char() will come, therefore the
|
||||
# current char is a \n and \n is given the line number of the line it
|
||||
# creates.
|
||||
try:
|
||||
offset = self.new_lines[lnum - 1] - pos
|
||||
except IndexError:
|
||||
offset = pos
|
||||
return (lnum + 1, offset)
|
||||
# }}}
|
||||
|
||||
if len("\U0010FFFF") == 1: # UCS4 build
|
||||
replace_chars = re.compile("[\uD800-\uDFFF]")
|
||||
else:
|
||||
replace_chars = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])")
|
||||
|
||||
def parse(raw, decoder=None, log=None, discard_namespaces=False, line_numbers=True):
|
||||
def parse(raw, decoder=None, log=None, discard_namespaces=False, line_numbers=True, linenumber_attribute=None):
|
||||
if isinstance(raw, bytes):
|
||||
raw = xml_to_unicode(raw)[0] if decoder is None else decoder(raw)
|
||||
raw = fix_self_closing_cdata_tags(raw) # TODO: Handle this in the parser
|
||||
@ -471,10 +503,10 @@ def parse(raw, decoder=None, log=None, discard_namespaces=False, line_numbers=Tr
|
||||
|
||||
stream_class = partial(FastStream, track_position=line_numbers)
|
||||
stream = stream_class(raw)
|
||||
builder = NoNamespaceTreeBuilder if discard_namespaces else TreeBuilder
|
||||
builder = partial(NoNamespaceTreeBuilder if discard_namespaces else TreeBuilder, linenumber_attribute=linenumber_attribute)
|
||||
while True:
|
||||
try:
|
||||
parser = HTMLParser(tree=builder, namespaceHTMLElements=not discard_namespaces)
|
||||
parser = HTMLParser(tree=builder, track_positions=line_numbers, namespaceHTMLElements=not discard_namespaces)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', category=DataLossWarning)
|
||||
try:
|
||||
@ -495,8 +527,8 @@ def parse(raw, decoder=None, log=None, discard_namespaces=False, line_numbers=Tr
|
||||
|
||||
if __name__ == '__main__':
|
||||
from lxml import etree
|
||||
# root = parse('\n<html><head><title>a\n</title><p> \n<b>b', discard_namespaces=False)
|
||||
root = parse('\n<html><p><svg viewbox="0 0 0 0"><image xlink:href="xxx"/><b></svg> \n<b>xxx', discard_namespaces=False)
|
||||
root = parse('\n<html><head><title>a\n</title><p> \n<b>b', discard_namespaces=False)
|
||||
# root = parse('\n<html><p><svg viewbox="0 0 0 0"><image xlink:href="xxx"/><b></svg> \n<b>xxx', discard_namespaces=False)
|
||||
print (etree.tostring(root, encoding='utf-8'))
|
||||
print()
|
||||
|
||||
|
||||
@ -141,4 +141,13 @@ class ParsingTests(BaseTest):
|
||||
test(self, parse)
|
||||
|
||||
root = parse('<html><p><svg><image /><b></svg> \n<b>xxx', discard_namespaces=True)
|
||||
self.assertTrue(root.xpath('//b'), 'Namespaces not discarded')
|
||||
self.assertFalse(root.xpath('//svg/b'), 'The <b> was not moved out of <svg>')
|
||||
|
||||
for ds in (False, True):
|
||||
src = '\n<html>\n<p>\n<svg><image />\n<b></svg> '
|
||||
root = parse(src, discard_namespaces=ds)
|
||||
for tag, lnum in {'html':2, 'head':3, 'body':3, 'p':3, 'svg':4, 'image':4, 'b':5}.iteritems():
|
||||
elem = root.xpath('//*[local-name()="%s"]' % tag)[0]
|
||||
self.assertEqual(lnum, elem.sourceline, 'Line number incorrect for %s, source: %s:' % (tag, src))
|
||||
|
||||
|
||||
@ -37,6 +37,7 @@ def parseFragment(doc, container="div", treebuilder="etree", encoding=None,
|
||||
|
||||
def method_decorator_metaclass(function):
|
||||
class Decorated(type):
|
||||
|
||||
def __new__(meta, classname, bases, classDict):
|
||||
for attributeName, attribute in classDict.items():
|
||||
if isinstance(attribute, types.FunctionType):
|
||||
@ -48,11 +49,12 @@ def method_decorator_metaclass(function):
|
||||
|
||||
|
||||
class HTMLParser(object):
|
||||
|
||||
"""HTML parser. Generates a tree structure from a stream of (possibly
|
||||
malformed) HTML"""
|
||||
|
||||
def __init__(self, tree=None, tokenizer=tokenizer.HTMLTokenizer,
|
||||
strict=False, namespaceHTMLElements=True, debug=False):
|
||||
strict=False, namespaceHTMLElements=True, debug=False, track_positions=False):
|
||||
"""
|
||||
strict - raise an exception when a parse error is encountered
|
||||
|
||||
@ -67,6 +69,7 @@ class HTMLParser(object):
|
||||
|
||||
# Raise an exception on the first error encountered
|
||||
self.strict = strict
|
||||
self.track_positions = track_positions
|
||||
|
||||
if tree is None:
|
||||
tree = treebuilders.getTreeBuilder("etree")
|
||||
@ -85,6 +88,7 @@ class HTMLParser(object):
|
||||
self.tokenizer = self.tokenizer_class(stream, encoding=encoding,
|
||||
parseMeta=parseMeta,
|
||||
useChardet=useChardet,
|
||||
track_positions=self.track_positions,
|
||||
parser=self, **kwargs)
|
||||
self.reset()
|
||||
|
||||
@ -406,6 +410,15 @@ class HTMLParser(object):
|
||||
|
||||
self.phase = self.phases["text"]
|
||||
|
||||
def impliedTagToken(self, name, type="EndTag", attributes=None,
|
||||
selfClosing=False):
|
||||
if attributes is None:
|
||||
attributes = {}
|
||||
ans = {"type": tokenTypes[type], "name": name, "data": attributes,
|
||||
"selfClosing": selfClosing}
|
||||
if self.track_positions:
|
||||
ans['position'] = (self.tokenizer.stream.position(), True)
|
||||
return ans
|
||||
|
||||
def getPhases(debug):
|
||||
def log(function):
|
||||
@ -440,12 +453,14 @@ def getPhases(debug):
|
||||
return type
|
||||
|
||||
class Phase(with_metaclass(getMetaclass(debug, log))):
|
||||
|
||||
"""Base class for helper object that implements each phase of processing
|
||||
"""
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
self.parser = parser
|
||||
self.tree = tree
|
||||
self.impliedTagToken = parser.impliedTagToken
|
||||
|
||||
def processEOF(self):
|
||||
raise NotImplementedError
|
||||
@ -479,6 +494,7 @@ def getPhases(debug):
|
||||
return self.endTagHandler[token["name"]](token)
|
||||
|
||||
class InitialPhase(Phase):
|
||||
|
||||
def processSpaceCharacters(self, token):
|
||||
pass
|
||||
|
||||
@ -609,8 +625,9 @@ def getPhases(debug):
|
||||
|
||||
class BeforeHtmlPhase(Phase):
|
||||
# helper methods
|
||||
|
||||
def insertHtmlElement(self):
|
||||
self.tree.insertRoot(impliedTagToken("html", "StartTag"))
|
||||
self.tree.insertRoot(self.impliedTagToken("html", "StartTag"))
|
||||
self.parser.phase = self.parser.phases["beforeHead"]
|
||||
|
||||
# other
|
||||
@ -643,6 +660,7 @@ def getPhases(debug):
|
||||
return token
|
||||
|
||||
class BeforeHeadPhase(Phase):
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -658,14 +676,14 @@ def getPhases(debug):
|
||||
self.endTagHandler.default = self.endTagOther
|
||||
|
||||
def processEOF(self):
|
||||
self.startTagHead(impliedTagToken("head", "StartTag"))
|
||||
self.startTagHead(self.impliedTagToken("head", "StartTag"))
|
||||
return True
|
||||
|
||||
def processSpaceCharacters(self, token):
|
||||
pass
|
||||
|
||||
def processCharacters(self, token):
|
||||
self.startTagHead(impliedTagToken("head", "StartTag"))
|
||||
self.startTagHead(self.impliedTagToken("head", "StartTag"))
|
||||
return token
|
||||
|
||||
def startTagHtml(self, token):
|
||||
@ -677,11 +695,11 @@ def getPhases(debug):
|
||||
self.parser.phase = self.parser.phases["inHead"]
|
||||
|
||||
def startTagOther(self, token):
|
||||
self.startTagHead(impliedTagToken("head", "StartTag"))
|
||||
self.startTagHead(self.impliedTagToken("head", "StartTag"))
|
||||
return token
|
||||
|
||||
def endTagImplyHead(self, token):
|
||||
self.startTagHead(impliedTagToken("head", "StartTag"))
|
||||
self.startTagHead(self.impliedTagToken("head", "StartTag"))
|
||||
return token
|
||||
|
||||
def endTagOther(self, token):
|
||||
@ -689,6 +707,7 @@ def getPhases(debug):
|
||||
{"name": token["name"]})
|
||||
|
||||
class InHeadPhase(Phase):
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -781,13 +800,14 @@ def getPhases(debug):
|
||||
self.parser.parseError("unexpected-end-tag", {"name": token["name"]})
|
||||
|
||||
def anythingElse(self):
|
||||
self.endTagHead(impliedTagToken("head"))
|
||||
self.endTagHead(self.impliedTagToken("head"))
|
||||
|
||||
# XXX If we implement a parser for which scripting is disabled we need to
|
||||
# implement this phase.
|
||||
#
|
||||
# class InHeadNoScriptPhase(Phase):
|
||||
class AfterHeadPhase(Phase):
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -850,13 +870,14 @@ def getPhases(debug):
|
||||
self.parser.parseError("unexpected-end-tag", {"name": token["name"]})
|
||||
|
||||
def anythingElse(self):
|
||||
self.tree.insertElement(impliedTagToken("body", "StartTag"))
|
||||
self.tree.insertElement(self.impliedTagToken("body", "StartTag"))
|
||||
self.parser.phase = self.parser.phases["inBody"]
|
||||
self.parser.framesetOK = True
|
||||
|
||||
class InBodyPhase(Phase):
|
||||
# http://www.whatwg.org/specs/web-apps/current-work/#parsing-main-inbody
|
||||
# the really-really-really-very crazy mode
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -1027,12 +1048,12 @@ def getPhases(debug):
|
||||
|
||||
def startTagCloseP(self, token):
|
||||
if self.tree.elementInScope("p", variant="button"):
|
||||
self.endTagP(impliedTagToken("p"))
|
||||
self.endTagP(self.impliedTagToken("p"))
|
||||
self.tree.insertElement(token)
|
||||
|
||||
def startTagPreListing(self, token):
|
||||
if self.tree.elementInScope("p", variant="button"):
|
||||
self.endTagP(impliedTagToken("p"))
|
||||
self.endTagP(self.impliedTagToken("p"))
|
||||
self.tree.insertElement(token)
|
||||
self.parser.framesetOK = False
|
||||
self.processSpaceCharacters = self.processSpaceCharactersDropNewline
|
||||
@ -1042,7 +1063,7 @@ def getPhases(debug):
|
||||
self.parser.parseError("unexpected-start-tag", {"name": "form"})
|
||||
else:
|
||||
if self.tree.elementInScope("p", variant="button"):
|
||||
self.endTagP(impliedTagToken("p"))
|
||||
self.endTagP(self.impliedTagToken("p"))
|
||||
self.tree.insertElement(token)
|
||||
self.tree.formPointer = self.tree.openElements[-1]
|
||||
|
||||
@ -1056,7 +1077,7 @@ def getPhases(debug):
|
||||
for node in reversed(self.tree.openElements):
|
||||
if node.name in stopNames:
|
||||
self.parser.phase.processEndTag(
|
||||
impliedTagToken(node.name, "EndTag"))
|
||||
self.impliedTagToken(node.name, "EndTag"))
|
||||
break
|
||||
if (node.nameTuple in specialElements and
|
||||
node.name not in ("address", "div", "p")):
|
||||
@ -1064,19 +1085,19 @@ def getPhases(debug):
|
||||
|
||||
if self.tree.elementInScope("p", variant="button"):
|
||||
self.parser.phase.processEndTag(
|
||||
impliedTagToken("p", "EndTag"))
|
||||
self.impliedTagToken("p", "EndTag"))
|
||||
|
||||
self.tree.insertElement(token)
|
||||
|
||||
def startTagPlaintext(self, token):
|
||||
if self.tree.elementInScope("p", variant="button"):
|
||||
self.endTagP(impliedTagToken("p"))
|
||||
self.endTagP(self.impliedTagToken("p"))
|
||||
self.tree.insertElement(token)
|
||||
self.parser.tokenizer.state = self.parser.tokenizer.plaintextState
|
||||
|
||||
def startTagHeading(self, token):
|
||||
if self.tree.elementInScope("p", variant="button"):
|
||||
self.endTagP(impliedTagToken("p"))
|
||||
self.endTagP(self.impliedTagToken("p"))
|
||||
if self.tree.openElements[-1].name in headingElements:
|
||||
self.parser.parseError("unexpected-start-tag", {"name": token["name"]})
|
||||
self.tree.openElements.pop()
|
||||
@ -1087,7 +1108,7 @@ def getPhases(debug):
|
||||
if afeAElement is not False:
|
||||
self.parser.parseError("unexpected-start-tag-implies-end-tag",
|
||||
{"startName": "a", "endName": "a"})
|
||||
self.endTagFormatting(impliedTagToken("a"))
|
||||
self.endTagFormatting(self.impliedTagToken("a"))
|
||||
if afeAElement in self.tree.openElements:
|
||||
self.tree.openElements.remove(afeAElement)
|
||||
if afeAElement in self.tree.activeFormattingElements:
|
||||
@ -1104,7 +1125,7 @@ def getPhases(debug):
|
||||
if self.tree.elementInScope("nobr"):
|
||||
self.parser.parseError("unexpected-start-tag-implies-end-tag",
|
||||
{"startName": "nobr", "endName": "nobr"})
|
||||
self.processEndTag(impliedTagToken("nobr"))
|
||||
self.processEndTag(self.impliedTagToken("nobr"))
|
||||
# XXX Need tests that trigger the following
|
||||
self.tree.reconstructActiveFormattingElements()
|
||||
self.addFormattingElement(token)
|
||||
@ -1113,7 +1134,7 @@ def getPhases(debug):
|
||||
if self.tree.elementInScope("button"):
|
||||
self.parser.parseError("unexpected-start-tag-implies-end-tag",
|
||||
{"startName": "button", "endName": "button"})
|
||||
self.processEndTag(impliedTagToken("button"))
|
||||
self.processEndTag(self.impliedTagToken("button"))
|
||||
return token
|
||||
else:
|
||||
self.tree.reconstructActiveFormattingElements()
|
||||
@ -1128,7 +1149,7 @@ def getPhases(debug):
|
||||
|
||||
def startTagXmp(self, token):
|
||||
if self.tree.elementInScope("p", variant="button"):
|
||||
self.endTagP(impliedTagToken("p"))
|
||||
self.endTagP(self.impliedTagToken("p"))
|
||||
self.tree.reconstructActiveFormattingElements()
|
||||
self.parser.framesetOK = False
|
||||
self.parser.parseRCDataRawtext(token, "RAWTEXT")
|
||||
@ -1136,7 +1157,7 @@ def getPhases(debug):
|
||||
def startTagTable(self, token):
|
||||
if self.parser.compatMode != "quirks":
|
||||
if self.tree.elementInScope("p", variant="button"):
|
||||
self.processEndTag(impliedTagToken("p"))
|
||||
self.processEndTag(self.impliedTagToken("p"))
|
||||
self.tree.insertElement(token)
|
||||
self.parser.framesetOK = False
|
||||
self.parser.phase = self.parser.phases["inTable"]
|
||||
@ -1163,7 +1184,7 @@ def getPhases(debug):
|
||||
|
||||
def startTagHr(self, token):
|
||||
if self.tree.elementInScope("p", variant="button"):
|
||||
self.endTagP(impliedTagToken("p"))
|
||||
self.endTagP(self.impliedTagToken("p"))
|
||||
self.tree.insertElement(token)
|
||||
self.tree.openElements.pop()
|
||||
token["selfClosingAcknowledged"] = True
|
||||
@ -1173,7 +1194,7 @@ def getPhases(debug):
|
||||
# No really...
|
||||
self.parser.parseError("unexpected-start-tag-treated-as",
|
||||
{"originalName": "image", "newName": "img"})
|
||||
self.processStartTag(impliedTagToken("img", "StartTag",
|
||||
self.processStartTag(self.impliedTagToken("img", "StartTag",
|
||||
attributes=token["data"],
|
||||
selfClosing=token["selfClosing"]))
|
||||
|
||||
@ -1184,10 +1205,10 @@ def getPhases(debug):
|
||||
form_attrs = {}
|
||||
if "action" in token["data"]:
|
||||
form_attrs["action"] = token["data"]["action"]
|
||||
self.processStartTag(impliedTagToken("form", "StartTag",
|
||||
self.processStartTag(self.impliedTagToken("form", "StartTag",
|
||||
attributes=form_attrs))
|
||||
self.processStartTag(impliedTagToken("hr", "StartTag"))
|
||||
self.processStartTag(impliedTagToken("label", "StartTag"))
|
||||
self.processStartTag(self.impliedTagToken("hr", "StartTag"))
|
||||
self.processStartTag(self.impliedTagToken("label", "StartTag"))
|
||||
# XXX Localization ...
|
||||
if "prompt" in token["data"]:
|
||||
prompt = token["data"]["prompt"]
|
||||
@ -1201,13 +1222,13 @@ def getPhases(debug):
|
||||
if "prompt" in attributes:
|
||||
del attributes["prompt"]
|
||||
attributes["name"] = "isindex"
|
||||
self.processStartTag(impliedTagToken("input", "StartTag",
|
||||
self.processStartTag(self.impliedTagToken("input", "StartTag",
|
||||
attributes=attributes,
|
||||
selfClosing=
|
||||
token["selfClosing"]))
|
||||
self.processEndTag(impliedTagToken("label"))
|
||||
self.processStartTag(impliedTagToken("hr", "StartTag"))
|
||||
self.processEndTag(impliedTagToken("form"))
|
||||
self.processEndTag(self.impliedTagToken("label"))
|
||||
self.processStartTag(self.impliedTagToken("hr", "StartTag"))
|
||||
self.processEndTag(self.impliedTagToken("form"))
|
||||
|
||||
def startTagTextarea(self, token):
|
||||
self.tree.insertElement(token)
|
||||
@ -1225,7 +1246,7 @@ def getPhases(debug):
|
||||
|
||||
def startTagOpt(self, token):
|
||||
if self.tree.openElements[-1].name == "option":
|
||||
self.parser.phase.processEndTag(impliedTagToken("option"))
|
||||
self.parser.phase.processEndTag(self.impliedTagToken("option"))
|
||||
self.tree.reconstructActiveFormattingElements()
|
||||
self.parser.tree.insertElement(token)
|
||||
|
||||
@ -1289,9 +1310,9 @@ def getPhases(debug):
|
||||
|
||||
def endTagP(self, token):
|
||||
if not self.tree.elementInScope("p", variant="button"):
|
||||
self.startTagCloseP(impliedTagToken("p", "StartTag"))
|
||||
self.startTagCloseP(self.impliedTagToken("p", "StartTag"))
|
||||
self.parser.parseError("unexpected-end-tag", {"name": "p"})
|
||||
self.endTagP(impliedTagToken("p", "EndTag"))
|
||||
self.endTagP(self.impliedTagToken("p", "EndTag"))
|
||||
else:
|
||||
self.tree.generateImpliedEndTags("p")
|
||||
if self.tree.openElements[-1].name != "p":
|
||||
@ -1321,7 +1342,7 @@ def getPhases(debug):
|
||||
def endTagHtml(self, token):
|
||||
# We repeat the test for the body end tag token being ignored here
|
||||
if self.tree.elementInScope("body"):
|
||||
self.endTagBody(impliedTagToken("body"))
|
||||
self.endTagBody(self.impliedTagToken("body"))
|
||||
return token
|
||||
|
||||
def endTagBlock(self, token):
|
||||
@ -1562,7 +1583,7 @@ def getPhases(debug):
|
||||
self.parser.parseError("unexpected-end-tag-treated-as",
|
||||
{"originalName": "br", "newName": "br element"})
|
||||
self.tree.reconstructActiveFormattingElements()
|
||||
self.tree.insertElement(impliedTagToken("br", "StartTag"))
|
||||
self.tree.insertElement(self.impliedTagToken("br", "StartTag"))
|
||||
self.tree.openElements.pop()
|
||||
|
||||
def endTagOther(self, token):
|
||||
@ -1580,6 +1601,7 @@ def getPhases(debug):
|
||||
break
|
||||
|
||||
class TextPhase(Phase):
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
self.startTagHandler = utils.MethodDispatcher([])
|
||||
@ -1614,6 +1636,7 @@ def getPhases(debug):
|
||||
|
||||
class InTablePhase(Phase):
|
||||
# http://www.whatwg.org/specs/web-apps/current-work/#in-table
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
self.startTagHandler = utils.MethodDispatcher([
|
||||
@ -1685,7 +1708,7 @@ def getPhases(debug):
|
||||
self.parser.phase = self.parser.phases["inColumnGroup"]
|
||||
|
||||
def startTagCol(self, token):
|
||||
self.startTagColgroup(impliedTagToken("colgroup", "StartTag"))
|
||||
self.startTagColgroup(self.impliedTagToken("colgroup", "StartTag"))
|
||||
return token
|
||||
|
||||
def startTagRowGroup(self, token):
|
||||
@ -1694,13 +1717,13 @@ def getPhases(debug):
|
||||
self.parser.phase = self.parser.phases["inTableBody"]
|
||||
|
||||
def startTagImplyTbody(self, token):
|
||||
self.startTagRowGroup(impliedTagToken("tbody", "StartTag"))
|
||||
self.startTagRowGroup(self.impliedTagToken("tbody", "StartTag"))
|
||||
return token
|
||||
|
||||
def startTagTable(self, token):
|
||||
self.parser.parseError("unexpected-start-tag-implies-end-tag",
|
||||
{"startName": "table", "endName": "table"})
|
||||
self.parser.phase.processEndTag(impliedTagToken("table"))
|
||||
self.parser.phase.processEndTag(self.impliedTagToken("table"))
|
||||
if not self.parser.innerHTML:
|
||||
return token
|
||||
|
||||
@ -1758,6 +1781,7 @@ def getPhases(debug):
|
||||
self.tree.insertFromTable = False
|
||||
|
||||
class InTableTextPhase(Phase):
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
self.originalPhase = None
|
||||
@ -1804,6 +1828,7 @@ def getPhases(debug):
|
||||
|
||||
class InCaptionPhase(Phase):
|
||||
# http://www.whatwg.org/specs/web-apps/current-work/#in-caption
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -1835,7 +1860,7 @@ def getPhases(debug):
|
||||
self.parser.parseError()
|
||||
# XXX Have to duplicate logic here to find out if the tag is ignored
|
||||
ignoreEndTag = self.ignoreEndTagCaption()
|
||||
self.parser.phase.processEndTag(impliedTagToken("caption"))
|
||||
self.parser.phase.processEndTag(self.impliedTagToken("caption"))
|
||||
if not ignoreEndTag:
|
||||
return token
|
||||
|
||||
@ -1863,7 +1888,7 @@ def getPhases(debug):
|
||||
def endTagTable(self, token):
|
||||
self.parser.parseError()
|
||||
ignoreEndTag = self.ignoreEndTagCaption()
|
||||
self.parser.phase.processEndTag(impliedTagToken("caption"))
|
||||
self.parser.phase.processEndTag(self.impliedTagToken("caption"))
|
||||
if not ignoreEndTag:
|
||||
return token
|
||||
|
||||
@ -1900,13 +1925,13 @@ def getPhases(debug):
|
||||
return
|
||||
else:
|
||||
ignoreEndTag = self.ignoreEndTagColgroup()
|
||||
self.endTagColgroup(impliedTagToken("colgroup"))
|
||||
self.endTagColgroup(self.impliedTagToken("colgroup"))
|
||||
if not ignoreEndTag:
|
||||
return True
|
||||
|
||||
def processCharacters(self, token):
|
||||
ignoreEndTag = self.ignoreEndTagColgroup()
|
||||
self.endTagColgroup(impliedTagToken("colgroup"))
|
||||
self.endTagColgroup(self.impliedTagToken("colgroup"))
|
||||
if not ignoreEndTag:
|
||||
return token
|
||||
|
||||
@ -1916,7 +1941,7 @@ def getPhases(debug):
|
||||
|
||||
def startTagOther(self, token):
|
||||
ignoreEndTag = self.ignoreEndTagColgroup()
|
||||
self.endTagColgroup(impliedTagToken("colgroup"))
|
||||
self.endTagColgroup(self.impliedTagToken("colgroup"))
|
||||
if not ignoreEndTag:
|
||||
return token
|
||||
|
||||
@ -1934,12 +1959,13 @@ def getPhases(debug):
|
||||
|
||||
def endTagOther(self, token):
|
||||
ignoreEndTag = self.ignoreEndTagColgroup()
|
||||
self.endTagColgroup(impliedTagToken("colgroup"))
|
||||
self.endTagColgroup(self.impliedTagToken("colgroup"))
|
||||
if not ignoreEndTag:
|
||||
return token
|
||||
|
||||
class InTableBodyPhase(Phase):
|
||||
# http://www.whatwg.org/specs/web-apps/current-work/#in-table0
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
self.startTagHandler = utils.MethodDispatcher([
|
||||
@ -1987,7 +2013,7 @@ def getPhases(debug):
|
||||
def startTagTableCell(self, token):
|
||||
self.parser.parseError("unexpected-cell-in-table-body",
|
||||
{"name": token["name"]})
|
||||
self.startTagTr(impliedTagToken("tr", "StartTag"))
|
||||
self.startTagTr(self.impliedTagToken("tr", "StartTag"))
|
||||
return token
|
||||
|
||||
def startTagTableOther(self, token):
|
||||
@ -1997,7 +2023,7 @@ def getPhases(debug):
|
||||
self.tree.elementInScope("tfoot", variant="table")):
|
||||
self.clearStackToTableBodyContext()
|
||||
self.endTagTableRowGroup(
|
||||
impliedTagToken(self.tree.openElements[-1].name))
|
||||
self.impliedTagToken(self.tree.openElements[-1].name))
|
||||
return token
|
||||
else:
|
||||
# innerHTML case
|
||||
@ -2022,7 +2048,7 @@ def getPhases(debug):
|
||||
self.tree.elementInScope("tfoot", variant="table")):
|
||||
self.clearStackToTableBodyContext()
|
||||
self.endTagTableRowGroup(
|
||||
impliedTagToken(self.tree.openElements[-1].name))
|
||||
self.impliedTagToken(self.tree.openElements[-1].name))
|
||||
return token
|
||||
else:
|
||||
# innerHTML case
|
||||
@ -2038,6 +2064,7 @@ def getPhases(debug):
|
||||
|
||||
class InRowPhase(Phase):
|
||||
# http://www.whatwg.org/specs/web-apps/current-work/#in-row
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
self.startTagHandler = utils.MethodDispatcher([
|
||||
@ -2085,7 +2112,7 @@ def getPhases(debug):
|
||||
|
||||
def startTagTableOther(self, token):
|
||||
ignoreEndTag = self.ignoreEndTagTr()
|
||||
self.endTagTr(impliedTagToken("tr"))
|
||||
self.endTagTr(self.impliedTagToken("tr"))
|
||||
# XXX how are we sure it's always ignored in the innerHTML case?
|
||||
if not ignoreEndTag:
|
||||
return token
|
||||
@ -2105,7 +2132,7 @@ def getPhases(debug):
|
||||
|
||||
def endTagTable(self, token):
|
||||
ignoreEndTag = self.ignoreEndTagTr()
|
||||
self.endTagTr(impliedTagToken("tr"))
|
||||
self.endTagTr(self.impliedTagToken("tr"))
|
||||
# Reprocess the current tag if the tr end tag was not ignored
|
||||
# XXX how are we sure it's always ignored in the innerHTML case?
|
||||
if not ignoreEndTag:
|
||||
@ -2113,7 +2140,7 @@ def getPhases(debug):
|
||||
|
||||
def endTagTableRowGroup(self, token):
|
||||
if self.tree.elementInScope(token["name"], variant="table"):
|
||||
self.endTagTr(impliedTagToken("tr"))
|
||||
self.endTagTr(self.impliedTagToken("tr"))
|
||||
return token
|
||||
else:
|
||||
self.parser.parseError()
|
||||
@ -2127,6 +2154,7 @@ def getPhases(debug):
|
||||
|
||||
class InCellPhase(Phase):
|
||||
# http://www.whatwg.org/specs/web-apps/current-work/#in-cell
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
self.startTagHandler = utils.MethodDispatcher([
|
||||
@ -2146,9 +2174,9 @@ def getPhases(debug):
|
||||
# helper
|
||||
def closeCell(self):
|
||||
if self.tree.elementInScope("td", variant="table"):
|
||||
self.endTagTableCell(impliedTagToken("td"))
|
||||
self.endTagTableCell(self.impliedTagToken("td"))
|
||||
elif self.tree.elementInScope("th", variant="table"):
|
||||
self.endTagTableCell(impliedTagToken("th"))
|
||||
self.endTagTableCell(self.impliedTagToken("th"))
|
||||
|
||||
# the rest
|
||||
def processEOF(self):
|
||||
@ -2202,6 +2230,7 @@ def getPhases(debug):
|
||||
return self.parser.phases["inBody"].processEndTag(token)
|
||||
|
||||
class InSelectPhase(Phase):
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -2249,12 +2278,12 @@ def getPhases(debug):
|
||||
|
||||
def startTagSelect(self, token):
|
||||
self.parser.parseError("unexpected-select-in-select")
|
||||
self.endTagSelect(impliedTagToken("select"))
|
||||
self.endTagSelect(self.impliedTagToken("select"))
|
||||
|
||||
def startTagInput(self, token):
|
||||
self.parser.parseError("unexpected-input-in-select")
|
||||
if self.tree.elementInScope("select", variant="select"):
|
||||
self.endTagSelect(impliedTagToken("select"))
|
||||
self.endTagSelect(self.impliedTagToken("select"))
|
||||
return token
|
||||
else:
|
||||
assert self.parser.innerHTML
|
||||
@ -2302,6 +2331,7 @@ def getPhases(debug):
|
||||
{"name": token["name"]})
|
||||
|
||||
class InSelectInTablePhase(Phase):
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -2325,7 +2355,7 @@ def getPhases(debug):
|
||||
|
||||
def startTagTable(self, token):
|
||||
self.parser.parseError("unexpected-table-element-start-tag-in-select-in-table", {"name": token["name"]})
|
||||
self.endTagOther(impliedTagToken("select"))
|
||||
self.endTagOther(self.impliedTagToken("select"))
|
||||
return token
|
||||
|
||||
def startTagOther(self, token):
|
||||
@ -2334,7 +2364,7 @@ def getPhases(debug):
|
||||
def endTagTable(self, token):
|
||||
self.parser.parseError("unexpected-table-element-end-tag-in-select-in-table", {"name": token["name"]})
|
||||
if self.tree.elementInScope(token["name"], variant="table"):
|
||||
self.endTagOther(impliedTagToken("select"))
|
||||
self.endTagOther(self.impliedTagToken("select"))
|
||||
return token
|
||||
|
||||
def endTagOther(self, token):
|
||||
@ -2456,6 +2486,7 @@ def getPhases(debug):
|
||||
return new_token
|
||||
|
||||
class AfterBodyPhase(Phase):
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -2504,6 +2535,7 @@ def getPhases(debug):
|
||||
|
||||
class InFramesetPhase(Phase):
|
||||
# http://www.whatwg.org/specs/web-apps/current-work/#in-frameset
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -2561,6 +2593,7 @@ def getPhases(debug):
|
||||
|
||||
class AfterFramesetPhase(Phase):
|
||||
# http://www.whatwg.org/specs/web-apps/current-work/#after3
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -2597,6 +2630,7 @@ def getPhases(debug):
|
||||
{"name": token["name"]})
|
||||
|
||||
class AfterAfterBodyPhase(Phase):
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -2635,6 +2669,7 @@ def getPhases(debug):
|
||||
return token
|
||||
|
||||
class AfterAfterFramesetPhase(Phase):
|
||||
|
||||
def __init__(self, parser, tree):
|
||||
Phase.__init__(self, parser, tree)
|
||||
|
||||
@ -2698,14 +2733,7 @@ def getPhases(debug):
|
||||
}
|
||||
|
||||
|
||||
def impliedTagToken(name, type="EndTag", attributes=None,
|
||||
selfClosing=False):
|
||||
if attributes is None:
|
||||
attributes = {}
|
||||
return {"type": tokenTypes[type], "name": name, "data": attributes,
|
||||
"selfClosing": selfClosing}
|
||||
|
||||
|
||||
class ParseError(Exception):
|
||||
|
||||
"""Error in parsed document"""
|
||||
pass
|
||||
|
||||
@ -35,10 +35,11 @@ class HTMLTokenizer(object):
|
||||
"""
|
||||
|
||||
def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True,
|
||||
lowercaseElementName=True, lowercaseAttrName=True, parser=None):
|
||||
lowercaseElementName=True, lowercaseAttrName=True, parser=None, track_positions=False):
|
||||
|
||||
self.stream = HTMLInputStream(stream, encoding, parseMeta, useChardet)
|
||||
self.parser = parser
|
||||
self.track_positions = track_positions
|
||||
|
||||
# Perform case conversions?
|
||||
self.lowercaseElementName = lowercaseElementName
|
||||
@ -378,6 +379,8 @@ class HTMLTokenizer(object):
|
||||
"name": data, "data": [],
|
||||
"selfClosing": False,
|
||||
"selfClosingAcknowledged": False}
|
||||
if self.track_positions:
|
||||
self.currentToken['position'] = (self.stream.position(), False)
|
||||
self.state = self.tagNameState
|
||||
elif data == ">":
|
||||
# XXX In theory it could be something besides a tag name. But
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user