123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473 |
- # Use of this source code is governed by the MIT license.
- __license__ = "MIT"
- __all__ = [
- 'HTML5TreeBuilder',
- ]
- import warnings
- import re
- from bs4.builder import (
- DetectsXMLParsedAsHTML,
- PERMISSIVE,
- HTML,
- HTML_5,
- HTMLTreeBuilder,
- )
- from bs4.element import (
- NamespacedAttribute,
- nonwhitespace_re,
- )
- import html5lib
- from html5lib.constants import (
- namespaces,
- prefixes,
- )
- from bs4.element import (
- Comment,
- Doctype,
- NavigableString,
- Tag,
- )
- try:
- # Pre-0.99999999
- from html5lib.treebuilders import _base as treebuilder_base
- new_html5lib = False
- except ImportError as e:
- # 0.99999999 and up
- from html5lib.treebuilders import base as treebuilder_base
- new_html5lib = True
- class HTML5TreeBuilder(HTMLTreeBuilder):
- """Use html5lib to build a tree.
- Note that this TreeBuilder does not support some features common
- to HTML TreeBuilders. Some of these features could theoretically
- be implemented, but at the very least it's quite difficult,
- because html5lib moves the parse tree around as it's being built.
- * This TreeBuilder doesn't use different subclasses of NavigableString
- based on the name of the tag in which the string was found.
- * You can't use a SoupStrainer to parse only part of a document.
- """
- NAME = "html5lib"
- features = [NAME, PERMISSIVE, HTML_5, HTML]
- # html5lib can tell us which line number and position in the
- # original file is the source of an element.
- TRACKS_LINE_NUMBERS = True
-
- def prepare_markup(self, markup, user_specified_encoding,
- document_declared_encoding=None, exclude_encodings=None):
- # Store the user-specified encoding for use later on.
- self.user_specified_encoding = user_specified_encoding
- # document_declared_encoding and exclude_encodings aren't used
- # ATM because the html5lib TreeBuilder doesn't use
- # UnicodeDammit.
- if exclude_encodings:
- warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
- # html5lib only parses HTML, so if it's given XML that's worth
- # noting.
- DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup)
- yield (markup, None, None, False)
- # These methods are defined by Beautiful Soup.
- def feed(self, markup):
- if self.soup.parse_only is not None:
- warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
- parser = html5lib.HTMLParser(tree=self.create_treebuilder)
- self.underlying_builder.parser = parser
- extra_kwargs = dict()
- if not isinstance(markup, str):
- if new_html5lib:
- extra_kwargs['override_encoding'] = self.user_specified_encoding
- else:
- extra_kwargs['encoding'] = self.user_specified_encoding
- doc = parser.parse(markup, **extra_kwargs)
-
- # Set the character encoding detected by the tokenizer.
- if isinstance(markup, str):
- # We need to special-case this because html5lib sets
- # charEncoding to UTF-8 if it gets Unicode input.
- doc.original_encoding = None
- else:
- original_encoding = parser.tokenizer.stream.charEncoding[0]
- if not isinstance(original_encoding, str):
- # In 0.99999999 and up, the encoding is an html5lib
- # Encoding object. We want to use a string for compatibility
- # with other tree builders.
- original_encoding = original_encoding.name
- doc.original_encoding = original_encoding
- self.underlying_builder.parser = None
-
- def create_treebuilder(self, namespaceHTMLElements):
- self.underlying_builder = TreeBuilderForHtml5lib(
- namespaceHTMLElements, self.soup,
- store_line_numbers=self.store_line_numbers
- )
- return self.underlying_builder
- def test_fragment_to_document(self, fragment):
- """See `TreeBuilder`."""
- return '<html><head></head><body>%s</body></html>' % fragment
- class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
-
- def __init__(self, namespaceHTMLElements, soup=None,
- store_line_numbers=True, **kwargs):
- if soup:
- self.soup = soup
- else:
- from bs4 import BeautifulSoup
- # TODO: Why is the parser 'html.parser' here? To avoid an
- # infinite loop?
- self.soup = BeautifulSoup(
- "", "html.parser", store_line_numbers=store_line_numbers,
- **kwargs
- )
- # TODO: What are **kwargs exactly? Should they be passed in
- # here in addition to/instead of being passed to the BeautifulSoup
- # constructor?
- super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
- # This will be set later to an html5lib.html5parser.HTMLParser
- # object, which we can use to track the current line number.
- self.parser = None
- self.store_line_numbers = store_line_numbers
-
- def documentClass(self):
- self.soup.reset()
- return Element(self.soup, self.soup, None)
- def insertDoctype(self, token):
- name = token["name"]
- publicId = token["publicId"]
- systemId = token["systemId"]
- doctype = Doctype.for_name_and_ids(name, publicId, systemId)
- self.soup.object_was_parsed(doctype)
- def elementClass(self, name, namespace):
- kwargs = {}
- if self.parser and self.store_line_numbers:
- # This represents the point immediately after the end of the
- # tag. We don't know when the tag started, but we do know
- # where it ended -- the character just before this one.
- sourceline, sourcepos = self.parser.tokenizer.stream.position()
- kwargs['sourceline'] = sourceline
- kwargs['sourcepos'] = sourcepos-1
- tag = self.soup.new_tag(name, namespace, **kwargs)
- return Element(tag, self.soup, namespace)
- def commentClass(self, data):
- return TextNode(Comment(data), self.soup)
- def fragmentClass(self):
- from bs4 import BeautifulSoup
- # TODO: Why is the parser 'html.parser' here? To avoid an
- # infinite loop?
- self.soup = BeautifulSoup("", "html.parser")
- self.soup.name = "[document_fragment]"
- return Element(self.soup, self.soup, None)
- def appendChild(self, node):
- # XXX This code is not covered by the BS4 tests.
- self.soup.append(node.element)
- def getDocument(self):
- return self.soup
- def getFragment(self):
- return treebuilder_base.TreeBuilder.getFragment(self).element
- def testSerializer(self, element):
- from bs4 import BeautifulSoup
- rv = []
- doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$')
- def serializeElement(element, indent=0):
- if isinstance(element, BeautifulSoup):
- pass
- if isinstance(element, Doctype):
- m = doctype_re.match(element)
- if m:
- name = m.group(1)
- if m.lastindex > 1:
- publicId = m.group(2) or ""
- systemId = m.group(3) or m.group(4) or ""
- rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" %
- (' ' * indent, name, publicId, systemId))
- else:
- rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name))
- else:
- rv.append("|%s<!DOCTYPE >" % (' ' * indent,))
- elif isinstance(element, Comment):
- rv.append("|%s<!-- %s -->" % (' ' * indent, element))
- elif isinstance(element, NavigableString):
- rv.append("|%s\"%s\"" % (' ' * indent, element))
- else:
- if element.namespace:
- name = "%s %s" % (prefixes[element.namespace],
- element.name)
- else:
- name = element.name
- rv.append("|%s<%s>" % (' ' * indent, name))
- if element.attrs:
- attributes = []
- for name, value in list(element.attrs.items()):
- if isinstance(name, NamespacedAttribute):
- name = "%s %s" % (prefixes[name.namespace], name.name)
- if isinstance(value, list):
- value = " ".join(value)
- attributes.append((name, value))
- for name, value in sorted(attributes):
- rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
- indent += 2
- for child in element.children:
- serializeElement(child, indent)
- serializeElement(element, 0)
- return "\n".join(rv)
- class AttrList(object):
- def __init__(self, element):
- self.element = element
- self.attrs = dict(self.element.attrs)
- def __iter__(self):
- return list(self.attrs.items()).__iter__()
- def __setitem__(self, name, value):
- # If this attribute is a multi-valued attribute for this element,
- # turn its value into a list.
- list_attr = self.element.cdata_list_attributes or {}
- if (name in list_attr.get('*')
- or (self.element.name in list_attr
- and name in list_attr[self.element.name])):
- # A node that is being cloned may have already undergone
- # this procedure.
- if not isinstance(value, list):
- value = nonwhitespace_re.findall(value)
- self.element[name] = value
- def items(self):
- return list(self.attrs.items())
- def keys(self):
- return list(self.attrs.keys())
- def __len__(self):
- return len(self.attrs)
- def __getitem__(self, name):
- return self.attrs[name]
- def __contains__(self, name):
- return name in list(self.attrs.keys())
- class Element(treebuilder_base.Node):
- def __init__(self, element, soup, namespace):
- treebuilder_base.Node.__init__(self, element.name)
- self.element = element
- self.soup = soup
- self.namespace = namespace
- def appendChild(self, node):
- string_child = child = None
- if isinstance(node, str):
- # Some other piece of code decided to pass in a string
- # instead of creating a TextElement object to contain the
- # string.
- string_child = child = node
- elif isinstance(node, Tag):
- # Some other piece of code decided to pass in a Tag
- # instead of creating an Element object to contain the
- # Tag.
- child = node
- elif node.element.__class__ == NavigableString:
- string_child = child = node.element
- node.parent = self
- else:
- child = node.element
- node.parent = self
- if not isinstance(child, str) and child.parent is not None:
- node.element.extract()
- if (string_child is not None and self.element.contents
- and self.element.contents[-1].__class__ == NavigableString):
- # We are appending a string onto another string.
- # TODO This has O(n^2) performance, for input like
- # "a</a>a</a>a</a>..."
- old_element = self.element.contents[-1]
- new_element = self.soup.new_string(old_element + string_child)
- old_element.replace_with(new_element)
- self.soup._most_recent_element = new_element
- else:
- if isinstance(node, str):
- # Create a brand new NavigableString from this string.
- child = self.soup.new_string(node)
- # Tell Beautiful Soup to act as if it parsed this element
- # immediately after the parent's last descendant. (Or
- # immediately after the parent, if it has no children.)
- if self.element.contents:
- most_recent_element = self.element._last_descendant(False)
- elif self.element.next_element is not None:
- # Something from further ahead in the parse tree is
- # being inserted into this earlier element. This is
- # very annoying because it means an expensive search
- # for the last element in the tree.
- most_recent_element = self.soup._last_descendant()
- else:
- most_recent_element = self.element
- self.soup.object_was_parsed(
- child, parent=self.element,
- most_recent_element=most_recent_element)
- def getAttributes(self):
- if isinstance(self.element, Comment):
- return {}
- return AttrList(self.element)
- def setAttributes(self, attributes):
- if attributes is not None and len(attributes) > 0:
- converted_attributes = []
- for name, value in list(attributes.items()):
- if isinstance(name, tuple):
- new_name = NamespacedAttribute(*name)
- del attributes[name]
- attributes[new_name] = value
- self.soup.builder._replace_cdata_list_attribute_values(
- self.name, attributes)
- for name, value in list(attributes.items()):
- self.element[name] = value
- # The attributes may contain variables that need substitution.
- # Call set_up_substitutions manually.
- #
- # The Tag constructor called this method when the Tag was created,
- # but we just set/changed the attributes, so call it again.
- self.soup.builder.set_up_substitutions(self.element)
- attributes = property(getAttributes, setAttributes)
- def insertText(self, data, insertBefore=None):
- text = TextNode(self.soup.new_string(data), self.soup)
- if insertBefore:
- self.insertBefore(text, insertBefore)
- else:
- self.appendChild(text)
- def insertBefore(self, node, refNode):
- index = self.element.index(refNode.element)
- if (node.element.__class__ == NavigableString and self.element.contents
- and self.element.contents[index-1].__class__ == NavigableString):
- # (See comments in appendChild)
- old_node = self.element.contents[index-1]
- new_str = self.soup.new_string(old_node + node.element)
- old_node.replace_with(new_str)
- else:
- self.element.insert(index, node.element)
- node.parent = self
- def removeChild(self, node):
- node.element.extract()
- def reparentChildren(self, new_parent):
- """Move all of this tag's children into another tag."""
- # print("MOVE", self.element.contents)
- # print("FROM", self.element)
- # print("TO", new_parent.element)
- element = self.element
- new_parent_element = new_parent.element
- # Determine what this tag's next_element will be once all the children
- # are removed.
- final_next_element = element.next_sibling
- new_parents_last_descendant = new_parent_element._last_descendant(False, False)
- if len(new_parent_element.contents) > 0:
- # The new parent already contains children. We will be
- # appending this tag's children to the end.
- new_parents_last_child = new_parent_element.contents[-1]
- new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
- else:
- # The new parent contains no children.
- new_parents_last_child = None
- new_parents_last_descendant_next_element = new_parent_element.next_element
- to_append = element.contents
- if len(to_append) > 0:
- # Set the first child's previous_element and previous_sibling
- # to elements within the new parent
- first_child = to_append[0]
- if new_parents_last_descendant is not None:
- first_child.previous_element = new_parents_last_descendant
- else:
- first_child.previous_element = new_parent_element
- first_child.previous_sibling = new_parents_last_child
- if new_parents_last_descendant is not None:
- new_parents_last_descendant.next_element = first_child
- else:
- new_parent_element.next_element = first_child
- if new_parents_last_child is not None:
- new_parents_last_child.next_sibling = first_child
- # Find the very last element being moved. It is now the
- # parent's last descendant. It has no .next_sibling and
- # its .next_element is whatever the previous last
- # descendant had.
- last_childs_last_descendant = to_append[-1]._last_descendant(False, True)
- last_childs_last_descendant.next_element = new_parents_last_descendant_next_element
- if new_parents_last_descendant_next_element is not None:
- # TODO: This code has no test coverage and I'm not sure
- # how to get html5lib to go through this path, but it's
- # just the other side of the previous line.
- new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant
- last_childs_last_descendant.next_sibling = None
- for child in to_append:
- child.parent = new_parent_element
- new_parent_element.contents.append(child)
- # Now that this element has no children, change its .next_element.
- element.contents = []
- element.next_element = final_next_element
- # print("DONE WITH MOVE")
- # print("FROM", self.element)
- # print("TO", new_parent_element)
- def cloneNode(self):
- tag = self.soup.new_tag(self.element.name, self.namespace)
- node = Element(tag, self.soup, self.namespace)
- for key,value in self.attributes:
- node.attributes[key] = value
- return node
- def hasContent(self):
- return self.element.contents
- def getNameTuple(self):
- if self.namespace == None:
- return namespaces["html"], self.name
- else:
- return self.namespace, self.name
- nameTuple = property(getNameTuple)
- class TextNode(Element):
- def __init__(self, element, soup):
- treebuilder_base.Node.__init__(self, None)
- self.element = element
- self.soup = soup
- def cloneNode(self):
- raise NotImplementedError
|