| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223 |
- """Tests to ensure that the html5lib tree builder generates good trees."""
- import warnings
- try:
- from bs4.builder import HTML5TreeBuilder
- HTML5LIB_PRESENT = True
- except ImportError as e:
- HTML5LIB_PRESENT = False
- from bs4.element import SoupStrainer
- from . import (
- HTML5TreeBuilderSmokeTest,
- SoupTest,
- skipIf,
- )
- @skipIf(
- not HTML5LIB_PRESENT,
- "html5lib seems not to be present, not testing its tree builder.")
- class TestHTML5LibBuilder(SoupTest, HTML5TreeBuilderSmokeTest):
- """See ``HTML5TreeBuilderSmokeTest``."""
- @property
- def default_builder(self):
- return HTML5TreeBuilder
- def test_soupstrainer(self):
- # The html5lib tree builder does not support SoupStrainers.
- strainer = SoupStrainer("b")
- markup = "<p>A <b>bold</b> statement.</p>"
- with warnings.catch_warnings(record=True) as w:
- soup = self.soup(markup, parse_only=strainer)
- assert soup.decode() == self.document_for(markup)
- assert "the html5lib tree builder doesn't support parse_only" in str(w[0].message)
- def test_correctly_nested_tables(self):
- """html5lib inserts <tbody> tags where other parsers don't."""
- markup = ('<table id="1">'
- '<tr>'
- "<td>Here's another table:"
- '<table id="2">'
- '<tr><td>foo</td></tr>'
- '</table></td>')
- self.assert_soup(
- markup,
- '<table id="1"><tbody><tr><td>Here\'s another table:'
- '<table id="2"><tbody><tr><td>foo</td></tr></tbody></table>'
- '</td></tr></tbody></table>')
- self.assert_soup(
- "<table><thead><tr><td>Foo</td></tr></thead>"
- "<tbody><tr><td>Bar</td></tr></tbody>"
- "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
- def test_xml_declaration_followed_by_doctype(self):
- markup = '''<?xml version="1.0" encoding="utf-8"?>
- <!DOCTYPE html>
- <html>
- <head>
- </head>
- <body>
- <p>foo</p>
- </body>
- </html>'''
- soup = self.soup(markup)
- # Verify that we can reach the <p> tag; this means the tree is connected.
- assert b"<p>foo</p>" == soup.p.encode()
- def test_reparented_markup(self):
- markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>'
- soup = self.soup(markup)
- assert "<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p></body>" == soup.body.decode()
- assert 2 == len(soup.find_all('p'))
- def test_reparented_markup_ends_with_whitespace(self):
- markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>\n'
- soup = self.soup(markup)
- assert "<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p>\n</body>" == soup.body.decode()
- assert 2 == len(soup.find_all('p'))
- def test_reparented_markup_containing_identical_whitespace_nodes(self):
- """Verify that we keep the two whitespace nodes in this
- document distinct when reparenting the adjacent <tbody> tags.
- """
- markup = '<table> <tbody><tbody><ims></tbody> </table>'
- soup = self.soup(markup)
- space1, space2 = soup.find_all(string=' ')
- tbody1, tbody2 = soup.find_all('tbody')
- assert space1.next_element is tbody1
- assert tbody2.next_element is space2
- def test_reparented_markup_containing_children(self):
- markup = '<div><a>aftermath<p><noscript>target</noscript>aftermath</a></p></div>'
- soup = self.soup(markup)
- noscript = soup.noscript
- assert "target" == noscript.next_element
- target = soup.find(string='target')
- # The 'aftermath' string was duplicated; we want the second one.
- final_aftermath = soup.find_all(string='aftermath')[-1]
- # The <noscript> tag was moved beneath a copy of the <a> tag,
- # but the 'target' string within is still connected to the
- # (second) 'aftermath' string.
- assert final_aftermath == target.next_element
- assert target == final_aftermath.previous_element
-
- def test_processing_instruction(self):
- """Processing instructions become comments."""
- markup = b"""<?PITarget PIContent?>"""
- soup = self.soup(markup)
- assert str(soup).startswith("<!--?PITarget PIContent?-->")
- def test_cloned_multivalue_node(self):
- markup = b"""<a class="my_class"><p></a>"""
- soup = self.soup(markup)
- a1, a2 = soup.find_all('a')
- assert a1 == a2
- assert a1 is not a2
- def test_foster_parenting(self):
- markup = b"""<table><td></tbody>A"""
- soup = self.soup(markup)
- assert "<body>A<table><tbody><tr><td></td></tr></tbody></table></body>" == soup.body.decode()
- def test_extraction(self):
- """
- Test that extraction does not destroy the tree.
- https://bugs.launchpad.net/beautifulsoup/+bug/1782928
- """
- markup = """
- <html><head></head>
- <style>
- </style><script></script><body><p>hello</p></body></html>
- """
- soup = self.soup(markup)
- [s.extract() for s in soup('script')]
- [s.extract() for s in soup('style')]
- assert len(soup.find_all("p")) == 1
- def test_empty_comment(self):
- """
- Test that empty comment does not break structure.
- https://bugs.launchpad.net/beautifulsoup/+bug/1806598
- """
- markup = """
- <html>
- <body>
- <form>
- <!----><input type="text">
- </form>
- </body>
- </html>
- """
- soup = self.soup(markup)
- inputs = []
- for form in soup.find_all('form'):
- inputs.extend(form.find_all('input'))
- assert len(inputs) == 1
- def test_tracking_line_numbers(self):
- # The html.parser TreeBuilder keeps track of line number and
- # position of each element.
- markup = "\n <p>\n\n<sourceline>\n<b>text</b></sourceline><sourcepos></p>"
- soup = self.soup(markup)
- assert 2 == soup.p.sourceline
- assert 5 == soup.p.sourcepos
- assert "sourceline" == soup.p.find('sourceline').name
- # You can deactivate this behavior.
- soup = self.soup(markup, store_line_numbers=False)
- assert "sourceline" == soup.p.sourceline.name
- assert "sourcepos" == soup.p.sourcepos.name
- def test_special_string_containers(self):
- # The html5lib tree builder doesn't support this standard feature,
- # because there's no way of knowing, when a string is created,
- # where in the tree it will eventually end up.
- pass
- def test_html5_attributes(self):
- # The html5lib TreeBuilder can convert any entity named in
- # the HTML5 spec to a sequence of Unicode characters, and
- # convert those Unicode characters to a (potentially
- # different) named entity on the way out.
- #
- # This is a copy of the same test from
- # HTMLParserTreeBuilderSmokeTest. It's not in the superclass
- # because the lxml HTML TreeBuilder _doesn't_ work this way.
- for input_element, output_unicode, output_element in (
- ("⇄", '\u21c4', b'⇄'),
- ('⊧', '\u22a7', b'⊧'),
- ('𝔑', '\U0001d511', b'𝔑'),
- ('≧̸', '\u2267\u0338', b'≧̸'),
- ('¬', '\xac', b'¬'),
- ('⫬', '\u2aec', b'⫬'),
- ('"', '"', b'"'),
- ('∴', '\u2234', b'∴'),
- ('∴', '\u2234', b'∴'),
- ('∴', '\u2234', b'∴'),
- ("fj", 'fj', b'fj'),
- ("⊔", '\u2294', b'⊔'),
- ("⊔︀", '\u2294\ufe00', b'⊔︀'),
- ("'", "'", b"'"),
- ("|", "|", b"|"),
- ):
- markup = '<div>%s</div>' % input_element
- div = self.soup(markup).div
- without_element = div.encode()
- expect = b"<div>%s</div>" % output_unicode.encode("utf8")
- assert without_element == expect
- with_element = div.encode(formatter="html")
- expect = b"<div>%s</div>" % output_element
- assert with_element == expect
|
