dammit.py 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095
  1. # -*- coding: utf-8 -*-
  2. """Beautiful Soup bonus library: Unicode, Dammit
  3. This library converts a bytestream to Unicode through any means
  4. necessary. It is heavily based on code from Mark Pilgrim's Universal
  5. Feed Parser. It works best on XML and HTML, but it does not rewrite the
  6. XML or HTML to reflect a new encoding; that's the tree builder's job.
  7. """
  8. # Use of this source code is governed by the MIT license.
  9. __license__ = "MIT"
  10. from html.entities import codepoint2name
  11. from collections import defaultdict
  12. import codecs
  13. import re
  14. import logging
  15. import string
  16. # Import a library to autodetect character encodings. We'll support
  17. # any of a number of libraries that all support the same API:
  18. #
  19. # * cchardet
  20. # * chardet
  21. # * charset-normalizer
  22. chardet_module = None
  23. try:
  24. # PyPI package: cchardet
  25. import cchardet as chardet_module
  26. except ImportError:
  27. try:
  28. # Debian package: python-chardet
  29. # PyPI package: chardet
  30. import chardet as chardet_module
  31. except ImportError:
  32. try:
  33. # PyPI package: charset-normalizer
  34. import charset_normalizer as chardet_module
  35. except ImportError:
  36. # No chardet available.
  37. chardet_module = None
  38. if chardet_module:
  39. def chardet_dammit(s):
  40. if isinstance(s, str):
  41. return None
  42. return chardet_module.detect(s)['encoding']
  43. else:
  44. def chardet_dammit(s):
  45. return None
  46. # Build bytestring and Unicode versions of regular expressions for finding
  47. # a declared encoding inside an XML or HTML document.
  48. xml_encoding = '^\\s*<\\?.*encoding=[\'"](.*?)[\'"].*\\?>'
  49. html_meta = '<\\s*meta[^>]+charset\\s*=\\s*["\']?([^>]*?)[ /;\'">]'
  50. encoding_res = dict()
  51. encoding_res[bytes] = {
  52. 'html' : re.compile(html_meta.encode("ascii"), re.I),
  53. 'xml' : re.compile(xml_encoding.encode("ascii"), re.I),
  54. }
  55. encoding_res[str] = {
  56. 'html' : re.compile(html_meta, re.I),
  57. 'xml' : re.compile(xml_encoding, re.I)
  58. }
  59. from html.entities import html5
  60. class EntitySubstitution(object):
  61. """The ability to substitute XML or HTML entities for certain characters."""
  62. def _populate_class_variables():
  63. """Initialize variables used by this class to manage the plethora of
  64. HTML5 named entities.
  65. This function returns a 3-tuple containing two dictionaries
  66. and a regular expression:
  67. unicode_to_name - A mapping of Unicode strings like "⦨" to
  68. entity names like "angmsdaa". When a single Unicode string has
  69. multiple entity names, we try to choose the most commonly-used
  70. name.
  71. name_to_unicode: A mapping of entity names like "angmsdaa" to
  72. Unicode strings like "⦨".
  73. named_entity_re: A regular expression matching (almost) any
  74. Unicode string that corresponds to an HTML5 named entity.
  75. """
  76. unicode_to_name = {}
  77. name_to_unicode = {}
  78. short_entities = set()
  79. long_entities_by_first_character = defaultdict(set)
  80. for name_with_semicolon, character in sorted(html5.items()):
  81. # "It is intentional, for legacy compatibility, that many
  82. # code points have multiple character reference names. For
  83. # example, some appear both with and without the trailing
  84. # semicolon, or with different capitalizations."
  85. # - https://html.spec.whatwg.org/multipage/named-characters.html#named-character-references
  86. #
  87. # The parsers are in charge of handling (or not) character
  88. # references with no trailing semicolon, so we remove the
  89. # semicolon whenever it appears.
  90. if name_with_semicolon.endswith(';'):
  91. name = name_with_semicolon[:-1]
  92. else:
  93. name = name_with_semicolon
  94. # When parsing HTML, we want to recognize any known named
  95. # entity and convert it to a sequence of Unicode
  96. # characters.
  97. if name not in name_to_unicode:
  98. name_to_unicode[name] = character
  99. # When _generating_ HTML, we want to recognize special
  100. # character sequences that _could_ be converted to named
  101. # entities.
  102. unicode_to_name[character] = name
  103. # We also need to build a regular expression that lets us
  104. # _find_ those characters in output strings so we can
  105. # replace them.
  106. #
  107. # This is tricky, for two reasons.
  108. if (len(character) == 1 and ord(character) < 128
  109. and character not in '<>&'):
  110. # First, it would be annoying to turn single ASCII
  111. # characters like | into named entities like
  112. # &verbar;. The exceptions are <>&, which we _must_
  113. # turn into named entities to produce valid HTML.
  114. continue
  115. if len(character) > 1 and all(ord(x) < 128 for x in character):
  116. # We also do not want to turn _combinations_ of ASCII
  117. # characters like 'fj' into named entities like '&fjlig;',
  118. # though that's more debateable.
  119. continue
  120. # Second, some named entities have a Unicode value that's
  121. # a subset of the Unicode value for some _other_ named
  122. # entity. As an example, \u2267' is &GreaterFullEqual;,
  123. # but '\u2267\u0338' is &NotGreaterFullEqual;. Our regular
  124. # expression needs to match the first two characters of
  125. # "\u2267\u0338foo", but only the first character of
  126. # "\u2267foo".
  127. #
  128. # In this step, we build two sets of characters that
  129. # _eventually_ need to go into the regular expression. But
  130. # we won't know exactly what the regular expression needs
  131. # to look like until we've gone through the entire list of
  132. # named entities.
  133. if len(character) == 1:
  134. short_entities.add(character)
  135. else:
  136. long_entities_by_first_character[character[0]].add(character)
  137. # Now that we've been through the entire list of entities, we
  138. # can create a regular expression that matches any of them.
  139. particles = set()
  140. for short in short_entities:
  141. long_versions = long_entities_by_first_character[short]
  142. if not long_versions:
  143. particles.add(short)
  144. else:
  145. ignore = "".join([x[1] for x in long_versions])
  146. # This finds, e.g. \u2267 but only if it is _not_
  147. # followed by \u0338.
  148. particles.add("%s(?![%s])" % (short, ignore))
  149. for long_entities in list(long_entities_by_first_character.values()):
  150. for long_entity in long_entities:
  151. particles.add(long_entity)
  152. re_definition = "(%s)" % "|".join(particles)
  153. # If an entity shows up in both html5 and codepoint2name, it's
  154. # likely that HTML5 gives it several different names, such as
  155. # 'rsquo' and 'rsquor'. When converting Unicode characters to
  156. # named entities, the codepoint2name name should take
  157. # precedence where possible, since that's the more easily
  158. # recognizable one.
  159. for codepoint, name in list(codepoint2name.items()):
  160. character = chr(codepoint)
  161. unicode_to_name[character] = name
  162. return unicode_to_name, name_to_unicode, re.compile(re_definition)
  163. (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
  164. CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
  165. CHARACTER_TO_XML_ENTITY = {
  166. "'": "apos",
  167. '"': "quot",
  168. "&": "amp",
  169. "<": "lt",
  170. ">": "gt",
  171. }
  172. BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
  173. "&(?!#\\d+;|#x[0-9a-fA-F]+;|\\w+;)"
  174. ")")
  175. AMPERSAND_OR_BRACKET = re.compile("([<>&])")
  176. @classmethod
  177. def _substitute_html_entity(cls, matchobj):
  178. """Used with a regular expression to substitute the
  179. appropriate HTML entity for a special character string."""
  180. entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
  181. return "&%s;" % entity
  182. @classmethod
  183. def _substitute_xml_entity(cls, matchobj):
  184. """Used with a regular expression to substitute the
  185. appropriate XML entity for a special character string."""
  186. entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
  187. return "&%s;" % entity
  188. @classmethod
  189. def quoted_attribute_value(self, value):
  190. """Make a value into a quoted XML attribute, possibly escaping it.
  191. Most strings will be quoted using double quotes.
  192. Bob's Bar -> "Bob's Bar"
  193. If a string contains double quotes, it will be quoted using
  194. single quotes.
  195. Welcome to "my bar" -> 'Welcome to "my bar"'
  196. If a string contains both single and double quotes, the
  197. double quotes will be escaped, and the string will be quoted
  198. using double quotes.
  199. Welcome to "Bob's Bar" -> "Welcome to &quot;Bob's bar&quot;
  200. """
  201. quote_with = '"'
  202. if '"' in value:
  203. if "'" in value:
  204. # The string contains both single and double
  205. # quotes. Turn the double quotes into
  206. # entities. We quote the double quotes rather than
  207. # the single quotes because the entity name is
  208. # "&quot;" whether this is HTML or XML. If we
  209. # quoted the single quotes, we'd have to decide
  210. # between &apos; and &squot;.
  211. replace_with = "&quot;"
  212. value = value.replace('"', replace_with)
  213. else:
  214. # There are double quotes but no single quotes.
  215. # We can use single quotes to quote the attribute.
  216. quote_with = "'"
  217. return quote_with + value + quote_with
  218. @classmethod
  219. def substitute_xml(cls, value, make_quoted_attribute=False):
  220. """Substitute XML entities for special XML characters.
  221. :param value: A string to be substituted. The less-than sign
  222. will become &lt;, the greater-than sign will become &gt;,
  223. and any ampersands will become &amp;. If you want ampersands
  224. that appear to be part of an entity definition to be left
  225. alone, use substitute_xml_containing_entities() instead.
  226. :param make_quoted_attribute: If True, then the string will be
  227. quoted, as befits an attribute value.
  228. """
  229. # Escape angle brackets and ampersands.
  230. value = cls.AMPERSAND_OR_BRACKET.sub(
  231. cls._substitute_xml_entity, value)
  232. if make_quoted_attribute:
  233. value = cls.quoted_attribute_value(value)
  234. return value
  235. @classmethod
  236. def substitute_xml_containing_entities(
  237. cls, value, make_quoted_attribute=False):
  238. """Substitute XML entities for special XML characters.
  239. :param value: A string to be substituted. The less-than sign will
  240. become &lt;, the greater-than sign will become &gt;, and any
  241. ampersands that are not part of an entity defition will
  242. become &amp;.
  243. :param make_quoted_attribute: If True, then the string will be
  244. quoted, as befits an attribute value.
  245. """
  246. # Escape angle brackets, and ampersands that aren't part of
  247. # entities.
  248. value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
  249. cls._substitute_xml_entity, value)
  250. if make_quoted_attribute:
  251. value = cls.quoted_attribute_value(value)
  252. return value
  253. @classmethod
  254. def substitute_html(cls, s):
  255. """Replace certain Unicode characters with named HTML entities.
  256. This differs from data.encode(encoding, 'xmlcharrefreplace')
  257. in that the goal is to make the result more readable (to those
  258. with ASCII displays) rather than to recover from
  259. errors. There's absolutely nothing wrong with a UTF-8 string
  260. containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
  261. character with "&eacute;" will make it more readable to some
  262. people.
  263. :param s: A Unicode string.
  264. """
  265. return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
  266. cls._substitute_html_entity, s)
  267. class EncodingDetector:
  268. """Suggests a number of possible encodings for a bytestring.
  269. Order of precedence:
  270. 1. Encodings you specifically tell EncodingDetector to try first
  271. (the known_definite_encodings argument to the constructor).
  272. 2. An encoding determined by sniffing the document's byte-order mark.
  273. 3. Encodings you specifically tell EncodingDetector to try if
  274. byte-order mark sniffing fails (the user_encodings argument to the
  275. constructor).
  276. 4. An encoding declared within the bytestring itself, either in an
  277. XML declaration (if the bytestring is to be interpreted as an XML
  278. document), or in a <meta> tag (if the bytestring is to be
  279. interpreted as an HTML document.)
  280. 5. An encoding detected through textual analysis by chardet,
  281. cchardet, or a similar external library.
  282. 4. UTF-8.
  283. 5. Windows-1252.
  284. """
  285. def __init__(self, markup, known_definite_encodings=None,
  286. is_html=False, exclude_encodings=None,
  287. user_encodings=None, override_encodings=None):
  288. """Constructor.
  289. :param markup: Some markup in an unknown encoding.
  290. :param known_definite_encodings: When determining the encoding
  291. of `markup`, these encodings will be tried first, in
  292. order. In HTML terms, this corresponds to the "known
  293. definite encoding" step defined here:
  294. https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding
  295. :param user_encodings: These encodings will be tried after the
  296. `known_definite_encodings` have been tried and failed, and
  297. after an attempt to sniff the encoding by looking at a
  298. byte order mark has failed. In HTML terms, this
  299. corresponds to the step "user has explicitly instructed
  300. the user agent to override the document's character
  301. encoding", defined here:
  302. https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
  303. :param override_encodings: A deprecated alias for
  304. known_definite_encodings. Any encodings here will be tried
  305. immediately after the encodings in
  306. known_definite_encodings.
  307. :param is_html: If True, this markup is considered to be
  308. HTML. Otherwise it's assumed to be XML.
  309. :param exclude_encodings: These encodings will not be tried,
  310. even if they otherwise would be.
  311. """
  312. self.known_definite_encodings = list(known_definite_encodings or [])
  313. if override_encodings:
  314. self.known_definite_encodings += override_encodings
  315. self.user_encodings = user_encodings or []
  316. exclude_encodings = exclude_encodings or []
  317. self.exclude_encodings = set([x.lower() for x in exclude_encodings])
  318. self.chardet_encoding = None
  319. self.is_html = is_html
  320. self.declared_encoding = None
  321. # First order of business: strip a byte-order mark.
  322. self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup)
  323. def _usable(self, encoding, tried):
  324. """Should we even bother to try this encoding?
  325. :param encoding: Name of an encoding.
  326. :param tried: Encodings that have already been tried. This will be modified
  327. as a side effect.
  328. """
  329. if encoding is not None:
  330. encoding = encoding.lower()
  331. if encoding in self.exclude_encodings:
  332. return False
  333. if encoding not in tried:
  334. tried.add(encoding)
  335. return True
  336. return False
  337. @property
  338. def encodings(self):
  339. """Yield a number of encodings that might work for this markup.
  340. :yield: A sequence of strings.
  341. """
  342. tried = set()
  343. # First, try the known definite encodings
  344. for e in self.known_definite_encodings:
  345. if self._usable(e, tried):
  346. yield e
  347. # Did the document originally start with a byte-order mark
  348. # that indicated its encoding?
  349. if self._usable(self.sniffed_encoding, tried):
  350. yield self.sniffed_encoding
  351. # Sniffing the byte-order mark did nothing; try the user
  352. # encodings.
  353. for e in self.user_encodings:
  354. if self._usable(e, tried):
  355. yield e
  356. # Look within the document for an XML or HTML encoding
  357. # declaration.
  358. if self.declared_encoding is None:
  359. self.declared_encoding = self.find_declared_encoding(
  360. self.markup, self.is_html)
  361. if self._usable(self.declared_encoding, tried):
  362. yield self.declared_encoding
  363. # Use third-party character set detection to guess at the
  364. # encoding.
  365. if self.chardet_encoding is None:
  366. self.chardet_encoding = chardet_dammit(self.markup)
  367. if self._usable(self.chardet_encoding, tried):
  368. yield self.chardet_encoding
  369. # As a last-ditch effort, try utf-8 and windows-1252.
  370. for e in ('utf-8', 'windows-1252'):
  371. if self._usable(e, tried):
  372. yield e
  373. @classmethod
  374. def strip_byte_order_mark(cls, data):
  375. """If a byte-order mark is present, strip it and return the encoding it implies.
  376. :param data: Some markup.
  377. :return: A 2-tuple (modified data, implied encoding)
  378. """
  379. encoding = None
  380. if isinstance(data, str):
  381. # Unicode data cannot have a byte-order mark.
  382. return data, encoding
  383. if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \
  384. and (data[2:4] != '\x00\x00'):
  385. encoding = 'utf-16be'
  386. data = data[2:]
  387. elif (len(data) >= 4) and (data[:2] == b'\xff\xfe') \
  388. and (data[2:4] != '\x00\x00'):
  389. encoding = 'utf-16le'
  390. data = data[2:]
  391. elif data[:3] == b'\xef\xbb\xbf':
  392. encoding = 'utf-8'
  393. data = data[3:]
  394. elif data[:4] == b'\x00\x00\xfe\xff':
  395. encoding = 'utf-32be'
  396. data = data[4:]
  397. elif data[:4] == b'\xff\xfe\x00\x00':
  398. encoding = 'utf-32le'
  399. data = data[4:]
  400. return data, encoding
  401. @classmethod
  402. def find_declared_encoding(cls, markup, is_html=False, search_entire_document=False):
  403. """Given a document, tries to find its declared encoding.
  404. An XML encoding is declared at the beginning of the document.
  405. An HTML encoding is declared in a <meta> tag, hopefully near the
  406. beginning of the document.
  407. :param markup: Some markup.
  408. :param is_html: If True, this markup is considered to be HTML. Otherwise
  409. it's assumed to be XML.
  410. :param search_entire_document: Since an encoding is supposed to declared near the beginning
  411. of the document, most of the time it's only necessary to search a few kilobytes of data.
  412. Set this to True to force this method to search the entire document.
  413. """
  414. if search_entire_document:
  415. xml_endpos = html_endpos = len(markup)
  416. else:
  417. xml_endpos = 1024
  418. html_endpos = max(2048, int(len(markup) * 0.05))
  419. if isinstance(markup, bytes):
  420. res = encoding_res[bytes]
  421. else:
  422. res = encoding_res[str]
  423. xml_re = res['xml']
  424. html_re = res['html']
  425. declared_encoding = None
  426. declared_encoding_match = xml_re.search(markup, endpos=xml_endpos)
  427. if not declared_encoding_match and is_html:
  428. declared_encoding_match = html_re.search(markup, endpos=html_endpos)
  429. if declared_encoding_match is not None:
  430. declared_encoding = declared_encoding_match.groups()[0]
  431. if declared_encoding:
  432. if isinstance(declared_encoding, bytes):
  433. declared_encoding = declared_encoding.decode('ascii', 'replace')
  434. return declared_encoding.lower()
  435. return None
  436. class UnicodeDammit:
  437. """A class for detecting the encoding of a *ML document and
  438. converting it to a Unicode string. If the source encoding is
  439. windows-1252, can replace MS smart quotes with their HTML or XML
  440. equivalents."""
  441. # This dictionary maps commonly seen values for "charset" in HTML
  442. # meta tags to the corresponding Python codec names. It only covers
  443. # values that aren't in Python's aliases and can't be determined
  444. # by the heuristics in find_codec.
  445. CHARSET_ALIASES = {"macintosh": "mac-roman",
  446. "x-sjis": "shift-jis"}
  447. ENCODINGS_WITH_SMART_QUOTES = [
  448. "windows-1252",
  449. "iso-8859-1",
  450. "iso-8859-2",
  451. ]
  452. def __init__(self, markup, known_definite_encodings=[],
  453. smart_quotes_to=None, is_html=False, exclude_encodings=[],
  454. user_encodings=None, override_encodings=None
  455. ):
  456. """Constructor.
  457. :param markup: A bytestring representing markup in an unknown encoding.
  458. :param known_definite_encodings: When determining the encoding
  459. of `markup`, these encodings will be tried first, in
  460. order. In HTML terms, this corresponds to the "known
  461. definite encoding" step defined here:
  462. https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding
  463. :param user_encodings: These encodings will be tried after the
  464. `known_definite_encodings` have been tried and failed, and
  465. after an attempt to sniff the encoding by looking at a
  466. byte order mark has failed. In HTML terms, this
  467. corresponds to the step "user has explicitly instructed
  468. the user agent to override the document's character
  469. encoding", defined here:
  470. https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
  471. :param override_encodings: A deprecated alias for
  472. known_definite_encodings. Any encodings here will be tried
  473. immediately after the encodings in
  474. known_definite_encodings.
  475. :param smart_quotes_to: By default, Microsoft smart quotes will, like all other characters, be converted
  476. to Unicode characters. Setting this to 'ascii' will convert them to ASCII quotes instead.
  477. Setting it to 'xml' will convert them to XML entity references, and setting it to 'html'
  478. will convert them to HTML entity references.
  479. :param is_html: If True, this markup is considered to be HTML. Otherwise
  480. it's assumed to be XML.
  481. :param exclude_encodings: These encodings will not be considered, even
  482. if the sniffing code thinks they might make sense.
  483. """
  484. self.smart_quotes_to = smart_quotes_to
  485. self.tried_encodings = []
  486. self.contains_replacement_characters = False
  487. self.is_html = is_html
  488. self.log = logging.getLogger(__name__)
  489. self.detector = EncodingDetector(
  490. markup, known_definite_encodings, is_html, exclude_encodings,
  491. user_encodings, override_encodings
  492. )
  493. # Short-circuit if the data is in Unicode to begin with.
  494. if isinstance(markup, str) or markup == '':
  495. self.markup = markup
  496. self.unicode_markup = str(markup)
  497. self.original_encoding = None
  498. return
  499. # The encoding detector may have stripped a byte-order mark.
  500. # Use the stripped markup from this point on.
  501. self.markup = self.detector.markup
  502. u = None
  503. for encoding in self.detector.encodings:
  504. markup = self.detector.markup
  505. u = self._convert_from(encoding)
  506. if u is not None:
  507. break
  508. if not u:
  509. # None of the encodings worked. As an absolute last resort,
  510. # try them again with character replacement.
  511. for encoding in self.detector.encodings:
  512. if encoding != "ascii":
  513. u = self._convert_from(encoding, "replace")
  514. if u is not None:
  515. self.log.warning(
  516. "Some characters could not be decoded, and were "
  517. "replaced with REPLACEMENT CHARACTER."
  518. )
  519. self.contains_replacement_characters = True
  520. break
  521. # If none of that worked, we could at this point force it to
  522. # ASCII, but that would destroy so much data that I think
  523. # giving up is better.
  524. self.unicode_markup = u
  525. if not u:
  526. self.original_encoding = None
  527. def _sub_ms_char(self, match):
  528. """Changes a MS smart quote character to an XML or HTML
  529. entity, or an ASCII character."""
  530. orig = match.group(1)
  531. if self.smart_quotes_to == 'ascii':
  532. sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
  533. else:
  534. sub = self.MS_CHARS.get(orig)
  535. if type(sub) == tuple:
  536. if self.smart_quotes_to == 'xml':
  537. sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
  538. else:
  539. sub = '&'.encode() + sub[0].encode() + ';'.encode()
  540. else:
  541. sub = sub.encode()
  542. return sub
  543. def _convert_from(self, proposed, errors="strict"):
  544. """Attempt to convert the markup to the proposed encoding.
  545. :param proposed: The name of a character encoding.
  546. """
  547. proposed = self.find_codec(proposed)
  548. if not proposed or (proposed, errors) in self.tried_encodings:
  549. return None
  550. self.tried_encodings.append((proposed, errors))
  551. markup = self.markup
  552. # Convert smart quotes to HTML if coming from an encoding
  553. # that might have them.
  554. if (self.smart_quotes_to is not None
  555. and proposed in self.ENCODINGS_WITH_SMART_QUOTES):
  556. smart_quotes_re = b"([\x80-\x9f])"
  557. smart_quotes_compiled = re.compile(smart_quotes_re)
  558. markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
  559. try:
  560. #print("Trying to convert document to %s (errors=%s)" % (
  561. # proposed, errors))
  562. u = self._to_unicode(markup, proposed, errors)
  563. self.markup = u
  564. self.original_encoding = proposed
  565. except Exception as e:
  566. #print("That didn't work!")
  567. #print(e)
  568. return None
  569. #print("Correct encoding: %s" % proposed)
  570. return self.markup
  571. def _to_unicode(self, data, encoding, errors="strict"):
  572. """Given a string and its encoding, decodes the string into Unicode.
  573. :param encoding: The name of an encoding.
  574. """
  575. return str(data, encoding, errors)
  576. @property
  577. def declared_html_encoding(self):
  578. """If the markup is an HTML document, returns the encoding declared _within_
  579. the document.
  580. """
  581. if not self.is_html:
  582. return None
  583. return self.detector.declared_encoding
  584. def find_codec(self, charset):
  585. """Convert the name of a character set to a codec name.
  586. :param charset: The name of a character set.
  587. :return: The name of a codec.
  588. """
  589. value = (self._codec(self.CHARSET_ALIASES.get(charset, charset))
  590. or (charset and self._codec(charset.replace("-", "")))
  591. or (charset and self._codec(charset.replace("-", "_")))
  592. or (charset and charset.lower())
  593. or charset
  594. )
  595. if value:
  596. return value.lower()
  597. return None
  598. def _codec(self, charset):
  599. if not charset:
  600. return charset
  601. codec = None
  602. try:
  603. codecs.lookup(charset)
  604. codec = charset
  605. except (LookupError, ValueError):
  606. pass
  607. return codec
  608. # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
  609. MS_CHARS = {b'\x80': ('euro', '20AC'),
  610. b'\x81': ' ',
  611. b'\x82': ('sbquo', '201A'),
  612. b'\x83': ('fnof', '192'),
  613. b'\x84': ('bdquo', '201E'),
  614. b'\x85': ('hellip', '2026'),
  615. b'\x86': ('dagger', '2020'),
  616. b'\x87': ('Dagger', '2021'),
  617. b'\x88': ('circ', '2C6'),
  618. b'\x89': ('permil', '2030'),
  619. b'\x8A': ('Scaron', '160'),
  620. b'\x8B': ('lsaquo', '2039'),
  621. b'\x8C': ('OElig', '152'),
  622. b'\x8D': '?',
  623. b'\x8E': ('#x17D', '17D'),
  624. b'\x8F': '?',
  625. b'\x90': '?',
  626. b'\x91': ('lsquo', '2018'),
  627. b'\x92': ('rsquo', '2019'),
  628. b'\x93': ('ldquo', '201C'),
  629. b'\x94': ('rdquo', '201D'),
  630. b'\x95': ('bull', '2022'),
  631. b'\x96': ('ndash', '2013'),
  632. b'\x97': ('mdash', '2014'),
  633. b'\x98': ('tilde', '2DC'),
  634. b'\x99': ('trade', '2122'),
  635. b'\x9a': ('scaron', '161'),
  636. b'\x9b': ('rsaquo', '203A'),
  637. b'\x9c': ('oelig', '153'),
  638. b'\x9d': '?',
  639. b'\x9e': ('#x17E', '17E'),
  640. b'\x9f': ('Yuml', ''),}
  641. # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
  642. # horrors like stripping diacritical marks to turn á into a, but also
  643. # contains non-horrors like turning “ into ".
  644. MS_CHARS_TO_ASCII = {
  645. b'\x80' : 'EUR',
  646. b'\x81' : ' ',
  647. b'\x82' : ',',
  648. b'\x83' : 'f',
  649. b'\x84' : ',,',
  650. b'\x85' : '...',
  651. b'\x86' : '+',
  652. b'\x87' : '++',
  653. b'\x88' : '^',
  654. b'\x89' : '%',
  655. b'\x8a' : 'S',
  656. b'\x8b' : '<',
  657. b'\x8c' : 'OE',
  658. b'\x8d' : '?',
  659. b'\x8e' : 'Z',
  660. b'\x8f' : '?',
  661. b'\x90' : '?',
  662. b'\x91' : "'",
  663. b'\x92' : "'",
  664. b'\x93' : '"',
  665. b'\x94' : '"',
  666. b'\x95' : '*',
  667. b'\x96' : '-',
  668. b'\x97' : '--',
  669. b'\x98' : '~',
  670. b'\x99' : '(TM)',
  671. b'\x9a' : 's',
  672. b'\x9b' : '>',
  673. b'\x9c' : 'oe',
  674. b'\x9d' : '?',
  675. b'\x9e' : 'z',
  676. b'\x9f' : 'Y',
  677. b'\xa0' : ' ',
  678. b'\xa1' : '!',
  679. b'\xa2' : 'c',
  680. b'\xa3' : 'GBP',
  681. b'\xa4' : '$', #This approximation is especially parochial--this is the
  682. #generic currency symbol.
  683. b'\xa5' : 'YEN',
  684. b'\xa6' : '|',
  685. b'\xa7' : 'S',
  686. b'\xa8' : '..',
  687. b'\xa9' : '',
  688. b'\xaa' : '(th)',
  689. b'\xab' : '<<',
  690. b'\xac' : '!',
  691. b'\xad' : ' ',
  692. b'\xae' : '(R)',
  693. b'\xaf' : '-',
  694. b'\xb0' : 'o',
  695. b'\xb1' : '+-',
  696. b'\xb2' : '2',
  697. b'\xb3' : '3',
  698. b'\xb4' : ("'", 'acute'),
  699. b'\xb5' : 'u',
  700. b'\xb6' : 'P',
  701. b'\xb7' : '*',
  702. b'\xb8' : ',',
  703. b'\xb9' : '1',
  704. b'\xba' : '(th)',
  705. b'\xbb' : '>>',
  706. b'\xbc' : '1/4',
  707. b'\xbd' : '1/2',
  708. b'\xbe' : '3/4',
  709. b'\xbf' : '?',
  710. b'\xc0' : 'A',
  711. b'\xc1' : 'A',
  712. b'\xc2' : 'A',
  713. b'\xc3' : 'A',
  714. b'\xc4' : 'A',
  715. b'\xc5' : 'A',
  716. b'\xc6' : 'AE',
  717. b'\xc7' : 'C',
  718. b'\xc8' : 'E',
  719. b'\xc9' : 'E',
  720. b'\xca' : 'E',
  721. b'\xcb' : 'E',
  722. b'\xcc' : 'I',
  723. b'\xcd' : 'I',
  724. b'\xce' : 'I',
  725. b'\xcf' : 'I',
  726. b'\xd0' : 'D',
  727. b'\xd1' : 'N',
  728. b'\xd2' : 'O',
  729. b'\xd3' : 'O',
  730. b'\xd4' : 'O',
  731. b'\xd5' : 'O',
  732. b'\xd6' : 'O',
  733. b'\xd7' : '*',
  734. b'\xd8' : 'O',
  735. b'\xd9' : 'U',
  736. b'\xda' : 'U',
  737. b'\xdb' : 'U',
  738. b'\xdc' : 'U',
  739. b'\xdd' : 'Y',
  740. b'\xde' : 'b',
  741. b'\xdf' : 'B',
  742. b'\xe0' : 'a',
  743. b'\xe1' : 'a',
  744. b'\xe2' : 'a',
  745. b'\xe3' : 'a',
  746. b'\xe4' : 'a',
  747. b'\xe5' : 'a',
  748. b'\xe6' : 'ae',
  749. b'\xe7' : 'c',
  750. b'\xe8' : 'e',
  751. b'\xe9' : 'e',
  752. b'\xea' : 'e',
  753. b'\xeb' : 'e',
  754. b'\xec' : 'i',
  755. b'\xed' : 'i',
  756. b'\xee' : 'i',
  757. b'\xef' : 'i',
  758. b'\xf0' : 'o',
  759. b'\xf1' : 'n',
  760. b'\xf2' : 'o',
  761. b'\xf3' : 'o',
  762. b'\xf4' : 'o',
  763. b'\xf5' : 'o',
  764. b'\xf6' : 'o',
  765. b'\xf7' : '/',
  766. b'\xf8' : 'o',
  767. b'\xf9' : 'u',
  768. b'\xfa' : 'u',
  769. b'\xfb' : 'u',
  770. b'\xfc' : 'u',
  771. b'\xfd' : 'y',
  772. b'\xfe' : 'b',
  773. b'\xff' : 'y',
  774. }
  775. # A map used when removing rogue Windows-1252/ISO-8859-1
  776. # characters in otherwise UTF-8 documents.
  777. #
  778. # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
  779. # Windows-1252.
  780. WINDOWS_1252_TO_UTF8 = {
  781. 0x80 : b'\xe2\x82\xac', # €
  782. 0x82 : b'\xe2\x80\x9a', # ‚
  783. 0x83 : b'\xc6\x92', # ƒ
  784. 0x84 : b'\xe2\x80\x9e', # „
  785. 0x85 : b'\xe2\x80\xa6', # …
  786. 0x86 : b'\xe2\x80\xa0', # †
  787. 0x87 : b'\xe2\x80\xa1', # ‡
  788. 0x88 : b'\xcb\x86', # ˆ
  789. 0x89 : b'\xe2\x80\xb0', # ‰
  790. 0x8a : b'\xc5\xa0', # Š
  791. 0x8b : b'\xe2\x80\xb9', # ‹
  792. 0x8c : b'\xc5\x92', # Œ
  793. 0x8e : b'\xc5\xbd', # Ž
  794. 0x91 : b'\xe2\x80\x98', # ‘
  795. 0x92 : b'\xe2\x80\x99', # ’
  796. 0x93 : b'\xe2\x80\x9c', # “
  797. 0x94 : b'\xe2\x80\x9d', # ”
  798. 0x95 : b'\xe2\x80\xa2', # •
  799. 0x96 : b'\xe2\x80\x93', # –
  800. 0x97 : b'\xe2\x80\x94', # —
  801. 0x98 : b'\xcb\x9c', # ˜
  802. 0x99 : b'\xe2\x84\xa2', # ™
  803. 0x9a : b'\xc5\xa1', # š
  804. 0x9b : b'\xe2\x80\xba', # ›
  805. 0x9c : b'\xc5\x93', # œ
  806. 0x9e : b'\xc5\xbe', # ž
  807. 0x9f : b'\xc5\xb8', # Ÿ
  808. 0xa0 : b'\xc2\xa0', #  
  809. 0xa1 : b'\xc2\xa1', # ¡
  810. 0xa2 : b'\xc2\xa2', # ¢
  811. 0xa3 : b'\xc2\xa3', # £
  812. 0xa4 : b'\xc2\xa4', # ¤
  813. 0xa5 : b'\xc2\xa5', # ¥
  814. 0xa6 : b'\xc2\xa6', # ¦
  815. 0xa7 : b'\xc2\xa7', # §
  816. 0xa8 : b'\xc2\xa8', # ¨
  817. 0xa9 : b'\xc2\xa9', # ©
  818. 0xaa : b'\xc2\xaa', # ª
  819. 0xab : b'\xc2\xab', # «
  820. 0xac : b'\xc2\xac', # ¬
  821. 0xad : b'\xc2\xad', # ­
  822. 0xae : b'\xc2\xae', # ®
  823. 0xaf : b'\xc2\xaf', # ¯
  824. 0xb0 : b'\xc2\xb0', # °
  825. 0xb1 : b'\xc2\xb1', # ±
  826. 0xb2 : b'\xc2\xb2', # ²
  827. 0xb3 : b'\xc2\xb3', # ³
  828. 0xb4 : b'\xc2\xb4', # ´
  829. 0xb5 : b'\xc2\xb5', # µ
  830. 0xb6 : b'\xc2\xb6', # ¶
  831. 0xb7 : b'\xc2\xb7', # ·
  832. 0xb8 : b'\xc2\xb8', # ¸
  833. 0xb9 : b'\xc2\xb9', # ¹
  834. 0xba : b'\xc2\xba', # º
  835. 0xbb : b'\xc2\xbb', # »
  836. 0xbc : b'\xc2\xbc', # ¼
  837. 0xbd : b'\xc2\xbd', # ½
  838. 0xbe : b'\xc2\xbe', # ¾
  839. 0xbf : b'\xc2\xbf', # ¿
  840. 0xc0 : b'\xc3\x80', # À
  841. 0xc1 : b'\xc3\x81', # Á
  842. 0xc2 : b'\xc3\x82', # Â
  843. 0xc3 : b'\xc3\x83', # Ã
  844. 0xc4 : b'\xc3\x84', # Ä
  845. 0xc5 : b'\xc3\x85', # Å
  846. 0xc6 : b'\xc3\x86', # Æ
  847. 0xc7 : b'\xc3\x87', # Ç
  848. 0xc8 : b'\xc3\x88', # È
  849. 0xc9 : b'\xc3\x89', # É
  850. 0xca : b'\xc3\x8a', # Ê
  851. 0xcb : b'\xc3\x8b', # Ë
  852. 0xcc : b'\xc3\x8c', # Ì
  853. 0xcd : b'\xc3\x8d', # Í
  854. 0xce : b'\xc3\x8e', # Î
  855. 0xcf : b'\xc3\x8f', # Ï
  856. 0xd0 : b'\xc3\x90', # Ð
  857. 0xd1 : b'\xc3\x91', # Ñ
  858. 0xd2 : b'\xc3\x92', # Ò
  859. 0xd3 : b'\xc3\x93', # Ó
  860. 0xd4 : b'\xc3\x94', # Ô
  861. 0xd5 : b'\xc3\x95', # Õ
  862. 0xd6 : b'\xc3\x96', # Ö
  863. 0xd7 : b'\xc3\x97', # ×
  864. 0xd8 : b'\xc3\x98', # Ø
  865. 0xd9 : b'\xc3\x99', # Ù
  866. 0xda : b'\xc3\x9a', # Ú
  867. 0xdb : b'\xc3\x9b', # Û
  868. 0xdc : b'\xc3\x9c', # Ü
  869. 0xdd : b'\xc3\x9d', # Ý
  870. 0xde : b'\xc3\x9e', # Þ
  871. 0xdf : b'\xc3\x9f', # ß
  872. 0xe0 : b'\xc3\xa0', # à
  873. 0xe1 : b'\xa1', # á
  874. 0xe2 : b'\xc3\xa2', # â
  875. 0xe3 : b'\xc3\xa3', # ã
  876. 0xe4 : b'\xc3\xa4', # ä
  877. 0xe5 : b'\xc3\xa5', # å
  878. 0xe6 : b'\xc3\xa6', # æ
  879. 0xe7 : b'\xc3\xa7', # ç
  880. 0xe8 : b'\xc3\xa8', # è
  881. 0xe9 : b'\xc3\xa9', # é
  882. 0xea : b'\xc3\xaa', # ê
  883. 0xeb : b'\xc3\xab', # ë
  884. 0xec : b'\xc3\xac', # ì
  885. 0xed : b'\xc3\xad', # í
  886. 0xee : b'\xc3\xae', # î
  887. 0xef : b'\xc3\xaf', # ï
  888. 0xf0 : b'\xc3\xb0', # ð
  889. 0xf1 : b'\xc3\xb1', # ñ
  890. 0xf2 : b'\xc3\xb2', # ò
  891. 0xf3 : b'\xc3\xb3', # ó
  892. 0xf4 : b'\xc3\xb4', # ô
  893. 0xf5 : b'\xc3\xb5', # õ
  894. 0xf6 : b'\xc3\xb6', # ö
  895. 0xf7 : b'\xc3\xb7', # ÷
  896. 0xf8 : b'\xc3\xb8', # ø
  897. 0xf9 : b'\xc3\xb9', # ù
  898. 0xfa : b'\xc3\xba', # ú
  899. 0xfb : b'\xc3\xbb', # û
  900. 0xfc : b'\xc3\xbc', # ü
  901. 0xfd : b'\xc3\xbd', # ý
  902. 0xfe : b'\xc3\xbe', # þ
  903. }
  904. MULTIBYTE_MARKERS_AND_SIZES = [
  905. (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
  906. (0xe0, 0xef, 3), # 3-byte characters start with E0-EF
  907. (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
  908. ]
  909. FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
  910. LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]
  911. @classmethod
  912. def detwingle(cls, in_bytes, main_encoding="utf8",
  913. embedded_encoding="windows-1252"):
  914. """Fix characters from one encoding embedded in some other encoding.
  915. Currently the only situation supported is Windows-1252 (or its
  916. subset ISO-8859-1), embedded in UTF-8.
  917. :param in_bytes: A bytestring that you suspect contains
  918. characters from multiple encodings. Note that this _must_
  919. be a bytestring. If you've already converted the document
  920. to Unicode, you're too late.
  921. :param main_encoding: The primary encoding of `in_bytes`.
  922. :param embedded_encoding: The encoding that was used to embed characters
  923. in the main document.
  924. :return: A bytestring in which `embedded_encoding`
  925. characters have been converted to their `main_encoding`
  926. equivalents.
  927. """
  928. if embedded_encoding.replace('_', '-').lower() not in (
  929. 'windows-1252', 'windows_1252'):
  930. raise NotImplementedError(
  931. "Windows-1252 and ISO-8859-1 are the only currently supported "
  932. "embedded encodings.")
  933. if main_encoding.lower() not in ('utf8', 'utf-8'):
  934. raise NotImplementedError(
  935. "UTF-8 is the only currently supported main encoding.")
  936. byte_chunks = []
  937. chunk_start = 0
  938. pos = 0
  939. while pos < len(in_bytes):
  940. byte = in_bytes[pos]
  941. if not isinstance(byte, int):
  942. # Python 2.x
  943. byte = ord(byte)
  944. if (byte >= cls.FIRST_MULTIBYTE_MARKER
  945. and byte <= cls.LAST_MULTIBYTE_MARKER):
  946. # This is the start of a UTF-8 multibyte character. Skip
  947. # to the end.
  948. for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
  949. if byte >= start and byte <= end:
  950. pos += size
  951. break
  952. elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
  953. # We found a Windows-1252 character!
  954. # Save the string up to this point as a chunk.
  955. byte_chunks.append(in_bytes[chunk_start:pos])
  956. # Now translate the Windows-1252 character into UTF-8
  957. # and add it as another, one-byte chunk.
  958. byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
  959. pos += 1
  960. chunk_start = pos
  961. else:
  962. # Go on to the next character.
  963. pos += 1
  964. if chunk_start == 0:
  965. # The string is unchanged.
  966. return in_bytes
  967. else:
  968. # Store the final chunk.
  969. byte_chunks.append(in_bytes[chunk_start:])
  970. return b''.join(byte_chunks)