From 9a5ad6f64378219eafcddb348ac91fa1c3521f38 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 23 Aug 2012 15:32:16 -0400 Subject: [PATCH 001/502] add code (mostly ported from metanl) --- .gitignore | 2 + README.txt | 4 + fixtext.py | 442 +++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 13 ++ 4 files changed, 461 insertions(+) create mode 100644 .gitignore create mode 100644 README.txt create mode 100644 fixtext.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..a295864e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +__pycache__ diff --git a/README.txt b/README.txt new file mode 100644 index 00000000..c52ae640 --- /dev/null +++ b/README.txt @@ -0,0 +1,4 @@ +A simple module for making text less broken. Works in Python 2.7, +Python 3.2, or later. + +Author: Rob Speer, rob@lumino.so diff --git a/fixtext.py b/fixtext.py new file mode 100644 index 00000000..b43a8537 --- /dev/null +++ b/fixtext.py @@ -0,0 +1,442 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import unicodedata +import re +import sys + +# Adjust names that have changed in Python 3 +if sys.version_info.major == 3: + from html import entities as htmlentitydefs + unichr = chr +else: + import htmlentitydefs + +def fixtext(text, normalization='NFKC'): + """ + Given any basestring as input, make its representation consistent: + + - Ensure that it is a Unicode string, converting from UTF-8 if + necessary. + - Detect whether the text was incorrectly encoded into UTF-8 and fix it, + as defined in `fix_bad_encoding`. + - If the text is HTML-escaped but has no HTML tags, replace HTML entities + with their equivalent characters. + - Remove terminal escapes and color codes. + - Remove control characters except for newlines and tabs. + - Normalize it with Unicode normalization form KC, which applies the + following relevant transformations: + - Combine characters and diacritics that are written using separate + code points, such as converting "e" plus an acute accent modifier + into "é", or converting "ka" (か) plus a dakuten into the + single character "ga" (が). + - Replace characters that are functionally equivalent with the most + common form: for example, half-width katakana will be replaced with + full-width, full-width Roman characters will be replaced with + ASCII characters, ellipsis characters will be replaced by three + periods, and the ligature 'fl' will be replaced by 'fl'. + - Replace curly quotation marks with straight ones. + + You may change the `normalization` argument to apply a different kind of + Unicode normalization, such as NFC or NFKD, or set it to None to skip this + step. + """ + if isinstance(text, bytes): + text = text.decode('utf-8') + text = remove_terminal_escapes(text) + text = fix_bad_encoding(text) + if '<' not in text or '>' not in text: + text = unescape_html(text) + text = text.translate(CONTROL_CHARS) + if normalization is not None: + text = unicodedata.normalize(normalization, text) + text = uncurl_quotes(text) + return text + +def fix_bad_encoding(text): + """ + Something you will find all over the place, in real-world text, is text + that's mistakenly encoded as utf-8, decoded in some ugly format like + latin-1 or even Windows codepage 1252, and encoded as utf-8 again. + + This causes your perfectly good Unicode-aware code to end up with garbage + text because someone else (or maybe "someone else") made a mistake. + + This function looks for the evidence of that having happened and fixes it. + It determines whether it should replace nonsense sequences of single-byte + characters that were really meant to be UTF-8 characters, and if so, turns + them into the correctly-encoded Unicode character that they were meant to + represent. + + The input to the function must be Unicode. It's not going to try to + auto-decode bytes for you -- then it would just create the problems it's + supposed to fix. + + NOTE: the following examples are written using unmarked literal strings, + but they are Unicode text. In Python 2 we have "unicode_literals" turned + on, and in Python 3 this is always the case. + + >>> print(fix_bad_encoding('único')) + único + + >>> print(fix_bad_encoding('This text is fine already :þ')) + This text is fine already :þ + + Because these characters often come from Microsoft products, we allow + for the possibility that we get not just Unicode characters 128-255, but + also Windows's conflicting idea of what characters 128-160 are. + + >>> print(fix_bad_encoding('This — should be an em dash')) + This — should be an em dash + + We might have to deal with both Windows characters and raw control + characters at the same time, especially when dealing with characters like + \x81 that have no mapping in Windows. + + >>> print(fix_bad_encoding('This text is sad .â\x81”.')) + This text is sad .⁔. + + This function even fixes multiple levels of badness: + + >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' + >>> print(fix_bad_encoding(wtf)) + ಠ_ಠ + + However, it has safeguards against fixing sequences of letters and + punctuation that can occur in valid text: + + >>> print(fix_bad_encoding('not such a fan of Charlotte Brontë…”')) + not such a fan of Charlotte Brontë…” + + Cases of genuine ambiguity can sometimes be addressed by finding other + characters that are not double-encoding, and expecting the encoding to + be consistent: + + >>> print(fix_bad_encoding('AHÅ™, the new sofa from IKEA®')) + AHÅ™, the new sofa from IKEA® + + Finally, we handle the case where the text is in a single-byte encoding + that was intended as Windows-1252 all along but read as Latin-1: + + >>> print(fix_bad_encoding('This text was never UTF-8 at all\x85')) + This text was never UTF-8 at all… + """ + if isinstance(text, bytes): + raise TypeError("This isn't even decoded into Unicode yet. " + "Decode it first.") + if len(text) == 0: + return text + + maxord = max(ord(char) for char in text) + tried_fixing = [] + if maxord < 128: + # Hooray! It's ASCII! + return text + else: + attempts = [(text, text_badness(text) + len(text))] + if maxord < 256: + tried_fixing = reinterpret_latin1_as_utf8(text) + tried_fixing2 = reinterpret_latin1_as_windows1252(text) + attempts.append((tried_fixing, text_cost(tried_fixing))) + attempts.append((tried_fixing2, text_cost(tried_fixing2))) + elif all(ord(char) in WINDOWS_1252_CODEPOINTS for char in text): + tried_fixing = reinterpret_windows1252_as_utf8(text) + attempts.append((tried_fixing, text_cost(tried_fixing))) + else: + # We can't imagine how this would be anything but valid text. + return text + + # Sort the results by badness + attempts.sort(key=lambda x: x[1]) + goodtext = attempts[0][0] + if goodtext == text: + return goodtext + else: + return fix_bad_encoding(goodtext) + + +def reinterpret_latin1_as_utf8(wrongtext): + newbytes = wrongtext.encode('latin-1', 'replace') + return newbytes.decode('utf-8', 'replace') + + +def reinterpret_windows1252_as_utf8(wrongtext): + altered_bytes = [] + for char in wrongtext: + if ord(char) in WINDOWS_1252_GREMLINS: + altered_bytes.append(char.encode('WINDOWS_1252')) + else: + altered_bytes.append(char.encode('latin-1', 'replace')) + return b''.join(altered_bytes).decode('utf-8', 'replace') + + +def reinterpret_latin1_as_windows1252(wrongtext): + """ + Maybe this was always meant to be in a single-byte encoding, and it + makes the most sense in Windows-1252. + """ + return wrongtext.encode('latin-1').decode('WINDOWS_1252', 'replace') + + +def text_badness(text): + ''' + Look for red flags that text is encoded incorrectly: + + Obvious problems: + - The replacement character \ufffd, indicating a decoding error + - Unassigned or private-use Unicode characters + + Very weird things: + - Adjacent letters from two different scripts + - Letters in scripts that are very rarely used on computers (and + therefore, someone who is using them will probably get Unicode right) + - Improbable control characters, such as 0x81 + + Moderately weird things: + - Improbable single-byte characters, such as ƒ or ¬ + - Letters in somewhat rare scripts + ''' + errors = 0 + very_weird_things = 0 + weird_things = 0 + prev_letter_script = None + for char in text: + index = ord(char) + if index < 256: + # Deal quickly with the first 256 characters. + weird_things += SINGLE_BYTE_WEIRDNESS[index] + if SINGLE_BYTE_LETTERS[index]: + prev_letter_script = 'latin' + else: + prev_letter_script = None + else: + category = unicodedata.category(char) + if category == 'Co': + # Unassigned or private use + errors += 1 + elif index == 0xfffd: + # Replacement character + errors += 1 + elif index in WINDOWS_1252_GREMLINS: + lowchar = char.encode('WINDOWS_1252').decode('latin-1') + weird_things += SINGLE_BYTE_WEIRDNESS[ord(lowchar)] - 0.5 + + if category.startswith('L'): + # It's a letter. What kind of letter? This is typically found + # in the first word of the letter's Unicode name. + name = unicodedata.name(char) + scriptname = name.split()[0] + freq, script = SCRIPT_TABLE.get(scriptname, (0, 'other')) + if prev_letter_script: + if script != prev_letter_script: + very_weird_things += 1 + if freq == 1: + weird_things += 2 + elif freq == 0: + very_weird_things += 1 + prev_letter_script = script + else: + prev_letter_script = None + + return 100 * errors + 10 * very_weird_things + weird_things + + +def text_cost(text): + """ + Assign a cost function to the length plus weirdness of a text string. + """ + return text_badness(text) + len(text) + + +HTML_ENTITY_RE = re.compile("&#?\w+;") +def unescape_html(text): + """ + Decode all three types of HTML entities/character references. + + Code by Fredrik Lundh of effbot.org. + + >>> print(unescape_html('<tag>')) + + """ + def fixup(m): + text = m.group(0) + if text[:2] == "&#": + # character reference + try: + if text[:3] == "&#x": + return unichr(int(text[3:-1], 16)) + else: + return unichr(int(text[2:-1])) + except ValueError: + pass + else: + # named entity + try: + text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) + except KeyError: + pass + return text # leave as is + return HTML_ENTITY_RE.sub(fixup, text) + + +ANSI_RE = re.compile('\033\[((?:\d|;)*)([a-zA-Z])') +def remove_terminal_escapes(text): + """ + Strip out "ANSI" terminal escape sequences, such as those that produce + colored text on Unix. + """ + return ANSI_RE.sub('', text) + + +SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') +DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') +def uncurl_quotes(text): + """ + Replace curly quotation marks with straight equivalents. + + >>> print(uncurl_quotes('\u201chere\u2019s a test\u201d')) + "here's a test" + """ + return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text)) + + +####################################################################### +# The rest of this file is esoteric info about characters, scripts, and their +# frequencies. +# +# Start with an inventory of "gremlins", which are characters from all over +# Unicode that Windows has instead assigned to the control characters +# 0x80-0x9F. We might encounter them in their Unicode forms and have to figure +# out what they were originally. + +WINDOWS_1252_GREMLINS = [ + # adapted from http://effbot.org/zone/unicode-gremlins.htm + 0x0152, # LATIN CAPITAL LIGATURE OE + 0x0153, # LATIN SMALL LIGATURE OE + 0x0160, # LATIN CAPITAL LETTER S WITH CARON + 0x0161, # LATIN SMALL LETTER S WITH CARON + 0x0178, # LATIN CAPITAL LETTER Y WITH DIAERESIS + 0x017E, # LATIN SMALL LETTER Z WITH CARON + 0x017D, # LATIN CAPITAL LETTER Z WITH CARON + 0x0192, # LATIN SMALL LETTER F WITH HOOK + 0x02C6, # MODIFIER LETTER CIRCUMFLEX ACCENT + 0x02DC, # SMALL TILDE + 0x2013, # EN DASH + 0x2014, # EM DASH + 0x201A, # SINGLE LOW-9 QUOTATION MARK + 0x201C, # LEFT DOUBLE QUOTATION MARK + 0x201D, # RIGHT DOUBLE QUOTATION MARK + 0x201E, # DOUBLE LOW-9 QUOTATION MARK + 0x2018, # LEFT SINGLE QUOTATION MARK + 0x2019, # RIGHT SINGLE QUOTATION MARK + 0x2020, # DAGGER + 0x2021, # DOUBLE DAGGER + 0x2022, # BULLET + 0x2026, # HORIZONTAL ELLIPSIS + 0x2030, # PER MILLE SIGN + 0x2039, # SINGLE LEFT-POINTING ANGLE QUOTATION MARK + 0x203A, # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + 0x20AC, # EURO SIGN + 0x2122, # TRADE MARK SIGN +] + +# a list of Unicode characters that might appear in Windows-1252 text +WINDOWS_1252_CODEPOINTS = list(range(256)) + WINDOWS_1252_GREMLINS + +# Rank the characters typically represented by a single byte -- that is, in +# Latin-1 or Windows-1252 -- by how weird it would be to see them in running +# text. +# +# 0 = not weird at all +# 1 = rare punctuation or rare letter that someone could certainly +# have a good reason to use. All Windows-1252 gremlins are at least +# weirdness 1. +# 2 = things that probably don't appear next to letters or other +# symbols, such as math or currency symbols +# 3 = obscure symbols that nobody would go out of their way to use +# (includes symbols that were replaced in ISO-8859-15) +# 4 = why would you use this? +# 5 = unprintable control character +# +# The Portuguese letter à (0xc3) is marked as weird because it would usually +# appear in the middle of a word in actual Portuguese, and meanwhile it +# appears in the mis-encodings of many common characters. + +SINGLE_BYTE_WEIRDNESS = ( +# 0 1 2 3 4 5 6 7 8 9 a b c d e f + 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 5, 5, 5, 5, 5, # 0x00 + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, # 0x10 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x20 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x30 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x40 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x50 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x60 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, # 0x70 + 2, 5, 1, 4, 1, 1, 3, 3, 4, 3, 1, 1, 1, 5, 1, 5, # 0x80 + 5, 1, 1, 1, 1, 3, 1, 1, 4, 1, 1, 1, 1, 5, 1, 1, # 0x90 + 1, 0, 2, 2, 3, 2, 4, 2, 4, 2, 2, 0, 3, 1, 1, 4, # 0xa0 + 2, 2, 3, 3, 4, 3, 3, 2, 4, 4, 4, 0, 3, 3, 3, 0, # 0xb0 + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0xc0 + 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, # 0xd0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0xe0 + 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, # 0xf0 +) + +# Pre-cache the Unicode data saying which of these first 256 characters are +# letters. We'll need it often. +SINGLE_BYTE_LETTERS = [ + unicodedata.category(unichr(i)).startswith('L') + for i in range(256) +] + +# A table telling us how to interpret the first word of a letter's Unicode +# name. The number indicates how frequently we expect this script to be used +# on computers. Many scripts not included here are assumed to have a frequency +# of "0" -- if you're going to write in Linear B using Unicode, you're +# probably aware enough of encoding issues to get it right. +# +# The lowercase name is a general category -- for example, Han characters and +# Hiragana characters are very frequently adjacent in Japanese, so they all go +# into category 'cjk'. Letters of different categories are assumed not to +# appear next to each other often. +SCRIPT_TABLE = { + 'LATIN': (3, 'latin'), + 'CJK': (2, 'cjk'), + 'ARABIC': (2, 'arabic'), + 'CYRILLIC': (2, 'cyrillic'), + 'GREEK': (2, 'greek'), + 'HEBREW': (2, 'hebrew'), + 'KATAKANA': (2, 'cjk'), + 'HIRAGANA': (2, 'cjk'), + 'HIRAGANA-KATAKANA': (2, 'cjk'), + 'HANGUL': (2, 'cjk'), + 'DEVANAGARI': (2, 'devanagari'), + 'THAI': (2, 'thai'), + 'FULLWIDTH': (2, 'cjk'), + 'MODIFIER': (2, None), + 'HALFWIDTH': (1, 'cjk'), + 'BENGALI': (1, 'bengali'), + 'LAO': (1, 'lao'), + 'KHMER': (1, 'khmer'), + 'TELUGU': (1, 'telugu'), + 'MALAYALAM': (1, 'malayalam'), + 'SINHALA': (1, 'sinhala'), + 'TAMIL': (1, 'tamil'), + 'GEORGIAN': (1, 'georgian'), + 'ARMENIAN': (1, 'armenian'), + 'KANNADA': (1, 'kannada'), # mostly used for looks of disapproval + 'MASCULINE': (1, 'latin'), + 'FEMININE': (1, 'latin') +} + +# A translate mapping that will strip all control characters except \t and \n. +# This incidentally has the effect of normalizing Windows \r\n line endings to +# Unix \n line endings. +CONTROL_CHARS = {} +for i in range(256): + if unicodedata.category(unichr(i)) == 'Cc': + CONTROL_CHARS[i] = None + +CONTROL_CHARS[ord('\t')] = '\t' +CONTROL_CHARS[ord('\n')] = '\n' + diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..70525b1a --- /dev/null +++ b/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup + +setup( + name="fixtext", + version = '1.0', + maintainer='Luminoso, LLC', + maintainer_email='dev@lumino.so', + license = "MIT", + url = 'http://github.com/LuminosoInsight/fixtext', + platforms = ["any"], + description = "Fixes some problems with Unicode text after the fact", + py_modules=['fixtext'], +) From d10819dbd5f8286bf039dab816002babc087adac Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 24 Aug 2012 12:05:41 -0400 Subject: [PATCH 002/502] change order of operations, add doctests to top level --- fixtext.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/fixtext.py b/fixtext.py index b43a8537..7d17b495 100644 --- a/fixtext.py +++ b/fixtext.py @@ -14,7 +14,8 @@ def fixtext(text, normalization='NFKC'): """ - Given any basestring as input, make its representation consistent: + Given any basestring as input, make its representation consistent and + possibly less broken: - Ensure that it is a Unicode string, converting from UTF-8 if necessary. @@ -40,13 +41,33 @@ def fixtext(text, normalization='NFKC'): You may change the `normalization` argument to apply a different kind of Unicode normalization, such as NFC or NFKD, or set it to None to skip this step. + + >>> print(fixtext('ünicode')) + ünicode + + >>> print(fixtext('Broken text… it’s flubberific!')) + Broken text... it's flubberific! + + >>> print(fixtext('HTML entities <3')) + HTML entities <3 + + >>> print(fixtext('HTML entities <3')) + HTML entities <3 + + >>> print(fixtext('\001\033[36;44mI’m blue, da ba dee da ba ' + ... 'doo…\033[0m')) + I'm blue, da ba dee da ba doo... + + >>> len(fixtext('')) + 0 + """ if isinstance(text, bytes): text = text.decode('utf-8') text = remove_terminal_escapes(text) - text = fix_bad_encoding(text) if '<' not in text or '>' not in text: text = unescape_html(text) + text = fix_bad_encoding(text) text = text.translate(CONTROL_CHARS) if normalization is not None: text = unicodedata.normalize(normalization, text) From ff498be1e3a9ba6e1e8c9014b3cb986f633143d7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 24 Aug 2012 12:13:01 -0400 Subject: [PATCH 003/502] don't auto-decode utf-8 as the first step, that's dangerous --- fixtext.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fixtext.py b/fixtext.py index 7d17b495..e980b369 100644 --- a/fixtext.py +++ b/fixtext.py @@ -14,7 +14,7 @@ def fixtext(text, normalization='NFKC'): """ - Given any basestring as input, make its representation consistent and + Given Unicode text as input, make its representation consistent and possibly less broken: - Ensure that it is a Unicode string, converting from UTF-8 if @@ -62,8 +62,6 @@ def fixtext(text, normalization='NFKC'): 0 """ - if isinstance(text, bytes): - text = text.decode('utf-8') text = remove_terminal_escapes(text) if '<' not in text or '>' not in text: text = unescape_html(text) From 2624885ec9a6eaab4a7c60bba1158556fef5f658 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 24 Aug 2012 12:15:47 -0400 Subject: [PATCH 004/502] raise an error given bytes --- fixtext.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fixtext.py b/fixtext.py index e980b369..3cc1cd78 100644 --- a/fixtext.py +++ b/fixtext.py @@ -62,6 +62,9 @@ def fixtext(text, normalization='NFKC'): 0 """ + if isinstance(text, bytes): + raise TypeError("fixtext works on Unicode text. Please decode " + "your text first.") text = remove_terminal_escapes(text) if '<' not in text or '>' not in text: text = unescape_html(text) From 99e8d98c8864ec9b38daca120280a71b097a4940 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 24 Aug 2012 12:17:07 -0400 Subject: [PATCH 005/502] add LICENSE.txt --- .gitignore | 1 + LICENSE.txt | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 LICENSE.txt diff --git a/.gitignore b/.gitignore index a295864e..2aae0d04 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc __pycache__ +.coverage diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..767a2405 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,8 @@ +Copyright (C) 2012 Rob Speer (rob@lumino.so) +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From 273966c68586aba6aac59bf4de657cdf174576e8 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 24 Aug 2012 15:22:04 -0400 Subject: [PATCH 006/502] rewrite readme --- README.md | 34 ++++++++++++++++++++++++++++++++++ README.txt | 4 ---- fixtext.py => ftfy.py | 16 ++++++++-------- setup.py | 6 +++--- 4 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 README.md delete mode 100644 README.txt rename fixtext.py => ftfy.py (97%) diff --git a/README.md b/README.md new file mode 100644 index 00000000..5733cbfb --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +A simple module for making text less broken. Works in Python 2.7, Python 3.2, +or later. + +Its main function, `fix_text` aka `ftfy`, performs the following operations on +a Unicode string: + +* Detect whether the text was incorrectly encoded into UTF-8 and fix it, + as defined in `fix_bad_encoding`. +* If the text is HTML-escaped but has no HTML tags, replace HTML entities + with their equivalent characters. +* Remove terminal escapes and color codes. +* Remove control characters except for newlines and tabs. +* Normalize it with Unicode normalization form KC, which applies the + following relevant transformations: + * Combine characters and diacritics that are written using separate + code points, such as converting "e" plus an acute accent modifier + into "é", or converting "ka" (か) plus a dakuten into the + single character "ga" (が). + * Replace characters that are functionally equivalent with the most + common form: for example, half-width katakana will be replaced with + full-width, full-width Roman characters will be replaced with + ASCII characters, ellipsis characters will be replaced by three + periods, and the ligature 'fl' will be replaced by 'fl'. +* Replace curly quotation marks with straight ones. + +If your text is inconsistently encoded, but breaks up into coherent pieces (for +example, line-by-line), then you will be better off feeding the lines into +`fix_text` individually instead of as one huge string. + +ftfy will not decode bytes into Unicode for you. You should use Python's +built-in `.decode` for that. If you don't know the encoding, you should use the +`chardet` library. + +Author: Rob Speer, rob@lumino.so diff --git a/README.txt b/README.txt deleted file mode 100644 index c52ae640..00000000 --- a/README.txt +++ /dev/null @@ -1,4 +0,0 @@ -A simple module for making text less broken. Works in Python 2.7, -Python 3.2, or later. - -Author: Rob Speer, rob@lumino.so diff --git a/fixtext.py b/ftfy.py similarity index 97% rename from fixtext.py rename to ftfy.py index 3cc1cd78..da1b38c7 100644 --- a/fixtext.py +++ b/ftfy.py @@ -12,7 +12,7 @@ else: import htmlentitydefs -def fixtext(text, normalization='NFKC'): +def fix_text(text, normalization='NFKC'): """ Given Unicode text as input, make its representation consistent and possibly less broken: @@ -42,28 +42,28 @@ def fixtext(text, normalization='NFKC'): Unicode normalization, such as NFC or NFKD, or set it to None to skip this step. - >>> print(fixtext('ünicode')) + >>> print(fix_text('ünicode')) ünicode - >>> print(fixtext('Broken text… it’s flubberific!')) + >>> print(fix_text('Broken text… it’s flubberific!')) Broken text... it's flubberific! - >>> print(fixtext('HTML entities <3')) + >>> print(fix_text('HTML entities <3')) HTML entities <3 - >>> print(fixtext('HTML entities <3')) + >>> print(fix_text('HTML entities <3')) HTML entities <3 - >>> print(fixtext('\001\033[36;44mI’m blue, da ba dee da ba ' + >>> print(fix_text('\001\033[36;44mI’m blue, da ba dee da ba ' ... 'doo…\033[0m')) I'm blue, da ba dee da ba doo... - >>> len(fixtext('')) + >>> len(fix_text('')) 0 """ if isinstance(text, bytes): - raise TypeError("fixtext works on Unicode text. Please decode " + raise TypeError("fix_text works on Unicode text. Please decode " "your text first.") text = remove_terminal_escapes(text) if '<' not in text or '>' not in text: diff --git a/setup.py b/setup.py index 70525b1a..7effe1fd 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,13 @@ from setuptools import setup setup( - name="fixtext", + name="ftfy", version = '1.0', maintainer='Luminoso, LLC', maintainer_email='dev@lumino.so', license = "MIT", - url = 'http://github.com/LuminosoInsight/fixtext', + url = 'http://github.com/LuminosoInsight/ftfy', platforms = ["any"], description = "Fixes some problems with Unicode text after the fact", - py_modules=['fixtext'], + py_modules=['ftfy'], ) From 48821dfd13f1c2a4f7ac2bc6680bc47d2d638758 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 24 Aug 2012 15:22:58 -0400 Subject: [PATCH 007/502] alias fix_text to ftfy --- ftfy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ftfy.py b/ftfy.py index da1b38c7..95e12982 100644 --- a/ftfy.py +++ b/ftfy.py @@ -74,6 +74,7 @@ def fix_text(text, normalization='NFKC'): text = unicodedata.normalize(normalization, text) text = uncurl_quotes(text) return text +ftfy = fix_text def fix_bad_encoding(text): """ From 948a98937229a4593dba468bb92c137a7f7e8d9d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 24 Aug 2012 15:24:34 -0400 Subject: [PATCH 008/502] update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 2aae0d04..b8f6c51a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.pyc __pycache__ .coverage +dist +*.egg-info From 1f3eacdff3d85d3fb44a0f3189d4c45e12edc729 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 24 Aug 2012 16:26:21 -0300 Subject: [PATCH 009/502] add heading --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5733cbfb..474faf68 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ -A simple module for making text less broken. Works in Python 2.7, Python 3.2, -or later. +## ftfy: fixes text for you + +This is a simple module for making text less broken. Works in Python 2.7, +Python 3.2, or later. Its main function, `fix_text` aka `ftfy`, performs the following operations on a Unicode string: From 5e3102886b5647feb01b540c44c9204baeaad257 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Aug 2012 23:39:06 -0400 Subject: [PATCH 010/502] changed version check to work on python 2.6 --- ftfy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy.py b/ftfy.py index 95e12982..a1b731c6 100644 --- a/ftfy.py +++ b/ftfy.py @@ -6,7 +6,7 @@ import sys # Adjust names that have changed in Python 3 -if sys.version_info.major == 3: +if sys.version_info[0] >= 3: from html import entities as htmlentitydefs unichr = chr else: From 3322d1d407da1301312ae60a1f7e04fdc87fa052 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 30 Aug 2012 01:13:39 -0300 Subject: [PATCH 011/502] update README now that it works on 2.6 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 474faf68..169a8fff 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## ftfy: fixes text for you -This is a simple module for making text less broken. Works in Python 2.7, +This is a simple module for making text less broken. Works in Python 2.6, Python 3.2, or later. Its main function, `fix_text` aka `ftfy`, performs the following operations on From b314272e1a480a8eeaffe678aacd8c80a870d145 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 29 Jan 2013 15:19:28 -0500 Subject: [PATCH 012/502] Update to version 2 - command line utility - prevents DoS from long strings - faster code that uses Python built-ins for inner loops --- ftfy.py | 465 ----------------------------------------------- ftfy/__init__.py | 349 +++++++++++++++++++++++++++++++++++ ftfy/badness.py | 180 ++++++++++++++++++ ftfy/chardata.py | 174 ++++++++++++++++++ ftfy/cli.py | 26 +++ setup.py | 17 +- 6 files changed, 740 insertions(+), 471 deletions(-) delete mode 100644 ftfy.py create mode 100644 ftfy/__init__.py create mode 100644 ftfy/badness.py create mode 100644 ftfy/chardata.py create mode 100644 ftfy/cli.py diff --git a/ftfy.py b/ftfy.py deleted file mode 100644 index 95e12982..00000000 --- a/ftfy.py +++ /dev/null @@ -1,465 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -import unicodedata -import re -import sys - -# Adjust names that have changed in Python 3 -if sys.version_info.major == 3: - from html import entities as htmlentitydefs - unichr = chr -else: - import htmlentitydefs - -def fix_text(text, normalization='NFKC'): - """ - Given Unicode text as input, make its representation consistent and - possibly less broken: - - - Ensure that it is a Unicode string, converting from UTF-8 if - necessary. - - Detect whether the text was incorrectly encoded into UTF-8 and fix it, - as defined in `fix_bad_encoding`. - - If the text is HTML-escaped but has no HTML tags, replace HTML entities - with their equivalent characters. - - Remove terminal escapes and color codes. - - Remove control characters except for newlines and tabs. - - Normalize it with Unicode normalization form KC, which applies the - following relevant transformations: - - Combine characters and diacritics that are written using separate - code points, such as converting "e" plus an acute accent modifier - into "é", or converting "ka" (か) plus a dakuten into the - single character "ga" (が). - - Replace characters that are functionally equivalent with the most - common form: for example, half-width katakana will be replaced with - full-width, full-width Roman characters will be replaced with - ASCII characters, ellipsis characters will be replaced by three - periods, and the ligature 'fl' will be replaced by 'fl'. - - Replace curly quotation marks with straight ones. - - You may change the `normalization` argument to apply a different kind of - Unicode normalization, such as NFC or NFKD, or set it to None to skip this - step. - - >>> print(fix_text('ünicode')) - ünicode - - >>> print(fix_text('Broken text… it’s flubberific!')) - Broken text... it's flubberific! - - >>> print(fix_text('HTML entities <3')) - HTML entities <3 - - >>> print(fix_text('HTML entities <3')) - HTML entities <3 - - >>> print(fix_text('\001\033[36;44mI’m blue, da ba dee da ba ' - ... 'doo…\033[0m')) - I'm blue, da ba dee da ba doo... - - >>> len(fix_text('')) - 0 - - """ - if isinstance(text, bytes): - raise TypeError("fix_text works on Unicode text. Please decode " - "your text first.") - text = remove_terminal_escapes(text) - if '<' not in text or '>' not in text: - text = unescape_html(text) - text = fix_bad_encoding(text) - text = text.translate(CONTROL_CHARS) - if normalization is not None: - text = unicodedata.normalize(normalization, text) - text = uncurl_quotes(text) - return text -ftfy = fix_text - -def fix_bad_encoding(text): - """ - Something you will find all over the place, in real-world text, is text - that's mistakenly encoded as utf-8, decoded in some ugly format like - latin-1 or even Windows codepage 1252, and encoded as utf-8 again. - - This causes your perfectly good Unicode-aware code to end up with garbage - text because someone else (or maybe "someone else") made a mistake. - - This function looks for the evidence of that having happened and fixes it. - It determines whether it should replace nonsense sequences of single-byte - characters that were really meant to be UTF-8 characters, and if so, turns - them into the correctly-encoded Unicode character that they were meant to - represent. - - The input to the function must be Unicode. It's not going to try to - auto-decode bytes for you -- then it would just create the problems it's - supposed to fix. - - NOTE: the following examples are written using unmarked literal strings, - but they are Unicode text. In Python 2 we have "unicode_literals" turned - on, and in Python 3 this is always the case. - - >>> print(fix_bad_encoding('único')) - único - - >>> print(fix_bad_encoding('This text is fine already :þ')) - This text is fine already :þ - - Because these characters often come from Microsoft products, we allow - for the possibility that we get not just Unicode characters 128-255, but - also Windows's conflicting idea of what characters 128-160 are. - - >>> print(fix_bad_encoding('This — should be an em dash')) - This — should be an em dash - - We might have to deal with both Windows characters and raw control - characters at the same time, especially when dealing with characters like - \x81 that have no mapping in Windows. - - >>> print(fix_bad_encoding('This text is sad .â\x81”.')) - This text is sad .⁔. - - This function even fixes multiple levels of badness: - - >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' - >>> print(fix_bad_encoding(wtf)) - ಠ_ಠ - - However, it has safeguards against fixing sequences of letters and - punctuation that can occur in valid text: - - >>> print(fix_bad_encoding('not such a fan of Charlotte Brontë…”')) - not such a fan of Charlotte Brontë…” - - Cases of genuine ambiguity can sometimes be addressed by finding other - characters that are not double-encoding, and expecting the encoding to - be consistent: - - >>> print(fix_bad_encoding('AHÅ™, the new sofa from IKEA®')) - AHÅ™, the new sofa from IKEA® - - Finally, we handle the case where the text is in a single-byte encoding - that was intended as Windows-1252 all along but read as Latin-1: - - >>> print(fix_bad_encoding('This text was never UTF-8 at all\x85')) - This text was never UTF-8 at all… - """ - if isinstance(text, bytes): - raise TypeError("This isn't even decoded into Unicode yet. " - "Decode it first.") - if len(text) == 0: - return text - - maxord = max(ord(char) for char in text) - tried_fixing = [] - if maxord < 128: - # Hooray! It's ASCII! - return text - else: - attempts = [(text, text_badness(text) + len(text))] - if maxord < 256: - tried_fixing = reinterpret_latin1_as_utf8(text) - tried_fixing2 = reinterpret_latin1_as_windows1252(text) - attempts.append((tried_fixing, text_cost(tried_fixing))) - attempts.append((tried_fixing2, text_cost(tried_fixing2))) - elif all(ord(char) in WINDOWS_1252_CODEPOINTS for char in text): - tried_fixing = reinterpret_windows1252_as_utf8(text) - attempts.append((tried_fixing, text_cost(tried_fixing))) - else: - # We can't imagine how this would be anything but valid text. - return text - - # Sort the results by badness - attempts.sort(key=lambda x: x[1]) - goodtext = attempts[0][0] - if goodtext == text: - return goodtext - else: - return fix_bad_encoding(goodtext) - - -def reinterpret_latin1_as_utf8(wrongtext): - newbytes = wrongtext.encode('latin-1', 'replace') - return newbytes.decode('utf-8', 'replace') - - -def reinterpret_windows1252_as_utf8(wrongtext): - altered_bytes = [] - for char in wrongtext: - if ord(char) in WINDOWS_1252_GREMLINS: - altered_bytes.append(char.encode('WINDOWS_1252')) - else: - altered_bytes.append(char.encode('latin-1', 'replace')) - return b''.join(altered_bytes).decode('utf-8', 'replace') - - -def reinterpret_latin1_as_windows1252(wrongtext): - """ - Maybe this was always meant to be in a single-byte encoding, and it - makes the most sense in Windows-1252. - """ - return wrongtext.encode('latin-1').decode('WINDOWS_1252', 'replace') - - -def text_badness(text): - ''' - Look for red flags that text is encoded incorrectly: - - Obvious problems: - - The replacement character \ufffd, indicating a decoding error - - Unassigned or private-use Unicode characters - - Very weird things: - - Adjacent letters from two different scripts - - Letters in scripts that are very rarely used on computers (and - therefore, someone who is using them will probably get Unicode right) - - Improbable control characters, such as 0x81 - - Moderately weird things: - - Improbable single-byte characters, such as ƒ or ¬ - - Letters in somewhat rare scripts - ''' - errors = 0 - very_weird_things = 0 - weird_things = 0 - prev_letter_script = None - for char in text: - index = ord(char) - if index < 256: - # Deal quickly with the first 256 characters. - weird_things += SINGLE_BYTE_WEIRDNESS[index] - if SINGLE_BYTE_LETTERS[index]: - prev_letter_script = 'latin' - else: - prev_letter_script = None - else: - category = unicodedata.category(char) - if category == 'Co': - # Unassigned or private use - errors += 1 - elif index == 0xfffd: - # Replacement character - errors += 1 - elif index in WINDOWS_1252_GREMLINS: - lowchar = char.encode('WINDOWS_1252').decode('latin-1') - weird_things += SINGLE_BYTE_WEIRDNESS[ord(lowchar)] - 0.5 - - if category.startswith('L'): - # It's a letter. What kind of letter? This is typically found - # in the first word of the letter's Unicode name. - name = unicodedata.name(char) - scriptname = name.split()[0] - freq, script = SCRIPT_TABLE.get(scriptname, (0, 'other')) - if prev_letter_script: - if script != prev_letter_script: - very_weird_things += 1 - if freq == 1: - weird_things += 2 - elif freq == 0: - very_weird_things += 1 - prev_letter_script = script - else: - prev_letter_script = None - - return 100 * errors + 10 * very_weird_things + weird_things - - -def text_cost(text): - """ - Assign a cost function to the length plus weirdness of a text string. - """ - return text_badness(text) + len(text) - - -HTML_ENTITY_RE = re.compile("&#?\w+;") -def unescape_html(text): - """ - Decode all three types of HTML entities/character references. - - Code by Fredrik Lundh of effbot.org. - - >>> print(unescape_html('<tag>')) - - """ - def fixup(m): - text = m.group(0) - if text[:2] == "&#": - # character reference - try: - if text[:3] == "&#x": - return unichr(int(text[3:-1], 16)) - else: - return unichr(int(text[2:-1])) - except ValueError: - pass - else: - # named entity - try: - text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) - except KeyError: - pass - return text # leave as is - return HTML_ENTITY_RE.sub(fixup, text) - - -ANSI_RE = re.compile('\033\[((?:\d|;)*)([a-zA-Z])') -def remove_terminal_escapes(text): - """ - Strip out "ANSI" terminal escape sequences, such as those that produce - colored text on Unix. - """ - return ANSI_RE.sub('', text) - - -SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') -DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') -def uncurl_quotes(text): - """ - Replace curly quotation marks with straight equivalents. - - >>> print(uncurl_quotes('\u201chere\u2019s a test\u201d')) - "here's a test" - """ - return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text)) - - -####################################################################### -# The rest of this file is esoteric info about characters, scripts, and their -# frequencies. -# -# Start with an inventory of "gremlins", which are characters from all over -# Unicode that Windows has instead assigned to the control characters -# 0x80-0x9F. We might encounter them in their Unicode forms and have to figure -# out what they were originally. - -WINDOWS_1252_GREMLINS = [ - # adapted from http://effbot.org/zone/unicode-gremlins.htm - 0x0152, # LATIN CAPITAL LIGATURE OE - 0x0153, # LATIN SMALL LIGATURE OE - 0x0160, # LATIN CAPITAL LETTER S WITH CARON - 0x0161, # LATIN SMALL LETTER S WITH CARON - 0x0178, # LATIN CAPITAL LETTER Y WITH DIAERESIS - 0x017E, # LATIN SMALL LETTER Z WITH CARON - 0x017D, # LATIN CAPITAL LETTER Z WITH CARON - 0x0192, # LATIN SMALL LETTER F WITH HOOK - 0x02C6, # MODIFIER LETTER CIRCUMFLEX ACCENT - 0x02DC, # SMALL TILDE - 0x2013, # EN DASH - 0x2014, # EM DASH - 0x201A, # SINGLE LOW-9 QUOTATION MARK - 0x201C, # LEFT DOUBLE QUOTATION MARK - 0x201D, # RIGHT DOUBLE QUOTATION MARK - 0x201E, # DOUBLE LOW-9 QUOTATION MARK - 0x2018, # LEFT SINGLE QUOTATION MARK - 0x2019, # RIGHT SINGLE QUOTATION MARK - 0x2020, # DAGGER - 0x2021, # DOUBLE DAGGER - 0x2022, # BULLET - 0x2026, # HORIZONTAL ELLIPSIS - 0x2030, # PER MILLE SIGN - 0x2039, # SINGLE LEFT-POINTING ANGLE QUOTATION MARK - 0x203A, # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK - 0x20AC, # EURO SIGN - 0x2122, # TRADE MARK SIGN -] - -# a list of Unicode characters that might appear in Windows-1252 text -WINDOWS_1252_CODEPOINTS = list(range(256)) + WINDOWS_1252_GREMLINS - -# Rank the characters typically represented by a single byte -- that is, in -# Latin-1 or Windows-1252 -- by how weird it would be to see them in running -# text. -# -# 0 = not weird at all -# 1 = rare punctuation or rare letter that someone could certainly -# have a good reason to use. All Windows-1252 gremlins are at least -# weirdness 1. -# 2 = things that probably don't appear next to letters or other -# symbols, such as math or currency symbols -# 3 = obscure symbols that nobody would go out of their way to use -# (includes symbols that were replaced in ISO-8859-15) -# 4 = why would you use this? -# 5 = unprintable control character -# -# The Portuguese letter à (0xc3) is marked as weird because it would usually -# appear in the middle of a word in actual Portuguese, and meanwhile it -# appears in the mis-encodings of many common characters. - -SINGLE_BYTE_WEIRDNESS = ( -# 0 1 2 3 4 5 6 7 8 9 a b c d e f - 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 5, 5, 5, 5, 5, # 0x00 - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, # 0x10 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x20 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x30 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x40 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x50 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x60 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, # 0x70 - 2, 5, 1, 4, 1, 1, 3, 3, 4, 3, 1, 1, 1, 5, 1, 5, # 0x80 - 5, 1, 1, 1, 1, 3, 1, 1, 4, 1, 1, 1, 1, 5, 1, 1, # 0x90 - 1, 0, 2, 2, 3, 2, 4, 2, 4, 2, 2, 0, 3, 1, 1, 4, # 0xa0 - 2, 2, 3, 3, 4, 3, 3, 2, 4, 4, 4, 0, 3, 3, 3, 0, # 0xb0 - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0xc0 - 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, # 0xd0 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0xe0 - 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, # 0xf0 -) - -# Pre-cache the Unicode data saying which of these first 256 characters are -# letters. We'll need it often. -SINGLE_BYTE_LETTERS = [ - unicodedata.category(unichr(i)).startswith('L') - for i in range(256) -] - -# A table telling us how to interpret the first word of a letter's Unicode -# name. The number indicates how frequently we expect this script to be used -# on computers. Many scripts not included here are assumed to have a frequency -# of "0" -- if you're going to write in Linear B using Unicode, you're -# probably aware enough of encoding issues to get it right. -# -# The lowercase name is a general category -- for example, Han characters and -# Hiragana characters are very frequently adjacent in Japanese, so they all go -# into category 'cjk'. Letters of different categories are assumed not to -# appear next to each other often. -SCRIPT_TABLE = { - 'LATIN': (3, 'latin'), - 'CJK': (2, 'cjk'), - 'ARABIC': (2, 'arabic'), - 'CYRILLIC': (2, 'cyrillic'), - 'GREEK': (2, 'greek'), - 'HEBREW': (2, 'hebrew'), - 'KATAKANA': (2, 'cjk'), - 'HIRAGANA': (2, 'cjk'), - 'HIRAGANA-KATAKANA': (2, 'cjk'), - 'HANGUL': (2, 'cjk'), - 'DEVANAGARI': (2, 'devanagari'), - 'THAI': (2, 'thai'), - 'FULLWIDTH': (2, 'cjk'), - 'MODIFIER': (2, None), - 'HALFWIDTH': (1, 'cjk'), - 'BENGALI': (1, 'bengali'), - 'LAO': (1, 'lao'), - 'KHMER': (1, 'khmer'), - 'TELUGU': (1, 'telugu'), - 'MALAYALAM': (1, 'malayalam'), - 'SINHALA': (1, 'sinhala'), - 'TAMIL': (1, 'tamil'), - 'GEORGIAN': (1, 'georgian'), - 'ARMENIAN': (1, 'armenian'), - 'KANNADA': (1, 'kannada'), # mostly used for looks of disapproval - 'MASCULINE': (1, 'latin'), - 'FEMININE': (1, 'latin') -} - -# A translate mapping that will strip all control characters except \t and \n. -# This incidentally has the effect of normalizing Windows \r\n line endings to -# Unix \n line endings. -CONTROL_CHARS = {} -for i in range(256): - if unicodedata.category(unichr(i)) == 'Cc': - CONTROL_CHARS[i] = None - -CONTROL_CHARS[ord('\t')] = '\t' -CONTROL_CHARS[ord('\n')] = '\n' - diff --git a/ftfy/__init__.py b/ftfy/__init__.py new file mode 100644 index 00000000..d37095f6 --- /dev/null +++ b/ftfy/__init__.py @@ -0,0 +1,349 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import unicodedata +import re +import sys +from ftfy.chardata import (CONTROL_CHARS, WINDOWS_1252_CODEPOINTS, + WINDOWS_1252_GREMLINS) +from ftfy.badness import text_badness + +# Adjust names that have changed in Python 3 +if sys.version_info.major == 3: + from html import entities + htmlentitydefs = entities + unichr = chr +else: + import htmlentitydefs + +# Don't try to ftfy more than 64 KiB of text at a time, to prevent +# denial of service. +MAXLEN = 0x10000 + +def fix_text(text, normalization='NFKC'): + """ + Given Unicode text as input, make its representation consistent and + possibly less broken: + + - Ensure that it is a Unicode string, converting from UTF-8 if + necessary. + - Detect whether the text was incorrectly encoded into UTF-8 and fix it, + as defined in `fix_bad_encoding`. + - If the text is HTML-escaped but has no HTML tags, replace HTML entities + with their equivalent characters. + - Remove terminal escapes and color codes. + - Remove control characters except for newlines and tabs. + - Normalize it with Unicode normalization form KC, which applies the + following relevant transformations: + - Combine characters and diacritics that are written using separate + code points, such as converting "e" plus an acute accent modifier + into "é", or converting "ka" (か) plus a dakuten into the + single character "ga" (が). + - Replace characters that are functionally equivalent with the most + common form: for example, half-width katakana will be replaced with + full-width, full-width Roman characters will be replaced with + ASCII characters, ellipsis characters will be replaced by three + periods, and the ligature 'fl' will be replaced by 'fl'. + - Replace curly quotation marks with straight ones. + - Remove the Byte-Order Mark if it exists. + + You may change the `normalization` argument to apply a different kind of + Unicode normalization, such as NFC or NFKD, or set it to None to skip this + step. + + >>> print(fix_text('ünicode')) + ünicode + + >>> print(fix_text('Broken text… it’s flubberific!')) + Broken text... it's flubberific! + + >>> print(fix_text('HTML entities <3')) + HTML entities <3 + + >>> print(fix_text('HTML entities <3')) + HTML entities <3 + + >>> print(fix_text('\001\033[36;44mI’m blue, da ba dee da ba ' + ... 'doo…\033[0m')) + I'm blue, da ba dee da ba doo... + + >>> print(fix_text('\\ufeffParty like\\nit’s 1999!')) + Party like + it's 1999! + + >>> len(fix_text('fi' * 100000)) + 200000 + + >>> len(fix_text('')) + 0 + + `fix_text` will work one line at a time, with the possibility that some + lines are in different encodings. + + If you are certain your entire text is in the same encoding (though that + encoding is possibly flawed), and do not mind performing operations on + the whole text at once, use `fix_text_segment`. + """ + out = [] + pos = 0 + entities = True + while pos < len(text): + textbreak = text.find('\n', pos, pos + MAXLEN) + if textbreak == -1: + if pos + MAXLEN >= len(text): + textbreak = len(text) - 1 + else: + textbreak = text.find(' ', pos, pos + MAXLEN) + if textbreak == -1: + # oh well. We might not be able to fix the 2**16th character. + textbreak = MAXLEN + + substring = text[pos:textbreak + 1] + + if '<' in substring and '>' in substring: + # we see angle brackets together; this could be HTML + entities = False + + out.append(fix_text_segment(substring, normalization, entities)) + pos = textbreak + 1 + + return ''.join(out) + + +def fix_file(file, normalization='NFKC'): + entities = True + for line in file: + if '<' in line and '>' in line: + entities = False + yield fix_text_segment(line, normalization, entities) + + +def fix_text_segment(text, normalization='NFKC', entities=True): + """ + Apply fixes to a single line of text. + """ + if isinstance(text, bytes): + raise TypeError("fix_text works on Unicode text. Please decode " + "your text first.") + text = remove_terminal_escapes(text) + if entities: + text = unescape_html(text) + text = fix_bad_encoding(text) + text = text.translate(CONTROL_CHARS) + if normalization is not None: + text = unicodedata.normalize(normalization, text) + text = uncurl_quotes(text) + text = remove_bom(text) + return text + +ftfy = fix_text_segment + + +def fix_bad_encoding(text): + """ + Something you will find all over the place, in real-world text, is text + that's mistakenly encoded as utf-8, decoded in some ugly format like + latin-1 or even Windows codepage 1252, and encoded as utf-8 again. + + This causes your perfectly good Unicode-aware code to end up with garbage + text because someone else (or maybe "someone else") made a mistake. + + This function looks for the evidence of that having happened and fixes it. + It determines whether it should replace nonsense sequences of single-byte + characters that were really meant to be UTF-8 characters, and if so, turns + them into the correctly-encoded Unicode character that they were meant to + represent. + + The input to the function must be Unicode. It's not going to try to + auto-decode bytes for you -- then it would just create the problems it's + supposed to fix. + + NOTE: the following examples are written using unmarked literal strings, + but they are Unicode text. In Python 2 we have "unicode_literals" turned + on, and in Python 3 this is always the case. + + >>> print(fix_bad_encoding('único')) + único + + >>> print(fix_bad_encoding('This text is fine already :þ')) + This text is fine already :þ + + Because these characters often come from Microsoft products, we allow + for the possibility that we get not just Unicode characters 128-255, but + also Windows's conflicting idea of what characters 128-160 are. + + >>> print(fix_bad_encoding('This — should be an em dash')) + This — should be an em dash + + We might have to deal with both Windows characters and raw control + characters at the same time, especially when dealing with characters like + \x81 that have no mapping in Windows. + + >>> print(fix_bad_encoding('This text is sad .â\x81”.')) + This text is sad .⁔. + + This function even fixes multiple levels of badness: + + >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' + >>> print(fix_bad_encoding(wtf)) + ಠ_ಠ + + However, it has safeguards against fixing sequences of letters and + punctuation that can occur in valid text: + + >>> print(fix_bad_encoding('not such a fan of Charlotte Brontë…”')) + not such a fan of Charlotte Brontë…” + + Cases of genuine ambiguity can sometimes be addressed by finding other + characters that are not double-encoded, and expecting the encoding to + be consistent: + + >>> print(fix_bad_encoding('AHÅ™, the new sofa from IKEA®')) + AHÅ™, the new sofa from IKEA® + + Finally, we handle the case where the text is in a single-byte encoding + that was intended as Windows-1252 all along but read as Latin-1: + + >>> print(fix_bad_encoding('This text was never UTF-8 at all\x85')) + This text was never UTF-8 at all… + """ + if isinstance(text, bytes): + raise TypeError("This isn't even decoded into Unicode yet. " + "Decode it first.") + if len(text) == 0: + return text + + maxord = max(ord(char) for char in text) + if maxord < 128: + # Hooray! It's ASCII! + return text + else: + attempts = [] + if maxord < 256: + attempts = [ + (text, 0), + (reinterpret_latin1_as_utf8(text), 0.5), + (reinterpret_latin1_as_windows1252(text), 0.5), + #(reinterpret_latin1_as_macroman(text), 2), + ] + elif all(ord(char) in WINDOWS_1252_CODEPOINTS for char in text): + attempts = [ + (text, 0), + (reinterpret_windows1252_as_utf8(text), 0.5), + #(reinterpret_windows1252_as_macroman(text), 2), + ] + else: + # We can't imagine how this would be anything but valid text. + return text + + results = [(newtext, penalty + text_cost(newtext)) + for newtext, penalty in attempts] + + # Sort the results by badness + results.sort(key=lambda x: x[1]) + goodtext = results[0][0] + if goodtext == text: + return goodtext + else: + return fix_bad_encoding(goodtext) + + +def reinterpret_latin1_as_utf8(wrongtext): + newbytes = wrongtext.encode('latin-1', 'replace') + return newbytes.decode('utf-8', 'replace') + + +def reinterpret_windows1252_as_utf8(wrongtext): + altered_bytes = [] + for char in wrongtext: + if ord(char) in WINDOWS_1252_GREMLINS: + altered_bytes.append(char.encode('WINDOWS_1252')) + else: + altered_bytes.append(char.encode('latin-1', 'replace')) + return b''.join(altered_bytes).decode('utf-8', 'replace') + + +def reinterpret_latin1_as_macroman(wrongtext): + "Not used, because it's too risky." + newbytes = wrongtext.encode('latin1', 'replace') + return newbytes.decode('macroman', 'replace') + + +def reinterpret_windows1252_as_macroman(wrongtext): + "Not used, because it's too risky." + newbytes = wrongtext.encode('WINDOWS_1252', 'replace') + return newbytes.decode('macroman', 'replace') + + +def reinterpret_latin1_as_windows1252(wrongtext): + """ + Maybe this was always meant to be in a single-byte encoding, and it + makes the most sense in Windows-1252. + """ + return wrongtext.encode('latin-1').decode('WINDOWS_1252', 'replace') + + +def text_cost(text): + """ + Assign a cost function to the length plus weirdness of a text string. + """ + return text_badness(text) + len(text) * 4 + + +HTML_ENTITY_RE = re.compile("&#?\w+;") +def unescape_html(text): + """ + Decode all three types of HTML entities/character references. + + Code by Fredrik Lundh of effbot.org. + + >>> print(unescape_html('<tag>')) + + """ + def fixup(m): + text = m.group(0) + if text[:2] == "&#": + # character reference + try: + if text[:3] == "&#x": + return unichr(int(text[3:-1], 16)) + else: + return unichr(int(text[2:-1])) + except ValueError: + pass + else: + # named entity + try: + text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) + except KeyError: + pass + return text # leave as is + return HTML_ENTITY_RE.sub(fixup, text) + + +ANSI_RE = re.compile('\033\[((?:\d|;)*)([a-zA-Z])') +def remove_terminal_escapes(text): + """ + Strip out "ANSI" terminal escape sequences, such as those that produce + colored text on Unix. + """ + return ANSI_RE.sub('', text) + + +SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') +DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') +def uncurl_quotes(text): + """ + Replace curly quotation marks with straight equivalents. + + >>> print(uncurl_quotes('\u201chere\u2019s a test\u201d')) + "here's a test" + """ + return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text)) + + +def remove_bom(text): + """ + Remove a left-over byte-order mark. + """ + return text.lstrip(unichr(0xfeff)) diff --git a/ftfy/badness.py b/ftfy/badness.py new file mode 100644 index 00000000..b2fdda98 --- /dev/null +++ b/ftfy/badness.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import unicodedata +import re +from ftfy.chardata import (SCRIPT_LETTERS, SINGLE_BYTE_WEIRDNESS, + WINDOWS_1252_GREMLINS) + +import logging +LOG = logging.getLogger(__name__) +logging.basicConfig() + +SCRIPT_MAP = {} + +# Create a fast mapping that converts a Unicode string to a string describing +# its character classes, particularly the scripts its letters are in. +# +# Capital letters represent groups of commonly-used scripts. +# Lowercase letters represent rare scripts. +# . represents non-letters. +# Whitespace represents whitespace. +# ? represents errors. +# +# Astral characters pass through unmodified; we don't count them as script +# conflicts. They are probably intentional. + +for codepoint in xrange(0x10000): + char = unichr(codepoint) + if unicodedata.category(char).startswith('L'): + name = unicodedata.name(char) + script = name.split()[0] + if script in SCRIPT_LETTERS: + SCRIPT_MAP[codepoint] = unicode(SCRIPT_LETTERS[script]) + else: + SCRIPT_MAP[codepoint] = 'z' + elif unicodedata.category(char).startswith('Z'): + SCRIPT_MAP[codepoint] = ' ' + elif unicodedata.category(char) in ('Cn', 'Co'): + SCRIPT_MAP[codepoint] = '?' + else: + SCRIPT_MAP[codepoint] = '.' + +SCRIPT_MAP[0x09] = ' ' +SCRIPT_MAP[0x0a] = '\n' +SCRIPT_MAP[0xfffd] = '?' + +# mark weird extended characters as their own script +for codepoint in xrange(0x100): + if SINGLE_BYTE_WEIRDNESS[codepoint] >= 2: + SCRIPT_MAP[codepoint] = 'W' + +CONSISTENT_SCRIPTS_RE = re.compile(r'([A-Za-z])(\1+)') +LETTER_SEGMENTS_RE = re.compile(r'([A-Za-z]+)') +LOWERCASE_RE = re.compile(r'([a-z])') +DOUBLE_WEIRD_RE = re.compile(r'(WW+)') +GREMLINS_RE = re.compile('[' + + ''.join([unichr(codepoint) for codepoint in WINDOWS_1252_GREMLINS]) + + ']') + +WEIRD_CHARACTER_RES = [] +for i in xrange(5): + chars = [unichr(codepoint) for codepoint in range(0x80, 0x100) + if SINGLE_BYTE_WEIRDNESS[codepoint] > i] + WEIRD_CHARACTER_RES.append(re.compile(u'[' + u''.join(chars) + u']')) + + +def num_consistent_scripts(scriptdata): + """ + Count the number of times two adjacent letters are in the same script. + + Uses a "scriptdata" string as input, not the actual text. + + >>> num_consistent_scripts(u'LL AAA.') + 3 + >>> num_consistent_scripts(u'LLAAA ...') + 3 + >>> num_consistent_scripts(u'LAL') + 0 + >>> num_consistent_scripts(u'..LLL..') + 2 + >>> num_consistent_scripts(u'LWWW') + 2 + """ + matches = CONSISTENT_SCRIPTS_RE.findall(scriptdata) + total = 0 + for first, rest in matches: + total += len(rest) + return total + + +def num_inconsistent_scripts(scriptdata): + """ + Count the number of times two adjacent letters are in different scripts, + or are both marked as 'weird'. + + Uses a "scriptdata" string as input, not the actual text. + + >>> num_inconsistent_scripts(u'LL AAA.') + 0 + >>> num_inconsistent_scripts(u'LLAAA ...') + 1 + >>> num_inconsistent_scripts(u'LAL') + 2 + >>> num_inconsistent_scripts(u'..LLL..') + 0 + >>> num_inconsistent_scripts(u'LWWW') + 3 + """ + # First, count the number of times two letters are adjacent + letter_segments = LETTER_SEGMENTS_RE.findall(scriptdata) + adjacent_letters = 0 + for seg in letter_segments: + adjacent_letters += len(seg) - 1 + + # Then subtract out the number of times the scripts are consistent, + # but first add back in adjacent weird characters + double_weird_segments = DOUBLE_WEIRD_RE.findall(scriptdata) + for seg in double_weird_segments: + adjacent_letters += len(seg) - 1 + + return adjacent_letters - num_consistent_scripts(scriptdata) + + +def script_obscurity(scriptdata): + """ + Count the number of characters in obscure scripts. Characters in very + obscure scripts count twice as much. + + >>> script_obscurity(u'LWWW') + 0 + >>> script_obscurity(u'Llkzz') + 6 + """ + return len(LOWERCASE_RE.findall(scriptdata)) + scriptdata.count('z') + + +def character_weirdness(text): + """ + Sum the weirdness of all the single-byte characters in this text. + + >>> character_weirdness(u'test') + 0 + >>> character_weirdness(u'wúút') + 0 + >>> character_weirdness(u'\x81\x81') + 10 + """ + total = 0 + for weird_re in WEIRD_CHARACTER_RES: + found = weird_re.findall(text) + total += len(found) + return total + + +def text_badness(text): + """ + Count the total badness of a string, which helps to determine when an + encoding has gone wrong. + + Obvious problems (badness = 100): + - The replacement character \ufffd, indicating a decoding error + - Unassigned or private-use Unicode characters + + Very weird things (badness = 10): + - Adjacent letters from two different scripts + - Letters adjacent to obscure single-byte symbols + - Obscure single-byte symbols adjacent to each other + - Improbable control characters, such as 0x81 + + Moderately weird things: + - Improbable single-byte characters, such as ƒ or ¬ + - Letters in somewhat rare scripts (they'll still probably look better than + they would in the wrong encoding) + """ + scriptdata = text.translate(SCRIPT_MAP) + badness = character_weirdness(text) + script_obscurity(scriptdata) + badness += 10 * num_inconsistent_scripts(scriptdata) + badness += 100 * scriptdata.count(u'?') + LOG.info('%r has badness %r' % (text, badness)) + return badness diff --git a/ftfy/chardata.py b/ftfy/chardata.py new file mode 100644 index 00000000..b82c37a4 --- /dev/null +++ b/ftfy/chardata.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import unicodedata +import sys +if sys.version_info.major == 3: + unichr = chr + +# Start with an inventory of "gremlins", which are characters from all over +# Unicode that Windows has instead assigned to the control characters +# 0x80-0x9F. We might encounter them in their Unicode forms and have to figure +# out what they were originally. + +WINDOWS_1252_GREMLINS = [ + # adapted from http://effbot.org/zone/unicode-gremlins.htm + 0x0152, # LATIN CAPITAL LIGATURE OE + 0x0153, # LATIN SMALL LIGATURE OE + 0x0160, # LATIN CAPITAL LETTER S WITH CARON + 0x0161, # LATIN SMALL LETTER S WITH CARON + 0x0178, # LATIN CAPITAL LETTER Y WITH DIAERESIS + 0x017E, # LATIN SMALL LETTER Z WITH CARON + 0x017D, # LATIN CAPITAL LETTER Z WITH CARON + 0x0192, # LATIN SMALL LETTER F WITH HOOK + 0x02C6, # MODIFIER LETTER CIRCUMFLEX ACCENT + 0x02DC, # SMALL TILDE + 0x2013, # EN DASH + 0x2014, # EM DASH + 0x201A, # SINGLE LOW-9 QUOTATION MARK + 0x201C, # LEFT DOUBLE QUOTATION MARK + 0x201D, # RIGHT DOUBLE QUOTATION MARK + 0x201E, # DOUBLE LOW-9 QUOTATION MARK + 0x2018, # LEFT SINGLE QUOTATION MARK + 0x2019, # RIGHT SINGLE QUOTATION MARK + 0x2020, # DAGGER + 0x2021, # DOUBLE DAGGER + 0x2022, # BULLET + 0x2026, # HORIZONTAL ELLIPSIS + 0x2030, # PER MILLE SIGN + 0x2039, # SINGLE LEFT-POINTING ANGLE QUOTATION MARK + 0x203A, # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + 0x20AC, # EURO SIGN + 0x2122, # TRADE MARK SIGN +] + +# a list of Unicode characters that might appear in Windows-1252 text +WINDOWS_1252_CODEPOINTS = list(range(256)) + WINDOWS_1252_GREMLINS + +# Rank the characters typically represented by a single byte -- that is, in +# Latin-1 or Windows-1252 -- by how weird it would be to see them in running +# text. +# +# 0 = not weird at all +# 1 = rare punctuation or rare letter that someone could certainly +# have a good reason to use. All Windows-1252 gremlins are at least +# weirdness 1. +# 2 = things that probably don't appear next to letters or other +# symbols, such as math or currency symbols +# 3 = obscure symbols that nobody would go out of their way to use +# (includes symbols that were replaced in ISO-8859-15) +# 4 = why would you use this? +# 5 = unprintable control character +# +# The Portuguese letter à (0xc3) is marked as weird because it would usually +# appear in the middle of a word in actual Portuguese, and meanwhile it +# appears in the mis-encodings of many common characters. + +SINGLE_BYTE_WEIRDNESS = ( +# 0 1 2 3 4 5 6 7 8 9 a b c d e f + 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 5, 5, 5, 5, 5, # 0x00 + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, # 0x10 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x20 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x30 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x40 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x50 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x60 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, # 0x70 + 2, 5, 1, 4, 1, 1, 3, 3, 4, 3, 1, 1, 1, 5, 1, 5, # 0x80 + 5, 1, 1, 1, 1, 3, 1, 1, 4, 1, 1, 1, 1, 5, 1, 1, # 0x90 + 1, 0, 2, 2, 3, 2, 4, 2, 4, 2, 2, 0, 3, 1, 1, 4, # 0xa0 + 2, 2, 3, 3, 4, 3, 3, 2, 4, 4, 4, 0, 3, 3, 3, 0, # 0xb0 + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0xc0 + 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, # 0xd0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0xe0 + 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, # 0xf0 +) + +# Pre-cache the Unicode data saying which of these first 256 characters are +# letters. We'll need it often. +SINGLE_BYTE_LETTERS = [ + unicodedata.category(unichr(i)).startswith('L') + for i in range(256) +] + +# A table telling us how to interpret the first word of a letter's Unicode +# name. The number indicates how frequently we expect this script to be used +# on computers. Many scripts not included here are assumed to have a frequency +# of "0" -- if you're going to write in Linear B using Unicode, you're +# probably aware enough of encoding issues to get it right. +# +# The lowercase name is a general category -- for example, Han characters and +# Hiragana characters are very frequently adjacent in Japanese, so they all go +# into category 'cjk'. Letters of different categories are assumed not to +# appear next to each other often. +SCRIPT_TABLE = { + 'LATIN': (3, 'latin'), + 'CJK': (2, 'cjk'), + 'ARABIC': (2, 'arabic'), + 'CYRILLIC': (2, 'cyrillic'), + 'GREEK': (2, 'greek'), + 'HEBREW': (2, 'hebrew'), + 'KATAKANA': (2, 'cjk'), + 'HIRAGANA': (2, 'cjk'), + 'HIRAGANA-KATAKANA': (2, 'cjk'), + 'HANGUL': (2, 'cjk'), + 'DEVANAGARI': (2, 'devanagari'), + 'THAI': (2, 'thai'), + 'FULLWIDTH': (2, 'cjk'), + 'MODIFIER': (2, None), + 'HALFWIDTH': (1, 'cjk'), + 'BENGALI': (1, 'bengali'), + 'LAO': (1, 'lao'), + 'KHMER': (1, 'khmer'), + 'TELUGU': (1, 'telugu'), + 'MALAYALAM': (1, 'malayalam'), + 'SINHALA': (1, 'sinhala'), + 'TAMIL': (1, 'tamil'), + 'GEORGIAN': (1, 'georgian'), + 'ARMENIAN': (1, 'armenian'), + 'KANNADA': (1, 'kannada'), # mostly used for looks of disapproval + 'MASCULINE': (1, 'latin'), + 'FEMININE': (1, 'latin') +} + +SCRIPT_LETTERS = { + 'LATIN': 'L', + 'CJK': 'E', + 'ARABIC': 'A', + 'CYRILLIC': 'C', + 'GREEK': 'G', + 'HEBREW': 'H', + 'KATAKANA': 'E', + 'HIRAGANA': 'E', + 'HIRAGANA-KATAKANA': 'E', + 'HANGUL': 'E', + 'DEVANAGARI': 'D', + 'THAI': 'T', + 'FULLWIDTH': 'E', + 'MASCULINE': 'L', + 'FEMININE': 'L', + 'MODIFIER': '.', + 'HALFWIDTH': 'C', + 'BENGALI': 'b', + 'LAO': 'l', + 'KHMER': 'k', + 'TELUGU': 't', + 'MALAYALAM': 'm', + 'SINHALA': 's', + 'TAMIL': 'a', + 'GEORGIAN': 'g', + 'ARMENIAN': 'r', + 'KANNADA': 'n', # mostly used for looks of disapproval +} + + +# A translate mapping that will strip all control characters except \t and \n. +# This incidentally has the effect of normalizing Windows \r\n line endings to +# Unix \n line endings. +CONTROL_CHARS = {} +for i in range(256): + if unicodedata.category(unichr(i)) == 'Cc': + CONTROL_CHARS[i] = None + +CONTROL_CHARS[ord('\t')] = '\t' +CONTROL_CHARS[ord('\n')] = '\n' diff --git a/ftfy/cli.py b/ftfy/cli.py new file mode 100644 index 00000000..a58a2155 --- /dev/null +++ b/ftfy/cli.py @@ -0,0 +1,26 @@ +from ftfy import fix_file +import codecs +import sys + + +def main(): + """ + Run ftfy as a command-line utility. (Requires Python 2.7 or later, or + the 'argparse' module.) + """ + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument('filename', help='file to transcode') + + args = parser.parse_args() + + # Why open in Latin-1? Because it'll never make anything worse, and + # we're about to make things better. + file = codecs.open(args.filename, encoding='latin-1') + for line in fix_file(file): + sys.stdout.write(line.encode('utf-8')) + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 7effe1fd..740554ff 100644 --- a/setup.py +++ b/setup.py @@ -2,12 +2,17 @@ setup( name="ftfy", - version = '1.0', + version='2.0', maintainer='Luminoso, LLC', maintainer_email='dev@lumino.so', - license = "MIT", - url = 'http://github.com/LuminosoInsight/ftfy', - platforms = ["any"], - description = "Fixes some problems with Unicode text after the fact", - py_modules=['ftfy'], + license="MIT", + url='http://github.com/LuminosoInsight/ftfy', + platforms=["any"], + description="Fixes some problems with Unicode text after the fact", + packages=['ftfy'], + entry_points={ + 'console_scripts': [ + 'ftfy = ftfy.cli:main' + ] + } ) From 51152868d8c1693863b92a0b92bda84526cde1fc Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 29 Jan 2013 20:46:31 -0500 Subject: [PATCH 013/502] fix python2-isms and stop logging --- .gitignore | 1 + ftfy/badness.py | 49 ++++++++++++++++++++++++------------------------- ftfy/cli.py | 2 +- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index b8f6c51a..452b164d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__ .coverage dist *.egg-info +build diff --git a/ftfy/badness.py b/ftfy/badness.py index b2fdda98..03b94690 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -6,9 +6,9 @@ from ftfy.chardata import (SCRIPT_LETTERS, SINGLE_BYTE_WEIRDNESS, WINDOWS_1252_GREMLINS) -import logging -LOG = logging.getLogger(__name__) -logging.basicConfig() +import sys +if sys.version_info.major == 3: + unichr = chr SCRIPT_MAP = {} @@ -24,13 +24,13 @@ # Astral characters pass through unmodified; we don't count them as script # conflicts. They are probably intentional. -for codepoint in xrange(0x10000): +for codepoint in range(0x10000): char = unichr(codepoint) if unicodedata.category(char).startswith('L'): name = unicodedata.name(char) script = name.split()[0] if script in SCRIPT_LETTERS: - SCRIPT_MAP[codepoint] = unicode(SCRIPT_LETTERS[script]) + SCRIPT_MAP[codepoint] = SCRIPT_LETTERS[script] else: SCRIPT_MAP[codepoint] = 'z' elif unicodedata.category(char).startswith('Z'): @@ -45,7 +45,7 @@ SCRIPT_MAP[0xfffd] = '?' # mark weird extended characters as their own script -for codepoint in xrange(0x100): +for codepoint in range(0x100): if SINGLE_BYTE_WEIRDNESS[codepoint] >= 2: SCRIPT_MAP[codepoint] = 'W' @@ -58,10 +58,10 @@ + ']') WEIRD_CHARACTER_RES = [] -for i in xrange(5): +for i in range(5): chars = [unichr(codepoint) for codepoint in range(0x80, 0x100) if SINGLE_BYTE_WEIRDNESS[codepoint] > i] - WEIRD_CHARACTER_RES.append(re.compile(u'[' + u''.join(chars) + u']')) + WEIRD_CHARACTER_RES.append(re.compile('[' + ''.join(chars) + ']')) def num_consistent_scripts(scriptdata): @@ -70,15 +70,15 @@ def num_consistent_scripts(scriptdata): Uses a "scriptdata" string as input, not the actual text. - >>> num_consistent_scripts(u'LL AAA.') + >>> num_consistent_scripts('LL AAA.') 3 - >>> num_consistent_scripts(u'LLAAA ...') + >>> num_consistent_scripts('LLAAA ...') 3 - >>> num_consistent_scripts(u'LAL') + >>> num_consistent_scripts('LAL') 0 - >>> num_consistent_scripts(u'..LLL..') + >>> num_consistent_scripts('..LLL..') 2 - >>> num_consistent_scripts(u'LWWW') + >>> num_consistent_scripts('LWWW') 2 """ matches = CONSISTENT_SCRIPTS_RE.findall(scriptdata) @@ -95,15 +95,15 @@ def num_inconsistent_scripts(scriptdata): Uses a "scriptdata" string as input, not the actual text. - >>> num_inconsistent_scripts(u'LL AAA.') + >>> num_inconsistent_scripts('LL AAA.') 0 - >>> num_inconsistent_scripts(u'LLAAA ...') + >>> num_inconsistent_scripts('LLAAA ...') 1 - >>> num_inconsistent_scripts(u'LAL') + >>> num_inconsistent_scripts('LAL') 2 - >>> num_inconsistent_scripts(u'..LLL..') + >>> num_inconsistent_scripts('..LLL..') 0 - >>> num_inconsistent_scripts(u'LWWW') + >>> num_inconsistent_scripts('LWWW') 3 """ # First, count the number of times two letters are adjacent @@ -126,9 +126,9 @@ def script_obscurity(scriptdata): Count the number of characters in obscure scripts. Characters in very obscure scripts count twice as much. - >>> script_obscurity(u'LWWW') + >>> script_obscurity('LWWW') 0 - >>> script_obscurity(u'Llkzz') + >>> script_obscurity('Llkzz') 6 """ return len(LOWERCASE_RE.findall(scriptdata)) + scriptdata.count('z') @@ -138,11 +138,11 @@ def character_weirdness(text): """ Sum the weirdness of all the single-byte characters in this text. - >>> character_weirdness(u'test') + >>> character_weirdness('test') 0 - >>> character_weirdness(u'wúút') + >>> character_weirdness('wúút') 0 - >>> character_weirdness(u'\x81\x81') + >>> character_weirdness('\x81\x81') 10 """ total = 0 @@ -175,6 +175,5 @@ def text_badness(text): scriptdata = text.translate(SCRIPT_MAP) badness = character_weirdness(text) + script_obscurity(scriptdata) badness += 10 * num_inconsistent_scripts(scriptdata) - badness += 100 * scriptdata.count(u'?') - LOG.info('%r has badness %r' % (text, badness)) + badness += 100 * scriptdata.count('?') return badness diff --git a/ftfy/cli.py b/ftfy/cli.py index a58a2155..872cea2c 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -19,7 +19,7 @@ def main(): # we're about to make things better. file = codecs.open(args.filename, encoding='latin-1') for line in fix_file(file): - sys.stdout.write(line.encode('utf-8')) + print(line.rstrip('\n')) if __name__ == '__main__': From 31df87c6c1cb1bcb56935bc3250c695e6682f8dc Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 29 Jan 2013 20:51:22 -0500 Subject: [PATCH 014/502] protect against differences between python2 and python3 standard out --- ftfy/cli.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ftfy/cli.py b/ftfy/cli.py index 872cea2c..62806e82 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -1,6 +1,8 @@ from ftfy import fix_file import codecs + import sys +ENCODE_STDOUT = (sys.version_info.major < 3) def main(): @@ -19,7 +21,10 @@ def main(): # we're about to make things better. file = codecs.open(args.filename, encoding='latin-1') for line in fix_file(file): - print(line.rstrip('\n')) + if ENCODE_STDOUT: + sys.stdout.write(line.encode('utf-8')) + else: + sys.stdout.write(line) if __name__ == '__main__': From 3a9929eae2d986c86b68290638fec7e270682a17 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 30 Jan 2013 15:46:42 -0500 Subject: [PATCH 015/502] Update documentation and error messages --- README.md | 79 ++++++++++++++++++++++++++++++++++++++++++------ ftfy/__init__.py | 47 +++++++++++++++++++++++----- 2 files changed, 109 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 169a8fff..8a72b463 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ ## ftfy: fixes text for you -This is a simple module for making text less broken. Works in Python 2.6, +This is a module for making text less broken. It works in Python 2.6, Python 3.2, or later. Its main function, `fix_text` aka `ftfy`, performs the following operations on a Unicode string: -* Detect whether the text was incorrectly encoded into UTF-8 and fix it, - as defined in `fix_bad_encoding`. +* Detect whether the text was incorrectly encoded and fix it, as defined + in `fix_bad_encoding`. * If the text is HTML-escaped but has no HTML tags, replace HTML entities with their equivalent characters. * Remove terminal escapes and color codes. @@ -24,13 +24,72 @@ a Unicode string: ASCII characters, ellipsis characters will be replaced by three periods, and the ligature 'fl' will be replaced by 'fl'. * Replace curly quotation marks with straight ones. +* Remove an unnecessary byte-order mark (which might come from Microsoft's version + of UTF-8) from the start. -If your text is inconsistently encoded, but breaks up into coherent pieces (for -example, line-by-line), then you will be better off feeding the lines into -`fix_text` individually instead of as one huge string. +### Examples -ftfy will not decode bytes into Unicode for you. You should use Python's -built-in `.decode` for that. If you don't know the encoding, you should use the -`chardet` library. + >>> from __future__ import unicode_literals + >>> from ftfy import fix_text + >>> print(fix_text('This — should be an em dash')) + This — should be an em dash -Author: Rob Speer, rob@lumino.so + >>> print(fix_text('ünicode')) + ünicode + + >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' + >>> print(fix_text(wtf)) + ಠ_ಠ + +### Non-Unicode strings + +When first using ftfy, you might be confused to find that you can't give it a +bytestring (the type of object called `str` in Python 2). + +ftfy fixes text. Treating bytestrings as text is exactly the kind of thing that +causes the Unicode problems that ftfy has to fix. Hypothetically, it might even +be able to let you do such a silly thing, and then fix the problem you just +created. But that creates a cycle of dependence on tools such as ftfy. + +So instead of silently fixing your error, ftfy ensures first that *you* are +doing it right -- which means you must give it a Unicode string as input. If +you don't, it'll point you to the +[Python Unicode HOWTO](http://docs.python.org/3/howto/unicode.html). + +If you have been given a mess of bytes from somewhere else and you need to handle +them, you can decode those bytes as Latin-1 and let ftfy take it from there. + + >>> print(fix_text(b'\x85test')) + UnicodeError: [informative error message] + + >>> print(fix_text(b'\x85test'.decode('latin-1'))) + —test + +### Encodings supported + +`ftfy` will resolve errors that result from mixing up any of the +following encodings: + +- UTF-8 +- Latin-1 +- Windows-1252 (cp1252) + +These are the three most commonly-confused encodings, because on English text +they look nearly the same. + +(Do you ever encounter files in MacRoman or DOS cp437? I could add support for +detecting and fixing text that has been through one of these encodings, but it +would come with an increased risk of "fixing" other text incorrectly. Open a +bug report on GitHub if you think it would be worthwhile to write the +heuristics to handle these encodings.) + +### Command-line usage + +ftfy now installs itself as a command line tool, for cleaning up text files +that are a mess of mangled bytes. + +You can type `ftfy FILENAME`, and it will read in FILENAME as Latin-1 text, fix +encoding problems (including re-interpreting it as UTF-8 if appropriate), and +write the result to standard out as UTF-8. + +Author: Rob Speer, rob@luminoso.com diff --git a/ftfy/__init__.py b/ftfy/__init__.py index d37095f6..258fb6fd 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -1,4 +1,11 @@ # -*- coding: utf-8 -*- +""" +ftfy: fixes text for you + +This is a module for making text less broken. See the `fix_text` function +for more information. +""" + from __future__ import unicode_literals import unicodedata @@ -19,6 +26,23 @@ # Don't try to ftfy more than 64 KiB of text at a time, to prevent # denial of service. MAXLEN = 0x10000 +BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode! + +ftfy is designed to fix problems that were introduced by handling Unicode +incorrectly. It might be able to fix the bytes you just handed it, but the +fact that you just gave a pile of bytes to a function that fixes text means +that your code is *also* handling Unicode incorrectly. + +ftfy takes Unicode text as input. You should take these bytes and decode +them from the encoding you think they are in. If you're not sure, you can +try decoding it as 'latin-1', which may work and at least won't make +anything worse. + +If you're confused by this, please read the Python Unicode HOWTO: + + http://docs.python.org/%d/howto/unicode.html +""" % sys.version_info.major + def fix_text(text, normalization='NFKC'): """ @@ -109,10 +133,21 @@ def fix_text(text, normalization='NFKC'): return ''.join(out) +ftfy = fix_text + def fix_file(file, normalization='NFKC'): + """ + Fix a file that is being read in as Unicode text (for example, using + the `codecs.open` function), and stream the resulting text one line + at a time. + """ entities = True for line in file: + if isinstance(line, bytes): + raise UnicodeError("fix_file wants Unicode as input, so that " + "it knows what to fix.\n" + "Try this: codecs.open(file, encoding='utf-8')") if '<' in line and '>' in line: entities = False yield fix_text_segment(line, normalization, entities) @@ -120,11 +155,12 @@ def fix_file(file, normalization='NFKC'): def fix_text_segment(text, normalization='NFKC', entities=True): """ - Apply fixes to a single line of text. + Apply fixes to text in a single chunk. This could be a line of text + within a larger run of `fix_text`, or it could be a larger amount + of text that you are certain is all in the same encoding. """ if isinstance(text, bytes): - raise TypeError("fix_text works on Unicode text. Please decode " - "your text first.") + raise UnicodeError(BYTES_ERROR_TEXT) text = remove_terminal_escapes(text) if entities: text = unescape_html(text) @@ -136,8 +172,6 @@ def fix_text_segment(text, normalization='NFKC', entities=True): text = remove_bom(text) return text -ftfy = fix_text_segment - def fix_bad_encoding(text): """ @@ -208,8 +242,7 @@ def fix_bad_encoding(text): This text was never UTF-8 at all… """ if isinstance(text, bytes): - raise TypeError("This isn't even decoded into Unicode yet. " - "Decode it first.") + raise UnicodeError(BYTES_ERROR_TEXT) if len(text) == 0: return text From 446ea3233c804955a401f3ca67eb0ff2378e66b9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 30 Jan 2013 18:46:05 -0500 Subject: [PATCH 016/502] add Trove classifiers; package as ftfy 2.0 --- setup.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/setup.py b/setup.py index 740554ff..2fd34d06 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,19 @@ platforms=["any"], description="Fixes some problems with Unicode text after the fact", packages=['ftfy'], + classifiers=[ + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Text Processing :: Filters", + "Development Status :: 5 - Production/Stable" + ], entry_points={ 'console_scripts': [ 'ftfy = ftfy.cli:main' From def991850ea25da54b091457ac52d3b2149e7d53 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 30 Jan 2013 19:24:47 -0500 Subject: [PATCH 017/502] refactor the script table; fix halfwidth kana bug - The script table now appears in ftfy.chardata, not ftfy.badness. - Halfwidth kana are now listed as East Asian, not typoed as Cyrillic. --- ftfy/badness.py | 43 +------------------------------------------ ftfy/chardata.py | 46 +++++++++++++++++++++++++++++++++++++++++++++- setup.py | 2 +- 3 files changed, 47 insertions(+), 44 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 03b94690..7a314356 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -1,54 +1,14 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import unicodedata import re -from ftfy.chardata import (SCRIPT_LETTERS, SINGLE_BYTE_WEIRDNESS, +from ftfy.chardata import (SCRIPT_MAP, SINGLE_BYTE_WEIRDNESS, WINDOWS_1252_GREMLINS) import sys if sys.version_info.major == 3: unichr = chr -SCRIPT_MAP = {} - -# Create a fast mapping that converts a Unicode string to a string describing -# its character classes, particularly the scripts its letters are in. -# -# Capital letters represent groups of commonly-used scripts. -# Lowercase letters represent rare scripts. -# . represents non-letters. -# Whitespace represents whitespace. -# ? represents errors. -# -# Astral characters pass through unmodified; we don't count them as script -# conflicts. They are probably intentional. - -for codepoint in range(0x10000): - char = unichr(codepoint) - if unicodedata.category(char).startswith('L'): - name = unicodedata.name(char) - script = name.split()[0] - if script in SCRIPT_LETTERS: - SCRIPT_MAP[codepoint] = SCRIPT_LETTERS[script] - else: - SCRIPT_MAP[codepoint] = 'z' - elif unicodedata.category(char).startswith('Z'): - SCRIPT_MAP[codepoint] = ' ' - elif unicodedata.category(char) in ('Cn', 'Co'): - SCRIPT_MAP[codepoint] = '?' - else: - SCRIPT_MAP[codepoint] = '.' - -SCRIPT_MAP[0x09] = ' ' -SCRIPT_MAP[0x0a] = '\n' -SCRIPT_MAP[0xfffd] = '?' - -# mark weird extended characters as their own script -for codepoint in range(0x100): - if SINGLE_BYTE_WEIRDNESS[codepoint] >= 2: - SCRIPT_MAP[codepoint] = 'W' - CONSISTENT_SCRIPTS_RE = re.compile(r'([A-Za-z])(\1+)') LETTER_SEGMENTS_RE = re.compile(r'([A-Za-z]+)') LOWERCASE_RE = re.compile(r'([a-z])') @@ -63,7 +23,6 @@ if SINGLE_BYTE_WEIRDNESS[codepoint] > i] WEIRD_CHARACTER_RES.append(re.compile('[' + ''.join(chars) + ']')) - def num_consistent_scripts(scriptdata): """ Count the number of times two adjacent letters are in the same script. diff --git a/ftfy/chardata.py b/ftfy/chardata.py index b82c37a4..fd1f1b0a 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -131,6 +131,23 @@ 'FEMININE': (1, 'latin') } +# Create a fast mapping that converts a Unicode string to a string describing +# its character classes, particularly the scripts its letters are in. +# +# Capital letters represent groups of commonly-used scripts: +# L = Latin +# E = several East Asian scripts including hanzi, kana, and Hangul +# C = Cyrillic +# etc. +# +# Lowercase letters represent rare scripts. +# . represents non-letters. +# Whitespace represents whitespace. +# ? represents errors. +# +# Astral characters pass through unmodified; we don't count them as script +# conflicts. They are probably intentional. + SCRIPT_LETTERS = { 'LATIN': 'L', 'CJK': 'E', @@ -148,7 +165,7 @@ 'MASCULINE': 'L', 'FEMININE': 'L', 'MODIFIER': '.', - 'HALFWIDTH': 'C', + 'HALFWIDTH': 'E', 'BENGALI': 'b', 'LAO': 'l', 'KHMER': 'k', @@ -162,6 +179,33 @@ } +SCRIPT_MAP = {} + +for codepoint in range(0x10000): + char = unichr(codepoint) + if unicodedata.category(char).startswith('L'): + name = unicodedata.name(char) + script = name.split()[0] + if script in SCRIPT_LETTERS: + SCRIPT_MAP[codepoint] = SCRIPT_LETTERS[script] + else: + SCRIPT_MAP[codepoint] = 'z' + elif unicodedata.category(char).startswith('Z'): + SCRIPT_MAP[codepoint] = ' ' + elif unicodedata.category(char) in ('Cn', 'Co'): + SCRIPT_MAP[codepoint] = '?' + else: + SCRIPT_MAP[codepoint] = '.' + +SCRIPT_MAP[0x09] = ' ' +SCRIPT_MAP[0x0a] = '\n' +SCRIPT_MAP[0xfffd] = '?' + +# mark weird extended characters as their own script +for codepoint in range(0x100): + if SINGLE_BYTE_WEIRDNESS[codepoint] >= 2: + SCRIPT_MAP[codepoint] = 'W' + # A translate mapping that will strip all control characters except \t and \n. # This incidentally has the effect of normalizing Windows \r\n line endings to # Unix \n line endings. diff --git a/setup.py b/setup.py index 2fd34d06..6ca6b7d6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ maintainer='Luminoso, LLC', maintainer_email='dev@lumino.so', license="MIT", - url='http://github.com/LuminosoInsight/ftfy', + url='http://github.com/LuminosoInsight/python-ftfy', platforms=["any"], description="Fixes some problems with Unicode text after the fact", packages=['ftfy'], From 66e5071d767afe033c0402feb9ac581dc6a5ee03 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 19 Mar 2013 16:20:58 -0400 Subject: [PATCH 018/502] Use sys.hexversion for version checks --- ftfy/__init__.py | 2 +- ftfy/badness.py | 2 +- ftfy/chardata.py | 42 +----------------------------------------- ftfy/cli.py | 6 +++--- 4 files changed, 6 insertions(+), 46 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 258fb6fd..73663cd7 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -16,7 +16,7 @@ from ftfy.badness import text_badness # Adjust names that have changed in Python 3 -if sys.version_info.major == 3: +if sys.hexversion >= 0x03000000: from html import entities htmlentitydefs = entities unichr = chr diff --git a/ftfy/badness.py b/ftfy/badness.py index 7a314356..59b46493 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -6,7 +6,7 @@ WINDOWS_1252_GREMLINS) import sys -if sys.version_info.major == 3: +if sys.hexversion >= 0x03000000: unichr = chr CONSISTENT_SCRIPTS_RE = re.compile(r'([A-Za-z])(\1+)') diff --git a/ftfy/chardata.py b/ftfy/chardata.py index fd1f1b0a..498e29b1 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -3,7 +3,7 @@ import unicodedata import sys -if sys.version_info.major == 3: +if sys.hexversion >= 0x03000000: unichr = chr # Start with an inventory of "gremlins", which are characters from all over @@ -91,46 +91,6 @@ for i in range(256) ] -# A table telling us how to interpret the first word of a letter's Unicode -# name. The number indicates how frequently we expect this script to be used -# on computers. Many scripts not included here are assumed to have a frequency -# of "0" -- if you're going to write in Linear B using Unicode, you're -# probably aware enough of encoding issues to get it right. -# -# The lowercase name is a general category -- for example, Han characters and -# Hiragana characters are very frequently adjacent in Japanese, so they all go -# into category 'cjk'. Letters of different categories are assumed not to -# appear next to each other often. -SCRIPT_TABLE = { - 'LATIN': (3, 'latin'), - 'CJK': (2, 'cjk'), - 'ARABIC': (2, 'arabic'), - 'CYRILLIC': (2, 'cyrillic'), - 'GREEK': (2, 'greek'), - 'HEBREW': (2, 'hebrew'), - 'KATAKANA': (2, 'cjk'), - 'HIRAGANA': (2, 'cjk'), - 'HIRAGANA-KATAKANA': (2, 'cjk'), - 'HANGUL': (2, 'cjk'), - 'DEVANAGARI': (2, 'devanagari'), - 'THAI': (2, 'thai'), - 'FULLWIDTH': (2, 'cjk'), - 'MODIFIER': (2, None), - 'HALFWIDTH': (1, 'cjk'), - 'BENGALI': (1, 'bengali'), - 'LAO': (1, 'lao'), - 'KHMER': (1, 'khmer'), - 'TELUGU': (1, 'telugu'), - 'MALAYALAM': (1, 'malayalam'), - 'SINHALA': (1, 'sinhala'), - 'TAMIL': (1, 'tamil'), - 'GEORGIAN': (1, 'georgian'), - 'ARMENIAN': (1, 'armenian'), - 'KANNADA': (1, 'kannada'), # mostly used for looks of disapproval - 'MASCULINE': (1, 'latin'), - 'FEMININE': (1, 'latin') -} - # Create a fast mapping that converts a Unicode string to a string describing # its character classes, particularly the scripts its letters are in. # diff --git a/ftfy/cli.py b/ftfy/cli.py index 62806e82..f2b970da 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -2,7 +2,7 @@ import codecs import sys -ENCODE_STDOUT = (sys.version_info.major < 3) +ENCODE_STDOUT = (sys.hexversion < 0x03000000) def main(): @@ -17,8 +17,8 @@ def main(): args = parser.parse_args() - # Why open in Latin-1? Because it'll never make anything worse, and - # we're about to make things better. + # Why open in Latin-1? Because it at least won't make encoding problems + # worse, and we're about to make things better. file = codecs.open(args.filename, encoding='latin-1') for line in fix_file(file): if ENCODE_STDOUT: From 6be73acb13dc21dbf491493f0ff681e2c34ed480 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 19 Mar 2013 17:05:17 -0400 Subject: [PATCH 019/502] Python 2.6 support is back - bump to version 2.0.1 --- ftfy/__init__.py | 2 +- ftfy/chardata.py | 20 +++++++++++++++----- setup.py | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 73663cd7..bedcd636 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -41,7 +41,7 @@ If you're confused by this, please read the Python Unicode HOWTO: http://docs.python.org/%d/howto/unicode.html -""" % sys.version_info.major +""" % sys.version_info[0] def fix_text(text, normalization='NFKC'): diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 498e29b1..8049fcc2 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -144,11 +144,21 @@ for codepoint in range(0x10000): char = unichr(codepoint) if unicodedata.category(char).startswith('L'): - name = unicodedata.name(char) - script = name.split()[0] - if script in SCRIPT_LETTERS: - SCRIPT_MAP[codepoint] = SCRIPT_LETTERS[script] - else: + try: + name = unicodedata.name(char) + script = name.split()[0] + if script in SCRIPT_LETTERS: + SCRIPT_MAP[codepoint] = SCRIPT_LETTERS[script] + else: + SCRIPT_MAP[codepoint] = 'z' + except ValueError: + # it's unfortunate that this gives subtly different results + # on Python 2.6, which is confused about the Unicode 5.1 + # Chinese range. It knows they're letters but it has no idea + # what they are named. + # + # This could be something to fix in the future, or maybe we + # just stop supporting Python 2.6 eventually. SCRIPT_MAP[codepoint] = 'z' elif unicodedata.category(char).startswith('Z'): SCRIPT_MAP[codepoint] = ' ' diff --git a/setup.py b/setup.py index 6ca6b7d6..c9706ddf 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='2.0', + version='2.0.1', maintainer='Luminoso, LLC', maintainer_email='dev@lumino.so', license="MIT", From 264bd5630c634fd37343cf5ade506aa3447f4751 Mon Sep 17 00:00:00 2001 From: Andrew Lin Date: Thu, 25 Apr 2013 17:01:45 -0400 Subject: [PATCH 020/502] Move this test from metanl. --- ftfy/test_unicode.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 ftfy/test_unicode.py diff --git a/ftfy/test_unicode.py b/ftfy/test_unicode.py new file mode 100644 index 00000000..f69084df --- /dev/null +++ b/ftfy/test_unicode.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +from ftfy import fix_bad_encoding, WINDOWS_1252_GREMLINS +import unicodedata + +# Most single-character strings which have been misencoded should be restored. +def test_all_bmp_characters(): + for index in xrange(0xa0, 0xfffd): + char = unichr(index) + # Exclude code points that are not assigned + if unicodedata.category(char) not in ('Co', 'Cn'): + garble = char.encode('utf-8').decode('latin-1') + assert fix_bad_encoding(garble) == char + +phrases = [ + u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", + u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", + u"\u2039ALLÍ ESTÁ\u203A", + u"\u2014ALLÍ ESTÁ\u2014", + u"AHÅ™, the new sofa from IKEA®", + #u"\u2014a radius of 10 Å\u2014", +] +# These phrases should not be erroneously "fixed" +def test_valid_phrases(): + for phrase in phrases: + print phrase + yield check_phrase, phrase + # make it not just confirm based on the opening punctuation + yield check_phrase, phrase[1:] + +def check_phrase(text): + assert fix_bad_encoding(text) == text, text + From 4a52f74d1fb662bc299b9c8c8fd8ff8cb91825f7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 20 Jun 2013 15:29:51 -0400 Subject: [PATCH 021/502] Fix #1: endless loop on lines longer than 64K. --- ftfy/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index bedcd636..c15ebfee 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -49,8 +49,6 @@ def fix_text(text, normalization='NFKC'): Given Unicode text as input, make its representation consistent and possibly less broken: - - Ensure that it is a Unicode string, converting from UTF-8 if - necessary. - Detect whether the text was incorrectly encoded into UTF-8 and fix it, as defined in `fix_bad_encoding`. - If the text is HTML-escaped but has no HTML tags, replace HTML entities @@ -122,14 +120,14 @@ def fix_text(text, normalization='NFKC'): # oh well. We might not be able to fix the 2**16th character. textbreak = MAXLEN - substring = text[pos:textbreak + 1] + substring = text[pos:pos + textbreak + 1] if '<' in substring and '>' in substring: # we see angle brackets together; this could be HTML entities = False out.append(fix_text_segment(substring, normalization, entities)) - pos = textbreak + 1 + pos += textbreak + 1 return ''.join(out) @@ -253,6 +251,8 @@ def fix_bad_encoding(text): else: attempts = [] if maxord < 256: + # All characters have codepoints below 256, so this could be in + # Latin-1. attempts = [ (text, 0), (reinterpret_latin1_as_utf8(text), 0.5), @@ -260,6 +260,8 @@ def fix_bad_encoding(text): #(reinterpret_latin1_as_macroman(text), 2), ] elif all(ord(char) in WINDOWS_1252_CODEPOINTS for char in text): + # All characters are in the Windows-1252 character set, so let's + # try that. attempts = [ (text, 0), (reinterpret_windows1252_as_utf8(text), 0.5), @@ -278,6 +280,8 @@ def fix_bad_encoding(text): if goodtext == text: return goodtext else: + # We changed the text. Re-run fix_bad_encoding, so we can fix + # recursive mis-encodings. return fix_bad_encoding(goodtext) From 04de9b7b5ed72caa4acd865b926a4ea5e260b4b7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 20 Jun 2013 15:32:21 -0400 Subject: [PATCH 022/502] bugfix release 2.0.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c9706ddf..2e210d83 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='2.0.1', + version='2.0.2', maintainer='Luminoso, LLC', maintainer_email='dev@lumino.so', license="MIT", From 1f576467c7aa2df54bb4210672ca8592950df664 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 28 Jun 2013 23:45:31 -0500 Subject: [PATCH 023/502] Huh. People *use* private-use characters. On StackOverflow, for example, some of Apple's private-use characters are used to illustrate Mac OS keyboard commands. It would be erroneous to consider these characters to be in need of correction. Therefore, the "unassigned" penalty now only applies to characters that are really unassigned (class Cn), not simply in a private use block (class Co). --- ftfy/badness.py | 2 +- ftfy/chardata.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 59b46493..8930b91f 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -118,7 +118,7 @@ def text_badness(text): Obvious problems (badness = 100): - The replacement character \ufffd, indicating a decoding error - - Unassigned or private-use Unicode characters + - Unassigned Unicode characters Very weird things (badness = 10): - Adjacent letters from two different scripts diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 8049fcc2..6d571f09 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -162,7 +162,7 @@ SCRIPT_MAP[codepoint] = 'z' elif unicodedata.category(char).startswith('Z'): SCRIPT_MAP[codepoint] = ' ' - elif unicodedata.category(char) in ('Cn', 'Co'): + elif unicodedata.category(char) == 'Cn': SCRIPT_MAP[codepoint] = '?' else: SCRIPT_MAP[codepoint] = '.' From 4f6a2d05ab395a43aad3541bd3571bf2aa701307 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 29 Jun 2013 02:24:11 -0500 Subject: [PATCH 024/502] A very common Japanese character was showing up as weird. The KATAKANA-HIRAGANA PROLONGED SOUND MARK is not in any sort of unusual script. --- ftfy/chardata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 6d571f09..e8d9cabb 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -118,6 +118,7 @@ 'KATAKANA': 'E', 'HIRAGANA': 'E', 'HIRAGANA-KATAKANA': 'E', + 'KATAKANA-HIRAGANA': 'E', 'HANGUL': 'E', 'DEVANAGARI': 'D', 'THAI': 'T', From 7773d86e824244ab4cf2f032e10048f1fa1fdd00 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 30 Jun 2013 02:31:24 -0500 Subject: [PATCH 025/502] WIP: simplify the logic and expand to more encodings --- ftfy/__init__.py | 290 ++++++++++++-------- ftfy/chardata.py | 251 ++++++++++++----- ftfy/single_byte_characters.txt | 471 ++++++++++++++++++++++++++++++++ 3 files changed, 832 insertions(+), 180 deletions(-) create mode 100644 ftfy/single_byte_characters.txt diff --git a/ftfy/__init__.py b/ftfy/__init__.py index c15ebfee..9b28cc34 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -11,9 +11,7 @@ import unicodedata import re import sys -from ftfy.chardata import (CONTROL_CHARS, WINDOWS_1252_CODEPOINTS, - WINDOWS_1252_GREMLINS) -from ftfy.badness import text_badness +from ftfy.chardata import possible_encoding, CHARMAPS, CHARMAP_ENCODINGS # Adjust names that have changed in Python 3 if sys.hexversion >= 0x03000000: @@ -23,6 +21,180 @@ else: import htmlentitydefs + +# Is this the right name? +def fix_text_encoding(text): + """ + TODO documentation. + + Returns a fixed version of the text, plus a data structure that + explains how it was fixed. + """ + if isinstance(text, bytes): + raise UnicodeError(BYTES_ERROR_TEXT) + if len(text) == 0: + return text, [] + + # The first plan is to return ASCII text unchanged. + if possible_encoding(text, 'ascii'): + return text, [] + + # As we go through the next step, remember the possible encodings + # that we encounter but don't successfully fix yet. We may need them + # later. + possible_1byte_encodings = [] + + # Suppose the text was supposed to be UTF-8, but it was decoded using + # a single-byte encoding instead. When these cases can be fixed, they + # are usually the correct thing to do, so try them next. + for encoding in CHARMAP_ENCODINGS: + if possible_encoding(text, encoding): + # This is an ugly-looking way to get the bytes that represent + # the text in this encoding. The reason we can't necessarily + # use .encode(encoding) is that the decoder is very likely + # to have been sloppier than Python. + # + # The decoder might have left bytes unchanged when they're not + # part of the encoding. It might represent b'\x81' as u'\x81' + # in Windows-1252, while Python would claim that using byte + # 0x81 in Windows-1252 is an error. + # + # So what we do here is we use the .translate method of Unicode + # strings, which gives us back a Unicode string using only code + # points up to 0xff. This can then be converted into the intended + # bytes by encoding it as Latin-1. + sorta_encoded_text = text.translate(CHARMAPS[encoding]) + encoded_bytes = sorta_encoded_text.encode('latin-1') + + # Now, find out if it's UTF-8. + try: + fixed = encoded_bytes.decode('utf-8') + steps = [('sloppy_encode', encoding), ('decode', 'utf-8')] + return fixed, steps + except UnicodeDecodeError: + possible_1byte_encodings.append(encoding) + + # The next most likely case is that this is Latin-1 that was intended to + # be read as Windows-1252, because those two encodings in particular are + # easily confused. + # + # We don't need to check for possibilites such as Latin-1 that was + # intended to be read as MacRoman, because it is unlikely that any + # software has that confusion. + if 'latin-1' in possible_1byte_encodings: + if 'windows-1252' in possible_1byte_encodings: + # This text is in the intersection of Latin-1 and + # Windows-1252, so it's probably legit. + return text, [] + else: + # Otherwise, it means we have characters that are in Latin-1 but + # not in Windows-1252. Those are C1 control characters. Nobody + # wants those. + encoded = text.encode('latin-1') + fixed = encoded.decode('windows-1252', errors='replace') + steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] + return fixed, steps + + # Now we need to check for a mixup between Windows-1252, cp437, + # and MacRoman. We know Latin-1 isn't in the list, because if it were, + # we would have returned already. + # + # We aren't crazy enough to try to turn text *into* Windows-1251. + # Russia, if you screw things up that much, you're on you're own. + for encoding in possible_1byte_encodings: + for target in ['windows-1252', 'cp437', 'macroman']: + result, goodness = try_reencoding(text, encoding, target) + + # A sort of prior: if it's not Windows-1252, it's less likely + if target != 'windows-1252': + goodness -= 5 + if goodness > 0: + fixed = text.encode(encoding).decode(target) + steps = [('encode', encoding), ('decode', target)] + return fixed, steps + + # We haven't returned anything by now? That's probably fine. It means + # this is probably perfectly good Unicode. + return text, [] + + +def fix_bad_encoding(text): + """ + Something you will find all over the place, in real-world text, is text + that's mistakenly encoded as utf-8, decoded in some ugly format like + latin-1 or even Windows codepage 1252, and encoded as utf-8 again. + + This causes your perfectly good Unicode-aware code to end up with garbage + text because someone else (or maybe "someone else") made a mistake. + + This function looks for the evidence of that having happened and fixes it. + It determines whether it should replace nonsense sequences of single-byte + characters that were really meant to be UTF-8 characters, and if so, turns + them into the correctly-encoded Unicode character that they were meant to + represent. + + The input to the function must be Unicode. If you don't have Unicode text, + you're not using the right tool to solve your problem. + + NOTE: the following examples are written using unmarked literal strings, + but they are Unicode text. In Python 2 we have "unicode_literals" turned + on, and in Python 3 this is always the case. + + >>> print(fix_bad_encoding('único')) + único + + >>> print(fix_bad_encoding('This text is fine already :þ')) + This text is fine already :þ + + Because these characters often come from Microsoft products, we allow + for the possibility that we get not just Unicode characters 128-255, but + also Windows's conflicting idea of what characters 128-160 are. + + >>> print(fix_bad_encoding('This — should be an em dash')) + This — should be an em dash + + We might have to deal with both Windows characters and raw control + characters at the same time, especially when dealing with characters like + \x81 that have no mapping in Windows. + + >>> print(fix_bad_encoding('This text is sad .â\x81”.')) + This text is sad .⁔. + + This function even fixes multiple levels of badness: + + >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' + >>> print(fix_bad_encoding(wtf)) + ಠ_ಠ + + However, it has safeguards against fixing sequences of letters and + punctuation that can occur in valid text: + + >>> print(fix_bad_encoding('not such a fan of Charlotte Brontë…”')) + not such a fan of Charlotte Brontë…” + + Cases of genuine ambiguity can sometimes be addressed by finding other + characters that are not double-encoded, and expecting the encoding to + be consistent: + + >>> print(fix_bad_encoding('AHÅ™, the new sofa from IKEA®')) + AHÅ™, the new sofa from IKEA® + + Finally, we handle the case where the text is in a single-byte encoding + that was intended as Windows-1252 all along but read as Latin-1: + + >>> print(fix_bad_encoding('This text was never UTF-8 at all\x85')) + This text was never UTF-8 at all… + """ + keep_going = True + while keep_going: + fixed, info = fix_text_encoding(text) + keep_going = (fixed != text) + text = fixed + + return fixed + + +# ---------------- # Don't try to ftfy more than 64 KiB of text at a time, to prevent # denial of service. MAXLEN = 0x10000 @@ -171,118 +343,6 @@ def fix_text_segment(text, normalization='NFKC', entities=True): return text -def fix_bad_encoding(text): - """ - Something you will find all over the place, in real-world text, is text - that's mistakenly encoded as utf-8, decoded in some ugly format like - latin-1 or even Windows codepage 1252, and encoded as utf-8 again. - - This causes your perfectly good Unicode-aware code to end up with garbage - text because someone else (or maybe "someone else") made a mistake. - - This function looks for the evidence of that having happened and fixes it. - It determines whether it should replace nonsense sequences of single-byte - characters that were really meant to be UTF-8 characters, and if so, turns - them into the correctly-encoded Unicode character that they were meant to - represent. - - The input to the function must be Unicode. It's not going to try to - auto-decode bytes for you -- then it would just create the problems it's - supposed to fix. - - NOTE: the following examples are written using unmarked literal strings, - but they are Unicode text. In Python 2 we have "unicode_literals" turned - on, and in Python 3 this is always the case. - - >>> print(fix_bad_encoding('único')) - único - - >>> print(fix_bad_encoding('This text is fine already :þ')) - This text is fine already :þ - - Because these characters often come from Microsoft products, we allow - for the possibility that we get not just Unicode characters 128-255, but - also Windows's conflicting idea of what characters 128-160 are. - - >>> print(fix_bad_encoding('This — should be an em dash')) - This — should be an em dash - - We might have to deal with both Windows characters and raw control - characters at the same time, especially when dealing with characters like - \x81 that have no mapping in Windows. - - >>> print(fix_bad_encoding('This text is sad .â\x81”.')) - This text is sad .⁔. - - This function even fixes multiple levels of badness: - - >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' - >>> print(fix_bad_encoding(wtf)) - ಠ_ಠ - - However, it has safeguards against fixing sequences of letters and - punctuation that can occur in valid text: - - >>> print(fix_bad_encoding('not such a fan of Charlotte Brontë…”')) - not such a fan of Charlotte Brontë…” - - Cases of genuine ambiguity can sometimes be addressed by finding other - characters that are not double-encoded, and expecting the encoding to - be consistent: - - >>> print(fix_bad_encoding('AHÅ™, the new sofa from IKEA®')) - AHÅ™, the new sofa from IKEA® - - Finally, we handle the case where the text is in a single-byte encoding - that was intended as Windows-1252 all along but read as Latin-1: - - >>> print(fix_bad_encoding('This text was never UTF-8 at all\x85')) - This text was never UTF-8 at all… - """ - if isinstance(text, bytes): - raise UnicodeError(BYTES_ERROR_TEXT) - if len(text) == 0: - return text - - maxord = max(ord(char) for char in text) - if maxord < 128: - # Hooray! It's ASCII! - return text - else: - attempts = [] - if maxord < 256: - # All characters have codepoints below 256, so this could be in - # Latin-1. - attempts = [ - (text, 0), - (reinterpret_latin1_as_utf8(text), 0.5), - (reinterpret_latin1_as_windows1252(text), 0.5), - #(reinterpret_latin1_as_macroman(text), 2), - ] - elif all(ord(char) in WINDOWS_1252_CODEPOINTS for char in text): - # All characters are in the Windows-1252 character set, so let's - # try that. - attempts = [ - (text, 0), - (reinterpret_windows1252_as_utf8(text), 0.5), - #(reinterpret_windows1252_as_macroman(text), 2), - ] - else: - # We can't imagine how this would be anything but valid text. - return text - - results = [(newtext, penalty + text_cost(newtext)) - for newtext, penalty in attempts] - - # Sort the results by badness - results.sort(key=lambda x: x[1]) - goodtext = results[0][0] - if goodtext == text: - return goodtext - else: - # We changed the text. Re-run fix_bad_encoding, so we can fix - # recursive mis-encodings. - return fix_bad_encoding(goodtext) def reinterpret_latin1_as_utf8(wrongtext): diff --git a/ftfy/chardata.py b/ftfy/chardata.py index e8d9cabb..2748e3e9 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -3,47 +3,92 @@ import unicodedata import sys +import re + if sys.hexversion >= 0x03000000: unichr = chr -# Start with an inventory of "gremlins", which are characters from all over -# Unicode that Windows has instead assigned to the control characters -# 0x80-0x9F. We might encounter them in their Unicode forms and have to figure -# out what they were originally. - -WINDOWS_1252_GREMLINS = [ - # adapted from http://effbot.org/zone/unicode-gremlins.htm - 0x0152, # LATIN CAPITAL LIGATURE OE - 0x0153, # LATIN SMALL LIGATURE OE - 0x0160, # LATIN CAPITAL LETTER S WITH CARON - 0x0161, # LATIN SMALL LETTER S WITH CARON - 0x0178, # LATIN CAPITAL LETTER Y WITH DIAERESIS - 0x017E, # LATIN SMALL LETTER Z WITH CARON - 0x017D, # LATIN CAPITAL LETTER Z WITH CARON - 0x0192, # LATIN SMALL LETTER F WITH HOOK - 0x02C6, # MODIFIER LETTER CIRCUMFLEX ACCENT - 0x02DC, # SMALL TILDE - 0x2013, # EN DASH - 0x2014, # EM DASH - 0x201A, # SINGLE LOW-9 QUOTATION MARK - 0x201C, # LEFT DOUBLE QUOTATION MARK - 0x201D, # RIGHT DOUBLE QUOTATION MARK - 0x201E, # DOUBLE LOW-9 QUOTATION MARK - 0x2018, # LEFT SINGLE QUOTATION MARK - 0x2019, # RIGHT SINGLE QUOTATION MARK - 0x2020, # DAGGER - 0x2021, # DOUBLE DAGGER - 0x2022, # BULLET - 0x2026, # HORIZONTAL ELLIPSIS - 0x2030, # PER MILLE SIGN - 0x2039, # SINGLE LEFT-POINTING ANGLE QUOTATION MARK - 0x203A, # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK - 0x20AC, # EURO SIGN - 0x2122, # TRADE MARK SIGN +# These are encodings that map each byte to a particular character. +# They are listed in order of frequency, so that more frequent +# encodings will be tried first. +CHARMAP_ENCODINGS = [ + 'windows-1252', + 'latin-1', + 'macroman', + 'cp437', + 'windows-1251', ] +CHARMAPS = {} +ENCODING_REGULAR_EXPRESSIONS = {} + +for encoding in CHARMAP_ENCODINGS: + charmap = {} + for codepoint in range(0, 0x80): + charmap[unichr(codepoint)] = unichr(codepoint) + for codepoint in range(0x80, 0x100): + char = unichr(codepoint) + encoded_char = char.encode('latin-1') + try: + decoded_char = encoded_char.decode(encoding) + except ValueError: + decoded_char = char + charmap[decoded_char] = char + + charlist = [char for char in sorted(charmap.keys()) if ord(char) >= 0x80] + regex = '^[\x00-\x7f{}]*$'.format(''.join(charlist)) + CHARMAPS[encoding] = charmap + ENCODING_REGULAR_EXPRESSIONS[encoding] = re.compile(regex) + +ENCODING_REGULAR_EXPRESSIONS['ascii'] = re.compile('^[\x00-\x7f]*') + -# a list of Unicode characters that might appear in Windows-1252 text -WINDOWS_1252_CODEPOINTS = list(range(256)) + WINDOWS_1252_GREMLINS +# Challenge questions: +# (1) How does this differ from the UTF-8 standard? +# (2) Does it matter? + +UTF8_AS_BYTES = re.compile( + b'^(' + b'[\x00-\x7f]|' + b'([\xc2-\xdf][\x80-\xbf])|' + b'(\xe0[\xa0-\xbf][\x80-\xbf])|' + b'([\xe1-\xef][\x80-\xbf][\x80-\xbf])|' + b'(\xf0[\x90-\xbf][\x80-\xbf][\x80-\xbf])|' + b'([\xf1-\xf4][\x80-\xbf][\x80-\xbf][\x80-\xbf])|' + b')*$' +) + +def possible_encoding(text, encoding): + """ + Given text and a single-byte encoding, check whether that text could have + been decoded from that single-byte encoding. + """ + return bool(ENCODING_REGULAR_EXPRESSIONS[encoding].match(text)) + + +# ---------------- + + +# What is the heuristic for comparing single-byte encodings to each other? +encodable_chars = set() +for codepoint in range(0x80, 0x100): + encoded_char = chr(codepoint).encode('latin-1') + possibilities = [] + for encoding in CHARMAP_ENCODINGS: + try: + decoded_char = encoded_char.decode(encoding) + encodable_chars.add(decoded_char) + except ValueError: + decoded_char = encoded_char.decode('latin-1') + possibilities.append(decoded_char) + print(' '.join(possibilities)) +print() + +for char in sorted(encodable_chars): + try: + name = unicodedata.name(char) + except ValueError: + name = '[unknown]' + print(char, hex(ord(char)), name) # Rank the characters typically represented by a single byte -- that is, in # Latin-1 or Windows-1252 -- by how weird it would be to see them in running @@ -64,32 +109,112 @@ # appear in the middle of a word in actual Portuguese, and meanwhile it # appears in the mis-encodings of many common characters. -SINGLE_BYTE_WEIRDNESS = ( -# 0 1 2 3 4 5 6 7 8 9 a b c d e f - 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 5, 5, 5, 5, 5, # 0x00 - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, # 0x10 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x20 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x30 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x40 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x50 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0x60 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, # 0x70 - 2, 5, 1, 4, 1, 1, 3, 3, 4, 3, 1, 1, 1, 5, 1, 5, # 0x80 - 5, 1, 1, 1, 1, 3, 1, 1, 4, 1, 1, 1, 1, 5, 1, 1, # 0x90 - 1, 0, 2, 2, 3, 2, 4, 2, 4, 2, 2, 0, 3, 1, 1, 4, # 0xa0 - 2, 2, 3, 3, 4, 3, 3, 2, 4, 4, 4, 0, 3, 3, 3, 0, # 0xb0 - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0xc0 - 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, # 0xd0 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, # 0xe0 - 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, # 0xf0 -) -# Pre-cache the Unicode data saying which of these first 256 characters are -# letters. We'll need it often. -SINGLE_BYTE_LETTERS = [ - unicodedata.category(unichr(i)).startswith('L') - for i in range(256) -] +CHARACTER_WEIRDNESS = { + # \xa0 to \xaf + '\N{NO-BREAK SPACE}': 1, + '\N{INVERTED EXCLAMATION MARK}': 1, + '\N{CENT SIGN}': 2, + '\N{POUND SIGN}': 2, + '\N{CURRENCY SIGN}': 3, + '\N{YEN SIGN}': 2, + '\N{BROKEN BAR}': 4, + '\N{SECTION SIGN}': 2, + '\N{DIAERESIS}': 4, + '\N{COPYRIGHT SIGN}': 2, + '\N{FEMININE ORDINAL INDICATOR}': 3, + '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}': 0, + '\N{NOT SIGN}': 3, + '\N{SOFT HYPHEN}': 1, + '\N{REGISTERED SIGN}': 1, + '\N{MACRON}': 4, + + # \xb0 to \xbf + '\N{DEGREE SIGN}': 2, + '\N{PLUS-MINUS SIGN}': 2, + '\N{SUPERSCRIPT TWO}': 2, + '\N{SUPERSCRIPT THREE}': 3, + '\N{ACUTE ACCENT}': 4, + '\N{MICRO SIGN}': 3, + '\N{PILCROW SIGN}': 3, + '\N{MIDDLE DOT}': 2, + '\N{CEDILLA}': 4, + '\N{SUPERSCRIPT ONE}': 4, + '\N{MASCULINE ORDINAL INDICATOR}': 4, + '\N{RIGHT POINTING DOUBLE ANGLE QUOTATION MARK}': 0, + '\N{VULGAR FRACTION ONE QUARTER}': 2, + '\N{VULGAR FRACTION ONE HALF}': 2, + '\N{VULGAR FRACTION THREE QUARTERS}': 2, + '\N{INVERTED QUESTION MARK}': 0, + + # \xc0 to \xff + '\N{LATIN CAPITAL LETTER A WITH TILDE}': 1, + '\N{LATIN CAPITAL LETTER ETH}': 1, + '\N{MULTIPLICATION SIGN}': 1, + '\N{LATIN SMALL LETTER ETH}': 1, + '\N{DIVISION SIGN}': 1, + '\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}': 1, + + # \U0100 to \U0200 + '\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}': 1, + '\N{LATIN SMALL LETTER F WITH HOOK}': 4, + + # \U0200 to \U0300 + '\N{MODIFIER LETTER CIRCUMFLEX ACCENT}': 1, + '\N{CARON}': 3, + '\N{BREVE}': 3, + '\N{DOT ABOVE}': 3, + '\N{RING ABOVE}': 3, + '\N{OGONEK}': 3, + '\N{SMALL TILDE}': 4, + '\N{DOUBLE ACUTE ACCENT}': 1, + + # \U0300 to \U0400 + '\N{GREEK CAPITAL LETTER GAMMA}': 1, + '\N{GREEK CAPITAL LETTER THETA}': 1, + '\N{GREEK CAPITAL LETTER SIGMA}': 1, + '\N{GREEK CAPITAL LETTER PHI}': 1, + '\N{GREEK CAPITAL LETTER OMEGA}': 1, + '\N{GREEK SMALL LETTER ALPHA}': 1, + '\N{GREEK SMALL LETTER DELTA}': 1, + '\N{GREEK SMALL LETTER EPSILON}': 1, + '\N{GREEK SMALL LETTER PI}': 1, + '\N{GREEK SMALL LETTER SIGMA}': 1, + '\N{GREEK SMALL LETTER TAU}': 1, + '\N{GREEK SMALL LETTER PHI}': 1, + + # \U0400 to \U0500 is fine (Cyrillic) + # \U2000 to \U2400 + '\N{DAGGER}': 2, + '\N{DOUBLE DAGGER}': 3, + '\N{PER MILLE SIGN}': 4, + '\N{FRACTION SLASH}': 2, + '\N{SUPERSCRIPT LATIN SMALL LETTER N}': 3, + '\N{PESETA SIGN}': 3, + '\N{NUMERO SIGN}': 2, + '\N{PARTIAL DIFFERENTIAL}': 1, + '\N{INCREMENT}': 1, + '\N{N-ARY PRODUCT}': 2, + '\N{N-ARY SUMMATION}': 2, + '\N{SQUARE ROOT}': 1, + '\N{INFINITY}': 1, + '\N{INTERSECTION}': 2, + '\N{INTEGRAL}': 2, + '\N{ALMOST EQUAL TO}': 1, + '\N{NOT EQUAL TO}': 1, + '\N{IDENTICAL TO}': 2, + '\N{LESS-THAN OR EQUAL TO}': 1, + '\N{GREATER-THAN OR EQUAL TO}': 1, + '\N{REVERSED NOT SIGN}': 4, + '\N{TOP HALF INTEGRAL}': 3, + '\N{BOTTOM HALF INTEGRAL}': 3, + + # \U2500 to \U2600 are mostly box drawings; they're okay + '\N{LOZENGE}': 2, + '\uf8ff': 2, # the Apple symbol + '\N{LATIN SMALL LIGATURE FI}': 1, + '\N{LATIN SMALL LIGATURE FL}': 1, +} # Create a fast mapping that converts a Unicode string to a string describing # its character classes, particularly the scripts its letters are in. @@ -172,10 +297,6 @@ SCRIPT_MAP[0x0a] = '\n' SCRIPT_MAP[0xfffd] = '?' -# mark weird extended characters as their own script -for codepoint in range(0x100): - if SINGLE_BYTE_WEIRDNESS[codepoint] >= 2: - SCRIPT_MAP[codepoint] = 'W' # A translate mapping that will strip all control characters except \t and \n. # This incidentally has the effect of normalizing Windows \r\n line endings to diff --git a/ftfy/single_byte_characters.txt b/ftfy/single_byte_characters.txt new file mode 100644 index 00000000..ea86c417 --- /dev/null +++ b/ftfy/single_byte_characters.txt @@ -0,0 +1,471 @@ +€ € Ä Ç Ђ +  Å ü Ѓ +‚ ‚ Ç é ‚ +ƒ ƒ É â ѓ +„ „ Ñ ä „ +… … Ö à … +† † Ü å † +‡ ‡ á ç ‡ +ˆ ˆ à ê € +‰ ‰ â ë ‰ +Š Š ä è Љ +‹ ‹ ã ï ‹ +Œ Œ å î Њ +  ç ì Ќ +Ž Ž é Ä Ћ +  è Å Џ +  ê É ђ +‘ ‘ ë æ ‘ +’ ’ í Æ ’ +“ “ ì ô “ +” ” î ö ” +• • ï ò • +– – ñ û – +— — ó ù — +˜ ˜ ò ÿ ˜ +™ ™ ô Ö ™ +š š ö Ü љ +› › õ ¢ › +œ œ ú £ њ +  ù ¥ ќ +ž ž û ₧ ћ +Ÿ Ÿ ü ƒ џ +    † á   +¡ ¡ ° í Ў +¢ ¢ ¢ ó ў +£ £ £ ú Ј +¤ ¤ § ñ ¤ +¥ ¥ • Ñ Ґ +¦ ¦ ¶ ª ¦ +§ § ß º § +¨ ¨ ® ¿ Ё +© © © ⌐ © +ª ª ™ ¬ Є +« « ´ ½ « +¬ ¬ ¨ ¼ ¬ +­ ­ ≠ ¡ ­ +® ® Æ « ® +¯ ¯ Ø » Ї +° ° ∞ ░ ° +± ± ± ▒ ± +² ² ≤ ▓ І +³ ³ ≥ │ і +´ ´ ¥ ┤ ґ +µ µ µ ╡ µ +¶ ¶ ∂ ╢ ¶ +· · ∑ ╖ · +¸ ¸ ∏ ╕ ё +¹ ¹ π ╣ № +º º ∫ ║ є +» » ª ╗ » +¼ ¼ º ╝ ј +½ ½ Ω ╜ Ѕ +¾ ¾ æ ╛ ѕ +¿ ¿ ø ┐ ї +À À ¿ └ А +Á Á ¡ ┴ Б +  ¬ ┬ В +à à √ ├ Г +Ä Ä ƒ ─ Д +Å Å ≈ ┼ Е +Æ Æ ∆ ╞ Ж +Ç Ç « ╟ З +È È » ╚ И +É É … ╔ Й +Ê Ê   ╩ К +Ë Ë À ╦ Л +Ì Ì Ã ╠ М +Í Í Õ ═ Н +Î Î Œ ╬ О +Ï Ï œ ╧ П +Ð Ð – ╨ Р +Ñ Ñ — ╤ С +Ò Ò “ ╥ Т +Ó Ó ” ╙ У +Ô Ô ‘ ╘ Ф +Õ Õ ’ ╒ Х +Ö Ö ÷ ╓ Ц +× × ◊ ╫ Ч +Ø Ø ÿ ╪ Ш +Ù Ù Ÿ ┘ Щ +Ú Ú ⁄ ┌ Ъ +Û Û € █ Ы +Ü Ü ‹ ▄ Ь +Ý Ý › ▌ Э +Þ Þ fi ▐ Ю +ß ß fl ▀ Я +à à ‡ α а +á á · ß б +â â ‚ Γ в +ã ã „ π г +ä ä ‰ Σ д +å å  σ е +æ æ Ê µ ж +ç ç Á τ з +è è Ë Φ и +é é È Θ й +ê ê Í Ω к +ë ë Î δ л +ì ì Ï ∞ м +í í Ì φ н +î î Ó ε о +ï ï Ô ∩ п +ð ð  ≡ р +ñ ñ Ò ± с +ò ò Ú ≥ т +ó ó Û ≤ у +ô ô Ù ⌠ ф +õ õ ı ⌡ х +ö ö ˆ ÷ ц +÷ ÷ ˜ ≈ ч +ø ø ¯ ° ш +ù ù ˘ ∙ щ +ú ú ˙ · ъ +û û ˚ √ ы +ü ü ¸ ⁿ ь +ý ý ˝ ² э +þ þ ˛ ■ ю +ÿ ÿ ˇ   я + +€ 0x80 [unknown] + 0x81 [unknown] +‚ 0x82 [unknown] +ƒ 0x83 [unknown] +„ 0x84 [unknown] +… 0x85 [unknown] +† 0x86 [unknown] +‡ 0x87 [unknown] +ˆ 0x88 [unknown] +‰ 0x89 [unknown] +Š 0x8a [unknown] +‹ 0x8b [unknown] +Œ 0x8c [unknown] + 0x8d [unknown] +Ž 0x8e [unknown] + 0x8f [unknown] + 0x90 [unknown] +‘ 0x91 [unknown] +’ 0x92 [unknown] +“ 0x93 [unknown] +” 0x94 [unknown] +• 0x95 [unknown] +– 0x96 [unknown] +— 0x97 [unknown] +˜ 0x98 [unknown] +™ 0x99 [unknown] +š 0x9a [unknown] +› 0x9b [unknown] +œ 0x9c [unknown] + 0x9d [unknown] +ž 0x9e [unknown] +Ÿ 0x9f [unknown] +  0xa0 NO-BREAK SPACE +¡ 0xa1 INVERTED EXCLAMATION MARK +¢ 0xa2 CENT SIGN +£ 0xa3 POUND SIGN +¤ 0xa4 CURRENCY SIGN +¥ 0xa5 YEN SIGN +¦ 0xa6 BROKEN BAR +§ 0xa7 SECTION SIGN +¨ 0xa8 DIAERESIS +© 0xa9 COPYRIGHT SIGN +ª 0xaa FEMININE ORDINAL INDICATOR +« 0xab LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +¬ 0xac NOT SIGN +­ 0xad SOFT HYPHEN +® 0xae REGISTERED SIGN +¯ 0xaf MACRON +° 0xb0 DEGREE SIGN +± 0xb1 PLUS-MINUS SIGN +² 0xb2 SUPERSCRIPT TWO +³ 0xb3 SUPERSCRIPT THREE +´ 0xb4 ACUTE ACCENT +µ 0xb5 MICRO SIGN +¶ 0xb6 PILCROW SIGN +· 0xb7 MIDDLE DOT +¸ 0xb8 CEDILLA +¹ 0xb9 SUPERSCRIPT ONE +º 0xba MASCULINE ORDINAL INDICATOR +» 0xbb RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +¼ 0xbc VULGAR FRACTION ONE QUARTER +½ 0xbd VULGAR FRACTION ONE HALF +¾ 0xbe VULGAR FRACTION THREE QUARTERS +¿ 0xbf INVERTED QUESTION MARK +À 0xc0 LATIN CAPITAL LETTER A WITH GRAVE +Á 0xc1 LATIN CAPITAL LETTER A WITH ACUTE + 0xc2 LATIN CAPITAL LETTER A WITH CIRCUMFLEX +à 0xc3 LATIN CAPITAL LETTER A WITH TILDE +Ä 0xc4 LATIN CAPITAL LETTER A WITH DIAERESIS +Å 0xc5 LATIN CAPITAL LETTER A WITH RING ABOVE +Æ 0xc6 LATIN CAPITAL LETTER AE +Ç 0xc7 LATIN CAPITAL LETTER C WITH CEDILLA +È 0xc8 LATIN CAPITAL LETTER E WITH GRAVE +É 0xc9 LATIN CAPITAL LETTER E WITH ACUTE +Ê 0xca LATIN CAPITAL LETTER E WITH CIRCUMFLEX +Ë 0xcb LATIN CAPITAL LETTER E WITH DIAERESIS +Ì 0xcc LATIN CAPITAL LETTER I WITH GRAVE +Í 0xcd LATIN CAPITAL LETTER I WITH ACUTE +Î 0xce LATIN CAPITAL LETTER I WITH CIRCUMFLEX +Ï 0xcf LATIN CAPITAL LETTER I WITH DIAERESIS +Ð 0xd0 LATIN CAPITAL LETTER ETH +Ñ 0xd1 LATIN CAPITAL LETTER N WITH TILDE +Ò 0xd2 LATIN CAPITAL LETTER O WITH GRAVE +Ó 0xd3 LATIN CAPITAL LETTER O WITH ACUTE +Ô 0xd4 LATIN CAPITAL LETTER O WITH CIRCUMFLEX +Õ 0xd5 LATIN CAPITAL LETTER O WITH TILDE +Ö 0xd6 LATIN CAPITAL LETTER O WITH DIAERESIS +× 0xd7 MULTIPLICATION SIGN +Ø 0xd8 LATIN CAPITAL LETTER O WITH STROKE +Ù 0xd9 LATIN CAPITAL LETTER U WITH GRAVE +Ú 0xda LATIN CAPITAL LETTER U WITH ACUTE +Û 0xdb LATIN CAPITAL LETTER U WITH CIRCUMFLEX +Ü 0xdc LATIN CAPITAL LETTER U WITH DIAERESIS +Ý 0xdd LATIN CAPITAL LETTER Y WITH ACUTE +Þ 0xde LATIN CAPITAL LETTER THORN +ß 0xdf LATIN SMALL LETTER SHARP S +à 0xe0 LATIN SMALL LETTER A WITH GRAVE +á 0xe1 LATIN SMALL LETTER A WITH ACUTE +â 0xe2 LATIN SMALL LETTER A WITH CIRCUMFLEX +ã 0xe3 LATIN SMALL LETTER A WITH TILDE +ä 0xe4 LATIN SMALL LETTER A WITH DIAERESIS +å 0xe5 LATIN SMALL LETTER A WITH RING ABOVE +æ 0xe6 LATIN SMALL LETTER AE +ç 0xe7 LATIN SMALL LETTER C WITH CEDILLA +è 0xe8 LATIN SMALL LETTER E WITH GRAVE +é 0xe9 LATIN SMALL LETTER E WITH ACUTE +ê 0xea LATIN SMALL LETTER E WITH CIRCUMFLEX +ë 0xeb LATIN SMALL LETTER E WITH DIAERESIS +ì 0xec LATIN SMALL LETTER I WITH GRAVE +í 0xed LATIN SMALL LETTER I WITH ACUTE +î 0xee LATIN SMALL LETTER I WITH CIRCUMFLEX +ï 0xef LATIN SMALL LETTER I WITH DIAERESIS +ð 0xf0 LATIN SMALL LETTER ETH +ñ 0xf1 LATIN SMALL LETTER N WITH TILDE +ò 0xf2 LATIN SMALL LETTER O WITH GRAVE +ó 0xf3 LATIN SMALL LETTER O WITH ACUTE +ô 0xf4 LATIN SMALL LETTER O WITH CIRCUMFLEX +õ 0xf5 LATIN SMALL LETTER O WITH TILDE +ö 0xf6 LATIN SMALL LETTER O WITH DIAERESIS +÷ 0xf7 DIVISION SIGN +ø 0xf8 LATIN SMALL LETTER O WITH STROKE +ù 0xf9 LATIN SMALL LETTER U WITH GRAVE +ú 0xfa LATIN SMALL LETTER U WITH ACUTE +û 0xfb LATIN SMALL LETTER U WITH CIRCUMFLEX +ü 0xfc LATIN SMALL LETTER U WITH DIAERESIS +ý 0xfd LATIN SMALL LETTER Y WITH ACUTE +þ 0xfe LATIN SMALL LETTER THORN +ÿ 0xff LATIN SMALL LETTER Y WITH DIAERESIS +ı 0x131 LATIN SMALL LETTER DOTLESS I +Œ 0x152 LATIN CAPITAL LIGATURE OE +œ 0x153 LATIN SMALL LIGATURE OE +Š 0x160 LATIN CAPITAL LETTER S WITH CARON +š 0x161 LATIN SMALL LETTER S WITH CARON +Ÿ 0x178 LATIN CAPITAL LETTER Y WITH DIAERESIS +Ž 0x17d LATIN CAPITAL LETTER Z WITH CARON +ž 0x17e LATIN SMALL LETTER Z WITH CARON +ƒ 0x192 LATIN SMALL LETTER F WITH HOOK +ˆ 0x2c6 MODIFIER LETTER CIRCUMFLEX ACCENT +ˇ 0x2c7 CARON +˘ 0x2d8 BREVE +˙ 0x2d9 DOT ABOVE +˚ 0x2da RING ABOVE +˛ 0x2db OGONEK +˜ 0x2dc SMALL TILDE +˝ 0x2dd DOUBLE ACUTE ACCENT +Γ 0x393 GREEK CAPITAL LETTER GAMMA +Θ 0x398 GREEK CAPITAL LETTER THETA +Σ 0x3a3 GREEK CAPITAL LETTER SIGMA +Φ 0x3a6 GREEK CAPITAL LETTER PHI +Ω 0x3a9 GREEK CAPITAL LETTER OMEGA +α 0x3b1 GREEK SMALL LETTER ALPHA +δ 0x3b4 GREEK SMALL LETTER DELTA +ε 0x3b5 GREEK SMALL LETTER EPSILON +π 0x3c0 GREEK SMALL LETTER PI +σ 0x3c3 GREEK SMALL LETTER SIGMA +τ 0x3c4 GREEK SMALL LETTER TAU +φ 0x3c6 GREEK SMALL LETTER PHI +Ё 0x401 CYRILLIC CAPITAL LETTER IO +Ђ 0x402 CYRILLIC CAPITAL LETTER DJE +Ѓ 0x403 CYRILLIC CAPITAL LETTER GJE +Є 0x404 CYRILLIC CAPITAL LETTER UKRAINIAN IE +Ѕ 0x405 CYRILLIC CAPITAL LETTER DZE +І 0x406 CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I +Ї 0x407 CYRILLIC CAPITAL LETTER YI +Ј 0x408 CYRILLIC CAPITAL LETTER JE +Љ 0x409 CYRILLIC CAPITAL LETTER LJE +Њ 0x40a CYRILLIC CAPITAL LETTER NJE +Ћ 0x40b CYRILLIC CAPITAL LETTER TSHE +Ќ 0x40c CYRILLIC CAPITAL LETTER KJE +Ў 0x40e CYRILLIC CAPITAL LETTER SHORT U +Џ 0x40f CYRILLIC CAPITAL LETTER DZHE +А 0x410 CYRILLIC CAPITAL LETTER A +Б 0x411 CYRILLIC CAPITAL LETTER BE +В 0x412 CYRILLIC CAPITAL LETTER VE +Г 0x413 CYRILLIC CAPITAL LETTER GHE +Д 0x414 CYRILLIC CAPITAL LETTER DE +Е 0x415 CYRILLIC CAPITAL LETTER IE +Ж 0x416 CYRILLIC CAPITAL LETTER ZHE +З 0x417 CYRILLIC CAPITAL LETTER ZE +И 0x418 CYRILLIC CAPITAL LETTER I +Й 0x419 CYRILLIC CAPITAL LETTER SHORT I +К 0x41a CYRILLIC CAPITAL LETTER KA +Л 0x41b CYRILLIC CAPITAL LETTER EL +М 0x41c CYRILLIC CAPITAL LETTER EM +Н 0x41d CYRILLIC CAPITAL LETTER EN +О 0x41e CYRILLIC CAPITAL LETTER O +П 0x41f CYRILLIC CAPITAL LETTER PE +Р 0x420 CYRILLIC CAPITAL LETTER ER +С 0x421 CYRILLIC CAPITAL LETTER ES +Т 0x422 CYRILLIC CAPITAL LETTER TE +У 0x423 CYRILLIC CAPITAL LETTER U +Ф 0x424 CYRILLIC CAPITAL LETTER EF +Х 0x425 CYRILLIC CAPITAL LETTER HA +Ц 0x426 CYRILLIC CAPITAL LETTER TSE +Ч 0x427 CYRILLIC CAPITAL LETTER CHE +Ш 0x428 CYRILLIC CAPITAL LETTER SHA +Щ 0x429 CYRILLIC CAPITAL LETTER SHCHA +Ъ 0x42a CYRILLIC CAPITAL LETTER HARD SIGN +Ы 0x42b CYRILLIC CAPITAL LETTER YERU +Ь 0x42c CYRILLIC CAPITAL LETTER SOFT SIGN +Э 0x42d CYRILLIC CAPITAL LETTER E +Ю 0x42e CYRILLIC CAPITAL LETTER YU +Я 0x42f CYRILLIC CAPITAL LETTER YA +а 0x430 CYRILLIC SMALL LETTER A +б 0x431 CYRILLIC SMALL LETTER BE +в 0x432 CYRILLIC SMALL LETTER VE +г 0x433 CYRILLIC SMALL LETTER GHE +д 0x434 CYRILLIC SMALL LETTER DE +е 0x435 CYRILLIC SMALL LETTER IE +ж 0x436 CYRILLIC SMALL LETTER ZHE +з 0x437 CYRILLIC SMALL LETTER ZE +и 0x438 CYRILLIC SMALL LETTER I +й 0x439 CYRILLIC SMALL LETTER SHORT I +к 0x43a CYRILLIC SMALL LETTER KA +л 0x43b CYRILLIC SMALL LETTER EL +м 0x43c CYRILLIC SMALL LETTER EM +н 0x43d CYRILLIC SMALL LETTER EN +о 0x43e CYRILLIC SMALL LETTER O +п 0x43f CYRILLIC SMALL LETTER PE +р 0x440 CYRILLIC SMALL LETTER ER +с 0x441 CYRILLIC SMALL LETTER ES +т 0x442 CYRILLIC SMALL LETTER TE +у 0x443 CYRILLIC SMALL LETTER U +ф 0x444 CYRILLIC SMALL LETTER EF +х 0x445 CYRILLIC SMALL LETTER HA +ц 0x446 CYRILLIC SMALL LETTER TSE +ч 0x447 CYRILLIC SMALL LETTER CHE +ш 0x448 CYRILLIC SMALL LETTER SHA +щ 0x449 CYRILLIC SMALL LETTER SHCHA +ъ 0x44a CYRILLIC SMALL LETTER HARD SIGN +ы 0x44b CYRILLIC SMALL LETTER YERU +ь 0x44c CYRILLIC SMALL LETTER SOFT SIGN +э 0x44d CYRILLIC SMALL LETTER E +ю 0x44e CYRILLIC SMALL LETTER YU +я 0x44f CYRILLIC SMALL LETTER YA +ё 0x451 CYRILLIC SMALL LETTER IO +ђ 0x452 CYRILLIC SMALL LETTER DJE +ѓ 0x453 CYRILLIC SMALL LETTER GJE +є 0x454 CYRILLIC SMALL LETTER UKRAINIAN IE +ѕ 0x455 CYRILLIC SMALL LETTER DZE +і 0x456 CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I +ї 0x457 CYRILLIC SMALL LETTER YI +ј 0x458 CYRILLIC SMALL LETTER JE +љ 0x459 CYRILLIC SMALL LETTER LJE +њ 0x45a CYRILLIC SMALL LETTER NJE +ћ 0x45b CYRILLIC SMALL LETTER TSHE +ќ 0x45c CYRILLIC SMALL LETTER KJE +ў 0x45e CYRILLIC SMALL LETTER SHORT U +џ 0x45f CYRILLIC SMALL LETTER DZHE +Ґ 0x490 CYRILLIC CAPITAL LETTER GHE WITH UPTURN +ґ 0x491 CYRILLIC SMALL LETTER GHE WITH UPTURN +– 0x2013 EN DASH +— 0x2014 EM DASH +‘ 0x2018 LEFT SINGLE QUOTATION MARK +’ 0x2019 RIGHT SINGLE QUOTATION MARK +‚ 0x201a SINGLE LOW-9 QUOTATION MARK +“ 0x201c LEFT DOUBLE QUOTATION MARK +” 0x201d RIGHT DOUBLE QUOTATION MARK +„ 0x201e DOUBLE LOW-9 QUOTATION MARK +† 0x2020 DAGGER +‡ 0x2021 DOUBLE DAGGER +• 0x2022 BULLET +… 0x2026 HORIZONTAL ELLIPSIS +‰ 0x2030 PER MILLE SIGN +‹ 0x2039 SINGLE LEFT-POINTING ANGLE QUOTATION MARK +› 0x203a SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +⁄ 0x2044 FRACTION SLASH +ⁿ 0x207f SUPERSCRIPT LATIN SMALL LETTER N +₧ 0x20a7 PESETA SIGN +€ 0x20ac EURO SIGN +№ 0x2116 NUMERO SIGN +™ 0x2122 TRADE MARK SIGN +∂ 0x2202 PARTIAL DIFFERENTIAL +∆ 0x2206 INCREMENT +∏ 0x220f N-ARY PRODUCT +∑ 0x2211 N-ARY SUMMATION +∙ 0x2219 BULLET OPERATOR +√ 0x221a SQUARE ROOT +∞ 0x221e INFINITY +∩ 0x2229 INTERSECTION +∫ 0x222b INTEGRAL +≈ 0x2248 ALMOST EQUAL TO +≠ 0x2260 NOT EQUAL TO +≡ 0x2261 IDENTICAL TO +≤ 0x2264 LESS-THAN OR EQUAL TO +≥ 0x2265 GREATER-THAN OR EQUAL TO +⌐ 0x2310 REVERSED NOT SIGN +⌠ 0x2320 TOP HALF INTEGRAL +⌡ 0x2321 BOTTOM HALF INTEGRAL +─ 0x2500 BOX DRAWINGS LIGHT HORIZONTAL +│ 0x2502 BOX DRAWINGS LIGHT VERTICAL +┌ 0x250c BOX DRAWINGS LIGHT DOWN AND RIGHT +┐ 0x2510 BOX DRAWINGS LIGHT DOWN AND LEFT +└ 0x2514 BOX DRAWINGS LIGHT UP AND RIGHT +┘ 0x2518 BOX DRAWINGS LIGHT UP AND LEFT +├ 0x251c BOX DRAWINGS LIGHT VERTICAL AND RIGHT +┤ 0x2524 BOX DRAWINGS LIGHT VERTICAL AND LEFT +┬ 0x252c BOX DRAWINGS LIGHT DOWN AND HORIZONTAL +┴ 0x2534 BOX DRAWINGS LIGHT UP AND HORIZONTAL +┼ 0x253c BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL +═ 0x2550 BOX DRAWINGS DOUBLE HORIZONTAL +║ 0x2551 BOX DRAWINGS DOUBLE VERTICAL +╒ 0x2552 BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE +╓ 0x2553 BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE +╔ 0x2554 BOX DRAWINGS DOUBLE DOWN AND RIGHT +╕ 0x2555 BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE +╖ 0x2556 BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE +╗ 0x2557 BOX DRAWINGS DOUBLE DOWN AND LEFT +╘ 0x2558 BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE +╙ 0x2559 BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE +╚ 0x255a BOX DRAWINGS DOUBLE UP AND RIGHT +╛ 0x255b BOX DRAWINGS UP SINGLE AND LEFT DOUBLE +╜ 0x255c BOX DRAWINGS UP DOUBLE AND LEFT SINGLE +╝ 0x255d BOX DRAWINGS DOUBLE UP AND LEFT +╞ 0x255e BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE +╟ 0x255f BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE +╠ 0x2560 BOX DRAWINGS DOUBLE VERTICAL AND RIGHT +╡ 0x2561 BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE +╢ 0x2562 BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE +╣ 0x2563 BOX DRAWINGS DOUBLE VERTICAL AND LEFT +╤ 0x2564 BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE +╥ 0x2565 BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE +╦ 0x2566 BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL +╧ 0x2567 BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE +╨ 0x2568 BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE +╩ 0x2569 BOX DRAWINGS DOUBLE UP AND HORIZONTAL +╪ 0x256a BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE +╫ 0x256b BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE +╬ 0x256c BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL +▀ 0x2580 UPPER HALF BLOCK +▄ 0x2584 LOWER HALF BLOCK +█ 0x2588 FULL BLOCK +▌ 0x258c LEFT HALF BLOCK +▐ 0x2590 RIGHT HALF BLOCK +░ 0x2591 LIGHT SHADE +▒ 0x2592 MEDIUM SHADE +▓ 0x2593 DARK SHADE +■ 0x25a0 BLACK SQUARE +◊ 0x25ca LOZENGE + 0xf8ff [unknown] +fi 0xfb01 LATIN SMALL LIGATURE FI +fl 0xfb02 LATIN SMALL LIGATURE FL From 6155023037aadca9ed6db9702a855e71e7f163b4 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 11 Jul 2013 18:20:42 -0400 Subject: [PATCH 026/502] WIP: simplifying the re-encoding step --- ftfy/__init__.py | 50 +- ftfy/chardata.py | 112 +++-- ftfy/single_byte_characters.txt | 846 ++++++++++++++------------------ ftfy/test_real_tweets.py | 47 ++ 4 files changed, 501 insertions(+), 554 deletions(-) create mode 100644 ftfy/test_real_tweets.py diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 9b28cc34..e92f9c61 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -11,7 +11,8 @@ import unicodedata import re import sys -from ftfy.chardata import possible_encoding, CHARMAPS, CHARMAP_ENCODINGS +from ftfy.chardata import (possible_encoding, CHARMAPS, CHARMAP_ENCODINGS, + CONFUSABLE_1BYTE_ENCODINGS) # Adjust names that have changed in Python 3 if sys.hexversion >= 0x03000000: @@ -22,7 +23,6 @@ import htmlentitydefs -# Is this the right name? def fix_text_encoding(text): """ TODO documentation. @@ -60,7 +60,8 @@ def fix_text_encoding(text): # 0x81 in Windows-1252 is an error. # # So what we do here is we use the .translate method of Unicode - # strings, which gives us back a Unicode string using only code + # strings. Using it with the character maps we have computed will + # give us back a Unicode string using only code # points up to 0xff. This can then be converted into the intended # bytes by encoding it as Latin-1. sorta_encoded_text = text.translate(CHARMAPS[encoding]) @@ -78,7 +79,7 @@ def fix_text_encoding(text): # be read as Windows-1252, because those two encodings in particular are # easily confused. # - # We don't need to check for possibilites such as Latin-1 that was + # We don't need to check for possibilities such as Latin-1 that was # intended to be read as MacRoman, because it is unlikely that any # software has that confusion. if 'latin-1' in possible_1byte_encodings: @@ -100,9 +101,9 @@ def fix_text_encoding(text): # we would have returned already. # # We aren't crazy enough to try to turn text *into* Windows-1251. - # Russia, if you screw things up that much, you're on you're own. + # Russians, if you screw things up that much, you're on you're own. for encoding in possible_1byte_encodings: - for target in ['windows-1252', 'cp437', 'macroman']: + for target in CONFUSABLE_1BYTE_ENCODINGS: result, goodness = try_reencoding(text, encoding, target) # A sort of prior: if it's not Windows-1252, it's less likely @@ -343,43 +344,6 @@ def fix_text_segment(text, normalization='NFKC', entities=True): return text - - -def reinterpret_latin1_as_utf8(wrongtext): - newbytes = wrongtext.encode('latin-1', 'replace') - return newbytes.decode('utf-8', 'replace') - - -def reinterpret_windows1252_as_utf8(wrongtext): - altered_bytes = [] - for char in wrongtext: - if ord(char) in WINDOWS_1252_GREMLINS: - altered_bytes.append(char.encode('WINDOWS_1252')) - else: - altered_bytes.append(char.encode('latin-1', 'replace')) - return b''.join(altered_bytes).decode('utf-8', 'replace') - - -def reinterpret_latin1_as_macroman(wrongtext): - "Not used, because it's too risky." - newbytes = wrongtext.encode('latin1', 'replace') - return newbytes.decode('macroman', 'replace') - - -def reinterpret_windows1252_as_macroman(wrongtext): - "Not used, because it's too risky." - newbytes = wrongtext.encode('WINDOWS_1252', 'replace') - return newbytes.decode('macroman', 'replace') - - -def reinterpret_latin1_as_windows1252(wrongtext): - """ - Maybe this was always meant to be in a single-byte encoding, and it - makes the most sense in Windows-1252. - """ - return wrongtext.encode('latin-1').decode('WINDOWS_1252', 'replace') - - def text_cost(text): """ Assign a cost function to the length plus weirdness of a text string. diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 2748e3e9..d3c7e414 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -4,6 +4,7 @@ import unicodedata import sys import re +from collections import defaultdict if sys.hexversion >= 0x03000000: unichr = chr @@ -18,53 +19,78 @@ 'cp437', 'windows-1251', ] +CONFUSABLE_1BYTE_ENCODINGS = [ + 'windows-1252', + 'macroman', + 'cp437' +] CHARMAPS = {} -ENCODING_REGULAR_EXPRESSIONS = {} -for encoding in CHARMAP_ENCODINGS: - charmap = {} - for codepoint in range(0, 0x80): - charmap[unichr(codepoint)] = unichr(codepoint) - for codepoint in range(0x80, 0x100): - char = unichr(codepoint) - encoded_char = char.encode('latin-1') - try: - decoded_char = encoded_char.decode(encoding) - except ValueError: - decoded_char = char - charmap[decoded_char] = char - charlist = [char for char in sorted(charmap.keys()) if ord(char) >= 0x80] - regex = '^[\x00-\x7f{}]*$'.format(''.join(charlist)) - CHARMAPS[encoding] = charmap - ENCODING_REGULAR_EXPRESSIONS[encoding] = re.compile(regex) +def _make_encoding_regexes(): + encoding_regular_expressions = {} -ENCODING_REGULAR_EXPRESSIONS['ascii'] = re.compile('^[\x00-\x7f]*') + for encoding in CHARMAP_ENCODINGS: + charmap = {} + for codepoint in range(0, 0x80): + charmap[unichr(codepoint)] = unichr(codepoint) + for codepoint in range(0x80, 0x100): + char = unichr(codepoint) + encoded_char = char.encode('latin-1') + try: + decoded_char = encoded_char.decode(encoding) + except ValueError: + decoded_char = char + charmap[decoded_char] = char + + charlist = [char for char in sorted(charmap.keys()) if ord(char) >= 0x80] + regex = '^[\x00-\x7f{}]*$'.format(''.join(charlist)) + CHARMAPS[encoding] = charmap + encoding_regular_expressions[encoding] = re.compile(regex) + encoding_regular_expressions['ascii'] = re.compile('^[\x00-\x7f]*') + return encoding_regular_expressions -# Challenge questions: -# (1) How does this differ from the UTF-8 standard? -# (2) Does it matter? +ENCODING_REGEXES = _make_encoding_regexes() -UTF8_AS_BYTES = re.compile( - b'^(' - b'[\x00-\x7f]|' - b'([\xc2-\xdf][\x80-\xbf])|' - b'(\xe0[\xa0-\xbf][\x80-\xbf])|' - b'([\xe1-\xef][\x80-\xbf][\x80-\xbf])|' - b'(\xf0[\x90-\xbf][\x80-\xbf][\x80-\xbf])|' - b'([\xf1-\xf4][\x80-\xbf][\x80-\xbf][\x80-\xbf])|' - b')*$' -) def possible_encoding(text, encoding): """ Given text and a single-byte encoding, check whether that text could have been decoded from that single-byte encoding. """ - return bool(ENCODING_REGULAR_EXPRESSIONS[encoding].match(text)) + return bool(ENCODING_REGEXES[encoding].match(text)) +# Make regular expressions out of categories of Unicode characters. +def _make_category_regex_ranges(): + encodable_char_set = set([unichr(codepoint) for codepoint in range(0, 0x100)]) + for codepoint in range(0x80, 0x100): + encoded_char = chr(codepoint).encode('latin-1') + for encoding in ['windows-1252', 'macroman', 'cp437']: + try: + decoded_char = encoded_char.decode(encoding) + encodable_char_set.add(decoded_char) + except ValueError: + pass + + categories = defaultdict(list) + for char in sorted(encodable_char_set): + category = unicodedata.category(char) + categories[category].append(char) + + ranges = {} + for category in categories: + ranges[category] = (''.join(categories[category]) + .replace('\\', '\\\\') + .replace('[', r'\[') + .replace(']', r'\]') + .replace('^', r'\^') + .replace('-', r'\-')) + return ranges +CATEGORY_RANGES = _make_category_regex_ranges() +print(CATEGORY_RANGES) + # ---------------- @@ -73,7 +99,7 @@ def possible_encoding(text, encoding): for codepoint in range(0x80, 0x100): encoded_char = chr(codepoint).encode('latin-1') possibilities = [] - for encoding in CHARMAP_ENCODINGS: + for encoding in ['latin-1', 'windows-1252', 'macroman', 'cp437']: try: decoded_char = encoded_char.decode(encoding) encodable_chars.add(decoded_char) @@ -88,7 +114,7 @@ def possible_encoding(text, encoding): name = unicodedata.name(char) except ValueError: name = '[unknown]' - print(char, hex(ord(char)), name) + print(char, hex(ord(char)), unicodedata.category(char), name) # Rank the characters typically represented by a single byte -- that is, in # Latin-1 or Windows-1252 -- by how weird it would be to see them in running @@ -123,7 +149,7 @@ def possible_encoding(text, encoding): '\N{DIAERESIS}': 4, '\N{COPYRIGHT SIGN}': 2, '\N{FEMININE ORDINAL INDICATOR}': 3, - '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}': 0, + # '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}': 0, '\N{NOT SIGN}': 3, '\N{SOFT HYPHEN}': 1, '\N{REGISTERED SIGN}': 1, @@ -134,14 +160,14 @@ def possible_encoding(text, encoding): '\N{PLUS-MINUS SIGN}': 2, '\N{SUPERSCRIPT TWO}': 2, '\N{SUPERSCRIPT THREE}': 3, - '\N{ACUTE ACCENT}': 4, + '\N{ACUTE ACCENT}': 3, '\N{MICRO SIGN}': 3, '\N{PILCROW SIGN}': 3, '\N{MIDDLE DOT}': 2, '\N{CEDILLA}': 4, '\N{SUPERSCRIPT ONE}': 4, '\N{MASCULINE ORDINAL INDICATOR}': 4, - '\N{RIGHT POINTING DOUBLE ANGLE QUOTATION MARK}': 0, + # '\N{RIGHT POINTING DOUBLE ANGLE QUOTATION MARK}': 0, '\N{VULGAR FRACTION ONE QUARTER}': 2, '\N{VULGAR FRACTION ONE HALF}': 2, '\N{VULGAR FRACTION THREE QUARTERS}': 2, @@ -167,9 +193,11 @@ def possible_encoding(text, encoding): '\N{RING ABOVE}': 3, '\N{OGONEK}': 3, '\N{SMALL TILDE}': 4, - '\N{DOUBLE ACUTE ACCENT}': 1, + '\N{DOUBLE ACUTE ACCENT}': 3, - # \U0300 to \U0400 + # \U0300 to \U0400: selected Greek letters that were used in cp437. + # You are probably not writing actual Greek with this small set of + # letters. '\N{GREEK CAPITAL LETTER GAMMA}': 1, '\N{GREEK CAPITAL LETTER THETA}': 1, '\N{GREEK CAPITAL LETTER SIGMA}': 1, @@ -183,7 +211,6 @@ def possible_encoding(text, encoding): '\N{GREEK SMALL LETTER TAU}': 1, '\N{GREEK SMALL LETTER PHI}': 1, - # \U0400 to \U0500 is fine (Cyrillic) # \U2000 to \U2400 '\N{DAGGER}': 2, '\N{DOUBLE DAGGER}': 3, @@ -209,13 +236,16 @@ def possible_encoding(text, encoding): '\N{TOP HALF INTEGRAL}': 3, '\N{BOTTOM HALF INTEGRAL}': 3, - # \U2500 to \U2600 are mostly box drawings; they're okay + # \U2500 to \U2600 are mostly box drawings; they're okay, or at least + # require a different heuristic '\N{LOZENGE}': 2, '\uf8ff': 2, # the Apple symbol '\N{LATIN SMALL LIGATURE FI}': 1, '\N{LATIN SMALL LIGATURE FL}': 1, } + + # Create a fast mapping that converts a Unicode string to a string describing # its character classes, particularly the scripts its letters are in. # diff --git a/ftfy/single_byte_characters.txt b/ftfy/single_byte_characters.txt index ea86c417..af46fb91 100644 --- a/ftfy/single_byte_characters.txt +++ b/ftfy/single_byte_characters.txt @@ -1,471 +1,377 @@ -€ € Ä Ç Ђ -  Å ü Ѓ -‚ ‚ Ç é ‚ -ƒ ƒ É â ѓ -„ „ Ñ ä „ -… … Ö à … -† † Ü å † -‡ ‡ á ç ‡ -ˆ ˆ à ê € -‰ ‰ â ë ‰ -Š Š ä è Љ -‹ ‹ ã ï ‹ -Œ Œ å î Њ -  ç ì Ќ -Ž Ž é Ä Ћ -  è Å Џ -  ê É ђ -‘ ‘ ë æ ‘ -’ ’ í Æ ’ -“ “ ì ô “ -” ” î ö ” -• • ï ò • -– – ñ û – -— — ó ù — -˜ ˜ ò ÿ ˜ -™ ™ ô Ö ™ -š š ö Ü љ -› › õ ¢ › -œ œ ú £ њ -  ù ¥ ќ -ž ž û ₧ ћ -Ÿ Ÿ ü ƒ џ -    † á   -¡ ¡ ° í Ў -¢ ¢ ¢ ó ў -£ £ £ ú Ј -¤ ¤ § ñ ¤ -¥ ¥ • Ñ Ґ -¦ ¦ ¶ ª ¦ -§ § ß º § -¨ ¨ ® ¿ Ё -© © © ⌐ © -ª ª ™ ¬ Є -« « ´ ½ « -¬ ¬ ¨ ¼ ¬ -­ ­ ≠ ¡ ­ -® ® Æ « ® -¯ ¯ Ø » Ї -° ° ∞ ░ ° -± ± ± ▒ ± -² ² ≤ ▓ І -³ ³ ≥ │ і -´ ´ ¥ ┤ ґ -µ µ µ ╡ µ -¶ ¶ ∂ ╢ ¶ -· · ∑ ╖ · -¸ ¸ ∏ ╕ ё -¹ ¹ π ╣ № -º º ∫ ║ є -» » ª ╗ » -¼ ¼ º ╝ ј -½ ½ Ω ╜ Ѕ -¾ ¾ æ ╛ ѕ -¿ ¿ ø ┐ ї -À À ¿ └ А -Á Á ¡ ┴ Б -  ¬ ┬ В -à à √ ├ Г -Ä Ä ƒ ─ Д -Å Å ≈ ┼ Е -Æ Æ ∆ ╞ Ж -Ç Ç « ╟ З -È È » ╚ И -É É … ╔ Й -Ê Ê   ╩ К -Ë Ë À ╦ Л -Ì Ì Ã ╠ М -Í Í Õ ═ Н -Î Î Œ ╬ О -Ï Ï œ ╧ П -Ð Ð – ╨ Р -Ñ Ñ — ╤ С -Ò Ò “ ╥ Т -Ó Ó ” ╙ У -Ô Ô ‘ ╘ Ф -Õ Õ ’ ╒ Х -Ö Ö ÷ ╓ Ц -× × ◊ ╫ Ч -Ø Ø ÿ ╪ Ш -Ù Ù Ÿ ┘ Щ -Ú Ú ⁄ ┌ Ъ -Û Û € █ Ы -Ü Ü ‹ ▄ Ь -Ý Ý › ▌ Э -Þ Þ fi ▐ Ю -ß ß fl ▀ Я -à à ‡ α а -á á · ß б -â â ‚ Γ в -ã ã „ π г -ä ä ‰ Σ д -å å  σ е -æ æ Ê µ ж -ç ç Á τ з -è è Ë Φ и -é é È Θ й -ê ê Í Ω к -ë ë Î δ л -ì ì Ï ∞ м -í í Ì φ н -î î Ó ε о -ï ï Ô ∩ п -ð ð  ≡ р -ñ ñ Ò ± с -ò ò Ú ≥ т -ó ó Û ≤ у -ô ô Ù ⌠ ф -õ õ ı ⌡ х -ö ö ˆ ÷ ц -÷ ÷ ˜ ≈ ч -ø ø ¯ ° ш -ù ù ˘ ∙ щ -ú ú ˙ · ъ -û û ˚ √ ы -ü ü ¸ ⁿ ь -ý ý ˝ ² э -þ þ ˛ ■ ю -ÿ ÿ ˇ   я +{'Sc': '$¢£¤¥₧€', 'Pi': '«‘“‹', 'Po': '!"#%&\'*,./:;?@\\\\¡§¶·¿†‡•…‰', 'Cc': '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f', 'Pc': '_', 'Sk': '\\^`¨¯´¸˘˙˚˛˜˝', 'Lu': 'ABCDEFGHIJKLMNOPQRSTUVWXYZÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞŒŠŸŽΓΘΣΦΩ', 'So': '¦©®°™⌐─│┌┐└┘├┤┬┴┼═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬▀▄█▌▐░▒▓■◊', 'Pe': ')\\]}', 'Pd': '\\-–—', 'Lo': 'ªº', 'Nd': '0123456789', 'Lm': 'ˆˇⁿ', 'Ll': 'abcdefghijklmnopqrstuvwxyzµßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿıœšžƒαδεπστφfifl', 'Cf': '\xad', 'Pf': '»’”›', 'No': '²³¹¼½¾', 'Ps': '(\\[{‚„', 'Zs': ' \xa0', 'Sm': '+<=>|~¬±×÷⁄∂∆∏∑∙√∞∩∫≈≠≡≤≥⌠⌡', 'Co': '\uf8ff'} +€ € Ä Ç +  Å ü +‚ ‚ Ç é +ƒ ƒ É â +„ „ Ñ ä +… … Ö à +† † Ü å +‡ ‡ á ç +ˆ ˆ à ê +‰ ‰ â ë +Š Š ä è +‹ ‹ ã ï +Œ Œ å î +  ç ì +Ž Ž é Ä +  è Å +  ê É +‘ ‘ ë æ +’ ’ í Æ +“ “ ì ô +” ” î ö +• • ï ò +– – ñ û +— — ó ù +˜ ˜ ò ÿ +™ ™ ô Ö +š š ö Ü +› › õ ¢ +œ œ ú £ +  ù ¥ +ž ž û ₧ +Ÿ Ÿ ü ƒ +    † á +¡ ¡ ° í +¢ ¢ ¢ ó +£ £ £ ú +¤ ¤ § ñ +¥ ¥ • Ñ +¦ ¦ ¶ ª +§ § ß º +¨ ¨ ® ¿ +© © © ⌐ +ª ª ™ ¬ +« « ´ ½ +¬ ¬ ¨ ¼ +­ ­ ≠ ¡ +® ® Æ « +¯ ¯ Ø » +° ° ∞ ░ +± ± ± ▒ +² ² ≤ ▓ +³ ³ ≥ │ +´ ´ ¥ ┤ +µ µ µ ╡ +¶ ¶ ∂ ╢ +· · ∑ ╖ +¸ ¸ ∏ ╕ +¹ ¹ π ╣ +º º ∫ ║ +» » ª ╗ +¼ ¼ º ╝ +½ ½ Ω ╜ +¾ ¾ æ ╛ +¿ ¿ ø ┐ +À À ¿ └ +Á Á ¡ ┴ +  ¬ ┬ +à à √ ├ +Ä Ä ƒ ─ +Å Å ≈ ┼ +Æ Æ ∆ ╞ +Ç Ç « ╟ +È È » ╚ +É É … ╔ +Ê Ê   ╩ +Ë Ë À ╦ +Ì Ì Ã ╠ +Í Í Õ ═ +Î Î Œ ╬ +Ï Ï œ ╧ +Ð Ð – ╨ +Ñ Ñ — ╤ +Ò Ò “ ╥ +Ó Ó ” ╙ +Ô Ô ‘ ╘ +Õ Õ ’ ╒ +Ö Ö ÷ ╓ +× × ◊ ╫ +Ø Ø ÿ ╪ +Ù Ù Ÿ ┘ +Ú Ú ⁄ ┌ +Û Û € █ +Ü Ü ‹ ▄ +Ý Ý › ▌ +Þ Þ fi ▐ +ß ß fl ▀ +à à ‡ α +á á · ß +â â ‚ Γ +ã ã „ π +ä ä ‰ Σ +å å  σ +æ æ Ê µ +ç ç Á τ +è è Ë Φ +é é È Θ +ê ê Í Ω +ë ë Î δ +ì ì Ï ∞ +í í Ì φ +î î Ó ε +ï ï Ô ∩ +ð ð  ≡ +ñ ñ Ò ± +ò ò Ú ≥ +ó ó Û ≤ +ô ô Ù ⌠ +õ õ ı ⌡ +ö ö ˆ ÷ +÷ ÷ ˜ ≈ +ø ø ¯ ° +ù ù ˘ ∙ +ú ú ˙ · +û û ˚ √ +ü ü ¸ ⁿ +ý ý ˝ ² +þ þ ˛ ■ +ÿ ÿ ˇ   -€ 0x80 [unknown] - 0x81 [unknown] -‚ 0x82 [unknown] -ƒ 0x83 [unknown] -„ 0x84 [unknown] -… 0x85 [unknown] -† 0x86 [unknown] -‡ 0x87 [unknown] -ˆ 0x88 [unknown] -‰ 0x89 [unknown] -Š 0x8a [unknown] -‹ 0x8b [unknown] -Œ 0x8c [unknown] - 0x8d [unknown] -Ž 0x8e [unknown] - 0x8f [unknown] - 0x90 [unknown] -‘ 0x91 [unknown] -’ 0x92 [unknown] -“ 0x93 [unknown] -” 0x94 [unknown] -• 0x95 [unknown] -– 0x96 [unknown] -— 0x97 [unknown] -˜ 0x98 [unknown] -™ 0x99 [unknown] -š 0x9a [unknown] -› 0x9b [unknown] -œ 0x9c [unknown] - 0x9d [unknown] -ž 0x9e [unknown] -Ÿ 0x9f [unknown] -  0xa0 NO-BREAK SPACE -¡ 0xa1 INVERTED EXCLAMATION MARK -¢ 0xa2 CENT SIGN -£ 0xa3 POUND SIGN -¤ 0xa4 CURRENCY SIGN -¥ 0xa5 YEN SIGN -¦ 0xa6 BROKEN BAR -§ 0xa7 SECTION SIGN -¨ 0xa8 DIAERESIS -© 0xa9 COPYRIGHT SIGN -ª 0xaa FEMININE ORDINAL INDICATOR -« 0xab LEFT-POINTING DOUBLE ANGLE QUOTATION MARK -¬ 0xac NOT SIGN -­ 0xad SOFT HYPHEN -® 0xae REGISTERED SIGN -¯ 0xaf MACRON -° 0xb0 DEGREE SIGN -± 0xb1 PLUS-MINUS SIGN -² 0xb2 SUPERSCRIPT TWO -³ 0xb3 SUPERSCRIPT THREE -´ 0xb4 ACUTE ACCENT -µ 0xb5 MICRO SIGN -¶ 0xb6 PILCROW SIGN -· 0xb7 MIDDLE DOT -¸ 0xb8 CEDILLA -¹ 0xb9 SUPERSCRIPT ONE -º 0xba MASCULINE ORDINAL INDICATOR -» 0xbb RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK -¼ 0xbc VULGAR FRACTION ONE QUARTER -½ 0xbd VULGAR FRACTION ONE HALF -¾ 0xbe VULGAR FRACTION THREE QUARTERS -¿ 0xbf INVERTED QUESTION MARK -À 0xc0 LATIN CAPITAL LETTER A WITH GRAVE -Á 0xc1 LATIN CAPITAL LETTER A WITH ACUTE - 0xc2 LATIN CAPITAL LETTER A WITH CIRCUMFLEX -à 0xc3 LATIN CAPITAL LETTER A WITH TILDE -Ä 0xc4 LATIN CAPITAL LETTER A WITH DIAERESIS -Å 0xc5 LATIN CAPITAL LETTER A WITH RING ABOVE -Æ 0xc6 LATIN CAPITAL LETTER AE -Ç 0xc7 LATIN CAPITAL LETTER C WITH CEDILLA -È 0xc8 LATIN CAPITAL LETTER E WITH GRAVE -É 0xc9 LATIN CAPITAL LETTER E WITH ACUTE -Ê 0xca LATIN CAPITAL LETTER E WITH CIRCUMFLEX -Ë 0xcb LATIN CAPITAL LETTER E WITH DIAERESIS -Ì 0xcc LATIN CAPITAL LETTER I WITH GRAVE -Í 0xcd LATIN CAPITAL LETTER I WITH ACUTE -Î 0xce LATIN CAPITAL LETTER I WITH CIRCUMFLEX -Ï 0xcf LATIN CAPITAL LETTER I WITH DIAERESIS -Ð 0xd0 LATIN CAPITAL LETTER ETH -Ñ 0xd1 LATIN CAPITAL LETTER N WITH TILDE -Ò 0xd2 LATIN CAPITAL LETTER O WITH GRAVE -Ó 0xd3 LATIN CAPITAL LETTER O WITH ACUTE -Ô 0xd4 LATIN CAPITAL LETTER O WITH CIRCUMFLEX -Õ 0xd5 LATIN CAPITAL LETTER O WITH TILDE -Ö 0xd6 LATIN CAPITAL LETTER O WITH DIAERESIS -× 0xd7 MULTIPLICATION SIGN -Ø 0xd8 LATIN CAPITAL LETTER O WITH STROKE -Ù 0xd9 LATIN CAPITAL LETTER U WITH GRAVE -Ú 0xda LATIN CAPITAL LETTER U WITH ACUTE -Û 0xdb LATIN CAPITAL LETTER U WITH CIRCUMFLEX -Ü 0xdc LATIN CAPITAL LETTER U WITH DIAERESIS -Ý 0xdd LATIN CAPITAL LETTER Y WITH ACUTE -Þ 0xde LATIN CAPITAL LETTER THORN -ß 0xdf LATIN SMALL LETTER SHARP S -à 0xe0 LATIN SMALL LETTER A WITH GRAVE -á 0xe1 LATIN SMALL LETTER A WITH ACUTE -â 0xe2 LATIN SMALL LETTER A WITH CIRCUMFLEX -ã 0xe3 LATIN SMALL LETTER A WITH TILDE -ä 0xe4 LATIN SMALL LETTER A WITH DIAERESIS -å 0xe5 LATIN SMALL LETTER A WITH RING ABOVE -æ 0xe6 LATIN SMALL LETTER AE -ç 0xe7 LATIN SMALL LETTER C WITH CEDILLA -è 0xe8 LATIN SMALL LETTER E WITH GRAVE -é 0xe9 LATIN SMALL LETTER E WITH ACUTE -ê 0xea LATIN SMALL LETTER E WITH CIRCUMFLEX -ë 0xeb LATIN SMALL LETTER E WITH DIAERESIS -ì 0xec LATIN SMALL LETTER I WITH GRAVE -í 0xed LATIN SMALL LETTER I WITH ACUTE -î 0xee LATIN SMALL LETTER I WITH CIRCUMFLEX -ï 0xef LATIN SMALL LETTER I WITH DIAERESIS -ð 0xf0 LATIN SMALL LETTER ETH -ñ 0xf1 LATIN SMALL LETTER N WITH TILDE -ò 0xf2 LATIN SMALL LETTER O WITH GRAVE -ó 0xf3 LATIN SMALL LETTER O WITH ACUTE -ô 0xf4 LATIN SMALL LETTER O WITH CIRCUMFLEX -õ 0xf5 LATIN SMALL LETTER O WITH TILDE -ö 0xf6 LATIN SMALL LETTER O WITH DIAERESIS -÷ 0xf7 DIVISION SIGN -ø 0xf8 LATIN SMALL LETTER O WITH STROKE -ù 0xf9 LATIN SMALL LETTER U WITH GRAVE -ú 0xfa LATIN SMALL LETTER U WITH ACUTE -û 0xfb LATIN SMALL LETTER U WITH CIRCUMFLEX -ü 0xfc LATIN SMALL LETTER U WITH DIAERESIS -ý 0xfd LATIN SMALL LETTER Y WITH ACUTE -þ 0xfe LATIN SMALL LETTER THORN -ÿ 0xff LATIN SMALL LETTER Y WITH DIAERESIS -ı 0x131 LATIN SMALL LETTER DOTLESS I -Œ 0x152 LATIN CAPITAL LIGATURE OE -œ 0x153 LATIN SMALL LIGATURE OE -Š 0x160 LATIN CAPITAL LETTER S WITH CARON -š 0x161 LATIN SMALL LETTER S WITH CARON -Ÿ 0x178 LATIN CAPITAL LETTER Y WITH DIAERESIS -Ž 0x17d LATIN CAPITAL LETTER Z WITH CARON -ž 0x17e LATIN SMALL LETTER Z WITH CARON -ƒ 0x192 LATIN SMALL LETTER F WITH HOOK -ˆ 0x2c6 MODIFIER LETTER CIRCUMFLEX ACCENT -ˇ 0x2c7 CARON -˘ 0x2d8 BREVE -˙ 0x2d9 DOT ABOVE -˚ 0x2da RING ABOVE -˛ 0x2db OGONEK -˜ 0x2dc SMALL TILDE -˝ 0x2dd DOUBLE ACUTE ACCENT -Γ 0x393 GREEK CAPITAL LETTER GAMMA -Θ 0x398 GREEK CAPITAL LETTER THETA -Σ 0x3a3 GREEK CAPITAL LETTER SIGMA -Φ 0x3a6 GREEK CAPITAL LETTER PHI -Ω 0x3a9 GREEK CAPITAL LETTER OMEGA -α 0x3b1 GREEK SMALL LETTER ALPHA -δ 0x3b4 GREEK SMALL LETTER DELTA -ε 0x3b5 GREEK SMALL LETTER EPSILON -π 0x3c0 GREEK SMALL LETTER PI -σ 0x3c3 GREEK SMALL LETTER SIGMA -τ 0x3c4 GREEK SMALL LETTER TAU -φ 0x3c6 GREEK SMALL LETTER PHI -Ё 0x401 CYRILLIC CAPITAL LETTER IO -Ђ 0x402 CYRILLIC CAPITAL LETTER DJE -Ѓ 0x403 CYRILLIC CAPITAL LETTER GJE -Є 0x404 CYRILLIC CAPITAL LETTER UKRAINIAN IE -Ѕ 0x405 CYRILLIC CAPITAL LETTER DZE -І 0x406 CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I -Ї 0x407 CYRILLIC CAPITAL LETTER YI -Ј 0x408 CYRILLIC CAPITAL LETTER JE -Љ 0x409 CYRILLIC CAPITAL LETTER LJE -Њ 0x40a CYRILLIC CAPITAL LETTER NJE -Ћ 0x40b CYRILLIC CAPITAL LETTER TSHE -Ќ 0x40c CYRILLIC CAPITAL LETTER KJE -Ў 0x40e CYRILLIC CAPITAL LETTER SHORT U -Џ 0x40f CYRILLIC CAPITAL LETTER DZHE -А 0x410 CYRILLIC CAPITAL LETTER A -Б 0x411 CYRILLIC CAPITAL LETTER BE -В 0x412 CYRILLIC CAPITAL LETTER VE -Г 0x413 CYRILLIC CAPITAL LETTER GHE -Д 0x414 CYRILLIC CAPITAL LETTER DE -Е 0x415 CYRILLIC CAPITAL LETTER IE -Ж 0x416 CYRILLIC CAPITAL LETTER ZHE -З 0x417 CYRILLIC CAPITAL LETTER ZE -И 0x418 CYRILLIC CAPITAL LETTER I -Й 0x419 CYRILLIC CAPITAL LETTER SHORT I -К 0x41a CYRILLIC CAPITAL LETTER KA -Л 0x41b CYRILLIC CAPITAL LETTER EL -М 0x41c CYRILLIC CAPITAL LETTER EM -Н 0x41d CYRILLIC CAPITAL LETTER EN -О 0x41e CYRILLIC CAPITAL LETTER O -П 0x41f CYRILLIC CAPITAL LETTER PE -Р 0x420 CYRILLIC CAPITAL LETTER ER -С 0x421 CYRILLIC CAPITAL LETTER ES -Т 0x422 CYRILLIC CAPITAL LETTER TE -У 0x423 CYRILLIC CAPITAL LETTER U -Ф 0x424 CYRILLIC CAPITAL LETTER EF -Х 0x425 CYRILLIC CAPITAL LETTER HA -Ц 0x426 CYRILLIC CAPITAL LETTER TSE -Ч 0x427 CYRILLIC CAPITAL LETTER CHE -Ш 0x428 CYRILLIC CAPITAL LETTER SHA -Щ 0x429 CYRILLIC CAPITAL LETTER SHCHA -Ъ 0x42a CYRILLIC CAPITAL LETTER HARD SIGN -Ы 0x42b CYRILLIC CAPITAL LETTER YERU -Ь 0x42c CYRILLIC CAPITAL LETTER SOFT SIGN -Э 0x42d CYRILLIC CAPITAL LETTER E -Ю 0x42e CYRILLIC CAPITAL LETTER YU -Я 0x42f CYRILLIC CAPITAL LETTER YA -а 0x430 CYRILLIC SMALL LETTER A -б 0x431 CYRILLIC SMALL LETTER BE -в 0x432 CYRILLIC SMALL LETTER VE -г 0x433 CYRILLIC SMALL LETTER GHE -д 0x434 CYRILLIC SMALL LETTER DE -е 0x435 CYRILLIC SMALL LETTER IE -ж 0x436 CYRILLIC SMALL LETTER ZHE -з 0x437 CYRILLIC SMALL LETTER ZE -и 0x438 CYRILLIC SMALL LETTER I -й 0x439 CYRILLIC SMALL LETTER SHORT I -к 0x43a CYRILLIC SMALL LETTER KA -л 0x43b CYRILLIC SMALL LETTER EL -м 0x43c CYRILLIC SMALL LETTER EM -н 0x43d CYRILLIC SMALL LETTER EN -о 0x43e CYRILLIC SMALL LETTER O -п 0x43f CYRILLIC SMALL LETTER PE -р 0x440 CYRILLIC SMALL LETTER ER -с 0x441 CYRILLIC SMALL LETTER ES -т 0x442 CYRILLIC SMALL LETTER TE -у 0x443 CYRILLIC SMALL LETTER U -ф 0x444 CYRILLIC SMALL LETTER EF -х 0x445 CYRILLIC SMALL LETTER HA -ц 0x446 CYRILLIC SMALL LETTER TSE -ч 0x447 CYRILLIC SMALL LETTER CHE -ш 0x448 CYRILLIC SMALL LETTER SHA -щ 0x449 CYRILLIC SMALL LETTER SHCHA -ъ 0x44a CYRILLIC SMALL LETTER HARD SIGN -ы 0x44b CYRILLIC SMALL LETTER YERU -ь 0x44c CYRILLIC SMALL LETTER SOFT SIGN -э 0x44d CYRILLIC SMALL LETTER E -ю 0x44e CYRILLIC SMALL LETTER YU -я 0x44f CYRILLIC SMALL LETTER YA -ё 0x451 CYRILLIC SMALL LETTER IO -ђ 0x452 CYRILLIC SMALL LETTER DJE -ѓ 0x453 CYRILLIC SMALL LETTER GJE -є 0x454 CYRILLIC SMALL LETTER UKRAINIAN IE -ѕ 0x455 CYRILLIC SMALL LETTER DZE -і 0x456 CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I -ї 0x457 CYRILLIC SMALL LETTER YI -ј 0x458 CYRILLIC SMALL LETTER JE -љ 0x459 CYRILLIC SMALL LETTER LJE -њ 0x45a CYRILLIC SMALL LETTER NJE -ћ 0x45b CYRILLIC SMALL LETTER TSHE -ќ 0x45c CYRILLIC SMALL LETTER KJE -ў 0x45e CYRILLIC SMALL LETTER SHORT U -џ 0x45f CYRILLIC SMALL LETTER DZHE -Ґ 0x490 CYRILLIC CAPITAL LETTER GHE WITH UPTURN -ґ 0x491 CYRILLIC SMALL LETTER GHE WITH UPTURN -– 0x2013 EN DASH -— 0x2014 EM DASH -‘ 0x2018 LEFT SINGLE QUOTATION MARK -’ 0x2019 RIGHT SINGLE QUOTATION MARK -‚ 0x201a SINGLE LOW-9 QUOTATION MARK -“ 0x201c LEFT DOUBLE QUOTATION MARK -” 0x201d RIGHT DOUBLE QUOTATION MARK -„ 0x201e DOUBLE LOW-9 QUOTATION MARK -† 0x2020 DAGGER -‡ 0x2021 DOUBLE DAGGER -• 0x2022 BULLET -… 0x2026 HORIZONTAL ELLIPSIS -‰ 0x2030 PER MILLE SIGN -‹ 0x2039 SINGLE LEFT-POINTING ANGLE QUOTATION MARK -› 0x203a SINGLE RIGHT-POINTING ANGLE QUOTATION MARK -⁄ 0x2044 FRACTION SLASH -ⁿ 0x207f SUPERSCRIPT LATIN SMALL LETTER N -₧ 0x20a7 PESETA SIGN -€ 0x20ac EURO SIGN -№ 0x2116 NUMERO SIGN -™ 0x2122 TRADE MARK SIGN -∂ 0x2202 PARTIAL DIFFERENTIAL -∆ 0x2206 INCREMENT -∏ 0x220f N-ARY PRODUCT -∑ 0x2211 N-ARY SUMMATION -∙ 0x2219 BULLET OPERATOR -√ 0x221a SQUARE ROOT -∞ 0x221e INFINITY -∩ 0x2229 INTERSECTION -∫ 0x222b INTEGRAL -≈ 0x2248 ALMOST EQUAL TO -≠ 0x2260 NOT EQUAL TO -≡ 0x2261 IDENTICAL TO -≤ 0x2264 LESS-THAN OR EQUAL TO -≥ 0x2265 GREATER-THAN OR EQUAL TO -⌐ 0x2310 REVERSED NOT SIGN -⌠ 0x2320 TOP HALF INTEGRAL -⌡ 0x2321 BOTTOM HALF INTEGRAL -─ 0x2500 BOX DRAWINGS LIGHT HORIZONTAL -│ 0x2502 BOX DRAWINGS LIGHT VERTICAL -┌ 0x250c BOX DRAWINGS LIGHT DOWN AND RIGHT -┐ 0x2510 BOX DRAWINGS LIGHT DOWN AND LEFT -└ 0x2514 BOX DRAWINGS LIGHT UP AND RIGHT -┘ 0x2518 BOX DRAWINGS LIGHT UP AND LEFT -├ 0x251c BOX DRAWINGS LIGHT VERTICAL AND RIGHT -┤ 0x2524 BOX DRAWINGS LIGHT VERTICAL AND LEFT -┬ 0x252c BOX DRAWINGS LIGHT DOWN AND HORIZONTAL -┴ 0x2534 BOX DRAWINGS LIGHT UP AND HORIZONTAL -┼ 0x253c BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL -═ 0x2550 BOX DRAWINGS DOUBLE HORIZONTAL -║ 0x2551 BOX DRAWINGS DOUBLE VERTICAL -╒ 0x2552 BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE -╓ 0x2553 BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE -╔ 0x2554 BOX DRAWINGS DOUBLE DOWN AND RIGHT -╕ 0x2555 BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE -╖ 0x2556 BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE -╗ 0x2557 BOX DRAWINGS DOUBLE DOWN AND LEFT -╘ 0x2558 BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE -╙ 0x2559 BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE -╚ 0x255a BOX DRAWINGS DOUBLE UP AND RIGHT -╛ 0x255b BOX DRAWINGS UP SINGLE AND LEFT DOUBLE -╜ 0x255c BOX DRAWINGS UP DOUBLE AND LEFT SINGLE -╝ 0x255d BOX DRAWINGS DOUBLE UP AND LEFT -╞ 0x255e BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE -╟ 0x255f BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE -╠ 0x2560 BOX DRAWINGS DOUBLE VERTICAL AND RIGHT -╡ 0x2561 BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE -╢ 0x2562 BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE -╣ 0x2563 BOX DRAWINGS DOUBLE VERTICAL AND LEFT -╤ 0x2564 BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE -╥ 0x2565 BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE -╦ 0x2566 BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL -╧ 0x2567 BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE -╨ 0x2568 BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE -╩ 0x2569 BOX DRAWINGS DOUBLE UP AND HORIZONTAL -╪ 0x256a BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE -╫ 0x256b BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE -╬ 0x256c BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL -▀ 0x2580 UPPER HALF BLOCK -▄ 0x2584 LOWER HALF BLOCK -█ 0x2588 FULL BLOCK -▌ 0x258c LEFT HALF BLOCK -▐ 0x2590 RIGHT HALF BLOCK -░ 0x2591 LIGHT SHADE -▒ 0x2592 MEDIUM SHADE -▓ 0x2593 DARK SHADE -■ 0x25a0 BLACK SQUARE -◊ 0x25ca LOZENGE - 0xf8ff [unknown] -fi 0xfb01 LATIN SMALL LIGATURE FI -fl 0xfb02 LATIN SMALL LIGATURE FL +€ 0x80 Cc [unknown] + 0x81 Cc [unknown] +‚ 0x82 Cc [unknown] +ƒ 0x83 Cc [unknown] +„ 0x84 Cc [unknown] +… 0x85 Cc [unknown] +† 0x86 Cc [unknown] +‡ 0x87 Cc [unknown] +ˆ 0x88 Cc [unknown] +‰ 0x89 Cc [unknown] +Š 0x8a Cc [unknown] +‹ 0x8b Cc [unknown] +Œ 0x8c Cc [unknown] + 0x8d Cc [unknown] +Ž 0x8e Cc [unknown] + 0x8f Cc [unknown] + 0x90 Cc [unknown] +‘ 0x91 Cc [unknown] +’ 0x92 Cc [unknown] +“ 0x93 Cc [unknown] +” 0x94 Cc [unknown] +• 0x95 Cc [unknown] +– 0x96 Cc [unknown] +— 0x97 Cc [unknown] +˜ 0x98 Cc [unknown] +™ 0x99 Cc [unknown] +š 0x9a Cc [unknown] +› 0x9b Cc [unknown] +œ 0x9c Cc [unknown] + 0x9d Cc [unknown] +ž 0x9e Cc [unknown] +Ÿ 0x9f Cc [unknown] +  0xa0 Zs NO-BREAK SPACE +¡ 0xa1 Po INVERTED EXCLAMATION MARK +¢ 0xa2 Sc CENT SIGN +£ 0xa3 Sc POUND SIGN +¤ 0xa4 Sc CURRENCY SIGN +¥ 0xa5 Sc YEN SIGN +¦ 0xa6 So BROKEN BAR +§ 0xa7 Po SECTION SIGN +¨ 0xa8 Sk DIAERESIS +© 0xa9 So COPYRIGHT SIGN +ª 0xaa Lo FEMININE ORDINAL INDICATOR +« 0xab Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +¬ 0xac Sm NOT SIGN +­ 0xad Cf SOFT HYPHEN +® 0xae So REGISTERED SIGN +¯ 0xaf Sk MACRON +° 0xb0 So DEGREE SIGN +± 0xb1 Sm PLUS-MINUS SIGN +² 0xb2 No SUPERSCRIPT TWO +³ 0xb3 No SUPERSCRIPT THREE +´ 0xb4 Sk ACUTE ACCENT +µ 0xb5 Ll MICRO SIGN +¶ 0xb6 Po PILCROW SIGN +· 0xb7 Po MIDDLE DOT +¸ 0xb8 Sk CEDILLA +¹ 0xb9 No SUPERSCRIPT ONE +º 0xba Lo MASCULINE ORDINAL INDICATOR +» 0xbb Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +¼ 0xbc No VULGAR FRACTION ONE QUARTER +½ 0xbd No VULGAR FRACTION ONE HALF +¾ 0xbe No VULGAR FRACTION THREE QUARTERS +¿ 0xbf Po INVERTED QUESTION MARK +À 0xc0 Lu LATIN CAPITAL LETTER A WITH GRAVE +Á 0xc1 Lu LATIN CAPITAL LETTER A WITH ACUTE + 0xc2 Lu LATIN CAPITAL LETTER A WITH CIRCUMFLEX +à 0xc3 Lu LATIN CAPITAL LETTER A WITH TILDE +Ä 0xc4 Lu LATIN CAPITAL LETTER A WITH DIAERESIS +Å 0xc5 Lu LATIN CAPITAL LETTER A WITH RING ABOVE +Æ 0xc6 Lu LATIN CAPITAL LETTER AE +Ç 0xc7 Lu LATIN CAPITAL LETTER C WITH CEDILLA +È 0xc8 Lu LATIN CAPITAL LETTER E WITH GRAVE +É 0xc9 Lu LATIN CAPITAL LETTER E WITH ACUTE +Ê 0xca Lu LATIN CAPITAL LETTER E WITH CIRCUMFLEX +Ë 0xcb Lu LATIN CAPITAL LETTER E WITH DIAERESIS +Ì 0xcc Lu LATIN CAPITAL LETTER I WITH GRAVE +Í 0xcd Lu LATIN CAPITAL LETTER I WITH ACUTE +Î 0xce Lu LATIN CAPITAL LETTER I WITH CIRCUMFLEX +Ï 0xcf Lu LATIN CAPITAL LETTER I WITH DIAERESIS +Ð 0xd0 Lu LATIN CAPITAL LETTER ETH +Ñ 0xd1 Lu LATIN CAPITAL LETTER N WITH TILDE +Ò 0xd2 Lu LATIN CAPITAL LETTER O WITH GRAVE +Ó 0xd3 Lu LATIN CAPITAL LETTER O WITH ACUTE +Ô 0xd4 Lu LATIN CAPITAL LETTER O WITH CIRCUMFLEX +Õ 0xd5 Lu LATIN CAPITAL LETTER O WITH TILDE +Ö 0xd6 Lu LATIN CAPITAL LETTER O WITH DIAERESIS +× 0xd7 Sm MULTIPLICATION SIGN +Ø 0xd8 Lu LATIN CAPITAL LETTER O WITH STROKE +Ù 0xd9 Lu LATIN CAPITAL LETTER U WITH GRAVE +Ú 0xda Lu LATIN CAPITAL LETTER U WITH ACUTE +Û 0xdb Lu LATIN CAPITAL LETTER U WITH CIRCUMFLEX +Ü 0xdc Lu LATIN CAPITAL LETTER U WITH DIAERESIS +Ý 0xdd Lu LATIN CAPITAL LETTER Y WITH ACUTE +Þ 0xde Lu LATIN CAPITAL LETTER THORN +ß 0xdf Ll LATIN SMALL LETTER SHARP S +à 0xe0 Ll LATIN SMALL LETTER A WITH GRAVE +á 0xe1 Ll LATIN SMALL LETTER A WITH ACUTE +â 0xe2 Ll LATIN SMALL LETTER A WITH CIRCUMFLEX +ã 0xe3 Ll LATIN SMALL LETTER A WITH TILDE +ä 0xe4 Ll LATIN SMALL LETTER A WITH DIAERESIS +å 0xe5 Ll LATIN SMALL LETTER A WITH RING ABOVE +æ 0xe6 Ll LATIN SMALL LETTER AE +ç 0xe7 Ll LATIN SMALL LETTER C WITH CEDILLA +è 0xe8 Ll LATIN SMALL LETTER E WITH GRAVE +é 0xe9 Ll LATIN SMALL LETTER E WITH ACUTE +ê 0xea Ll LATIN SMALL LETTER E WITH CIRCUMFLEX +ë 0xeb Ll LATIN SMALL LETTER E WITH DIAERESIS +ì 0xec Ll LATIN SMALL LETTER I WITH GRAVE +í 0xed Ll LATIN SMALL LETTER I WITH ACUTE +î 0xee Ll LATIN SMALL LETTER I WITH CIRCUMFLEX +ï 0xef Ll LATIN SMALL LETTER I WITH DIAERESIS +ð 0xf0 Ll LATIN SMALL LETTER ETH +ñ 0xf1 Ll LATIN SMALL LETTER N WITH TILDE +ò 0xf2 Ll LATIN SMALL LETTER O WITH GRAVE +ó 0xf3 Ll LATIN SMALL LETTER O WITH ACUTE +ô 0xf4 Ll LATIN SMALL LETTER O WITH CIRCUMFLEX +õ 0xf5 Ll LATIN SMALL LETTER O WITH TILDE +ö 0xf6 Ll LATIN SMALL LETTER O WITH DIAERESIS +÷ 0xf7 Sm DIVISION SIGN +ø 0xf8 Ll LATIN SMALL LETTER O WITH STROKE +ù 0xf9 Ll LATIN SMALL LETTER U WITH GRAVE +ú 0xfa Ll LATIN SMALL LETTER U WITH ACUTE +û 0xfb Ll LATIN SMALL LETTER U WITH CIRCUMFLEX +ü 0xfc Ll LATIN SMALL LETTER U WITH DIAERESIS +ý 0xfd Ll LATIN SMALL LETTER Y WITH ACUTE +þ 0xfe Ll LATIN SMALL LETTER THORN +ÿ 0xff Ll LATIN SMALL LETTER Y WITH DIAERESIS +ı 0x131 Ll LATIN SMALL LETTER DOTLESS I +Œ 0x152 Lu LATIN CAPITAL LIGATURE OE +œ 0x153 Ll LATIN SMALL LIGATURE OE +Š 0x160 Lu LATIN CAPITAL LETTER S WITH CARON +š 0x161 Ll LATIN SMALL LETTER S WITH CARON +Ÿ 0x178 Lu LATIN CAPITAL LETTER Y WITH DIAERESIS +Ž 0x17d Lu LATIN CAPITAL LETTER Z WITH CARON +ž 0x17e Ll LATIN SMALL LETTER Z WITH CARON +ƒ 0x192 Ll LATIN SMALL LETTER F WITH HOOK +ˆ 0x2c6 Lm MODIFIER LETTER CIRCUMFLEX ACCENT +ˇ 0x2c7 Lm CARON +˘ 0x2d8 Sk BREVE +˙ 0x2d9 Sk DOT ABOVE +˚ 0x2da Sk RING ABOVE +˛ 0x2db Sk OGONEK +˜ 0x2dc Sk SMALL TILDE +˝ 0x2dd Sk DOUBLE ACUTE ACCENT +Γ 0x393 Lu GREEK CAPITAL LETTER GAMMA +Θ 0x398 Lu GREEK CAPITAL LETTER THETA +Σ 0x3a3 Lu GREEK CAPITAL LETTER SIGMA +Φ 0x3a6 Lu GREEK CAPITAL LETTER PHI +Ω 0x3a9 Lu GREEK CAPITAL LETTER OMEGA +α 0x3b1 Ll GREEK SMALL LETTER ALPHA +δ 0x3b4 Ll GREEK SMALL LETTER DELTA +ε 0x3b5 Ll GREEK SMALL LETTER EPSILON +π 0x3c0 Ll GREEK SMALL LETTER PI +σ 0x3c3 Ll GREEK SMALL LETTER SIGMA +τ 0x3c4 Ll GREEK SMALL LETTER TAU +φ 0x3c6 Ll GREEK SMALL LETTER PHI +– 0x2013 Pd EN DASH +— 0x2014 Pd EM DASH +‘ 0x2018 Pi LEFT SINGLE QUOTATION MARK +’ 0x2019 Pf RIGHT SINGLE QUOTATION MARK +‚ 0x201a Ps SINGLE LOW-9 QUOTATION MARK +“ 0x201c Pi LEFT DOUBLE QUOTATION MARK +” 0x201d Pf RIGHT DOUBLE QUOTATION MARK +„ 0x201e Ps DOUBLE LOW-9 QUOTATION MARK +† 0x2020 Po DAGGER +‡ 0x2021 Po DOUBLE DAGGER +• 0x2022 Po BULLET +… 0x2026 Po HORIZONTAL ELLIPSIS +‰ 0x2030 Po PER MILLE SIGN +‹ 0x2039 Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK +› 0x203a Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +⁄ 0x2044 Sm FRACTION SLASH +ⁿ 0x207f Lm SUPERSCRIPT LATIN SMALL LETTER N +₧ 0x20a7 Sc PESETA SIGN +€ 0x20ac Sc EURO SIGN +™ 0x2122 So TRADE MARK SIGN +∂ 0x2202 Sm PARTIAL DIFFERENTIAL +∆ 0x2206 Sm INCREMENT +∏ 0x220f Sm N-ARY PRODUCT +∑ 0x2211 Sm N-ARY SUMMATION +∙ 0x2219 Sm BULLET OPERATOR +√ 0x221a Sm SQUARE ROOT +∞ 0x221e Sm INFINITY +∩ 0x2229 Sm INTERSECTION +∫ 0x222b Sm INTEGRAL +≈ 0x2248 Sm ALMOST EQUAL TO +≠ 0x2260 Sm NOT EQUAL TO +≡ 0x2261 Sm IDENTICAL TO +≤ 0x2264 Sm LESS-THAN OR EQUAL TO +≥ 0x2265 Sm GREATER-THAN OR EQUAL TO +⌐ 0x2310 So REVERSED NOT SIGN +⌠ 0x2320 Sm TOP HALF INTEGRAL +⌡ 0x2321 Sm BOTTOM HALF INTEGRAL +─ 0x2500 So BOX DRAWINGS LIGHT HORIZONTAL +│ 0x2502 So BOX DRAWINGS LIGHT VERTICAL +┌ 0x250c So BOX DRAWINGS LIGHT DOWN AND RIGHT +┐ 0x2510 So BOX DRAWINGS LIGHT DOWN AND LEFT +└ 0x2514 So BOX DRAWINGS LIGHT UP AND RIGHT +┘ 0x2518 So BOX DRAWINGS LIGHT UP AND LEFT +├ 0x251c So BOX DRAWINGS LIGHT VERTICAL AND RIGHT +┤ 0x2524 So BOX DRAWINGS LIGHT VERTICAL AND LEFT +┬ 0x252c So BOX DRAWINGS LIGHT DOWN AND HORIZONTAL +┴ 0x2534 So BOX DRAWINGS LIGHT UP AND HORIZONTAL +┼ 0x253c So BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL +═ 0x2550 So BOX DRAWINGS DOUBLE HORIZONTAL +║ 0x2551 So BOX DRAWINGS DOUBLE VERTICAL +╒ 0x2552 So BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE +╓ 0x2553 So BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE +╔ 0x2554 So BOX DRAWINGS DOUBLE DOWN AND RIGHT +╕ 0x2555 So BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE +╖ 0x2556 So BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE +╗ 0x2557 So BOX DRAWINGS DOUBLE DOWN AND LEFT +╘ 0x2558 So BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE +╙ 0x2559 So BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE +╚ 0x255a So BOX DRAWINGS DOUBLE UP AND RIGHT +╛ 0x255b So BOX DRAWINGS UP SINGLE AND LEFT DOUBLE +╜ 0x255c So BOX DRAWINGS UP DOUBLE AND LEFT SINGLE +╝ 0x255d So BOX DRAWINGS DOUBLE UP AND LEFT +╞ 0x255e So BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE +╟ 0x255f So BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE +╠ 0x2560 So BOX DRAWINGS DOUBLE VERTICAL AND RIGHT +╡ 0x2561 So BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE +╢ 0x2562 So BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE +╣ 0x2563 So BOX DRAWINGS DOUBLE VERTICAL AND LEFT +╤ 0x2564 So BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE +╥ 0x2565 So BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE +╦ 0x2566 So BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL +╧ 0x2567 So BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE +╨ 0x2568 So BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE +╩ 0x2569 So BOX DRAWINGS DOUBLE UP AND HORIZONTAL +╪ 0x256a So BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE +╫ 0x256b So BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE +╬ 0x256c So BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL +▀ 0x2580 So UPPER HALF BLOCK +▄ 0x2584 So LOWER HALF BLOCK +█ 0x2588 So FULL BLOCK +▌ 0x258c So LEFT HALF BLOCK +▐ 0x2590 So RIGHT HALF BLOCK +░ 0x2591 So LIGHT SHADE +▒ 0x2592 So MEDIUM SHADE +▓ 0x2593 So DARK SHADE +■ 0x25a0 So BLACK SQUARE +◊ 0x25ca So LOZENGE + 0xf8ff Co [unknown] +fi 0xfb01 Ll LATIN SMALL LIGATURE FI +fl 0xfb02 Ll LATIN SMALL LIGATURE FL diff --git a/ftfy/test_real_tweets.py b/ftfy/test_real_tweets.py new file mode 100644 index 00000000..f839247f --- /dev/null +++ b/ftfy/test_real_tweets.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +TEST_CASES = [ + ("He's Justin❤", "He's Justin❤"), + ("Le Schtroumpf Docteur conseille g√¢teaux et baies schtroumpfantes pour un r√©gime √©quilibr√©.", + "Le Schtroumpf Docteur conseille gâteaux et baies schtroumpfantes pour un régime équilibré."), + ("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", + "Deja dos heridos hundimiento de barco turístico en Acapulco."), + ("✔ No problems", "✔ No problems"), + ('4288×…', '4288×…'), + ("РґРѕСЂРѕРіРµ Из-РїРѕРґ http://t.co/A0eJAMTuJ1 #футбол", + "дороге Из-под http://t.co/A0eJAMTuJ1 #футбол"), + ("Hi guys 😍", "Hi guys 😍"), + ("\x84Handwerk bringt dich \xfcberall hin\x93: Von der YOU bis nach Monaco", + '„Handwerk bringt dich überall hin“: Von der YOU bis nach Monaco') +] + + +# Possible cases: +# (1) Actual text is UTF-8. Decoded in 1-byte encoding. +# - Use a heuristic to shrink the text size and decrease badness. +# - It will work very consistently if it works. +# (2) Actual text is in a 1-byte encoding, but was decoded with a different one. +# - Use a single-byte heuristic if it helps. Prefer to leave the text alone. +# (3) Text is in a mix of Windows-1252 and Latin-1. +# - Replace control characters with Windows-1252 equivalents and proceed. +# (4) Text is in a mix of Windows-1251 and Latin-1. +# - Oh wow. But, same deal. + +# Strategy: +# If maxchar < 128: +# ASCII! \o/ +# If maxchar < 256: +# Try Latin-1 => UTF-8. +# If all chars in Latin-1 + Windows-1252: +# Convert Latin-1 control characters to Windows-1252. +# Try Windows-1252 => UTF-8. +# Try other 1-byte encodings. +# If all chars in another 1-byte encoding: +# Try other 1-byte encodings. +# +# UTF-8 heuristic: +# * How many positive examples? +# * How many script clashes were introduced? +# * No failures? +# * Did it create modifiers out of nowhere? From 24e93a376d77a6622ca2b2e2cb96a9dc86767c68 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 23 Jul 2013 15:08:17 -0400 Subject: [PATCH 027/502] WIP on version 3 --- ftfy/__init__.py | 94 ++++++++++++++++++++++++++---------------------- ftfy/chardata.py | 21 +++++------ 2 files changed, 60 insertions(+), 55 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index e92f9c61..0559ebf2 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -71,6 +71,7 @@ def fix_text_encoding(text): try: fixed = encoded_bytes.decode('utf-8') steps = [('sloppy_encode', encoding), ('decode', 'utf-8')] + # Now do we check for the nyeoss problem? return fixed, steps except UnicodeDecodeError: possible_1byte_encodings.append(encoding) @@ -90,34 +91,30 @@ def fix_text_encoding(text): else: # Otherwise, it means we have characters that are in Latin-1 but # not in Windows-1252. Those are C1 control characters. Nobody - # wants those. + # wants those. Assume they were meant to be Windows-1252. encoded = text.encode('latin-1') fixed = encoded.decode('windows-1252', errors='replace') steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] return fixed, steps - # Now we need to check for a mixup between Windows-1252, cp437, - # and MacRoman. We know Latin-1 isn't in the list, because if it were, - # we would have returned already. + # The cases that remain are mixups between two different single-byte + # encodings, neither of which is Latin-1. # - # We aren't crazy enough to try to turn text *into* Windows-1251. - # Russians, if you screw things up that much, you're on you're own. - for encoding in possible_1byte_encodings: - for target in CONFUSABLE_1BYTE_ENCODINGS: - result, goodness = try_reencoding(text, encoding, target) - - # A sort of prior: if it's not Windows-1252, it's less likely - if target != 'windows-1252': - goodness -= 5 - if goodness > 0: - fixed = text.encode(encoding).decode(target) - steps = [('encode', encoding), ('decode', target)] - return fixed, steps + # Those cases are somewhat rare, and impossible to solve without false + # positives. If you're in one of these situations, you don't need an + # encoding fixer. You need something that heuristically guesses what + # the encoding is in the first place. + # + # It's a different problem, the one that the 'chardet' module is + # theoretically designed to solve. It probably *won't* solve it in + # such an ambiguous case, but perhaps a version of it with better + # heuristics would. Anyway, ftfy should not claim to solve it. - # We haven't returned anything by now? That's probably fine. It means - # this is probably perfectly good Unicode. - return text, [] + return text, [('give up', None)] +# TODO: +# - write a function that just continues to use the same decoding step +# - if the decoding step encounters a problem, back up and try line-by-line def fix_bad_encoding(text): """ @@ -217,34 +214,45 @@ def fix_bad_encoding(text): """ % sys.version_info[0] -def fix_text(text, normalization='NFKC'): +def fix_text_segment(text, + fix_entities=True, + remove_terminal_escapes=True, + fix_encoding=True, + normalization='NFKC', + uncurl_quotes=True, + remove_control_chars=True, + remove_bom=True): """ Given Unicode text as input, make its representation consistent and - possibly less broken: - - - Detect whether the text was incorrectly encoded into UTF-8 and fix it, - as defined in `fix_bad_encoding`. - - If the text is HTML-escaped but has no HTML tags, replace HTML entities - with their equivalent characters. - - Remove terminal escapes and color codes. - - Remove control characters except for newlines and tabs. - - Normalize it with Unicode normalization form KC, which applies the - following relevant transformations: - - Combine characters and diacritics that are written using separate + possibly less broken, by applying these steps in order: + + - If `remove_terminal_escapes` is True, remove sequences of bytes that are + instructions for Unix terminals, such as the codes that make text appear + in different colors. + - If `fix_entities` is True, consider replacing HTML entities with their + equivalent characters. However, this never applies to text with a pair + of angle brackets in it already; you're probably not supposed to decode + entities there, and you'd make things ambiguous if you did. + - If `fix_encoding` is True, look for common mistakes that come from + encoding or decoding Unicode text incorrectly, and fix them if they are + reasonably fixable. See `fix_text_encoding` for details. + - If `normalization` is not None, apply the specified form of Unicode + normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. + The default, 'NFKC', applies the following relevant transformations: + - C: Combine characters and diacritics that are written using separate code points, such as converting "e" plus an acute accent modifier into "é", or converting "ka" (か) plus a dakuten into the single character "ga" (が). - - Replace characters that are functionally equivalent with the most - common form: for example, half-width katakana will be replaced with - full-width, full-width Roman characters will be replaced with - ASCII characters, ellipsis characters will be replaced by three - periods, and the ligature 'fl' will be replaced by 'fl'. - - Replace curly quotation marks with straight ones. - - Remove the Byte-Order Mark if it exists. - - You may change the `normalization` argument to apply a different kind of - Unicode normalization, such as NFC or NFKD, or set it to None to skip this - step. + - K: Replace characters that are functionally equivalent with the most + common form. For example, half-width katakana will be replaced with + full-width versions, full-width Roman characters will be replaced with + ASCII characters, ellipsis characters will be replaced with three + periods, and the ligature 'fl' will be replaced with 'fl'. + - If `uncurl_quotes` is True, replace various curly quotation marks with + plain-ASCII straight quotes. + - If `remove_bom` is True, remove the Byte-Order Mark if it exists. + - If anything was changed, repeat all the steps, so that the function is + idempotent. "&amp;" will become "&", for example, not "&". >>> print(fix_text('ünicode')) ünicode diff --git a/ftfy/chardata.py b/ftfy/chardata.py index d3c7e414..e76a3de5 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -63,16 +63,15 @@ def possible_encoding(text, encoding): # Make regular expressions out of categories of Unicode characters. +# Except we probably need broader ranges. +# +# Ll Lu: weirdness=1 +# Ll [Lo Lt]: weirdness=2 +# Lm Sk Sm Sc No So Po def _make_category_regex_ranges(): - encodable_char_set = set([unichr(codepoint) for codepoint in range(0, 0x100)]) - for codepoint in range(0x80, 0x100): - encoded_char = chr(codepoint).encode('latin-1') - for encoding in ['windows-1252', 'macroman', 'cp437']: - try: - decoded_char = encoded_char.decode(encoding) - encodable_char_set.add(decoded_char) - except ValueError: - pass + encodable_char_set = [] + for codepoint in range(0x20, 0x10000): + encodable_char_set.append(unichr(codepoint)) categories = defaultdict(list) for char in sorted(encodable_char_set): @@ -89,7 +88,6 @@ def _make_category_regex_ranges(): .replace('-', r'\-')) return ranges CATEGORY_RANGES = _make_category_regex_ranges() -print(CATEGORY_RANGES) # ---------------- @@ -122,8 +120,7 @@ def _make_category_regex_ranges(): # # 0 = not weird at all # 1 = rare punctuation or rare letter that someone could certainly -# have a good reason to use. All Windows-1252 gremlins are at least -# weirdness 1. +# have a good reason to use. # 2 = things that probably don't appear next to letters or other # symbols, such as math or currency symbols # 3 = obscure symbols that nobody would go out of their way to use From 2c093a848d437c001b75de4dcde6c7048a2eec79 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 28 Jul 2013 16:00:42 -0400 Subject: [PATCH 028/502] Big refactor - Individual fixes are in fixes.py. (Now we can give options nice names without conflicting with the functions that implement them.) - Top-level functions revised to take new options. - fix_text is idempotent. - "Weirdness" is now determined using general Unicode properties, instead of a mapping of expected and unexpected scripts. This should be faster and more general. - It is considered slightly weird to create modifier characters that weren't there before. --- ftfy/__init__.py | 330 +++++---------------------------------- ftfy/badness.py | 153 ++++-------------- ftfy/chardata.py | 115 +------------- ftfy/fixes.py | 250 +++++++++++++++++++++++++++++ ftfy/test_real_tweets.py | 38 +---- ftfy/test_unicode.py | 8 +- 6 files changed, 336 insertions(+), 558 deletions(-) create mode 100644 ftfy/fixes.py diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 0559ebf2..c1ac21da 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -7,221 +7,20 @@ """ from __future__ import unicode_literals - +from ftfy import fixes import unicodedata -import re -import sys -from ftfy.chardata import (possible_encoding, CHARMAPS, CHARMAP_ENCODINGS, - CONFUSABLE_1BYTE_ENCODINGS) - -# Adjust names that have changed in Python 3 -if sys.hexversion >= 0x03000000: - from html import entities - htmlentitydefs = entities - unichr = chr -else: - import htmlentitydefs - - -def fix_text_encoding(text): - """ - TODO documentation. - - Returns a fixed version of the text, plus a data structure that - explains how it was fixed. - """ - if isinstance(text, bytes): - raise UnicodeError(BYTES_ERROR_TEXT) - if len(text) == 0: - return text, [] - - # The first plan is to return ASCII text unchanged. - if possible_encoding(text, 'ascii'): - return text, [] - - # As we go through the next step, remember the possible encodings - # that we encounter but don't successfully fix yet. We may need them - # later. - possible_1byte_encodings = [] - - # Suppose the text was supposed to be UTF-8, but it was decoded using - # a single-byte encoding instead. When these cases can be fixed, they - # are usually the correct thing to do, so try them next. - for encoding in CHARMAP_ENCODINGS: - if possible_encoding(text, encoding): - # This is an ugly-looking way to get the bytes that represent - # the text in this encoding. The reason we can't necessarily - # use .encode(encoding) is that the decoder is very likely - # to have been sloppier than Python. - # - # The decoder might have left bytes unchanged when they're not - # part of the encoding. It might represent b'\x81' as u'\x81' - # in Windows-1252, while Python would claim that using byte - # 0x81 in Windows-1252 is an error. - # - # So what we do here is we use the .translate method of Unicode - # strings. Using it with the character maps we have computed will - # give us back a Unicode string using only code - # points up to 0xff. This can then be converted into the intended - # bytes by encoding it as Latin-1. - sorta_encoded_text = text.translate(CHARMAPS[encoding]) - encoded_bytes = sorta_encoded_text.encode('latin-1') - - # Now, find out if it's UTF-8. - try: - fixed = encoded_bytes.decode('utf-8') - steps = [('sloppy_encode', encoding), ('decode', 'utf-8')] - # Now do we check for the nyeoss problem? - return fixed, steps - except UnicodeDecodeError: - possible_1byte_encodings.append(encoding) - - # The next most likely case is that this is Latin-1 that was intended to - # be read as Windows-1252, because those two encodings in particular are - # easily confused. - # - # We don't need to check for possibilities such as Latin-1 that was - # intended to be read as MacRoman, because it is unlikely that any - # software has that confusion. - if 'latin-1' in possible_1byte_encodings: - if 'windows-1252' in possible_1byte_encodings: - # This text is in the intersection of Latin-1 and - # Windows-1252, so it's probably legit. - return text, [] - else: - # Otherwise, it means we have characters that are in Latin-1 but - # not in Windows-1252. Those are C1 control characters. Nobody - # wants those. Assume they were meant to be Windows-1252. - encoded = text.encode('latin-1') - fixed = encoded.decode('windows-1252', errors='replace') - steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] - return fixed, steps - - # The cases that remain are mixups between two different single-byte - # encodings, neither of which is Latin-1. - # - # Those cases are somewhat rare, and impossible to solve without false - # positives. If you're in one of these situations, you don't need an - # encoding fixer. You need something that heuristically guesses what - # the encoding is in the first place. - # - # It's a different problem, the one that the 'chardet' module is - # theoretically designed to solve. It probably *won't* solve it in - # such an ambiguous case, but perhaps a version of it with better - # heuristics would. Anyway, ftfy should not claim to solve it. - - return text, [('give up', None)] - -# TODO: -# - write a function that just continues to use the same decoding step -# - if the decoding step encounters a problem, back up and try line-by-line - -def fix_bad_encoding(text): - """ - Something you will find all over the place, in real-world text, is text - that's mistakenly encoded as utf-8, decoded in some ugly format like - latin-1 or even Windows codepage 1252, and encoded as utf-8 again. - - This causes your perfectly good Unicode-aware code to end up with garbage - text because someone else (or maybe "someone else") made a mistake. - - This function looks for the evidence of that having happened and fixes it. - It determines whether it should replace nonsense sequences of single-byte - characters that were really meant to be UTF-8 characters, and if so, turns - them into the correctly-encoded Unicode character that they were meant to - represent. - - The input to the function must be Unicode. If you don't have Unicode text, - you're not using the right tool to solve your problem. - - NOTE: the following examples are written using unmarked literal strings, - but they are Unicode text. In Python 2 we have "unicode_literals" turned - on, and in Python 3 this is always the case. - - >>> print(fix_bad_encoding('único')) - único - - >>> print(fix_bad_encoding('This text is fine already :þ')) - This text is fine already :þ - - Because these characters often come from Microsoft products, we allow - for the possibility that we get not just Unicode characters 128-255, but - also Windows's conflicting idea of what characters 128-160 are. - - >>> print(fix_bad_encoding('This — should be an em dash')) - This — should be an em dash - - We might have to deal with both Windows characters and raw control - characters at the same time, especially when dealing with characters like - \x81 that have no mapping in Windows. - - >>> print(fix_bad_encoding('This text is sad .â\x81”.')) - This text is sad .⁔. - - This function even fixes multiple levels of badness: - - >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' - >>> print(fix_bad_encoding(wtf)) - ಠ_ಠ - - However, it has safeguards against fixing sequences of letters and - punctuation that can occur in valid text: - - >>> print(fix_bad_encoding('not such a fan of Charlotte Brontë…”')) - not such a fan of Charlotte Brontë…” - - Cases of genuine ambiguity can sometimes be addressed by finding other - characters that are not double-encoded, and expecting the encoding to - be consistent: - - >>> print(fix_bad_encoding('AHÅ™, the new sofa from IKEA®')) - AHÅ™, the new sofa from IKEA® - - Finally, we handle the case where the text is in a single-byte encoding - that was intended as Windows-1252 all along but read as Latin-1: - - >>> print(fix_bad_encoding('This text was never UTF-8 at all\x85')) - This text was never UTF-8 at all… - """ - keep_going = True - while keep_going: - fixed, info = fix_text_encoding(text) - keep_going = (fixed != text) - text = fixed - - return fixed - - -# ---------------- -# Don't try to ftfy more than 64 KiB of text at a time, to prevent -# denial of service. -MAXLEN = 0x10000 -BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode! - -ftfy is designed to fix problems that were introduced by handling Unicode -incorrectly. It might be able to fix the bytes you just handed it, but the -fact that you just gave a pile of bytes to a function that fixes text means -that your code is *also* handling Unicode incorrectly. -ftfy takes Unicode text as input. You should take these bytes and decode -them from the encoding you think they are in. If you're not sure, you can -try decoding it as 'latin-1', which may work and at least won't make -anything worse. -If you're confused by this, please read the Python Unicode HOWTO: +MAXLEN = 65536 - http://docs.python.org/%d/howto/unicode.html -""" % sys.version_info[0] - - -def fix_text_segment(text, - fix_entities=True, - remove_terminal_escapes=True, - fix_encoding=True, - normalization='NFKC', - uncurl_quotes=True, - remove_control_chars=True, - remove_bom=True): +def fix_text(text, + fix_entities=True, + remove_terminal_escapes=True, + fix_encoding=True, + normalization='NFKC', + uncurl_quotes=True, + remove_control_chars=True, + remove_bom=True): """ Given Unicode text as input, make its representation consistent and possibly less broken, by applying these steps in order: @@ -332,87 +131,42 @@ def fix_file(file, normalization='NFKC'): yield fix_text_segment(line, normalization, entities) -def fix_text_segment(text, normalization='NFKC', entities=True): +def fix_text_segment(text, + fix_entities=True, + remove_terminal_escapes=True, + fix_encoding=True, + normalization='NFKC', + uncurl_quotes=True, + remove_control_chars=True, + remove_bom=True): """ Apply fixes to text in a single chunk. This could be a line of text within a larger run of `fix_text`, or it could be a larger amount of text that you are certain is all in the same encoding. - """ - if isinstance(text, bytes): - raise UnicodeError(BYTES_ERROR_TEXT) - text = remove_terminal_escapes(text) - if entities: - text = unescape_html(text) - text = fix_bad_encoding(text) - text = text.translate(CONTROL_CHARS) - if normalization is not None: - text = unicodedata.normalize(normalization, text) - text = uncurl_quotes(text) - text = remove_bom(text) - return text - -def text_cost(text): + See `fix_text` for a description of the parameters. """ - Assign a cost function to the length plus weirdness of a text string. - """ - return text_badness(text) + len(text) * 4 - - -HTML_ENTITY_RE = re.compile("&#?\w+;") -def unescape_html(text): - """ - Decode all three types of HTML entities/character references. - - Code by Fredrik Lundh of effbot.org. - - >>> print(unescape_html('<tag>')) - - """ - def fixup(m): - text = m.group(0) - if text[:2] == "&#": - # character reference - try: - if text[:3] == "&#x": - return unichr(int(text[3:-1], 16)) - else: - return unichr(int(text[2:-1])) - except ValueError: - pass - else: - # named entity - try: - text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) - except KeyError: - pass - return text # leave as is - return HTML_ENTITY_RE.sub(fixup, text) - - -ANSI_RE = re.compile('\033\[((?:\d|;)*)([a-zA-Z])') -def remove_terminal_escapes(text): - """ - Strip out "ANSI" terminal escape sequences, such as those that produce - colored text on Unix. - """ - return ANSI_RE.sub('', text) - - -SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') -DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') -def uncurl_quotes(text): - """ - Replace curly quotation marks with straight equivalents. - - >>> print(uncurl_quotes('\u201chere\u2019s a test\u201d')) - "here's a test" - """ - return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text)) - + if isinstance(text, bytes): + raise UnicodeError(fixes.BYTES_ERROR_TEXT) + + unsafe_entities = ('<' in text and '>' in text) + while True: + origtext = text + if fix_entities and not unsafe_entities: + text = fixes.unescape_html(text) + if remove_terminal_escapes: + text = fixes.remove_terminal_escapes(text) + if fix_encoding: + text = fixes.fix_text_encoding(text) + if normalization is not None: + text = unicodedata.normalize(normalization, text) + if uncurl_quotes: + text = fixes.uncurl_quotes(text) + if remove_control_chars: + text = fixes.remove_control_chars(text) + if remove_bom: + text = fixes.remove_bom(text) + + if text == origtext: + return text -def remove_bom(text): - """ - Remove a left-over byte-order mark. - """ - return text.lstrip(unichr(0xfeff)) diff --git a/ftfy/badness.py b/ftfy/badness.py index 8930b91f..b28f8d7a 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -1,138 +1,43 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals - +from ftfy.chardata import CATEGORY_RANGES import re -from ftfy.chardata import (SCRIPT_MAP, SINGLE_BYTE_WEIRDNESS, - WINDOWS_1252_GREMLINS) - -import sys -if sys.hexversion >= 0x03000000: - unichr = chr - -CONSISTENT_SCRIPTS_RE = re.compile(r'([A-Za-z])(\1+)') -LETTER_SEGMENTS_RE = re.compile(r'([A-Za-z]+)') -LOWERCASE_RE = re.compile(r'([a-z])') -DOUBLE_WEIRD_RE = re.compile(r'(WW+)') -GREMLINS_RE = re.compile('[' + - ''.join([unichr(codepoint) for codepoint in WINDOWS_1252_GREMLINS]) - + ']') - -WEIRD_CHARACTER_RES = [] -for i in range(5): - chars = [unichr(codepoint) for codepoint in range(0x80, 0x100) - if SINGLE_BYTE_WEIRDNESS[codepoint] > i] - WEIRD_CHARACTER_RES.append(re.compile('[' + ''.join(chars) + ']')) - -def num_consistent_scripts(scriptdata): - """ - Count the number of times two adjacent letters are in the same script. - Uses a "scriptdata" string as input, not the actual text. +def _make_weirdness_regex(): + groups = [] + groups.append(u'([{Ll}][{Lu}{Lo}{Lt}])'.format(**CATEGORY_RANGES)) + groups.append(u'([{Mn}{Mc}])'.format(**CATEGORY_RANGES)) + exclusive_categories = ['Lm', 'Sk', 'Sm', 'Sc', 'No', 'So'] + for cat1 in exclusive_categories: + others = exclusive_categories[:] + others.remove(cat1) + cat1_range = CATEGORY_RANGES[cat1] + others_range = u''.join([CATEGORY_RANGES[other] for other in others]) + groups.append( + u'([{cat1_range}][{others_range}])'.format( + cat1_range=cat1_range, + others_range=others_range + ) + ) + regex = '|'.join(groups) + return re.compile(regex) - >>> num_consistent_scripts('LL AAA.') - 3 - >>> num_consistent_scripts('LLAAA ...') - 3 - >>> num_consistent_scripts('LAL') - 0 - >>> num_consistent_scripts('..LLL..') - 2 - >>> num_consistent_scripts('LWWW') - 2 - """ - matches = CONSISTENT_SCRIPTS_RE.findall(scriptdata) - total = 0 - for first, rest in matches: - total += len(rest) - return total +WEIRDNESS_RE = _make_weirdness_regex() -def num_inconsistent_scripts(scriptdata): +def sequence_weirdness(text): """ - Count the number of times two adjacent letters are in different scripts, - or are both marked as 'weird'. - - Uses a "scriptdata" string as input, not the actual text. + TODO: better documentation - >>> num_inconsistent_scripts('LL AAA.') - 0 - >>> num_inconsistent_scripts('LLAAA ...') - 1 - >>> num_inconsistent_scripts('LAL') - 2 - >>> num_inconsistent_scripts('..LLL..') - 0 - >>> num_inconsistent_scripts('LWWW') - 3 + Determine how often a text has unexpected sequences of characters, + which might indicate an incorrect encoding. """ - # First, count the number of times two letters are adjacent - letter_segments = LETTER_SEGMENTS_RE.findall(scriptdata) - adjacent_letters = 0 - for seg in letter_segments: - adjacent_letters += len(seg) - 1 - - # Then subtract out the number of times the scripts are consistent, - # but first add back in adjacent weird characters - double_weird_segments = DOUBLE_WEIRD_RE.findall(scriptdata) - for seg in double_weird_segments: - adjacent_letters += len(seg) - 1 + return len(WEIRDNESS_RE.findall(text)) - return adjacent_letters - num_consistent_scripts(scriptdata) +def better_text(newtext, oldtext): + return text_cost(newtext) < text_cost(oldtext) -def script_obscurity(scriptdata): - """ - Count the number of characters in obscure scripts. Characters in very - obscure scripts count twice as much. - - >>> script_obscurity('LWWW') - 0 - >>> script_obscurity('Llkzz') - 6 - """ - return len(LOWERCASE_RE.findall(scriptdata)) + scriptdata.count('z') - -def character_weirdness(text): - """ - Sum the weirdness of all the single-byte characters in this text. - - >>> character_weirdness('test') - 0 - >>> character_weirdness('wúút') - 0 - >>> character_weirdness('\x81\x81') - 10 - """ - total = 0 - for weird_re in WEIRD_CHARACTER_RES: - found = weird_re.findall(text) - total += len(found) - return total - - -def text_badness(text): - """ - Count the total badness of a string, which helps to determine when an - encoding has gone wrong. - - Obvious problems (badness = 100): - - The replacement character \ufffd, indicating a decoding error - - Unassigned Unicode characters - - Very weird things (badness = 10): - - Adjacent letters from two different scripts - - Letters adjacent to obscure single-byte symbols - - Obscure single-byte symbols adjacent to each other - - Improbable control characters, such as 0x81 - - Moderately weird things: - - Improbable single-byte characters, such as ƒ or ¬ - - Letters in somewhat rare scripts (they'll still probably look better than - they would in the wrong encoding) - """ - scriptdata = text.translate(SCRIPT_MAP) - badness = character_weirdness(text) + script_obscurity(scriptdata) - badness += 10 * num_inconsistent_scripts(scriptdata) - badness += 100 * scriptdata.count('?') - return badness +def text_cost(text): + return sequence_weirdness(text) * 2 + len(text) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index e76a3de5..e58b1a12 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -33,7 +33,7 @@ def _make_encoding_regexes(): for encoding in CHARMAP_ENCODINGS: charmap = {} for codepoint in range(0, 0x80): - charmap[unichr(codepoint)] = unichr(codepoint) + charmap[codepoint] = unichr(codepoint) for codepoint in range(0x80, 0x100): char = unichr(codepoint) encoded_char = char.encode('latin-1') @@ -41,14 +41,14 @@ def _make_encoding_regexes(): decoded_char = encoded_char.decode(encoding) except ValueError: decoded_char = char - charmap[decoded_char] = char + charmap[ord(decoded_char)] = char - charlist = [char for char in sorted(charmap.keys()) if ord(char) >= 0x80] + charlist = [unichr(codept) for codept in sorted(charmap.keys()) if codept >= 0x80] regex = '^[\x00-\x7f{}]*$'.format(''.join(charlist)) CHARMAPS[encoding] = charmap encoding_regular_expressions[encoding] = re.compile(regex) - encoding_regular_expressions['ascii'] = re.compile('^[\x00-\x7f]*') + encoding_regular_expressions['ascii'] = re.compile('^[\x00-\x7f]*$') return encoding_regular_expressions ENCODING_REGEXES = _make_encoding_regexes() @@ -89,30 +89,6 @@ def _make_category_regex_ranges(): return ranges CATEGORY_RANGES = _make_category_regex_ranges() -# ---------------- - - -# What is the heuristic for comparing single-byte encodings to each other? -encodable_chars = set() -for codepoint in range(0x80, 0x100): - encoded_char = chr(codepoint).encode('latin-1') - possibilities = [] - for encoding in ['latin-1', 'windows-1252', 'macroman', 'cp437']: - try: - decoded_char = encoded_char.decode(encoding) - encodable_chars.add(decoded_char) - except ValueError: - decoded_char = encoded_char.decode('latin-1') - possibilities.append(decoded_char) - print(' '.join(possibilities)) -print() - -for char in sorted(encodable_chars): - try: - name = unicodedata.name(char) - except ValueError: - name = '[unknown]' - print(char, hex(ord(char)), unicodedata.category(char), name) # Rank the characters typically represented by a single byte -- that is, in # Latin-1 or Windows-1252 -- by how weird it would be to see them in running @@ -242,89 +218,6 @@ def _make_category_regex_ranges(): } - -# Create a fast mapping that converts a Unicode string to a string describing -# its character classes, particularly the scripts its letters are in. -# -# Capital letters represent groups of commonly-used scripts: -# L = Latin -# E = several East Asian scripts including hanzi, kana, and Hangul -# C = Cyrillic -# etc. -# -# Lowercase letters represent rare scripts. -# . represents non-letters. -# Whitespace represents whitespace. -# ? represents errors. -# -# Astral characters pass through unmodified; we don't count them as script -# conflicts. They are probably intentional. - -SCRIPT_LETTERS = { - 'LATIN': 'L', - 'CJK': 'E', - 'ARABIC': 'A', - 'CYRILLIC': 'C', - 'GREEK': 'G', - 'HEBREW': 'H', - 'KATAKANA': 'E', - 'HIRAGANA': 'E', - 'HIRAGANA-KATAKANA': 'E', - 'KATAKANA-HIRAGANA': 'E', - 'HANGUL': 'E', - 'DEVANAGARI': 'D', - 'THAI': 'T', - 'FULLWIDTH': 'E', - 'MASCULINE': 'L', - 'FEMININE': 'L', - 'MODIFIER': '.', - 'HALFWIDTH': 'E', - 'BENGALI': 'b', - 'LAO': 'l', - 'KHMER': 'k', - 'TELUGU': 't', - 'MALAYALAM': 'm', - 'SINHALA': 's', - 'TAMIL': 'a', - 'GEORGIAN': 'g', - 'ARMENIAN': 'r', - 'KANNADA': 'n', # mostly used for looks of disapproval -} - - -SCRIPT_MAP = {} - -for codepoint in range(0x10000): - char = unichr(codepoint) - if unicodedata.category(char).startswith('L'): - try: - name = unicodedata.name(char) - script = name.split()[0] - if script in SCRIPT_LETTERS: - SCRIPT_MAP[codepoint] = SCRIPT_LETTERS[script] - else: - SCRIPT_MAP[codepoint] = 'z' - except ValueError: - # it's unfortunate that this gives subtly different results - # on Python 2.6, which is confused about the Unicode 5.1 - # Chinese range. It knows they're letters but it has no idea - # what they are named. - # - # This could be something to fix in the future, or maybe we - # just stop supporting Python 2.6 eventually. - SCRIPT_MAP[codepoint] = 'z' - elif unicodedata.category(char).startswith('Z'): - SCRIPT_MAP[codepoint] = ' ' - elif unicodedata.category(char) == 'Cn': - SCRIPT_MAP[codepoint] = '?' - else: - SCRIPT_MAP[codepoint] = '.' - -SCRIPT_MAP[0x09] = ' ' -SCRIPT_MAP[0x0a] = '\n' -SCRIPT_MAP[0xfffd] = '?' - - # A translate mapping that will strip all control characters except \t and \n. # This incidentally has the effect of normalizing Windows \r\n line endings to # Unix \n line endings. diff --git a/ftfy/fixes.py b/ftfy/fixes.py new file mode 100644 index 00000000..ad03bd60 --- /dev/null +++ b/ftfy/fixes.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from ftfy.chardata import (possible_encoding, CHARMAPS, CHARMAP_ENCODINGS, + CONTROL_CHARS) +from ftfy.badness import text_cost +import re +import sys +import logging +LOG = logging.getLogger(__name__) + +if sys.hexversion >= 0x03000000: + from html import entities + htmlentitydefs = entities + unichr = chr +else: + import htmlentitydefs + + +BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode! + +ftfy is designed to fix problems that were introduced by handling Unicode +incorrectly. It might be able to fix the bytes you just handed it, but the +fact that you just gave a pile of bytes to a function that fixes text means +that your code is *also* handling Unicode incorrectly. + +ftfy takes Unicode text as input. You should take these bytes and decode +them from the encoding you think they are in. If you're not sure, you can +try decoding it as 'latin-1', which may work and at least won't make +anything worse. + +If you're confused by this, please read the Python Unicode HOWTO: + + http://docs.python.org/%d/howto/unicode.html +""" % sys.version_info[0] + + +def fix_text_encoding(text): + """ + Something you will find all over the place, in real-world text, is text + that's mistakenly encoded as utf-8, decoded in some ugly format like + latin-1 or even Windows codepage 1252, and encoded as utf-8 again. + + This causes your perfectly good Unicode-aware code to end up with garbage + text because someone else (or maybe "someone else") made a mistake. + + This function looks for the evidence of that having happened and fixes it. + It determines whether it should replace nonsense sequences of single-byte + characters that were really meant to be UTF-8 characters, and if so, turns + them into the correctly-encoded Unicode character that they were meant to + represent. + + The input to the function must be Unicode. If you don't have Unicode text, + you're not using the right tool to solve your problem. + + NOTE: the following examples are written using unmarked literal strings, + but they are Unicode text. In Python 2 we have "unicode_literals" turned + on, and in Python 3 this is always the case. + + >>> print(fix_text_encoding('único')) + único + + >>> print(fix_text_encoding('This text is fine already :þ')) + This text is fine already :þ + + Because these characters often come from Microsoft products, we allow + for the possibility that we get not just Unicode characters 128-255, but + also Windows's conflicting idea of what characters 128-160 are. + + >>> print(fix_text_encoding('This — should be an em dash')) + This — should be an em dash + + We might have to deal with both Windows characters and raw control + characters at the same time, especially when dealing with characters like + \x81 that have no mapping in Windows. + + >>> print(fix_text_encoding('This text is sad .â\x81”.')) + This text is sad .⁔. + + However, it has safeguards against fixing sequences of letters and + punctuation that can occur in valid text: + + >>> print(fix_text_encoding('not such a fan of Charlotte Brontë…”')) + not such a fan of Charlotte Brontë…” + + Cases of genuine ambiguity can sometimes be addressed by finding other + characters that are not double-encoded, and expecting the encoding to + be consistent: + + >>> print(fix_text_encoding('AHÅ™, the new sofa from IKEA®')) + AHÅ™, the new sofa from IKEA® + + Finally, we handle the case where the text is in a single-byte encoding + that was intended as Windows-1252 all along but read as Latin-1: + + >>> print(fix_text_encoding('This text was never UTF-8 at all\x85')) + This text was never UTF-8 at all… + """ + fixed, _ = fix_text_and_explain(text) + return fixed + +def fix_text_and_explain(text): + if isinstance(text, bytes): + raise UnicodeError(BYTES_ERROR_TEXT) + if len(text) == 0: + return text, [] + + # The first plan is to return ASCII text unchanged. + if possible_encoding(text, 'ascii'): + return text, [] + + # As we go through the next step, remember the possible encodings + # that we encounter but don't successfully fix yet. We may need them + # later. + possible_1byte_encodings = [] + + # Suppose the text was supposed to be UTF-8, but it was decoded using + # a single-byte encoding instead. When these cases can be fixed, they + # are usually the correct thing to do, so try them next. + for encoding in CHARMAP_ENCODINGS: + if possible_encoding(text, encoding): + # This is an ugly-looking way to get the bytes that represent + # the text in this encoding. The reason we can't necessarily + # use .encode(encoding) is that the decoder is very likely + # to have been sloppier than Python. + # + # The decoder might have left bytes unchanged when they're not + # part of the encoding. It might represent b'\x81' as u'\x81' + # in Windows-1252, while Python would claim that using byte + # 0x81 in Windows-1252 is an error. + # + # So what we do here is we use the .translate method of Unicode + # strings. Using it with the character maps we have computed will + # give us back a Unicode string using only code + # points up to 0xff. This can then be converted into the intended + # bytes by encoding it as Latin-1. + sorta_encoded_text = text.translate(CHARMAPS[encoding]) + if max(sorta_encoded_text) > unichr(255): + LOG.warn("Charmap: %s" % CHARMAPS[encoding]) + LOG.warn("Encoding: %s" % encoding) + LOG.warn("Original text: %s" % text) + LOG.warn("encoded text: %s" % sorta_encoded_text) + encoded_bytes = sorta_encoded_text.encode('latin-1') + + # Now, find out if it's UTF-8. + try: + fixed = encoded_bytes.decode('utf-8') + steps = [('sloppy_encode', encoding), ('decode', 'utf-8')] + if text_cost(fixed) < text_cost(text): + return fixed, steps + except UnicodeDecodeError: + possible_1byte_encodings.append(encoding) + + # The next most likely case is that this is Latin-1 that was intended to + # be read as Windows-1252, because those two encodings in particular are + # easily confused. + # + # We don't need to check for possibilities such as Latin-1 that was + # intended to be read as MacRoman, because it is unlikely that any + # software has that confusion. + if 'latin-1' in possible_1byte_encodings: + if 'windows-1252' in possible_1byte_encodings: + # This text is in the intersection of Latin-1 and + # Windows-1252, so it's probably legit. + return text, [] + else: + # Otherwise, it means we have characters that are in Latin-1 but + # not in Windows-1252. Those are C1 control characters. Nobody + # wants those. Assume they were meant to be Windows-1252. + encoded = text.encode('latin-1') + fixed = encoded.decode('windows-1252', errors='replace') + steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] + return fixed, steps + + # The cases that remain are mixups between two different single-byte + # encodings, neither of which is Latin-1. + # + # Those cases are somewhat rare, and impossible to solve without false + # positives. If you're in one of these situations, you don't need an + # encoding fixer. You need something that heuristically guesses what + # the encoding is in the first place. + # + # It's a different problem, the one that the 'chardet' module is + # theoretically designed to solve. It probably *won't* solve it in + # such an ambiguous case, but perhaps a version of it with better + # heuristics would. Anyway, ftfy should not claim to solve it. + + return text, [('give up', None)] + + +HTML_ENTITY_RE = re.compile("&#?\w+;") +def unescape_html(text): + """ + Decode all three types of HTML entities/character references. + + Code by Fredrik Lundh of effbot.org. + + >>> print(unescape_html('<tag>')) + + """ + def fixup(m): + text = m.group(0) + if text[:2] == "&#": + # character reference + try: + if text[:3] == "&#x": + return unichr(int(text[3:-1], 16)) + else: + return unichr(int(text[2:-1])) + except ValueError: + pass + else: + # named entity + try: + text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) + except KeyError: + pass + return text # leave as is + return HTML_ENTITY_RE.sub(fixup, text) + + +ANSI_RE = re.compile('\033\[((?:\d|;)*)([a-zA-Z])') +def remove_terminal_escapes(text): + """ + Strip out "ANSI" terminal escape sequences, such as those that produce + colored text on Unix. + """ + return ANSI_RE.sub('', text) + + +SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') +DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') +def uncurl_quotes(text): + """ + Replace curly quotation marks with straight equivalents. + + >>> print(uncurl_quotes('\u201chere\u2019s a test\u201d')) + "here's a test" + """ + return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text)) + + +def remove_control_chars(text): + return text.translate(CONTROL_CHARS) + + +def remove_bom(text): + """ + Remove a left-over byte-order mark. + """ + return text.lstrip(unichr(0xfeff)) diff --git a/ftfy/test_real_tweets.py b/ftfy/test_real_tweets.py index f839247f..f93a50b6 100644 --- a/ftfy/test_real_tweets.py +++ b/ftfy/test_real_tweets.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +from ftfy.fixes import fix_text_encoding +from nose.tools import eq_ TEST_CASES = [ ("He's Justin❤", "He's Justin❤"), ("Le Schtroumpf Docteur conseille g√¢teaux et baies schtroumpfantes pour un r√©gime √©quilibr√©.", "Le Schtroumpf Docteur conseille gâteaux et baies schtroumpfantes pour un régime équilibré."), - ("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", - "Deja dos heridos hundimiento de barco turístico en Acapulco."), + #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", + # "Deja dos heridos hundimiento de barco turístico en Acapulco."), ("✔ No problems", "✔ No problems"), ('4288×…', '4288×…'), ("РґРѕСЂРѕРіРµ Из-РїРѕРґ http://t.co/A0eJAMTuJ1 #футбол", @@ -16,32 +18,6 @@ '„Handwerk bringt dich überall hin“: Von der YOU bis nach Monaco') ] - -# Possible cases: -# (1) Actual text is UTF-8. Decoded in 1-byte encoding. -# - Use a heuristic to shrink the text size and decrease badness. -# - It will work very consistently if it works. -# (2) Actual text is in a 1-byte encoding, but was decoded with a different one. -# - Use a single-byte heuristic if it helps. Prefer to leave the text alone. -# (3) Text is in a mix of Windows-1252 and Latin-1. -# - Replace control characters with Windows-1252 equivalents and proceed. -# (4) Text is in a mix of Windows-1251 and Latin-1. -# - Oh wow. But, same deal. - -# Strategy: -# If maxchar < 128: -# ASCII! \o/ -# If maxchar < 256: -# Try Latin-1 => UTF-8. -# If all chars in Latin-1 + Windows-1252: -# Convert Latin-1 control characters to Windows-1252. -# Try Windows-1252 => UTF-8. -# Try other 1-byte encodings. -# If all chars in another 1-byte encoding: -# Try other 1-byte encodings. -# -# UTF-8 heuristic: -# * How many positive examples? -# * How many script clashes were introduced? -# * No failures? -# * Did it create modifiers out of nowhere? +def test_real_tweets(): + for orig, target in TEST_CASES: + eq_(fix_text_encoding(orig), target) \ No newline at end of file diff --git a/ftfy/test_unicode.py b/ftfy/test_unicode.py index f69084df..78362ea7 100644 --- a/ftfy/test_unicode.py +++ b/ftfy/test_unicode.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from ftfy import fix_bad_encoding, WINDOWS_1252_GREMLINS +from ftfy.fixes import fix_text_encoding import unicodedata # Most single-character strings which have been misencoded should be restored. @@ -7,9 +7,9 @@ def test_all_bmp_characters(): for index in xrange(0xa0, 0xfffd): char = unichr(index) # Exclude code points that are not assigned - if unicodedata.category(char) not in ('Co', 'Cn'): + if unicodedata.category(char) not in ('Co', 'Cn', 'Mc', 'Mn'): garble = char.encode('utf-8').decode('latin-1') - assert fix_bad_encoding(garble) == char + assert fix_text_encoding(garble) == char phrases = [ u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", @@ -28,5 +28,5 @@ def test_valid_phrases(): yield check_phrase, phrase[1:] def check_phrase(text): - assert fix_bad_encoding(text) == text, text + assert fix_text_encoding(text) == text, text From 785653fe42f3432315615aa3d18d37b999568fc3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 28 Jul 2013 16:24:12 -0400 Subject: [PATCH 029/502] Clarify error text --- ftfy/fixes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index ad03bd60..cbae6d17 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -16,7 +16,7 @@ import htmlentitydefs -BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode! +BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. ftfy is designed to fix problems that were introduced by handling Unicode incorrectly. It might be able to fix the bytes you just handed it, but the @@ -25,8 +25,8 @@ ftfy takes Unicode text as input. You should take these bytes and decode them from the encoding you think they are in. If you're not sure, you can -try decoding it as 'latin-1', which may work and at least won't make -anything worse. +try decoding it as 'latin-1' and letting ftfy take it from there. This may +or may not work, but at least it shouldn't make the situation worse. If you're confused by this, please read the Python Unicode HOWTO: From ee3fa96db7ed52feb6fe203c236223b5acbd5fb3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 28 Jul 2013 16:24:22 -0400 Subject: [PATCH 030/502] Remove single-character weirdness measure (It doesn't fit into what ftfy currently does.) --- ftfy/chardata.py | 128 ----------------------------------------------- 1 file changed, 128 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index e58b1a12..948c46ee 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -90,134 +90,6 @@ def _make_category_regex_ranges(): CATEGORY_RANGES = _make_category_regex_ranges() -# Rank the characters typically represented by a single byte -- that is, in -# Latin-1 or Windows-1252 -- by how weird it would be to see them in running -# text. -# -# 0 = not weird at all -# 1 = rare punctuation or rare letter that someone could certainly -# have a good reason to use. -# 2 = things that probably don't appear next to letters or other -# symbols, such as math or currency symbols -# 3 = obscure symbols that nobody would go out of their way to use -# (includes symbols that were replaced in ISO-8859-15) -# 4 = why would you use this? -# 5 = unprintable control character -# -# The Portuguese letter à (0xc3) is marked as weird because it would usually -# appear in the middle of a word in actual Portuguese, and meanwhile it -# appears in the mis-encodings of many common characters. - - -CHARACTER_WEIRDNESS = { - # \xa0 to \xaf - '\N{NO-BREAK SPACE}': 1, - '\N{INVERTED EXCLAMATION MARK}': 1, - '\N{CENT SIGN}': 2, - '\N{POUND SIGN}': 2, - '\N{CURRENCY SIGN}': 3, - '\N{YEN SIGN}': 2, - '\N{BROKEN BAR}': 4, - '\N{SECTION SIGN}': 2, - '\N{DIAERESIS}': 4, - '\N{COPYRIGHT SIGN}': 2, - '\N{FEMININE ORDINAL INDICATOR}': 3, - # '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}': 0, - '\N{NOT SIGN}': 3, - '\N{SOFT HYPHEN}': 1, - '\N{REGISTERED SIGN}': 1, - '\N{MACRON}': 4, - - # \xb0 to \xbf - '\N{DEGREE SIGN}': 2, - '\N{PLUS-MINUS SIGN}': 2, - '\N{SUPERSCRIPT TWO}': 2, - '\N{SUPERSCRIPT THREE}': 3, - '\N{ACUTE ACCENT}': 3, - '\N{MICRO SIGN}': 3, - '\N{PILCROW SIGN}': 3, - '\N{MIDDLE DOT}': 2, - '\N{CEDILLA}': 4, - '\N{SUPERSCRIPT ONE}': 4, - '\N{MASCULINE ORDINAL INDICATOR}': 4, - # '\N{RIGHT POINTING DOUBLE ANGLE QUOTATION MARK}': 0, - '\N{VULGAR FRACTION ONE QUARTER}': 2, - '\N{VULGAR FRACTION ONE HALF}': 2, - '\N{VULGAR FRACTION THREE QUARTERS}': 2, - '\N{INVERTED QUESTION MARK}': 0, - - # \xc0 to \xff - '\N{LATIN CAPITAL LETTER A WITH TILDE}': 1, - '\N{LATIN CAPITAL LETTER ETH}': 1, - '\N{MULTIPLICATION SIGN}': 1, - '\N{LATIN SMALL LETTER ETH}': 1, - '\N{DIVISION SIGN}': 1, - '\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}': 1, - - # \U0100 to \U0200 - '\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}': 1, - '\N{LATIN SMALL LETTER F WITH HOOK}': 4, - - # \U0200 to \U0300 - '\N{MODIFIER LETTER CIRCUMFLEX ACCENT}': 1, - '\N{CARON}': 3, - '\N{BREVE}': 3, - '\N{DOT ABOVE}': 3, - '\N{RING ABOVE}': 3, - '\N{OGONEK}': 3, - '\N{SMALL TILDE}': 4, - '\N{DOUBLE ACUTE ACCENT}': 3, - - # \U0300 to \U0400: selected Greek letters that were used in cp437. - # You are probably not writing actual Greek with this small set of - # letters. - '\N{GREEK CAPITAL LETTER GAMMA}': 1, - '\N{GREEK CAPITAL LETTER THETA}': 1, - '\N{GREEK CAPITAL LETTER SIGMA}': 1, - '\N{GREEK CAPITAL LETTER PHI}': 1, - '\N{GREEK CAPITAL LETTER OMEGA}': 1, - '\N{GREEK SMALL LETTER ALPHA}': 1, - '\N{GREEK SMALL LETTER DELTA}': 1, - '\N{GREEK SMALL LETTER EPSILON}': 1, - '\N{GREEK SMALL LETTER PI}': 1, - '\N{GREEK SMALL LETTER SIGMA}': 1, - '\N{GREEK SMALL LETTER TAU}': 1, - '\N{GREEK SMALL LETTER PHI}': 1, - - # \U2000 to \U2400 - '\N{DAGGER}': 2, - '\N{DOUBLE DAGGER}': 3, - '\N{PER MILLE SIGN}': 4, - '\N{FRACTION SLASH}': 2, - '\N{SUPERSCRIPT LATIN SMALL LETTER N}': 3, - '\N{PESETA SIGN}': 3, - '\N{NUMERO SIGN}': 2, - '\N{PARTIAL DIFFERENTIAL}': 1, - '\N{INCREMENT}': 1, - '\N{N-ARY PRODUCT}': 2, - '\N{N-ARY SUMMATION}': 2, - '\N{SQUARE ROOT}': 1, - '\N{INFINITY}': 1, - '\N{INTERSECTION}': 2, - '\N{INTEGRAL}': 2, - '\N{ALMOST EQUAL TO}': 1, - '\N{NOT EQUAL TO}': 1, - '\N{IDENTICAL TO}': 2, - '\N{LESS-THAN OR EQUAL TO}': 1, - '\N{GREATER-THAN OR EQUAL TO}': 1, - '\N{REVERSED NOT SIGN}': 4, - '\N{TOP HALF INTEGRAL}': 3, - '\N{BOTTOM HALF INTEGRAL}': 3, - - # \U2500 to \U2600 are mostly box drawings; they're okay, or at least - # require a different heuristic - '\N{LOZENGE}': 2, - '\uf8ff': 2, # the Apple symbol - '\N{LATIN SMALL LIGATURE FI}': 1, - '\N{LATIN SMALL LIGATURE FL}': 1, -} - - # A translate mapping that will strip all control characters except \t and \n. # This incidentally has the effect of normalizing Windows \r\n line endings to # Unix \n line endings. From 3626fb9eba1441290970ad32d8357e64e69044de Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 28 Jul 2013 16:25:44 -0400 Subject: [PATCH 031/502] bump version to 3.0b for upcoming version 3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2e210d83..c29393f2 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='2.0.2', + version='3.0b', maintainer='Luminoso, LLC', maintainer_email='dev@lumino.so', license="MIT", From 7f384f3eae786c8517965e4d24deb9a40a61c18b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 28 Jul 2013 16:40:47 -0400 Subject: [PATCH 032/502] preserve options from fix_text in fix_text_segment --- ftfy/__init__.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index c1ac21da..437d59d2 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -88,7 +88,6 @@ def fix_text(text, """ out = [] pos = 0 - entities = True while pos < len(text): textbreak = text.find('\n', pos, pos + MAXLEN) if textbreak == -1: @@ -104,9 +103,20 @@ def fix_text(text, if '<' in substring and '>' in substring: # we see angle brackets together; this could be HTML - entities = False - - out.append(fix_text_segment(substring, normalization, entities)) + fix_entities = False + + out.append( + fix_text_segment( + substring, + fix_entities=fix_entities, + remove_terminal_escapes=remove_terminal_escapes, + fix_encoding=fix_encoding, + normalization=normalization, + uncurl_quotes=uncurl_quotes, + remove_control_chars=remove_control_chars, + remove_bom=remove_bom + ) + ) pos += textbreak + 1 return ''.join(out) @@ -170,3 +180,5 @@ def fix_text_segment(text, if text == origtext: return text + +fix_text_encoding = fixes.fix_text_encoding From daea08a9be3ef1a22a4c41c17989594f05c231c5 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 28 Jul 2013 16:54:13 -0400 Subject: [PATCH 033/502] python 3 fixes --- ftfy/test_unicode.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ftfy/test_unicode.py b/ftfy/test_unicode.py index 78362ea7..5438f3d3 100644 --- a/ftfy/test_unicode.py +++ b/ftfy/test_unicode.py @@ -1,13 +1,17 @@ # -*- coding: utf-8 -*- from ftfy.fixes import fix_text_encoding import unicodedata +import sys + +if sys.hexversion >= 0x03000000: + unichr = chr # Most single-character strings which have been misencoded should be restored. def test_all_bmp_characters(): - for index in xrange(0xa0, 0xfffd): + for index in range(0xa0, 0xfffd): char = unichr(index) # Exclude code points that are not assigned - if unicodedata.category(char) not in ('Co', 'Cn', 'Mc', 'Mn'): + if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): garble = char.encode('utf-8').decode('latin-1') assert fix_text_encoding(garble) == char @@ -22,7 +26,6 @@ def test_all_bmp_characters(): # These phrases should not be erroneously "fixed" def test_valid_phrases(): for phrase in phrases: - print phrase yield check_phrase, phrase # make it not just confirm based on the opening punctuation yield check_phrase, phrase[1:] From 6b18c333e171696f5bdf25fa1188f62b2869fbf9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 28 Jul 2013 17:50:40 -0400 Subject: [PATCH 034/502] Remove debug code --- ftfy/fixes.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index cbae6d17..00bb7e3f 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -5,8 +5,6 @@ from ftfy.badness import text_cost import re import sys -import logging -LOG = logging.getLogger(__name__) if sys.hexversion >= 0x03000000: from html import entities @@ -134,11 +132,6 @@ def fix_text_and_explain(text): # points up to 0xff. This can then be converted into the intended # bytes by encoding it as Latin-1. sorta_encoded_text = text.translate(CHARMAPS[encoding]) - if max(sorta_encoded_text) > unichr(255): - LOG.warn("Charmap: %s" % CHARMAPS[encoding]) - LOG.warn("Encoding: %s" % encoding) - LOG.warn("Original text: %s" % text) - LOG.warn("encoded text: %s" % sorta_encoded_text) encoded_bytes = sorta_encoded_text.encode('latin-1') # Now, find out if it's UTF-8. From 2edc6cfefdd3ad92032a38b919a0016b2c686a77 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 29 Jul 2013 13:21:00 -0400 Subject: [PATCH 035/502] remove debugging output --- ftfy/single_byte_characters.txt | 377 -------------------------------- 1 file changed, 377 deletions(-) delete mode 100644 ftfy/single_byte_characters.txt diff --git a/ftfy/single_byte_characters.txt b/ftfy/single_byte_characters.txt deleted file mode 100644 index af46fb91..00000000 --- a/ftfy/single_byte_characters.txt +++ /dev/null @@ -1,377 +0,0 @@ -{'Sc': '$¢£¤¥₧€', 'Pi': '«‘“‹', 'Po': '!"#%&\'*,./:;?@\\\\¡§¶·¿†‡•…‰', 'Cc': '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f', 'Pc': '_', 'Sk': '\\^`¨¯´¸˘˙˚˛˜˝', 'Lu': 'ABCDEFGHIJKLMNOPQRSTUVWXYZÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞŒŠŸŽΓΘΣΦΩ', 'So': '¦©®°™⌐─│┌┐└┘├┤┬┴┼═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬▀▄█▌▐░▒▓■◊', 'Pe': ')\\]}', 'Pd': '\\-–—', 'Lo': 'ªº', 'Nd': '0123456789', 'Lm': 'ˆˇⁿ', 'Ll': 'abcdefghijklmnopqrstuvwxyzµßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿıœšžƒαδεπστφfifl', 'Cf': '\xad', 'Pf': '»’”›', 'No': '²³¹¼½¾', 'Ps': '(\\[{‚„', 'Zs': ' \xa0', 'Sm': '+<=>|~¬±×÷⁄∂∆∏∑∙√∞∩∫≈≠≡≤≥⌠⌡', 'Co': '\uf8ff'} -€ € Ä Ç -  Å ü -‚ ‚ Ç é -ƒ ƒ É â -„ „ Ñ ä -… … Ö à -† † Ü å -‡ ‡ á ç -ˆ ˆ à ê -‰ ‰ â ë -Š Š ä è -‹ ‹ ã ï -Œ Œ å î -  ç ì -Ž Ž é Ä -  è Å -  ê É -‘ ‘ ë æ -’ ’ í Æ -“ “ ì ô -” ” î ö -• • ï ò -– – ñ û -— — ó ù -˜ ˜ ò ÿ -™ ™ ô Ö -š š ö Ü -› › õ ¢ -œ œ ú £ -  ù ¥ -ž ž û ₧ -Ÿ Ÿ ü ƒ -    † á -¡ ¡ ° í -¢ ¢ ¢ ó -£ £ £ ú -¤ ¤ § ñ -¥ ¥ • Ñ -¦ ¦ ¶ ª -§ § ß º -¨ ¨ ® ¿ -© © © ⌐ -ª ª ™ ¬ -« « ´ ½ -¬ ¬ ¨ ¼ -­ ­ ≠ ¡ -® ® Æ « -¯ ¯ Ø » -° ° ∞ ░ -± ± ± ▒ -² ² ≤ ▓ -³ ³ ≥ │ -´ ´ ¥ ┤ -µ µ µ ╡ -¶ ¶ ∂ ╢ -· · ∑ ╖ -¸ ¸ ∏ ╕ -¹ ¹ π ╣ -º º ∫ ║ -» » ª ╗ -¼ ¼ º ╝ -½ ½ Ω ╜ -¾ ¾ æ ╛ -¿ ¿ ø ┐ -À À ¿ └ -Á Á ¡ ┴ -  ¬ ┬ -à à √ ├ -Ä Ä ƒ ─ -Å Å ≈ ┼ -Æ Æ ∆ ╞ -Ç Ç « ╟ -È È » ╚ -É É … ╔ -Ê Ê   ╩ -Ë Ë À ╦ -Ì Ì Ã ╠ -Í Í Õ ═ -Î Î Œ ╬ -Ï Ï œ ╧ -Ð Ð – ╨ -Ñ Ñ — ╤ -Ò Ò “ ╥ -Ó Ó ” ╙ -Ô Ô ‘ ╘ -Õ Õ ’ ╒ -Ö Ö ÷ ╓ -× × ◊ ╫ -Ø Ø ÿ ╪ -Ù Ù Ÿ ┘ -Ú Ú ⁄ ┌ -Û Û € █ -Ü Ü ‹ ▄ -Ý Ý › ▌ -Þ Þ fi ▐ -ß ß fl ▀ -à à ‡ α -á á · ß -â â ‚ Γ -ã ã „ π -ä ä ‰ Σ -å å  σ -æ æ Ê µ -ç ç Á τ -è è Ë Φ -é é È Θ -ê ê Í Ω -ë ë Î δ -ì ì Ï ∞ -í í Ì φ -î î Ó ε -ï ï Ô ∩ -ð ð  ≡ -ñ ñ Ò ± -ò ò Ú ≥ -ó ó Û ≤ -ô ô Ù ⌠ -õ õ ı ⌡ -ö ö ˆ ÷ -÷ ÷ ˜ ≈ -ø ø ¯ ° -ù ù ˘ ∙ -ú ú ˙ · -û û ˚ √ -ü ü ¸ ⁿ -ý ý ˝ ² -þ þ ˛ ■ -ÿ ÿ ˇ   - -€ 0x80 Cc [unknown] - 0x81 Cc [unknown] -‚ 0x82 Cc [unknown] -ƒ 0x83 Cc [unknown] -„ 0x84 Cc [unknown] -… 0x85 Cc [unknown] -† 0x86 Cc [unknown] -‡ 0x87 Cc [unknown] -ˆ 0x88 Cc [unknown] -‰ 0x89 Cc [unknown] -Š 0x8a Cc [unknown] -‹ 0x8b Cc [unknown] -Œ 0x8c Cc [unknown] - 0x8d Cc [unknown] -Ž 0x8e Cc [unknown] - 0x8f Cc [unknown] - 0x90 Cc [unknown] -‘ 0x91 Cc [unknown] -’ 0x92 Cc [unknown] -“ 0x93 Cc [unknown] -” 0x94 Cc [unknown] -• 0x95 Cc [unknown] -– 0x96 Cc [unknown] -— 0x97 Cc [unknown] -˜ 0x98 Cc [unknown] -™ 0x99 Cc [unknown] -š 0x9a Cc [unknown] -› 0x9b Cc [unknown] -œ 0x9c Cc [unknown] - 0x9d Cc [unknown] -ž 0x9e Cc [unknown] -Ÿ 0x9f Cc [unknown] -  0xa0 Zs NO-BREAK SPACE -¡ 0xa1 Po INVERTED EXCLAMATION MARK -¢ 0xa2 Sc CENT SIGN -£ 0xa3 Sc POUND SIGN -¤ 0xa4 Sc CURRENCY SIGN -¥ 0xa5 Sc YEN SIGN -¦ 0xa6 So BROKEN BAR -§ 0xa7 Po SECTION SIGN -¨ 0xa8 Sk DIAERESIS -© 0xa9 So COPYRIGHT SIGN -ª 0xaa Lo FEMININE ORDINAL INDICATOR -« 0xab Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK -¬ 0xac Sm NOT SIGN -­ 0xad Cf SOFT HYPHEN -® 0xae So REGISTERED SIGN -¯ 0xaf Sk MACRON -° 0xb0 So DEGREE SIGN -± 0xb1 Sm PLUS-MINUS SIGN -² 0xb2 No SUPERSCRIPT TWO -³ 0xb3 No SUPERSCRIPT THREE -´ 0xb4 Sk ACUTE ACCENT -µ 0xb5 Ll MICRO SIGN -¶ 0xb6 Po PILCROW SIGN -· 0xb7 Po MIDDLE DOT -¸ 0xb8 Sk CEDILLA -¹ 0xb9 No SUPERSCRIPT ONE -º 0xba Lo MASCULINE ORDINAL INDICATOR -» 0xbb Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK -¼ 0xbc No VULGAR FRACTION ONE QUARTER -½ 0xbd No VULGAR FRACTION ONE HALF -¾ 0xbe No VULGAR FRACTION THREE QUARTERS -¿ 0xbf Po INVERTED QUESTION MARK -À 0xc0 Lu LATIN CAPITAL LETTER A WITH GRAVE -Á 0xc1 Lu LATIN CAPITAL LETTER A WITH ACUTE - 0xc2 Lu LATIN CAPITAL LETTER A WITH CIRCUMFLEX -à 0xc3 Lu LATIN CAPITAL LETTER A WITH TILDE -Ä 0xc4 Lu LATIN CAPITAL LETTER A WITH DIAERESIS -Å 0xc5 Lu LATIN CAPITAL LETTER A WITH RING ABOVE -Æ 0xc6 Lu LATIN CAPITAL LETTER AE -Ç 0xc7 Lu LATIN CAPITAL LETTER C WITH CEDILLA -È 0xc8 Lu LATIN CAPITAL LETTER E WITH GRAVE -É 0xc9 Lu LATIN CAPITAL LETTER E WITH ACUTE -Ê 0xca Lu LATIN CAPITAL LETTER E WITH CIRCUMFLEX -Ë 0xcb Lu LATIN CAPITAL LETTER E WITH DIAERESIS -Ì 0xcc Lu LATIN CAPITAL LETTER I WITH GRAVE -Í 0xcd Lu LATIN CAPITAL LETTER I WITH ACUTE -Î 0xce Lu LATIN CAPITAL LETTER I WITH CIRCUMFLEX -Ï 0xcf Lu LATIN CAPITAL LETTER I WITH DIAERESIS -Ð 0xd0 Lu LATIN CAPITAL LETTER ETH -Ñ 0xd1 Lu LATIN CAPITAL LETTER N WITH TILDE -Ò 0xd2 Lu LATIN CAPITAL LETTER O WITH GRAVE -Ó 0xd3 Lu LATIN CAPITAL LETTER O WITH ACUTE -Ô 0xd4 Lu LATIN CAPITAL LETTER O WITH CIRCUMFLEX -Õ 0xd5 Lu LATIN CAPITAL LETTER O WITH TILDE -Ö 0xd6 Lu LATIN CAPITAL LETTER O WITH DIAERESIS -× 0xd7 Sm MULTIPLICATION SIGN -Ø 0xd8 Lu LATIN CAPITAL LETTER O WITH STROKE -Ù 0xd9 Lu LATIN CAPITAL LETTER U WITH GRAVE -Ú 0xda Lu LATIN CAPITAL LETTER U WITH ACUTE -Û 0xdb Lu LATIN CAPITAL LETTER U WITH CIRCUMFLEX -Ü 0xdc Lu LATIN CAPITAL LETTER U WITH DIAERESIS -Ý 0xdd Lu LATIN CAPITAL LETTER Y WITH ACUTE -Þ 0xde Lu LATIN CAPITAL LETTER THORN -ß 0xdf Ll LATIN SMALL LETTER SHARP S -à 0xe0 Ll LATIN SMALL LETTER A WITH GRAVE -á 0xe1 Ll LATIN SMALL LETTER A WITH ACUTE -â 0xe2 Ll LATIN SMALL LETTER A WITH CIRCUMFLEX -ã 0xe3 Ll LATIN SMALL LETTER A WITH TILDE -ä 0xe4 Ll LATIN SMALL LETTER A WITH DIAERESIS -å 0xe5 Ll LATIN SMALL LETTER A WITH RING ABOVE -æ 0xe6 Ll LATIN SMALL LETTER AE -ç 0xe7 Ll LATIN SMALL LETTER C WITH CEDILLA -è 0xe8 Ll LATIN SMALL LETTER E WITH GRAVE -é 0xe9 Ll LATIN SMALL LETTER E WITH ACUTE -ê 0xea Ll LATIN SMALL LETTER E WITH CIRCUMFLEX -ë 0xeb Ll LATIN SMALL LETTER E WITH DIAERESIS -ì 0xec Ll LATIN SMALL LETTER I WITH GRAVE -í 0xed Ll LATIN SMALL LETTER I WITH ACUTE -î 0xee Ll LATIN SMALL LETTER I WITH CIRCUMFLEX -ï 0xef Ll LATIN SMALL LETTER I WITH DIAERESIS -ð 0xf0 Ll LATIN SMALL LETTER ETH -ñ 0xf1 Ll LATIN SMALL LETTER N WITH TILDE -ò 0xf2 Ll LATIN SMALL LETTER O WITH GRAVE -ó 0xf3 Ll LATIN SMALL LETTER O WITH ACUTE -ô 0xf4 Ll LATIN SMALL LETTER O WITH CIRCUMFLEX -õ 0xf5 Ll LATIN SMALL LETTER O WITH TILDE -ö 0xf6 Ll LATIN SMALL LETTER O WITH DIAERESIS -÷ 0xf7 Sm DIVISION SIGN -ø 0xf8 Ll LATIN SMALL LETTER O WITH STROKE -ù 0xf9 Ll LATIN SMALL LETTER U WITH GRAVE -ú 0xfa Ll LATIN SMALL LETTER U WITH ACUTE -û 0xfb Ll LATIN SMALL LETTER U WITH CIRCUMFLEX -ü 0xfc Ll LATIN SMALL LETTER U WITH DIAERESIS -ý 0xfd Ll LATIN SMALL LETTER Y WITH ACUTE -þ 0xfe Ll LATIN SMALL LETTER THORN -ÿ 0xff Ll LATIN SMALL LETTER Y WITH DIAERESIS -ı 0x131 Ll LATIN SMALL LETTER DOTLESS I -Œ 0x152 Lu LATIN CAPITAL LIGATURE OE -œ 0x153 Ll LATIN SMALL LIGATURE OE -Š 0x160 Lu LATIN CAPITAL LETTER S WITH CARON -š 0x161 Ll LATIN SMALL LETTER S WITH CARON -Ÿ 0x178 Lu LATIN CAPITAL LETTER Y WITH DIAERESIS -Ž 0x17d Lu LATIN CAPITAL LETTER Z WITH CARON -ž 0x17e Ll LATIN SMALL LETTER Z WITH CARON -ƒ 0x192 Ll LATIN SMALL LETTER F WITH HOOK -ˆ 0x2c6 Lm MODIFIER LETTER CIRCUMFLEX ACCENT -ˇ 0x2c7 Lm CARON -˘ 0x2d8 Sk BREVE -˙ 0x2d9 Sk DOT ABOVE -˚ 0x2da Sk RING ABOVE -˛ 0x2db Sk OGONEK -˜ 0x2dc Sk SMALL TILDE -˝ 0x2dd Sk DOUBLE ACUTE ACCENT -Γ 0x393 Lu GREEK CAPITAL LETTER GAMMA -Θ 0x398 Lu GREEK CAPITAL LETTER THETA -Σ 0x3a3 Lu GREEK CAPITAL LETTER SIGMA -Φ 0x3a6 Lu GREEK CAPITAL LETTER PHI -Ω 0x3a9 Lu GREEK CAPITAL LETTER OMEGA -α 0x3b1 Ll GREEK SMALL LETTER ALPHA -δ 0x3b4 Ll GREEK SMALL LETTER DELTA -ε 0x3b5 Ll GREEK SMALL LETTER EPSILON -π 0x3c0 Ll GREEK SMALL LETTER PI -σ 0x3c3 Ll GREEK SMALL LETTER SIGMA -τ 0x3c4 Ll GREEK SMALL LETTER TAU -φ 0x3c6 Ll GREEK SMALL LETTER PHI -– 0x2013 Pd EN DASH -— 0x2014 Pd EM DASH -‘ 0x2018 Pi LEFT SINGLE QUOTATION MARK -’ 0x2019 Pf RIGHT SINGLE QUOTATION MARK -‚ 0x201a Ps SINGLE LOW-9 QUOTATION MARK -“ 0x201c Pi LEFT DOUBLE QUOTATION MARK -” 0x201d Pf RIGHT DOUBLE QUOTATION MARK -„ 0x201e Ps DOUBLE LOW-9 QUOTATION MARK -† 0x2020 Po DAGGER -‡ 0x2021 Po DOUBLE DAGGER -• 0x2022 Po BULLET -… 0x2026 Po HORIZONTAL ELLIPSIS -‰ 0x2030 Po PER MILLE SIGN -‹ 0x2039 Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK -› 0x203a Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK -⁄ 0x2044 Sm FRACTION SLASH -ⁿ 0x207f Lm SUPERSCRIPT LATIN SMALL LETTER N -₧ 0x20a7 Sc PESETA SIGN -€ 0x20ac Sc EURO SIGN -™ 0x2122 So TRADE MARK SIGN -∂ 0x2202 Sm PARTIAL DIFFERENTIAL -∆ 0x2206 Sm INCREMENT -∏ 0x220f Sm N-ARY PRODUCT -∑ 0x2211 Sm N-ARY SUMMATION -∙ 0x2219 Sm BULLET OPERATOR -√ 0x221a Sm SQUARE ROOT -∞ 0x221e Sm INFINITY -∩ 0x2229 Sm INTERSECTION -∫ 0x222b Sm INTEGRAL -≈ 0x2248 Sm ALMOST EQUAL TO -≠ 0x2260 Sm NOT EQUAL TO -≡ 0x2261 Sm IDENTICAL TO -≤ 0x2264 Sm LESS-THAN OR EQUAL TO -≥ 0x2265 Sm GREATER-THAN OR EQUAL TO -⌐ 0x2310 So REVERSED NOT SIGN -⌠ 0x2320 Sm TOP HALF INTEGRAL -⌡ 0x2321 Sm BOTTOM HALF INTEGRAL -─ 0x2500 So BOX DRAWINGS LIGHT HORIZONTAL -│ 0x2502 So BOX DRAWINGS LIGHT VERTICAL -┌ 0x250c So BOX DRAWINGS LIGHT DOWN AND RIGHT -┐ 0x2510 So BOX DRAWINGS LIGHT DOWN AND LEFT -└ 0x2514 So BOX DRAWINGS LIGHT UP AND RIGHT -┘ 0x2518 So BOX DRAWINGS LIGHT UP AND LEFT -├ 0x251c So BOX DRAWINGS LIGHT VERTICAL AND RIGHT -┤ 0x2524 So BOX DRAWINGS LIGHT VERTICAL AND LEFT -┬ 0x252c So BOX DRAWINGS LIGHT DOWN AND HORIZONTAL -┴ 0x2534 So BOX DRAWINGS LIGHT UP AND HORIZONTAL -┼ 0x253c So BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL -═ 0x2550 So BOX DRAWINGS DOUBLE HORIZONTAL -║ 0x2551 So BOX DRAWINGS DOUBLE VERTICAL -╒ 0x2552 So BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE -╓ 0x2553 So BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE -╔ 0x2554 So BOX DRAWINGS DOUBLE DOWN AND RIGHT -╕ 0x2555 So BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE -╖ 0x2556 So BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE -╗ 0x2557 So BOX DRAWINGS DOUBLE DOWN AND LEFT -╘ 0x2558 So BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE -╙ 0x2559 So BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE -╚ 0x255a So BOX DRAWINGS DOUBLE UP AND RIGHT -╛ 0x255b So BOX DRAWINGS UP SINGLE AND LEFT DOUBLE -╜ 0x255c So BOX DRAWINGS UP DOUBLE AND LEFT SINGLE -╝ 0x255d So BOX DRAWINGS DOUBLE UP AND LEFT -╞ 0x255e So BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE -╟ 0x255f So BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE -╠ 0x2560 So BOX DRAWINGS DOUBLE VERTICAL AND RIGHT -╡ 0x2561 So BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE -╢ 0x2562 So BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE -╣ 0x2563 So BOX DRAWINGS DOUBLE VERTICAL AND LEFT -╤ 0x2564 So BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE -╥ 0x2565 So BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE -╦ 0x2566 So BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL -╧ 0x2567 So BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE -╨ 0x2568 So BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE -╩ 0x2569 So BOX DRAWINGS DOUBLE UP AND HORIZONTAL -╪ 0x256a So BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE -╫ 0x256b So BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE -╬ 0x256c So BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL -▀ 0x2580 So UPPER HALF BLOCK -▄ 0x2584 So LOWER HALF BLOCK -█ 0x2588 So FULL BLOCK -▌ 0x258c So LEFT HALF BLOCK -▐ 0x2590 So RIGHT HALF BLOCK -░ 0x2591 So LIGHT SHADE -▒ 0x2592 So MEDIUM SHADE -▓ 0x2593 So DARK SHADE -■ 0x25a0 So BLACK SQUARE -◊ 0x25ca So LOZENGE - 0xf8ff Co [unknown] -fi 0xfb01 Ll LATIN SMALL LIGATURE FI -fl 0xfb02 Ll LATIN SMALL LIGATURE FL From bcbfaebabcccc77372f43bc1a640efa753bae310 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 29 Jul 2013 13:36:33 -0400 Subject: [PATCH 036/502] fix_text_encoding should decode repeatedly (The new idempotence code is not sufficient, because one of the intermediate characters might be a curly quotation mark or something.) --- ftfy/fixes.py | 7 +++++-- ftfy/test_real_tweets.py | 7 ++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 00bb7e3f..f677383b 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -93,8 +93,11 @@ def fix_text_encoding(text): >>> print(fix_text_encoding('This text was never UTF-8 at all\x85')) This text was never UTF-8 at all… """ - fixed, _ = fix_text_and_explain(text) - return fixed + while True: + origtext = text + text, _ = fix_text_and_explain(text) + if text == origtext: + return text def fix_text_and_explain(text): if isinstance(text, bytes): diff --git a/ftfy/test_real_tweets.py b/ftfy/test_real_tweets.py index f93a50b6..6825ac7d 100644 --- a/ftfy/test_real_tweets.py +++ b/ftfy/test_real_tweets.py @@ -20,4 +20,9 @@ def test_real_tweets(): for orig, target in TEST_CASES: - eq_(fix_text_encoding(orig), target) \ No newline at end of file + # make sure we can decode the text as intended + eq_(fix_text_encoding(orig), target) + + # make sure we can decode as intended even with an extra layer of badness + extra_bad = orig.encode('utf-8').decode('latin-1') + eq_(fix_text_encoding(extra_bad), target) From 7b5bb2d0c0220dfca7ea710570d2244b9bdbe2bc Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 29 Jul 2013 15:46:38 -0400 Subject: [PATCH 037/502] added fix_surrogate_encoding fixer This allows the last test in test_real_tweets to pass, and not just by coincidence on narrow Python builds. --- ftfy/__init__.py | 11 +++++++++- ftfy/fixes.py | 45 +++++++++++++++++++++++++++++++++++++++- ftfy/test_real_tweets.py | 25 ++++++++++++++++------ 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 437d59d2..97335b54 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -20,6 +20,7 @@ def fix_text(text, normalization='NFKC', uncurl_quotes=True, remove_control_chars=True, + fix_surrogate_encoding=True, remove_bom=True): """ Given Unicode text as input, make its representation consistent and @@ -49,6 +50,11 @@ def fix_text(text, periods, and the ligature 'fl' will be replaced with 'fl'. - If `uncurl_quotes` is True, replace various curly quotation marks with plain-ASCII straight quotes. + - If `fix_surrogate_encoding` is true, replace surrogate encodings of + "astral" Unicode characters with the actual characters... but only + on Python builds ("wide" builds or Python >= 3.3) that can handle + astral characters. On "narrow" builds, it'll keep using surrogate + characters and give a warning. - If `remove_bom` is True, remove the Byte-Order Mark if it exists. - If anything was changed, repeat all the steps, so that the function is idempotent. "&amp;" will become "&", for example, not "&". @@ -114,6 +120,7 @@ def fix_text(text, normalization=normalization, uncurl_quotes=uncurl_quotes, remove_control_chars=remove_control_chars, + fix_surrogate_encoding=fix_surrogate_encoding, remove_bom=remove_bom ) ) @@ -148,6 +155,7 @@ def fix_text_segment(text, normalization='NFKC', uncurl_quotes=True, remove_control_chars=True, + fix_surrogate_encoding=True, remove_bom=True): """ Apply fixes to text in a single chunk. This could be a line of text @@ -176,7 +184,8 @@ def fix_text_segment(text, text = fixes.remove_control_chars(text) if remove_bom: text = fixes.remove_bom(text) - + if fix_surrogate_encoding: + text = fixes.fix_surrogate_encoding(text) if text == origtext: return text diff --git a/ftfy/fixes.py b/ftfy/fixes.py index f677383b..1427dd22 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals from ftfy.chardata import (possible_encoding, CHARMAPS, CHARMAP_ENCODINGS, - CONTROL_CHARS) + CONTROL_CHARS, CATEGORY_RANGES) from ftfy.badness import text_cost import re import sys +import warnings if sys.hexversion >= 0x03000000: from html import entities @@ -31,6 +32,20 @@ http://docs.python.org/%d/howto/unicode.html """ % sys.version_info[0] +SURROGATES_WARNING_TEXT = """ +This text contains characters that this version of Python can't handle. + +You are using a "narrow" build of Python, which can only represent the first +2^16 characters in Unicode. There are characters called "astral" characters +above this range, including emoji and some rare Chinese characters. + +Python will instead represent these as pairs of surrogate characters. Your +string will be the wrong length and will cause problems if you try to refer +to its individual characters. + +Unfortunately, this can't be fixed from within Python. You should download +or compile a "wide" build, or upgrade to Python 3.3 or later. +""" def fix_text_encoding(text): """ @@ -236,9 +251,37 @@ def uncurl_quotes(text): def remove_control_chars(text): + """ + Remove all control characters except for the important ones (line feed + and tab). + + This removes all characters from U+0000 to U+001F and U+0080 to U+009F, + but it of course leaves TAB (U+0009) and LF (U+000A) alone. + + Note that, for example, this will convert Windows-style CRLF line breaks + into Unix-style LF line breaks, because CR (U+000D) is a control + character that does nothing on its own. You are likely to want this + transformation anyway. + """ return text.translate(CONTROL_CHARS) +SURROGATES_RE = re.compile('[{0}]'.format(CATEGORY_RANGES['Cs'])) +def fix_surrogate_encoding(text): + # Python has no functions that manipulate surrogate pairs + # directly. However, if they can be fixed at all, they will + # be fixed by round-tripping to UTF-8 encoding. + # + # On the other hand, on a narrow build of Python, this can't possibly + # work. The fixed text simply cannot be represented by the unicode + # type in narrow Python. In that case, we'll let it do its thing with + # surrogates, but also warn the user. + if sys.maxunicode < 0x10000 and SURROGATES_RE.search(text): + warnings.warn(SURROGATES_WARNING_TEXT, UnicodeWarning) + + return text.encode('utf-8').decode('utf-8') + + def remove_bom(text): """ Remove a left-over byte-order mark. diff --git a/ftfy/test_real_tweets.py b/ftfy/test_real_tweets.py index 6825ac7d..defa6a78 100644 --- a/ftfy/test_real_tweets.py +++ b/ftfy/test_real_tweets.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from ftfy.fixes import fix_text_encoding +from ftfy import fix_text from nose.tools import eq_ TEST_CASES = [ @@ -10,19 +10,32 @@ #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", # "Deja dos heridos hundimiento de barco turístico en Acapulco."), ("✔ No problems", "✔ No problems"), - ('4288×…', '4288×…'), + ('4288×…', '4288×...'), ("РґРѕСЂРѕРіРµ Из-РїРѕРґ http://t.co/A0eJAMTuJ1 #футбол", "дороге Из-под http://t.co/A0eJAMTuJ1 #футбол"), - ("Hi guys 😍", "Hi guys 😍"), ("\x84Handwerk bringt dich \xfcberall hin\x93: Von der YOU bis nach Monaco", - '„Handwerk bringt dich überall hin“: Von der YOU bis nach Monaco') + '"Handwerk bringt dich überall hin": Von der YOU bis nach Monaco'), + ("Hi guys 😍", "Hi guys 😍"), ] def test_real_tweets(): + """ + Test with text actually found on Twitter. + + I collected these test cases by listening to the Twitter streaming API for + a million or so tweets, picking out examples with high weirdness according + to ftfy version 2, and seeing what ftfy decoded them to. There are some + impressive things that can happen to text, even in an ecosystem that is + supposedly entirely UTF-8. + + The tweets that appear in TEST_CASES are the most interesting examples of + these, with some trickiness of how to decode them into the actually intended + text. + """ for orig, target in TEST_CASES: # make sure we can decode the text as intended - eq_(fix_text_encoding(orig), target) + eq_(fix_text(orig), target) # make sure we can decode as intended even with an extra layer of badness extra_bad = orig.encode('utf-8').decode('latin-1') - eq_(fix_text_encoding(extra_bad), target) + eq_(fix_text(extra_bad), target) From e7fdfe3409d524d91df2bc64fde06f8821f951d8 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 30 Jul 2013 14:07:24 -0400 Subject: [PATCH 038/502] test that we can decode all doubly-encoded characters (doesn't pass yet) --- ftfy/test_unicode.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ftfy/test_unicode.py b/ftfy/test_unicode.py index 5438f3d3..5c7b141d 100644 --- a/ftfy/test_unicode.py +++ b/ftfy/test_unicode.py @@ -2,6 +2,7 @@ from ftfy.fixes import fix_text_encoding import unicodedata import sys +from nose.tools import eq_ if sys.hexversion >= 0x03000000: unichr = chr @@ -13,7 +14,9 @@ def test_all_bmp_characters(): # Exclude code points that are not assigned if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): garble = char.encode('utf-8').decode('latin-1') - assert fix_text_encoding(garble) == char + garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') + eq_(fix_text_encoding(garble), char) + eq_(fix_text_encoding(garble2), char) phrases = [ u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", @@ -31,5 +34,5 @@ def test_valid_phrases(): yield check_phrase, phrase[1:] def check_phrase(text): - assert fix_text_encoding(text) == text, text + eq_(fix_text_encoding(text), text) From e4fd539e8bbebaf8debd709b34f76ce03a34fd81 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 1 Aug 2013 23:07:59 -0400 Subject: [PATCH 039/502] Don't reinterpret latin-1 as windows-1252 when it breaks --- ftfy/fixes.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 1427dd22..a9a9e7aa 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -178,9 +178,13 @@ def fix_text_and_explain(text): # not in Windows-1252. Those are C1 control characters. Nobody # wants those. Assume they were meant to be Windows-1252. encoded = text.encode('latin-1') - fixed = encoded.decode('windows-1252', errors='replace') - steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] - return fixed, steps + try: + fixed = encoded.decode('windows-1252') + steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] + return fixed, steps + except UnicodeDecodeError: + # Well, never mind. + pass # The cases that remain are mixups between two different single-byte # encodings, neither of which is Latin-1. From 7e584597820b087698858ae085a15031b52e0bcf Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 1 Aug 2013 23:08:39 -0400 Subject: [PATCH 040/502] Unexpected capital letters are only weird when they're not ASCII. CamelCase exists, after all. --- ftfy/__init__.py | 1 + ftfy/badness.py | 4 ++-- ftfy/chardata.py | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 97335b54..f769e265 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -191,3 +191,4 @@ def fix_text_segment(text, fix_text_encoding = fixes.fix_text_encoding +fix_bad_encoding = fix_text_encoding diff --git a/ftfy/badness.py b/ftfy/badness.py index b28f8d7a..9a5cec78 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -5,8 +5,8 @@ def _make_weirdness_regex(): groups = [] - groups.append(u'([{Ll}][{Lu}{Lo}{Lt}])'.format(**CATEGORY_RANGES)) - groups.append(u'([{Mn}{Mc}])'.format(**CATEGORY_RANGES)) + groups.append(u'([{Ll}][{Lun}{Lo}{Lt}])'.format(**CATEGORY_RANGES)) + groups.append(u'([^{Lo}][{Mn}{Mc}])'.format(**CATEGORY_RANGES)) exclusive_categories = ['Lm', 'Sk', 'Sm', 'Sc', 'No', 'So'] for cat1 in exclusive_categories: others = exclusive_categories[:] diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 948c46ee..d0f0881f 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -89,6 +89,12 @@ def _make_category_regex_ranges(): return ranges CATEGORY_RANGES = _make_category_regex_ranges() +# Separate out non-ASCII uppercase characters +_non_ascii_upper = ''.join( + ch for ch in CATEGORY_RANGES['Lu'] + if ord(ch) >= 0x80 +) +CATEGORY_RANGES['Lun'] = _non_ascii_upper # A translate mapping that will strip all control characters except \t and \n. # This incidentally has the effect of normalizing Windows \r\n line endings to From dd6a7b9765bb4502a559864f840c3287e8b90d21 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 3 Aug 2013 23:30:56 -0400 Subject: [PATCH 041/502] Keep decoding, even if the first decoding looks worse. Additional changes: - Refined the sequence_weirdness metric. - Added more inline documentation. --- ftfy/badness.py | 43 +++++++++++++++++++++++++++++++++++++------ ftfy/fixes.py | 29 +++++++++++++++++++---------- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 9a5cec78..73d65ed9 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -4,10 +4,42 @@ import re def _make_weirdness_regex(): + """ + Creates a list of regexes that match 'weird' character sequences. + The more matches there are, the weirder the text is. + """ groups = [] + + # Match lowercase letters that are followed by: + # - non-ASCII uppercase letters, such as accented letters + # - letters from a different script that doesn't have case + # - title-case ligatures groups.append(u'([{Ll}][{Lun}{Lo}{Lt}])'.format(**CATEGORY_RANGES)) - groups.append(u'([^{Lo}][{Mn}{Mc}])'.format(**CATEGORY_RANGES)) - exclusive_categories = ['Lm', 'Sk', 'Sm', 'Sc', 'No', 'So'] + + # Match diacritic marks, except when they modify a non-cased letter. + # + # You wouldn't put a diacritic mark on a digit or a space, for example. + # You might, of course, put diacritics on Latin letters, but in that case + # it's a lot more common to see the pre-combined version. Modifier + # characters are most often used in other scripts where the letters have + # category 'Lo'. + groups.append(u'([^{Lo}][{Mn}{Mc}{Me}])'.format(**CATEGORY_RANGES)) + + # Match C1 control characters, which are almost always the result of + # decoding Latin-1 that was meant to be Windows-1252. + groups.append(u'[\x80-\x9f]') + + # Match adjacent characters from any different pair of these categories: + # - Letter modifiers (Lm) + # - Spacing combining marks (Mc) + # - Enclosing marks (Me) + # - Nonspacing marks (Mn) + # - Miscellaneous numbers (No) + # - Symbol modifiers (Sk) + # - Mathematical symbols (Sm) + # - Currency symbols (Sc) + # - Other symbols (So) + exclusive_categories = ['Lm', 'Mc', 'Me', 'Mn', 'No', 'Sk', 'Sm', 'Sc', 'So'] for cat1 in exclusive_categories: others = exclusive_categories[:] others.remove(cat1) @@ -27,10 +59,9 @@ def _make_weirdness_regex(): def sequence_weirdness(text): """ - TODO: better documentation - - Determine how often a text has unexpected sequences of characters, - which might indicate an incorrect encoding. + Determine how often a text has unexpected characters or sequences of + characters. This metric is used to disambiguate when text should be + re-decoded or left as is. """ return len(WEIRDNESS_RE.findall(text)) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index a9a9e7aa..6c028145 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -6,6 +6,7 @@ import re import sys import warnings +import logging if sys.hexversion >= 0x03000000: from html import entities @@ -33,15 +34,18 @@ """ % sys.version_info[0] SURROGATES_WARNING_TEXT = """ -This text contains characters that this version of Python can't handle. +This text contains characters that this installation of Python can't +properly decode. You are using a "narrow" build of Python, which can only represent the first 2^16 characters in Unicode. There are characters called "astral" characters -above this range, including emoji and some rare Chinese characters. +above this range, including emoji (single-character emoticons) and some +Chinese characters. -Python will instead represent these as pairs of surrogate characters. Your -string will be the wrong length and will cause problems if you try to refer -to its individual characters. +Python will instead represent these as pairs of surrogate characters. +If you are just going to re-encode the string and output it, this is fine. +If you slice it, or refer to its length or individual characters, however, +you will get incorrect results. Unfortunately, this can't be fixed from within Python. You should download or compile a "wide" build, or upgrade to Python 3.3 or later. @@ -108,11 +112,17 @@ def fix_text_encoding(text): >>> print(fix_text_encoding('This text was never UTF-8 at all\x85')) This text was never UTF-8 at all… """ + best_version = text + best_cost = text_cost(text) while True: - origtext = text + prevtext = text text, _ = fix_text_and_explain(text) - if text == origtext: - return text + cost = text_cost(text) + if cost < best_cost: + best_cost = cost + best_version = text + if text == prevtext: + return best_version def fix_text_and_explain(text): if isinstance(text, bytes): @@ -156,8 +166,7 @@ def fix_text_and_explain(text): try: fixed = encoded_bytes.decode('utf-8') steps = [('sloppy_encode', encoding), ('decode', 'utf-8')] - if text_cost(fixed) < text_cost(text): - return fixed, steps + return fixed, steps except UnicodeDecodeError: possible_1byte_encodings.append(encoding) From c3ad8e2805d7b897a6df5ab647c8b5dd9234a2cd Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 3 Aug 2013 23:50:39 -0400 Subject: [PATCH 042/502] Convert lone CR to LF - fixes #5 --- ftfy/__init__.py | 10 ++++++++++ ftfy/chardata.py | 2 ++ ftfy/fixes.py | 27 ++++++++++++++++++--------- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index f769e265..6b6c4103 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -19,6 +19,7 @@ def fix_text(text, fix_encoding=True, normalization='NFKC', uncurl_quotes=True, + fix_line_breaks=True, remove_control_chars=True, fix_surrogate_encoding=True, remove_bom=True): @@ -50,6 +51,11 @@ def fix_text(text, periods, and the ligature 'fl' will be replaced with 'fl'. - If `uncurl_quotes` is True, replace various curly quotation marks with plain-ASCII straight quotes. + - If `fix_line_breaks` is true, convert all line breaks to Unix style + (CRLF and CR line breaks become LF line breaks). + - If `fix_control_characters` is true, remove all control characters + except the common useful ones: TAB, CR, LF, and FF. (CR characters + may have already been removed by the `fix_line_breaks` step.) - If `fix_surrogate_encoding` is true, replace surrogate encodings of "astral" Unicode characters with the actual characters... but only on Python builds ("wide" builds or Python >= 3.3) that can handle @@ -119,6 +125,7 @@ def fix_text(text, fix_encoding=fix_encoding, normalization=normalization, uncurl_quotes=uncurl_quotes, + fix_line_breaks=fix_line_breaks, remove_control_chars=remove_control_chars, fix_surrogate_encoding=fix_surrogate_encoding, remove_bom=remove_bom @@ -154,6 +161,7 @@ def fix_text_segment(text, fix_encoding=True, normalization='NFKC', uncurl_quotes=True, + fix_line_breaks=True, remove_control_chars=True, fix_surrogate_encoding=True, remove_bom=True): @@ -180,6 +188,8 @@ def fix_text_segment(text, text = unicodedata.normalize(normalization, text) if uncurl_quotes: text = fixes.uncurl_quotes(text) + if fix_line_breaks: + text = fixes.fix_line_breaks(text) if remove_control_chars: text = fixes.remove_control_chars(text) if remove_bom: diff --git a/ftfy/chardata.py b/ftfy/chardata.py index d0f0881f..020f3bd7 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -106,3 +106,5 @@ def _make_category_regex_ranges(): CONTROL_CHARS[ord('\t')] = '\t' CONTROL_CHARS[ord('\n')] = '\n' +CONTROL_CHARS[ord('\f')] = '\f' +CONTROL_CHARS[ord('\r')] = '\r' diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 6c028145..98e4490b 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -263,18 +263,27 @@ def uncurl_quotes(text): return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text)) -def remove_control_chars(text): +def fix_line_breaks(text): + r""" + Convert line breaks to Unix style. + + In particular, this replaces CRLF (\r\n) with LF (\n), then + additionally replaces CR (\r) with LF (\n). """ - Remove all control characters except for the important ones (line feed - and tab). + return text.replace('\r\n', '\n').replace('\r', '\n') + + +def remove_control_chars(text): + r""" + Remove all control characters except for the important ones. - This removes all characters from U+0000 to U+001F and U+0080 to U+009F, - but it of course leaves TAB (U+0009) and LF (U+000A) alone. + This removes characters from U+0000 to U+001F and U+0080 to U+009F, except + it leaves alone these characters that are commonly used for formatting: - Note that, for example, this will convert Windows-style CRLF line breaks - into Unix-style LF line breaks, because CR (U+000D) is a control - character that does nothing on its own. You are likely to want this - transformation anyway. + - TAB (\t, U+0009) + - LF (\n, U+000A) + - FF (\f, U+000C) + - CR (\r, U+000D) """ return text.translate(CONTROL_CHARS) From 3d467412ad89787e14e49e0084f48ee01ee7e0b7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 6 Aug 2013 02:58:31 -0400 Subject: [PATCH 043/502] more tricky real-world test cases --- ftfy/test_real_tweets.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ftfy/test_real_tweets.py b/ftfy/test_real_tweets.py index defa6a78..185b8d28 100644 --- a/ftfy/test_real_tweets.py +++ b/ftfy/test_real_tweets.py @@ -11,11 +11,20 @@ # "Deja dos heridos hundimiento de barco turístico en Acapulco."), ("✔ No problems", "✔ No problems"), ('4288×…', '4288×...'), + ('RETWEET SE VOCÊ…', 'RETWEET SE VOCÊ...'), + ('L’épisode 8 est trop fou ouahh', 'L’épisode 8 est trop fou ouahh'), ("РґРѕСЂРѕРіРµ Из-РїРѕРґ http://t.co/A0eJAMTuJ1 #футбол", "дороге Из-под http://t.co/A0eJAMTuJ1 #футбол"), ("\x84Handwerk bringt dich \xfcberall hin\x93: Von der YOU bis nach Monaco", '"Handwerk bringt dich überall hin": Von der YOU bis nach Monaco'), ("Hi guys 😍", "Hi guys 😍"), + ("@rakryanM hihi RT damnitstrue: ☺😘 http://t.co/DqSCy26POe", + "@rakryanM hihi RT damnitstrue: ☺😘 http://t.co/DqSCy26POe"), + ("Beta Haber: Hırsızı Büyü Korkuttu http://t.co/rMkt5yz7Si", + "Beta Haber: Hırsızı Büyü Korkuttu http://t.co/rMkt5yz7Si"), + ## Egad. Was this intentional? + # ("歮爠汥∽摥瑩•楴汴㵥ꢷ鯩⊆栠敲㵦⼢⽷湩敤⹸桰㽰楴汴㵥䔥┶㘹㠥┷㕅䄥╄㜹䔥┵䌸㤥┶㍅㠥┱ㄹ愦灭愻瑣潩㵮摥瑩•㸯㰊楬歮爠汥∽灡汰ⵥ潴捵⵨捩湯•牨晥∽⼯慪眮歩灩摥慩漮杲", + # 'nk rel="edit" title="\u7de8\u96c6" href="/w/index.php?title=%E6%96%87%E5%AD%97%E5%8C%96%E3%81%91&action=edit" />\n\n= 0x80 ) +_non_ascii_lower = ''.join( + ch for ch in CATEGORY_RANGES['Ll'] + if ord(ch) >= 0x80 +) CATEGORY_RANGES['Lun'] = _non_ascii_upper +CATEGORY_RANGES['Lln'] = _non_ascii_lower + # A translate mapping that will strip all control characters except \t and \n. # This incidentally has the effect of normalizing Windows \r\n line endings to diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 98e4490b..6f0148e2 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -116,8 +116,14 @@ def fix_text_encoding(text): best_cost = text_cost(text) while True: prevtext = text - text, _ = fix_text_and_explain(text) + text, plan = fix_text_and_explain(text) cost = text_cost(text) + + # Add a small penalty if we used a particularly obsolete encoding. + if ('sloppy_encode', 'macroman') in plan or\ + ('sloppy_encode', 'cp437') in plan: + cost += 1 + if cost < best_cost: best_cost = cost best_version = text From dc6dfa278a459aa1baa66e90f851b64003f6cf8d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 21 Aug 2013 15:16:41 -0400 Subject: [PATCH 046/502] another potentially ambiguous tweet --- ftfy/__init__.py | 15 ++++++--------- ftfy/test_real_tweets.py | 1 + 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 6b6c4103..d0ce04b8 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -11,8 +11,6 @@ import unicodedata -MAXLEN = 65536 - def fix_text(text, fix_entities=True, remove_terminal_escapes=True, @@ -22,7 +20,8 @@ def fix_text(text, fix_line_breaks=True, remove_control_chars=True, fix_surrogate_encoding=True, - remove_bom=True): + remove_bom=True, + max_line_length=2**20): """ Given Unicode text as input, make its representation consistent and possibly less broken, by applying these steps in order: @@ -101,15 +100,13 @@ def fix_text(text, out = [] pos = 0 while pos < len(text): - textbreak = text.find('\n', pos, pos + MAXLEN) + should_fix = True + textbreak = text.find('\n', pos, pos + max_line_length) if textbreak == -1: - if pos + MAXLEN >= len(text): + if pos + max_line_length >= len(text): textbreak = len(text) - 1 else: - textbreak = text.find(' ', pos, pos + MAXLEN) - if textbreak == -1: - # oh well. We might not be able to fix the 2**16th character. - textbreak = MAXLEN + should_fix = False substring = text[pos:pos + textbreak + 1] diff --git a/ftfy/test_real_tweets.py b/ftfy/test_real_tweets.py index ab1c4b0d..c8f27b1b 100644 --- a/ftfy/test_real_tweets.py +++ b/ftfy/test_real_tweets.py @@ -13,6 +13,7 @@ ('4288×…', '4288×...'), ('RETWEET SE VOCÊ…', 'RETWEET SE VOCÊ...'), ('PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…', 'PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ...'), + ('TEM QUE SEGUIR, SDV SÓ…', 'TEM QUE SEGUIR, SDV SÓ...'), ('Join ZZAJÉ’s Official Fan List and receive news, events, and more!', "Join ZZAJÉ's Official Fan List and receive news, events, and more!"), ('L’épisode 8 est trop fou ouahh', "L'épisode 8 est trop fou ouahh"), ("РґРѕСЂРѕРіРµ Из-РїРѕРґ http://t.co/A0eJAMTuJ1 #футбол", From 195487702e30ebae1f34bca2358e2c42be6b6d73 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 21 Aug 2013 15:21:14 -0400 Subject: [PATCH 047/502] temporary band-aid: make fix_file at least try utf-8 first --- ftfy/__init__.py | 7 ++++--- ftfy/cli.py | 4 +--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index d0ce04b8..58280156 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -144,9 +144,10 @@ def fix_file(file, normalization='NFKC'): entities = True for line in file: if isinstance(line, bytes): - raise UnicodeError("fix_file wants Unicode as input, so that " - "it knows what to fix.\n" - "Try this: codecs.open(file, encoding='utf-8')") + try: + line = line.decode('utf-8') + except UnicodeDecodeError: + line = line.decode('latin-1') if '<' in line and '>' in line: entities = False yield fix_text_segment(line, normalization, entities) diff --git a/ftfy/cli.py b/ftfy/cli.py index f2b970da..5881bd76 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -17,9 +17,7 @@ def main(): args = parser.parse_args() - # Why open in Latin-1? Because it at least won't make encoding problems - # worse, and we're about to make things better. - file = codecs.open(args.filename, encoding='latin-1') + file = open(args.filename) for line in fix_file(file): if ENCODE_STDOUT: sys.stdout.write(line.encode('utf-8')) From f6c75901e8507b890bbe6592166d5fd50989e331 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 22 Aug 2013 00:45:30 -0400 Subject: [PATCH 048/502] Massive (over 16x) speed improvement. Fixes #8. Now we pre-calculate all the classes of characters that we might handle differently, and use the unicode.translate() trick that v2 used. In addition, these classes of characters appear in a pre-made string that is loaded from a compressed data file. --- ftfy/badness.py | 67 ++++++++++++++---------- ftfy/char_classes.dat | Bin 0 -> 3566 bytes ftfy/chardata.py | 115 +++++++++++++++++++++++++++--------------- ftfy/fixes.py | 4 +- 4 files changed, 114 insertions(+), 72 deletions(-) create mode 100644 ftfy/char_classes.dat diff --git a/ftfy/badness.py b/ftfy/badness.py index 88096c6c..c1c160e2 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -1,8 +1,29 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from ftfy.chardata import CATEGORY_RANGES +from ftfy.chardata import chars_to_classes import re +# The following regex uses the mapping of character classes to ASCII +# characters defined in chardata.py: +# +# L = Latin capital letter +# l = Latin lowercase letter +# A = Non-latin capital or title-case letter +# a = Non-latin lowercase letter +# C = Non-cased letter (Lo) +# X = Control character (Cc) +# m = Letter modifier (Lm) +# M = Mark (Mc, Me, Mn) +# N = Miscellaneous numbers (No) +# 0 = Math symbol (Sm) +# 1 = Currency symbol (Sc) +# 2 = Symbol modifier (Sk) +# 3 = Other symbol (So) +# S = UTF-16 surrogate +# _ = Unassigned character +# = Whitespace +# o = Other + def _make_weirdness_regex(): """ Creates a list of regexes that match 'weird' character sequences. @@ -11,7 +32,7 @@ def _make_weirdness_regex(): groups = [] # Match lowercase letters that are followed by non-ASCII uppercase letters - groups.append(u'([{Ll}][{Lun}{Lt}])'.format(**CATEGORY_RANGES)) + groups.append('lA') # Match diacritic marks, except when they modify a non-cased letter. # @@ -20,43 +41,33 @@ def _make_weirdness_regex(): # it's a lot more common to see the pre-combined version. Modifier # characters are most often used in other scripts where the letters have # category 'Lo'. - groups.append(u'([^{Lo}][{Mn}{Mc}{Me}])'.format(**CATEGORY_RANGES)) + groups.append('[^C]M') # Match non-Latin characters adjacent to Latin characters. # # This is a simplification from ftfy version 2, which compared all # adjacent scripts. However, the ambiguities we need to resolve come from # encodings designed to represent Latin characters. - groups.append(u'([{latin}][{nonlatin}])'.format(**CATEGORY_RANGES)) - groups.append(u'([{nonlatin}][{latin}])'.format(**CATEGORY_RANGES)) + groups.append('[Ll][AaC]') + groups.append('[AaC][Ll]') # Match C1 control characters, which are almost always the result of # decoding Latin-1 that was meant to be Windows-1252. - groups.append(u'([\x80-\x9f])') + groups.append(u'X') # Match adjacent characters from any different pair of these categories: - # - Letter modifiers (Lm) - # - Spacing combining marks (Mc) - # - Enclosing marks (Me) - # - Nonspacing marks (Mn) - # - Miscellaneous numbers (No) - # - Symbol modifiers (Sk) - # - Mathematical symbols (Sm) - # - Currency symbols (Sc) - # - Other symbols (So) - exclusive_categories = ['Lm', 'Mc', 'Me', 'Mn', 'No', 'Sk', 'Sm', 'Sc', 'So'] + # - Modifier marks (M) + # - Letter modifiers (m) + # - Miscellaneous numbers (N) + # - Symbols (0123) + + exclusive_categories = 'MmN0123' for cat1 in exclusive_categories: - others = exclusive_categories[:] - others.remove(cat1) - cat1_range = CATEGORY_RANGES[cat1] - others_range = u''.join([CATEGORY_RANGES[other] for other in others]) - groups.append( - u'([{cat1_range}][{others_range}])'.format( - cat1_range=cat1_range, - others_range=others_range - ) - ) - regex = '|'.join(groups) + others_range = ''.join(c for c in exclusive_categories if c != cat1) + groups.append('{cat1}[{others_range}]'.format( + cat1=cat1, others_range=others_range + )) + regex = '|'.join('({0})'.format(group) for group in groups) return re.compile(regex) WEIRDNESS_RE = _make_weirdness_regex() @@ -68,7 +79,7 @@ def sequence_weirdness(text): characters. This metric is used to disambiguate when text should be re-decoded or left as is. """ - return len(WEIRDNESS_RE.findall(text)) + return len(WEIRDNESS_RE.findall(chars_to_classes(text))) def better_text(newtext, oldtext): diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat new file mode 100644 index 0000000000000000000000000000000000000000..1b5871b508bdca3fac5e493043d3da0cc073b66a GIT binary patch literal 3566 zcmeHIc~lbk7It2{PFks1X*o^nSfP|=Iw&uD<(T`Pno=5?nvx5W3(%{~#wD7pa0!{o zkQ5PjO96Z35^9=|k`gK+8Xztpq9E{QdFRZTdGpShf8RNC|M;Er-R0ch_uX^Pz4lnb zbxnEF^EaI_@}&(HhvrV?dCW}U6OO|4Rrt_@2Pf3b%S<(L*!%luwriek)T#qjO9%0* z%u`FuWn=dgX8P@weJ-c(_@b1@%J0lsxm~@dskt{nobb8j>A~Qoe2p{)jslh?DaB_s zUacMkyl{fl=- z%i>QD;JQ=nvTRv#2#x*wxF+skIr&g_ z?6r`sni8z*{qxPst}?>sz1AI91mLoVUHc@T9z;R+20b$3)fH;CIB{Rw`brimbcv4`ErpLu7v!u8HdA38Ban|OJAjwtS4w7xbxyqxf+&6bm7R20-%yi|(oPH4=4&8hT zSiGY-Ag=iKLJMu;8g&f=7~qjc;o1m0qk;!U8nOx5ilQ;25e#I%{I7+rvBS&6Q^JFoNv_)D6OUVXtvyAi|e@@6acaVu>cS^#Yg zp_-J92vO^mrLltC2k2%cM;5Wna-}FEDt33xD=~J>1Gs8^>LIn*O$ishURYQ9rfb1D z(fTkNp)x&$W+`2ApK`S!37(UsvUrMe6_`u3i5*C|Q1o8ZQwB$oaw(M>`4R9F#EKEd>D3e0It;abF*~9Eay>R+ zodq%fl7P2pXEaGI2cpC0;5k}ngR}O#D^}(9o@7}SReSdV@OfZd5S_`qc+$H|$|CZi6K`6M zEj`5@D-C~(Ik{v!G}K_qJ+Bc%5lD9;eG_EZ;k>JJS$_OZz~{vIYjB84!NO$u5k8QD zJ4w))t?qmMv9NaH+5Ff*@1B(RPfn;c$AP$pDF@#c{U#F%XDxSyJ>$<^4VxV|@txgQ z-$;BbIPnM>cbM;PyL7Nxb2T+oQ!yK}3)UYGE7yN4}n-#~or zcgJR;tR#NWvg}m0Voun}z3?f9WP(|1@8*g8XxkgpaU`en3*OhnX%uKn8Ovh|Flg_w z!`z@TeuGOQzS+gGZ|t{`H6`-)$VyXLWPL{o&kGlM>5}i8W!t%EjbM#F zho&fAK!21|#_j=;U=!ZO!A8946(H-&7&gN%Z>_{(Isb4yE4;*nk6+J~Xi07}XzZ(8 zf&}Hp;3#T5cyItGcbqFw&Jj0kA}Z(roId#kIJ$e-uf6-yzLoxMHHe!9fhBLS;1PNTUy%>>K8`A{-Zu@1 zhLo#jTRXS?Ah%(Qqkq|ND-Jp|{SoJ_oeQwMi?d*I2I7`N^t>W%O1Q!sTc1 zy9f1frvhS@cOX51{@Eg=F9iiM-5eiLBpAJx-aksDzJQ@e)E?FR7Hk|MoPI0hP2-H? zWB1I;QhYO5dL~s2Xym4w;5(rG!uKuMg`0{ez2ip+B-O>yslW{ca;-v)=o~=?KSB<` zlgQf8GLe0~-^VxOp@A4iql7%V18C0Pc?9(QTexopk6WIFpvFtD3ae`Nw==x>Ltj0? zANdB?nw35J!U0hP?{3Y&%;3r!*{vft>&10xqnI+s`ok0uipxS*VZAQG3C5OeM(K#v z_i|US9!717p?~*n3Rf8&hAy)4-hR^>+s}<-j z6?83!DoslgSK$L8c}Z3zWsSR-nVS_qAs=1v0YixBa;5`z^@QDRHao+*g9fY=OSm7p zW1lZl?xvY zcU(?Ayih7}Hp>)k>2heghwslRf>jGEglxeQo#Ue?i5)mVfV2fPDUz2WIbqcI(z^>L zRpwla#QdDW544Vjxj`F;9ypmXoAz34ysFQzsV=VbQnbx+Pc7mWwSltI-ys{FKt`@? zB!NK|apm%pA|c=8sU<}pKiH|54_hsSYBtgIO~aIukLFcTdI>Ewq~B<$>gFERjP#tP z;Q&_iKoS92CZ1gr+D-JKvKBdgejVWB77=E#lfu#^vJgW|ZL)ThC)FNRk8oklM44rk zmM6&#F1$)#y|OjEllGiGudj2Vn0x^LGJbaC2z+hfp#2$C}VthrMxHrTQby|fTQqncc++CL1XZtmrm!^7OkAsA<%yE0+U3drc ztHi(wD{`1_bULoyr409xF7>Oi?eDbRBltw|&1z*&OxveRBXLhMi7Mg6M8a#QY&56n z^btWZEq&#)pZbrb-{P?QxWn{}t^Z>7R$A3*@O-SOkD!}mFxsQXB41*9?B!l=iDqwi zn|cIGJ^?Bp*K7%CeZNvR>$*#MU;_thv-HpKG!M!psvd(NKpy z-!AH1iOWozyd6`7N&kugpU)K}hpWi~$k}DA=ElKGDxQ=GENWL~R2QSNI-a= 0x03000000: unichr = chr + xrange = range # These are encodings that map each byte to a particular character. # They are listed in order of frequency, so that more frequent @@ -62,51 +64,80 @@ def possible_encoding(text, encoding): return bool(ENCODING_REGEXES[encoding].match(text)) -# Make regular expressions out of categories of Unicode characters. -# Except we probably need broader ranges. -# -# Ll Lu: weirdness=1 -# Ll [Lo Lt]: weirdness=2 -# Lm Sk Sm Sc No So Po -def _make_category_regex_ranges(): - categories = defaultdict(list) - for codepoint in range(0x20, 0x10000): +# L = Latin capital letter +# l = Latin lowercase letter +# A = Non-latin capital or title-case letter +# a = Non-latin lowercase letter +# C = Non-cased letter (Lo) +# X = Control character (Cc) +# m = Letter modifier (Lm) +# M = Mark (Mc, Me, Mn) +# N = Miscellaneous numbers (No) +# 0 = Math symbol (Sm) +# 1 = Currency symbol (Sc) +# 2 = Symbol modifier (Sk) +# 3 = Other symbol (So) +# S = UTF-16 surrogate +# _ = Unassigned character +# = Whitespace +# o = Other + +def _make_char_class_letters(): + cclasses = [None] * 0x110000 + for codepoint in range(0x0, 0x110000): char = unichr(codepoint) category = unicodedata.category(char) - categories[category].append(char) - - # Find Latin vs. non-Latin letters - if category.startswith('L'): + + if category.startswith('L'): # letters if unicodedata.name(char).startswith('LATIN')\ and codepoint < 0x200: - categories['latin'].append(char) - else: - categories['nonlatin'].append(char) - - ranges = {} - for category in categories: - ranges[category] = (''.join(categories[category]) - .replace('\\', '\\\\') - .replace('[', r'\[') - .replace(']', r'\]') - .replace('^', r'\^') - .replace('-', r'\-')) - - return ranges -CATEGORY_RANGES = _make_category_regex_ranges() - -# Separate out non-ASCII uppercase characters -_non_ascii_upper = ''.join( - ch for ch in CATEGORY_RANGES['Lu'] - if ord(ch) >= 0x80 -) -_non_ascii_lower = ''.join( - ch for ch in CATEGORY_RANGES['Ll'] - if ord(ch) >= 0x80 -) -CATEGORY_RANGES['Lun'] = _non_ascii_upper -CATEGORY_RANGES['Lln'] = _non_ascii_lower - + if category == 'Lu': + cclasses[codepoint] = 'L' + else: + cclasses[codepoint] = 'l' + else: # non-Latin letter, or close enough + if category == 'Lu' or category == 'Lt': + cclasses[codepoint] = 'A' + elif category == 'Ll': + cclasses[codepoint] = 'a' + elif category == 'Lo': + cclasses[codepoint] = 'C' + elif category == 'Lm': + cclasses[codepoint] = 'm' + else: + raise ValueError('got some weird kind of letter') + elif category.startswith('M'): # marks + cclasses[codepoint] = 'M' + elif category == 'No': + cclasses[codepoint] = 'N' + elif category == 'Sm': + cclasses[codepoint] = '0' + elif category == 'Sc': + cclasses[codepoint] = '1' + elif category == 'Sk': + cclasses[codepoint] = '2' + elif category == 'So': + cclasses[codepoint] = '3' + elif category == 'Cn': + cclasses[codepoint] = '_' + elif category == 'Cc': + cclasses[codepoint] = 'X' + elif category == 'Cs': + cclasses[codepoint] = 'S' + elif category.startswith('Z'): + cclasses[codepoint] = ' ' + else: + cclasses[codepoint] = 'o' + + cclasses[9] = cclasses[10] = cclasses[12] = cclasses[13] = ' ' + out = open('char_classes.dat', 'wb') + out.write(zlib.compress(bytes(''.join(cclasses), 'ascii'))) + out.close() + +CHAR_CLASS_STRING = zlib.decompress(resource_string(__name__, 'char_classes.dat')).decode('ascii') + +def chars_to_classes(string): + return string.translate(CHAR_CLASS_STRING) # A translate mapping that will strip all control characters except \t and \n. # This incidentally has the effect of normalizing Windows \r\n line endings to diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 6f0148e2..487b8936 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals from ftfy.chardata import (possible_encoding, CHARMAPS, CHARMAP_ENCODINGS, - CONTROL_CHARS, CATEGORY_RANGES) + CONTROL_CHARS) from ftfy.badness import text_cost import re import sys @@ -294,7 +294,7 @@ def remove_control_chars(text): return text.translate(CONTROL_CHARS) -SURROGATES_RE = re.compile('[{0}]'.format(CATEGORY_RANGES['Cs'])) +SURROGATES_RE = re.compile(u'[\ud800-\udfff]') def fix_surrogate_encoding(text): # Python has no functions that manipulate surrogate pairs # directly. However, if they can be fixed at all, they will From e786f26c78ed0c842b3321a330076e5de1b8f0af Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 23 Aug 2013 18:10:26 -0400 Subject: [PATCH 049/502] Add a code path for decoding CESU-8 and its ilk. Fixes #3. --- ftfy/__init__.py | 12 +----- ftfy/chardata.py | 5 +-- ftfy/fixes.py | 98 +++++++++++++++++++++++++++++------------------- 3 files changed, 61 insertions(+), 54 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 58280156..898a5c66 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -11,7 +11,7 @@ import unicodedata -def fix_text(text, +def fix_text(text, fix_entities=True, remove_terminal_escapes=True, fix_encoding=True, @@ -19,7 +19,6 @@ def fix_text(text, uncurl_quotes=True, fix_line_breaks=True, remove_control_chars=True, - fix_surrogate_encoding=True, remove_bom=True, max_line_length=2**20): """ @@ -55,11 +54,6 @@ def fix_text(text, - If `fix_control_characters` is true, remove all control characters except the common useful ones: TAB, CR, LF, and FF. (CR characters may have already been removed by the `fix_line_breaks` step.) - - If `fix_surrogate_encoding` is true, replace surrogate encodings of - "astral" Unicode characters with the actual characters... but only - on Python builds ("wide" builds or Python >= 3.3) that can handle - astral characters. On "narrow" builds, it'll keep using surrogate - characters and give a warning. - If `remove_bom` is True, remove the Byte-Order Mark if it exists. - If anything was changed, repeat all the steps, so that the function is idempotent. "&amp;" will become "&", for example, not "&". @@ -124,7 +118,6 @@ def fix_text(text, uncurl_quotes=uncurl_quotes, fix_line_breaks=fix_line_breaks, remove_control_chars=remove_control_chars, - fix_surrogate_encoding=fix_surrogate_encoding, remove_bom=remove_bom ) ) @@ -161,7 +154,6 @@ def fix_text_segment(text, uncurl_quotes=True, fix_line_breaks=True, remove_control_chars=True, - fix_surrogate_encoding=True, remove_bom=True): """ Apply fixes to text in a single chunk. This could be a line of text @@ -192,8 +184,6 @@ def fix_text_segment(text, text = fixes.remove_control_chars(text) if remove_bom: text = fixes.remove_bom(text) - if fix_surrogate_encoding: - text = fixes.fix_surrogate_encoding(text) if text == origtext: return text diff --git a/ftfy/chardata.py b/ftfy/chardata.py index b50b197b..082afb3a 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -6,10 +6,8 @@ import re import zlib from pkg_resources import resource_string +from ftfy.compatibility import htmlentitydefs, unichr, xrange -if sys.hexversion >= 0x03000000: - unichr = chr - xrange = range # These are encodings that map each byte to a particular character. # They are listed in order of frequency, so that more frequent @@ -28,7 +26,6 @@ ] CHARMAPS = {} - def _make_encoding_regexes(): encoding_regular_expressions = {} diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 487b8936..2e8757c1 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -7,13 +7,7 @@ import sys import warnings import logging - -if sys.hexversion >= 0x03000000: - from html import entities - htmlentitydefs = entities - unichr = chr -else: - import htmlentitydefs +from ftfy.compatibility import htmlentitydefs, unichr, bytes_to_ints BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. @@ -33,23 +27,6 @@ http://docs.python.org/%d/howto/unicode.html """ % sys.version_info[0] -SURROGATES_WARNING_TEXT = """ -This text contains characters that this installation of Python can't -properly decode. - -You are using a "narrow" build of Python, which can only represent the first -2^16 characters in Unicode. There are characters called "astral" characters -above this range, including emoji (single-character emoticons) and some -Chinese characters. - -Python will instead represent these as pairs of surrogate characters. -If you are just going to re-encode the string and output it, this is fine. -If you slice it, or refer to its length or individual characters, however, -you will get incorrect results. - -Unfortunately, this can't be fixed from within Python. You should download -or compile a "wide" build, or upgrade to Python 3.3 or later. -""" def fix_text_encoding(text): """ @@ -168,9 +145,10 @@ def fix_text_and_explain(text): sorta_encoded_text = text.translate(CHARMAPS[encoding]) encoded_bytes = sorta_encoded_text.encode('latin-1') - # Now, find out if it's UTF-8. + # Now, find out if it's UTF-8. ...Or at least something that + # resembles UTF-8. See fix_java_encoding for more details. try: - fixed = encoded_bytes.decode('utf-8') + fixed = fix_java_encoding(encoded_bytes).decode('utf-8') steps = [('sloppy_encode', encoding), ('decode', 'utf-8')] return fixed, steps except UnicodeDecodeError: @@ -294,20 +272,62 @@ def remove_control_chars(text): return text.translate(CONTROL_CHARS) -SURROGATES_RE = re.compile(u'[\ud800-\udfff]') -def fix_surrogate_encoding(text): - # Python has no functions that manipulate surrogate pairs - # directly. However, if they can be fixed at all, they will - # be fixed by round-tripping to UTF-8 encoding. +CESU8_RE = re.compile(b'\xed[\xa0-\xaf][\x80-\xbf]\xed[\xb0-\xbf][\x80-\xbf]') +def fix_java_encoding(bytestring): + """ + Convert a bytestring that might contain "Java UTF8" into valid UTF-8. + + There are two things that Java is known to do with its "UTF8" encoder + that are incompatible with UTF-8. (If you happen to be writing Java + code, apparently the standards-compliant encoder is named "AS32UTF8".) + + - Every UTF-16 character is separately encoded as UTF-8. This is wrong + when the UTF-16 string contains surrogates; the character they actually + represent should have been encoded as UTF-8 instead. Unicode calls this + "CESU-8", the Compatibility Encoding Scheme for Unicode. Python 2 will + decode it as if it's UTF-8, but Python 3 refuses to. + + - The null codepoint, U+0000, is encoded as 0xc0 0x80, which avoids + outputting a null byte by breaking the UTF shortest-form rule. + Unicode does not even deign to give this scheme a name, and no version + of Python will decode it. + """ + assert isinstance(bytestring, bytes) + # Replace the sloppy encoding of U+0000 with the correct one. + bytestring = bytestring.replace(b'\xc0\x80', b'\x00') + + # When we have improperly encoded surrogates, we can still see the + # bits that they were meant to represent. + # + # The surrogates were meant to encode a 20-bit number, to which we + # add 0x10000 to get a codepoint. That 20-bit number now appears in + # this form: + # + # 11101101 1010abcd 10efghij 11101101 1011klmn 10opqrst # - # On the other hand, on a narrow build of Python, this can't possibly - # work. The fixed text simply cannot be represented by the unicode - # type in narrow Python. In that case, we'll let it do its thing with - # surrogates, but also warn the user. - if sys.maxunicode < 0x10000 and SURROGATES_RE.search(text): - warnings.warn(SURROGATES_WARNING_TEXT, UnicodeWarning) - - return text.encode('utf-8').decode('utf-8') + # The CESU8_RE above matches byte sequences of this form. Then we need + # to extract the bits and assemble a codepoint number from them. + match = CESU8_RE.search(bytestring) + fixed_pieces = [] + while match: + pos = match.start() + cesu8_sequence = bytes_to_ints(bytestring[pos:pos + 6]) + assert cesu8_sequence[0] == cesu8_sequence[3] == 0xed + codepoint = ( + ((cesu8_sequence[1] & 0x0f) << 16) + + ((cesu8_sequence[2] & 0x3f) << 10) + + ((cesu8_sequence[4] & 0x0f) << 6) + + (cesu8_sequence[5] & 0x3f) + + 0x10000 + ) + # For the reason why this will work on all Python builds, see + # compatibility.py. + new_bytes = unichr(codepoint).encode('utf-8') + fixed_pieces.append(bytestring[:pos] + new_bytes) + bytestring = bytestring[pos + 6:] + match = CESU8_RE.match(bytestring) + + return b''.join(fixed_pieces) + bytestring def remove_bom(text): From e7b65deffc34838e90daef6cd8fdd142e0bb6ea6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 03:18:13 -0400 Subject: [PATCH 050/502] Build the data file in a separate script --- ftfy/build_data.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++ ftfy/chardata.py | 70 ------------------------------------- 2 files changed, 86 insertions(+), 70 deletions(-) create mode 100644 ftfy/build_data.py diff --git a/ftfy/build_data.py b/ftfy/build_data.py new file mode 100644 index 00000000..1a29e399 --- /dev/null +++ b/ftfy/build_data.py @@ -0,0 +1,86 @@ +""" +A script to make the char_classes.dat file. + +This never needs to run in normal usage. It needs to be run if the character +classes we care about change, or if a new version of Python supports a new +Unicode standard and we want it to affect our string decoding. + +The file that we generate is based on Unicode 6.1, as supported by Python 3.3. +You can certainly use it in earlier versions. This simply makes sure that we +get consistent results from running ftfy on different versions of Python. + +The file will be written to the current directory. +""" +# L = Latin capital letter +# l = Latin lowercase letter +# A = Non-latin capital or title-case letter +# a = Non-latin lowercase letter +# C = Non-cased letter (Lo) +# X = Control character (Cc) +# m = Letter modifier (Lm) +# M = Mark (Mc, Me, Mn) +# N = Miscellaneous numbers (No) +# 0 = Math symbol (Sm) +# 1 = Currency symbol (Sc) +# 2 = Symbol modifier (Sk) +# 3 = Other symbol (So) +# S = UTF-16 surrogate +# _ = Unassigned character +# = Whitespace +# o = Other + +def make_char_data_file(): + cclasses = [None] * 0x110000 + for codepoint in range(0x0, 0x110000): + char = unichr(codepoint) + category = unicodedata.category(char) + + if category.startswith('L'): # letters + if unicodedata.name(char).startswith('LATIN')\ + and codepoint < 0x200: + if category == 'Lu': + cclasses[codepoint] = 'L' + else: + cclasses[codepoint] = 'l' + else: # non-Latin letter, or close enough + if category == 'Lu' or category == 'Lt': + cclasses[codepoint] = 'A' + elif category == 'Ll': + cclasses[codepoint] = 'a' + elif category == 'Lo': + cclasses[codepoint] = 'C' + elif category == 'Lm': + cclasses[codepoint] = 'm' + else: + raise ValueError('got some weird kind of letter') + elif category.startswith('M'): # marks + cclasses[codepoint] = 'M' + elif category == 'No': + cclasses[codepoint] = 'N' + elif category == 'Sm': + cclasses[codepoint] = '0' + elif category == 'Sc': + cclasses[codepoint] = '1' + elif category == 'Sk': + cclasses[codepoint] = '2' + elif category == 'So': + cclasses[codepoint] = '3' + elif category == 'Cn': + cclasses[codepoint] = '_' + elif category == 'Cc': + cclasses[codepoint] = 'X' + elif category == 'Cs': + cclasses[codepoint] = 'S' + elif category.startswith('Z'): + cclasses[codepoint] = ' ' + else: + cclasses[codepoint] = 'o' + + cclasses[9] = cclasses[10] = cclasses[12] = cclasses[13] = ' ' + out = open('char_classes.dat', 'wb') + out.write(zlib.compress(bytes(''.join(cclasses), 'ascii'))) + out.close() + +if __name__ == '__main__': + make_char_data_file() + diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 082afb3a..7a4ff792 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -61,76 +61,6 @@ def possible_encoding(text, encoding): return bool(ENCODING_REGEXES[encoding].match(text)) -# L = Latin capital letter -# l = Latin lowercase letter -# A = Non-latin capital or title-case letter -# a = Non-latin lowercase letter -# C = Non-cased letter (Lo) -# X = Control character (Cc) -# m = Letter modifier (Lm) -# M = Mark (Mc, Me, Mn) -# N = Miscellaneous numbers (No) -# 0 = Math symbol (Sm) -# 1 = Currency symbol (Sc) -# 2 = Symbol modifier (Sk) -# 3 = Other symbol (So) -# S = UTF-16 surrogate -# _ = Unassigned character -# = Whitespace -# o = Other - -def _make_char_class_letters(): - cclasses = [None] * 0x110000 - for codepoint in range(0x0, 0x110000): - char = unichr(codepoint) - category = unicodedata.category(char) - - if category.startswith('L'): # letters - if unicodedata.name(char).startswith('LATIN')\ - and codepoint < 0x200: - if category == 'Lu': - cclasses[codepoint] = 'L' - else: - cclasses[codepoint] = 'l' - else: # non-Latin letter, or close enough - if category == 'Lu' or category == 'Lt': - cclasses[codepoint] = 'A' - elif category == 'Ll': - cclasses[codepoint] = 'a' - elif category == 'Lo': - cclasses[codepoint] = 'C' - elif category == 'Lm': - cclasses[codepoint] = 'm' - else: - raise ValueError('got some weird kind of letter') - elif category.startswith('M'): # marks - cclasses[codepoint] = 'M' - elif category == 'No': - cclasses[codepoint] = 'N' - elif category == 'Sm': - cclasses[codepoint] = '0' - elif category == 'Sc': - cclasses[codepoint] = '1' - elif category == 'Sk': - cclasses[codepoint] = '2' - elif category == 'So': - cclasses[codepoint] = '3' - elif category == 'Cn': - cclasses[codepoint] = '_' - elif category == 'Cc': - cclasses[codepoint] = 'X' - elif category == 'Cs': - cclasses[codepoint] = 'S' - elif category.startswith('Z'): - cclasses[codepoint] = ' ' - else: - cclasses[codepoint] = 'o' - - cclasses[9] = cclasses[10] = cclasses[12] = cclasses[13] = ' ' - out = open('char_classes.dat', 'wb') - out.write(zlib.compress(bytes(''.join(cclasses), 'ascii'))) - out.close() - CHAR_CLASS_STRING = zlib.decompress(resource_string(__name__, 'char_classes.dat')).decode('ascii') def chars_to_classes(string): From 5e99dfa94227f6b6ab1d1e7d0f73f918ce8e0998 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 03:18:37 -0400 Subject: [PATCH 051/502] compatibility.py puts Python version checks in one place It also distinguishes narrow vs. wide builds as a separate issue from Python 2 vs. Python 3, and provides a workaround for unichr() on narrow builds. --- ftfy/compatibility.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 ftfy/compatibility.py diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py new file mode 100644 index 00000000..f154a56a --- /dev/null +++ b/ftfy/compatibility.py @@ -0,0 +1,38 @@ +import sys + +if sys.hexversion >= 0x03000000: + from html import entities + htmlentitydefs = entities + unichr = chr + xrange = range + PYTHON3 = True +else: + import htmlentitydefs + unichr = unichr + xrange = xrange + PYTHON3 = False + + +def _narrow_unichr_workaround(codepoint): + """ + A replacement for unichr() on narrow builds of Python. This will get + us the narrow representation of an astral character, which will be + a string of length two, containing two UTF-16 surrogates. + """ + escaped = b'\\U%08x' % codepoint + return escaped.decode('unicode-escape') + +if sys.maxunicode < 0x10000: + unichr = _narrow_unichr_workaround + + +def bytes_to_ints(bytestring): + """ + No matter what version of Python this is, make a sequence of integers from + a bytestring. On Python 3, this is easy, because a 'bytes' object _is_ a + sequence of integers. + """ + if PYTHON3: + return bytestring + else: + return [ord(b) for b in bytestring] From a1b77dc55be50b7880fcb5e978246af1dcc7513b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 03:19:45 -0400 Subject: [PATCH 052/502] Fix logic that finds segments of text to decode and join together. Fixes #6, by making it utterly predictable what will happen to a sequence of greater than 2**16 characters. It will go through all the fixes except the one that figures out if it's in the wrong encoding, because that's the complicated one that needs more than linear time. This is also a fix to bugs that hadn't been discovered yet, in that it would simply break up text into lines at the wrong locations. --- ftfy/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 898a5c66..e4e286ba 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -20,7 +20,7 @@ def fix_text(text, fix_line_breaks=True, remove_control_chars=True, remove_bom=True, - max_line_length=2**20): + max_decode_length=2**16): """ Given Unicode text as input, make its representation consistent and possibly less broken, by applying these steps in order: @@ -94,15 +94,14 @@ def fix_text(text, out = [] pos = 0 while pos < len(text): - should_fix = True - textbreak = text.find('\n', pos, pos + max_line_length) - if textbreak == -1: - if pos + max_line_length >= len(text): - textbreak = len(text) - 1 - else: - should_fix = False + textbreak = text.find('\n', pos) + 1 + fix_encoding_this_time = fix_encoding + if textbreak == 0: + textbreak = len(text) + if (textbreak - pos) > max_decode_length: + fix_encoding_this_time = False - substring = text[pos:pos + textbreak + 1] + substring = text[pos:textbreak] if '<' in substring and '>' in substring: # we see angle brackets together; this could be HTML @@ -113,7 +112,7 @@ def fix_text(text, substring, fix_entities=fix_entities, remove_terminal_escapes=remove_terminal_escapes, - fix_encoding=fix_encoding, + fix_encoding=fix_encoding_this_time, normalization=normalization, uncurl_quotes=uncurl_quotes, fix_line_breaks=fix_line_breaks, @@ -121,7 +120,7 @@ def fix_text(text, remove_bom=remove_bom ) ) - pos += textbreak + 1 + pos = textbreak return ''.join(out) From ea920f4d5cd6641863a415187f878238c3ff1f58 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 03:42:25 -0400 Subject: [PATCH 053/502] Fix regression: we want to be able to recompose diacritics If we get the encoding of a string in NFD form, it very well may have a floating diacritical mark next to a Latin letter, so fix the assumption that we only cared about other kinds of letters. --- ftfy/badness.py | 8 ++------ ftfy/fixes.py | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index c1c160e2..23fa9bf5 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -34,14 +34,10 @@ def _make_weirdness_regex(): # Match lowercase letters that are followed by non-ASCII uppercase letters groups.append('lA') - # Match diacritic marks, except when they modify a non-cased letter. + # Match diacritic marks, except when they modify a letter. # # You wouldn't put a diacritic mark on a digit or a space, for example. - # You might, of course, put diacritics on Latin letters, but in that case - # it's a lot more common to see the pre-combined version. Modifier - # characters are most often used in other scripts where the letters have - # category 'Lo'. - groups.append('[^C]M') + groups.append('[^LlAaC]M') # Match non-Latin characters adjacent to Latin characters. # diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 2e8757c1..8fac499b 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -143,12 +143,12 @@ def fix_text_and_explain(text): # points up to 0xff. This can then be converted into the intended # bytes by encoding it as Latin-1. sorta_encoded_text = text.translate(CHARMAPS[encoding]) - encoded_bytes = sorta_encoded_text.encode('latin-1') + encoded_bytes = fix_java_encoding(sorta_encoded_text.encode('latin-1')) # Now, find out if it's UTF-8. ...Or at least something that # resembles UTF-8. See fix_java_encoding for more details. try: - fixed = fix_java_encoding(encoded_bytes).decode('utf-8') + fixed = encoded_bytes.decode('utf-8') steps = [('sloppy_encode', encoding), ('decode', 'utf-8')] return fixed, steps except UnicodeDecodeError: From 30a3ec4ef2b3b2cef91256ad3cade130ffa5c08f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 04:16:22 -0400 Subject: [PATCH 054/502] Marks on Latin letters can still be weird if they don't form an existing character. --- ftfy/badness.py | 15 +++++++++++---- ftfy/test_unicode.py | 12 ++++++++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 23fa9bf5..f1824834 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from ftfy.chardata import chars_to_classes import re +import unicodedata # The following regex uses the mapping of character classes to ASCII # characters defined in chardata.py: @@ -34,10 +35,15 @@ def _make_weirdness_regex(): # Match lowercase letters that are followed by non-ASCII uppercase letters groups.append('lA') - # Match diacritic marks, except when they modify a letter. + # Match diacritical marks, except when they modify a non-cased letter or + # another mark. # - # You wouldn't put a diacritic mark on a digit or a space, for example. - groups.append('[^LlAaC]M') + # You wouldn't put a diacritical mark on a digit or a space, for example. + # You might put it on a Latin letter, but in that case there will almost + # always be a pre-composed version, and we normalize to pre-composed + # versions first. The cases that can't be pre-composed tend to be in + # large scripts without case, which are in class C. + groups.append('[^CM]M') # Match non-Latin characters adjacent to Latin characters. # @@ -75,7 +81,8 @@ def sequence_weirdness(text): characters. This metric is used to disambiguate when text should be re-decoded or left as is. """ - return len(WEIRDNESS_RE.findall(chars_to_classes(text))) + text2 = unicodedata.normalize('NFC', text) + return len(WEIRDNESS_RE.findall(chars_to_classes(text2))) def better_text(newtext, oldtext): diff --git a/ftfy/test_unicode.py b/ftfy/test_unicode.py index 5c7b141d..eb1f3c66 100644 --- a/ftfy/test_unicode.py +++ b/ftfy/test_unicode.py @@ -7,6 +7,13 @@ if sys.hexversion >= 0x03000000: unichr = chr +def char_names(text): + """ + Show the names of the characters involved. Helpful for debugging when + characters themselves are not visually distinguishable. + """ + return [unicodedata.name(c) for c in text] + # Most single-character strings which have been misencoded should be restored. def test_all_bmp_characters(): for index in range(0xa0, 0xfffd): @@ -15,8 +22,8 @@ def test_all_bmp_characters(): if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): garble = char.encode('utf-8').decode('latin-1') garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') - eq_(fix_text_encoding(garble), char) - eq_(fix_text_encoding(garble2), char) + eq_(char_names(fix_text_encoding(garble)), char_names(char)) + eq_(char_names(fix_text_encoding(garble2)), char_names(char)) phrases = [ u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", @@ -35,4 +42,5 @@ def test_valid_phrases(): def check_phrase(text): eq_(fix_text_encoding(text), text) + eq_(fix_text_encoding(text.encode('utf-8').decode('latin-1')), text) From 58ff625f7f2220d7d92aa3603eb56759bac315f3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 15:13:34 -0400 Subject: [PATCH 055/502] Don't remove C1 control characters that we can't fix. They were probably meant to be *some* sort of printable character, probably in Windows-1252 or MacRoman, but we just failed to decode it. Usually this happens due to mixed encodings. Leaving the characters alone at least avoids losing information. --- ftfy/__init__.py | 2 +- ftfy/chardata.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index e4e286ba..7c01edc7 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -51,7 +51,7 @@ def fix_text(text, plain-ASCII straight quotes. - If `fix_line_breaks` is true, convert all line breaks to Unix style (CRLF and CR line breaks become LF line breaks). - - If `fix_control_characters` is true, remove all control characters + - If `fix_control_characters` is true, remove all C0 control characters except the common useful ones: TAB, CR, LF, and FF. (CR characters may have already been removed by the `fix_line_breaks` step.) - If `remove_bom` is True, remove the Byte-Order Mark if it exists. diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 7a4ff792..1d2a6a37 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import unicodedata -import sys import re import zlib from pkg_resources import resource_string -from ftfy.compatibility import htmlentitydefs, unichr, xrange +from ftfy.compatibility import unichr # These are encodings that map each byte to a particular character. @@ -66,13 +64,12 @@ def possible_encoding(text, encoding): def chars_to_classes(string): return string.translate(CHAR_CLASS_STRING) -# A translate mapping that will strip all control characters except \t and \n. -# This incidentally has the effect of normalizing Windows \r\n line endings to -# Unix \n line endings. +# A translate mapping that will strip all C0 control characters except +# \t and \n. This incidentally has the effect of normalizing Windows \r\n +# line endings to Unix \n line endings. CONTROL_CHARS = {} -for i in range(256): - if unicodedata.category(unichr(i)) == 'Cc': - CONTROL_CHARS[i] = None +for i in range(32): + CONTROL_CHARS[i] = None CONTROL_CHARS[ord('\t')] = '\t' CONTROL_CHARS[ord('\n')] = '\n' From 5aedecb586a9cae9fa428b737fb216b42b9b74d9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 15:42:51 -0400 Subject: [PATCH 056/502] Tabs to spaces. I must have a word with my text editor. --- ftfy/compatibility.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index f154a56a..16a06ac1 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -23,16 +23,16 @@ def _narrow_unichr_workaround(codepoint): return escaped.decode('unicode-escape') if sys.maxunicode < 0x10000: - unichr = _narrow_unichr_workaround + unichr = _narrow_unichr_workaround def bytes_to_ints(bytestring): - """ - No matter what version of Python this is, make a sequence of integers from - a bytestring. On Python 3, this is easy, because a 'bytes' object _is_ a - sequence of integers. - """ - if PYTHON3: - return bytestring - else: - return [ord(b) for b in bytestring] + """ + No matter what version of Python this is, make a sequence of integers from + a bytestring. On Python 3, this is easy, because a 'bytes' object _is_ a + sequence of integers. + """ + if PYTHON3: + return bytestring + else: + return [ord(b) for b in bytestring] From ce7cf7f2b6efcf6c0d2d8ecfcc5c5ea62960dcb6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 15:58:59 -0400 Subject: [PATCH 057/502] Clean up, and fix some backslashes in regular expressions. --- ftfy/fixes.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 8fac499b..20009d49 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -138,15 +138,21 @@ def fix_text_and_explain(text): # 0x81 in Windows-1252 is an error. # # So what we do here is we use the .translate method of Unicode - # strings. Using it with the character maps we have computed will + # strings. Using it with the character maps we have computed will # give us back a Unicode string using only code # points up to 0xff. This can then be converted into the intended # bytes by encoding it as Latin-1. sorta_encoded_text = text.translate(CHARMAPS[encoding]) - encoded_bytes = fix_java_encoding(sorta_encoded_text.encode('latin-1')) - - # Now, find out if it's UTF-8. ...Or at least something that - # resembles UTF-8. See fix_java_encoding for more details. + + # When we get the bytes, run them through fix_java_encoding, + # because we can only reliably do that at the byte level. (See + # its documentation for details.) + encoded_bytes = fix_java_encoding( + sorta_encoded_text.encode('latin-1') + ) + + # Now, find out if it's UTF-8. Otherwise, remember the encoding + # for later. try: fixed = encoded_bytes.decode('utf-8') steps = [('sloppy_encode', encoding), ('decode', 'utf-8')] @@ -195,7 +201,7 @@ def fix_text_and_explain(text): return text, [('give up', None)] -HTML_ENTITY_RE = re.compile("&#?\w+;") +HTML_ENTITY_RE = re.compile(r"&#?\w+;") def unescape_html(text): """ Decode all three types of HTML entities/character references. @@ -226,7 +232,7 @@ def fixup(m): return HTML_ENTITY_RE.sub(fixup, text) -ANSI_RE = re.compile('\033\[((?:\d|;)*)([a-zA-Z])') +ANSI_RE = re.compile('\033\\[((?:\\d|;)*)([a-zA-Z])') def remove_terminal_escapes(text): """ Strip out "ANSI" terminal escape sequences, such as those that produce From c7a32678e056ad239dc1abcd652ed18a09fe1ee3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 16:05:50 -0400 Subject: [PATCH 058/502] Formatting fixes and docstrings --- ftfy/__init__.py | 16 ++++++++++--- ftfy/badness.py | 21 ++++++++++++++-- ftfy/chardata.py | 62 +++++++++++++++++++++++++++++++++--------------- ftfy/fixes.py | 37 +++++++++++++++++++++-------- 4 files changed, 102 insertions(+), 34 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 7c01edc7..b2a2f36a 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -8,7 +8,9 @@ from __future__ import unicode_literals from ftfy import fixes +from ftfy.fixes import fix_text_encoding import unicodedata +import warnings def fix_text(text, @@ -145,7 +147,7 @@ def fix_file(file, normalization='NFKC'): yield fix_text_segment(line, normalization, entities) -def fix_text_segment(text, +def fix_text_segment(text, fix_entities=True, remove_terminal_escapes=True, fix_encoding=True, @@ -187,5 +189,13 @@ def fix_text_segment(text, return text -fix_text_encoding = fixes.fix_text_encoding -fix_bad_encoding = fix_text_encoding +def fix_bad_encoding(text): + """ + Kept for compatibility with previous versions of ftfy. + """ + warnings.warn( + 'fix_bad_encoding is now known as fix_text_encoding', + DeprecationWarning + ) + return fix_text_encoding(text) + diff --git a/ftfy/badness.py b/ftfy/badness.py index f1824834..34261ed0 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -1,11 +1,16 @@ # -*- coding: utf-8 -*- +""" +Heuristics to determine whether re-encoding text is actually making it +more reasonable. +""" + from __future__ import unicode_literals from ftfy.chardata import chars_to_classes import re import unicodedata # The following regex uses the mapping of character classes to ASCII -# characters defined in chardata.py: +# characters defined in chardata.py and build_data.py: # # L = Latin capital letter # l = Latin lowercase letter @@ -62,7 +67,7 @@ def _make_weirdness_regex(): # - Letter modifiers (m) # - Miscellaneous numbers (N) # - Symbols (0123) - + exclusive_categories = 'MmN0123' for cat1 in exclusive_categories: others_range = ''.join(c for c in exclusive_categories if c != cat1) @@ -80,14 +85,26 @@ def sequence_weirdness(text): Determine how often a text has unexpected characters or sequences of characters. This metric is used to disambiguate when text should be re-decoded or left as is. + + We start by normalizing text in NFC form, so that penalties for + diacritical marks don't apply to characters that know what to do with + them. """ text2 = unicodedata.normalize('NFC', text) return len(WEIRDNESS_RE.findall(chars_to_classes(text2))) def better_text(newtext, oldtext): + """ + Is newtext better than oldtext? + """ return text_cost(newtext) < text_cost(oldtext) def text_cost(text): + """ + An overall cost function for text. All else being equal, shorter strings + are better. + """ return sequence_weirdness(text) * 2 + len(text) + diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 1d2a6a37..50f85515 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals +""" +This file contains the gritty details about characters and the encodings that +use them. +""" +from __future__ import unicode_literals import re import zlib from pkg_resources import resource_string @@ -17,16 +21,31 @@ 'cp437', 'windows-1251', ] -CONFUSABLE_1BYTE_ENCODINGS = [ - 'windows-1252', - 'macroman', - 'cp437' -] -CHARMAPS = {} -def _make_encoding_regexes(): - encoding_regular_expressions = {} +def _build_charmaps(): + """ + CHARMAPS contains mappings from bytes to characters, for each single-byte + encoding we know about. + + We don't use Python's decoders here because they're too strict. Many + non-Python programs will leave mysterious bytes alone instead of raising + an error or removing them. For example, Python will not decode 0x81 in + Windows-1252 because it doesn't map to anything. Other systems will decode + it to U+0081, which actually makes no sense because that's a meaningless + control character from Latin-1, but I guess at least it preserves some + information that ftfy can take advantage of. + + So that's what we do. When other systems decode 0x81 as U+0081, we match + their behavior in case it helps us get reasonable text. + + Meanwhile, ENCODING_REGEXES contain reasonably fast ways to detect if we + could represent a given string in a given encoding. The simplest one is + the 'ascii' detector, which of course just determines if all characters + are between U+0000 and U+007F. + """ + charmaps = {} + encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')} for encoding in CHARMAP_ENCODINGS: charmap = {} for codepoint in range(0, 0x80): @@ -40,16 +59,14 @@ def _make_encoding_regexes(): decoded_char = char charmap[ord(decoded_char)] = char - charlist = [unichr(codept) for codept in sorted(charmap.keys()) if codept >= 0x80] + charlist = [unichr(codept) for codept in sorted(charmap.keys()) + if codept >= 0x80] regex = '^[\x00-\x7f{}]*$'.format(''.join(charlist)) - CHARMAPS[encoding] = charmap - encoding_regular_expressions[encoding] = re.compile(regex) - - encoding_regular_expressions['ascii'] = re.compile('^[\x00-\x7f]*$') - return encoding_regular_expressions - -ENCODING_REGEXES = _make_encoding_regexes() + charmaps[encoding] = charmap + encoding_regexes[encoding] = re.compile(regex) + return charmaps, encoding_regexes +CHARMAPS, ENCODING_REGEXES = _build_charmaps() def possible_encoding(text, encoding): """ @@ -59,9 +76,16 @@ def possible_encoding(text, encoding): return bool(ENCODING_REGEXES[encoding].match(text)) -CHAR_CLASS_STRING = zlib.decompress(resource_string(__name__, 'char_classes.dat')).decode('ascii') - +CHAR_CLASS_STRING = zlib.decompress( + resource_string(__name__, 'char_classes.dat') +).decode('ascii') def chars_to_classes(string): + """ + Convert each Unicode character to a letter indicating which of many + classes it's in. + + See build_data.py for where this data comes from and what it means. + """ return string.translate(CHAR_CLASS_STRING) # A translate mapping that will strip all C0 control characters except diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 20009d49..f6c161e0 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -1,12 +1,15 @@ # -*- coding: utf-8 -*- +""" +This module contains the individual fixes that the main fix_text function +can perform. +""" + from __future__ import unicode_literals from ftfy.chardata import (possible_encoding, CHARMAPS, CHARMAP_ENCODINGS, CONTROL_CHARS) from ftfy.badness import text_cost import re import sys -import warnings -import logging from ftfy.compatibility import htmlentitydefs, unichr, bytes_to_ints @@ -108,6 +111,14 @@ def fix_text_encoding(text): return best_version def fix_text_and_explain(text): + """ + Performs a single step of re-encoding text that's been decoded incorrectly. + It returns the decoded text, plus a structure explaining what it did. + + This structure could be used for more than it currently is, but we at least + use it to track whether we had to intepret text as an old encoding such as + MacRoman or cp437. + """ if isinstance(text, bytes): raise UnicodeError(BYTES_ERROR_TEXT) if len(text) == 0: @@ -143,7 +154,7 @@ def fix_text_and_explain(text): # points up to 0xff. This can then be converted into the intended # bytes by encoding it as Latin-1. sorta_encoded_text = text.translate(CHARMAPS[encoding]) - + # When we get the bytes, run them through fix_java_encoding, # because we can only reliably do that at the byte level. (See # its documentation for details.) @@ -201,18 +212,24 @@ def fix_text_and_explain(text): return text, [('give up', None)] -HTML_ENTITY_RE = re.compile(r"&#?\w+;") +HTML_ENTITY_RE = re.compile(r"&#?\w{0,8};") def unescape_html(text): """ Decode all three types of HTML entities/character references. - Code by Fredrik Lundh of effbot.org. + Code by Fredrik Lundh of effbot.org. Rob Speer made a slight change + to it for efficiency: it won't match entities longer than 8 characters, + because there are no valid entities like that. >>> print(unescape_html('<tag>')) """ - def fixup(m): - text = m.group(0) + def fixup(match): + """ + Replace one matched HTML entity with the character it represents, + if possible. + """ + text = match.group(0) if text[:2] == "&#": # character reference try: @@ -266,7 +283,7 @@ def fix_line_breaks(text): def remove_control_chars(text): r""" Remove all control characters except for the important ones. - + This removes characters from U+0000 to U+001F and U+0080 to U+009F, except it leaves alone these characters that are commonly used for formatting: @@ -304,11 +321,11 @@ def fix_java_encoding(bytestring): # When we have improperly encoded surrogates, we can still see the # bits that they were meant to represent. - # + # # The surrogates were meant to encode a 20-bit number, to which we # add 0x10000 to get a codepoint. That 20-bit number now appears in # this form: - # + # # 11101101 1010abcd 10efghij 11101101 1011klmn 10opqrst # # The CESU8_RE above matches byte sequences of this form. Then we need From 079e87133d196a284456ab1cd993d60f394decf7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 16:06:46 -0400 Subject: [PATCH 059/502] Move tests out of the package --- {ftfy => tests}/test_real_tweets.py | 0 {ftfy => tests}/test_unicode.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {ftfy => tests}/test_real_tweets.py (100%) rename {ftfy => tests}/test_unicode.py (100%) diff --git a/ftfy/test_real_tweets.py b/tests/test_real_tweets.py similarity index 100% rename from ftfy/test_real_tweets.py rename to tests/test_real_tweets.py diff --git a/ftfy/test_unicode.py b/tests/test_unicode.py similarity index 100% rename from ftfy/test_unicode.py rename to tests/test_unicode.py From 2b3375f67a7ed2b3cf690f83ff5b34b330d57ddd Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 16:11:05 -0400 Subject: [PATCH 060/502] fix build_data script so it actually runs on its own --- ftfy/build_data.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 1a29e399..69afb557 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -11,6 +11,13 @@ The file will be written to the current directory. """ +from __future__ import unicode_literals +import unicodedata +import sys +import zlib +if sys.hexversion >= 0x03000000: + unichr = chr + # L = Latin capital letter # l = Latin lowercase letter # A = Non-latin capital or title-case letter @@ -75,10 +82,10 @@ def make_char_data_file(): cclasses[codepoint] = ' ' else: cclasses[codepoint] = 'o' - + cclasses[9] = cclasses[10] = cclasses[12] = cclasses[13] = ' ' out = open('char_classes.dat', 'wb') - out.write(zlib.compress(bytes(''.join(cclasses), 'ascii'))) + out.write(zlib.compress(''.join(cclasses).encode('ascii'))) out.close() if __name__ == '__main__': From 2430c91d726151b2740eb0765466086925a7f72b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 16:21:13 -0400 Subject: [PATCH 061/502] Restore the behavior of fix_file and cli.py. --- ftfy/__init__.py | 18 +++++++++++------- ftfy/cli.py | 11 ++++++++++- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index b2a2f36a..5fd3f715 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -131,17 +131,21 @@ def fix_text(text, def fix_file(file, normalization='NFKC'): """ - Fix a file that is being read in as Unicode text (for example, using - the `codecs.open` function), and stream the resulting text one line - at a time. + Fix text that is found in a file. + + If the file is being read as Unicode text, use that. If it's being read as + bytes, then we'll try to decode each line as Latin-1, and then fix it if it + was really UTF-8 or something. + + That's kind of awful, but it will have to suffice until we have a working + encoding detector. + + The output is a stream of fixed lines of text. """ entities = True for line in file: if isinstance(line, bytes): - try: - line = line.decode('utf-8') - except UnicodeDecodeError: - line = line.decode('latin-1') + line = line.decode('latin-1') if '<' in line and '>' in line: entities = False yield fix_text_segment(line, normalization, entities) diff --git a/ftfy/cli.py b/ftfy/cli.py index 5881bd76..5c04103c 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -1,5 +1,14 @@ +""" +A simple command-line utility for fixing text found in a file. + +This utility knows nothing about the initial encoding of the file, so it will +guess Latin-1 and then try to fix it in the very likely event that that guess +is wrong. + +That behavior is slower than strictly necessary, and possibly a bit risky, +so consider this utility a proof of concept. +""" from ftfy import fix_file -import codecs import sys ENCODE_STDOUT = (sys.hexversion < 0x03000000) From da9bfea2c07ef4e7aa27e8913f2289fd22970444 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 16:21:35 -0400 Subject: [PATCH 062/502] More docstrings. pylint is now reasonably happy with this code. --- ftfy/build_data.py | 8 ++++++++ ftfy/compatibility.py | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 69afb557..b97260e4 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -37,6 +37,14 @@ # o = Other def make_char_data_file(): + """ + Build the compressed data file 'char_classes.dat' and write it to the + current directory. + + If you run this, run it in Python 3.3 or later. It will run in earlier + versions, but you won't get the current Unicode standard, leading to + inconsistent behavior. + """ cclasses = [None] * 0x110000 for codepoint in range(0x0, 0x110000): char = unichr(codepoint) diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index 16a06ac1..6c894c5e 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -1,3 +1,7 @@ +""" +Makes some function names and behavior consistent between Python 2 and +Python 3, and also between narrow and wide builds. +""" import sys if sys.hexversion >= 0x03000000: From e913426a029159e054c7b457579de6769ee0fa2f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 16:23:48 -0400 Subject: [PATCH 063/502] update formatting and comments in chardata --- ftfy/chardata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 50f85515..580271a6 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -65,9 +65,9 @@ def _build_charmaps(): charmaps[encoding] = charmap encoding_regexes[encoding] = re.compile(regex) return charmaps, encoding_regexes - CHARMAPS, ENCODING_REGEXES = _build_charmaps() + def possible_encoding(text, encoding): """ Given text and a single-byte encoding, check whether that text could have @@ -88,9 +88,9 @@ def chars_to_classes(string): """ return string.translate(CHAR_CLASS_STRING) + # A translate mapping that will strip all C0 control characters except -# \t and \n. This incidentally has the effect of normalizing Windows \r\n -# line endings to Unix \n line endings. +# those that represent whitespace. CONTROL_CHARS = {} for i in range(32): CONTROL_CHARS[i] = None From 2f6f97c2c7468ddc140f86bc0fcb2e122cc9d923 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 16:25:26 -0400 Subject: [PATCH 064/502] update MIT license --- LICENSE.txt | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 767a2405..4b80d192 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,8 +1,20 @@ -Copyright (C) 2012 Rob Speer (rob@lumino.so) +Copyright (C) 2013 Rob Speer (rob@luminoso.com) MIT License -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 9bc04bd3fc1629fd94ab23e4380e66c6165533c0 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 16:30:00 -0400 Subject: [PATCH 065/502] fix unnecessarily marked Unicode literal --- ftfy/badness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 34261ed0..8ba615b2 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -60,7 +60,7 @@ def _make_weirdness_regex(): # Match C1 control characters, which are almost always the result of # decoding Latin-1 that was meant to be Windows-1252. - groups.append(u'X') + groups.append('X') # Match adjacent characters from any different pair of these categories: # - Modifier marks (M) From a4f0e5c7b08fc93e404f5566f72f2a0caa9a56ca Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 16:30:13 -0400 Subject: [PATCH 066/502] include char_classes.dat in package_data --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index c29393f2..6f88458a 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ platforms=["any"], description="Fixes some problems with Unicode text after the fact", packages=['ftfy'], + package_data={'ftfy': ['char_classes.dat']}, classifiers=[ "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", From d0c0aa3d189966065bf5bf53aa68f648fc44f6c8 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 18:15:11 -0400 Subject: [PATCH 067/502] update the README --- README.md | 199 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 148 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 8a72b463..40d07f69 100644 --- a/README.md +++ b/README.md @@ -1,63 +1,150 @@ -## ftfy: fixes text for you +# ftfy: fixes text for you This is a module for making text less broken. It works in Python 2.6, Python 3.2, or later. -Its main function, `fix_text` aka `ftfy`, performs the following operations on -a Unicode string: - -* Detect whether the text was incorrectly encoded and fix it, as defined - in `fix_bad_encoding`. -* If the text is HTML-escaped but has no HTML tags, replace HTML entities - with their equivalent characters. -* Remove terminal escapes and color codes. -* Remove control characters except for newlines and tabs. -* Normalize it with Unicode normalization form KC, which applies the - following relevant transformations: - * Combine characters and diacritics that are written using separate - code points, such as converting "e" plus an acute accent modifier - into "é", or converting "ka" (か) plus a dakuten into the - single character "ga" (が). - * Replace characters that are functionally equivalent with the most - common form: for example, half-width katakana will be replaced with - full-width, full-width Roman characters will be replaced with - ASCII characters, ellipsis characters will be replaced by three - periods, and the ligature 'fl' will be replaced by 'fl'. -* Replace curly quotation marks with straight ones. -* Remove an unnecessary byte-order mark (which might come from Microsoft's version - of UTF-8) from the start. - -### Examples +What does it mean for text to be broken? + + - The text could have been encoded with one encoding standard, and decoded + with a different one. The result is that some characters turn into + nonsense sequences that look like this: `—` + - The text could contain HTML entities, but be passed into a system that + was not designed to read HTML. + - For that matter, it could contain instructions for a text terminal to + move the cursor or change colors or something, but you are not sending + the text to a terminal. + - The text could write words in non-standard ways for display purposes, + such as using the three characters `fl` `o` `p` for the word "flop". + - The text could contain control characters that are designed for a + certain operating system. + +Of course you're better off if all the text you take as input is in the right +format for your application and operating system. But often, your input is +something you have no control over. Somebody else's minor mistake becomes +your problem. + +ftfy will do everything it can to fix the problem. + +## Examples + +In these examples, `unicode_literals` are turned on. ftfy always expects +Unicode strings as input. >>> from __future__ import unicode_literals >>> from ftfy import fix_text + >>> print(fix_text('This — should be an em dash')) This — should be an em dash >>> print(fix_text('ünicode')) ünicode + >>> print(fix_text('Broken text… it’s flubberific!')) + Broken text... it's flubberific! + + >>> print(fix_text('HTML entities <3')) + HTML entities <3 + + >>> print(fix_text('HTML entities <3')) + HTML entities <3 + >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' >>> print(fix_text(wtf)) ಠ_ಠ -### Non-Unicode strings +## Using ftfy + +The main function, `fix_text`, will run text through a sequence of fixes. If +the text changed, it will run them through again, so that you can be sure +the output ends up in a standard form that will be unchanged by `fix_text`. +All the fixes are on by default, but you can pass options to turn them off. + + - If `remove_terminal_escapes` is True, remove sequences of bytes that are + instructions for Unix terminals, such as the codes that make text appear + in different colors. + - If `fix_entities` is True, consider replacing HTML entities with their + equivalent characters. However, this never applies to text with a pair + of angle brackets in it already; you're probably not supposed to decode + entities there, and you'd make things ambiguous if you did. + - If `fix_encoding` is True, look for common mistakes that come from + encoding or decoding Unicode text incorrectly, and fix them if they are + reasonably fixable. + - If `normalization` is not None, apply the specified form of Unicode + normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. + The default, 'NFKC', applies the following relevant transformations: + - C: Combine characters and diacritics that are written using separate + code points, such as converting "e" plus an acute accent modifier + into "é", or converting "ka" (か) plus a dakuten into the + single character "ga" (が). + - K: Replace characters that are functionally equivalent with the most + common form. For example, half-width katakana will be replaced with + full-width versions, full-width Roman characters will be replaced with + ASCII characters, ellipsis characters will be replaced with three + periods, and the ligature 'fl' will be replaced with 'fl'. + - If `uncurl_quotes` is True, replace various curly quotation marks with + plain-ASCII straight quotes. + - If `fix_line_breaks` is true, convert all line breaks to Unix style + (CRLF and CR line breaks become LF line breaks). + - If `fix_control_characters` is true, remove all C0 control characters + except the common useful ones: TAB, CR, LF, and FF. (CR characters + may have already been removed by the `fix_line_breaks` step.) + - If `remove_bom` is True, remove the Byte-Order Mark if it exists. + (It's an instruction to a UTF-16 decoder. It's not meant to actually + end up in your string.) + - If anything was changed, repeat all the steps, so that the function is + idempotent. "&amp;" will become "&", for example, not "&". + +## Encodings ftfy can handle + +`ftfy` can understand text that was decoded as any of these single-byte +encodings: + +- Latin-1 (ISO-8859-1) +- Windows-1252 (cp1252 -- used in Microsoft products) +- Windows-1251 (cp1251 -- the Russian version of cp1252) +- MacRoman (used on Mac OS 9 and earlier) +- cp437 (used in MS-DOS) + +when it was actually intended to be decoded as a variable-length encoding: + +- UTF-8 +- CESU-8 (what some programmers think is UTF-8) + +It can also understand text that was intended as Windows-1252 but decoded as +Latin-1 -- that's when things like smart-quotes and bullets turn into weird +control characters. + +## Encodings ftfy can't handle + +`ftfy` cannot understand other mixups between single-byte encodings besides +Latin-1 for Windows-1252, because it is extremely difficult to detect which +mixup in particular is the one that happened. + +It cannot handle Windows-1250 and ISO-8859-2, and it never will. Look at their +rows on http://en.wikipedia.org/wiki/Polish_code_pages and you might recognize +why there is no hope of ever being able to tell these apart. + +## Non-Unicode strings When first using ftfy, you might be confused to find that you can't give it a bytestring (the type of object called `str` in Python 2). ftfy fixes text. Treating bytestrings as text is exactly the kind of thing that -causes the Unicode problems that ftfy has to fix. Hypothetically, it might even -be able to let you do such a silly thing, and then fix the problem you just -created. But that creates a cycle of dependence on tools such as ftfy. +causes the Unicode problems that ftfy has to fix. So instead of silently fixing your error, ftfy ensures first that *you* are doing it right -- which means you must give it a Unicode string as input. If you don't, it'll point you to the [Python Unicode HOWTO](http://docs.python.org/3/howto/unicode.html). -If you have been given a mess of bytes from somewhere else and you need to handle -them, you can decode those bytes as Latin-1 and let ftfy take it from there. +But what if you all you have is a mess of bytes? Well, you've got a problem, +and ftfy is not quite the right tool to solve it. Maybe you should try getting +your input from a protocol that understands encoding issues, such as JSON or +XML. + +As a sort of half-measure that covers a few common cases, you can decode the +bytes as Latin-1 and let ftfy take it from there, which might include +reinterpreting the Latin-1 text as Windows-1252 or UTF-8. >>> print(fix_text(b'\x85test')) UnicodeError: [informative error message] @@ -65,31 +152,41 @@ them, you can decode those bytes as Latin-1 and let ftfy take it from there. >>> print(fix_text(b'\x85test'.decode('latin-1'))) —test -### Encodings supported +### A note on encoding detection -`ftfy` will resolve errors that result from mixing up any of the -following encodings: +If your input is a mess of unmarked bytes, you might want a tool that can just +statistically analyze those bytes and predict what encoding they're in. -- UTF-8 -- Latin-1 -- Windows-1252 (cp1252) +ftfy is not that tool. I might want to write that tool someday. -These are the three most commonly-confused encodings, because on English text -they look nearly the same. +You may have heard of chardet. Chardet is admirable, but it is not that tool +either. Its heuristics only work on multi-byte encodings, such as UTF-8 and the +language-specific encodings used in East Asian languages. It works very badly +on single-byte encodings, and outputs wrong answers with high confidence. -(Do you ever encounter files in MacRoman or DOS cp437? I could add support for -detecting and fixing text that has been through one of these encodings, but it -would come with an increased risk of "fixing" other text incorrectly. Open a -bug report on GitHub if you think it would be worthwhile to write the -heuristics to handle these encodings.) +There is lots of real-world text that's in an unknown single-byte encoding. +There might be enough information to statistically sort out which encoding is +which. But nothing, so far, actually does that. -### Command-line usage +## Command-line usage -ftfy now installs itself as a command line tool, for cleaning up text files -that are a mess of mangled bytes. +ftfy installs itself as a command line tool, for cleaning up text files that +are a mess of mangled bytes. You can type `ftfy FILENAME`, and it will read in FILENAME as Latin-1 text, fix -encoding problems (including re-interpreting it as UTF-8 if appropriate), and -write the result to standard out as UTF-8. +everything that `fix_text` fixes (including re-interpreting it as UTF-8 if +appropriate), and write the result to standard out as UTF-8. + +This is not necessarily a good idea, but it's kind of convenient. Consider this +a proof of concept until we get a real encoding detector. + +## Who's behind this tool? + +I'm Rob Speer (rob@luminoso.com). I develop this tool as part of my +text-understanding company, [Luminoso](http://luminoso.com), where it has +proven essential. + +Luminoso provides ftfy as free, open source software under the extremely +permissive MIT license. -Author: Rob Speer, rob@luminoso.com +You can report bugs regarding ftfy on GitHub and we'll handle them. From e9c8b4d28130b684cb75c97af286a6c9a516e09a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 18:16:41 -0400 Subject: [PATCH 068/502] Markdown formatting fixes --- README.md | 99 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 40d07f69..03693c57 100644 --- a/README.md +++ b/README.md @@ -5,18 +5,18 @@ Python 3.2, or later. What does it mean for text to be broken? - - The text could have been encoded with one encoding standard, and decoded - with a different one. The result is that some characters turn into - nonsense sequences that look like this: `—` - - The text could contain HTML entities, but be passed into a system that - was not designed to read HTML. - - For that matter, it could contain instructions for a text terminal to - move the cursor or change colors or something, but you are not sending - the text to a terminal. - - The text could write words in non-standard ways for display purposes, - such as using the three characters `fl` `o` `p` for the word "flop". - - The text could contain control characters that are designed for a - certain operating system. +- The text could have been encoded with one encoding standard, and decoded + with a different one. The result is that some characters turn into + nonsense sequences that look like this: `—` +- The text could contain HTML entities, but be passed into a system that + was not designed to read HTML. +- For that matter, it could contain instructions for a text terminal to + move the cursor or change colors or something, but you are not sending + the text to a terminal. +- The text could write words in non-standard ways for display purposes, + such as using the three characters `fl` `o` `p` for the word "flop". +- The text could contain control characters that are designed for a + certain operating system. Of course you're better off if all the text you take as input is in the right format for your application and operating system. But often, your input is @@ -57,44 +57,45 @@ Unicode strings as input. The main function, `fix_text`, will run text through a sequence of fixes. If the text changed, it will run them through again, so that you can be sure the output ends up in a standard form that will be unchanged by `fix_text`. + All the fixes are on by default, but you can pass options to turn them off. - - If `remove_terminal_escapes` is True, remove sequences of bytes that are - instructions for Unix terminals, such as the codes that make text appear - in different colors. - - If `fix_entities` is True, consider replacing HTML entities with their - equivalent characters. However, this never applies to text with a pair - of angle brackets in it already; you're probably not supposed to decode - entities there, and you'd make things ambiguous if you did. - - If `fix_encoding` is True, look for common mistakes that come from - encoding or decoding Unicode text incorrectly, and fix them if they are - reasonably fixable. - - If `normalization` is not None, apply the specified form of Unicode - normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. - The default, 'NFKC', applies the following relevant transformations: - - C: Combine characters and diacritics that are written using separate - code points, such as converting "e" plus an acute accent modifier - into "é", or converting "ka" (か) plus a dakuten into the - single character "ga" (が). - - K: Replace characters that are functionally equivalent with the most - common form. For example, half-width katakana will be replaced with - full-width versions, full-width Roman characters will be replaced with - ASCII characters, ellipsis characters will be replaced with three - periods, and the ligature 'fl' will be replaced with 'fl'. - - If `uncurl_quotes` is True, replace various curly quotation marks with - plain-ASCII straight quotes. - - If `fix_line_breaks` is true, convert all line breaks to Unix style - (CRLF and CR line breaks become LF line breaks). - - If `fix_control_characters` is true, remove all C0 control characters - except the common useful ones: TAB, CR, LF, and FF. (CR characters - may have already been removed by the `fix_line_breaks` step.) - - If `remove_bom` is True, remove the Byte-Order Mark if it exists. - (It's an instruction to a UTF-16 decoder. It's not meant to actually - end up in your string.) - - If anything was changed, repeat all the steps, so that the function is - idempotent. "&amp;" will become "&", for example, not "&". - -## Encodings ftfy can handle +- If `remove_terminal_escapes` is True, remove sequences of bytes that are + instructions for Unix terminals, such as the codes that make text appear + in different colors. +- If `fix_entities` is True, consider replacing HTML entities with their + equivalent characters. However, this never applies to text with a pair + of angle brackets in it already; you're probably not supposed to decode + entities there, and you'd make things ambiguous if you did. +- If `fix_encoding` is True, look for common mistakes that come from + encoding or decoding Unicode text incorrectly, and fix them if they are + reasonably fixable. +- If `normalization` is not None, apply the specified form of Unicode + normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. + The default, 'NFKC', applies the following relevant transformations: + - C: Combine characters and diacritics that are written using separate + code points, such as converting "e" plus an acute accent modifier + into "é", or converting "ka" (か) plus a dakuten into the + single character "ga" (が). + - K: Replace characters that are functionally equivalent with the most + common form. For example, half-width katakana will be replaced with + full-width versions, full-width Roman characters will be replaced with + ASCII characters, ellipsis characters will be replaced with three + periods, and the ligature 'fl' will be replaced with 'fl'. +- If `uncurl_quotes` is True, replace various curly quotation marks with + plain-ASCII straight quotes. +- If `fix_line_breaks` is true, convert all line breaks to Unix style + (CRLF and CR line breaks become LF line breaks). +- If `fix_control_characters` is true, remove all C0 control characters + except the common useful ones: TAB, CR, LF, and FF. (CR characters + may have already been removed by the `fix_line_breaks` step.) +- If `remove_bom` is True, remove the Byte-Order Mark if it exists. + (It's an instruction to a UTF-16 decoder. It's not meant to actually + end up in your string.) +- If anything was changed, repeat all the steps, so that the function is + idempotent. "&amp;" will become "&", for example, not "&". + +### Encodings ftfy can handle `ftfy` can understand text that was decoded as any of these single-byte encodings: @@ -114,7 +115,7 @@ It can also understand text that was intended as Windows-1252 but decoded as Latin-1 -- that's when things like smart-quotes and bullets turn into weird control characters. -## Encodings ftfy can't handle +### Encodings ftfy can't handle `ftfy` cannot understand other mixups between single-byte encodings besides Latin-1 for Windows-1252, because it is extremely difficult to detect which From fb7543f2f3e539d84479e3f4e83136bcc6431d35 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 18:17:47 -0400 Subject: [PATCH 069/502] yes, Markdown, you're very clever, now stop it --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 03693c57..840257b5 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ All the fixes are on by default, but you can pass options to turn them off. (It's an instruction to a UTF-16 decoder. It's not meant to actually end up in your string.) - If anything was changed, repeat all the steps, so that the function is - idempotent. "&amp;" will become "&", for example, not "&". + idempotent. `"&amp;"` will become `"&"`, for example, not `"&"`. ### Encodings ftfy can handle From 3fab47535032dc307d6f1939d2dba370bd115a7c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 24 Aug 2013 20:12:28 -0400 Subject: [PATCH 070/502] Copyedit the README. --- README.md | 54 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 840257b5..0f7a6c53 100644 --- a/README.md +++ b/README.md @@ -112,18 +112,12 @@ when it was actually intended to be decoded as a variable-length encoding: - CESU-8 (what some programmers think is UTF-8) It can also understand text that was intended as Windows-1252 but decoded as -Latin-1 -- that's when things like smart-quotes and bullets turn into weird -control characters. +Latin-1 -- that's the very common case where things like smart-quotes and +bullets turn into weird control characters. -### Encodings ftfy can't handle - -`ftfy` cannot understand other mixups between single-byte encodings besides -Latin-1 for Windows-1252, because it is extremely difficult to detect which -mixup in particular is the one that happened. - -It cannot handle Windows-1250 and ISO-8859-2, and it never will. Look at their -rows on http://en.wikipedia.org/wiki/Polish_code_pages and you might recognize -why there is no hope of ever being able to tell these apart. +However, `ftfy` cannot understand other mixups between single-byte encodings, +because it is extremely difficult to detect which mixup in particular is the +one that happened. ## Non-Unicode strings @@ -131,17 +125,16 @@ When first using ftfy, you might be confused to find that you can't give it a bytestring (the type of object called `str` in Python 2). ftfy fixes text. Treating bytestrings as text is exactly the kind of thing that -causes the Unicode problems that ftfy has to fix. +causes the Unicode problems that ftfy has to fix. So if you don't give it a +Unicode string, ftfy will point you to the [Python Unicode +HOWTO](http://docs.python.org/3/howto/unicode.html). -So instead of silently fixing your error, ftfy ensures first that *you* are -doing it right -- which means you must give it a Unicode string as input. If -you don't, it'll point you to the -[Python Unicode HOWTO](http://docs.python.org/3/howto/unicode.html). +Reasonable ways that you might exchange data, such as JSON or XML, already have +perfectly good ways of expressing Unicode strings. Given a Unicode string, ftfy +can apply fixes that are very likely to work without false positives. -But what if you all you have is a mess of bytes? Well, you've got a problem, -and ftfy is not quite the right tool to solve it. Maybe you should try getting -your input from a protocol that understands encoding issues, such as JSON or -XML. +But what if you all you actually have is a mess of bytes on a disk? Well, +you've got a problem, and ftfy is not quite the right tool to solve it. As a sort of half-measure that covers a few common cases, you can decode the bytes as Latin-1 and let ftfy take it from there, which might include @@ -163,7 +156,8 @@ ftfy is not that tool. I might want to write that tool someday. You may have heard of chardet. Chardet is admirable, but it is not that tool either. Its heuristics only work on multi-byte encodings, such as UTF-8 and the language-specific encodings used in East Asian languages. It works very badly -on single-byte encodings, and outputs wrong answers with high confidence. +on single-byte encodings, to the point where it will output wrong answers with +high confidence. There is lots of real-world text that's in an unknown single-byte encoding. There might be enough information to statistically sort out which encoding is @@ -171,17 +165,25 @@ which. But nothing, so far, actually does that. ## Command-line usage -ftfy installs itself as a command line tool, for cleaning up text files that -are a mess of mangled bytes. +ftfy installs itself as a command line tool that reads a file and applies +`fix_text` to it. + +This has exactly the problem described above: a file on a disk is made of bytes +in an unspecified encoding. It could assume the file is UTF-8, but if you had +totally valid UTF-8 you probably wouldn't need this command line utility, and +there's a slight chance that the file could contain Latin-1 that coincidentally +looks like UTF-8. + +Instead, it will follow the "half-measure" above. You can type `ftfy FILENAME`, and it will read in FILENAME as Latin-1 text, fix everything that `fix_text` fixes (including re-interpreting it as UTF-8 if appropriate), and write the result to standard out as UTF-8. -This is not necessarily a good idea, but it's kind of convenient. Consider this -a proof of concept until we get a real encoding detector. +This is not necessarily a good idea, but it's convenient. Consider this a proof +of concept until we get a real encoding detector. -## Who's behind this tool? +## Who maintains ftfy? I'm Rob Speer (rob@luminoso.com). I develop this tool as part of my text-understanding company, [Luminoso](http://luminoso.com), where it has From de1147b4de4af35e07c21c6ddaef7f520b43f66b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 25 Aug 2013 23:55:49 -0400 Subject: [PATCH 071/502] more editing of docs --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0f7a6c53..006d6bb3 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # ftfy: fixes text for you -This is a module for making text less broken. It works in Python 2.6, -Python 3.2, or later. +This is a module for making text less broken and more consistent. It works in +Python 2.6, Python 3.2, or later. What does it mean for text to be broken? @@ -45,9 +45,14 @@ Unicode strings as input. >>> print(fix_text('HTML entities <3')) HTML entities <3 +If any HTML tags appear in your input, ftfy will make sure to leave the HTML +entities alone: + >>> print(fix_text('HTML entities <3')) HTML entities <3 +ftfy repeats its process until it reaches a result that it won't change: + >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' >>> print(fix_text(wtf)) ಠ_ಠ From c3f706dddbe3c519312b31a9ae26cf01ccdbf43d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 26 Aug 2013 11:31:46 -0400 Subject: [PATCH 072/502] Other README changes --- README.md | 223 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 169 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 8a72b463..f6c18fe0 100644 --- a/README.md +++ b/README.md @@ -1,63 +1,159 @@ -## ftfy: fixes text for you +# ftfy: fixes text for you This is a module for making text less broken. It works in Python 2.6, Python 3.2, or later. -Its main function, `fix_text` aka `ftfy`, performs the following operations on -a Unicode string: - -* Detect whether the text was incorrectly encoded and fix it, as defined - in `fix_bad_encoding`. -* If the text is HTML-escaped but has no HTML tags, replace HTML entities - with their equivalent characters. -* Remove terminal escapes and color codes. -* Remove control characters except for newlines and tabs. -* Normalize it with Unicode normalization form KC, which applies the - following relevant transformations: - * Combine characters and diacritics that are written using separate - code points, such as converting "e" plus an acute accent modifier - into "é", or converting "ka" (か) plus a dakuten into the - single character "ga" (が). - * Replace characters that are functionally equivalent with the most - common form: for example, half-width katakana will be replaced with - full-width, full-width Roman characters will be replaced with - ASCII characters, ellipsis characters will be replaced by three - periods, and the ligature 'fl' will be replaced by 'fl'. -* Replace curly quotation marks with straight ones. -* Remove an unnecessary byte-order mark (which might come from Microsoft's version - of UTF-8) from the start. +The most interesting kind of brokenness that this resolves is when someone +has encoded Unicode with one standard and decoded it with a different one. +This often shows up as characters that turn into nonsense sequences: + +- The word `schön` might become `schön`. +- An em dash (`—`) might become `—`. +- Text that was meant to be enclosed in quotation marks might end up + instead enclosed in `“` and `”` (and that last character + probably won't even display as anything meaningful). + +This happens very often to real text. Fortunately, the nonsense sequences +usually contain all the information you need to reconstruct what character was +supposed to be there. + +Any given text string might have other irritating properties, possibly even +interacting with the erroneous decoding: + +- The text could contain HTML entities such as `&` in place of certain + characters, when you would rather see what the characters actually are. +- For that matter, it could contain instructions for a text terminal to + do something like change colors, but you are not sending the text to a + terminal, so those instructions are just going to look like `^[[30m;` + or something in the middle of the text. +- The text could write words in non-standard ways for display purposes, + such as using the three characters `fl` `o` `p` for the word "flop". + This can happen when you copy text out of a PDF. +- The text could contain control characters that are designed for a + particular operating system, which your application does not care about. + +Of course you're better off if all the text you take as input is decoded +properly, and in the right format for your application and operating system. +But often, your input is something you have no control over. Somebody else's +minor mistake becomes your problem. -### Examples +ftfy will do everything it can to fix the problem. + +## Examples + +In these examples, `unicode_literals` are turned on. ftfy always expects +Unicode strings as input. >>> from __future__ import unicode_literals >>> from ftfy import fix_text + >>> print(fix_text('This — should be an em dash')) This — should be an em dash >>> print(fix_text('ünicode')) ünicode + >>> print(fix_text('Broken text… it’s flubberific!')) + Broken text... it's flubberific! + + >>> print(fix_text('HTML entities <3')) + HTML entities <3 + + >>> print(fix_text('HTML entities <3')) + HTML entities <3 + >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' >>> print(fix_text(wtf)) ಠ_ಠ -### Non-Unicode strings +## Using ftfy + +The main function, `fix_text`, will run text through a sequence of fixes. If +the text changed, it will run them through again, so that you can be sure +the output ends up in a standard form that will be unchanged by `fix_text`. + +All the fixes are on by default, but you can pass options to turn them off. + +- If `remove_terminal_escapes` is True, remove sequences of bytes that are + instructions for Unix terminals, such as the codes that make text appear + in different colors. +- If `fix_entities` is True, consider replacing HTML entities with their + equivalent characters. However, this never applies to text with a pair + of angle brackets in it already; you're probably not supposed to decode + entities there, and you'd make things ambiguous if you did. +- If `fix_encoding` is True, look for common mistakes that come from + encoding or decoding Unicode text incorrectly, and fix them if they are + reasonably fixable. +- If `normalization` is not None, apply the specified form of Unicode + normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. + The default, 'NFKC', applies the following relevant transformations: + - C: Combine characters and diacritics that are written using separate + code points, such as converting "e" plus an acute accent modifier + into "é", or converting "ka" (か) plus a dakuten into the + single character "ga" (が). + - K: Replace characters that are functionally equivalent with the most + common form. For example, half-width katakana will be replaced with + full-width versions, full-width Roman characters will be replaced with + ASCII characters, ellipsis characters will be replaced with three + periods, and the ligature 'fl' will be replaced with 'fl'. +- If `uncurl_quotes` is True, replace various curly quotation marks with + plain-ASCII straight quotes. +- If `fix_line_breaks` is true, convert all line breaks to Unix style + (CRLF and CR line breaks become LF line breaks). +- If `fix_control_characters` is true, remove all C0 control characters + except the common useful ones: TAB, CR, LF, and FF. (CR characters + may have already been removed by the `fix_line_breaks` step.) +- If `remove_bom` is True, remove the Byte-Order Mark if it exists. + (It's a hint for a UTF-16 decoder. It's not meant to actually + end up in your string.) +- If anything was changed, repeat all the steps, so that the function is + idempotent. `"&amp;"` will become `"&"`, for example, not `"&"`. + +### Encodings ftfy can handle + +ftfy can understand text that was decoded as any of these single-byte +encodings: + +- Latin-1 (ISO-8859-1) +- Windows-1252 (cp1252 -- used in Microsoft products) +- Windows-1251 (cp1251 -- the Russian version of cp1252) +- MacRoman (used on Mac OS 9 and earlier) +- cp437 (used in MS-DOS) + +when it was actually intended to be decoded as one of these variable-length +encodings: + +- UTF-8 +- CESU-8 (what some people, especially Java programmers, think is UTF-8) + +It can also understand text that was intended as Windows-1252 but decoded as +Latin-1. That's the very common case where things like smart-quotes and +bullets turn into single weird control characters. + +However, ftfy cannot understand other mixups between single-byte encodings, +because it is extremely difficult to detect which mixup in particular is the +one that happened. + +## Non-Unicode strings When first using ftfy, you might be confused to find that you can't give it a bytestring (the type of object called `str` in Python 2). ftfy fixes text. Treating bytestrings as text is exactly the kind of thing that -causes the Unicode problems that ftfy has to fix. Hypothetically, it might even -be able to let you do such a silly thing, and then fix the problem you just -created. But that creates a cycle of dependence on tools such as ftfy. +causes the Unicode problems that ftfy has to fix. So if you don't give it a +Unicode string, ftfy will point you to the [Python Unicode +HOWTO](http://docs.python.org/3/howto/unicode.html). + +Reasonable ways that you might exchange data, such as JSON or XML, already have +perfectly good ways of expressing Unicode strings. Given a Unicode string, ftfy +can apply fixes that are very likely to work without false positives. -So instead of silently fixing your error, ftfy ensures first that *you* are -doing it right -- which means you must give it a Unicode string as input. If -you don't, it'll point you to the -[Python Unicode HOWTO](http://docs.python.org/3/howto/unicode.html). +But what if you all you actually have is a mess of bytes on a disk? Well, +you've got a problem, and ftfy is not quite the right tool to solve it. -If you have been given a mess of bytes from somewhere else and you need to handle -them, you can decode those bytes as Latin-1 and let ftfy take it from there. +As a sort of half-measure that covers a few common cases, you can decode the +bytes as Latin-1 and let ftfy take it from there, which might include +reinterpreting the Latin-1 text as Windows-1252 or UTF-8. >>> print(fix_text(b'\x85test')) UnicodeError: [informative error message] @@ -65,31 +161,50 @@ them, you can decode those bytes as Latin-1 and let ftfy take it from there. >>> print(fix_text(b'\x85test'.decode('latin-1'))) —test -### Encodings supported +### A note on encoding detection -`ftfy` will resolve errors that result from mixing up any of the -following encodings: +If your input is a mess of unmarked bytes, you might want a tool that can just +statistically analyze those bytes and predict what encoding they're in. -- UTF-8 -- Latin-1 -- Windows-1252 (cp1252) +ftfy is not that tool. I might want to write that tool someday. + +You may have heard of chardet. Chardet is admirable, but it is not that tool +either. Its heuristics only work on multi-byte encodings, such as UTF-8 and the +language-specific encodings used in East Asian languages. It works very badly +on single-byte encodings, to the point where it will output wrong answers with +high confidence. -These are the three most commonly-confused encodings, because on English text -they look nearly the same. +There is lots of real-world text that's in an unknown single-byte encoding. +There might be enough information to statistically sort out which encoding is +which. But nothing, so far, actually does that. -(Do you ever encounter files in MacRoman or DOS cp437? I could add support for -detecting and fixing text that has been through one of these encodings, but it -would come with an increased risk of "fixing" other text incorrectly. Open a -bug report on GitHub if you think it would be worthwhile to write the -heuristics to handle these encodings.) +## Command-line usage -### Command-line usage +ftfy installs itself as a command line tool that reads a file and applies +`fix_text` to it. -ftfy now installs itself as a command line tool, for cleaning up text files -that are a mess of mangled bytes. +This has exactly the problem described above: a file on a disk is made of bytes +in an unspecified encoding. It could assume the file is UTF-8, but if you had +totally valid UTF-8 you probably wouldn't need this command line utility, and +there's a slight chance that the file could contain Latin-1 that coincidentally +looks like UTF-8. + +Instead, it will follow the "half-measure" above. You can type `ftfy FILENAME`, and it will read in FILENAME as Latin-1 text, fix -encoding problems (including re-interpreting it as UTF-8 if appropriate), and -write the result to standard out as UTF-8. +everything that `fix_text` fixes (including re-interpreting it as UTF-8 if +appropriate), and write the result to standard out as UTF-8. + +This is not necessarily a good idea, but it's convenient. Consider this a proof +of concept until we get a real encoding detector. + +## Who maintains ftfy? + +I'm Rob Speer (rob@luminoso.com). I develop this tool as part of my +text-understanding company, [Luminoso](http://luminoso.com), where it has +proven essential. + +Luminoso provides ftfy as free, open source software under the extremely +permissive MIT license. -Author: Rob Speer, rob@luminoso.com +You can report bugs regarding ftfy on GitHub and we'll handle them. From 30d2895edfba22909247ba761b52af4d061cba09 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 26 Aug 2013 11:38:33 -0400 Subject: [PATCH 073/502] remove extraneous print statement from tests --- ftfy/test_unicode.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ftfy/test_unicode.py b/ftfy/test_unicode.py index f69084df..170caf29 100644 --- a/ftfy/test_unicode.py +++ b/ftfy/test_unicode.py @@ -22,7 +22,6 @@ def test_all_bmp_characters(): # These phrases should not be erroneously "fixed" def test_valid_phrases(): for phrase in phrases: - print phrase yield check_phrase, phrase # make it not just confirm based on the opening punctuation yield check_phrase, phrase[1:] From b208b7e98d03ad3560e5d75cba86352382d9f755 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 26 Aug 2013 11:40:25 -0400 Subject: [PATCH 074/502] make tests run on Python 3 again --- ftfy/test_unicode.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ftfy/test_unicode.py b/ftfy/test_unicode.py index 170caf29..b8ac5e53 100644 --- a/ftfy/test_unicode.py +++ b/ftfy/test_unicode.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- -from ftfy import fix_bad_encoding, WINDOWS_1252_GREMLINS +from ftfy import fix_bad_encoding import unicodedata +import sys + +if sys.hexversion >= 0x03000000: + xrange = range + unichr = chr # Most single-character strings which have been misencoded should be restored. def test_all_bmp_characters(): From f6375292a59d3294321346703293225a53ebee06 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 26 Aug 2013 12:32:59 -0400 Subject: [PATCH 075/502] add changelog --- CHANGELOG.txt | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 CHANGELOG.txt diff --git a/CHANGELOG.txt b/CHANGELOG.txt new file mode 100644 index 00000000..1433d36e --- /dev/null +++ b/CHANGELOG.txt @@ -0,0 +1,31 @@ +## Version 3.0 (August 26, 2013) + +- Generally runs faster +- Idempotent +- Simplified decoding logic +- Understands more encodings and more kinds of mistakes +- Takes options that enable or disable particular normalization steps +- Long line handling: now the time-consuming step (fix_text_encoding) will be + consistently skipped on long lines, but all other fixes will apply +- Tested on millions of examples from Twitter, ensuring a near-zero rate of + false positives + +## Version 2.0.2 (June 20, 2013) + +- Fix breaking up of long lines, so it can't go into an infinite loop + +## Version 2.0.1 (March 19, 2013) + +- Restored Python 2.6 support + +## Version 2.0 (January 30, 2013) + +- Python 3 support +- Use fast Python built-ins to speed up fixes +- Bugfixes + +## Version 1.0 (August 24, 2012) + +- Made into its own package with no dependencies, instead of a part of + `metanl` + From bf0bf94ede821323364f5141a96734aa89de0cc9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 26 Aug 2013 14:09:45 -0400 Subject: [PATCH 076/502] version 3.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6f88458a..3fb4a799 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.0b', + version='3.0', maintainer='Luminoso, LLC', maintainer_email='dev@lumino.so', license="MIT", From f521c32bc99d8d61343486bcf035e376d4d6aa50 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 26 Aug 2013 17:18:13 -0400 Subject: [PATCH 077/502] Raise an error for non-Unicode text in the main method. This is better than an unexplained error about decoding ASCII. --- ftfy/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 5fd3f715..57360961 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -93,6 +93,9 @@ def fix_text(text, encoding is possibly flawed), and do not mind performing operations on the whole text at once, use `fix_text_segment`. """ + if isinstance(text, bytes): + raise UnicodeError(fixes.BYTES_ERROR_TEXT) + out = [] pos = 0 while pos < len(text): From 3eca9020a7636d168eb4235ee0974c79d95d9f71 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 26 Aug 2013 17:58:53 -0400 Subject: [PATCH 078/502] Try adding those pypipin badges. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index a75b2c2f..4172692e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # ftfy: fixes text for you +[![Downloads](https://pypip.in/d/ftfy/badge.png)](https://crate.io/package/ftfy) +[![Version](https://pypip.in/v/ftfy/badge.png)](https://crate.io/package/ftfy) + This is a module for making text less broken and more consistent. It works in Python 2.6, Python 3.2, or later. From 9e503b3c82204d456f7f615df917f3604fc77fd6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 26 Aug 2013 18:00:32 -0400 Subject: [PATCH 079/502] Fix the link target on the badges. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4172692e..6a2c411a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # ftfy: fixes text for you -[![Downloads](https://pypip.in/d/ftfy/badge.png)](https://crate.io/package/ftfy) -[![Version](https://pypip.in/v/ftfy/badge.png)](https://crate.io/package/ftfy) +[![Downloads](https://pypip.in/d/ftfy/badge.png)](https://crate.io/packages/ftfy) +[![Version](https://pypip.in/v/ftfy/badge.png)](https://crate.io/packages/ftfy) This is a module for making text less broken and more consistent. It works in Python 2.6, Python 3.2, or later. From dd96906c929c9251fc2d6440f46f81a3c76b7baf Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 28 Aug 2013 04:28:41 -0400 Subject: [PATCH 080/502] added Sphinx documentation --- .gitignore | 1 + docs/Makefile | 177 +++++++++++++++++++++++++++++++++ docs/conf.py | 248 +++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 157 ++++++++++++++++++++++++++++++ ftfy/__init__.py | 68 +++++++------ ftfy/badness.py | 31 ++++-- ftfy/chardata.py | 4 +- ftfy/fixes.py | 65 +++++++++---- 8 files changed, 693 insertions(+), 58 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.rst diff --git a/.gitignore b/.gitignore index 452b164d..402af6e9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __pycache__ dist *.egg-info build +_build diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..e828bc69 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,177 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ftfy.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ftfy.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/ftfy" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ftfy" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..3da434ad --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- +# +# ftfy documentation build configuration file, created by +# sphinx-quickstart on Wed Aug 28 03:18:27 2013. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'ftfy' +copyright = u'2013, Rob Speer' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '3.0' +# The full version, including alpha/beta/rc tags. +release = '3.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +default_role = 'code' + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'nature' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'ftfydoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'ftfy.tex', u'ftfy Documentation', + u'Rob Speer', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'ftfy', u'ftfy Documentation', + [u'Rob Speer'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'ftfy', u'ftfy Documentation', + u'Rob Speer', 'ftfy', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..816b0905 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,157 @@ +.. ftfy documentation master file, created by + sphinx-quickstart on Wed Aug 28 03:18:27 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +ftfy: fixes text for you +======================== + +This is a module for making text less broken and more consistent. It works in +Python 2.6, Python 3.2, or later. + +The most interesting kind of brokenness that this resolves is when someone +has encoded Unicode with one standard and decoded it with a different one. +This often shows up as characters that turn into nonsense sequences: + +- The word `schön` might appear as `schön`. +- An em dash (`—`) might appear as `—`. +- Text that was meant to be enclosed in quotation marks might end up + instead enclosed in `“` and `”` (and that last character + probably won't even display as anything meaningful). + +This happens very often to real text. Fortunately, the nonsense sequences +usually contain all the information you need to reconstruct what character was +supposed to be there. + +Any given text string might have other irritating properties, possibly even +interacting with the erroneous decoding: + +- The text could contain HTML entities such as `&` in place of certain + characters, when you would rather see what the characters actually are. +- For that matter, it could contain instructions for a text terminal to + do something like change colors, but you are not sending the text to a + terminal, so those instructions are just going to look like `^[[30m;` + or something in the middle of the text. +- The text could write words in non-standard ways for display purposes, + such as using the three characters `fl` `o` `p` for the word "flop". + This can happen when you copy text out of a PDF, for example. + +Of course you're better off if all the text you take as input is decoded +properly and written in standard ways. But often, your input is something you +have no control over. Somebody else's minor mistake becomes your problem. + +ftfy will do everything it can to fix the problem. + +Using ftfy +---------- + +The main function, `fix_text`, will run text through a sequence of fixes. If +the text changed, it will run them through again, so that you can be sure +the output ends up in a standard form that will be unchanged by `fix_text`. + +All the fixes are on by default, but you can pass options to turn them off. + +.. autofunction:: ftfy.fix_text + +Encodings ftfy can handle +------------------------- + +ftfy can understand text that was decoded as any of these single-byte +encodings: + +- Latin-1 (ISO-8859-1) +- Windows-1252 (cp1252 -- used in Microsoft products) +- Windows-1251 (cp1251 -- the Russian version of cp1252) +- MacRoman (used on Mac OS 9 and earlier) +- cp437 (used in MS-DOS) + +when it was actually intended to be decoded as one of these variable-length +encodings: + +- UTF-8 +- CESU-8 (what some people think is UTF-8) + +It can also understand text that was intended as Windows-1252 but decoded as +Latin-1. That's the very common case where things like smart-quotes and +bullets turn into single weird control characters. + +However, ftfy cannot understand other mixups between single-byte encodings, +because it is extremely difficult to detect which mixup in particular is the +one that happened. + +Non-Unicode strings +------------------- + +When first using ftfy, you might be confused to find that you can't give it a +bytestring (the type of object called `str` in Python 2). + +ftfy fixes text. Treating bytestrings as text is exactly the kind of thing that +causes the Unicode problems that ftfy has to fix. So if you don't give it a +Unicode string, ftfy will point you to the `Python Unicode HOWTO`_. + +.. _`Python Unicode Howto`: http://docs.python.org/3/howto/unicode.html + +Reasonable ways that you might exchange data, such as JSON or XML, already have +perfectly good ways of expressing Unicode strings. Given a Unicode string, ftfy +can apply fixes that are very likely to work without false positives. + +But what if you all you actually have is a mess of bytes on a disk? Well, +you've got a problem, and ftfy is not quite the right tool to solve it. + +As a sort of half-measure that covers a few common cases, you can decode the +bytes as Latin-1 and let ftfy take it from there, which might include +reinterpreting the Latin-1 text as Windows-1252 or UTF-8. + + >>> print(fix_text(b'\x85test')) + UnicodeError: [informative error message] + + >>> print(fix_text(b'\x85test'.decode('latin-1'))) + —test + +A note on encoding detection +---------------------------- + +If your input is a mess of unmarked bytes, you might want a tool that can just +statistically analyze those bytes and predict what encoding they're in. + +ftfy is not that tool. I might want to write that tool someday. + +You may have heard of chardet. Chardet is admirable, but it is not that tool +either. Its heuristics only work on multi-byte encodings, such as UTF-8 and the +language-specific encodings used in East Asian languages. It works very badly +on single-byte encodings, to the point where it will output wrong answers with +high confidence. + +There is lots of real-world text that's in an unknown single-byte encoding. +There might be enough information to statistically sort out which encoding is +which. But nothing, so far, actually does that. + + +Module documentation +==================== + +`ftfy.fixes`: how individual fixes are implemented +-------------------------------------------------- + +.. automodule:: ftfy.fixes + :members: + + +`ftfy.badness`: measures the "badness" of text +---------------------------------------------- + +.. automodule:: ftfy.badness + :members: + + +`ftfy.chardata` and `ftfy.build_data`: trivia about characters +-------------------------------------------------------------- +These files load information about the character properties in Unicode 6.1. +Yes, even if your version of Python doesn't support Unicode 6.1. This ensures +that ftfy's behavior is consistent across versions. + +.. automodule:: ftfy.chardata + :members: + +.. autofunction:: ftfy.build_data.make_char_data_file + diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 57360961..ba827b36 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -23,9 +23,41 @@ def fix_text(text, remove_control_chars=True, remove_bom=True, max_decode_length=2**16): - """ + ur""" Given Unicode text as input, make its representation consistent and - possibly less broken, by applying these steps in order: + possibly less broken. + + Let's start with some examples: + + >>> print(fix_text('ünicode')) + ünicode + + >>> print(fix_text('Broken text… it’s flubberific!')) + Broken text... it's flubberific! + + >>> print(fix_text('HTML entities <3')) + HTML entities <3 + + >>> print(fix_text('HTML entities <3')) + HTML entities <3 + + >>> print(fix_text('\001\033[36;44mI’m blue, da ba dee da ba ' + ... 'doo…\033[0m')) + I'm blue, da ba dee da ba doo... + + >>> # This example string starts with a byte-order mark, even if you can't + >>> # see it on the Web. + >>> print(fix_text('\ufeffParty like\nit’s 1999!')) + Party like + it's 1999! + + >>> len(fix_text('fi' * 100000)) + 200000 + + >>> len(fix_text('')) + 0 + + Based on the options you provide, ftfy applies these steps in order: - If `remove_terminal_escapes` is True, remove sequences of bytes that are instructions for Unix terminals, such as the codes that make text appear @@ -40,6 +72,7 @@ def fix_text(text, - If `normalization` is not None, apply the specified form of Unicode normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. The default, 'NFKC', applies the following relevant transformations: + - C: Combine characters and diacritics that are written using separate code points, such as converting "e" plus an acute accent modifier into "é", or converting "ka" (か) plus a dakuten into the @@ -49,6 +82,7 @@ def fix_text(text, full-width versions, full-width Roman characters will be replaced with ASCII characters, ellipsis characters will be replaced with three periods, and the ligature 'fl' will be replaced with 'fl'. + - If `uncurl_quotes` is True, replace various curly quotation marks with plain-ASCII straight quotes. - If `fix_line_breaks` is true, convert all line breaks to Unix style @@ -60,34 +94,10 @@ def fix_text(text, - If anything was changed, repeat all the steps, so that the function is idempotent. "&amp;" will become "&", for example, not "&". - >>> print(fix_text('ünicode')) - ünicode - - >>> print(fix_text('Broken text… it’s flubberific!')) - Broken text... it's flubberific! - - >>> print(fix_text('HTML entities <3')) - HTML entities <3 - - >>> print(fix_text('HTML entities <3')) - HTML entities <3 - - >>> print(fix_text('\001\033[36;44mI’m blue, da ba dee da ba ' - ... 'doo…\033[0m')) - I'm blue, da ba dee da ba doo... - - >>> print(fix_text('\\ufeffParty like\\nit’s 1999!')) - Party like - it's 1999! - - >>> len(fix_text('fi' * 100000)) - 200000 - - >>> len(fix_text('')) - 0 - `fix_text` will work one line at a time, with the possibility that some - lines are in different encodings. + lines are in different encodings. When it encounters lines longer than + `max_decode_length`, it will not run the `fix_encoding` step, to avoid + unbounded slowdowns. If you are certain your entire text is in the same encoding (though that encoding is possibly flawed), and do not mind performing operations on diff --git a/ftfy/badness.py b/ftfy/badness.py index 8ba615b2..95f31b6c 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -89,22 +89,35 @@ def sequence_weirdness(text): We start by normalizing text in NFC form, so that penalties for diacritical marks don't apply to characters that know what to do with them. - """ - text2 = unicodedata.normalize('NFC', text) - return len(WEIRDNESS_RE.findall(chars_to_classes(text2))) + The following things are deemed weird: -def better_text(newtext, oldtext): - """ - Is newtext better than oldtext? + - Lowercase letters followed by non-ASCII uppercase letters + - Non-Latin characters next to Latin characters + - Un-combined diacritical marks, unless they're stacking on non-alphabetic + characters (in languages that do that kind of thing a lot) or other + marks + - C1 control characters + - Adjacent symbols from any different pair of these categories: + + - Modifier marks + - Letter modifiers + - Non-digit numbers + - Symbols (including math and currency) + + The return value is the number of instances of weirdness. """ - return text_cost(newtext) < text_cost(oldtext) + text2 = unicodedata.normalize('NFC', text) + return len(WEIRDNESS_RE.findall(chars_to_classes(text2))) def text_cost(text): """ - An overall cost function for text. All else being equal, shorter strings - are better. + An overall cost function for text. Weirder is worse, but all else being + equal, shorter strings are better. + + The overall cost is measured as twice the "weirdness" + (see :func:`sequence_weirdness`) plus the length. """ return sequence_weirdness(text) * 2 + len(text) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 580271a6..0d8f403b 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -This file contains the gritty details about characters and the encodings that -use them. +This gives other modules access to the gritty details about characters and the +encodings that use them. """ from __future__ import unicode_literals diff --git a/ftfy/fixes.py b/ftfy/fixes.py index f6c161e0..9d5c905b 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -32,7 +32,7 @@ def fix_text_encoding(text): - """ + r""" Something you will find all over the place, in real-world text, is text that's mistakenly encoded as utf-8, decoded in some ugly format like latin-1 or even Windows codepage 1252, and encoded as utf-8 again. @@ -49,9 +49,13 @@ def fix_text_encoding(text): The input to the function must be Unicode. If you don't have Unicode text, you're not using the right tool to solve your problem. - NOTE: the following examples are written using unmarked literal strings, - but they are Unicode text. In Python 2 we have "unicode_literals" turned - on, and in Python 3 this is always the case. + .. note:: + The following examples are written using unmarked literal strings, + but they are Unicode text. In Python 2 we have "unicode_literals" turned + on, and in Python 3 this is always the case. + + ftfy decodes text that looks like it was decoded incorrectly. It leaves + alone text that doesn't. >>> print(fix_text_encoding('único')) único @@ -68,7 +72,8 @@ def fix_text_encoding(text): We might have to deal with both Windows characters and raw control characters at the same time, especially when dealing with characters like - \x81 that have no mapping in Windows. + \x81 that have no mapping in Windows. This is a string that Python's + standard `.encode` and `.decode` methods cannot correct. >>> print(fix_text_encoding('This text is sad .â\x81”.')) This text is sad .⁔. @@ -91,6 +96,9 @@ def fix_text_encoding(text): >>> print(fix_text_encoding('This text was never UTF-8 at all\x85')) This text was never UTF-8 at all… + + The best version of the text is found using + :func:`ftfy.badness.text_cost`. """ best_version = text best_cost = text_cost(text) @@ -251,9 +259,14 @@ def fixup(match): ANSI_RE = re.compile('\033\\[((?:\\d|;)*)([a-zA-Z])') def remove_terminal_escapes(text): - """ + r""" Strip out "ANSI" terminal escape sequences, such as those that produce colored text on Unix. + + >>> print(remove_terminal_escapes( + ... "\033[36;44mI'm blue, da ba dee da ba doo...\033[0m" + ... )) + I'm blue, da ba dee da ba doo... """ return ANSI_RE.sub('', text) @@ -261,7 +274,7 @@ def remove_terminal_escapes(text): SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') def uncurl_quotes(text): - """ + r""" Replace curly quotation marks with straight equivalents. >>> print(uncurl_quotes('\u201chere\u2019s a test\u201d')) @@ -274,31 +287,38 @@ def fix_line_breaks(text): r""" Convert line breaks to Unix style. - In particular, this replaces CRLF (\r\n) with LF (\n), then - additionally replaces CR (\r) with LF (\n). + In particular, this replaces CRLF (\\r\\n) with LF (\\n), then + additionally replaces CR (\\r) with LF (\\n). """ return text.replace('\r\n', '\n').replace('\r', '\n') def remove_control_chars(text): - r""" + """ Remove all control characters except for the important ones. - This removes characters from U+0000 to U+001F and U+0080 to U+009F, except + This removes characters in these ranges: + + - U+0000 to U+0008 + - U+000B + - U+000E to U+001F + - U+007F + it leaves alone these characters that are commonly used for formatting: - - TAB (\t, U+0009) - - LF (\n, U+000A) - - FF (\f, U+000C) - - CR (\r, U+000D) + - TAB (U+0009) + - LF (U+000A) + - FF (U+000C) + - CR (U+000D) """ return text.translate(CONTROL_CHARS) CESU8_RE = re.compile(b'\xed[\xa0-\xaf][\x80-\xbf]\xed[\xb0-\xbf][\x80-\xbf]') def fix_java_encoding(bytestring): - """ - Convert a bytestring that might contain "Java UTF8" into valid UTF-8. + r""" + Convert a bytestring that might contain "Java Modified UTF8" into valid + UTF-8. There are two things that Java is known to do with its "UTF8" encoder that are incompatible with UTF-8. (If you happen to be writing Java @@ -314,6 +334,12 @@ def fix_java_encoding(bytestring): outputting a null byte by breaking the UTF shortest-form rule. Unicode does not even deign to give this scheme a name, and no version of Python will decode it. + + >>> fix_java_encoding(b'\xed\xa0\xbd\xed\xb8\x8d') + b'\xf0\x9f\x98\x8d' + + >>> fix_java_encoding(b'Here comes a null! \xc0\x80') + b'Here comes a null! \x00' """ assert isinstance(bytestring, bytes) # Replace the sloppy encoding of U+0000 with the correct one. @@ -354,7 +380,10 @@ def fix_java_encoding(bytestring): def remove_bom(text): - """ + r""" Remove a left-over byte-order mark. + + >>> print(remove_bom(unichr(0xfeff) + "Where do you want to go today?")) + Where do you want to go today? """ return text.lstrip(unichr(0xfeff)) From 7f95bdd6d3d85855f1e66d489062bca1210900bb Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 28 Aug 2013 04:44:46 -0400 Subject: [PATCH 081/502] link to readthedocs --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 6a2c411a..63f2d3d0 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ [![Downloads](https://pypip.in/d/ftfy/badge.png)](https://crate.io/packages/ftfy) [![Version](https://pypip.in/v/ftfy/badge.png)](https://crate.io/packages/ftfy) +Full documentation: **http://ftfy.readthedocs.org** + This is a module for making text less broken and more consistent. It works in Python 2.6, Python 3.2, or later. @@ -208,3 +210,11 @@ Luminoso provides ftfy as free, open source software under the extremely permissive MIT license. You can report bugs regarding ftfy on GitHub and we'll handle them. + +## More details + +To learn how the pieces of ftfy work: + +- You could just read the code. I recommend it. There are copious comments and + everything has a docstring. +- You can read nicely-formatted documentation at http://ftfy.readthedocs.org. From c84e332229855d694a54e2f7f6ac0688ffe14043 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 28 Aug 2013 11:32:19 -0400 Subject: [PATCH 082/502] Fixed doctests and Python 2/3 compatibility --- ftfy/__init__.py | 2 +- ftfy/compatibility.py | 10 +++++----- ftfy/fixes.py | 9 +++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index ba827b36..7bd9662d 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -23,7 +23,7 @@ def fix_text(text, remove_control_chars=True, remove_bom=True, max_decode_length=2**16): - ur""" + r""" Given Unicode text as input, make its representation consistent and possibly less broken. diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index 6c894c5e..c6462586 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -9,12 +9,12 @@ htmlentitydefs = entities unichr = chr xrange = range - PYTHON3 = True + PYTHON2 = False else: import htmlentitydefs unichr = unichr xrange = xrange - PYTHON3 = False + PYTHON2 = True def _narrow_unichr_workaround(codepoint): @@ -36,7 +36,7 @@ def bytes_to_ints(bytestring): a bytestring. On Python 3, this is easy, because a 'bytes' object _is_ a sequence of integers. """ - if PYTHON3: - return bytestring - else: + if PYTHON2: return [ord(b) for b in bytestring] + else: + return bytestring diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 9d5c905b..517ea9d6 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -335,11 +335,12 @@ def fix_java_encoding(bytestring): Unicode does not even deign to give this scheme a name, and no version of Python will decode it. - >>> fix_java_encoding(b'\xed\xa0\xbd\xed\xb8\x8d') - b'\xf0\x9f\x98\x8d' + >>> len(fix_java_encoding(b'\xed\xa0\xbd\xed\xb8\x8d')) + 4 - >>> fix_java_encoding(b'Here comes a null! \xc0\x80') - b'Here comes a null! \x00' + >>> ends_with_null = fix_java_encoding(b'Here comes a null! \xc0\x80') + >>> bytes_to_ints(ends_with_null)[-1] + 0 """ assert isinstance(bytestring, bytes) # Replace the sloppy encoding of U+0000 with the correct one. From 97931f04fbe0ae1fa2a1e8e043a624f030846ed5 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 28 Aug 2013 13:50:35 -0400 Subject: [PATCH 083/502] Update docs, including slightly tongue-in-cheek testimonials --- README.md | 23 +++++++++++++++++++---- docs/index.rst | 12 ++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 63f2d3d0..d7a07806 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,22 @@ Full documentation: **http://ftfy.readthedocs.org** -This is a module for making text less broken and more consistent. It works in +## Testimonials + +- “My life is livable again!” + — [@planarrowspace](http://twitter.com/planarrowspace) +- “A handy piece of magic” + — [@simonw](http://twitter.com/simonw) +- “This package might help you skip the legwork?” + — [@batouchu](http://twitter.com/batouchu) +- “I have no idea when I’m gonna need this, but I’m definitely bookmarking it.” + — [/u/ocrow](http://reddit.com/u/ocrow) +- “9.64/10” + — [pylint](https://bitbucket.org/logilab/pylint/) + +## What it does + +ftfy makes Unicode text less broken and more consistent. It works in Python 2.6, Python 3.2, or later. The most interesting kind of brokenness that this resolves is when someone @@ -18,9 +33,9 @@ This often shows up as characters that turn into nonsense sequences: instead enclosed in `“` and `”` (and that last character probably won't even display as anything meaningful). -This happens very often to real text. Fortunately, the nonsense sequences -usually contain all the information you need to reconstruct what character was -supposed to be there. +This is called "mojibake", and it happens very often to real text. Fortunately, +the nonsense sequences usually contain all the information you need to +reconstruct what character was supposed to be there. Any given text string might have other irritating properties, possibly even interacting with the erroneous decoding: diff --git a/docs/index.rst b/docs/index.rst index 816b0905..e443840f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,10 +13,10 @@ The most interesting kind of brokenness that this resolves is when someone has encoded Unicode with one standard and decoded it with a different one. This often shows up as characters that turn into nonsense sequences: -- The word `schön` might appear as `schön`. -- An em dash (`—`) might appear as `—`. +- The word ``schön`` might appear as ``schön``. +- An em dash (``—``) might appear as ``—``. - Text that was meant to be enclosed in quotation marks might end up - instead enclosed in `“` and `”` (and that last character + instead enclosed in ``“`` and ``”`` (and that last character probably won't even display as anything meaningful). This happens very often to real text. Fortunately, the nonsense sequences @@ -26,14 +26,14 @@ supposed to be there. Any given text string might have other irritating properties, possibly even interacting with the erroneous decoding: -- The text could contain HTML entities such as `&` in place of certain +- The text could contain HTML entities such as ``&`` in place of certain characters, when you would rather see what the characters actually are. - For that matter, it could contain instructions for a text terminal to do something like change colors, but you are not sending the text to a - terminal, so those instructions are just going to look like `^[[30m;` + terminal, so those instructions are just going to look like ``^[[30m;`` or something in the middle of the text. - The text could write words in non-standard ways for display purposes, - such as using the three characters `fl` `o` `p` for the word "flop". + such as using the three characters ``fl`` ``o`` ``p`` for the word "flop". This can happen when you copy text out of a PDF, for example. Of course you're better off if all the text you take as input is decoded From db90a7647df09279ca557a9d447b677605dc19f1 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 28 Aug 2013 13:56:17 -0400 Subject: [PATCH 084/502] formatting fixes and stuff, German quote, etc. --- README.md | 7 +++---- ftfy/__init__.py | 8 ++++---- ftfy/fixes.py | 12 ++++++------ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d7a07806..e8a26d31 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,12 @@ Full documentation: **http://ftfy.readthedocs.org** — [@planarrowspace](http://twitter.com/planarrowspace) - “A handy piece of magic” — [@simonw](http://twitter.com/simonw) -- “This package might help you skip the legwork?” - — [@batouchu](http://twitter.com/batouchu) +- “Hat mir die Tage geholfen. Im Übrigen bin ich der Meinung, dass wir keine komplexen Maschinen mit Computern bauen sollten solange wir nicht einmal Umlaute sicher verarbeiten können. :D” + — [Bruno Ranieri](http://yrrsinn.de/2012/09/17/gelesen-kw37/) - “I have no idea when I’m gonna need this, but I’m definitely bookmarking it.” — [/u/ocrow](http://reddit.com/u/ocrow) -- “9.64/10” +- “9.41/10” — [pylint](https://bitbucket.org/logilab/pylint/) - ## What it does ftfy makes Unicode text less broken and more consistent. It works in diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 7bd9662d..b675ab5c 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -26,9 +26,9 @@ def fix_text(text, r""" Given Unicode text as input, make its representation consistent and possibly less broken. - + Let's start with some examples: - + >>> print(fix_text('ünicode')) ünicode @@ -142,7 +142,7 @@ def fix_text(text, ftfy = fix_text -def fix_file(file, normalization='NFKC'): +def fix_file(input_file, normalization='NFKC'): """ Fix text that is found in a file. @@ -156,7 +156,7 @@ def fix_file(file, normalization='NFKC'): The output is a stream of fixed lines of text. """ entities = True - for line in file: + for line in input_file: if isinstance(line, bytes): line = line.decode('latin-1') if '<' in line and '>' in line: diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 517ea9d6..42749764 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -262,7 +262,7 @@ def remove_terminal_escapes(text): r""" Strip out "ANSI" terminal escape sequences, such as those that produce colored text on Unix. - + >>> print(remove_terminal_escapes( ... "\033[36;44mI'm blue, da ba dee da ba doo...\033[0m" ... )) @@ -271,8 +271,8 @@ def remove_terminal_escapes(text): return ANSI_RE.sub('', text) -SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') -DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') +SINGLE_QUOTE_RE = re.compile(u'[\u2018-\u201b]') +DOUBLE_QUOTE_RE = re.compile(u'[\u201c-\u201f]') def uncurl_quotes(text): r""" Replace curly quotation marks with straight equivalents. @@ -303,8 +303,8 @@ def remove_control_chars(text): - U+000B - U+000E to U+001F - U+007F - - it leaves alone these characters that are commonly used for formatting: + + It leaves alone these characters that are commonly used for formatting: - TAB (U+0009) - LF (U+000A) @@ -337,7 +337,7 @@ def fix_java_encoding(bytestring): >>> len(fix_java_encoding(b'\xed\xa0\xbd\xed\xb8\x8d')) 4 - + >>> ends_with_null = fix_java_encoding(b'Here comes a null! \xc0\x80') >>> bytes_to_ints(ends_with_null)[-1] 0 From e4c680edd49fc9b2403e1b93afeb0bd3552bd76b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 28 Aug 2013 14:19:52 -0400 Subject: [PATCH 085/502] fix markdown formatting --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e8a26d31..39cb5297 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Full documentation: **http://ftfy.readthedocs.org** — [/u/ocrow](http://reddit.com/u/ocrow) - “9.41/10” — [pylint](https://bitbucket.org/logilab/pylint/) + ## What it does ftfy makes Unicode text less broken and more consistent. It works in From dca826260f911806fee2040347895d67dbac9d90 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 28 Aug 2013 20:31:06 -0400 Subject: [PATCH 086/502] fix false negative in fix_java_encoding loop Previously, fix_java_encoding would only fix the first CESU-8-encoded character, leaving the rest untouched, because the loop wasn't looping correctly. Now it should. --- ftfy/fixes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 42749764..7eecc919 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -375,7 +375,7 @@ def fix_java_encoding(bytestring): new_bytes = unichr(codepoint).encode('utf-8') fixed_pieces.append(bytestring[:pos] + new_bytes) bytestring = bytestring[pos + 6:] - match = CESU8_RE.match(bytestring) + match = CESU8_RE.search(bytestring) return b''.join(fixed_pieces) + bytestring From e4f2c62316b3a3514d8712f34978e4a89f974b38 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 29 Aug 2013 13:09:31 -0400 Subject: [PATCH 087/502] Include a fixer that works around a Python 3.3 bug. --- README.md | 10 +++++++--- ftfy/__init__.py | 17 ++++++++++++++--- ftfy/compatibility.py | 2 ++ ftfy/fixes.py | 32 ++++++++++++++++++++++++++++++-- setup.py | 2 +- 5 files changed, 54 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 39cb5297..d4d49a17 100644 --- a/README.md +++ b/README.md @@ -91,13 +91,17 @@ the output ends up in a standard form that will be unchanged by `fix_text`. All the fixes are on by default, but you can pass options to turn them off. -- If `remove_terminal_escapes` is True, remove sequences of bytes that are - instructions for Unix terminals, such as the codes that make text appear - in different colors. +- If `remove_unsafe_private_use` is True, remove a range of unassigned + characters that can crash Python via + [bug 18183](http://bugs.python.org/issue18183). This fix will turn itself + off when you're using Python 3.4 or better, which you probably aren't. - If `fix_entities` is True, consider replacing HTML entities with their equivalent characters. However, this never applies to text with a pair of angle brackets in it already; you're probably not supposed to decode entities there, and you'd make things ambiguous if you did. +- If `remove_terminal_escapes` is True, remove sequences of bytes that are + instructions for Unix terminals, such as the codes that make text appear + in different colors. - If `fix_encoding` is True, look for common mistakes that come from encoding or decoding Unicode text incorrectly, and fix them if they are reasonably fixable. diff --git a/ftfy/__init__.py b/ftfy/__init__.py index b675ab5c..86eb110a 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -9,11 +9,13 @@ from __future__ import unicode_literals from ftfy import fixes from ftfy.fixes import fix_text_encoding +from ftfy.compatibility import PYTHON34_OR_LATER import unicodedata import warnings def fix_text(text, + remove_unsafe_private_use=(not PYTHON34_OR_LATER), fix_entities=True, remove_terminal_escapes=True, fix_encoding=True, @@ -59,13 +61,16 @@ def fix_text(text, Based on the options you provide, ftfy applies these steps in order: - - If `remove_terminal_escapes` is True, remove sequences of bytes that are - instructions for Unix terminals, such as the codes that make text appear - in different colors. + - If `remove_unsafe_private_use` is True, it removes a range of unassigned + characters that could trigger Python `bug 18183`_. This will default + to False starting on Python 3.4, when the bugfix will be released. - If `fix_entities` is True, consider replacing HTML entities with their equivalent characters. However, this never applies to text with a pair of angle brackets in it already; you're probably not supposed to decode entities there, and you'd make things ambiguous if you did. + - If `remove_terminal_escapes` is True, remove sequences of bytes that are + instructions for Unix terminals, such as the codes that make text appear + in different colors. - If `fix_encoding` is True, look for common mistakes that come from encoding or decoding Unicode text incorrectly, and fix them if they are reasonably fixable. See `fix_text_encoding` for details. @@ -102,6 +107,8 @@ def fix_text(text, If you are certain your entire text is in the same encoding (though that encoding is possibly flawed), and do not mind performing operations on the whole text at once, use `fix_text_segment`. + + _`bug 18183`: http://bugs.python.org/issue18183 """ if isinstance(text, bytes): raise UnicodeError(fixes.BYTES_ERROR_TEXT) @@ -125,6 +132,7 @@ def fix_text(text, out.append( fix_text_segment( substring, + remove_unsafe_private_use=remove_unsafe_private_use, fix_entities=fix_entities, remove_terminal_escapes=remove_terminal_escapes, fix_encoding=fix_encoding_this_time, @@ -165,6 +173,7 @@ def fix_file(input_file, normalization='NFKC'): def fix_text_segment(text, + remove_unsafe_private_use=True, fix_entities=True, remove_terminal_escapes=True, fix_encoding=True, @@ -186,6 +195,8 @@ def fix_text_segment(text, unsafe_entities = ('<' in text and '>' in text) while True: origtext = text + if remove_unsafe_private_use: + text = fixes.remove_unsafe_private_use(text) if fix_entities and not unsafe_entities: text = fixes.unescape_html(text) if remove_terminal_escapes: diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index c6462586..fb225b6a 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -16,6 +16,8 @@ xrange = xrange PYTHON2 = True +PYTHON34_OR_LATER = (sys.hexversion >= 0x03040000) + def _narrow_unichr_workaround(codepoint): """ diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 7eecc919..457cdf0a 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -271,8 +271,8 @@ def remove_terminal_escapes(text): return ANSI_RE.sub('', text) -SINGLE_QUOTE_RE = re.compile(u'[\u2018-\u201b]') -DOUBLE_QUOTE_RE = re.compile(u'[\u201c-\u201f]') +SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') +DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') def uncurl_quotes(text): r""" Replace curly quotation marks with straight equivalents. @@ -388,3 +388,31 @@ def remove_bom(text): Where do you want to go today? """ return text.lstrip(unichr(0xfeff)) + +UNSAFE_3_3_RE = re.compile('[\U00100000-\U0010ffff]') +def remove_unsafe_private_use(text): + r""" + Python 3.3's Unicode support isn't perfect, and in fact there are certain + string operations that will crash it with a SystemError: + http://bugs.python.org/issue18183 + + You can trigger the bug by running `` '\U00010000\U00100000'.lower() ``. + + The best solution on Python 3.3 is to remove all characters from + Supplementary Private Use Area B, using a regex that is known not to crash + given those characters. + + These are the characters from U+100000 to U+10FFFF. It's sad to lose an + entire plane of Unicode, but on the other hand, these characters are not + assigned and never will be. If you get one of these characters and don't + know what its purpose is, its purpose is probably to crash your code. + + If you were using these for actual private use, this might be inconvenient. + You can turn off this fixer, of course, but I kind of encourage using + Supplementary Private Use Area A instead. + + >>> print(remove_unsafe_private_use('\U0001F4A9\U00100000')) + 💩 + """ + return UNSAFE_3_3_RE.sub('', text) + diff --git a/setup.py b/setup.py index 3fb4a799..4a8b8ef2 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.0', + version='3.0.1', maintainer='Luminoso, LLC', maintainer_email='dev@lumino.so', license="MIT", From 2b5e3d33ca9289936d0550fd088955ddeb74cb03 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 30 Aug 2013 11:32:43 -0400 Subject: [PATCH 088/502] add another test tweet --- tests/test_real_tweets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index c8f27b1b..1efa6ed3 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -25,6 +25,8 @@ "@rakryanM hihi RT damnitstrue: ☺😘 http://t.co/DqSCy26POe"), ("Beta Haber: Hırsızı Büyü Korkuttu http://t.co/rMkt5yz7Si", "Beta Haber: Hırsızı Büyü Korkuttu http://t.co/rMkt5yz7Si"), + ("Blog Traffic Tip 2 – Broadcast Email Your Blog", + "Blog Traffic Tip 2 – Broadcast Email Your Blog"), ] def test_real_tweets(): From cc94c1b0daf3146ede2b62e8a737570853d478c5 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 30 Aug 2013 11:36:53 -0400 Subject: [PATCH 089/502] version 3.0.1 --- CHANGELOG.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 1433d36e..10ca790d 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,10 @@ +## Version 3.0.1 (August 30, 2013) + +- Fix bug in `fix_java_encoding` that led to only the first instance of + CESU-8 badness per line being fixed +- Add a fixer that removes unassigned characters that can break Python 3.3 + (http://bugs.python.org/issue18183) + ## Version 3.0 (August 26, 2013) - Generally runs faster From 8fc84d68ce092841372298da2ed4b40a6496e04d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 30 Aug 2013 17:44:06 -0400 Subject: [PATCH 090/502] Fix an actual false positive. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The text "Ôôô", which appears to be an interjection that Brazilian Portuguese speakers sometimes use online, appeared to be the MacRoman mis-encoding of a private use character. The heuristics now count private use and unassigned characters (that is, unassigned in Unicode 6.1; sorry, Turkish lira) as adding 1 weirdness. They will still decode, but they will not decode on their own when the rare encoding penalty also applies. Note that this is an update to char_classes.dat. --- ftfy/badness.py | 4 ++++ ftfy/build_data.py | 3 +++ ftfy/char_classes.dat | Bin 3566 -> 3568 bytes setup.py | 2 +- tests/test_real_tweets.py | 1 + 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 95f31b6c..f336bcd8 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -62,6 +62,10 @@ def _make_weirdness_regex(): # decoding Latin-1 that was meant to be Windows-1252. groups.append('X') + # Match private use and unassigned characters. + groups.append('P') + groups.append('_') + # Match adjacent characters from any different pair of these categories: # - Modifier marks (M) # - Letter modifiers (m) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index b97260e4..d9e004b0 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -27,6 +27,7 @@ # m = Letter modifier (Lm) # M = Mark (Mc, Me, Mn) # N = Miscellaneous numbers (No) +# P = Private use (Co) # 0 = Math symbol (Sm) # 1 = Currency symbol (Sc) # 2 = Symbol modifier (Sk) @@ -86,6 +87,8 @@ def make_char_data_file(): cclasses[codepoint] = 'X' elif category == 'Cs': cclasses[codepoint] = 'S' + elif category == 'Co': + cclasses[codepoint] = 'P' elif category.startswith('Z'): cclasses[codepoint] = ' ' else: diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index 1b5871b508bdca3fac5e493043d3da0cc073b66a..84155cd5201b19cbdc66d6a977a356a1bf1732e8 100644 GIT binary patch literal 3568 zcmeHHYg7{07IyC3TaA@Yr8!yZ^wO-9EZ>^eG>uv+llXq2QkMD3%tQgXJ+*vLzAJ&z z@`WPhBN9X!O-WHH2uTr{3^XM~P(TIb-deNP%$hapuK9b{n*HPKv-h|6I_LZLItN-I z=AA68>i>t%EBwMn|G+3mtNa3Jk;~5WORW4doHR2PTep6j!88cV8W?3m>4rt*?N zeP%P9`c{4EAZ>^y%e&!03DnfUOH<(RF^d(r?ayv8y6mdCNd^{Gvu-fe255tEM68Uh z)KxHqEfpzl%Cf_g2V3@Ubu?UNrHeyG>WPmpAf7ZAG_#ft`LtBoW`sJUhcZrO$4TqX zx9m(D$JdckN35sjVoBm5Y&J>!lsj5Vr&z|Z2MRn0P!~uD{LyRD`Mvjv#yeRh^5V%# zz&4f-#S&o3J`^iCXk|_C!v)-1w==_Lz{1N#c5&Jpe)KZZ1=C%yh$k21KXU z8&h1TEw2{pXEeIHoO zw?e-3EXksx7U)>BY%~O-(Pt%>x#zQOBDHTHIX9j? zrSqHK`D)76n{6w)D+2~0VOjf;nbQGCZ@cfEBDFRiMAD|=E9uopv=`MEQN9vo5OQzD zv*4ysl7xf^lSAEa-ECDN_s#afTZyIS{@w>CYq!iZuRMlO!y?8;POGlLhS{5j3i4+A*Bp z#{m3$_QCXA^Fkb*I035eYirmO>vH~7MHccw`OxS*ramH@o6AYOmb4PD#3fxNrb~0{ zl^a93tjUYp=$-bZ(wePsgJ9&d4r?Sk-3~=OMMKq>W z*t(e}@HQ3-!V6yXU>QG>h1iQTU+=9mu3@krR_MtUA$mKpJ;pt6Dh)z+I~k-Fu2_yI zR0j@WD`Rl#>x0`6ItHQzyr*mI71b6QX_>jhNjj(*y(NZ7S-GJbD4lFo47tz>sq9IU z-uZkpL)Yz0;Q@VfnGy5^@>YwBs=F^@lGr?}YU{0;7hW*-Hwz3x$q?nb&5ie0WT_pv zqy5Bpc@74y>$)>Ko4;W@V^!|R#s(J=;ujnFJK&~oeIcSk!x5W~q+GK*a?7%a2rySPiw`G1hBKr_L#=+<_UX{FM$BI*OJK;Lg5)e?`2L z$U`_j&<*i$PeTgwWy28(WTmxRKD~-npAMK_Ejf~0Ua$+Reu8%DK@acgo3gO z^u2r79cQGzAvF~?Ybz>6E}8ONK#2q-Wpy0qti03A2XQVWGDA(O)rJEqyJk-8i)rb~!`Nc3jOrSRR7qZ4q0_&{I=h$8$hg z%N72m@C%6}iDko#(yO&BqXYO-Q&cwSqHB%Usq$kY=aDn*^giFyhoTFe~Q|t z8d%&Kh=lr&`iZW+VbK7d;7AiUJWBGOvW7`lZ_hNb?^xiryN5G2Ph5tn>N?4a)gB6- z@JXaO4xq8-(mG|p5?xX$uo_w`CSvRJkO}_1j>7{i4`~?73^<<@2cMZbeP)?L=3fF% zfy1@k+%0Vf-@(6Hv{V$pQV{P=be`LX0o|_f#wqRk30i4??6rk)e}5gF#VLyF`kFG|ha&;mDO!K5F6>hs~E?WH6^5^`tRP zM0wR9LDS3VP>KOl;T%`w?b41A*2g4Qp^|^-e+{nHLlXK%(iDP0=i_UEN>;bvMD`5> z?Nhtl(0&J)a`DSg=RfDHAri3UHpJ5YtJUa`=va+A4Zy~pG@LZ;yk}Gs*7peQNqT*m z)pV$Q$cq2r0@SZZv-@dQ{nBxMnI7}rOE9wcz;o|#$TE1q`b;7$j+8O-joJ|3;U;*Q zsR_p17aP4D)s+vdXWN=7{R%}BTjIF{wme~bk9ZA@f4SbpGY~#8St#f%MgP@g8u*))so== literal 3566 zcmeHIc~lbk7It2{PFks1X*o^nSfP|=Iw&uD<(T`Pno=5?nvx5W3(%{~#wD7pa0!{o zkQ5PjO96Z35^9=|k`gK+8Xztpq9E{QdFRZTdGpShf8RNC|M;Er-R0ch_uX^Pz4lnb zbxnEF^EaI_@}&(HhvrV?dCW}U6OO|4Rrt_@2Pf3b%S<(L*!%luwriek)T#qjO9%0* z%u`FuWn=dgX8P@weJ-c(_@b1@%J0lsxm~@dskt{nobb8j>A~Qoe2p{)jslh?DaB_s zUacMkyl{fl=- z%i>QD;JQ=nvTRv#2#x*wxF+skIr&g_ z?6r`sni8z*{qxPst}?>sz1AI91mLoVUHc@T9z;R+20b$3)fH;CIB{Rw`brimbcv4`ErpLu7v!u8HdA38Ban|OJAjwtS4w7xbxyqxf+&6bm7R20-%yi|(oPH4=4&8hT zSiGY-Ag=iKLJMu;8g&f=7~qjc;o1m0qk;!U8nOx5ilQ;25e#I%{I7+rvBS&6Q^JFoNv_)D6OUVXtvyAi|e@@6acaVu>cS^#Yg zp_-J92vO^mrLltC2k2%cM;5Wna-}FEDt33xD=~J>1Gs8^>LIn*O$ishURYQ9rfb1D z(fTkNp)x&$W+`2ApK`S!37(UsvUrMe6_`u3i5*C|Q1o8ZQwB$oaw(M>`4R9F#EKEd>D3e0It;abF*~9Eay>R+ zodq%fl7P2pXEaGI2cpC0;5k}ngR}O#D^}(9o@7}SReSdV@OfZd5S_`qc+$H|$|CZi6K`6M zEj`5@D-C~(Ik{v!G}K_qJ+Bc%5lD9;eG_EZ;k>JJS$_OZz~{vIYjB84!NO$u5k8QD zJ4w))t?qmMv9NaH+5Ff*@1B(RPfn;c$AP$pDF@#c{U#F%XDxSyJ>$<^4VxV|@txgQ z-$;BbIPnM>cbM;PyL7Nxb2T+oQ!yK}3)UYGE7yN4}n-#~or zcgJR;tR#NWvg}m0Voun}z3?f9WP(|1@8*g8XxkgpaU`en3*OhnX%uKn8Ovh|Flg_w z!`z@TeuGOQzS+gGZ|t{`H6`-)$VyXLWPL{o&kGlM>5}i8W!t%EjbM#F zho&fAK!21|#_j=;U=!ZO!A8946(H-&7&gN%Z>_{(Isb4yE4;*nk6+J~Xi07}XzZ(8 zf&}Hp;3#T5cyItGcbqFw&Jj0kA}Z(roId#kIJ$e-uf6-yzLoxMHHe!9fhBLS;1PNTUy%>>K8`A{-Zu@1 zhLo#jTRXS?Ah%(Qqkq|ND-Jp|{SoJ_oeQwMi?d*I2I7`N^t>W%O1Q!sTc1 zy9f1frvhS@cOX51{@Eg=F9iiM-5eiLBpAJx-aksDzJQ@e)E?FR7Hk|MoPI0hP2-H? zWB1I;QhYO5dL~s2Xym4w;5(rG!uKuMg`0{ez2ip+B-O>yslW{ca;-v)=o~=?KSB<` zlgQf8GLe0~-^VxOp@A4iql7%V18C0Pc?9(QTexopk6WIFpvFtD3ae`Nw==x>Ltj0? zANdB?nw35J!U0hP?{3Y&%;3r!*{vft>&10xqnI+s`ok0uipxS*VZAQG3C5OeM(K#v z_i|US9!717p?~*n3Rf8&hAy)4-hR^>+s}<-j z6?83!DoslgSK$L8c}Z3zWsSR-nVS_qAs=1v0YixBa;5`z^@QDRHao+*g9fY=OSm7p zW1lZl?xvY zcU(?Ayih7}Hp>)k>2heghwslRf>jGEglxeQo#Ue?i5)mVfV2fPDUz2WIbqcI(z^>L zRpwla#QdDW544Vjxj`F;9ypmXoAz34ysFQzsV=VbQnbx+Pc7mWwSltI-ys{FKt`@? zB!NK|apm%pA|c=8sU<}pKiH|54_hsSYBtgIO~aIukLFcTdI>Ewq~B<$>gFERjP#tP z;Q&_iKoS92CZ1gr+D-JKvKBdgejVWB77=E#lfu#^vJgW|ZL)ThC)FNRk8oklM44rk zmM6&#F1$)#y|OjEllGiGudj2Vn0x^LGJbaC2z+hfp#2$C}VthrMxHrTQby|fTQqncc++CL1XZtmrm!^7OkAsA<%yE0+U3drc ztHi(wD{`1_bULoyr409xF7>Oi?eDbRBltw|&1z*&OxveRBXLhMi7Mg6M8a#QY&56n z^btWZEq&#)pZbrb-{P?QxWn{}t^Z>7R$A3*@O-SOkD!}mFxsQXB41*9?B!l=iDqwi zn|cIGJ^?Bp*K7%CeZNvR>$*#MU;_thv-HpKG!M!psvd(NKpy z-!AH1iOWozyd6`7N&kugpU)K}hpWi~$k}DA=ElKGDxQ=GENWL~R2QSNI-a Date: Tue, 3 Sep 2013 14:26:13 -0400 Subject: [PATCH 091/502] Fix multiple false positives. - Increase the cost of macroman and cp437 by 1. - Also count the cost of macroman and cp437 correctly when there are multiple steps of decoding. - Add new test tweets. --- ftfy/fixes.py | 12 ++++++++---- tests/test_real_tweets.py | 30 ++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 457cdf0a..dfcaba47 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -102,15 +102,19 @@ def fix_text_encoding(text): """ best_version = text best_cost = text_cost(text) + plan_so_far = [] while True: prevtext = text text, plan = fix_text_and_explain(text) + plan_so_far.extend(plan) cost = text_cost(text) - # Add a small penalty if we used a particularly obsolete encoding. - if ('sloppy_encode', 'macroman') in plan or\ - ('sloppy_encode', 'cp437') in plan: - cost += 1 + # Add a penalty if we used a particularly obsolete encoding. The result + # is that we won't use these encodings unless they can successfully + # replace multiple characters. + if ('sloppy_encode', 'macroman') in plan_so_far or\ + ('sloppy_encode', 'cp437') in plan_so_far: + cost += 2 if cost < best_cost: best_cost = cost diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index d9d00521..e7b917dd 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -4,11 +4,11 @@ from nose.tools import eq_ TEST_CASES = [ + ## These are excerpts from tweets actually seen on the public Twitter + ## stream. Usernames and links have been removed. ("He's Justin❤", "He's Justin❤"), ("Le Schtroumpf Docteur conseille g√¢teaux et baies schtroumpfantes pour un r√©gime √©quilibr√©.", "Le Schtroumpf Docteur conseille gâteaux et baies schtroumpfantes pour un régime équilibré."), - #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", - # "Deja dos heridos hundimiento de barco turístico en Acapulco."), ("✔ No problems", "✔ No problems"), ('4288×…', '4288×...'), ('RETWEET SE VOCÊ…', 'RETWEET SE VOCÊ...'), @@ -16,18 +16,32 @@ ('TEM QUE SEGUIR, SDV SÓ…', 'TEM QUE SEGUIR, SDV SÓ...'), ('Join ZZAJÉ’s Official Fan List and receive news, events, and more!', "Join ZZAJÉ's Official Fan List and receive news, events, and more!"), ('L’épisode 8 est trop fou ouahh', "L'épisode 8 est trop fou ouahh"), - ("РґРѕСЂРѕРіРµ Из-РїРѕРґ http://t.co/A0eJAMTuJ1 #футбол", - "дороге Из-под http://t.co/A0eJAMTuJ1 #футбол"), + ("РґРѕСЂРѕРіРµ Из-РїРѕРґ #футбол", + "дороге Из-под #футбол"), ("\x84Handwerk bringt dich \xfcberall hin\x93: Von der YOU bis nach Monaco", '"Handwerk bringt dich überall hin": Von der YOU bis nach Monaco'), ("Hi guys 😍", "Hi guys 😍"), - ("@rakryanM hihi RT damnitstrue: ☺😘 http://t.co/DqSCy26POe", - "@rakryanM hihi RT damnitstrue: ☺😘 http://t.co/DqSCy26POe"), - ("Beta Haber: Hırsızı Büyü Korkuttu http://t.co/rMkt5yz7Si", - "Beta Haber: Hırsızı Büyü Korkuttu http://t.co/rMkt5yz7Si"), + ("hihi RT username: ☺😘", + "hihi RT username: ☺😘"), + ("Beta Haber: Hırsızı Büyü Korkuttu", + "Beta Haber: Hırsızı Büyü Korkuttu"), ("Blog Traffic Tip 2 – Broadcast Email Your Blog", "Blog Traffic Tip 2 – Broadcast Email Your Blog"), ("Ôôô VIDA MINHA", "Ôôô VIDA MINHA"), + ('[x]\xa0©', '[x] ©'), + ('2012—∞', '2012—∞'), + ('Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.', + 'Con il corpo e lo spirito ammaccato, è come se nel cuore avessi un vetro conficcato.') + + ## Current false positives: + #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", + # "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), + #('HUHLL Õ…', 'HUHLL Õ...'), + #("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), + + ## This kind of tweet can't be fixed without a full-blown encoding detector. + #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", + # "Deja dos heridos hundimiento de barco turístico en Acapulco."), ] def test_real_tweets(): From 6dd1e5628ed79c44aa6c23776841980652a1fce0 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 4 Sep 2013 02:47:46 -0400 Subject: [PATCH 092/502] Version 3.0.2 --- CHANGELOG.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 10ca790d..cc2f7149 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,13 @@ +## Version 3.0.2 (September 4, 2013) + +- Fixed some false positives. + - Basically, 3.0.1 was too eager to treat text as MacRoman or cp437 when + three consecutive characters coincidentally decoded as UTF-8. Increased the + cost of those encodings so that they have to successfully decode multiple + UTF-8 characters. + - See tests/test_real_tweets.py for the new test cases that were added as a + result. + ## Version 3.0.1 (August 30, 2013) - Fix bug in `fix_java_encoding` that led to only the first instance of From 5b403d4120668a1b93c636bddfa578aebc63e795 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 9 Sep 2013 17:07:50 -0400 Subject: [PATCH 093/502] bump version number and update company info --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d9501531..2b787e6f 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,8 @@ setup( name="ftfy", - version='3.0.2', - maintainer='Luminoso, LLC', + version='3.0.3', + maintainer='Luminoso Technologies, Inc.', maintainer_email='dev@lumino.so', license="MIT", url='http://github.com/LuminosoInsight/python-ftfy', From 86aa65814e768ea99646001ec19c77c1c3626de9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 9 Sep 2013 17:18:09 -0400 Subject: [PATCH 094/502] Restore compatibility with narrow builds of Python. You know how narrow builds are designed to replace astral characters with two surrogate characters and hope you don't notice? Regular expressions notice. Narrow builds turn them into different expressions that are completely wrong. Fortunately, in this case, it was so wrong that it prevented the module from being imported. --- ftfy/compatibility.py | 23 +++++++++++++++++++++++ ftfy/fixes.py | 6 +++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index fb225b6a..88f88b46 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -2,7 +2,9 @@ Makes some function names and behavior consistent between Python 2 and Python 3, and also between narrow and wide builds. """ +from __future__ import unicode_literals import sys +import re if sys.hexversion >= 0x03000000: from html import entities @@ -28,8 +30,29 @@ def _narrow_unichr_workaround(codepoint): escaped = b'\\U%08x' % codepoint return escaped.decode('unicode-escape') + if sys.maxunicode < 0x10000: unichr = _narrow_unichr_workaround + # In a narrow build of Python, we can't write a regex involving astral + # characters. If we want to write the regex: + # + # [\U00100000-\U0010ffff] + # + # The actual string that defines it quietly turns into: + # + # [\udbc0\udc00-\udbff\udfff] + # + # And now the range operator only applies to the middle two characters. + # It looks like a range that's going backwards from \dc00 to \dbff, + # which is an error. + # + # What we can do instead is rewrite the expression to be _about_ the two + # surrogates that make up the astral characters, instead of the characters + # themselves. This would be wrong on a wide build, but it works on a + # narrow build. + UNSAFE_PRIVATE_USE_RE = re.compile('[\udbc0-\udbff][\udc00-\udfff]') +else: + UNSAFE_PRIVATE_USE_RE = re.compile('[\U00100000-\U0010ffff]') def bytes_to_ints(bytestring): diff --git a/ftfy/fixes.py b/ftfy/fixes.py index dfcaba47..4fe6e8c7 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -10,7 +10,8 @@ from ftfy.badness import text_cost import re import sys -from ftfy.compatibility import htmlentitydefs, unichr, bytes_to_ints +from ftfy.compatibility import (htmlentitydefs, unichr, bytes_to_ints, + UNSAFE_PRIVATE_USE_RE) BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. @@ -393,7 +394,6 @@ def remove_bom(text): """ return text.lstrip(unichr(0xfeff)) -UNSAFE_3_3_RE = re.compile('[\U00100000-\U0010ffff]') def remove_unsafe_private_use(text): r""" Python 3.3's Unicode support isn't perfect, and in fact there are certain @@ -418,5 +418,5 @@ def remove_unsafe_private_use(text): >>> print(remove_unsafe_private_use('\U0001F4A9\U00100000')) 💩 """ - return UNSAFE_3_3_RE.sub('', text) + return UNSAFE_PRIVATE_USE_RE.sub('', text) From cd32d5e5c515048a2659815af55300074f66a88c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 9 Sep 2013 17:45:37 -0400 Subject: [PATCH 095/502] version 3.0.3 --- CHANGELOG.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index cc2f7149..00b35e62 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,8 @@ +## Version 3.0.3 (September 9, 2013) + +- Fixed an ugly regular expression bug that prevented ftfy from importing on a + narrow build of Python. + ## Version 3.0.2 (September 4, 2013) - Fixed some false positives. From cfa06895dc5a8a1e4911689f9f13e001427a538a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 25 Sep 2013 18:26:36 -0400 Subject: [PATCH 096/502] Correct Cyrillic characters less aggressively. Fixes #13. --- ftfy/fixes.py | 4 ++++ tests/test_real_tweets.py | 4 ++-- tests/test_unicode.py | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 4fe6e8c7..6ddca3a1 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -117,6 +117,10 @@ def fix_text_encoding(text): ('sloppy_encode', 'cp437') in plan_so_far: cost += 2 + # We need pretty solid evidence to decode from Windows-1251 (Cyrillic). + if ('sloppy_encode', 'windows-1251') in plan_so_far: + cost += 5 + if cost < best_cost: best_cost = cost best_version = text diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index e7b917dd..f7731813 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -25,8 +25,8 @@ "hihi RT username: ☺😘"), ("Beta Haber: Hırsızı Büyü Korkuttu", "Beta Haber: Hırsızı Büyü Korkuttu"), - ("Blog Traffic Tip 2 – Broadcast Email Your Blog", - "Blog Traffic Tip 2 – Broadcast Email Your Blog"), + #("Blog Traffic Tip 2 – Broadcast Email Your Blog", + # "Blog Traffic Tip 2 – Broadcast Email Your Blog"), ("Ôôô VIDA MINHA", "Ôôô VIDA MINHA"), ('[x]\xa0©', '[x] ©'), ('2012—∞', '2012—∞'), diff --git a/tests/test_unicode.py b/tests/test_unicode.py index eb1f3c66..8f9927bb 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -31,6 +31,7 @@ def test_all_bmp_characters(): u"\u2039ALLÍ ESTÁ\u203A", u"\u2014ALLÍ ESTÁ\u2014", u"AHÅ™, the new sofa from IKEA®", + u"ВІКІ is Ukrainian for WIKI", #u"\u2014a radius of 10 Å\u2014", ] # These phrases should not be erroneously "fixed" From de88012010709d16a9ffa62f4c6f8243fda5f594 Mon Sep 17 00:00:00 2001 From: Adriaan Tijsseling Date: Tue, 1 Oct 2013 15:52:07 +0200 Subject: [PATCH 097/502] 2.6 compatibility. --- ftfy/chardata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 0d8f403b..ffaff333 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -61,7 +61,7 @@ def _build_charmaps(): charlist = [unichr(codept) for codept in sorted(charmap.keys()) if codept >= 0x80] - regex = '^[\x00-\x7f{}]*$'.format(''.join(charlist)) + regex = '^[\x00-\x7f{0}]*$'.format(''.join(charlist)) charmaps[encoding] = charmap encoding_regexes[encoding] = re.compile(regex) return charmaps, encoding_regexes From e3605df0b5a67c8a451c3838ff76123462035b52 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 1 Oct 2013 11:45:02 -0400 Subject: [PATCH 098/502] v3.0.4 - restores python 2.6 compatibility --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2b787e6f..0981a7ff 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.0.3', + version='3.0.4', maintainer='Luminoso Technologies, Inc.', maintainer_email='dev@lumino.so', license="MIT", From e6eeb662c8bf377daa0867bfe90d0269d267741f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 1 Nov 2013 16:55:59 -0400 Subject: [PATCH 099/502] fix_file was using totally the wrong arguments. Fixed that. --- ftfy/__init__.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 86eb110a..b18efe24 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -150,7 +150,16 @@ def fix_text(text, ftfy = fix_text -def fix_file(input_file, normalization='NFKC'): +def fix_file(input_file, + remove_unsafe_private_use=True, + fix_entities=True, + remove_terminal_escapes=True, + fix_encoding=True, + normalization='NFKC', + uncurl_quotes=True, + fix_line_breaks=True, + remove_control_chars=True, + remove_bom=True): """ Fix text that is found in a file. @@ -163,13 +172,24 @@ def fix_file(input_file, normalization='NFKC'): The output is a stream of fixed lines of text. """ - entities = True + entities = fix_entities for line in input_file: if isinstance(line, bytes): line = line.decode('latin-1') if '<' in line and '>' in line: entities = False - yield fix_text_segment(line, normalization, entities) + yield fix_text_segment( + line, + remove_unsafe_private_use=remove_unsafe_private_use, + fix_entities=fix_entities, + remove_terminal_escapes=remove_terminal_escapes, + fix_encoding=entities, + normalization=normalization, + uncurl_quotes=uncurl_quotes, + fix_line_breaks=fix_line_breaks, + remove_control_chars=remove_control_chars, + remove_bom=remove_bom + ) def fix_text_segment(text, From 7fa37786a6a6c9af4e1c783361a76280c39f8166 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 1 Nov 2013 17:10:44 -0400 Subject: [PATCH 100/502] Version 3.0.5 --- CHANGELOG.txt | 8 ++++++++ README.md | 2 +- setup.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 00b35e62..b7cb97af 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +## Version 3.0.5 (November 1, 2013) + +- Fix the arguments to `fix_file`, because they were totally wrong. + +## Version 3.0.4 (October 1, 2013) + +- Restore compatibility with Python 2.6. + ## Version 3.0.3 (September 9, 2013) - Fixed an ugly regular expression bug that prevented ftfy from importing on a diff --git a/README.md b/README.md index d4d49a17..167eea05 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Full documentation: **http://ftfy.readthedocs.org** — [Bruno Ranieri](http://yrrsinn.de/2012/09/17/gelesen-kw37/) - “I have no idea when I’m gonna need this, but I’m definitely bookmarking it.” — [/u/ocrow](http://reddit.com/u/ocrow) -- “9.41/10” +- “Too many branches” — [pylint](https://bitbucket.org/logilab/pylint/) ## What it does diff --git a/setup.py b/setup.py index 0981a7ff..cda7dee2 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.0.4', + version='3.0.5', maintainer='Luminoso Technologies, Inc.', maintainer_email='dev@lumino.so', license="MIT", From 3b8c9373b496344a76cd9a6deac30762f6f9e496 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 5 Nov 2013 17:58:07 -0500 Subject: [PATCH 101/502] Document and clean up the character map building process --- ftfy/chardata.py | 75 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index ffaff333..a2961c67 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -25,19 +25,39 @@ def _build_charmaps(): """ - CHARMAPS contains mappings from bytes to characters, for each single-byte - encoding we know about. - - We don't use Python's decoders here because they're too strict. Many - non-Python programs will leave mysterious bytes alone instead of raising - an error or removing them. For example, Python will not decode 0x81 in - Windows-1252 because it doesn't map to anything. Other systems will decode - it to U+0081, which actually makes no sense because that's a meaningless - control character from Latin-1, but I guess at least it preserves some - information that ftfy can take advantage of. - - So that's what we do. When other systems decode 0x81 as U+0081, we match - their behavior in case it helps us get reasonable text. + CHARMAPS is a list of 'translate' mappings. For every character map that + we care about, it will contain a mapping *from* the Unicode characters in + that character set, *to* characters that stand in for bytes. + + It would be a lot cleaner if we could map the characters directly to + bytes, but the output of ``str.translate`` (``unicode.translate`` in + Python 2) has to be a Unicode string, not bytes. + + So when you use these mappings with Unicode, what you're doing is mapping + a certain character set onto the first 256 Unicode characters, like a + chump who doesn't know anything about Unicode. But then you can take the + resulting string, and encode it as `latin-1`, and you get back the + appropriate bytes. Two wrongs *do* make a right sometimes! + + We don't use Python's encoders here because they're too strict. For + example, the Windows-1252 character map has some holes in it, including + 0x81. There is no assigned meaning to byte 0x81, so if you try to decode + it, Python will raise an error. + + But some other environments, such as web browsers, don't have the luxury + of raising errors when they see something unexpected. What they will + typically do is map 0x81 to Unicode character U+0081, a character which + is basically reserved for that purpose anyway. We'll call this "sloppy + decoding". + + So now you've got some text that came from the Web, and it contains + U+0081 because it was sloppily decoded as Windows-1252. How do we get + back what the bytes were? Well, we need to sloppily *encode*. We need + a version of Windows-1252 where U+0081 gets encoded as 0x81, instead + of just raising an error. + + So that's what we do. When other systems decode 0x81 as U+0081, we support + that by encoding U+0081 as 0x81, in case it helps us get reasonable text. Meanwhile, ENCODING_REGEXES contain reasonably fast ways to detect if we could represent a given string in a given encoding. The simplest one is @@ -45,23 +65,42 @@ def _build_charmaps(): are between U+0000 and U+007F. """ charmaps = {} + + # Define a regex that matches ASCII text. encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')} + + # For each character map encoding we care about, make a regex that contains + # all the characters that that encoding supports, and a mapping from those + # characters to sorta-bytes. for encoding in CHARMAP_ENCODINGS: charmap = {} for codepoint in range(0, 0x80): + # ASCII characters map to themselves. charmap[codepoint] = unichr(codepoint) for codepoint in range(0x80, 0x100): + # The other characters map to characters from U+0080 to U+00FF, + # standing in for bytes. We call these sorta-bytes. char = unichr(codepoint) + + # Turn the sorta-byte into a byte, and try decoding it. encoded_char = char.encode('latin-1') try: + # It decoded, so that's the character we need to add to our + # character map. decoded_char = encoded_char.decode(encoding) except ValueError: + # It didn't decode, so we should be "sloppy" and add the + # sorta-byte itself to the character map. decoded_char = char charmap[ord(decoded_char)] = char + # Put together all the characters we got, and make a regular expression + # that matches all of them. charlist = [unichr(codept) for codept in sorted(charmap.keys()) if codept >= 0x80] regex = '^[\x00-\x7f{0}]*$'.format(''.join(charlist)) + + # Store the mapping and the regex in the dictionaries we're returning. charmaps[encoding] = charmap encoding_regexes[encoding] = re.compile(regex) return charmaps, encoding_regexes @@ -72,6 +111,9 @@ def possible_encoding(text, encoding): """ Given text and a single-byte encoding, check whether that text could have been decoded from that single-byte encoding. + + In other words, check whether it can be encoded in that encoding, possibly + sloppily. """ return bool(ENCODING_REGEXES[encoding].match(text)) @@ -95,7 +137,6 @@ def chars_to_classes(string): for i in range(32): CONTROL_CHARS[i] = None -CONTROL_CHARS[ord('\t')] = '\t' -CONTROL_CHARS[ord('\n')] = '\n' -CONTROL_CHARS[ord('\f')] = '\f' -CONTROL_CHARS[ord('\r')] = '\r' +# Map whitespace control characters to themselves. +for char in '\t\n\f\r': + del CONTROL_CHARS[ord(char)] From 1236365258e9f54264cf391abbc1b6246c33d247 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 5 Nov 2013 18:02:11 -0500 Subject: [PATCH 102/502] Document the case where we give up on decoding text as Windows-1252. This includes fixing an inaccurate comment that appeared after it. --- ftfy/fixes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 6ddca3a1..a0e37c1e 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -210,11 +210,13 @@ def fix_text_and_explain(text): steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] return fixed, steps except UnicodeDecodeError: - # Well, never mind. + # This text contained characters that don't even make sense + # if you assume they were supposed to be Windows-1252. In + # that case, let's not assume anything. pass # The cases that remain are mixups between two different single-byte - # encodings, neither of which is Latin-1. + # encodings, and not the common case of Latin-1 vs. Windows-1252. # # Those cases are somewhat rare, and impossible to solve without false # positives. If you're in one of these situations, you don't need an From 1e4d8bcbd1699b9960f851a029b9d8c313b3ff55 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 5 Nov 2013 18:15:04 -0500 Subject: [PATCH 103/502] fix_entities can now be True, False, or 'auto' --- ftfy/__init__.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index b18efe24..68c17752 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -16,7 +16,7 @@ def fix_text(text, remove_unsafe_private_use=(not PYTHON34_OR_LATER), - fix_entities=True, + fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, normalization='NFKC', @@ -64,10 +64,11 @@ def fix_text(text, - If `remove_unsafe_private_use` is True, it removes a range of unassigned characters that could trigger Python `bug 18183`_. This will default to False starting on Python 3.4, when the bugfix will be released. - - If `fix_entities` is True, consider replacing HTML entities with their - equivalent characters. However, this never applies to text with a pair - of angle brackets in it already; you're probably not supposed to decode - entities there, and you'd make things ambiguous if you did. + - If `fix_entities` is True, replace HTML entities with their equivalent + characters. If it's "auto" (the default), then consider replacing HTML + entities, but don't do so in text where you have seen a pair of actual + angle brackets (that's probably actually HTML and you shouldn't mess + with the entities). - If `remove_terminal_escapes` is True, remove sequences of bytes that are instructions for Unix terminals, such as the codes that make text appear in different colors. @@ -125,7 +126,7 @@ def fix_text(text, substring = text[pos:textbreak] - if '<' in substring and '>' in substring: + if fix_entities == 'auto' and '<' in substring and '>' in substring: # we see angle brackets together; this could be HTML fix_entities = False @@ -152,7 +153,7 @@ def fix_text(text, def fix_file(input_file, remove_unsafe_private_use=True, - fix_entities=True, + fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, normalization='NFKC', @@ -176,7 +177,7 @@ def fix_file(input_file, for line in input_file: if isinstance(line, bytes): line = line.decode('latin-1') - if '<' in line and '>' in line: + if fix_entities == 'auto' and '<' in line and '>' in line: entities = False yield fix_text_segment( line, @@ -194,7 +195,7 @@ def fix_file(input_file, def fix_text_segment(text, remove_unsafe_private_use=True, - fix_entities=True, + fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, normalization='NFKC', @@ -212,12 +213,13 @@ def fix_text_segment(text, if isinstance(text, bytes): raise UnicodeError(fixes.BYTES_ERROR_TEXT) - unsafe_entities = ('<' in text and '>' in text) + if fix_entities == 'auto' and '<' in text and '>' in text: + fix_entities = False while True: origtext = text if remove_unsafe_private_use: text = fixes.remove_unsafe_private_use(text) - if fix_entities and not unsafe_entities: + if fix_entities: text = fixes.unescape_html(text) if remove_terminal_escapes: text = fixes.remove_terminal_escapes(text) From 6e76b036b4e80abe323709d9d4de2fcc1a2eccd1 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 5 Nov 2013 18:19:46 -0500 Subject: [PATCH 104/502] Make sure to run build_data in Python 3.3 or later. --- ftfy/build_data.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index d9e004b0..c3665227 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -37,15 +37,22 @@ # = Whitespace # o = Other -def make_char_data_file(): +def make_char_data_file(do_it_anyway=False): """ Build the compressed data file 'char_classes.dat' and write it to the current directory. If you run this, run it in Python 3.3 or later. It will run in earlier versions, but you won't get the current Unicode standard, leading to - inconsistent behavior. + inconsistent behavior. To protect against this, running this in the + wrong version of Python will raise an error unless you pass + `do_it_anyway=True`. """ + if sys.hexversion < 0x03030000 and not do_it_anyway: + raise RuntimeError( + "This function should be run in Python 3.3 or later." + ) + cclasses = [None] * 0x110000 for codepoint in range(0x0, 0x110000): char = unichr(codepoint) @@ -101,4 +108,3 @@ def make_char_data_file(): if __name__ == '__main__': make_char_data_file() - From f7be5699265e123866f2e43e3e2920b39495dd80 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 5 Nov 2013 18:22:49 -0500 Subject: [PATCH 105/502] Add tests for the fix_entities parameter --- tests/test_entities.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/test_entities.py diff --git a/tests/test_entities.py b/tests/test_entities.py new file mode 100644 index 00000000..f23abffc --- /dev/null +++ b/tests/test_entities.py @@ -0,0 +1,18 @@ +from __future__ import unicode_literals +from ftfy import fix_text, fix_text_segment +from nose.tools import eq_ + +def test_entities(): + example = '&\n\n&' + eq_(fix_text(example), '&\n\n&') + eq_(fix_text_segment(example), '&\n\n&') + + eq_(fix_text(example, fix_entities=True), '&\n\n&') + eq_(fix_text_segment(example, fix_entities=True), '&\n\n&') + + eq_(fix_text(example, fix_entities=False), '&\n\n&') + eq_(fix_text_segment(example, fix_entities=False), '&\n\n&') + + eq_(fix_text_segment('<>', fix_entities=False), '<>') + eq_(fix_text_segment('<>', fix_entities=True), '<>') + eq_(fix_text_segment('<>'), '<>') From 4ef7df7f574dcca86eeb8bb395525f8899a4daf9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 5 Nov 2013 18:23:04 -0500 Subject: [PATCH 106/502] Version 3.0.6 --- CHANGELOG.txt | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index b7cb97af..34498199 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +## Version 3.0.6 (November 5, 2013) + +- `fix_entities` can now be True, False, or 'auto'. The new case is True, which + will decode all entities, even in text that already contains angle brackets. + This may also be faster, because it doesn't have to check. +- `build_data.py` will refuse to run on Python < 3.3, to prevent building + an inconsistent data file. + ## Version 3.0.5 (November 1, 2013) - Fix the arguments to `fix_file`, because they were totally wrong. diff --git a/setup.py b/setup.py index cda7dee2..8539c5e6 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.0.5', + version='3.0.6', maintainer='Luminoso Technologies, Inc.', maintainer_email='dev@lumino.so', license="MIT", From c215cffe7564902724066c1dd91cb861d0d1bdce Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 10 Nov 2013 03:22:06 -0500 Subject: [PATCH 107/502] let test_unicode be used as a script for profiling --- tests/test_unicode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 8f9927bb..2a616e39 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -45,3 +45,6 @@ def check_phrase(text): eq_(fix_text_encoding(text), text) eq_(fix_text_encoding(text.encode('utf-8').decode('latin-1')), text) +if __name__ == '__main__': + test_all_bmp_characters() + From a56dddfd2b51273952308c0ca3ecdc362d9631a7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 12 Sep 2013 22:16:50 -0400 Subject: [PATCH 108/502] Add "bad codecs" for decoding bad bytes. Simply by using a statement such as "import ftfy.bad_codecs", you will be able to decode popular, messy encodings that Python is too dignified to support, such as 'sloppy-windows-1252' and 'cesu-8'. --- .gitignore | 1 + ftfy/autodecode.py | 21 ++++ ftfy/bad_codecs/__init__.py | 35 +++++++ ftfy/bad_codecs/sloppy.py | 58 +++++++++++ ftfy/bad_codecs/utf8_variants.py | 169 +++++++++++++++++++++++++++++++ ftfy/chardata.py | 83 +++++++-------- ftfy/fixes.py | 119 ++++------------------ tests/test_encodings.py | 12 +++ tests/test_unicode.py | 4 +- 9 files changed, 351 insertions(+), 151 deletions(-) create mode 100644 ftfy/autodecode.py create mode 100644 ftfy/bad_codecs/__init__.py create mode 100644 ftfy/bad_codecs/sloppy.py create mode 100644 ftfy/bad_codecs/utf8_variants.py create mode 100644 tests/test_encodings.py diff --git a/.gitignore b/.gitignore index 402af6e9..684ffe97 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist *.egg-info build _build +.idea \ No newline at end of file diff --git a/ftfy/autodecode.py b/ftfy/autodecode.py new file mode 100644 index 00000000..35773797 --- /dev/null +++ b/ftfy/autodecode.py @@ -0,0 +1,21 @@ +from ftfy.chardata import possible_encoding +import codecs +import ftfy.bad_codecs.utf8_variants + +def decode_bytes(bstring): + if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): + return bstring.decode('utf-16'), 'utf-16' + + try: + return bstring.decode('utf-8'), 'utf-8') + except UnicodeDecodeError: + pass + + try: + return bstring.decode('utf-8-variants'), 'utf-8-variants') + except UnicodeDecodeError: + pass + + return bstring.decode('latin-1'), 'latin-1' + + diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py new file mode 100644 index 00000000..ff1fd58d --- /dev/null +++ b/ftfy/bad_codecs/__init__.py @@ -0,0 +1,35 @@ +import codecs +from encodings import normalize_encoding + +_cache = {} + +# Define some aliases for 'utf-8-variants' +UTF8_VAR_NAMES = ( + 'utf_8_variants', 'utf8_variants', + 'utf_8_variant', 'utf8_variant', + 'utf_8_var', 'utf8_var', + 'cesu_8', 'cesu8', + 'java_utf_8', 'java_utf8' +) + + +def search_function(encoding): + if encoding in _cache: + return _cache[encoding] + + norm_encoding = normalize_encoding(encoding) + codec = None + if norm_encoding in UTF8_VAR_NAMES: + from ftfy.bad_codecs.utf8_variants import CODEC_INFO + codec = CODEC_INFO + elif norm_encoding.startswith('sloppy_'): + from ftfy.bad_codecs.sloppy import CODECS + return CODECS.get(norm_encoding) + + if codec is not None: + _cache[encoding] = codec + + return codec + + +codecs.register(search_function) diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py new file mode 100644 index 00000000..e802507c --- /dev/null +++ b/ftfy/bad_codecs/sloppy.py @@ -0,0 +1,58 @@ +from __future__ import unicode_literals +import codecs +from encodings import normalize_encoding +from ftfy.compatibility import unichr + +REPLACEMENT_CHAR = '\ufffd' + +def make_sloppy_codec(encoding): + decoding_list = [] + for byte in range(0x100): + char = unichr(byte).encode('latin-1').decode(encoding, errors='replace') + if char == REPLACEMENT_CHAR: + char = unichr(byte) + decoding_list.append(char) + decoding_table = ''.join(decoding_list) + encoding_table = codecs.charmap_build(decoding_table) + + class Codec(codecs.Codec): + def encode(self,input,errors='strict'): + return codecs.charmap_encode(input,errors,encoding_table) + + def decode(self,input,errors='strict'): + return codecs.charmap_decode(input,errors,decoding_table) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return codecs.charmap_encode(input,self.errors,encoding_table)[0] + + class IncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return codecs.charmap_decode(input,self.errors,decoding_table)[0] + + class StreamWriter(Codec,codecs.StreamWriter): + pass + + class StreamReader(Codec,codecs.StreamReader): + pass + + return codecs.CodecInfo( + name='cp1252', + encode=Codec().encode, + decode=Codec().decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + ) + +CODECS = {} +INCOMPLETE_ENCODINGS = ( + ['windows-%s' % num for num in range(1250, 1259)] + + ['iso-8859-%s' % num for num in (3, 6, 7, 8, 11)] +) + +for _encoding in INCOMPLETE_ENCODINGS: + _new_name = normalize_encoding('sloppy-' + _encoding) + CODECS[_new_name] = make_sloppy_codec(_encoding) + diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py new file mode 100644 index 00000000..7478da47 --- /dev/null +++ b/ftfy/bad_codecs/utf8_variants.py @@ -0,0 +1,169 @@ +r""" +This file defines a codec called "utf-8-variants" (or "utf-8-var"), which can +decode text that's been encoded with a popular non-standard version of UTF-8. +This includes CESU-8, the accidental encoding made by layering UTF-8 on top of +UTF-16, as well as Java's twist on CESU-8 that contains a two-byte encoding for +codepoint 0. + +This is particularly relevant in Python 3, which provides no other way of +decoding CESU-8 or Java's encoding. [1] + +The easiest way to use the codec is simply to import `ftfy.bad_codecs`: + + >>> import ftfy.bad_codecs + >>> b'here comes a null! \xc0\x80'.decode('utf-8-var') + 'here comes a null! \x00' + +The codec does not at all enforce "correct" CESU-8. For example, the Unicode +Consortium's not-quite-standard describing CESU-8 requires that there is only +one possible encoding of any character, so it does not allow mixing of valid +UTF-8 and CESU-8. This codec *does* allow that, just like Python 2's UTF-8 +decoder does. + +Characters in the Basic Multilingual Plane still have only one encoding. This +codec still enforces the rule, within the BMP, that characters must appear in +their shortest form. There is one exception: the sequence of bytes `0xc0 0x80`, +instead of just `0x00`, may be used to encode the null character `U+0000`, like +in Java. + +If you encode with this codec, you get legitimate UTF-8. Decoding with this +codec and then re-encoding is not idempotent, although encoding and then +decoding is. So this module won't produce CESU-8 for you. Look for that +functionality in the sister module, "Broke That For You", coming approximately +never. + +[1] In a pinch, you can decode CESU-8 in Python 2 using the UTF-8 codec: first +decode the bytes (incorrectly), then encode them, then decode them again. +""" + +from __future__ import unicode_literals +from ftfy.compatibility import bytes_to_ints, unichr +from encodings.utf_8 import (IncrementalDecoder as UTF8IncrementalDecoder, + IncrementalEncoder as UTF8IncrementalEncoder) +import re +import codecs + +NAME = 'utf-8-variants' +CESU8_RE = re.compile(b'\xed[\xa0-\xaf][\x80-\xbf]\xed[\xb0-\xbf][\x80-\xbf]') + + +# TODO: document and write tests +class IncrementalDecoder(UTF8IncrementalDecoder): + def _buffer_decode(self, input, errors, final): + decoded_segments = [] + position = 0 + while True: + decoded, consumed = self._buffer_decode_step( + input[position:], + errors, + final + ) + if consumed == 0: + break + decoded_segments.append(decoded) + position += consumed + + if final: + assert position == len(input) + return ''.join(decoded_segments), position + + def _buffer_decode_step(self, input, errors, final): + sup = UTF8IncrementalDecoder._buffer_decode + cutoff1 = input.find(b'\xed') + cutoff2 = input.find(b'\xc0') + if cutoff1 != -1 and cutoff2 != -1: + cutoff = min(cutoff1, cutoff2) + elif cutoff1 != -1: + cutoff = cutoff1 + elif cutoff2 != -1: + cutoff = cutoff2 + else: + # The UTF-8 decoder can handle it from here. + return sup(input, errors, final) + + if input.startswith(b'\xc0'): + return self._buffer_decode_null(sup, input, errors, final) + elif input.startswith(b'\xed'): + return self._buffer_decode_surrogates(sup, input, errors, final) + else: + # at this point, we know cutoff > 0 + return sup(input[:cutoff], errors, False) + + @staticmethod + def _buffer_decode_null(sup, input, errors, final): + nextchar = input[1:2] + if nextchar == b'': + if final: + return sup(input, errors, True) + else: + return '', 0 + elif nextchar == b'\x80': + return '\u0000', 2 + else: + return sup(b'\xc0', errors, True) + + @staticmethod + def _buffer_decode_surrogates(sup, input, errors, final): + """ + When we have improperly encoded surrogates, we can still see the + bits that they were meant to represent. + + The surrogates were meant to encode a 20-bit number, to which we + add 0x10000 to get a codepoint. That 20-bit number now appears in + this form: + + 11101101 1010abcd 10efghij 11101101 1011klmn 10opqrst + + The CESU8_RE above matches byte sequences of this form. Then we need + to extract the bits and assemble a codepoint number from them. + """ + if len(input) < 6: + if final: + return sup(input, errors, True) + else: + return '', 0 + else: + if CESU8_RE.match(input): + bytenums = bytes_to_ints(input[:6]) + codepoint = ( + ((bytenums[1] & 0x0f) << 16) + + ((bytenums[2] & 0x3f) << 10) + + ((bytenums[4] & 0x0f) << 6) + + (bytenums[5] & 0x3f) + + 0x10000 + ) + return unichr(codepoint), 6 + else: + return sup(input[:3], errors, False) + + +# The encoder is identical to UTF8. +IncrementalEncoder = UTF8IncrementalEncoder + + +# Everything below here is basically boilerplate. +def encode(input, errors='strict'): + return IncrementalEncoder(errors).encode(input, final=True), len(input) + + +def decode(input, errors='strict'): + return IncrementalDecoder(errors).decode(input, final=True), len(input) + + +class StreamWriter(codecs.StreamWriter): + encode = encode + + +class StreamReader(codecs.StreamReader): + decode = decode + + +CODEC_INFO = codecs.CodecInfo( + name=NAME, + encode=encode, + decode=decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamreader=StreamReader, + streamwriter=StreamWriter, +) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index a2961c67..64485324 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -9,55 +9,36 @@ import zlib from pkg_resources import resource_string from ftfy.compatibility import unichr +import ftfy.bad_codecs - -# These are encodings that map each byte to a particular character. -# They are listed in order of frequency, so that more frequent -# encodings will be tried first. +# These are the five encodings we will try to fix in ftfy, in the +# order that they should be tried. CHARMAP_ENCODINGS = [ - 'windows-1252', 'latin-1', + 'sloppy-windows-1252', 'macroman', 'cp437', - 'windows-1251', + 'sloppy-windows-1251', ] -def _build_charmaps(): +def _build_regexes(): """ - CHARMAPS is a list of 'translate' mappings. For every character map that - we care about, it will contain a mapping *from* the Unicode characters in - that character set, *to* characters that stand in for bytes. - - It would be a lot cleaner if we could map the characters directly to - bytes, but the output of ``str.translate`` (``unicode.translate`` in - Python 2) has to be a Unicode string, not bytes. - - So when you use these mappings with Unicode, what you're doing is mapping - a certain character set onto the first 256 Unicode characters, like a - chump who doesn't know anything about Unicode. But then you can take the - resulting string, and encode it as `latin-1`, and you get back the - appropriate bytes. Two wrongs *do* make a right sometimes! - - We don't use Python's encoders here because they're too strict. For - example, the Windows-1252 character map has some holes in it, including - 0x81. There is no assigned meaning to byte 0x81, so if you try to decode - it, Python will raise an error. - - But some other environments, such as web browsers, don't have the luxury - of raising errors when they see something unexpected. What they will - typically do is map 0x81 to Unicode character U+0081, a character which - is basically reserved for that purpose anyway. We'll call this "sloppy - decoding". - - So now you've got some text that came from the Web, and it contains - U+0081 because it was sloppily decoded as Windows-1252. How do we get - back what the bytes were? Well, we need to sloppily *encode*. We need - a version of Windows-1252 where U+0081 gets encoded as 0x81, instead - of just raising an error. - - So that's what we do. When other systems decode 0x81 as U+0081, we support - that by encoding U+0081 as 0x81, in case it helps us get reasonable text. + [outdated docs] + + CHARMAPS contains mappings from bytes to characters, for each single-byte + encoding we know about. + + We don't use Python's decoders here because they're too strict. Many + non-Python programs will leave mysterious bytes alone instead of raising + an error or removing them. For example, Python will not decode 0x81 in + Windows-1252 because it doesn't map to anything. Other systems will decode + it to U+0081, which actually makes no sense because that's a meaningless + control character from Latin-1, but I guess at least it preserves some + information that ftfy can take advantage of. + + So that's what we do. When other systems decode 0x81 as U+0081, we match + their behavior in case it helps us get reasonable text. Meanwhile, ENCODING_REGEXES contain reasonably fast ways to detect if we could represent a given string in a given encoding. The simplest one is @@ -65,18 +46,17 @@ def _build_charmaps(): are between U+0000 and U+007F. """ charmaps = {} - - # Define a regex that matches ASCII text. - encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')} - - # For each character map encoding we care about, make a regex that contains - # all the characters that that encoding supports, and a mapping from those - # characters to sorta-bytes. + inverse_charmaps = {} + encoding_regexes = { + 'ascii': re.compile('^[\x00-\x7f]*$'), + } for encoding in CHARMAP_ENCODINGS: charmap = {} + inverse_charmap = {} for codepoint in range(0, 0x80): # ASCII characters map to themselves. charmap[codepoint] = unichr(codepoint) + inverse_charmap[codepoint] = unichr(codepoint) for codepoint in range(0x80, 0x100): # The other characters map to characters from U+0080 to U+00FF, # standing in for bytes. We call these sorta-bytes. @@ -93,6 +73,7 @@ def _build_charmaps(): # sorta-byte itself to the character map. decoded_char = char charmap[ord(decoded_char)] = char + inverse_charmap[ord(char)] = decoded_char # Put together all the characters we got, and make a regular expression # that matches all of them. @@ -102,9 +83,13 @@ def _build_charmaps(): # Store the mapping and the regex in the dictionaries we're returning. charmaps[encoding] = charmap + inverse_charmaps[encoding] = inverse_charmap + latin1table = ''.join(unichr(i) for i in range(256)) + charlist = latin1table.encode('latin-1').decode(encoding) + regex = '^[\x00-\x7f{}]*$'.format(charlist) encoding_regexes[encoding] = re.compile(regex) - return charmaps, encoding_regexes -CHARMAPS, ENCODING_REGEXES = _build_charmaps() + return encoding_regexes +ENCODING_REGEXES = _build_regexes() def possible_encoding(text, encoding): diff --git a/ftfy/fixes.py b/ftfy/fixes.py index a0e37c1e..3de9c263 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -5,13 +5,14 @@ """ from __future__ import unicode_literals -from ftfy.chardata import (possible_encoding, CHARMAPS, CHARMAP_ENCODINGS, - CONTROL_CHARS) +from ftfy.chardata import (possible_encoding, + CHARMAP_ENCODINGS, CONTROL_CHARS) from ftfy.badness import text_cost -import re -import sys from ftfy.compatibility import (htmlentitydefs, unichr, bytes_to_ints, UNSAFE_PRIVATE_USE_RE) +import ftfy.bad_codecs +import re +import sys BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. @@ -113,8 +114,8 @@ def fix_text_encoding(text): # Add a penalty if we used a particularly obsolete encoding. The result # is that we won't use these encodings unless they can successfully # replace multiple characters. - if ('sloppy_encode', 'macroman') in plan_so_far or\ - ('sloppy_encode', 'cp437') in plan_so_far: + if ('encode', 'macroman') in plan_so_far or\ + ('encode', 'cp437') in plan_so_far: cost += 2 # We need pretty solid evidence to decode from Windows-1251 (Cyrillic). @@ -155,35 +156,16 @@ def fix_text_and_explain(text): # are usually the correct thing to do, so try them next. for encoding in CHARMAP_ENCODINGS: if possible_encoding(text, encoding): - # This is an ugly-looking way to get the bytes that represent - # the text in this encoding. The reason we can't necessarily - # use .encode(encoding) is that the decoder is very likely - # to have been sloppier than Python. - # - # The decoder might have left bytes unchanged when they're not - # part of the encoding. It might represent b'\x81' as u'\x81' - # in Windows-1252, while Python would claim that using byte - # 0x81 in Windows-1252 is an error. - # - # So what we do here is we use the .translate method of Unicode - # strings. Using it with the character maps we have computed will - # give us back a Unicode string using only code - # points up to 0xff. This can then be converted into the intended - # bytes by encoding it as Latin-1. - sorta_encoded_text = text.translate(CHARMAPS[encoding]) - - # When we get the bytes, run them through fix_java_encoding, - # because we can only reliably do that at the byte level. (See - # its documentation for details.) - encoded_bytes = fix_java_encoding( - sorta_encoded_text.encode('latin-1') - ) - - # Now, find out if it's UTF-8. Otherwise, remember the encoding - # for later. + encoded_bytes = text.encode(encoding) + + # Now, find out if it's UTF-8 (or close enough). Otherwise, + # remember the encoding for later. try: - fixed = encoded_bytes.decode('utf-8') - steps = [('sloppy_encode', encoding), ('decode', 'utf-8')] + decoding = 'utf-8' + if b'\xed' in encoded_bytes or b'\xc0' in encoded_bytes: + decoding = 'utf-8-variants' + fixed = encoded_bytes.decode(decoding) + steps = [('encode', encoding), ('decode', decoding)] return fixed, steps except UnicodeDecodeError: possible_1byte_encodings.append(encoding) @@ -203,7 +185,9 @@ def fix_text_and_explain(text): else: # Otherwise, it means we have characters that are in Latin-1 but # not in Windows-1252. Those are C1 control characters. Nobody - # wants those. Assume they were meant to be Windows-1252. + # wants those. Assume they were meant to be Windows-1252. Don't + # use the sloppy codec, because bad Windows-1252 characters are + # a bad sign. encoded = text.encode('latin-1') try: fixed = encoded.decode('windows-1252') @@ -325,71 +309,6 @@ def remove_control_chars(text): return text.translate(CONTROL_CHARS) -CESU8_RE = re.compile(b'\xed[\xa0-\xaf][\x80-\xbf]\xed[\xb0-\xbf][\x80-\xbf]') -def fix_java_encoding(bytestring): - r""" - Convert a bytestring that might contain "Java Modified UTF8" into valid - UTF-8. - - There are two things that Java is known to do with its "UTF8" encoder - that are incompatible with UTF-8. (If you happen to be writing Java - code, apparently the standards-compliant encoder is named "AS32UTF8".) - - - Every UTF-16 character is separately encoded as UTF-8. This is wrong - when the UTF-16 string contains surrogates; the character they actually - represent should have been encoded as UTF-8 instead. Unicode calls this - "CESU-8", the Compatibility Encoding Scheme for Unicode. Python 2 will - decode it as if it's UTF-8, but Python 3 refuses to. - - - The null codepoint, U+0000, is encoded as 0xc0 0x80, which avoids - outputting a null byte by breaking the UTF shortest-form rule. - Unicode does not even deign to give this scheme a name, and no version - of Python will decode it. - - >>> len(fix_java_encoding(b'\xed\xa0\xbd\xed\xb8\x8d')) - 4 - - >>> ends_with_null = fix_java_encoding(b'Here comes a null! \xc0\x80') - >>> bytes_to_ints(ends_with_null)[-1] - 0 - """ - assert isinstance(bytestring, bytes) - # Replace the sloppy encoding of U+0000 with the correct one. - bytestring = bytestring.replace(b'\xc0\x80', b'\x00') - - # When we have improperly encoded surrogates, we can still see the - # bits that they were meant to represent. - # - # The surrogates were meant to encode a 20-bit number, to which we - # add 0x10000 to get a codepoint. That 20-bit number now appears in - # this form: - # - # 11101101 1010abcd 10efghij 11101101 1011klmn 10opqrst - # - # The CESU8_RE above matches byte sequences of this form. Then we need - # to extract the bits and assemble a codepoint number from them. - match = CESU8_RE.search(bytestring) - fixed_pieces = [] - while match: - pos = match.start() - cesu8_sequence = bytes_to_ints(bytestring[pos:pos + 6]) - assert cesu8_sequence[0] == cesu8_sequence[3] == 0xed - codepoint = ( - ((cesu8_sequence[1] & 0x0f) << 16) + - ((cesu8_sequence[2] & 0x3f) << 10) + - ((cesu8_sequence[4] & 0x0f) << 6) + - (cesu8_sequence[5] & 0x3f) + - 0x10000 - ) - # For the reason why this will work on all Python builds, see - # compatibility.py. - new_bytes = unichr(codepoint).encode('utf-8') - fixed_pieces.append(bytestring[:pos] + new_bytes) - bytestring = bytestring[pos + 6:] - match = CESU8_RE.search(bytestring) - - return b''.join(fixed_pieces) + bytestring - def remove_bom(text): r""" diff --git a/tests/test_encodings.py b/tests/test_encodings.py new file mode 100644 index 00000000..a2075801 --- /dev/null +++ b/tests/test_encodings.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from ftfy import bad_codecs +from nose.tools import eq_ + + +def test_cesu8(): + eq_(bad_codecs.search_function('cesu8').__class__, bad_codecs.search_function('cesu-8').__class__) + + test_bytes = b'\xed\xa6\x9d\xed\xbd\xb7 is an unassigned character, and \xc0\x80 is null' + test_text = '\U00077777 is an unassigned character, and \x00 is null' + eq_(test_bytes.decode('cesu8'), test_text) \ No newline at end of file diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 2a616e39..39f345f8 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from ftfy.fixes import fix_text_encoding +from ftfy.fixes import fix_text_encoding, fix_text_and_explain import unicodedata import sys from nose.tools import eq_ @@ -22,7 +22,7 @@ def test_all_bmp_characters(): if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): garble = char.encode('utf-8').decode('latin-1') garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') - eq_(char_names(fix_text_encoding(garble)), char_names(char)) + eq_(char_names(fix_text_encoding(garble)), char_names(char), fix_text_and_explain(garble)) eq_(char_names(fix_text_encoding(garble2)), char_names(char)) phrases = [ From 046522b69d41999f4e6804ebd7f4d67d0da3e0d9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 10 Nov 2013 16:30:02 -0500 Subject: [PATCH 109/502] Simplify how sloppy codecs are built --- ftfy/bad_codecs/sloppy.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index e802507c..f2afc28f 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -1,18 +1,18 @@ from __future__ import unicode_literals import codecs from encodings import normalize_encoding -from ftfy.compatibility import unichr REPLACEMENT_CHAR = '\ufffd' def make_sloppy_codec(encoding): - decoding_list = [] - for byte in range(0x100): - char = unichr(byte).encode('latin-1').decode(encoding, errors='replace') - if char == REPLACEMENT_CHAR: - char = unichr(byte) - decoding_list.append(char) - decoding_table = ''.join(decoding_list) + all_bytes = bytearray(range(256)) + sloppy_chars = list(all_bytes.decode('latin-1')) + decoded_chars = all_bytes.decode(encoding, errors='replace') + for i, char in enumerate(decoded_chars): + if char != REPLACEMENT_CHAR: + sloppy_chars[i] = char + + decoding_table = ''.join(sloppy_chars) encoding_table = codecs.charmap_build(decoding_table) class Codec(codecs.Codec): From f1ea0de73adadc60dac8167687c9692cf1b97c55 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 10 Nov 2013 17:05:47 -0500 Subject: [PATCH 110/502] allow sloppy codecs to be cached --- ftfy/bad_codecs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py index ff1fd58d..e0ad11f7 100644 --- a/ftfy/bad_codecs/__init__.py +++ b/ftfy/bad_codecs/__init__.py @@ -24,7 +24,7 @@ def search_function(encoding): codec = CODEC_INFO elif norm_encoding.startswith('sloppy_'): from ftfy.bad_codecs.sloppy import CODECS - return CODECS.get(norm_encoding) + codec = CODECS.get(norm_encoding) if codec is not None: _cache[encoding] = codec From d0b33cc0727bcd5d7d0fa9eaaaa76420e8b43762 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 11 Nov 2013 01:34:38 -0500 Subject: [PATCH 111/502] fix expected output in 'plan_so_far' --- ftfy/fixes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 3de9c263..6051377b 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -119,7 +119,7 @@ def fix_text_encoding(text): cost += 2 # We need pretty solid evidence to decode from Windows-1251 (Cyrillic). - if ('sloppy_encode', 'windows-1251') in plan_so_far: + if ('encode', 'sloppy-windows-1251') in plan_so_far: cost += 5 if cost < best_cost: From ec40797de51d4572efd3c05ce33271fda5141304 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 11 Nov 2013 01:48:34 -0500 Subject: [PATCH 112/502] Simplify how charmaps are generated --- ftfy/chardata.py | 44 +++++++------------------------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 64485324..1d652d9c 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -46,44 +46,14 @@ def _build_regexes(): are between U+0000 and U+007F. """ charmaps = {} - inverse_charmaps = {} - encoding_regexes = { - 'ascii': re.compile('^[\x00-\x7f]*$'), - } + + # Define a regex that matches ASCII text. + encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')} + + # For each character map encoding we care about, make a regex that contains + # all the characters that that encoding supports, and a mapping from those + # characters to sorta-bytes. for encoding in CHARMAP_ENCODINGS: - charmap = {} - inverse_charmap = {} - for codepoint in range(0, 0x80): - # ASCII characters map to themselves. - charmap[codepoint] = unichr(codepoint) - inverse_charmap[codepoint] = unichr(codepoint) - for codepoint in range(0x80, 0x100): - # The other characters map to characters from U+0080 to U+00FF, - # standing in for bytes. We call these sorta-bytes. - char = unichr(codepoint) - - # Turn the sorta-byte into a byte, and try decoding it. - encoded_char = char.encode('latin-1') - try: - # It decoded, so that's the character we need to add to our - # character map. - decoded_char = encoded_char.decode(encoding) - except ValueError: - # It didn't decode, so we should be "sloppy" and add the - # sorta-byte itself to the character map. - decoded_char = char - charmap[ord(decoded_char)] = char - inverse_charmap[ord(char)] = decoded_char - - # Put together all the characters we got, and make a regular expression - # that matches all of them. - charlist = [unichr(codept) for codept in sorted(charmap.keys()) - if codept >= 0x80] - regex = '^[\x00-\x7f{0}]*$'.format(''.join(charlist)) - - # Store the mapping and the regex in the dictionaries we're returning. - charmaps[encoding] = charmap - inverse_charmaps[encoding] = inverse_charmap latin1table = ''.join(unichr(i) for i in range(256)) charlist = latin1table.encode('latin-1').decode(encoding) regex = '^[\x00-\x7f{}]*$'.format(charlist) From e59510e116ebcdcc746726db6c8b29b9b2614487 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 11 Nov 2013 02:05:45 -0500 Subject: [PATCH 113/502] fix outdated docs --- ftfy/chardata.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 1d652d9c..6e434d7b 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -24,23 +24,7 @@ def _build_regexes(): """ - [outdated docs] - - CHARMAPS contains mappings from bytes to characters, for each single-byte - encoding we know about. - - We don't use Python's decoders here because they're too strict. Many - non-Python programs will leave mysterious bytes alone instead of raising - an error or removing them. For example, Python will not decode 0x81 in - Windows-1252 because it doesn't map to anything. Other systems will decode - it to U+0081, which actually makes no sense because that's a meaningless - control character from Latin-1, but I guess at least it preserves some - information that ftfy can take advantage of. - - So that's what we do. When other systems decode 0x81 as U+0081, we match - their behavior in case it helps us get reasonable text. - - Meanwhile, ENCODING_REGEXES contain reasonably fast ways to detect if we + ENCODING_REGEXES contain reasonably fast ways to detect if we could represent a given string in a given encoding. The simplest one is the 'ascii' detector, which of course just determines if all characters are between U+0000 and U+007F. @@ -50,9 +34,6 @@ def _build_regexes(): # Define a regex that matches ASCII text. encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')} - # For each character map encoding we care about, make a regex that contains - # all the characters that that encoding supports, and a mapping from those - # characters to sorta-bytes. for encoding in CHARMAP_ENCODINGS: latin1table = ''.join(unichr(i) for i in range(256)) charlist = latin1table.encode('latin-1').decode(encoding) From abef9f4c45ce99fae13f0946c48fff2dc3a9d04e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 11 Nov 2013 21:53:22 -0500 Subject: [PATCH 114/502] add apply_plan, which applies the result of fix_text_and_explain --- ftfy/fixes.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 6051377b..f9091ec8 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -102,12 +102,17 @@ def fix_text_encoding(text): The best version of the text is found using :func:`ftfy.badness.text_cost`. """ + text, plan = fix_text_and_explain(text) + return text + + +def fix_text_and_explain(text): best_version = text best_cost = text_cost(text) plan_so_far = [] while True: prevtext = text - text, plan = fix_text_and_explain(text) + text, plan = fix_one_step_and_explain(text) plan_so_far.extend(plan) cost = text_cost(text) @@ -126,12 +131,14 @@ def fix_text_encoding(text): best_cost = cost best_version = text if text == prevtext: - return best_version + return best_version, plan_so_far -def fix_text_and_explain(text): + +def fix_one_step_and_explain(text): """ Performs a single step of re-encoding text that's been decoded incorrectly. - It returns the decoded text, plus a structure explaining what it did. + It returns the decoded text, plus a 'plan' for how to reproduce what it + did. (`apply_plan` can use this object.) This structure could be used for more than it currently is, but we at least use it to track whether we had to intepret text as an old encoding such as @@ -212,9 +219,22 @@ def fix_text_and_explain(text): # such an ambiguous case, but perhaps a version of it with better # heuristics would. Anyway, ftfy should not claim to solve it. - return text, [('give up', None)] + # Return the text unchanged; the plan is empty. + return text, [] +def apply_plan(obj, plan): + for operation, encoding in plan: + if operation == 'encode': + obj = obj.encode(encoding) + elif operation == 'decode': + obj = obj.decode(encoding) + else: + raise ValueError("Unknown plan step: %s" % operation) + + return obj + + HTML_ENTITY_RE = re.compile(r"&#?\w{0,8};") def unescape_html(text): """ From 83311d2f7254195c5396287eeca443c14b65b473 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 23 Nov 2013 02:57:27 -0500 Subject: [PATCH 115/502] remove redundancy for a faster regex --- ftfy/chardata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 6e434d7b..dbf67049 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -37,7 +37,7 @@ def _build_regexes(): for encoding in CHARMAP_ENCODINGS: latin1table = ''.join(unichr(i) for i in range(256)) charlist = latin1table.encode('latin-1').decode(encoding) - regex = '^[\x00-\x7f{}]*$'.format(charlist) + regex = '^[{}]*$'.format(charlist) encoding_regexes[encoding] = re.compile(regex) return encoding_regexes ENCODING_REGEXES = _build_regexes() From 9414c21a8b651256cfc29c39bde578d2d38d0ece Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 23 Nov 2013 03:58:27 -0500 Subject: [PATCH 116/502] Remove extra output that was slowing down tests. (So *that's* why it was taking longer!) --- tests/test_unicode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 39f345f8..d17d27ad 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -22,7 +22,7 @@ def test_all_bmp_characters(): if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): garble = char.encode('utf-8').decode('latin-1') garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') - eq_(char_names(fix_text_encoding(garble)), char_names(char), fix_text_and_explain(garble)) + eq_(char_names(fix_text_encoding(garble)), char_names(char)) eq_(char_names(fix_text_encoding(garble2)), char_names(char)) phrases = [ From 3f05ce7fbea025b6e12fdb9b60dd2130a434f11d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 27 Nov 2013 02:15:37 -0500 Subject: [PATCH 117/502] finish the guess_bytes function --- .gitignore | 1 - ftfy/__init__.py | 11 +++++------ ftfy/autodecode.py | 21 -------------------- ftfy/guess_bytes.py | 47 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_bytes.py | 31 ++++++++++++++++++++++++++++++ 5 files changed, 83 insertions(+), 28 deletions(-) delete mode 100644 ftfy/autodecode.py create mode 100644 ftfy/guess_bytes.py create mode 100644 tests/test_bytes.py diff --git a/.gitignore b/.gitignore index 684ffe97..402af6e9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,3 @@ dist *.egg-info build _build -.idea \ No newline at end of file diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 68c17752..2bc32da4 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -8,6 +8,7 @@ from __future__ import unicode_literals from ftfy import fixes +from ftfy.guess_bytes import guess_bytes from ftfy.fixes import fix_text_encoding from ftfy.compatibility import PYTHON34_OR_LATER import unicodedata @@ -165,18 +166,16 @@ def fix_file(input_file, Fix text that is found in a file. If the file is being read as Unicode text, use that. If it's being read as - bytes, then we'll try to decode each line as Latin-1, and then fix it if it - was really UTF-8 or something. - - That's kind of awful, but it will have to suffice until we have a working - encoding detector. + bytes, then unfortunately, we have to guess what encoding it is. We'll try + a few common encodings, but we make no promises. See `guess_bytes.py` for + how this is done. The output is a stream of fixed lines of text. """ entities = fix_entities for line in input_file: if isinstance(line, bytes): - line = line.decode('latin-1') + line, encoding = guess_bytes(line) if fix_entities == 'auto' and '<' in line and '>' in line: entities = False yield fix_text_segment( diff --git a/ftfy/autodecode.py b/ftfy/autodecode.py deleted file mode 100644 index 35773797..00000000 --- a/ftfy/autodecode.py +++ /dev/null @@ -1,21 +0,0 @@ -from ftfy.chardata import possible_encoding -import codecs -import ftfy.bad_codecs.utf8_variants - -def decode_bytes(bstring): - if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): - return bstring.decode('utf-16'), 'utf-16' - - try: - return bstring.decode('utf-8'), 'utf-8') - except UnicodeDecodeError: - pass - - try: - return bstring.decode('utf-8-variants'), 'utf-8-variants') - except UnicodeDecodeError: - pass - - return bstring.decode('latin-1'), 'latin-1' - - diff --git a/ftfy/guess_bytes.py b/ftfy/guess_bytes.py new file mode 100644 index 00000000..5cb722f0 --- /dev/null +++ b/ftfy/guess_bytes.py @@ -0,0 +1,47 @@ +from ftfy.chardata import possible_encoding +import codecs +import ftfy.bad_codecs.utf8_variants + +def guess_bytes(bstring): + """ + If you have some bytes in an unknown encoding, here's a reasonable + strategy for decoding them, by trying a few common encodings that + can be distinguished from each other. + + This is not a magic bullet. If the bytes are coming from some MySQL + database with the "character set" set to ISO Elbonian, this won't figure + it out. + + The encodings we try are: + + - UTF-16 with a byte order mark, because a UTF-16 byte order mark looks + like nothing else + - UTF-8, because it's the global de facto standard + - "utf-8-variants", because it's what people actually implement when they + think they're doing UTF-8 + - MacRoman, because Microsoft Office thinks it's still a thing, and it + can be distinguished by its line breaks. (If there are no line breaks in + the string, though, you're out of luck.) + - "sloppy-windows-1252", the Latin-1-like encoding that is the most common + single-byte encoding + """ + if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): + return bstring.decode('utf-16'), 'utf-16' + + byteset = set(bytes(bstring)) + byte_ed, byte_c0, byte_CR, byte_LF = b'\xed\xc0\r\n' + + try: + if byte_ed in byteset or byte_c0 in byteset: + return bstring.decode('utf-8-variants'), 'utf-8-variants' + else: + return bstring.decode('utf-8'), 'utf-8' + except UnicodeDecodeError: + pass + + if byte_CR in bstring and byte_LF not in bstring: + return bstring.decode('macroman'), 'macroman' + else: + return bstring.decode('sloppy-windows-1252'), 'sloppy-windows-1252' + + diff --git a/tests/test_bytes.py b/tests/test_bytes.py new file mode 100644 index 00000000..0599e99b --- /dev/null +++ b/tests/test_bytes.py @@ -0,0 +1,31 @@ +from ftfy.guess_bytes import guess_bytes +from nose.tools import eq_ + +TEST_ENCODINGS = [ + 'utf-16', 'utf-8', 'sloppy-windows-1252' +] + +TEST_STRINGS = [ + u'Renée\nFleming', u'Noël\nCoward', u'Señor\nCardgage', + u'€ • £ • ¥', u'¿Qué?' +] + +def check_bytes_decoding(string): + for encoding in TEST_ENCODINGS: + result_str, result_encoding = guess_bytes(string.encode(encoding)) + eq_(result_str, string) + eq_(result_encoding, encoding) + + if '\n' in string: + old_mac_bytes = string.replace('\n', '\r').encode('macroman') + result_str, result_encoding = guess_bytes(old_mac_bytes) + eq_(result_str, string.replace('\n', '\r')) + +def test_guess_bytes(): + for string in TEST_STRINGS: + yield check_bytes_decoding, string + + bowdlerized_null = b'null\xc0\x80separated' + result_str, result_encoding = guess_bytes(bowdlerized_null) + eq_(result_str, u'null\x00separated') + eq_(result_encoding, u'utf-8-variants') From d27b474ee230a945cd7f2a440a98452825e48a87 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 27 Nov 2013 02:33:02 -0500 Subject: [PATCH 118/502] version 3.1.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8539c5e6..bf3012fd 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.0.6', + version='3.1.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='dev@lumino.so', license="MIT", From bc96f567b114944c6a3953edea97df9a55d03709 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 27 Nov 2013 02:59:29 -0500 Subject: [PATCH 119/502] put coding line in test_bytes --- tests/test_bytes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_bytes.py b/tests/test_bytes.py index 0599e99b..58453e1f 100644 --- a/tests/test_bytes.py +++ b/tests/test_bytes.py @@ -1,3 +1,4 @@ +# coding: utf-8 from ftfy.guess_bytes import guess_bytes from nose.tools import eq_ From 653321ae7daa8a64ac6aee16c9a88b7587118423 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 27 Nov 2013 03:14:03 -0500 Subject: [PATCH 120/502] Update docstrings based on existence of Python 3.3.3 --- ftfy/__init__.py | 7 ++++--- ftfy/fixes.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 2bc32da4..5c0a84eb 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -62,9 +62,10 @@ def fix_text(text, Based on the options you provide, ftfy applies these steps in order: - - If `remove_unsafe_private_use` is True, it removes a range of unassigned - characters that could trigger Python `bug 18183`_. This will default - to False starting on Python 3.4, when the bugfix will be released. + - If `remove_unsafe_private_use` is True, it removes a range of private-use + characters that could trigger a Python bug. The bug is fixed in + the most recent versions of Python, so this will default to False + starting on Python 3.4. - If `fix_entities` is True, replace HTML entities with their equivalent characters. If it's "auto" (the default), then consider replacing HTML entities, but don't do so in text where you have seen a pair of actual diff --git a/ftfy/fixes.py b/ftfy/fixes.py index f9091ec8..d5000ccf 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -342,14 +342,14 @@ def remove_bom(text): def remove_unsafe_private_use(text): r""" Python 3.3's Unicode support isn't perfect, and in fact there are certain - string operations that will crash it with a SystemError: + string operations that will crash some versions of it with a SystemError: http://bugs.python.org/issue18183 You can trigger the bug by running `` '\U00010000\U00100000'.lower() ``. - The best solution on Python 3.3 is to remove all characters from - Supplementary Private Use Area B, using a regex that is known not to crash - given those characters. + The best solution is to remove all characters from Supplementary Private + Use Area B, using a regex that is known not to crash given those + characters. These are the characters from U+100000 to U+10FFFF. It's sad to lose an entire plane of Unicode, but on the other hand, these characters are not @@ -362,6 +362,10 @@ def remove_unsafe_private_use(text): >>> print(remove_unsafe_private_use('\U0001F4A9\U00100000')) 💩 + + This fixer is off by default in Python 3.4 or later. (The bug is actually + fixed in 3.3.3 and 2.7.6, but I don't want the default behavior to change + based on a micro version upgrade of Python.) """ return UNSAFE_PRIVATE_USE_RE.sub('', text) From be1718812256eb3c38abc988a973daaa9b6883f8 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 27 Nov 2013 12:19:21 -0500 Subject: [PATCH 121/502] Output correct 'plans' and test them --- ftfy/bad_codecs/utf8_variants.py | 3 ++- ftfy/fixes.py | 8 ++++++-- tests/test_unicode.py | 8 +++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 7478da47..915b0014 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -11,7 +11,8 @@ The easiest way to use the codec is simply to import `ftfy.bad_codecs`: >>> import ftfy.bad_codecs - >>> b'here comes a null! \xc0\x80'.decode('utf-8-var') + >>> result = b'here comes a null! \xc0\x80'.decode('utf-8-var') + >>> print repr(result).lstrip('u') 'here comes a null! \x00' The codec does not at all enforce "correct" CESU-8. For example, the Unicode diff --git a/ftfy/fixes.py b/ftfy/fixes.py index f9091ec8..7cf67d1e 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -109,6 +109,7 @@ def fix_text_encoding(text): def fix_text_and_explain(text): best_version = text best_cost = text_cost(text) + best_plan = [] plan_so_far = [] while True: prevtext = text @@ -130,8 +131,9 @@ def fix_text_and_explain(text): if cost < best_cost: best_cost = cost best_version = text + best_plan = list(plan_so_far) if text == prevtext: - return best_version, plan_so_far + return best_version, best_plan def fix_one_step_and_explain(text): @@ -198,7 +200,9 @@ def fix_one_step_and_explain(text): encoded = text.encode('latin-1') try: fixed = encoded.decode('windows-1252') - steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] + steps = [] + if fixed != text: + steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] return fixed, steps except UnicodeDecodeError: # This text contained characters that don't even make sense diff --git a/tests/test_unicode.py b/tests/test_unicode.py index d17d27ad..a2c1cd5e 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from ftfy.fixes import fix_text_encoding, fix_text_and_explain +from ftfy.fixes import fix_text_encoding, fix_text_and_explain, apply_plan import unicodedata import sys from nose.tools import eq_ @@ -22,8 +22,10 @@ def test_all_bmp_characters(): if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): garble = char.encode('utf-8').decode('latin-1') garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') - eq_(char_names(fix_text_encoding(garble)), char_names(char)) - eq_(char_names(fix_text_encoding(garble2)), char_names(char)) + for garb in (garble, garble2): + fixed, plan = fix_text_and_explain(garb) + eq_(fixed, char) + eq_(apply_plan(garb, plan), char) phrases = [ u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", From 8aafcca01d804043e7df77d3e9ee69de28986513 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 2 Dec 2013 01:49:55 -0500 Subject: [PATCH 122/502] Clean up, add some missing pieces, document a lot For the purpose of documentation, and also because it's generally useful, this adds the `explain_unicode` function, which prints the name and codepoint of each character in a Unicode string. It also adds aliases such as 'sloppy-cp1252' for 'sloppy-windows-1252', and adds the 'sloppy-cp854' codec. --- ftfy/__init__.py | 19 +++++ ftfy/bad_codecs/__init__.py | 63 +++++++++++++++- ftfy/bad_codecs/sloppy.py | 121 ++++++++++++++++++++++++++++--- ftfy/bad_codecs/utf8_variants.py | 112 ++++++++++++++++++++++++---- ftfy/cli.py | 8 +- ftfy/fixes.py | 69 ++++++++++-------- tests/test_unicode.py | 2 +- 7 files changed, 327 insertions(+), 67 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 5c0a84eb..3d44e287 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -11,8 +11,10 @@ from ftfy.guess_bytes import guess_bytes from ftfy.fixes import fix_text_encoding from ftfy.compatibility import PYTHON34_OR_LATER +import ftfy.bad_codecs import unicodedata import warnings +ftfy.bad_codecs.ok() def fix_text(text, @@ -239,6 +241,23 @@ def fix_text_segment(text, return text +def explain_unicode(text): + """ + A utility method that's useful for debugging mysterious Unicode. + """ + for char in text: + if char.isprintable(): + display = char + else: + display = char.encode('unicode-escape').decode('ascii') + print('U+{code:04X} {display:<7} {name}'.format( + display=display, + code=ord(char), + name=unicodedata.name(char, '') + )) + + + def fix_bad_encoding(text): """ Kept for compatibility with previous versions of ftfy. diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py index e0ad11f7..c2386309 100644 --- a/ftfy/bad_codecs/__init__.py +++ b/ftfy/bad_codecs/__init__.py @@ -1,9 +1,41 @@ -import codecs +r""" +Give Python the ability to decode some common, flawed encodings. + +Python does not want you to be sloppy with your text. Its encoders and decoders +("codecs") follow the relevant standards whenever possible, which means that +when you get text that *doesn't* follow those standards, you'll probably fail +to decode it. Or you might succeed at decoding it for implementation-specific +reasons, which is perhaps worse. + +There are some encodings out there that Python wishes didn't exist, which are +widely used outside of Python: + +- "utf-8-variants", a family of not-quite-UTF-8 encodings, including the + ever-popular CESU-8 and "Java modified UTF-8". +- "Sloppy" versions of character map encodings, where bytes that don't map to + anything will instead map to the Unicode character with the same number. + +Simply importing this module, or in fact any part of the `ftfy` package, will +make these new "bad codecs" available to Python through the standard Codecs +API. You never have to actually call any functions inside `ftfy.bad_codecs`. + +However, if you want to call something because your code checker insists on it, +you can call ``ftfy.bad_codecs.ok()``. + +A quick example of decoding text that's encoded in CESU-8: + + >>> import ftfy.bad_codecs + >>> print(b'\xed\xa0\xbd\xed\xb8\x8d'.decode('utf-8-variants')) + 😍 +""" +from __future__ import unicode_literals from encodings import normalize_encoding +import codecs _cache = {} -# Define some aliases for 'utf-8-variants' +# Define some aliases for 'utf-8-variants'. All hyphens get turned into +# underscores, because of `normalize_encoding`. UTF8_VAR_NAMES = ( 'utf_8_variants', 'utf8_variants', 'utf_8_variant', 'utf8_variant', @@ -14,6 +46,19 @@ def search_function(encoding): + """ + Register our "bad codecs" with Python's codecs API. This involves adding + a search function that takes in an encoding name, and returns a codec + for that encoding if it knows one, or None if it doesn't. + + The encodings this will match are: + + - Encodings of the form 'sloppy-windows-NNNN' or 'sloppy-iso-8859-N', + where the non-sloppy version is an encoding that leaves some bytes + unmapped to characters. + - The 'utf-8-variants' encoding, which has the several aliases seen + above. + """ if encoding in _cache: return _cache[encoding] @@ -32,4 +77,18 @@ def search_function(encoding): return codec +def ok(): + """ + A feel-good function that gives you something to call after importing + this package. + + Why is this here? Pyflakes. Pyflakes gets upset when you import a module + and appear not to use it. It doesn't know that you're using it when + you use the ``unicode.encode`` and ``bytes.decode`` methods with certain + encodings. + """ + pass + + codecs.register(search_function) + diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index f2afc28f..5a03faed 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -1,43 +1,138 @@ +r""" +Decodes single-byte encodings, filling their "holes" in the same messy way that +everyone else does. + +A single-byte encoding maps each byte to a Unicode character, except that some +bytes are left unmapped. In the commonly-used Windows-1252 encoding, for +example, bytes 0x81 and 0x8D, among others, have no meaning. + +Python, wanting to preserve some sense of decorum, will handle these bytes +as errors. But Windows knows that 0x81 and 0x8D are possible bytes and they're +different from each other. It just hasn't defined what they are in terms of +Unicode. + +Software that has to interoperate with Windows-1252 and Unicode -- such as all +the common Web browsers -- will pick some Unicode characters for them to map +to, and the characters they pick are the Unicode characters with the same +numbers: U+0081 and U+008D. This is the same as what Latin-1 does, and the +resulting characters tend to fall into a range of Unicode that's set aside for +obselete Latin-1 control characters anyway. + +These sloppy codecs let Python do the same thing, thus interoperating with +other software that works this way. It defines a sloppy version of many +single-byte encodings with holes. (There is no need for a sloppy version of +an encoding without holes: for example, there is no such thing as +sloppy-iso-8859-2 or sloppy-macroman.) + +The following encodings will become defined: + +- sloppy-windows-1250 (Central European, sort of based on ISO-8859-2) +- sloppy-windows-1251 (Cyrillic) +- sloppy-windows-1252 (Western European, based on Latin-1) +- sloppy-windows-1253 (Greek, sort of based on ISO-8859-7) +- sloppy-windows-1254 (Turkish, based on ISO-8859-9) +- sloppy-windows-1255 (Hebrew, based on ISO-8859-8) +- sloppy-windows-1256 (Arabic) +- sloppy-windows-1257 (Baltic, based on ISO-8859-13) +- sloppy-windows-1258 (Vietnamese) +- sloppy-cp874 (Thai, based on ISO-8859-11) +- sloppy-iso-8859-3 (Maltese and Esperanto, I guess) +- sloppy-iso-8859-6 (different Arabic) +- sloppy-iso-8859-7 (Greek) +- sloppy-iso-8859-8 (Hebrew) +- sloppy-iso-8859-11 (Thai) + +Aliases such as "sloppy-cp1252" for "sloppy-windows-1252" will also be +defined. + +Only sloppy-windows-1251 and sloppy-windows-1252 are used by the rest of ftfy; +the rest are rather uncommon. + +Here are some examples, using `ftfy.explain_unicode` to illustrate how +sloppy-windows-1252 merges Windows-1252 with Latin-1: + + >>> from ftfy import explain_unicode + >>> some_bytes = b'\x80\x81\x82' + >>> explain_unicode(some_bytes.decode('latin-1')) + U+0080 \x80 + U+0081 \x81 + U+0082 \x82 + + >>> explain_unicode(some_bytes.decode('windows-1252', 'replace')) + U+20AC € EURO SIGN + U+FFFD � REPLACEMENT CHARACTER + U+201A ‚ SINGLE LOW-9 QUOTATION MARK + + >>> explain_unicode(some_bytes.decode('sloppy-windows-1252')) + U+20AC € EURO SIGN + U+0081 \x81 + U+201A ‚ SINGLE LOW-9 QUOTATION MARK +""" from __future__ import unicode_literals import codecs from encodings import normalize_encoding REPLACEMENT_CHAR = '\ufffd' + def make_sloppy_codec(encoding): + """ + Take a codec name, and return a 'sloppy' version of that codec that can + encode and decode the unassigned bytes in that encoding. + + Single-byte encodings in the standard library are defined using some + boilerplate classes surrounding the functions that do the actual work, + `codecs.charmap_decode` and `charmap_encode`. This function, given an + encoding name, *defines* those boilerplate classes. + """ + # Make an array of all 256 possible bytes. all_bytes = bytearray(range(256)) + + # Get a list of what they would decode to in Latin-1. sloppy_chars = list(all_bytes.decode('latin-1')) + + # Get a list of what they decode to in the given encoding. Use the + # replacement character for unassigned bytes. decoded_chars = all_bytes.decode(encoding, errors='replace') + + # Update the sloppy_chars list. Each byte that was successfully decoded + # gets its decoded value in the list. The unassigned bytes are left as + # they are, which gives their decoding in Latin-1. for i, char in enumerate(decoded_chars): if char != REPLACEMENT_CHAR: sloppy_chars[i] = char - + + # Create the data structures that tell the charmap methods how to encode + # and decode in this sloppy encoding. decoding_table = ''.join(sloppy_chars) encoding_table = codecs.charmap_build(decoding_table) + # Now produce all the class boilerplate. Look at the Python source for + # `encodings.cp1252` for comparison; this is almost exactly the same, + # except I made it follow pep8. class Codec(codecs.Codec): - def encode(self,input,errors='strict'): - return codecs.charmap_encode(input,errors,encoding_table) + def encode(self, input, errors='strict'): + return codecs.charmap_encode(input, errors, encoding_table) - def decode(self,input,errors='strict'): - return codecs.charmap_decode(input,errors,decoding_table) + def decode(self, input, errors='strict'): + return codecs.charmap_decode(input, errors, decoding_table) class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): - return codecs.charmap_encode(input,self.errors,encoding_table)[0] + return codecs.charmap_encode(input, self.errors, encoding_table)[0] class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): - return codecs.charmap_decode(input,self.errors,decoding_table)[0] + return codecs.charmap_decode(input, self.errors, decoding_table)[0] - class StreamWriter(Codec,codecs.StreamWriter): + class StreamWriter(Codec, codecs.StreamWriter): pass - class StreamReader(Codec,codecs.StreamReader): + class StreamReader(Codec, codecs.StreamReader): pass return codecs.CodecInfo( - name='cp1252', + name='sloppy-' + encoding, encode=Codec().encode, decode=Codec().decode, incrementalencoder=IncrementalEncoder, @@ -46,13 +141,15 @@ class StreamReader(Codec,codecs.StreamReader): streamwriter=StreamWriter, ) +# Define a codec for each incomplete encoding. The resulting CODECS dictionary +# can be used by the main module of ftfy.bad_codecs. CODECS = {} INCOMPLETE_ENCODINGS = ( ['windows-%s' % num for num in range(1250, 1259)] + - ['iso-8859-%s' % num for num in (3, 6, 7, 8, 11)] + ['iso-8859-%s' % num for num in (3, 6, 7, 8, 11)] + + ['cp%s' % num for num in range(1250, 1259)] + ['cp874'] ) for _encoding in INCOMPLETE_ENCODINGS: _new_name = normalize_encoding('sloppy-' + _encoding) CODECS[_new_name] = make_sloppy_codec(_encoding) - diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 7478da47..373c98c0 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -8,7 +8,7 @@ This is particularly relevant in Python 3, which provides no other way of decoding CESU-8 or Java's encoding. [1] -The easiest way to use the codec is simply to import `ftfy.bad_codecs`: +The easiest way to use the codec is to simply import `ftfy.bad_codecs`: >>> import ftfy.bad_codecs >>> b'here comes a null! \xc0\x80'.decode('utf-8-var') @@ -29,7 +29,7 @@ If you encode with this codec, you get legitimate UTF-8. Decoding with this codec and then re-encoding is not idempotent, although encoding and then decoding is. So this module won't produce CESU-8 for you. Look for that -functionality in the sister module, "Broke That For You", coming approximately +functionality in the sister module, "Breaks Text For You", coming approximately never. [1] In a pinch, you can decode CESU-8 in Python 2 using the UTF-8 codec: first @@ -44,33 +44,85 @@ import codecs NAME = 'utf-8-variants' +# This regular expression matches all possible six-byte CESU-8 sequences. CESU8_RE = re.compile(b'\xed[\xa0-\xaf][\x80-\xbf]\xed[\xb0-\xbf][\x80-\xbf]') -# TODO: document and write tests class IncrementalDecoder(UTF8IncrementalDecoder): + """ + An incremental decoder that extends Python's built-in UTF-8 decoder. + + This encoder needs to take in bytes, possibly arriving in a stream, and + output the correctly decoded text. The general strategy for doing this + is to fall back on the real UTF-8 decoder whenever possible, because + the real UTF-8 decoder is way optimized, but to call specialized methods + we define here for the cases the real encoder isn't expecting. + """ def _buffer_decode(self, input, errors, final): + """ + Decode bytes that may be arriving in a stream, following the Codecs + API. + + `input` is the incoming sequence of bytes. `errors` tells us how to + handle errors, though we delegate all error-handling cases to the real + UTF-8 decoder to ensure correct behavior. `final` indicates whether + this is the end of the sequence, in which case we should raise an + error given incomplete input. + + Returns as much decoded text as possible, and the number of bytes + consumed. + """ + # decoded_segments are the pieces of text we have decoded so far, + # and position is our current position in the byte string. (Bytes + # before this position have been consumed, and bytes after it have + # yet to be decoded.) decoded_segments = [] position = 0 while True: + # Use _buffer_decode_step to decode a segment of text. decoded, consumed = self._buffer_decode_step( input[position:], errors, final ) if consumed == 0: + # Either there's nothing left to decode, or we need to wait + # for more input. Either way, we're done for now. break + + # Append the decoded text to the list, and update our position. decoded_segments.append(decoded) position += consumed if final: + # _buffer_decode_step must consume all the bytes when `final` is + # true. assert position == len(input) + return ''.join(decoded_segments), position def _buffer_decode_step(self, input, errors, final): + """ + There are three possibilities for each decoding step: + + - Decode as much real UTF-8 as possible. + - Decode a six-byte CESU-8 sequence at the current position. + - Decode a Java-style null at the current position. + + This method figures out which step is appropriate, and does it. + """ + # Get a reference to the superclass method that we'll be using for + # most of the real work. sup = UTF8IncrementalDecoder._buffer_decode + + # Find the next byte position that indicates a variant of UTF-8. + # CESU-8 sequences always start with 0xed, and Java nulls always + # start with 0xc0, both of which are conveniently impossible in + # real UTF-8. cutoff1 = input.find(b'\xed') cutoff2 = input.find(b'\xc0') + + # Set `cutoff` to whichever cutoff comes first. if cutoff1 != -1 and cutoff2 != -1: cutoff = min(cutoff1, cutoff2) elif cutoff1 != -1: @@ -78,29 +130,43 @@ def _buffer_decode_step(self, input, errors, final): elif cutoff2 != -1: cutoff = cutoff2 else: - # The UTF-8 decoder can handle it from here. + # The entire input can be decoded as UTF-8, so just do so. return sup(input, errors, final) - if input.startswith(b'\xc0'): - return self._buffer_decode_null(sup, input, errors, final) - elif input.startswith(b'\xed'): + if cutoff1 == 0: + # Decode a possible six-byte sequence starting with 0xed. return self._buffer_decode_surrogates(sup, input, errors, final) + elif cutoff2 == 0: + # Decode a possible two-byte sequence, 0xc0 0x80. + return self._buffer_decode_null(sup, input, errors, final) else: - # at this point, we know cutoff > 0 + # Decode the bytes up until the next weird thing as UTF-8. return sup(input[:cutoff], errors, False) @staticmethod def _buffer_decode_null(sup, input, errors, final): - nextchar = input[1:2] - if nextchar == b'': + """ + Decode the bytes 0xc0 0x80 as U+0000, like Java does. + """ + nextbyte = input[1:2] + if nextbyte == b'': if final: - return sup(input, errors, True) + # We found 0xc0 at the end of the stream, which is an error. + # Delegate to the superclass method to handle that error. + return sup(input, errors, final) else: + # We found 0xc0 and we don't know what comes next, so consume + # no bytes and wait. return '', 0 - elif nextchar == b'\x80': + elif nextbyte == b'\x80': + # We found the usual 0xc0 0x80 sequence, so decode it and consume + # two bytes. return '\u0000', 2 else: - return sup(b'\xc0', errors, True) + # We found 0xc0 followed by something else, which is an error. + # Whatever should happen is equivalent to what happens when the + # superclass is given just the byte 0xc0, with final=True. + return sup(b'\xc0', errors, final=True) @staticmethod def _buffer_decode_surrogates(sup, input, errors, final): @@ -119,11 +185,19 @@ def _buffer_decode_surrogates(sup, input, errors, final): """ if len(input) < 6: if final: - return sup(input, errors, True) + # We found 0xed near the end of the stream, and there aren't + # six bytes to decode. Delegate to the superclass method + # to handle this error. + return sup(input, errors, final) else: + # We found 0xed, the stream isn't over yet, and we don't know + # enough of the following bytes to decode anything, so consume + # zero bytes and wait. return '', 0 else: if CESU8_RE.match(input): + # If this is a CESU-8 sequence, do some math to pull out + # the intended 20-bit value, and consume six bytes. bytenums = bytes_to_ints(input[:6]) codepoint = ( ((bytenums[1] & 0x0f) << 16) + @@ -134,14 +208,20 @@ def _buffer_decode_surrogates(sup, input, errors, final): ) return unichr(codepoint), 6 else: + # This looked like a CESU-8 sequence, but it wasn't one. + # 0xed indicates the start of a three-byte sequence, so give + # three bytes to the superclass, so it can either decode them + # as a surrogate codepoint (on Python 2) or handle the error + # (on Python 3). return sup(input[:3], errors, False) -# The encoder is identical to UTF8. +# The encoder is identical to UTF-8. IncrementalEncoder = UTF8IncrementalEncoder -# Everything below here is basically boilerplate. +# Everything below here is boilerplate that matches the modules in the +# built-in `encodings` package. def encode(input, errors='strict'): return IncrementalEncoder(errors).encode(input, final=True), len(input) diff --git a/ftfy/cli.py b/ftfy/cli.py index 5c04103c..6ac83706 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -1,12 +1,8 @@ """ A simple command-line utility for fixing text found in a file. -This utility knows nothing about the initial encoding of the file, so it will -guess Latin-1 and then try to fix it in the very likely event that that guess -is wrong. - -That behavior is slower than strictly necessary, and possibly a bit risky, -so consider this utility a proof of concept. +Because files do not come with their encoding marked, it first runs the file +through `ftfy.guess_bytes`, then runs it through `ftfy.fix_text`. """ from ftfy import fix_file diff --git a/ftfy/fixes.py b/ftfy/fixes.py index d5000ccf..1d8ee1b0 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -8,9 +8,7 @@ from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS, CONTROL_CHARS) from ftfy.badness import text_cost -from ftfy.compatibility import (htmlentitydefs, unichr, bytes_to_ints, - UNSAFE_PRIVATE_USE_RE) -import ftfy.bad_codecs +from ftfy.compatibility import (htmlentitydefs, unichr, UNSAFE_PRIVATE_USE_RE) import re import sys @@ -23,9 +21,12 @@ that your code is *also* handling Unicode incorrectly. ftfy takes Unicode text as input. You should take these bytes and decode -them from the encoding you think they are in. If you're not sure, you can -try decoding it as 'latin-1' and letting ftfy take it from there. This may -or may not work, but at least it shouldn't make the situation worse. +them from the encoding you think they are in. If you're not sure what encoding +they're in: + +- First, try to find out. 'utf-8' is a good assumption. +- If the encoding is simply unknowable, try running your bytes through + ftfy.guess_bytes. As the name implies, this may not always be accurate. If you're confused by this, please read the Python Unicode HOWTO: @@ -35,6 +36,8 @@ def fix_text_encoding(text): r""" + Fix text with incorrectly-decoded garbage ("mojibake") whenever possible. + Something you will find all over the place, in real-world text, is text that's mistakenly encoded as utf-8, decoded in some ugly format like latin-1 or even Windows codepage 1252, and encoded as utf-8 again. @@ -74,7 +77,7 @@ def fix_text_encoding(text): We might have to deal with both Windows characters and raw control characters at the same time, especially when dealing with characters like - \x81 that have no mapping in Windows. This is a string that Python's + 0x81 that have no mapping in Windows. This is a string that Python's standard `.encode` and `.decode` methods cannot correct. >>> print(fix_text_encoding('This text is sad .â\x81”.')) @@ -102,11 +105,18 @@ def fix_text_encoding(text): The best version of the text is found using :func:`ftfy.badness.text_cost`. """ - text, plan = fix_text_and_explain(text) + text, plan = fix_encoding_and_explain(text) return text -def fix_text_and_explain(text): +def fix_encoding_and_explain(text): + """ + Re-decodes text that has been decoded incorrectly, and also return a + "plan" indicating all the steps required to fix it. + + To fix similar text in the same way, without having to detect anything, + you can use the ``apply_plan`` function. + """ best_version = text best_cost = text_cost(text) plan_so_far = [] @@ -136,13 +146,10 @@ def fix_text_and_explain(text): def fix_one_step_and_explain(text): """ - Performs a single step of re-encoding text that's been decoded incorrectly. - It returns the decoded text, plus a 'plan' for how to reproduce what it - did. (`apply_plan` can use this object.) - - This structure could be used for more than it currently is, but we at least - use it to track whether we had to intepret text as an old encoding such as - MacRoman or cp437. + Performs a single step of re-decoding text that's been decoded incorrectly. + + Returns the decoded text, plus a "plan" for how to reproduce what it + did. """ if isinstance(text, bytes): raise UnicodeError(BYTES_ERROR_TEXT) @@ -180,10 +187,6 @@ def fix_one_step_and_explain(text): # The next most likely case is that this is Latin-1 that was intended to # be read as Windows-1252, because those two encodings in particular are # easily confused. - # - # We don't need to check for possibilities such as Latin-1 that was - # intended to be read as MacRoman, because it is unlikely that any - # software has that confusion. if 'latin-1' in possible_1byte_encodings: if 'windows-1252' in possible_1byte_encodings: # This text is in the intersection of Latin-1 and @@ -210,20 +213,26 @@ def fix_one_step_and_explain(text): # encodings, and not the common case of Latin-1 vs. Windows-1252. # # Those cases are somewhat rare, and impossible to solve without false - # positives. If you're in one of these situations, you don't need an - # encoding fixer. You need something that heuristically guesses what - # the encoding is in the first place. - # - # It's a different problem, the one that the 'chardet' module is - # theoretically designed to solve. It probably *won't* solve it in - # such an ambiguous case, but perhaps a version of it with better - # heuristics would. Anyway, ftfy should not claim to solve it. + # positives. If you're in one of these situations, you should try using + # the `ftfy.guess_bytes` function. # Return the text unchanged; the plan is empty. return text, [] -def apply_plan(obj, plan): +def apply_plan(text, plan): + """ + Apply a plan for fixing the encoding of text. + + The plan is a list of tuples of the form (operation, encoding), where + `operation` is either 'encode' or 'decode', and `encoding` is an encoding + name such as 'utf-8' or 'latin-1'. + + Because only text can be encoded, and only bytes can be decoded, the plan + should alternate 'encode' and 'decode' steps, or else this function will + encounter an error. + """ + obj = text for operation, encoding in plan: if operation == 'encode': obj = obj.encode(encoding) @@ -329,7 +338,6 @@ def remove_control_chars(text): return text.translate(CONTROL_CHARS) - def remove_bom(text): r""" Remove a left-over byte-order mark. @@ -339,6 +347,7 @@ def remove_bom(text): """ return text.lstrip(unichr(0xfeff)) + def remove_unsafe_private_use(text): r""" Python 3.3's Unicode support isn't perfect, and in fact there are certain diff --git a/tests/test_unicode.py b/tests/test_unicode.py index d17d27ad..6673b6e0 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from ftfy.fixes import fix_text_encoding, fix_text_and_explain +from ftfy.fixes import fix_text_encoding, fix_encoding_and_explain import unicodedata import sys from nose.tools import eq_ From 139a6554c6f7db44014c27fe3eea4f4b7f068951 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 2 Dec 2013 01:54:32 -0500 Subject: [PATCH 123/502] fix a doctest that broke on Python 3 --- ftfy/bad_codecs/utf8_variants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 58449a44..252263ad 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -12,7 +12,7 @@ >>> import ftfy.bad_codecs >>> result = b'here comes a null! \xc0\x80'.decode('utf-8-var') - >>> print repr(result).lstrip('u') + >>> print(repr(result).lstrip('u')) 'here comes a null! \x00' The codec does not at all enforce "correct" CESU-8. For example, the Unicode From 30f65c6d0a7a85a9937618b64a966dd89317ecae Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 2 Dec 2013 02:28:51 -0500 Subject: [PATCH 124/502] Add coding lines to fix python2 --- ftfy/bad_codecs/__init__.py | 2 +- ftfy/bad_codecs/sloppy.py | 1 + ftfy/bad_codecs/utf8_variants.py | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py index c2386309..91928475 100644 --- a/ftfy/bad_codecs/__init__.py +++ b/ftfy/bad_codecs/__init__.py @@ -1,3 +1,4 @@ +# coding: utf-8 r""" Give Python the ability to decode some common, flawed encodings. @@ -91,4 +92,3 @@ def ok(): codecs.register(search_function) - diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index 5a03faed..a95a6879 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -1,3 +1,4 @@ +# coding: utf-8 r""" Decodes single-byte encodings, filling their "holes" in the same messy way that everyone else does. diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 252263ad..7a5103a1 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -90,7 +90,7 @@ def _buffer_decode(self, input, errors, final): # Either there's nothing left to decode, or we need to wait # for more input. Either way, we're done for now. break - + # Append the decoded text to the list, and update our position. decoded_segments.append(decoded) position += consumed @@ -174,13 +174,13 @@ def _buffer_decode_surrogates(sup, input, errors, final): """ When we have improperly encoded surrogates, we can still see the bits that they were meant to represent. - + The surrogates were meant to encode a 20-bit number, to which we add 0x10000 to get a codepoint. That 20-bit number now appears in this form: - + 11101101 1010abcd 10efghij 11101101 1011klmn 10opqrst - + The CESU8_RE above matches byte sequences of this form. Then we need to extract the bits and assemble a codepoint number from them. """ From 93b6c30e567df8053b98c57153aa2bd6c291412c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 2 Dec 2013 02:29:05 -0500 Subject: [PATCH 125/502] Fixes to make flake8 mostly happy --- ftfy/__init__.py | 15 ++++++++------- ftfy/badness.py | 2 +- ftfy/build_data.py | 9 ++++++--- ftfy/chardata.py | 4 +--- ftfy/compatibility.py | 4 ++-- ftfy/fixes.py | 21 ++++++++++++--------- ftfy/guess_bytes.py | 12 +++++------- 7 files changed, 35 insertions(+), 32 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 3d44e287..2c018fcc 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -7,14 +7,17 @@ """ from __future__ import unicode_literals + +# See the docstring for ftfy.bad_codecs to see what we're doing here. +import ftfy.bad_codecs +ftfy.bad_codecs.ok() + from ftfy import fixes from ftfy.guess_bytes import guess_bytes from ftfy.fixes import fix_text_encoding from ftfy.compatibility import PYTHON34_OR_LATER -import ftfy.bad_codecs import unicodedata import warnings -ftfy.bad_codecs.ok() def fix_text(text, @@ -50,8 +53,8 @@ def fix_text(text, ... 'doo…\033[0m')) I'm blue, da ba dee da ba doo... - >>> # This example string starts with a byte-order mark, even if you can't - >>> # see it on the Web. + >>> # This example string starts with a byte-order mark, even if + >>> # you can't see it on the Web. >>> print(fix_text('\ufeffParty like\nit’s 1999!')) Party like it's 1999! @@ -112,7 +115,7 @@ def fix_text(text, If you are certain your entire text is in the same encoding (though that encoding is possibly flawed), and do not mind performing operations on the whole text at once, use `fix_text_segment`. - + _`bug 18183`: http://bugs.python.org/issue18183 """ if isinstance(text, bytes): @@ -257,7 +260,6 @@ def explain_unicode(text): )) - def fix_bad_encoding(text): """ Kept for compatibility with previous versions of ftfy. @@ -267,4 +269,3 @@ def fix_bad_encoding(text): DeprecationWarning ) return fix_text_encoding(text) - diff --git a/ftfy/badness.py b/ftfy/badness.py index f336bcd8..f345d6e4 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -30,6 +30,7 @@ # = Whitespace # o = Other + def _make_weirdness_regex(): """ Creates a list of regexes that match 'weird' character sequences. @@ -124,4 +125,3 @@ def text_cost(text): (see :func:`sequence_weirdness`) plus the length. """ return sequence_weirdness(text) * 2 + len(text) - diff --git a/ftfy/build_data.py b/ftfy/build_data.py index c3665227..b09d780c 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -37,6 +37,7 @@ # = Whitespace # o = Other + def make_char_data_file(do_it_anyway=False): """ Build the compressed data file 'char_classes.dat' and write it to the @@ -59,13 +60,15 @@ def make_char_data_file(do_it_anyway=False): category = unicodedata.category(char) if category.startswith('L'): # letters - if unicodedata.name(char).startswith('LATIN')\ - and codepoint < 0x200: + if ( + unicodedata.name(char).startswith('LATIN') + and codepoint < 0x20 + ): if category == 'Lu': cclasses[codepoint] = 'L' else: cclasses[codepoint] = 'l' - else: # non-Latin letter, or close enough + else: # non-Latin letter, or close enough if category == 'Lu' or category == 'Lt': cclasses[codepoint] = 'A' elif category == 'Ll': diff --git a/ftfy/chardata.py b/ftfy/chardata.py index dbf67049..d5286696 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -9,7 +9,6 @@ import zlib from pkg_resources import resource_string from ftfy.compatibility import unichr -import ftfy.bad_codecs # These are the five encodings we will try to fix in ftfy, in the # order that they should be tried. @@ -29,8 +28,6 @@ def _build_regexes(): the 'ascii' detector, which of course just determines if all characters are between U+0000 and U+007F. """ - charmaps = {} - # Define a regex that matches ASCII text. encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')} @@ -57,6 +54,7 @@ def possible_encoding(text, encoding): CHAR_CLASS_STRING = zlib.decompress( resource_string(__name__, 'char_classes.dat') ).decode('ascii') + def chars_to_classes(string): """ Convert each Unicode character to a letter indicating which of many diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index 88f88b46..d76824aa 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -8,15 +8,15 @@ if sys.hexversion >= 0x03000000: from html import entities - htmlentitydefs = entities unichr = chr xrange = range PYTHON2 = False else: - import htmlentitydefs + import htmlentitydefs as entities unichr = unichr xrange = xrange PYTHON2 = True +htmlentitydefs = entities PYTHON34_OR_LATER = (sys.hexversion >= 0x03040000) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 9ce50205..21e430fb 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -56,8 +56,8 @@ def fix_text_encoding(text): .. note:: The following examples are written using unmarked literal strings, - but they are Unicode text. In Python 2 we have "unicode_literals" turned - on, and in Python 3 this is always the case. + but they are Unicode text. In Python 2 we have "unicode_literals" + turned on, and in Python 3 this is always the case. ftfy decodes text that looks like it was decoded incorrectly. It leaves alone text that doesn't. @@ -149,7 +149,7 @@ def fix_encoding_and_explain(text): def fix_one_step_and_explain(text): """ Performs a single step of re-decoding text that's been decoded incorrectly. - + Returns the decoded text, plus a "plan" for how to reproduce what it did. """ @@ -218,7 +218,7 @@ def fix_one_step_and_explain(text): # # Those cases are somewhat rare, and impossible to solve without false # positives. If you're in one of these situations, you should try using - # the `ftfy.guess_bytes` function. + # the `ftfy.guess_bytes` function. # Return the text unchanged; the plan is empty. return text, [] @@ -246,9 +246,11 @@ def apply_plan(text, plan): raise ValueError("Unknown plan step: %s" % operation) return obj - + HTML_ENTITY_RE = re.compile(r"&#?\w{0,8};") + + def unescape_html(text): """ Decode all three types of HTML entities/character references. @@ -286,6 +288,7 @@ def fixup(match): ANSI_RE = re.compile('\033\\[((?:\\d|;)*)([a-zA-Z])') + def remove_terminal_escapes(text): r""" Strip out "ANSI" terminal escape sequences, such as those that produce @@ -301,6 +304,7 @@ def remove_terminal_escapes(text): SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') + def uncurl_quotes(text): r""" Replace curly quotation marks with straight equivalents. @@ -363,22 +367,21 @@ def remove_unsafe_private_use(text): The best solution is to remove all characters from Supplementary Private Use Area B, using a regex that is known not to crash given those characters. - + These are the characters from U+100000 to U+10FFFF. It's sad to lose an entire plane of Unicode, but on the other hand, these characters are not assigned and never will be. If you get one of these characters and don't know what its purpose is, its purpose is probably to crash your code. - + If you were using these for actual private use, this might be inconvenient. You can turn off this fixer, of course, but I kind of encourage using Supplementary Private Use Area A instead. >>> print(remove_unsafe_private_use('\U0001F4A9\U00100000')) 💩 - + This fixer is off by default in Python 3.4 or later. (The bug is actually fixed in 3.3.3 and 2.7.6, but I don't want the default behavior to change based on a micro version upgrade of Python.) """ return UNSAFE_PRIVATE_USE_RE.sub('', text) - diff --git a/ftfy/guess_bytes.py b/ftfy/guess_bytes.py index 5cb722f0..37fa766b 100644 --- a/ftfy/guess_bytes.py +++ b/ftfy/guess_bytes.py @@ -1,6 +1,6 @@ -from ftfy.chardata import possible_encoding -import codecs -import ftfy.bad_codecs.utf8_variants +import ftfy.bad_codecs +ftfy.bad_codecs.ok() + def guess_bytes(bstring): """ @@ -11,7 +11,7 @@ def guess_bytes(bstring): This is not a magic bullet. If the bytes are coming from some MySQL database with the "character set" set to ISO Elbonian, this won't figure it out. - + The encodings we try are: - UTF-16 with a byte order mark, because a UTF-16 byte order mark looks @@ -27,7 +27,7 @@ def guess_bytes(bstring): """ if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): return bstring.decode('utf-16'), 'utf-16' - + byteset = set(bytes(bstring)) byte_ed, byte_c0, byte_CR, byte_LF = b'\xed\xc0\r\n' @@ -43,5 +43,3 @@ def guess_bytes(bstring): return bstring.decode('macroman'), 'macroman' else: return bstring.decode('sloppy-windows-1252'), 'sloppy-windows-1252' - - From f7eb8956ee1cc16990f7c31672bfc8d10e95187f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 2 Dec 2013 02:31:42 -0500 Subject: [PATCH 126/502] Fix unnecessary modulization of guess_bytes --- ftfy/__init__.py | 44 +++++++++++++++++++++++++++++++++++++++++++- ftfy/guess_bytes.py | 45 --------------------------------------------- tests/test_bytes.py | 2 +- 3 files changed, 44 insertions(+), 47 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 2c018fcc..8661cf21 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -13,7 +13,6 @@ ftfy.bad_codecs.ok() from ftfy import fixes -from ftfy.guess_bytes import guess_bytes from ftfy.fixes import fix_text_encoding from ftfy.compatibility import PYTHON34_OR_LATER import unicodedata @@ -244,6 +243,49 @@ def fix_text_segment(text, return text +def guess_bytes(bstring): + """ + If you have some bytes in an unknown encoding, here's a reasonable + strategy for decoding them, by trying a few common encodings that + can be distinguished from each other. + + This is not a magic bullet. If the bytes are coming from some MySQL + database with the "character set" set to ISO Elbonian, this won't figure + it out. + + The encodings we try are: + + - UTF-16 with a byte order mark, because a UTF-16 byte order mark looks + like nothing else + - UTF-8, because it's the global de facto standard + - "utf-8-variants", because it's what people actually implement when they + think they're doing UTF-8 + - MacRoman, because Microsoft Office thinks it's still a thing, and it + can be distinguished by its line breaks. (If there are no line breaks in + the string, though, you're out of luck.) + - "sloppy-windows-1252", the Latin-1-like encoding that is the most common + single-byte encoding + """ + if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): + return bstring.decode('utf-16'), 'utf-16' + + byteset = set(bytes(bstring)) + byte_ed, byte_c0, byte_CR, byte_LF = b'\xed\xc0\r\n' + + try: + if byte_ed in byteset or byte_c0 in byteset: + return bstring.decode('utf-8-variants'), 'utf-8-variants' + else: + return bstring.decode('utf-8'), 'utf-8' + except UnicodeDecodeError: + pass + + if byte_CR in bstring and byte_LF not in bstring: + return bstring.decode('macroman'), 'macroman' + else: + return bstring.decode('sloppy-windows-1252'), 'sloppy-windows-1252' + + def explain_unicode(text): """ A utility method that's useful for debugging mysterious Unicode. diff --git a/ftfy/guess_bytes.py b/ftfy/guess_bytes.py index 37fa766b..e69de29b 100644 --- a/ftfy/guess_bytes.py +++ b/ftfy/guess_bytes.py @@ -1,45 +0,0 @@ -import ftfy.bad_codecs -ftfy.bad_codecs.ok() - - -def guess_bytes(bstring): - """ - If you have some bytes in an unknown encoding, here's a reasonable - strategy for decoding them, by trying a few common encodings that - can be distinguished from each other. - - This is not a magic bullet. If the bytes are coming from some MySQL - database with the "character set" set to ISO Elbonian, this won't figure - it out. - - The encodings we try are: - - - UTF-16 with a byte order mark, because a UTF-16 byte order mark looks - like nothing else - - UTF-8, because it's the global de facto standard - - "utf-8-variants", because it's what people actually implement when they - think they're doing UTF-8 - - MacRoman, because Microsoft Office thinks it's still a thing, and it - can be distinguished by its line breaks. (If there are no line breaks in - the string, though, you're out of luck.) - - "sloppy-windows-1252", the Latin-1-like encoding that is the most common - single-byte encoding - """ - if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): - return bstring.decode('utf-16'), 'utf-16' - - byteset = set(bytes(bstring)) - byte_ed, byte_c0, byte_CR, byte_LF = b'\xed\xc0\r\n' - - try: - if byte_ed in byteset or byte_c0 in byteset: - return bstring.decode('utf-8-variants'), 'utf-8-variants' - else: - return bstring.decode('utf-8'), 'utf-8' - except UnicodeDecodeError: - pass - - if byte_CR in bstring and byte_LF not in bstring: - return bstring.decode('macroman'), 'macroman' - else: - return bstring.decode('sloppy-windows-1252'), 'sloppy-windows-1252' diff --git a/tests/test_bytes.py b/tests/test_bytes.py index 58453e1f..9fa401af 100644 --- a/tests/test_bytes.py +++ b/tests/test_bytes.py @@ -1,5 +1,5 @@ # coding: utf-8 -from ftfy.guess_bytes import guess_bytes +from ftfy import guess_bytes from nose.tools import eq_ TEST_ENCODINGS = [ From 002f33eae43d76e89a94847fd067459527ead83a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 2 Dec 2013 02:33:53 -0500 Subject: [PATCH 127/502] actually remove guess_bytes as a module --- ftfy/guess_bytes.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ftfy/guess_bytes.py diff --git a/ftfy/guess_bytes.py b/ftfy/guess_bytes.py deleted file mode 100644 index e69de29b..00000000 From bc629d0d9078a40600bdfd16475c905137c3fab1 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 2 Dec 2013 02:40:49 -0500 Subject: [PATCH 128/502] port explain_unicode to Python 2 --- ftfy/__init__.py | 4 ++-- ftfy/compatibility.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 8661cf21..e66bec0b 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -14,7 +14,7 @@ from ftfy import fixes from ftfy.fixes import fix_text_encoding -from ftfy.compatibility import PYTHON34_OR_LATER +from ftfy.compatibility import PYTHON34_OR_LATER, is_printable import unicodedata import warnings @@ -291,7 +291,7 @@ def explain_unicode(text): A utility method that's useful for debugging mysterious Unicode. """ for char in text: - if char.isprintable(): + if is_printable(char): display = char else: display = char.encode('unicode-escape').decode('ascii') diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index d76824aa..c218a0dc 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import sys import re +import unicodedata if sys.hexversion >= 0x03000000: from html import entities @@ -65,3 +66,14 @@ def bytes_to_ints(bytestring): return [ord(b) for b in bytestring] else: return bytestring + + +def is_printable(char): + """ + str.isprintable() is new in Python 3. It's useful in `explain_unicode`, so + let's make a crude approximation in Python 2. + """ + if PYTHON2: + return not unicodedata.category(char).startswith('C') + else: + return char.isprintable() From 7620e8b9f3d799f4baaa4fb640550e40ce7a7f78 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 28 Dec 2013 23:19:21 -0500 Subject: [PATCH 129/502] Fix bug when analyzing text that isn't at all UTF-8 --- ftfy/bad_codecs/utf8_variants.py | 4 +++- tests/test_encodings.py | 18 ++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 7a5103a1..626d8d8b 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -142,7 +142,9 @@ def _buffer_decode_step(self, input, errors, final): return self._buffer_decode_null(sup, input, errors, final) else: # Decode the bytes up until the next weird thing as UTF-8. - return sup(input[:cutoff], errors, False) + # Set final=True because 0xc0 and 0xed don't make sense in the + # middle of a sequence, in any variant. + return sup(input[:cutoff], errors, True) @staticmethod def _buffer_decode_null(sup, input, errors, final): diff --git a/tests/test_encodings.py b/tests/test_encodings.py index a2075801..e7f9ef97 100644 --- a/tests/test_encodings.py +++ b/tests/test_encodings.py @@ -1,12 +1,22 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from ftfy import bad_codecs +from ftfy import bad_codecs, guess_bytes from nose.tools import eq_ def test_cesu8(): - eq_(bad_codecs.search_function('cesu8').__class__, bad_codecs.search_function('cesu-8').__class__) + eq_(bad_codecs.search_function('cesu8').__class__, + bad_codecs.search_function('cesu-8').__class__) - test_bytes = b'\xed\xa6\x9d\xed\xbd\xb7 is an unassigned character, and \xc0\x80 is null' + test_bytes = (b'\xed\xa6\x9d\xed\xbd\xb7 is an unassigned character, ' + b'and \xc0\x80 is null') test_text = '\U00077777 is an unassigned character, and \x00 is null' - eq_(test_bytes.decode('cesu8'), test_text) \ No newline at end of file + eq_(test_bytes.decode('cesu8'), test_text) + +def test_russian_crash(): + thebytes = b'\xe8\xed\xe2\xe5\xed\xf2\xe0\xf0\xe8\xe7\xe0\xf6\xe8\xff ' + # We don't care what the result is, but this shouldn't crash + thebytes.decode('utf-8-variants', errors='replace') + + # This shouldn't crash either + guess_bytes(thebytes) From b42794dc750fecbf6a1e407580bb0259c3a74747 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 10 Jan 2014 15:23:57 -0500 Subject: [PATCH 130/502] the internal UTF-8 decoder doesn't take keyword arguments --- ftfy/__init__.py | 3 ++- ftfy/bad_codecs/utf8_variants.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index e66bec0b..a5b218fd 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -295,9 +295,10 @@ def explain_unicode(text): display = char else: display = char.encode('unicode-escape').decode('ascii') - print('U+{code:04X} {display:<7} {name}'.format( + print('U+{code:04X} {display:<7} [{category}] {name}'.format( display=display, code=ord(char), + category=unicodedata.category(char), name=unicodedata.name(char, '') )) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 626d8d8b..081218e8 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -169,7 +169,7 @@ def _buffer_decode_null(sup, input, errors, final): # We found 0xc0 followed by something else, which is an error. # Whatever should happen is equivalent to what happens when the # superclass is given just the byte 0xc0, with final=True. - return sup(b'\xc0', errors, final=True) + return sup(b'\xc0', errors, True) @staticmethod def _buffer_decode_surrogates(sup, input, errors, final): From df479e622f6791bc6a9f336eb28bec57722872d1 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 26 Jan 2014 20:29:05 -0500 Subject: [PATCH 131/502] remove_unsafe_private_use isn't necessary in current Python. As of Python 3.3.3, the bug that made `remove_unsafe_private_use` necessary is fixed. The function will remain there, but it's not used as a part of the `fix_text` process anymore. Along the way, added more comments and expanded docstrings. --- ftfy/__init__.py | 32 ++++++++++++++++++++------------ ftfy/fixes.py | 19 +++++++++++++------ 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index a5b218fd..8f50905a 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -14,13 +14,12 @@ from ftfy import fixes from ftfy.fixes import fix_text_encoding -from ftfy.compatibility import PYTHON34_OR_LATER, is_printable +from ftfy.compatibility import is_printable import unicodedata import warnings def fix_text(text, - remove_unsafe_private_use=(not PYTHON34_OR_LATER), fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, @@ -66,10 +65,6 @@ def fix_text(text, Based on the options you provide, ftfy applies these steps in order: - - If `remove_unsafe_private_use` is True, it removes a range of private-use - characters that could trigger a Python bug. The bug is fixed in - the most recent versions of Python, so this will default to False - starting on Python 3.4. - If `fix_entities` is True, replace HTML entities with their equivalent characters. If it's "auto" (the default), then consider replacing HTML entities, but don't do so in text where you have seen a pair of actual @@ -139,7 +134,6 @@ def fix_text(text, out.append( fix_text_segment( substring, - remove_unsafe_private_use=remove_unsafe_private_use, fix_entities=fix_entities, remove_terminal_escapes=remove_terminal_escapes, fix_encoding=fix_encoding_this_time, @@ -158,7 +152,6 @@ def fix_text(text, def fix_file(input_file, - remove_unsafe_private_use=True, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, @@ -185,7 +178,6 @@ def fix_file(input_file, entities = False yield fix_text_segment( line, - remove_unsafe_private_use=remove_unsafe_private_use, fix_entities=fix_entities, remove_terminal_escapes=remove_terminal_escapes, fix_encoding=entities, @@ -198,7 +190,6 @@ def fix_file(input_file, def fix_text_segment(text, - remove_unsafe_private_use=True, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, @@ -221,8 +212,6 @@ def fix_text_segment(text, fix_entities = False while True: origtext = text - if remove_unsafe_private_use: - text = fixes.remove_unsafe_private_use(text) if fix_entities: text = fixes.unescape_html(text) if remove_terminal_escapes: @@ -266,14 +255,33 @@ def guess_bytes(bstring): - "sloppy-windows-1252", the Latin-1-like encoding that is the most common single-byte encoding """ + # Look for a byte-order mark to recognize UTF-16 if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): return bstring.decode('utf-16'), 'utf-16' + # Make a set out of the bytes, so we can quickly test for the presence of + # certain bytes byteset = set(bytes(bstring)) byte_ed, byte_c0, byte_CR, byte_LF = b'\xed\xc0\r\n' try: if byte_ed in byteset or byte_c0 in byteset: + # Byte 0xed can only be used to encode a range of codepoints that + # are UTF-16 surrogates. UTF-8 does not use UTF-16 surrogates, so + # this byte is impossible in real UTF-8. Therefore, if we see + # bytes that look like UTF-8 but contain 0xed, what we're seeing + # is CESU-8, the variant that encodes UTF-16 surrogates instead + # of the original characters themselves. + # + # Byte 0xc0 is impossible because, numerically, it would only + # encode characters lower than U+40. Those already have single-byte + # representations, and UTF-8 requires using the shortest possible + # representation. However, Java hides the null codepoint, U+0, in a + # non-standard longer representation -- it encodes it as 0xc0 0x80 + # instead of 0x00, guaranteeing that 0x00 will never appear in the + # encoded bytes. + # + # The 'utf-8-variants' decoder can handle both of these cases. return bstring.decode('utf-8-variants'), 'utf-8-variants' else: return bstring.decode('utf-8'), 'utf-8' diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 21e430fb..ce7468f7 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -336,19 +336,26 @@ def remove_control_chars(text): - U+000E to U+001F - U+007F - It leaves alone these characters that are commonly used for formatting: + It leaves alone these characters that are used for formatting: - - TAB (U+0009) - - LF (U+000A) - - FF (U+000C) - - CR (U+000D) + - Tab (U+0009) + - Line Feed (U+000A) + - Form Feed (U+000C) + - Carriage Return (U+000D) + + Form Feed is the most borderline of these, but I've seen it used in + plain text files -- for example, as a large whitespace to separate + major sections of code. It has a purpose in text that it's still + used for, so we might as well leave it alone. """ return text.translate(CONTROL_CHARS) def remove_bom(text): r""" - Remove a left-over byte-order mark. + Remove a left-over byte-order mark. Byte-order marks are metadata about + UTF-16 encoded text and shouldn't appear in the text itself, but they can + arise due to Microsoft's confusion between UTF-16 and Unicode. >>> print(remove_bom(unichr(0xfeff) + "Where do you want to go today?")) Where do you want to go today? From 23a5ff950eccb73bdd46a33fd5fdfa670e63784b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Jan 2014 11:40:42 -0500 Subject: [PATCH 132/502] new Russian example --- tests/test_real_tweets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index f7731813..02c55eea 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -31,7 +31,8 @@ ('[x]\xa0©', '[x] ©'), ('2012—∞', '2012—∞'), ('Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.', - 'Con il corpo e lo spirito ammaccato, è come se nel cuore avessi un vetro conficcato.') + 'Con il corpo e lo spirito ammaccato, è come se nel cuore avessi un vetro conficcato.'), + ('приятности. РІСњВ¤', 'приятности. ❤'), ## Current false positives: #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", From f04798ab68c32339715b192e38b57c0c63586841 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Jan 2014 11:43:36 -0500 Subject: [PATCH 133/502] fix doctests for sloppy codecs --- ftfy/bad_codecs/sloppy.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index a95a6879..39a56e0a 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -55,19 +55,19 @@ >>> from ftfy import explain_unicode >>> some_bytes = b'\x80\x81\x82' >>> explain_unicode(some_bytes.decode('latin-1')) - U+0080 \x80 - U+0081 \x81 - U+0082 \x82 + U+0080 \x80 [Cc] + U+0081 \x81 [Cc] + U+0082 \x82 [Cc] >>> explain_unicode(some_bytes.decode('windows-1252', 'replace')) - U+20AC € EURO SIGN - U+FFFD � REPLACEMENT CHARACTER - U+201A ‚ SINGLE LOW-9 QUOTATION MARK + U+20AC € [Sc] EURO SIGN + U+FFFD � [So] REPLACEMENT CHARACTER + U+201A ‚ [Ps] SINGLE LOW-9 QUOTATION MARK >>> explain_unicode(some_bytes.decode('sloppy-windows-1252')) - U+20AC € EURO SIGN - U+0081 \x81 - U+201A ‚ SINGLE LOW-9 QUOTATION MARK + U+20AC € [Sc] EURO SIGN + U+0081 \x81 [Cc] + U+201A ‚ [Ps] SINGLE LOW-9 QUOTATION MARK """ from __future__ import unicode_literals import codecs From 81bed3b0176e3f35b9f06804fc12b5ec2892f2e9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Jan 2014 11:47:18 -0500 Subject: [PATCH 134/502] List 'ftfy.bad_codecs' as a package --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bf3012fd..c513fddb 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ url='http://github.com/LuminosoInsight/python-ftfy', platforms=["any"], description="Fixes some problems with Unicode text after the fact", - packages=['ftfy'], + packages=['ftfy', 'ftfy.bad_codecs'], package_data={'ftfy': ['char_classes.dat']}, classifiers=[ "Programming Language :: Python :: 2", From 4ca6f4d9d40c72b0ddd03afb55b1e8b6d604ef61 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Jan 2014 12:59:58 -0500 Subject: [PATCH 135/502] v3.1.1 - fix packaging problem --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c513fddb..7f533374 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.1.0', + version='3.1.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='dev@lumino.so', license="MIT", From cbb310469b67f225a3fd3c95ce34224b9f0c65fb Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Jan 2014 15:36:16 -0500 Subject: [PATCH 136/502] bugfix: as of v3.1.0, strings containing a backslash were never fixed --- ftfy/chardata.py | 4 ++-- tests/test_unicode.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index d5286696..5ff28b3e 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -32,9 +32,9 @@ def _build_regexes(): encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')} for encoding in CHARMAP_ENCODINGS: - latin1table = ''.join(unichr(i) for i in range(256)) + latin1table = ''.join(unichr(i) for i in range(128, 256)) charlist = latin1table.encode('latin-1').decode(encoding) - regex = '^[{}]*$'.format(charlist) + regex = '^[\x00-\x7f{}]*$'.format(charlist.replace('\\', '\\\\')) encoding_regexes[encoding] = re.compile(regex) return encoding_regexes ENCODING_REGEXES = _build_regexes() diff --git a/tests/test_unicode.py b/tests/test_unicode.py index e78dd070..45c416e4 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from ftfy.fixes import fix_text_encoding, fix_encoding_and_explain, apply_plan +from ftfy.fixes import fix_text_encoding, fix_encoding_and_explain, apply_plan, possible_encoding import unicodedata import sys from nose.tools import eq_ @@ -47,6 +47,14 @@ def check_phrase(text): eq_(fix_text_encoding(text), text) eq_(fix_text_encoding(text.encode('utf-8').decode('latin-1')), text) +def test_possible_encoding(): + for codept in range(256): + char = chr(codept) + assert possible_encoding(char, 'latin-1') + +def test_fix_with_backslash(): + eq_(fix_text_encoding(u"<40\\% vs \xe2\x89\xa540\\%"), "<40\\% vs ≥40\\%") + if __name__ == '__main__': test_all_bmp_characters() From d05e874accce30111ed57c09d07bf4f4bccb534f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Jan 2014 15:38:55 -0500 Subject: [PATCH 137/502] fix unicode literal for Python 2 --- setup.py | 2 +- tests/test_unicode.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7f533374..bc8e08c4 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.1.1', + version='3.1.2', maintainer='Luminoso Technologies, Inc.', maintainer_email='dev@lumino.so', license="MIT", diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 45c416e4..24d3bbf6 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -53,7 +53,7 @@ def test_possible_encoding(): assert possible_encoding(char, 'latin-1') def test_fix_with_backslash(): - eq_(fix_text_encoding(u"<40\\% vs \xe2\x89\xa540\\%"), "<40\\% vs ≥40\\%") + eq_(fix_text_encoding(u"<40\\% vs \xe2\x89\xa540\\%"), u"<40\\% vs ≥40\\%") if __name__ == '__main__': test_all_bmp_characters() From d0c782deee718390b7989c2f6a79e0af79ea011c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Jan 2014 17:06:14 -0500 Subject: [PATCH 138/502] new test: an interesting pile-up on top of curly quotes --- tests/test_real_tweets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 02c55eea..0e0248ce 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -33,6 +33,8 @@ ('Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.', 'Con il corpo e lo spirito ammaccato, è come se nel cuore avessi un vetro conficcato.'), ('приятности. РІСњВ¤', 'приятности. ❤'), + ('Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New “ Romance”.', + 'Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New " Romance".'), ## Current false positives: #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", From 1f4b0b3fa3335c5f4a69b9a87f0b94ea9bf25339 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 30 Jan 2014 03:52:58 -0500 Subject: [PATCH 139/502] add the streaming Twitter tester --- .gitignore | 1 + ftfy/__init__.py | 12 +++--- ftfy/streamtester/__init__.py | 0 ftfy/streamtester/oauth.py | 67 +++++++++++++++++++++++++++++++ ftfy/streamtester/streamtester.py | 66 ++++++++++++++++++++++++++++++ 5 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 ftfy/streamtester/__init__.py create mode 100644 ftfy/streamtester/oauth.py create mode 100644 ftfy/streamtester/streamtester.py diff --git a/.gitignore b/.gitignore index 402af6e9..6dc3e66a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist *.egg-info build _build +twitterlogs diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 8f50905a..8cc97417 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -274,12 +274,12 @@ def guess_bytes(bstring): # of the original characters themselves. # # Byte 0xc0 is impossible because, numerically, it would only - # encode characters lower than U+40. Those already have single-byte - # representations, and UTF-8 requires using the shortest possible - # representation. However, Java hides the null codepoint, U+0, in a - # non-standard longer representation -- it encodes it as 0xc0 0x80 - # instead of 0x00, guaranteeing that 0x00 will never appear in the - # encoded bytes. + # encode characters lower than U+0040. Those already have + # single-byte representations, and UTF-8 requires using the + # shortest possible representation. However, Java hides the null + # codepoint, U+0000, in a non-standard longer representation -- it + # encodes it as 0xc0 0x80 instead of 0x00, guaranteeing that 0x00 + # will never appear in the encoded bytes. # # The 'utf-8-variants' decoder can handle both of these cases. return bstring.decode('utf-8-variants'), 'utf-8-variants' diff --git a/ftfy/streamtester/__init__.py b/ftfy/streamtester/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ftfy/streamtester/oauth.py b/ftfy/streamtester/oauth.py new file mode 100644 index 00000000..92eaab1d --- /dev/null +++ b/ftfy/streamtester/oauth.py @@ -0,0 +1,67 @@ +# coding: utf-8 +from __future__ import unicode_literals +from twitter.oauth import OAuth +from twitter import oauth_dance, read_token_file +import os + + +AUTH_TOKEN_PATH = os.path.expanduser('~/.cache/oauth/twitter_ftfy.auth') + +def get_auth(): + """ + Twitter has some bizarre requirements about how to authorize an "app" to + use its API. + + The user of the app has to log in to get a secret token. That's fine. But + the app itself has its own "consumer secret" token. The app has to know it, + and the user of the app has to not know it. + + This is, of course, impossible. It's equivalent to DRM. Your computer can't + *really* make use of secret information while hiding the same information + from you. + + The threat appears to be that, if you have this super-sekrit token, you can + impersonate the app while doing something different. Well, of course you + can do that, because you *have the source code* and you can change it to do + what you want. You still have to log in as a particular user who has a + token that's actually secret, you know. + + Even developers of closed-source applications that use the Twitter API are + unsure what to do, for good reason. These "secrets" are not secret in any + cryptographic sense. A bit of Googling shows that the secret tokens for + every popular Twitter app are already posted on the Web. + + Twitter wants us to pretend this string can be kept secret, and hide this + secret behind a fig leaf like everybody else does. So that's what we've + done. + """ + def unhide(s): + """ + Do something mysterious and exactly as secure as every other Twitter + app. + """ + return ''.join([chr(ord(c) - 0x2800) for c in s]) + + fig_leaf = '⠴⡹⠹⡩⠶⠴⡶⡅⡂⡩⡅⠳⡏⡉⡈⠰⠰⡹⡥⡶⡈⡐⡍⡂⡫⡍⡗⡬⡒⡧⡶⡣⡰⡄⡧⡸⡑⡣⠵⡓⠶⠴⡁' + consumer_key = 'OFhyNd2Zt4Ba6gJGJXfbsw' + + if os.path.exists(AUTH_TOKEN_PATH): + token, token_secret = read_token_file(AUTH_TOKEN_PATH) + else: + authdir = os.path.dirname(AUTH_TOKEN_PATH) + if not os.path.exists(authdir): + os.makedirs(authdir) + token, token_secret = oauth_dance( + app_name='ftfy-tester', + consumer_key=consumer_key, + consumer_secret=unhide(fig_leaf), + token_filename=AUTH_TOKEN_PATH + ) + + return OAuth( + token=token, + token_secret=token_secret, + consumer_key=consumer_key, + consumer_secret=unhide(fig_leaf) + ) + diff --git a/ftfy/streamtester/streamtester.py b/ftfy/streamtester/streamtester.py new file mode 100644 index 00000000..a6c64268 --- /dev/null +++ b/ftfy/streamtester/streamtester.py @@ -0,0 +1,66 @@ +from twitter import TwitterStream, OAuth +import json +import time +import unicodedata +import os +from ftfy.fixes import fix_text_encoding, remove_unsafe_private_use +from ftfy.badness import sequence_weirdness +from ftfy.chardata import possible_encoding +from ftfy.streamtester.oauth import get_auth +from collections import defaultdict + +OUTPUT_DIR = './twitterlogs' +class TwitterTester: + def __init__(self): + self.lines_by_lang = defaultdict(list) + self.num_fixed = 0 + + def save_files(self): + if not os.path.exists(OUTPUT_DIR): + os.makedirs(OUTPUT_DIR) + for lang, lines in self.lines_by_lang.items(): + filename = 'tweets.{}.txt'.format(lang) + fullname = os.path.join(OUTPUT_DIR, filename) + langfile = open(fullname, 'a') + for line in lines: + print(line.replace('\n', ' '), file=langfile) + langfile.close() + self.lines_by_lang = defaultdict(list) + + def run_sample(self): + twitter_stream = TwitterStream(auth=get_auth()) + iterator = twitter_stream.statuses.sample() + count = 0 + for tweet in iterator: + if 'text' in tweet: + self.handle_tweet(tweet) + if count % 10000 == 0: + print('%d/%d fixed' % (self.num_fixed, count)) + count += 1 + if count % 10000 == 100: + self.save_files() + + def check_ftfy(self, text): + if not possible_encoding(text, 'ascii'): + fixed = fix_text_encoding(text) + if text != fixed: + # possibly filter common bots before printing + print(u'Text:\t{text}\nFixed:\t{fixed}\n'.format(text=text, fixed=fixed)) + self.num_fixed += 1 + + def handle_tweet(self, tweet): + text = tweet['text'] + self.check_ftfy(text) + if 'user' in tweet: + lang = tweet['user'].get('lang', 'NONE') + self.lines_by_lang[lang].append(tweet['text']) + + +def main(): + tester = TwitterTester() + tester.run_sample() + + +if __name__ == '__main__': + main() + From e81d4ab07e8903ebf76662d0654f45ab93ae9e88 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 28 Feb 2014 13:16:54 -0500 Subject: [PATCH 140/502] Correct some comments about the byte 0xed. The comments here assumed, erroneously, that every instance of the byte 0xed in UTF-8 indicates that a UTF-16 surrogate is being encoded. This isn't true; there are about 2000 Hangul characters that are encoded with the byte 0xed. However, the code did not rely on this assumption. It appears to decode these Hangul characters correctly anyway, if a bit more slowly. (If it didn't work, Twitter tests probably would have noticed by now.) --- ftfy/bad_codecs/utf8_variants.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 081218e8..75aef452 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -189,8 +189,9 @@ def _buffer_decode_surrogates(sup, input, errors, final): if len(input) < 6: if final: # We found 0xed near the end of the stream, and there aren't - # six bytes to decode. Delegate to the superclass method - # to handle this error. + # six bytes to decode. Delegate to the superclass method to + # handle it as normal UTF-8. It might be a Hangul character + # or an error. return sup(input, errors, final) else: # We found 0xed, the stream isn't over yet, and we don't know @@ -213,9 +214,7 @@ def _buffer_decode_surrogates(sup, input, errors, final): else: # This looked like a CESU-8 sequence, but it wasn't one. # 0xed indicates the start of a three-byte sequence, so give - # three bytes to the superclass, so it can either decode them - # as a surrogate codepoint (on Python 2) or handle the error - # (on Python 3). + # three bytes to the superclass to decode as usual. return sup(input[:3], errors, False) From 35f98ece57ff90bda2f63835d8a16e7b2eb52d60 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 12 Mar 2014 11:34:35 -0400 Subject: [PATCH 141/502] Update version number in docs --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3da434ad..64f7cf8c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ # built documents. # # The short X.Y version. -version = '3.0' +version = '3.1' # The full version, including alpha/beta/rc tags. -release = '3.0' +release = '3.1.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From ed4ea113714995154af326e0f0d7357a916bdf11 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 18 Mar 2014 18:59:45 -0400 Subject: [PATCH 142/502] Tested on Python 3.4 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index bc8e08c4..c421fb32 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", From 1a11fcd77b2c19c0d7c48f6f84e72185828b8abe Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 19 Apr 2014 05:35:16 -0400 Subject: [PATCH 143/502] Be forgiving of some punctuation. Fixes #10. --- ftfy/badness.py | 21 +++++++++++++++++++-- tests/test_unicode.py | 12 +++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index f345d6e4..12e8ed1e 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -84,6 +84,21 @@ def _make_weirdness_regex(): WEIRDNESS_RE = _make_weirdness_regex() +# A few characters are common ending punctuation that can show up at the end +# of a mojibake sequence. It's plausible that such a character could appear +# after an accented capital letter, for example, so we'll want to add a +# slight preference to leave these characters alone. +# +# The match ends with a + so that we only give the bonus once for a +# consecutive sequence of these characters. +ENDING_PUNCT_RE = re.compile( + '[' + '\N{HORIZONTAL ELLIPSIS}\N{EM DASH}\N{EN DASH}' + '\N{RIGHT SINGLE QUOTATION MARK}\N{RIGHT DOUBLE QUOTATION MARK}' + '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' + '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' + ']+' +) def sequence_weirdness(text): """ @@ -113,7 +128,9 @@ def sequence_weirdness(text): The return value is the number of instances of weirdness. """ text2 = unicodedata.normalize('NFC', text) - return len(WEIRDNESS_RE.findall(chars_to_classes(text2))) + weirdness = len(WEIRDNESS_RE.findall(chars_to_classes(text2))) + punct_discount = len(ENDING_PUNCT_RE.findall(text2)) + return weirdness * 2 - punct_discount def text_cost(text): @@ -124,4 +141,4 @@ def text_cost(text): The overall cost is measured as twice the "weirdness" (see :func:`sequence_weirdness`) plus the length. """ - return sequence_weirdness(text) * 2 + len(text) + return sequence_weirdness(text) + len(text) diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 24d3bbf6..6554821a 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from ftfy.fixes import fix_text_encoding, fix_encoding_and_explain, apply_plan, possible_encoding +from ftfy.badness import ENDING_PUNCT_RE import unicodedata import sys from nose.tools import eq_ @@ -21,11 +22,12 @@ def test_all_bmp_characters(): # Exclude code points that are not assigned if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): garble = char.encode('utf-8').decode('latin-1') - garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') - for garb in (garble, garble2): - fixed, plan = fix_encoding_and_explain(garb) - eq_(fixed, char) - eq_(apply_plan(garb, plan), char) + if not (index < 0x800 and ENDING_PUNCT_RE.search(garble)): + garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') + for garb in (garble, garble2): + fixed, plan = fix_encoding_and_explain(garb) + eq_(fixed, char) + eq_(apply_plan(garb, plan), char) phrases = [ u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", From 84418050f512b7857dba301b8877de1fae6225b4 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 15 May 2014 15:25:54 -0400 Subject: [PATCH 144/502] Alter UTF-8 sequences for surrogates on Python 2. This ensures that text will decode the same in our `bad_codecs` decoders on Python 2 and Python 3. Fixes #21. --- ftfy/bad_codecs/utf8_variants.py | 36 +++++++++++++++++++++++++++++--- tests/test_bytes.py | 32 ++++++++++++++++++++++++++-- tests/test_real_tweets.py | 1 + tests/test_unicode.py | 8 ++++++- 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 75aef452..1d59357e 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -34,11 +34,12 @@ never. [1] In a pinch, you can decode CESU-8 in Python 2 using the UTF-8 codec: first -decode the bytes (incorrectly), then encode them, then decode them again. +decode the bytes (incorrectly), then encode them, then decode them again, using +UTF-8 as the codec every time. """ from __future__ import unicode_literals -from ftfy.compatibility import bytes_to_ints, unichr +from ftfy.compatibility import bytes_to_ints, unichr, PYTHON2 from encodings.utf_8 import (IncrementalDecoder as UTF8IncrementalDecoder, IncrementalEncoder as UTF8IncrementalEncoder) import re @@ -192,6 +193,10 @@ def _buffer_decode_surrogates(sup, input, errors, final): # six bytes to decode. Delegate to the superclass method to # handle it as normal UTF-8. It might be a Hangul character # or an error. + if PYTHON2 and len(input) >= 3: + # We can't trust Python 2 to raise an error when it's + # asked to decode a surrogate, so let's force the issue. + input = mangle_surrogates(input) return sup(input, errors, final) else: # We found 0xed, the stream isn't over yet, and we don't know @@ -214,10 +219,35 @@ def _buffer_decode_surrogates(sup, input, errors, final): else: # This looked like a CESU-8 sequence, but it wasn't one. # 0xed indicates the start of a three-byte sequence, so give - # three bytes to the superclass to decode as usual. + # three bytes to the superclass to decode as usual -- except + # for working around the Python 2 discrepancy as before. + if PYTHON2: + input = mangle_surrogates(input) return sup(input[:3], errors, False) +def mangle_surrogates(bytestring): + """ + When Python 3 sees the UTF-8 encoding of a surrogate codepoint, it treats + it as an error (which it is). In 'replace' mode, it will decode as three + replacement characters. But Python 2 will just output the surrogate + codepoint. + + To ensure consistency between Python 2 and Python 3, and protect downstream + applications from malformed strings, we turn surrogate sequences at the + start of the string into the bytes `ff ff ff`, which we're *sure* won't + decode, and which turn into three replacement characters in 'replace' mode. + """ + if PYTHON2: + if bytestring.startswith(b'\xed') and len(bytestring) >= 3: + decoded = bytestring[:3].decode('utf-8', 'replace') + if '\ud800' <= decoded <= '\udfff': + return b'\xff\xff\xff' + mangle_surrogates(bytestring[3:]) + return bytestring + else: + # On Python 3, nothing needs to be done. + return bytestring + # The encoder is identical to UTF-8. IncrementalEncoder = UTF8IncrementalEncoder diff --git a/tests/test_bytes.py b/tests/test_bytes.py index 9fa401af..204c786c 100644 --- a/tests/test_bytes.py +++ b/tests/test_bytes.py @@ -1,16 +1,23 @@ # coding: utf-8 +from __future__ import unicode_literals from ftfy import guess_bytes +from ftfy.bad_codecs.utf8_variants import mangle_surrogates from nose.tools import eq_ +import sys + + +PYTHON2 = sys.hexversion < 0x03000000 TEST_ENCODINGS = [ 'utf-16', 'utf-8', 'sloppy-windows-1252' ] TEST_STRINGS = [ - u'Renée\nFleming', u'Noël\nCoward', u'Señor\nCardgage', - u'€ • £ • ¥', u'¿Qué?' + 'Renée\nFleming', 'Noël\nCoward', 'Señor\nCardgage', + '€ • £ • ¥', '¿Qué?' ] + def check_bytes_decoding(string): for encoding in TEST_ENCODINGS: result_str, result_encoding = guess_bytes(string.encode(encoding)) @@ -22,6 +29,7 @@ def check_bytes_decoding(string): result_str, result_encoding = guess_bytes(old_mac_bytes) eq_(result_str, string.replace('\n', '\r')) + def test_guess_bytes(): for string in TEST_STRINGS: yield check_bytes_decoding, string @@ -30,3 +38,23 @@ def test_guess_bytes(): result_str, result_encoding = guess_bytes(bowdlerized_null) eq_(result_str, u'null\x00separated') eq_(result_encoding, u'utf-8-variants') + + +def test_mangle_surrogates(): + if PYTHON2: + # These are the encodings of a surrogate character, plus a similar-looking + # Korean character. Only the surrogate character's bytes should get mangled. + eq_(mangle_surrogates(b'\xed\xa0\x80\xed\x9e\x99'), b'\xff\xff\xff\xed\x9e\x99') + + # Mangle sequences of surrogates, but don't mangle surrogates later in + # the string (there's no need to in our decoders). + eq_(mangle_surrogates(b'\xed\xa0\xbd\xed\xb8\xb9test\xed\xb4\xb4'), + b'\xff\xff\xff\xff\xff\xfftest\xed\xb4\xb4') + eq_(mangle_surrogates(b'test\xed\xb4\xb4'), b'test\xed\xb4\xb4') + + # Handle short bytestrings correctly. + eq_(mangle_surrogates(b'\xed'), b'\xed') + eq_(mangle_surrogates(b''), b'') + else: + # Make sure mangle_surrogates doesn't do anything + eq_(mangle_surrogates(b'\xed\xa0\x80\xed\x9e\x99'), b'\xed\xa0\x80\xed\x9e\x99') diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 0e0248ce..5fb58d91 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -35,6 +35,7 @@ ('приятности. РІСњВ¤', 'приятности. ❤'), ('Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New “ Romance”.', 'Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New " Romance".'), + ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí \u0301 \u0301"), ## Current false positives: #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 24d3bbf6..93705648 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -7,6 +7,7 @@ if sys.hexversion >= 0x03000000: unichr = chr + def char_names(text): """ Show the names of the characters involved. Helpful for debugging when @@ -14,6 +15,7 @@ def char_names(text): """ return [unicodedata.name(c) for c in text] + # Most single-character strings which have been misencoded should be restored. def test_all_bmp_characters(): for index in range(0xa0, 0xfffd): @@ -27,6 +29,7 @@ def test_all_bmp_characters(): eq_(fixed, char) eq_(apply_plan(garb, plan), char) + phrases = [ u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", @@ -43,18 +46,21 @@ def test_valid_phrases(): # make it not just confirm based on the opening punctuation yield check_phrase, phrase[1:] + def check_phrase(text): eq_(fix_text_encoding(text), text) eq_(fix_text_encoding(text.encode('utf-8').decode('latin-1')), text) + def test_possible_encoding(): for codept in range(256): char = chr(codept) assert possible_encoding(char, 'latin-1') + def test_fix_with_backslash(): eq_(fix_text_encoding(u"<40\\% vs \xe2\x89\xa540\\%"), u"<40\\% vs ≥40\\%") + if __name__ == '__main__': test_all_bmp_characters() - From 58bdb7b812d2ffc88eb5ee99407895d06e70aca2 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 15 May 2014 15:33:35 -0400 Subject: [PATCH 145/502] Simplify the regex-building expression We don't need to be that paranoid about backslashes when we're just expressing ASCII as a range. Also, add an explanatory comment. --- ftfy/chardata.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 5ff28b3e..139f0ee1 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -34,7 +34,12 @@ def _build_regexes(): for encoding in CHARMAP_ENCODINGS: latin1table = ''.join(unichr(i) for i in range(128, 256)) charlist = latin1table.encode('latin-1').decode(encoding) - regex = '^[\x00-\x7f{}]*$'.format(charlist.replace('\\', '\\\\')) + + # Build a regex from the ASCII range, followed by the decodings of + # bytes 0x80-0xff in this character set. (This uses the fact that all + # regex special characters are ASCII, and therefore won't appear in the + # string.) + regex = '^[\x00-\x7f{}]*$'.format(charlist) encoding_regexes[encoding] = re.compile(regex) return encoding_regexes ENCODING_REGEXES = _build_regexes() From 67c674faebea844a0fe48a8112ceefb14f00d57a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 15 May 2014 15:52:56 -0400 Subject: [PATCH 146/502] package as v3.1.3 --- CHANGELOG.txt | 18 ++++++++++++++++++ setup.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 34498199..efd57717 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,21 @@ +## Version 3.1.3 (May 15, 2014) + +- Fix `utf-8-variants` so it never outputs surrogate codepoints, even on + Python 2 where that would otherwise be possible. + +## Version 3.1.2 (January 29, 2014) + +- Fix bug in 3.1.1 where strings with backslashes in them could never be fixed + +## Version 3.1.1 (January 29, 2014) + +- Add the `ftfy.bad_codecs` package, which registers new codecs that can + decoding things that Python may otherwise refuse to decode: + - `utf-8-variants`, which decodes CESU-8 and its Java lookalike + - `sloppy-windows-*`, which decodes character-map encodings while treating + unmapped characters as Latin-1 +- Simplify the code using `ftfy.bad_codecs`. + ## Version 3.0.6 (November 5, 2013) - `fix_entities` can now be True, False, or 'auto'. The new case is True, which diff --git a/setup.py b/setup.py index c421fb32..dc07789a 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.1.2', + version='3.1.3', maintainer='Luminoso Technologies, Inc.', maintainer_email='dev@lumino.so', license="MIT", From a4e9297cf6dea73734bf2b4af3aed5e5d9dea6fb Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 4 Jun 2014 11:28:32 -0400 Subject: [PATCH 147/502] Add hiring blurb to README --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 167eea05..cf067793 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,20 @@ Full documentation: **http://ftfy.readthedocs.org** - “Too many branches” — [pylint](https://bitbucket.org/logilab/pylint/) +## Developed at Luminoso + +[Luminoso](http://www.luminoso.com) makes groundbreaking software for text +analytics that really understands what words mean, in many languages. Our +software is used by enterprise customers such as Sony, Intel, Mars, and Scotts, +and it's built on Python and open-source technologies. + +We use ftfy every day at Luminoso, because the first step in understanding text +is making sure it has the correct characters in it! + +Luminoso is growing fast and hiring in all areas. If you're interested in +joining us, take a look at [our careers +page](http://www.luminoso.com/careers.html). + ## What it does ftfy makes Unicode text less broken and more consistent. It works in From 4f59d96d16bbc6b37b797f9367f8de21c06ec315 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 4 Jun 2014 14:10:35 -0400 Subject: [PATCH 148/502] Update `remove_unsafe_private_use` description in README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cf067793..e123fc06 100644 --- a/README.md +++ b/README.md @@ -106,9 +106,9 @@ the output ends up in a standard form that will be unchanged by `fix_text`. All the fixes are on by default, but you can pass options to turn them off. - If `remove_unsafe_private_use` is True, remove a range of unassigned - characters that can crash Python via - [bug 18183](http://bugs.python.org/issue18183). This fix will turn itself - off when you're using Python 3.4 or better, which you probably aren't. + characters that can crash Python < 3.3.3 via + [bug 18183](http://bugs.python.org/issue18183). (This fix is off by + default on Python 3.4, because it's no longer necessary.) - If `fix_entities` is True, consider replacing HTML entities with their equivalent characters. However, this never applies to text with a pair of angle brackets in it already; you're probably not supposed to decode From 350eb4605c0026ae61520dbaae234d32ebf7288e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 10 Jun 2014 14:24:32 -0400 Subject: [PATCH 149/502] Fix three more kinds of purported line breaks --- ftfy/fixes.py | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 21e430fb..b392f119 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -317,12 +317,45 @@ def uncurl_quotes(text): def fix_line_breaks(text): r""" - Convert line breaks to Unix style. + Convert all line breaks to Unix style. - In particular, this replaces CRLF (\\r\\n) with LF (\\n), then - additionally replaces CR (\\r) with LF (\\n). + This will convert the following sequences into the standard \\n + line break: + + - CRLF (\\r\\n), used on Windows and in some communication + protocols + - CR (\\r), once used on Mac OS Classic, and now kept alive + by misguided software such as Microsoft Office for Mac + - LINE SEPARATOR (\\u2028) and PARAGRAPH SEPARATOR (\\u2029), + defined by Unicode and used to sow confusion and discord + - NEXT LINE (\\x85), a C1 control character that is certainly + not what you meant + + The NEXT LINE character is a bit of an odd case, because it + usually won't show up if `fix_encoding` is also being run. + \\x85 is very common mojibake for \\u2026, HORIZONTAL ELLIPSIS. + + >>> fix_line_breaks("Content-type: text/plain\r\n\r\nHi.") + 'Content-type: text/plain\n\nHi.' + + >>> fix_line_breaks("This is how Microsoft \r trolls Mac users") + 'This is how Microsoft \n trolls Mac users' + + >>> fix_line_breaks("What is this \x85 I don't even") + "What is this \n I don't even" + + >>> print(fix_line_breaks( + ... "This string is made of two things:\u2029" + ... "1. Unicode\u2028" + ... "2. Spite" + ... )) + This string is made of two things: + 1. Unicode + 2. Spite """ - return text.replace('\r\n', '\n').replace('\r', '\n') + return text.replace('\r\n', '\n').replace('\r', '\n')\ + .replace('\u2028', '\n').replace('\u2029', '\n')\ + .replace('\u0085', '\n') def remove_control_chars(text): From 3dab08e6d516ebbe1e07506504945a9fe7fca33c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 17 Jun 2014 15:56:11 -0400 Subject: [PATCH 150/502] Refactor so we can have non-live-streaming testers --- ftfy/streamtester/__init__.py | 27 ++++++++++ ftfy/streamtester/streamtester.py | 66 ------------------------- ftfy/streamtester/twitter_tester.py | 76 +++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 66 deletions(-) delete mode 100644 ftfy/streamtester/streamtester.py create mode 100644 ftfy/streamtester/twitter_tester.py diff --git a/ftfy/streamtester/__init__.py b/ftfy/streamtester/__init__.py index e69de29b..d911563f 100644 --- a/ftfy/streamtester/__init__.py +++ b/ftfy/streamtester/__init__.py @@ -0,0 +1,27 @@ +from __future__ import print_function, unicode_literals +from ftfy.fixes import fix_text_encoding +from ftfy.chardata import possible_encoding + + +class StreamTester: + """ + Take in a sequence of texts, and show the ones that will be changed by ftfy. + This will also periodically show updates, such as the proportion of texts that + changed. + """ + def __init__(self): + self.num_fixed = 0 + self.count = 0 + + def check_ftfy(self, text): + self.count += 1 + if not possible_encoding(text, 'ascii'): + fixed = fix_text_encoding(text) + if text != fixed: + # possibly filter common bots before printing + print(u'\nText:\t{text}\nFixed:\t{fixed}\n'.format(text=text, fixed=fixed)) + self.num_fixed += 1 + if self.count % 100 == 0: + print('.', end='') + if self.count % 10000 == 0: + print('\n%d/%d fixed' % (self.num_fixed, self.count)) diff --git a/ftfy/streamtester/streamtester.py b/ftfy/streamtester/streamtester.py deleted file mode 100644 index a6c64268..00000000 --- a/ftfy/streamtester/streamtester.py +++ /dev/null @@ -1,66 +0,0 @@ -from twitter import TwitterStream, OAuth -import json -import time -import unicodedata -import os -from ftfy.fixes import fix_text_encoding, remove_unsafe_private_use -from ftfy.badness import sequence_weirdness -from ftfy.chardata import possible_encoding -from ftfy.streamtester.oauth import get_auth -from collections import defaultdict - -OUTPUT_DIR = './twitterlogs' -class TwitterTester: - def __init__(self): - self.lines_by_lang = defaultdict(list) - self.num_fixed = 0 - - def save_files(self): - if not os.path.exists(OUTPUT_DIR): - os.makedirs(OUTPUT_DIR) - for lang, lines in self.lines_by_lang.items(): - filename = 'tweets.{}.txt'.format(lang) - fullname = os.path.join(OUTPUT_DIR, filename) - langfile = open(fullname, 'a') - for line in lines: - print(line.replace('\n', ' '), file=langfile) - langfile.close() - self.lines_by_lang = defaultdict(list) - - def run_sample(self): - twitter_stream = TwitterStream(auth=get_auth()) - iterator = twitter_stream.statuses.sample() - count = 0 - for tweet in iterator: - if 'text' in tweet: - self.handle_tweet(tweet) - if count % 10000 == 0: - print('%d/%d fixed' % (self.num_fixed, count)) - count += 1 - if count % 10000 == 100: - self.save_files() - - def check_ftfy(self, text): - if not possible_encoding(text, 'ascii'): - fixed = fix_text_encoding(text) - if text != fixed: - # possibly filter common bots before printing - print(u'Text:\t{text}\nFixed:\t{fixed}\n'.format(text=text, fixed=fixed)) - self.num_fixed += 1 - - def handle_tweet(self, tweet): - text = tweet['text'] - self.check_ftfy(text) - if 'user' in tweet: - lang = tweet['user'].get('lang', 'NONE') - self.lines_by_lang[lang].append(tweet['text']) - - -def main(): - tester = TwitterTester() - tester.run_sample() - - -if __name__ == '__main__': - main() - diff --git a/ftfy/streamtester/twitter_tester.py b/ftfy/streamtester/twitter_tester.py new file mode 100644 index 00000000..75a8c2a0 --- /dev/null +++ b/ftfy/streamtester/twitter_tester.py @@ -0,0 +1,76 @@ +from __future__ import print_function, unicode_literals +import os +from ftfy.streamtester import StreamTester +from collections import defaultdict + + +class TwitterTester(StreamTester): + """ + This class uses the StreamTester code (defined in `__init__.py`) to + evaluate ftfy's real-world performance, by feeding it live data from + Twitter. + + This is a semi-manual evaluation. It requires a human to look at the + results and determine if they are good. The three possible cases we + can see here are: + + - Success: the process takes in mojibake and outputs correct text. + - False positive: the process takes in correct text, and outputs + mojibake. Every false positive should be considered a bug, and + reported on GitHub if it isn't already. + - Confusion: the process takes in mojibake and outputs different + mojibake. Not a great outcome, but not as dire as a false + positive. + + This tester cannot reveal false negatives. So far, that can only be + done by the unit tests. + """ + OUTPUT_DIR = './twitterlogs' + + def __init__(self): + self.lines_by_lang = defaultdict(list) + super().__init__() + + def save_files(self): + """ + When processing data from live Twitter, save it to log files so that + it can be replayed later. + """ + if not os.path.exists(self.OUTPUT_DIR): + os.makedirs(self.OUTPUT_DIR) + for lang, lines in self.lines_by_lang.items(): + filename = 'tweets.{}.txt'.format(lang) + fullname = os.path.join(self.OUTPUT_DIR, filename) + langfile = open(fullname, 'a') + for line in lines: + print(line.replace('\n', ' '), file=langfile) + langfile.close() + self.lines_by_lang = defaultdict(list) + + def run_sample(self): + """ + Listen to live data from Twitter, and pass on the fully-formed tweets to + `check_ftfy`. This requires the `twitter` Python package as a dependency. + """ + from twitter import TwitterStream + from ftfy.streamtester.oauth import get_auth + twitter_stream = TwitterStream(auth=get_auth()) + iterator = twitter_stream.statuses.sample() + for tweet in iterator: + if 'text' in tweet: + self.check_ftfy(tweet['text']) + if 'user' in tweet: + lang = tweet['user'].get('lang', 'NONE') + self.lines_by_lang[lang].append(tweet['text']) + if self.count % 10000 == 100: + self.save_files() + + +def main(): + tester = TwitterTester() + tester.run_sample() + + +if __name__ == '__main__': + main() + From 2edfd3b83e6ff4b974417bbd59108d65614c3f6e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 17 Jun 2014 18:26:37 -0400 Subject: [PATCH 151/502] weirdify the linebreak doctest for Python 2 compatibility --- ftfy/fixes.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index b392f119..4c2b6a9d 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -335,15 +335,6 @@ def fix_line_breaks(text): usually won't show up if `fix_encoding` is also being run. \\x85 is very common mojibake for \\u2026, HORIZONTAL ELLIPSIS. - >>> fix_line_breaks("Content-type: text/plain\r\n\r\nHi.") - 'Content-type: text/plain\n\nHi.' - - >>> fix_line_breaks("This is how Microsoft \r trolls Mac users") - 'This is how Microsoft \n trolls Mac users' - - >>> fix_line_breaks("What is this \x85 I don't even") - "What is this \n I don't even" - >>> print(fix_line_breaks( ... "This string is made of two things:\u2029" ... "1. Unicode\u2028" @@ -352,6 +343,21 @@ def fix_line_breaks(text): This string is made of two things: 1. Unicode 2. Spite + + For further testing and examples, let's define a function to make sure + we can see the control characters in their escaped form: + + >>> def eprint(text): + ... print(text.encode('unicode-escape').decode('ascii')) + + >>> eprint(fix_line_breaks("Content-type: text/plain\r\n\r\nHi.")) + Content-type: text/plain\n\nHi. + + >>> eprint(fix_line_breaks("This is how Microsoft \r trolls Mac users")) + This is how Microsoft \n trolls Mac users + + >>> eprint(fix_line_breaks("What is this \x85 I don't even")) + What is this \n I don't even """ return text.replace('\r\n', '\n').replace('\r', '\n')\ .replace('\u2028', '\n').replace('\u2029', '\n')\ From f05c3537bdbe766e919ce5d9a90c306559bd1855 Mon Sep 17 00:00:00 2001 From: Harry Brundage Date: Tue, 17 Jun 2014 23:31:50 -0400 Subject: [PATCH 152/502] Document real control character removal option --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 25ca940e..648822b6 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ All the fixes are on by default, but you can pass options to turn them off. plain-ASCII straight quotes. - If `fix_line_breaks` is true, convert all line breaks to Unix style (CRLF and CR line breaks become LF line breaks). -- If `fix_control_characters` is true, remove all C0 control characters +- If `remove_control_chars` is true, remove all C0 control characters except the common useful ones: TAB, CR, LF, and FF. (CR characters may have already been removed by the `fix_line_breaks` step.) - If `remove_bom` is True, remove the Byte-Order Mark if it exists. From a91b3854277cf22b2ea3e51a46bf4e32383bd529 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 18 Jun 2014 16:00:16 -0400 Subject: [PATCH 153/502] fix an inaccurate comment --- ftfy/__init__.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 8cc97417..1cf594ff 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -266,12 +266,15 @@ def guess_bytes(bstring): try: if byte_ed in byteset or byte_c0 in byteset: - # Byte 0xed can only be used to encode a range of codepoints that - # are UTF-16 surrogates. UTF-8 does not use UTF-16 surrogates, so - # this byte is impossible in real UTF-8. Therefore, if we see - # bytes that look like UTF-8 but contain 0xed, what we're seeing - # is CESU-8, the variant that encodes UTF-16 surrogates instead - # of the original characters themselves. + # Byte 0xed can be used to encode a range of codepoints that + # are UTF-16 surrogates. UTF-8 does not use UTF-16 surrogates, + # so when we see 0xed, it's very likely we're being asked to + # decode CESU-8, the variant that encodes UTF-16 surrogates + # instead of the original characters themselves. + # + # This will occasionally trigger on standard UTF-8, as there + # are some Korean characters that also use byte 0xed, but that's + # not harmful. # # Byte 0xc0 is impossible because, numerically, it would only # encode characters lower than U+0040. Those already have @@ -281,7 +284,8 @@ def guess_bytes(bstring): # encodes it as 0xc0 0x80 instead of 0x00, guaranteeing that 0x00 # will never appear in the encoded bytes. # - # The 'utf-8-variants' decoder can handle both of these cases. + # The 'utf-8-variants' decoder can handle both of these cases, as + # well as standard UTF-8, at the cost of a bit of speed. return bstring.decode('utf-8-variants'), 'utf-8-variants' else: return bstring.decode('utf-8'), 'utf-8' From 2745c0449632bf7386d6286f99de041a82c62f6e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 18 Jun 2014 16:03:03 -0400 Subject: [PATCH 154/502] Put back `remove_unsafe_private_use` to avoid an API change I was removing this option prematurely, because the characters it removes are no longer unsafe in up-to-date Python. However: - Removing the option would be an API change that breaks compatibility - Some versions of Linux distributions are still on Python 3.3.2, and will not update it without a distribution upgrade This reverts commit df479e622f6791bc6a9f336eb28bec57722872d1. Conflicts: ftfy/__init__.py --- ftfy/__init__.py | 16 ++++++++++++---- ftfy/fixes.py | 19 ++++++------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 1cf594ff..25719601 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -14,12 +14,13 @@ from ftfy import fixes from ftfy.fixes import fix_text_encoding -from ftfy.compatibility import is_printable +from ftfy.compatibility import PYTHON34_OR_LATER, is_printable import unicodedata import warnings def fix_text(text, + remove_unsafe_private_use=(not PYTHON34_OR_LATER), fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, @@ -65,6 +66,10 @@ def fix_text(text, Based on the options you provide, ftfy applies these steps in order: + - If `remove_unsafe_private_use` is True, it removes a range of private-use + characters that could trigger a Python bug. The bug is fixed in + the most recent versions of Python, so this will default to False + starting on Python 3.4. - If `fix_entities` is True, replace HTML entities with their equivalent characters. If it's "auto" (the default), then consider replacing HTML entities, but don't do so in text where you have seen a pair of actual @@ -134,6 +139,7 @@ def fix_text(text, out.append( fix_text_segment( substring, + remove_unsafe_private_use=remove_unsafe_private_use, fix_entities=fix_entities, remove_terminal_escapes=remove_terminal_escapes, fix_encoding=fix_encoding_this_time, @@ -152,6 +158,7 @@ def fix_text(text, def fix_file(input_file, + remove_unsafe_private_use=True, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, @@ -178,6 +185,7 @@ def fix_file(input_file, entities = False yield fix_text_segment( line, + remove_unsafe_private_use=remove_unsafe_private_use, fix_entities=fix_entities, remove_terminal_escapes=remove_terminal_escapes, fix_encoding=entities, @@ -190,6 +198,7 @@ def fix_file(input_file, def fix_text_segment(text, + remove_unsafe_private_use=True, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, @@ -212,6 +221,8 @@ def fix_text_segment(text, fix_entities = False while True: origtext = text + if remove_unsafe_private_use: + text = fixes.remove_unsafe_private_use(text) if fix_entities: text = fixes.unescape_html(text) if remove_terminal_escapes: @@ -255,12 +266,9 @@ def guess_bytes(bstring): - "sloppy-windows-1252", the Latin-1-like encoding that is the most common single-byte encoding """ - # Look for a byte-order mark to recognize UTF-16 if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): return bstring.decode('utf-16'), 'utf-16' - # Make a set out of the bytes, so we can quickly test for the presence of - # certain bytes byteset = set(bytes(bstring)) byte_ed, byte_c0, byte_CR, byte_LF = b'\xed\xc0\r\n' diff --git a/ftfy/fixes.py b/ftfy/fixes.py index f983ba4a..b392f119 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -369,26 +369,19 @@ def remove_control_chars(text): - U+000E to U+001F - U+007F - It leaves alone these characters that are used for formatting: + It leaves alone these characters that are commonly used for formatting: - - Tab (U+0009) - - Line Feed (U+000A) - - Form Feed (U+000C) - - Carriage Return (U+000D) - - Form Feed is the most borderline of these, but I've seen it used in - plain text files -- for example, as a large whitespace to separate - major sections of code. It has a purpose in text that it's still - used for, so we might as well leave it alone. + - TAB (U+0009) + - LF (U+000A) + - FF (U+000C) + - CR (U+000D) """ return text.translate(CONTROL_CHARS) def remove_bom(text): r""" - Remove a left-over byte-order mark. Byte-order marks are metadata about - UTF-16 encoded text and shouldn't appear in the text itself, but they can - arise due to Microsoft's confusion between UTF-16 and Unicode. + Remove a left-over byte-order mark. >>> print(remove_bom(unichr(0xfeff) + "Where do you want to go today?")) Where do you want to go today? From 73d4f1134b7b2bfe1764215b11d77af41485320b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 27 Jun 2014 16:45:59 -0400 Subject: [PATCH 155/502] Version 3.2.0 --- CHANGELOG.txt | 10 ++++++++++ setup.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index efd57717..24a3c9fb 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,13 @@ +## Version 3.2.0 (June 27, 2014) + +- `fix_line_breaks` fixes three additional characters that are considered line + breaks in some environments, such as Javascript, and Python's "codecs" + library. These are all now replaced with \n: + + U+0085 , with alias "NEXT LINE" + U+2028 LINE SEPARATOR + U+2029 PARAGRAPH SEPARATOR + ## Version 3.1.3 (May 15, 2014) - Fix `utf-8-variants` so it never outputs surrogate codepoints, even on diff --git a/setup.py b/setup.py index dc07789a..e50bd164 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,9 @@ setup( name="ftfy", - version='3.1.3', + version='3.2.0', maintainer='Luminoso Technologies, Inc.', - maintainer_email='dev@lumino.so', + maintainer_email='info@luminoso.com', license="MIT", url='http://github.com/LuminosoInsight/python-ftfy', platforms=["any"], From 0c260ebaf748217eda0ec1be8b641f2df6e8345c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sun, 3 Aug 2014 23:47:03 -0400 Subject: [PATCH 156/502] enable one more test that now works --- tests/test_real_tweets.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 5fb58d91..cbf1e464 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -25,8 +25,6 @@ "hihi RT username: ☺😘"), ("Beta Haber: Hırsızı Büyü Korkuttu", "Beta Haber: Hırsızı Büyü Korkuttu"), - #("Blog Traffic Tip 2 – Broadcast Email Your Blog", - # "Blog Traffic Tip 2 – Broadcast Email Your Blog"), ("Ôôô VIDA MINHA", "Ôôô VIDA MINHA"), ('[x]\xa0©', '[x] ©'), ('2012—∞', '2012—∞'), @@ -36,16 +34,20 @@ ('Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New “ Romance”.', 'Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New " Romance".'), ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí \u0301 \u0301"), + ('HUHLL Õ…', 'HUHLL Õ...'), ## Current false positives: #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", # "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), - #('HUHLL Õ…', 'HUHLL Õ...'), #("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), ## This kind of tweet can't be fixed without a full-blown encoding detector. #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", # "Deja dos heridos hundimiento de barco turístico en Acapulco."), + + ## The heuristics aren't confident enough to fix this text and its weird encoding. + #("Blog Traffic Tip 2 – Broadcast Email Your Blog", + # "Blog Traffic Tip 2 – Broadcast Email Your Blog"), ] def test_real_tweets(): From df26785806f193f03759c2512c367e251d806d30 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 4 Aug 2014 01:59:39 -0400 Subject: [PATCH 157/502] Add a manual fixer for escape sequences (fixes #19) --- ftfy/fixes.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index b392f119..6d19339e 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -8,9 +8,10 @@ from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS, CONTROL_CHARS) from ftfy.badness import text_cost -from ftfy.compatibility import (htmlentitydefs, unichr, UNSAFE_PRIVATE_USE_RE) +from ftfy.compatibility import htmlentitydefs, unichr, UNSAFE_PRIVATE_USE_RE import re import sys +import codecs BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. @@ -418,3 +419,50 @@ def remove_unsafe_private_use(text): based on a micro version upgrade of Python.) """ return UNSAFE_PRIVATE_USE_RE.sub('', text) + + +# Define a regex to match valid escape sequences in Python string literals. +ESCAPE_SEQUENCE_RE = re.compile(r''' + ( \\U........ # 8-digit hex escapes + | \\u.... # 4-digit hex escapes + | \\x.. # 2-digit hex escapes + | \\[0-7]{1,3} # Octal escapes + | \\N\{[^}]+\} # Unicode characters by name + | \\[\\'"abfnrtv] # Single-character escapes + )''', re.UNICODE | re.VERBOSE) + + +def decode_escapes(s): + r""" + Decode backslashed escape sequences, including \x, \u, and \U character + references, even in the presence of other Unicode. + + This is what Python's "string-escape" and "unicode-escape" codecs were + meant to do, but in contrast, this actually works. It will decode the + string exactly the same way that the Python interpreter decodes its string + literals. + + >>> factoid = '\\u20a1 is the currency symbol for the colón.' + >>> print(factoid) + \u20a1 is the currency symbol for the colón. + >>> print(decode_escapes(factoid)) + ₡ is the currency symbol for the colón. + + Even though Python itself can read string literals with a combination of + escapes and literal Unicode -- you're looking at one right now -- the + "unicode-escape" codec doesn't work on literal Unicode. (See + http://stackoverflow.com/a/24519338/773754 for more details.) + + Instead, this function searches for just the parts of a string that + represent escape sequences, and decodes them, leaving the rest alone. All + valid escape sequences are made of ASCII characters, and this allows + "unicode-escape" to work correctly. + + This fix cannot be automatically applied by the `ftfy.fix_text` function, + because escaped text is not necessarily a mistake, and there is no way + to distinguish text that's supposed to be escaped from text that isn't. + """ + def decode_match(match): + return codecs.decode(match.group(0), 'unicode-escape') + + return ESCAPE_SEQUENCE_RE.sub(decode_match, s) From 335f7b1b1ff3328d58215fb1aab2ecb90ca26532 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 4 Aug 2014 02:14:24 -0400 Subject: [PATCH 158/502] bugfix: build_data would not have generated the correct file build_data got refactored at some point without being tested. It would not generate the correct char_classes.dat file, because it was saying that reasonable Latin letters had to be lower than 0x20 (it meant 0x200). There are no letters below 0x20. Despite this bug, char_classes.dat was correct from a previous version. Manual testing shows that, with the bugfix, it generates the same char_classes.dat file that is being distributed. --- ftfy/build_data.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index b09d780c..f556b306 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -60,10 +60,8 @@ def make_char_data_file(do_it_anyway=False): category = unicodedata.category(char) if category.startswith('L'): # letters - if ( - unicodedata.name(char).startswith('LATIN') - and codepoint < 0x20 - ): + is_latin = unicodedata.name(char).startswith('LATIN') + if is_latin and codepoint < 0x200: if category == 'Lu': cclasses[codepoint] = 'L' else: From b4bbb57ac429208f169f009c24a9e0dd8a9d4b02 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 4 Aug 2014 02:43:22 -0400 Subject: [PATCH 159/502] bugfix: args were mixed up in ftfy.fix_file --- ftfy/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 25719601..a24ba61d 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -186,9 +186,9 @@ def fix_file(input_file, yield fix_text_segment( line, remove_unsafe_private_use=remove_unsafe_private_use, - fix_entities=fix_entities, + fix_entities=entities, remove_terminal_escapes=remove_terminal_escapes, - fix_encoding=entities, + fix_encoding=fix_encoding, normalization=normalization, uncurl_quotes=uncurl_quotes, fix_line_breaks=fix_line_breaks, From 5fdd7286570d36d48958ea3a332044dc5cd7817a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 4 Aug 2014 02:44:10 -0400 Subject: [PATCH 160/502] docstrings, pep8, and pylint fixes --- ftfy/bad_codecs/__init__.py | 8 ++++---- ftfy/chardata.py | 2 +- ftfy/compatibility.py | 2 +- ftfy/fixes.py | 7 ++++--- ftfy/streamtester/__init__.py | 20 ++++++++++++++++---- ftfy/streamtester/oauth.py | 8 ++++++-- ftfy/streamtester/twitter_tester.py | 19 ++++++++++++++++--- 7 files changed, 48 insertions(+), 18 deletions(-) diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py index 91928475..0984bd52 100644 --- a/ftfy/bad_codecs/__init__.py +++ b/ftfy/bad_codecs/__init__.py @@ -33,7 +33,7 @@ from encodings import normalize_encoding import codecs -_cache = {} +_CACHE = {} # Define some aliases for 'utf-8-variants'. All hyphens get turned into # underscores, because of `normalize_encoding`. @@ -60,8 +60,8 @@ def search_function(encoding): - The 'utf-8-variants' encoding, which has the several aliases seen above. """ - if encoding in _cache: - return _cache[encoding] + if encoding in _CACHE: + return _CACHE[encoding] norm_encoding = normalize_encoding(encoding) codec = None @@ -73,7 +73,7 @@ def search_function(encoding): codec = CODECS.get(norm_encoding) if codec is not None: - _cache[encoding] = codec + _CACHE[encoding] = codec return codec diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 139f0ee1..5c72dd40 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -34,7 +34,7 @@ def _build_regexes(): for encoding in CHARMAP_ENCODINGS: latin1table = ''.join(unichr(i) for i in range(128, 256)) charlist = latin1table.encode('latin-1').decode(encoding) - + # Build a regex from the ASCII range, followed by the decodings of # bytes 0x80-0xff in this character set. (This uses the fact that all # regex special characters are ASCII, and therefore won't appear in the diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index c218a0dc..1246248c 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -74,6 +74,6 @@ def is_printable(char): let's make a crude approximation in Python 2. """ if PYTHON2: - return not unicodedata.category(char).startswith('C') + return not unicodedata.category(char).startswith('C') else: return char.isprintable() diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 60bdd030..e7479dc9 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -106,7 +106,7 @@ def fix_text_encoding(text): The best version of the text is found using :func:`ftfy.badness.text_cost`. """ - text, plan = fix_encoding_and_explain(text) + text, _plan = fix_encoding_and_explain(text) return text @@ -438,7 +438,7 @@ def remove_unsafe_private_use(text): )''', re.UNICODE | re.VERBOSE) -def decode_escapes(s): +def decode_escapes(text): r""" Decode backslashed escape sequences, including \x, \u, and \U character references, even in the presence of other Unicode. @@ -469,6 +469,7 @@ def decode_escapes(s): to distinguish text that's supposed to be escaped from text that isn't. """ def decode_match(match): + "Given a regex match, decode the escape sequence it contains." return codecs.decode(match.group(0), 'unicode-escape') - return ESCAPE_SEQUENCE_RE.sub(decode_match, s) + return ESCAPE_SEQUENCE_RE.sub(decode_match, text) diff --git a/ftfy/streamtester/__init__.py b/ftfy/streamtester/__init__.py index d911563f..c7ee1084 100644 --- a/ftfy/streamtester/__init__.py +++ b/ftfy/streamtester/__init__.py @@ -1,3 +1,7 @@ +""" +This file defines a general method for evaluating ftfy using data that arrives +in a stream. A concrete implementation of it is found in `twitter_tester.py`. +""" from __future__ import print_function, unicode_literals from ftfy.fixes import fix_text_encoding from ftfy.chardata import possible_encoding @@ -5,22 +9,30 @@ class StreamTester: """ - Take in a sequence of texts, and show the ones that will be changed by ftfy. - This will also periodically show updates, such as the proportion of texts that - changed. + Take in a sequence of texts, and show the ones that will be changed by + ftfy. This will also periodically show updates, such as the proportion of + texts that changed. """ def __init__(self): self.num_fixed = 0 self.count = 0 def check_ftfy(self, text): + """ + Given a single text input, check whether `ftfy.fix_text_encoding` + would change it. If so, display the change. + """ self.count += 1 if not possible_encoding(text, 'ascii'): fixed = fix_text_encoding(text) if text != fixed: # possibly filter common bots before printing - print(u'\nText:\t{text}\nFixed:\t{fixed}\n'.format(text=text, fixed=fixed)) + print(u'\nText:\t{text}\nFixed:\t{fixed}\n'.format( + text=text, fixed=fixed + )) self.num_fixed += 1 + + # Print status updates once in a while if self.count % 100 == 0: print('.', end='') if self.count % 10000 == 0: diff --git a/ftfy/streamtester/oauth.py b/ftfy/streamtester/oauth.py index 92eaab1d..31a801bb 100644 --- a/ftfy/streamtester/oauth.py +++ b/ftfy/streamtester/oauth.py @@ -1,4 +1,8 @@ # coding: utf-8 +""" +Do what is necessary to authenticate this tester as a Twitter "app", using +somebody's Twitter account. +""" from __future__ import unicode_literals from twitter.oauth import OAuth from twitter import oauth_dance, read_token_file @@ -35,12 +39,12 @@ def get_auth(): secret behind a fig leaf like everybody else does. So that's what we've done. """ - def unhide(s): + def unhide(secret): """ Do something mysterious and exactly as secure as every other Twitter app. """ - return ''.join([chr(ord(c) - 0x2800) for c in s]) + return ''.join([chr(ord(c) - 0x2800) for c in secret]) fig_leaf = '⠴⡹⠹⡩⠶⠴⡶⡅⡂⡩⡅⠳⡏⡉⡈⠰⠰⡹⡥⡶⡈⡐⡍⡂⡫⡍⡗⡬⡒⡧⡶⡣⡰⡄⡧⡸⡑⡣⠵⡓⠶⠴⡁' consumer_key = 'OFhyNd2Zt4Ba6gJGJXfbsw' diff --git a/ftfy/streamtester/twitter_tester.py b/ftfy/streamtester/twitter_tester.py index 75a8c2a0..6ad125ee 100644 --- a/ftfy/streamtester/twitter_tester.py +++ b/ftfy/streamtester/twitter_tester.py @@ -1,7 +1,14 @@ +""" +Implements a StreamTester that runs over Twitter data. See the class +docstring. + +This module is written for Python 3 only. The __future__ imports you see here +are just to let Python 2 scan the file without crashing with a SyntaxError. +""" from __future__ import print_function, unicode_literals import os -from ftfy.streamtester import StreamTester from collections import defaultdict +from ftfy.streamtester import StreamTester class TwitterTester(StreamTester): @@ -49,8 +56,9 @@ def save_files(self): def run_sample(self): """ - Listen to live data from Twitter, and pass on the fully-formed tweets to - `check_ftfy`. This requires the `twitter` Python package as a dependency. + Listen to live data from Twitter, and pass on the fully-formed tweets + to `check_ftfy`. This requires the `twitter` Python package as a + dependency. """ from twitter import TwitterStream from ftfy.streamtester.oauth import get_auth @@ -67,6 +75,11 @@ def run_sample(self): def main(): + """ + When run from the command line, this script connects to the Twitter stream + and runs the TwitterTester on it forever. Or at least until the stream + drops. + """ tester = TwitterTester() tester.run_sample() From 70e939e9fce60235b397e5ed5def6bb978ffa7e2 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 4 Aug 2014 11:18:16 -0400 Subject: [PATCH 161/502] documentation updates --- docs/index.rst | 42 ++++++++++++++++++++++++++++++------------ ftfy/__init__.py | 9 ++++----- ftfy/fixes.py | 4 +--- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index e443840f..7fd7f872 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,9 +9,10 @@ ftfy: fixes text for you This is a module for making text less broken and more consistent. It works in Python 2.6, Python 3.2, or later. -The most interesting kind of brokenness that this resolves is when someone -has encoded Unicode with one standard and decoded it with a different one. -This often shows up as characters that turn into nonsense sequences: +The most interesting kind of brokenness that this resolves is when someone has +encoded Unicode with one standard and decoded it with a different one. This +often shows up as characters that turn into nonsense sequences (called +"mojibake"): - The word ``schön`` might appear as ``schön``. - An em dash (``—``) might appear as ``—``. @@ -53,6 +54,14 @@ All the fixes are on by default, but you can pass options to turn them off. .. autofunction:: ftfy.fix_text +.. autofunction:: ftfy.fix_text_segment + +.. autofunction:: ftfy.fix_file + +.. autofunction:: ftfy.guess_bytes + +.. autofunction:: ftfy.explain_unicode + Encodings ftfy can handle ------------------------- @@ -114,17 +123,19 @@ A note on encoding detection If your input is a mess of unmarked bytes, you might want a tool that can just statistically analyze those bytes and predict what encoding they're in. -ftfy is not that tool. I might want to write that tool someday. +ftfy is not that tool. The `guess_bytes` function will do this in very limited +cases, but to support more encodings from around the world, something more is +needed. -You may have heard of chardet. Chardet is admirable, but it is not that tool -either. Its heuristics only work on multi-byte encodings, such as UTF-8 and the -language-specific encodings used in East Asian languages. It works very badly -on single-byte encodings, to the point where it will output wrong answers with -high confidence. +You may have heard of chardet. Chardet is admirable, but it doesn't completely +do the job either. Its heuristics are designed for multi-byte encodings, such +as UTF-8 and the language-specific encodings used in East Asian languages. It +works badly on single-byte encodings, to the point where it will output wrong +answers with high confidence. -There is lots of real-world text that's in an unknown single-byte encoding. -There might be enough information to statistically sort out which encoding is -which. But nothing, so far, actually does that. +ftfy's `guess_bytes` doesn't even try the East Asian encodings, so the ideal thing +would combine the simple heuristic of `guess_bytes` with the multibyte character +set detection of `chardet`. This ideal thing doesn't exist yet. Module documentation @@ -137,6 +148,13 @@ Module documentation :members: +`ftfy.bad_codecs`: decode text from encodings Python doesn't like +----------------------------------------------------------------- + +.. automodule:: ftfy.bad_codecs + :members: + + `ftfy.badness`: measures the "badness" of text ---------------------------------------------- diff --git a/ftfy/__init__.py b/ftfy/__init__.py index a24ba61d..3938c520 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -114,8 +114,6 @@ def fix_text(text, If you are certain your entire text is in the same encoding (though that encoding is possibly flawed), and do not mind performing operations on the whole text at once, use `fix_text_segment`. - - _`bug 18183`: http://bugs.python.org/issue18183 """ if isinstance(text, bytes): raise UnicodeError(fixes.BYTES_ERROR_TEXT) @@ -172,8 +170,8 @@ def fix_file(input_file, If the file is being read as Unicode text, use that. If it's being read as bytes, then unfortunately, we have to guess what encoding it is. We'll try - a few common encodings, but we make no promises. See `guess_bytes.py` for - how this is done. + a few common encodings, but we make no promises. See the `guess_bytes` + function for how this is done. The output is a stream of fixed lines of text. """ @@ -251,7 +249,8 @@ def guess_bytes(bstring): This is not a magic bullet. If the bytes are coming from some MySQL database with the "character set" set to ISO Elbonian, this won't figure - it out. + it out. Perhaps more relevantly, this currently doesn't try East Asian + encodings. The encodings we try are: diff --git a/ftfy/fixes.py b/ftfy/fixes.py index e7479dc9..890cd0c8 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -402,8 +402,6 @@ def remove_unsafe_private_use(text): string operations that will crash some versions of it with a SystemError: http://bugs.python.org/issue18183 - You can trigger the bug by running `` '\U00010000\U00100000'.lower() ``. - The best solution is to remove all characters from Supplementary Private Use Area B, using a regex that is known not to crash given those characters. @@ -440,7 +438,7 @@ def remove_unsafe_private_use(text): def decode_escapes(text): r""" - Decode backslashed escape sequences, including \x, \u, and \U character + Decode backslashed escape sequences, including \\x, \\u, and \\U character references, even in the presence of other Unicode. This is what Python's "string-escape" and "unicode-escape" codecs were From 8f332f4d291b292191f19989253cac92686df788 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 4 Aug 2014 18:26:18 -0400 Subject: [PATCH 162/502] add another test, show progress of streamtesting --- ftfy/streamtester/__init__.py | 2 +- tests/test_real_tweets.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ftfy/streamtester/__init__.py b/ftfy/streamtester/__init__.py index c7ee1084..4b5c0614 100644 --- a/ftfy/streamtester/__init__.py +++ b/ftfy/streamtester/__init__.py @@ -34,6 +34,6 @@ def check_ftfy(self, text): # Print status updates once in a while if self.count % 100 == 0: - print('.', end='') + print('.', end='', flush=True) if self.count % 10000 == 0: print('\n%d/%d fixed' % (self.num_fixed, self.count)) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index cbf1e464..915d2570 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -35,6 +35,7 @@ 'Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New " Romance".'), ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí \u0301 \u0301"), ('HUHLL Õ…', 'HUHLL Õ...'), + ('Iggy Pop (né Jim Osterberg)', 'Iggy Pop (né Jim Osterberg)'), ## Current false positives: #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", From ada10d0cdcd4673b9d7cf43eba203c7c2ccaa725 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 6 Aug 2014 13:59:29 -0400 Subject: [PATCH 163/502] update the text_cost docstring --- ftfy/badness.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 12e8ed1e..f94fc552 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -138,7 +138,7 @@ def text_cost(text): An overall cost function for text. Weirder is worse, but all else being equal, shorter strings are better. - The overall cost is measured as twice the "weirdness" - (see :func:`sequence_weirdness`) plus the length. + The overall cost is measured as the "weirdness" (see + :func:`sequence_weirdness`) plus the length. """ return sequence_weirdness(text) + len(text) From 0b199c98369db7b4772e9e5caa95f2f15041dced Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 8 Aug 2014 11:58:24 -0400 Subject: [PATCH 164/502] update docs and version number --- README.md | 2 +- docs/accuracy.rst | 15 +++++++++ docs/conf.py | 6 ++-- docs/index.rst | 55 +++++++++++++------------------- ftfy/__init__.py | 18 +++++++++++ ftfy/bad_codecs/utf8_variants.py | 2 +- setup.py | 2 +- tests/test_real_tweets.py | 1 + 8 files changed, 63 insertions(+), 38 deletions(-) create mode 100644 docs/accuracy.rst diff --git a/README.md b/README.md index 648822b6..b9903d37 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Full documentation: **http://ftfy.readthedocs.org** — [Bruno Ranieri](http://yrrsinn.de/2012/09/17/gelesen-kw37/) - “I have no idea when I’m gonna need this, but I’m definitely bookmarking it.” — [/u/ocrow](http://reddit.com/u/ocrow) -- “Too many branches” +- “9.2/10” — [pylint](https://bitbucket.org/logilab/pylint/) ## Developed at Luminoso diff --git a/docs/accuracy.rst b/docs/accuracy.rst new file mode 100644 index 00000000..4e93f2a3 --- /dev/null +++ b/docs/accuracy.rst @@ -0,0 +1,15 @@ +This version of ftfy was evaluated on 12,200,000 tweets received from Twitter's streaming API, in early 2014. + +1627 (0.013%) of them were detected by ftfy as containing mojibake. These tweets fall into three categories: + +- True positives: ftfy turns incorrect text into correct text. +- False positives: ftfy turns correct text into incorrect text. +- Confusions: ftfy turns incorrect text into different incorrect text. + +The results for the 1627 tweets that ftfy found were: + +- True positives: 1587 (97.54%) +- False positives: 2 (0.12%) +- Confusions: 38 (2.34%) + +Two false positives in 12,200,000 examples corresponds to an overall false positive rate of 0.16 per million. diff --git a/docs/conf.py b/docs/conf.py index 64f7cf8c..3638da41 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ # General information about the project. project = u'ftfy' -copyright = u'2013, Rob Speer' +copyright = u'2014, Rob Speer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '3.1' +version = '3.3' # The full version, including alpha/beta/rc tags. -release = '3.1.2' +release = '3.3.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/index.rst b/docs/index.rst index 7fd7f872..da27f7b4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -43,25 +43,6 @@ have no control over. Somebody else's minor mistake becomes your problem. ftfy will do everything it can to fix the problem. -Using ftfy ----------- - -The main function, `fix_text`, will run text through a sequence of fixes. If -the text changed, it will run them through again, so that you can be sure -the output ends up in a standard form that will be unchanged by `fix_text`. - -All the fixes are on by default, but you can pass options to turn them off. - -.. autofunction:: ftfy.fix_text - -.. autofunction:: ftfy.fix_text_segment - -.. autofunction:: ftfy.fix_file - -.. autofunction:: ftfy.guess_bytes - -.. autofunction:: ftfy.explain_unicode - Encodings ftfy can handle ------------------------- @@ -88,6 +69,25 @@ However, ftfy cannot understand other mixups between single-byte encodings, because it is extremely difficult to detect which mixup in particular is the one that happened. +Using ftfy +---------- + +The main function, `fix_text`, will run text through a sequence of fixes. If +the text changed, it will run them through again, so that you can be sure +the output ends up in a standard form that will be unchanged by `fix_text`. + +All the fixes are on by default, but you can pass options to turn them off. + +.. autofunction:: ftfy.fix_text + +.. autofunction:: ftfy.fix_text_segment + +.. autofunction:: ftfy.fix_file + +.. autofunction:: ftfy.guess_bytes + +.. autofunction:: ftfy.explain_unicode + Non-Unicode strings ------------------- @@ -104,19 +104,6 @@ Reasonable ways that you might exchange data, such as JSON or XML, already have perfectly good ways of expressing Unicode strings. Given a Unicode string, ftfy can apply fixes that are very likely to work without false positives. -But what if you all you actually have is a mess of bytes on a disk? Well, -you've got a problem, and ftfy is not quite the right tool to solve it. - -As a sort of half-measure that covers a few common cases, you can decode the -bytes as Latin-1 and let ftfy take it from there, which might include -reinterpreting the Latin-1 text as Windows-1252 or UTF-8. - - >>> print(fix_text(b'\x85test')) - UnicodeError: [informative error message] - - >>> print(fix_text(b'\x85test'.decode('latin-1'))) - —test - A note on encoding detection ---------------------------- @@ -138,6 +125,10 @@ would combine the simple heuristic of `guess_bytes` with the multibyte character set detection of `chardet`. This ideal thing doesn't exist yet. +Accuracy +-------- +.. include:: accuracy.rst + Module documentation ==================== diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 3938c520..b39d00d6 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -308,6 +308,24 @@ def guess_bytes(bstring): def explain_unicode(text): """ A utility method that's useful for debugging mysterious Unicode. + + It breaks down a string, showing you for each codepoint its number in + hexadecimal, its glyph, its category in the Unicode standard, and its name + in the Unicode standard. + + >>> explain_unicode('(╯°□°)╯︵ ┻━┻') + U+0028 ( [Ps] LEFT PARENTHESIS + U+256F ╯ [So] BOX DRAWINGS LIGHT ARC UP AND LEFT + U+00B0 ° [So] DEGREE SIGN + U+25A1 □ [So] WHITE SQUARE + U+00B0 ° [So] DEGREE SIGN + U+0029 ) [Pe] RIGHT PARENTHESIS + U+256F ╯ [So] BOX DRAWINGS LIGHT ARC UP AND LEFT + U+FE35 ︵ [Ps] PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS + U+0020 [Zs] SPACE + U+253B ┻ [So] BOX DRAWINGS HEAVY UP AND HORIZONTAL + U+2501 ━ [So] BOX DRAWINGS HEAVY HORIZONTAL + U+253B ┻ [So] BOX DRAWINGS HEAVY UP AND HORIZONTAL """ for char in text: if is_printable(char): diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 1d59357e..565cb2b4 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -232,7 +232,7 @@ def mangle_surrogates(bytestring): it as an error (which it is). In 'replace' mode, it will decode as three replacement characters. But Python 2 will just output the surrogate codepoint. - + To ensure consistency between Python 2 and Python 3, and protect downstream applications from malformed strings, we turn surrogate sequences at the start of the string into the bytes `ff ff ff`, which we're *sure* won't diff --git a/setup.py b/setup.py index e50bd164..35fe4b62 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.2.0', + version='3.3.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 915d2570..eb1086cd 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -41,6 +41,7 @@ #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", # "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), #("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), + #("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿") ## This kind of tweet can't be fixed without a full-blown encoding detector. #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", From f6a20901793eefa06155fa757474556d753f489d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 12 Aug 2014 11:03:44 -0400 Subject: [PATCH 165/502] move optional imports to allow doctests to work The streamtester modules need to import things from the 'twitter' package, but these should not be run as part of normal testing. Doctests were failing because they tried to import twitter, so I've moved the import. This had already been done in the main twitter tester, but I neglected to also hide the import that's in "oauth.py". --- ftfy/streamtester/oauth.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ftfy/streamtester/oauth.py b/ftfy/streamtester/oauth.py index 31a801bb..8e300ed7 100644 --- a/ftfy/streamtester/oauth.py +++ b/ftfy/streamtester/oauth.py @@ -4,8 +4,6 @@ somebody's Twitter account. """ from __future__ import unicode_literals -from twitter.oauth import OAuth -from twitter import oauth_dance, read_token_file import os @@ -39,6 +37,10 @@ def get_auth(): secret behind a fig leaf like everybody else does. So that's what we've done. """ + + from twitter.oauth import OAuth + from twitter import oauth_dance, read_token_file + def unhide(secret): """ Do something mysterious and exactly as secure as every other Twitter From 540e58ca66783bf04fa5dee0b447bb3764cd087a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 14:10:59 -0400 Subject: [PATCH 166/502] Stop claiming support for 2.6 I don't even have a Python 2.6 interpreter. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 35fe4b62..727c2696 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,6 @@ package_data={'ftfy': ['char_classes.dat']}, classifiers=[ "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.2", From 04a2dca70f5bac8e98e7cbd02eb1c8b9cd437d33 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 14:23:54 -0400 Subject: [PATCH 167/502] fix a doctest for Python 2 For some reason, raw strings on Python 2 don't prevent \u escapes from being interpreted in some cases. This may be an unfortunate interaction between the r'' syntax and "from __future__ import unicode_literals". This makes things awkward when you're trying to give an example *about* \u escapes. --- ftfy/fixes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 890cd0c8..fb1f934b 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -447,8 +447,8 @@ def decode_escapes(text): literals. >>> factoid = '\\u20a1 is the currency symbol for the colón.' - >>> print(factoid) - \u20a1 is the currency symbol for the colón. + >>> print(factoid[1:]) + u20a1 is the currency symbol for the colón. >>> print(decode_escapes(factoid)) ₡ is the currency symbol for the colón. From 21fcb2f63514574a3a3116f47641aaf36b1a7fec Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 15:05:33 -0400 Subject: [PATCH 168/502] fix some trailing spaces, update changelog --- CHANGELOG.txt | 34 ++++++++++++++++++++++++++++++++++ ftfy/__init__.py | 2 +- tests/test_real_tweets.py | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 24a3c9fb..3ae578bc 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,34 @@ +## Version 3.3.0 (August 16, 2014) + +Heuristic changes: + +- Certain symbols are marked as "ending punctuation" that may naturally occur + after letters. When they follow an accented capital letter and look like + mojibake, they will not be "fixed" without further evidence. + An example is that "MARQUÉ…" will become "MARQUÉ...", and not "MARQUɅ". + +New features: + +- `ftfy.explain_unicode` is a diagnostic function that shows you what's going + on in a Unicode string. It shows you a table with each code point in + hexadecimal, its glyph, its name, and its Unicode category. + +- `ftfy.fixes.decode_escapes` adds a feature missing from the standard library: + it lets you decode a Unicode string with backslashed escape sequences in it + (such as "\u2014") the same way that Python itself would. + +- `ftfy.streamtester` is a release of the code that I use to test ftfy on + an endless stream of real-world data from Twitter. With the new heuristics, + the false positive rate of ftfy is about 1 per 6 million tweets. (See + the "Accuracy" section of the documentation.) + +Deprecations: + +- Python 2.6 is no longer supported. + +- `remove_unsafe_private_use` is no longer needed in any current version of + Python. This fixer will disappear in a later version of ftfy. + ## Version 3.2.0 (June 27, 2014) - `fix_line_breaks` fixes three additional characters that are considered line @@ -21,9 +52,12 @@ - Add the `ftfy.bad_codecs` package, which registers new codecs that can decoding things that Python may otherwise refuse to decode: + - `utf-8-variants`, which decodes CESU-8 and its Java lookalike + - `sloppy-windows-*`, which decodes character-map encodings while treating unmapped characters as Latin-1 + - Simplify the code using `ftfy.bad_codecs`. ## Version 3.0.6 (November 5, 2013) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index b39d00d6..2887c5b9 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -308,7 +308,7 @@ def guess_bytes(bstring): def explain_unicode(text): """ A utility method that's useful for debugging mysterious Unicode. - + It breaks down a string, showing you for each codepoint its number in hexadecimal, its glyph, its category in the Unicode standard, and its name in the Unicode standard. diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index eb1086cd..c1a748ac 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -36,7 +36,7 @@ ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí \u0301 \u0301"), ('HUHLL Õ…', 'HUHLL Õ...'), ('Iggy Pop (né Jim Osterberg)', 'Iggy Pop (né Jim Osterberg)'), - + ## Current false positives: #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", # "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), From 7a8e4ceaba1d7bfabd5f2a53bb445a830654dfde Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 15:14:50 -0400 Subject: [PATCH 169/502] Write an example differently because of Read The Docs Yuck. Read The Docs runs on Python 2, and it interacts badly with docstrings when "from __future__ import unicode_literals" is on. It was already hard to document things *about* Unicode without using Python 3 syntax. But when I use the Python 3 syntax, RTD makes it say the wrong thing. I'm working around this for now, but I really hope that Read The Docs offers a Python 3 option sometime. --- ftfy/fixes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index fb1f934b..917a789e 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -337,8 +337,8 @@ def fix_line_breaks(text): \\x85 is very common mojibake for \\u2026, HORIZONTAL ELLIPSIS. >>> print(fix_line_breaks( - ... "This string is made of two things:\u2029" - ... "1. Unicode\u2028" + ... "This string is made of two things:\N{PARAGRAPH SEPARATOR}" + ... "1. Unicode\N{LINE SEPARATOR}" ... "2. Spite" ... )) This string is made of two things: From facae0948a6fe0358bb05f5116787fc8b0d965e4 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 15:20:02 -0400 Subject: [PATCH 170/502] unindent docstrings outside of doctests --- ftfy/fixes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 917a789e..69c3bfb3 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -332,9 +332,9 @@ def fix_line_breaks(text): - NEXT LINE (\\x85), a C1 control character that is certainly not what you meant - The NEXT LINE character is a bit of an odd case, because it - usually won't show up if `fix_encoding` is also being run. - \\x85 is very common mojibake for \\u2026, HORIZONTAL ELLIPSIS. + The NEXT LINE character is a bit of an odd case, because it + usually won't show up if `fix_encoding` is also being run. + \\x85 is very common mojibake for \\u2026, HORIZONTAL ELLIPSIS. >>> print(fix_line_breaks( ... "This string is made of two things:\N{PARAGRAPH SEPARATOR}" @@ -345,8 +345,8 @@ def fix_line_breaks(text): 1. Unicode 2. Spite - For further testing and examples, let's define a function to make sure - we can see the control characters in their escaped form: + For further testing and examples, let's define a function to make sure + we can see the control characters in their escaped form: >>> def eprint(text): ... print(text.encode('unicode-escape').decode('ascii')) From 4c62c1fc5d28c5931e0991bbe578de9c8e08792e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 15:24:24 -0400 Subject: [PATCH 171/502] Oh cool. readthedocs *does* support Python 3. --- ftfy/fixes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 69c3bfb3..8da51aa4 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -337,8 +337,8 @@ def fix_line_breaks(text): \\x85 is very common mojibake for \\u2026, HORIZONTAL ELLIPSIS. >>> print(fix_line_breaks( - ... "This string is made of two things:\N{PARAGRAPH SEPARATOR}" - ... "1. Unicode\N{LINE SEPARATOR}" + ... "This string is made of two things:\u2029" + ... "1. Unicode\u2028" ... "2. Spite" ... )) This string is made of two things: @@ -390,7 +390,7 @@ def remove_bom(text): r""" Remove a left-over byte-order mark. - >>> print(remove_bom(unichr(0xfeff) + "Where do you want to go today?")) + >>> print(remove_bom("\ufeffWhere do you want to go today?")) Where do you want to go today? """ return text.lstrip(unichr(0xfeff)) From bbaaae8f987b703e0aa66530c08268519600bee0 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 15:35:06 -0400 Subject: [PATCH 172/502] start updating the README --- README.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b9903d37..6afb8789 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ page](http://www.luminoso.com/careers.html). ## What it does ftfy makes Unicode text less broken and more consistent. It works in -Python 2.6, Python 3.2, or later. +Python 2.7, Python 3.2, or later. The most interesting kind of brokenness that this resolves is when someone has encoded Unicode with one standard and decoded it with a different one. @@ -44,8 +44,8 @@ This often shows up as characters that turn into nonsense sequences: - The word `schön` might appear as `schön`. - An em dash (`—`) might appear as `—`. - Text that was meant to be enclosed in quotation marks might end up - instead enclosed in `“` and `”` (and that last character - probably won't even display as anything meaningful). + instead enclosed in `“` and `â€<9d>` (where <9d> is an unprintable + codepoint). This is called "mojibake", and it happens very often to real text. Fortunately, the nonsense sequences usually contain all the information you need to @@ -110,10 +110,6 @@ the output ends up in a standard form that will be unchanged by `fix_text`. All the fixes are on by default, but you can pass options to turn them off. -- If `remove_unsafe_private_use` is True, remove a range of unassigned - characters that can crash Python < 3.3.3 via - [bug 18183](http://bugs.python.org/issue18183). (This fix is off by - default on Python 3.4, because it's no longer necessary.) - If `fix_entities` is True, consider replacing HTML entities with their equivalent characters. However, this never applies to text with a pair of angle brackets in it already; you're probably not supposed to decode From 615022be1cf431acf2bce36308ac914321764b14 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 15:46:19 -0400 Subject: [PATCH 173/502] Update the README to match other docs --- README.md | 74 ++++++++++++++++++++++--------------------------------- 1 file changed, 30 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 6afb8789..f9399703 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Downloads](https://pypip.in/d/ftfy/badge.png)](https://crate.io/packages/ftfy) [![Version](https://pypip.in/v/ftfy/badge.png)](https://crate.io/packages/ftfy) +[![Docs](https://readthedocs.org/projects/ftfy/badge/?version=latest)](http://ftfy.readthedocs.org/en/latest/) Full documentation: **http://ftfy.readthedocs.org** @@ -109,20 +110,22 @@ the text changed, it will run them through again, so that you can be sure the output ends up in a standard form that will be unchanged by `fix_text`. All the fixes are on by default, but you can pass options to turn them off. - -- If `fix_entities` is True, consider replacing HTML entities with their - equivalent characters. However, this never applies to text with a pair - of angle brackets in it already; you're probably not supposed to decode - entities there, and you'd make things ambiguous if you did. + +- If `fix_entities` is True, replace HTML entities with their equivalent + characters. If it's "auto" (the default), then consider replacing HTML + entities, but don't do so in text where you have seen a pair of actual + angle brackets (that's probably actually HTML and you shouldn't mess + with the entities). - If `remove_terminal_escapes` is True, remove sequences of bytes that are instructions for Unix terminals, such as the codes that make text appear in different colors. - If `fix_encoding` is True, look for common mistakes that come from encoding or decoding Unicode text incorrectly, and fix them if they are - reasonably fixable. + reasonably fixable. See `fix_text_encoding` for details. - If `normalization` is not None, apply the specified form of Unicode normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. The default, 'NFKC', applies the following relevant transformations: + - C: Combine characters and diacritics that are written using separate code points, such as converting "e" plus an acute accent modifier into "é", or converting "ka" (か) plus a dakuten into the @@ -132,18 +135,18 @@ All the fixes are on by default, but you can pass options to turn them off. full-width versions, full-width Roman characters will be replaced with ASCII characters, ellipsis characters will be replaced with three periods, and the ligature 'fl' will be replaced with 'fl'. + - If `uncurl_quotes` is True, replace various curly quotation marks with plain-ASCII straight quotes. - If `fix_line_breaks` is true, convert all line breaks to Unix style (CRLF and CR line breaks become LF line breaks). -- If `remove_control_chars` is true, remove all C0 control characters +- If `fix_control_characters` is true, remove all C0 control characters except the common useful ones: TAB, CR, LF, and FF. (CR characters may have already been removed by the `fix_line_breaks` step.) - If `remove_bom` is True, remove the Byte-Order Mark if it exists. - (It's a hint for a UTF-16 decoder. It's not meant to actually - end up in your string.) - If anything was changed, repeat all the steps, so that the function is - idempotent. `"&amp;"` will become `"&"`, for example, not `"&"`. + idempotent. "&amp;" will become "&", for example, not "&". + ### Encodings ftfy can handle @@ -184,35 +187,24 @@ Reasonable ways that you might exchange data, such as JSON or XML, already have perfectly good ways of expressing Unicode strings. Given a Unicode string, ftfy can apply fixes that are very likely to work without false positives. -But what if you all you actually have is a mess of bytes on a disk? Well, -you've got a problem, and ftfy is not quite the right tool to solve it. - -As a sort of half-measure that covers a few common cases, you can decode the -bytes as Latin-1 and let ftfy take it from there, which might include -reinterpreting the Latin-1 text as Windows-1252 or UTF-8. - - >>> print(fix_text(b'\x85test')) - UnicodeError: [informative error message] - - >>> print(fix_text(b'\x85test'.decode('latin-1'))) - —test - ### A note on encoding detection If your input is a mess of unmarked bytes, you might want a tool that can just statistically analyze those bytes and predict what encoding they're in. -ftfy is not that tool. I might want to write that tool someday. +ftfy is not that tool. The `guess_bytes` function will do this in very limited +cases, but to support more encodings from around the world, something more is +needed. -You may have heard of chardet. Chardet is admirable, but it is not that tool -either. Its heuristics only work on multi-byte encodings, such as UTF-8 and the -language-specific encodings used in East Asian languages. It works very badly -on single-byte encodings, to the point where it will output wrong answers with -high confidence. +You may have heard of chardet. Chardet is admirable, but it doesn't completely +do the job either. Its heuristics are designed for multi-byte encodings, such +as UTF-8 and the language-specific encodings used in East Asian languages. It +works badly on single-byte encodings, to the point where it will output wrong +answers with high confidence. -There is lots of real-world text that's in an unknown single-byte encoding. -There might be enough information to statistically sort out which encoding is -which. But nothing, so far, actually does that. +ftfy's `guess_bytes` doesn't even try the East Asian encodings, so the ideal thing +would combine the simple heuristic of `guess_bytes` with the multibyte character +set detection of `chardet`. This ideal thing doesn't exist yet. ## Command-line usage @@ -220,19 +212,13 @@ ftfy installs itself as a command line tool that reads a file and applies `fix_text` to it. This has exactly the problem described above: a file on a disk is made of bytes -in an unspecified encoding. It could assume the file is UTF-8, but if you had -totally valid UTF-8 you probably wouldn't need this command line utility, and -there's a slight chance that the file could contain Latin-1 that coincidentally -looks like UTF-8. - -Instead, it will follow the "half-measure" above. - -You can type `ftfy FILENAME`, and it will read in FILENAME as Latin-1 text, fix -everything that `fix_text` fixes (including re-interpreting it as UTF-8 if -appropriate), and write the result to standard out as UTF-8. +in an unspecified encoding. It has to guess the encoding of the bytes in the +file. But if it guesses wrong, it might be able to fix it anyway, because +that's what ftfy does. -This is not necessarily a good idea, but it's convenient. Consider this a proof -of concept until we get a real encoding detector. +You can type `ftfy FILENAME`, and it will read in FILENAME, guess its encoding, +fix everything that `fix_text` fixes, and write the result to standard out as +UTF-8. ## Who maintains ftfy? From 1b452f6093c51a42f4b90668487bc98e32397fbd Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 15:47:29 -0400 Subject: [PATCH 174/502] fix thing that looked like an HTML tag --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f9399703..199bf641 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ This often shows up as characters that turn into nonsense sequences: - The word `schön` might appear as `schön`. - An em dash (`—`) might appear as `—`. - Text that was meant to be enclosed in quotation marks might end up - instead enclosed in `“` and `â€<9d>` (where <9d> is an unprintable + instead enclosed in `“` and `â€<9d>` (where `<9d>` is an unprintable codepoint). This is called "mojibake", and it happens very often to real text. Fortunately, From 9e14ca700000b54bf65c41ab0413fbb1e7b4add3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 15:50:34 -0400 Subject: [PATCH 175/502] try renaming CHANGELOG.txt to CHANGELOG.md --- CHANGELOG.txt => CHANGELOG.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CHANGELOG.txt => CHANGELOG.md (100%) diff --git a/CHANGELOG.txt b/CHANGELOG.md similarity index 100% rename from CHANGELOG.txt rename to CHANGELOG.md From 799946ac7c6e777199f57035e71bf491132073bf Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 15:51:56 -0400 Subject: [PATCH 176/502] okay, markdown, this is supposed to be a literal --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ae578bc..eafab461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,9 +35,9 @@ Deprecations: breaks in some environments, such as Javascript, and Python's "codecs" library. These are all now replaced with \n: - U+0085 , with alias "NEXT LINE" - U+2028 LINE SEPARATOR - U+2029 PARAGRAPH SEPARATOR + U+0085 , with alias "NEXT LINE" + U+2028 LINE SEPARATOR + U+2029 PARAGRAPH SEPARATOR ## Version 3.1.3 (May 15, 2014) From 7f025a25feef4f83fe2a6a34b41662cce2ab00d6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Sat, 16 Aug 2014 15:55:18 -0400 Subject: [PATCH 177/502] dropped support for 2.6 --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index da27f7b4..1450d81c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ ftfy: fixes text for you ======================== This is a module for making text less broken and more consistent. It works in -Python 2.6, Python 3.2, or later. +Python 2.7, Python 3.2, or later. The most interesting kind of brokenness that this resolves is when someone has encoded Unicode with one standard and decoded it with a different one. This From f3c5bf8b16d3e3f52980acd99e40adf9610c20e7 Mon Sep 17 00:00:00 2001 From: Jared Forsyth Date: Mon, 18 Aug 2014 09:21:56 -0700 Subject: [PATCH 178/502] fix & example to make sense when rendered as markdown Before it looked like `"&" will become "&", for example, not "&"` which doesn't make sense --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 199bf641..b2d5e611 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ All the fixes are on by default, but you can pass options to turn them off. may have already been removed by the `fix_line_breaks` step.) - If `remove_bom` is True, remove the Byte-Order Mark if it exists. - If anything was changed, repeat all the steps, so that the function is - idempotent. "&amp;" will become "&", for example, not "&". + idempotent. "&amp;amp;" will become "&", for example, not "&amp;". ### Encodings ftfy can handle From cf73d54ce623ecc85a716cf94b318120c13fca84 Mon Sep 17 00:00:00 2001 From: Tim Lewandowski Date: Tue, 2 Dec 2014 11:08:51 -0500 Subject: [PATCH 179/502] Explicitly specify indices for .format --- ftfy/chardata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 5c72dd40..e853ed3e 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -39,7 +39,7 @@ def _build_regexes(): # bytes 0x80-0xff in this character set. (This uses the fact that all # regex special characters are ASCII, and therefore won't appear in the # string.) - regex = '^[\x00-\x7f{}]*$'.format(charlist) + regex = '^[\x00-\x7f{0}]*$'.format(charlist) encoding_regexes[encoding] = re.compile(regex) return encoding_regexes ENCODING_REGEXES = _build_regexes() From 582fd64c292c70d137f2bbc2d4649eeda0a753b0 Mon Sep 17 00:00:00 2001 From: Tim Lewandowski Date: Tue, 2 Dec 2014 11:27:38 -0500 Subject: [PATCH 180/502] Test to see if Python 2.6 is being used, and use a positional argument for errors in decode, rather than the keyword 'errors=' --- ftfy/bad_codecs/sloppy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index 39a56e0a..073b1443 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -72,9 +72,10 @@ from __future__ import unicode_literals import codecs from encodings import normalize_encoding +import sys REPLACEMENT_CHAR = '\ufffd' - +PY26 = sys.version_info[:2] == (2,6) def make_sloppy_codec(encoding): """ @@ -94,7 +95,10 @@ def make_sloppy_codec(encoding): # Get a list of what they decode to in the given encoding. Use the # replacement character for unassigned bytes. - decoded_chars = all_bytes.decode(encoding, errors='replace') + if PY26: + decoded_chars = all_bytes.decode(encoding, 'replace') + else: + decoded_chars = all_bytes.decode(encoding, errors='replace') # Update the sloppy_chars list. Each byte that was successfully decoded # gets its decoded value in the list. The unassigned bytes are left as From 82faf50eb63af960b22627c7b1d6397424f83074 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 12 Dec 2014 18:43:10 -0500 Subject: [PATCH 181/502] another change for 2.6 compatibility --- tests/test_encodings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_encodings.py b/tests/test_encodings.py index e7f9ef97..3500ecc2 100644 --- a/tests/test_encodings.py +++ b/tests/test_encodings.py @@ -16,7 +16,7 @@ def test_cesu8(): def test_russian_crash(): thebytes = b'\xe8\xed\xe2\xe5\xed\xf2\xe0\xf0\xe8\xe7\xe0\xf6\xe8\xff ' # We don't care what the result is, but this shouldn't crash - thebytes.decode('utf-8-variants', errors='replace') + thebytes.decode('utf-8-variants', 'replace') # This shouldn't crash either guess_bytes(thebytes) From fcf821c1663466120174fca0de24253882360d74 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 12 Dec 2014 18:43:46 -0500 Subject: [PATCH 182/502] Bump the version, indicate 2.6 compatibility --- CHANGELOG.md | 4 ++++ setup.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eafab461..37637583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 3.3.1 (December 12, 2014) + +This version restores compatibility with Python 2.6. + ## Version 3.3.0 (August 16, 2014) Heuristic changes: diff --git a/setup.py b/setup.py index 727c2696..4a533bf6 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.3.0', + version='3.3.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", @@ -13,6 +13,7 @@ package_data={'ftfy': ['char_classes.dat']}, classifiers=[ "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.2", From 5041d33c49717253b5ffc5c2de1f6ac6aa9dadf3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 9 Jan 2015 15:01:45 -0500 Subject: [PATCH 183/502] add fixes.fix_surrogates This enables working with surrogate-encoded text more safely. It still needs tests. --- ftfy/__init__.py | 11 +++++++++++ ftfy/fixes.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 2887c5b9..181b5823 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -27,6 +27,7 @@ def fix_text(text, normalization='NFKC', uncurl_quotes=True, fix_line_breaks=True, + fix_surrogates=True, remove_control_chars=True, remove_bom=True, max_decode_length=2**16): @@ -99,6 +100,10 @@ def fix_text(text, plain-ASCII straight quotes. - If `fix_line_breaks` is true, convert all line breaks to Unix style (CRLF and CR line breaks become LF line breaks). + - If `fix_surrogates` is true, ensure that there are no UTF-16 surrogates + in the resulting string, by converting them to the correct characters + when they're appropriately paired, or replacing them with \ufffd + otherwise. - If `fix_control_characters` is true, remove all C0 control characters except the common useful ones: TAB, CR, LF, and FF. (CR characters may have already been removed by the `fix_line_breaks` step.) @@ -144,6 +149,7 @@ def fix_text(text, normalization=normalization, uncurl_quotes=uncurl_quotes, fix_line_breaks=fix_line_breaks, + fix_surrogates=fix_surrogates, remove_control_chars=remove_control_chars, remove_bom=remove_bom ) @@ -163,6 +169,7 @@ def fix_file(input_file, normalization='NFKC', uncurl_quotes=True, fix_line_breaks=True, + fix_surrogates=True, remove_control_chars=True, remove_bom=True): """ @@ -190,6 +197,7 @@ def fix_file(input_file, normalization=normalization, uncurl_quotes=uncurl_quotes, fix_line_breaks=fix_line_breaks, + fix_surrogates=fix_surrogates, remove_control_chars=remove_control_chars, remove_bom=remove_bom ) @@ -203,6 +211,7 @@ def fix_text_segment(text, normalization='NFKC', uncurl_quotes=True, fix_line_breaks=True, + fix_surrogates=True, remove_control_chars=True, remove_bom=True): """ @@ -233,6 +242,8 @@ def fix_text_segment(text, text = fixes.uncurl_quotes(text) if fix_line_breaks: text = fixes.fix_line_breaks(text) + if fix_surrogates: + text = fixes.fix_surrogates(text) if remove_control_chars: text = fixes.remove_control_chars(text) if remove_bom: diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 8da51aa4..4852eeac 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -365,6 +365,27 @@ def fix_line_breaks(text): .replace('\u0085', '\n') +SURROGATE_RE = re.compile('[\ud800-\udfff]') +SURROGATE_PAIR_RE = re.compile('[\ud800-\udbff][\udc00-\udfff]') + + +def convert_surrogate_pair(match): + pair = match.group(0) + codept = 0x10000 + (ord(pair[0]) - 0xd800) * 0x400 + (ord(pair[1]) - 0xdc00) + return chr(codept) + + +def fix_surrogates(text): + """ + Replace 16-bit surrogate codepoints with the characters they represent + (when properly paired), or with \ufffd otherwise. + """ + if SURROGATE_RE.search(text): + text = SURROGATE_PAIR_RE.sub(convert_surrogate_pair, text) + text = SURROGATE_RE.sub('\ufffd', text) + return text + + def remove_control_chars(text): """ Remove all control characters except for the important ones. From 6b72a6f6024a43b183a19ccadf719acaf67ef2bb Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 9 Jan 2015 17:58:50 -0500 Subject: [PATCH 184/502] test fix_surrogates, deprecate remove_unsafe_private_use --- ftfy/__init__.py | 18 +++++++++-------- ftfy/fixes.py | 47 ++++++++++++++++++++++++++----------------- setup.py | 2 +- tests/test_unicode.py | 8 +++++--- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 181b5823..c3fcbd90 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -14,13 +14,13 @@ from ftfy import fixes from ftfy.fixes import fix_text_encoding -from ftfy.compatibility import PYTHON34_OR_LATER, is_printable +from ftfy.compatibility import is_printable import unicodedata import warnings def fix_text(text, - remove_unsafe_private_use=(not PYTHON34_OR_LATER), + remove_unsafe_private_use=False, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, @@ -67,10 +67,6 @@ def fix_text(text, Based on the options you provide, ftfy applies these steps in order: - - If `remove_unsafe_private_use` is True, it removes a range of private-use - characters that could trigger a Python bug. The bug is fixed in - the most recent versions of Python, so this will default to False - starting on Python 3.4. - If `fix_entities` is True, replace HTML entities with their equivalent characters. If it's "auto" (the default), then consider replacing HTML entities, but don't do so in text where you have seen a pair of actual @@ -111,6 +107,12 @@ def fix_text(text, - If anything was changed, repeat all the steps, so that the function is idempotent. "&amp;" will become "&", for example, not "&". + There is one deprecated option: + + - If `remove_unsafe_private_use` is True, it issues a DeprecationWarning + and applies a workaround for an old Python bug. This option will go away + in ftfy 3.5. + `fix_text` will work one line at a time, with the possibility that some lines are in different encodings. When it encounters lines longer than `max_decode_length`, it will not run the `fix_encoding` step, to avoid @@ -162,7 +164,7 @@ def fix_text(text, def fix_file(input_file, - remove_unsafe_private_use=True, + remove_unsafe_private_use=False, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, @@ -204,7 +206,7 @@ def fix_file(input_file, def fix_text_segment(text, - remove_unsafe_private_use=True, + remove_unsafe_private_use=False, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 4852eeac..557516a5 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -12,6 +12,7 @@ import re import sys import codecs +import warnings BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. @@ -379,6 +380,18 @@ def fix_surrogates(text): """ Replace 16-bit surrogate codepoints with the characters they represent (when properly paired), or with \ufffd otherwise. + + >>> high_surrogate = chr(0xd83d) + >>> low_surrogate = chr(0xdca9) + >>> print(fix_surrogates(high_surrogate + low_surrogate)) + 💩 + >>> print(fix_surrogates(low_surrogate + high_surrogate)) + �� + + The above doctest had to be very carefully written, because even putting + the Unicode escapes of the surrogates in the docstring was causing + various tools to fail, which I think just goes to show why this fixer is + necessary. """ if SURROGATE_RE.search(text): text = SURROGATE_PAIR_RE.sub(convert_surrogate_pair, text) @@ -419,30 +432,28 @@ def remove_bom(text): def remove_unsafe_private_use(text): r""" - Python 3.3's Unicode support isn't perfect, and in fact there are certain - string operations that will crash some versions of it with a SystemError: - http://bugs.python.org/issue18183 - - The best solution is to remove all characters from Supplementary Private - Use Area B, using a regex that is known not to crash given those - characters. + This function is deprecated, because the Python bug it works around has + been fixed for a long time. It only affects old versions of Python that + are also insecure in more serious ways. - These are the characters from U+100000 to U+10FFFF. It's sad to lose an - entire plane of Unicode, but on the other hand, these characters are not - assigned and never will be. If you get one of these characters and don't - know what its purpose is, its purpose is probably to crash your code. + Certain string operations would crash Python < 3.3.3 and < 2.7.6 with a + SystemError: http://bugs.python.org/issue18183 - If you were using these for actual private use, this might be inconvenient. - You can turn off this fixer, of course, but I kind of encourage using - Supplementary Private Use Area A instead. + ftfy's solution was to remove all characters from Supplementary Private + Use Area B, using a regex that is known not to crash given those + characters. In retrospect, it would have been a better idea to indicate + that they were replaced, using the Unicode replacement character \ufffd. >>> print(remove_unsafe_private_use('\U0001F4A9\U00100000')) 💩 - - This fixer is off by default in Python 3.4 or later. (The bug is actually - fixed in 3.3.3 and 2.7.6, but I don't want the default behavior to change - based on a micro version upgrade of Python.) + + ftfy no longer applies this fix, and it will be removed in ftfy 3.5. """ + warnings.warn( + "remove_unsafe_private_use is deprecated and will be removed in " + "ftfy 3.5", + DeprecationWarning + ) return UNSAFE_PRIVATE_USE_RE.sub('', text) diff --git a/setup.py b/setup.py index 4a533bf6..876d3501 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="ftfy", - version='3.3.1', + version='3.4.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 3a7331ae..2cd68700 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from ftfy.fixes import fix_text_encoding, fix_encoding_and_explain, apply_plan, possible_encoding +from ftfy.fixes import fix_text_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, fix_surrogates from ftfy.badness import ENDING_PUNCT_RE import unicodedata import sys @@ -45,13 +45,15 @@ def test_all_bmp_characters(): def test_valid_phrases(): for phrase in phrases: yield check_phrase, phrase - # make it not just confirm based on the opening punctuation - yield check_phrase, phrase[1:] def check_phrase(text): eq_(fix_text_encoding(text), text) eq_(fix_text_encoding(text.encode('utf-8').decode('latin-1')), text) + # make sure that the opening punctuation is not the only thing that makes + # it work + eq_(fix_text_encoding(text[1:]), text[1:]) + eq_(fix_text_encoding(text[1:].encode('utf-8').decode('latin-1')), text[1:]) def test_possible_encoding(): From c15ac0b345cf5891b980d3c952d35ce115b4a04d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 9 Jan 2015 18:01:08 -0500 Subject: [PATCH 185/502] adjust fix_surrogates for python 2 --- ftfy/fixes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 557516a5..c0359630 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -373,7 +373,7 @@ def fix_line_breaks(text): def convert_surrogate_pair(match): pair = match.group(0) codept = 0x10000 + (ord(pair[0]) - 0xd800) * 0x400 + (ord(pair[1]) - 0xdc00) - return chr(codept) + return unichr(codept) def fix_surrogates(text): @@ -381,8 +381,8 @@ def fix_surrogates(text): Replace 16-bit surrogate codepoints with the characters they represent (when properly paired), or with \ufffd otherwise. - >>> high_surrogate = chr(0xd83d) - >>> low_surrogate = chr(0xdca9) + >>> high_surrogate = unichr(0xd83d) + >>> low_surrogate = unichr(0xdca9) >>> print(fix_surrogates(high_surrogate + low_surrogate)) 💩 >>> print(fix_surrogates(low_surrogate + high_surrogate)) From 87193c0a601e615599ef3cb50610ba2a66e76963 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 9 Jan 2015 18:04:14 -0500 Subject: [PATCH 186/502] update changelog for 3.4.0 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37637583..64bcbfd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## Version 3.4.0 (January 9, 2015) + +New features: + +- `ftfy.fixes.fix_surrogates` will fix all 16-bit surrogate codepoints, + which would otherwise break various encoding and output functions. + +Deprecations: + +- `remove_unsafe_private_use` emits a warning, and will disappear in the + next minor or major version. + ## Version 3.3.1 (December 12, 2014) This version restores compatibility with Python 2.6. From afb57058bf09b598d1318cbfc89c163d9ea59308 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 15 Jan 2015 18:09:26 -0500 Subject: [PATCH 187/502] code review fixes --- ftfy/fixes.py | 9 ++++++++- tests/test_unicode.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index c0359630..8b78eaad 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -371,6 +371,12 @@ def fix_line_breaks(text): def convert_surrogate_pair(match): + """ + Convert a surrogate pair to the single codepoint it represents. + + This implements the formula described at: + http://en.wikipedia.org/wiki/Universal_Character_Set_characters#Surrogates + """ pair = match.group(0) codept = 0x10000 + (ord(pair[0]) - 0xd800) * 0x400 + (ord(pair[1]) - 0xdc00) return unichr(codept) @@ -447,7 +453,8 @@ def remove_unsafe_private_use(text): >>> print(remove_unsafe_private_use('\U0001F4A9\U00100000')) 💩 - ftfy no longer applies this fix, and it will be removed in ftfy 3.5. + ftfy no longer applies this fix by default, and it will be removed in + ftfy 3.5. """ warnings.warn( "remove_unsafe_private_use is deprecated and will be removed in " diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 2cd68700..2d609207 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -66,5 +66,11 @@ def test_fix_with_backslash(): eq_(fix_text_encoding(u"<40\\% vs \xe2\x89\xa540\\%"), u"<40\\% vs ≥40\\%") +def test_surrogates(): + eq_(fix_surrogates(u'\udbff\udfff'), u'\U0010ffff') + eq_(fix_surrogates(u'\ud800\udc00'), u'\U00010000') + + + if __name__ == '__main__': test_all_bmp_characters() From c1cebeb174fc4a25236dd0d9bfb5e7f4720c6b1d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 15 Jan 2015 18:09:45 -0500 Subject: [PATCH 188/502] update changelog date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64bcbfd5..4bcca067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Version 3.4.0 (January 9, 2015) +## Version 3.4.0 (January 15, 2015) New features: From 791887955798033784504b2c66315d5adfa6c036 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 18 Mar 2015 20:23:14 -0400 Subject: [PATCH 189/502] Update to Unicode 7 The data was built using Python 3.5a2, so that the heuristics can take new characters in Unicode 7 into account, even when running on a version of Python that doesn't know what they are. 3.5 is the first version to support Unicode 7, which can already be found in some operating systems, and I'm assuming that Python's Unicode table is not going to change between now and release. --- ftfy/build_data.py | 16 +++++++++------- ftfy/char_classes.dat | Bin 3568 -> 3847 bytes 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index f556b306..8e78890e 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -43,15 +43,17 @@ def make_char_data_file(do_it_anyway=False): Build the compressed data file 'char_classes.dat' and write it to the current directory. - If you run this, run it in Python 3.3 or later. It will run in earlier - versions, but you won't get the current Unicode standard, leading to - inconsistent behavior. To protect against this, running this in the - wrong version of Python will raise an error unless you pass - `do_it_anyway=True`. + If you run this, run it in Python 3.5 or later, even though that requires + an alpha version at the time of writing this code. It will run in earlier + versions, but you won't get the Unicode 7 standard, leading to inconsistent + behavior. + + To protect against this, running this in the wrong version of Python will + raise an error unless you pass `do_it_anyway=True`. """ - if sys.hexversion < 0x03030000 and not do_it_anyway: + if sys.hexversion < 0x03050000 and not do_it_anyway: raise RuntimeError( - "This function should be run in Python 3.3 or later." + "This function should be run in Python 3.5 or later." ) cclasses = [None] * 0x110000 diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index 84155cd5201b19cbdc66d6a977a356a1bf1732e8..e948607c13bde716ea9ea2b01f53929c7e3c55da 100644 GIT binary patch literal 3847 zcmeHFc{JOJ7S=bNYG+1uS2ddH(9N1s`}Vpjm8v09Ta?;&N@Iznoi;TnX{kXgQbm;z z4MLG7oocAP1hK~!X#{B!LBub4GjINw_vWv6&iwVxneUH#&OP6~-@WI%H@;wL!Y-!x z!#@;05(akrMgId+f7I4?6T&y3ME)q{TZ=3;gK7wq*`0eH&N&!-_);$OkctWG`Rh6U zBet&Q{u*V@*1Wanmha6h$Xw9i|RKqVQlans#9m;ZGty%Y+v^Kx&SYg11kSvEb zdZyuhTiFKiTXPPwqF$Fv%KniLA{jI3ELhFf3mnIh9f%K$pg<-R~Rdv zm7aXQpm(9!vqQNv93e3u3~yjf){iek?5YEHQZCXh4qjf-4jZ~U7<8nn^}{YV$0N#z zvgR}v7a(r33e?%9rb;(>$-qL!oCJ*51U}Rvx%SA?I;knX<&Nb?*L%tjjojK7W=gnA zt|*py;-MJ!U!5Ydw}j0lJ$|wBxmIk>?%+Ax^>k@L_jAn z26ue)CF|=t>>Cg-vmydBZ{;c8@{zgN7&yj0mJUHRt#m#Kwk4d|i2L4&tK8`wP-2fB zw`gVeTED>E2m~A%{9K#UgQvXZ?c@pozK!Swm~aQQQ|6t?etG2jKqlt3xRX^qeFjib z^4_>K8eXg|j}bd^R7O;x=QL zErds~?|%$~cx@C_E^A#K-G{u|-{d=#-D`sBv({}_zAG%ogczzec#Z@|(h2I_7prQX zhZYqbE1ax+Is5kEz(Uu3a~iZ&gJ45t2R_KeKPi3~(?_1Lg(oG^1Ny@1m~cg{7ulM^ zsAfy77K(qO*ALdEzED+#J9LHbC7+DV1f}u2*X$jNY`#PFp;C-{;=_c*PVh?M7j19p8|FB`cl4!;}(j7NhFd!GubQ;1UZV>-Zh z&1>vDO!Dni;Z7E_HLu;eKyp1o_-p(nKC04C1IKiV#0ZQB`Cfve8nhLYY8-2^tHuN`sAn zm7A0b-!v>+HPY>K%)y?T556uuq&iJ@4Tdmpnj*yt%9K7jfRb6}>S?An+{=OUOS~uz zZ6GOWujvUr0Ix%&rgLYuTG>gxXO*$grqm|d$6L|y)?W| zT8-#3#>z)|$6$gz^CB-ll}7_sLAEvIkVU6pwGa+zak+T1t&j#3jiD*WSN#?KjptU& zQXV3A>+jir#VK2}&ztm-_rfP}OZX(lJm@kbo%&0^9X z5gB(Dg|rr4apRw?Z%H|T=agoM5>|A$7WfM|W9(yMVah~z^`5V^gZE)s zJLxpV4^lbI%cNn06<1E|{5iz9X1<&NYNsxRj~{|uj8Jrs7w+QKUt_fW;E0c(n4dQH z)Hp_`UlonC7kleo!qJ-3Y>@j9<~nE`%LDOG6Rk+3RCi#V%#2JM0aiF6?itXvneyhdppDq>hb((cL&K1;(gn&(YmgBN+Gk<7<>-zZ>@(V@?x2=s^fJENY z)bA=s;1{2d)91dc$%&okWCUW9_*B`tFT`kHBtGfhWO#>0eMlctr9e z&Cf42g>HmfPy@$8if&YmHdZyv25drJ(>LqKZq<7gx+0X>8Slh&Zuo!s>b?1UK5O_s@WFqBo}-#}Ez&Do6n)l>)c7tbQ7;p6~>bGTut6>I8F@j8l){J+VkhA za2L+A&K(%sl3DU9tZ_%I+CXi!isdM2W{ud%a+_@C+3>IdQl31h-Y)``bpOtUN#-kdIF+|e;z8)y8( zCSz!G!6sQ}sJRXF>+~rJT--);9Bl4t`t0eiaIs_hW_XY;N_gD|Oek=Z=VpY8dF~kQ zy9QAQ7jEFQs~tSr_X_S0uGD35V-v5^fg`J^a!CpQ@^P-tT%`Gp8;RN&0j$sl=%?y2iiGln!)`L1SQWJL)TF;*;XGjs zVU-*&MV~#4FMOLR;1_N>%#OjRuQ!IBzzWx$=JwzE&9eUqaEO48<96gibt7t5 zvg((A>U-VN%m=?ktS(RP_!IMY`nLoBFAgBT9%-F9o&-6PJ0UaR6C?lH-c!;I#lNPY WAlorLVfGg_rYyEk%L`YI$^QkR`ZQYr literal 3568 zcmeHHYg7{07IyC3TaA@Yr8!yZ^wO-9EZ>^eG>uv+llXq2QkMD3%tQgXJ+*vLzAJ&z z@`WPhBN9X!O-WHH2uTr{3^XM~P(TIb-deNP%$hapuK9b{n*HPKv-h|6I_LZLItN-I z=AA68>i>t%EBwMn|G+3mtNa3Jk;~5WORW4doHR2PTep6j!88cV8W?3m>4rt*?N zeP%P9`c{4EAZ>^y%e&!03DnfUOH<(RF^d(r?ayv8y6mdCNd^{Gvu-fe255tEM68Uh z)KxHqEfpzl%Cf_g2V3@Ubu?UNrHeyG>WPmpAf7ZAG_#ft`LtBoW`sJUhcZrO$4TqX zx9m(D$JdckN35sjVoBm5Y&J>!lsj5Vr&z|Z2MRn0P!~uD{LyRD`Mvjv#yeRh^5V%# zz&4f-#S&o3J`^iCXk|_C!v)-1w==_Lz{1N#c5&Jpe)KZZ1=C%yh$k21KXU z8&h1TEw2{pXEeIHoO zw?e-3EXksx7U)>BY%~O-(Pt%>x#zQOBDHTHIX9j? zrSqHK`D)76n{6w)D+2~0VOjf;nbQGCZ@cfEBDFRiMAD|=E9uopv=`MEQN9vo5OQzD zv*4ysl7xf^lSAEa-ECDN_s#afTZyIS{@w>CYq!iZuRMlO!y?8;POGlLhS{5j3i4+A*Bp z#{m3$_QCXA^Fkb*I035eYirmO>vH~7MHccw`OxS*ramH@o6AYOmb4PD#3fxNrb~0{ zl^a93tjUYp=$-bZ(wePsgJ9&d4r?Sk-3~=OMMKq>W z*t(e}@HQ3-!V6yXU>QG>h1iQTU+=9mu3@krR_MtUA$mKpJ;pt6Dh)z+I~k-Fu2_yI zR0j@WD`Rl#>x0`6ItHQzyr*mI71b6QX_>jhNjj(*y(NZ7S-GJbD4lFo47tz>sq9IU z-uZkpL)Yz0;Q@VfnGy5^@>YwBs=F^@lGr?}YU{0;7hW*-Hwz3x$q?nb&5ie0WT_pv zqy5Bpc@74y>$)>Ko4;W@V^!|R#s(J=;ujnFJK&~oeIcSk!x5W~q+GK*a?7%a2rySPiw`G1hBKr_L#=+<_UX{FM$BI*OJK;Lg5)e?`2L z$U`_j&<*i$PeTgwWy28(WTmxRKD~-npAMK_Ejf~0Ua$+Reu8%DK@acgo3gO z^u2r79cQGzAvF~?Ybz>6E}8ONK#2q-Wpy0qti03A2XQVWGDA(O)rJEqyJk-8i)rb~!`Nc3jOrSRR7qZ4q0_&{I=h$8$hg z%N72m@C%6}iDko#(yO&BqXYO-Q&cwSqHB%Usq$kY=aDn*^giFyhoTFe~Q|t z8d%&Kh=lr&`iZW+VbK7d;7AiUJWBGOvW7`lZ_hNb?^xiryN5G2Ph5tn>N?4a)gB6- z@JXaO4xq8-(mG|p5?xX$uo_w`CSvRJkO}_1j>7{i4`~?73^<<@2cMZbeP)?L=3fF% zfy1@k+%0Vf-@(6Hv{V$pQV{P=be`LX0o|_f#wqRk30i4??6rk)e}5gF#VLyF`kFG|ha&;mDO!K5F6>hs~E?WH6^5^`tRP zM0wR9LDS3VP>KOl;T%`w?b41A*2g4Qp^|^-e+{nHLlXK%(iDP0=i_UEN>;bvMD`5> z?Nhtl(0&J)a`DSg=RfDHAri3UHpJ5YtJUa`=va+A4Zy~pG@LZ;yk}Gs*7peQNqT*m z)pV$Q$cq2r0@SZZv-@dQ{nBxMnI7}rOE9wcz;o|#$TE1q`b;7$j+8O-joJ|3;U;*Q zsR_p17aP4D)s+vdXWN=7{R%}BTjIF{wme~bk9ZA@f4SbpGY~#8St#f%MgP@g8u*))so== From 0a0298534db203578516f2cd5e45ceac95753788 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 20 Mar 2015 15:09:05 -0400 Subject: [PATCH 190/502] add fix_latin_ligatures, rename fix_encoding, rm deprecated Rearranging fixes.py for v4, including removing the deprecated `fix_unsafe_private_use` and renaming `fix_text_encoding` to the simpler `fix_encoding`. Information about ligatures has been added to chardata.py. --- ftfy/chardata.py | 19 +++++++++++++++++ ftfy/fixes.py | 54 ++++++++++++++++++------------------------------ 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index e853ed3e..0d875fea 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -79,3 +79,22 @@ def chars_to_classes(string): # Map whitespace control characters to themselves. for char in '\t\n\f\r': del CONTROL_CHARS[ord(char)] + +# A translate mapping that breaks ligatures made of Latin letters. While +# ligatures may be important to the representation of other languages, in +# Latin letters they tend to represent a copy/paste error. +# +# Ligatures may also be separated by NFKC normalization, but that is sometimes +# more normalization than you want. +LIGATURES = { + ord('IJ'): 'IJ', + ord('ij'): 'ij', + ord('ff'): 'ff', + ord('fi'): 'fi', + ord('fl'): 'fl', + ord('ffi'): 'ffi', + ord('ffl'): 'ffl', + ord('ſt'): 'ſt', + ord('st'): 'st' +} + diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 8b78eaad..3cb9fb3b 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals from ftfy.chardata import (possible_encoding, - CHARMAP_ENCODINGS, CONTROL_CHARS) + CHARMAP_ENCODINGS, CONTROL_CHARS, LIGATURES) from ftfy.badness import text_cost from ftfy.compatibility import htmlentitydefs, unichr, UNSAFE_PRIVATE_USE_RE import re @@ -36,7 +36,7 @@ """ % sys.version_info[0] -def fix_text_encoding(text): +def fix_encoding(text): r""" Fix text with incorrectly-decoded garbage ("mojibake") whenever possible. @@ -111,6 +111,12 @@ def fix_text_encoding(text): return text +def fix_text_encoding(text): + warnings.warn('fix_text_encoding is now known as fix_encoding', + DeprecationWarning) + return fix_encoding(text) + + def fix_encoding_and_explain(text): """ Re-decodes text that has been decoded incorrectly, and also return a @@ -317,6 +323,10 @@ def uncurl_quotes(text): return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text)) +def fix_latin_ligatures(text): + return text.translate(LIGATURES) + + def fix_line_breaks(text): r""" Convert all line breaks to Unix style. @@ -386,7 +396,7 @@ def fix_surrogates(text): """ Replace 16-bit surrogate codepoints with the characters they represent (when properly paired), or with \ufffd otherwise. - + >>> high_surrogate = unichr(0xd83d) >>> low_surrogate = unichr(0xdca9) >>> print(fix_surrogates(high_surrogate + low_surrogate)) @@ -407,7 +417,7 @@ def fix_surrogates(text): def remove_control_chars(text): """ - Remove all control characters except for the important ones. + Remove all ASCII control characters except for the important ones. This removes characters in these ranges: @@ -422,13 +432,17 @@ def remove_control_chars(text): - LF (U+000A) - FF (U+000C) - CR (U+000D) + + Feel free to object that FF isn't "commonly" used for formatting. I've at + least seen it used. """ return text.translate(CONTROL_CHARS) def remove_bom(text): r""" - Remove a left-over byte-order mark. + Remove a byte-order mark that was accidentally decoded as if it were part + of the text. >>> print(remove_bom("\ufeffWhere do you want to go today?")) Where do you want to go today? @@ -436,34 +450,6 @@ def remove_bom(text): return text.lstrip(unichr(0xfeff)) -def remove_unsafe_private_use(text): - r""" - This function is deprecated, because the Python bug it works around has - been fixed for a long time. It only affects old versions of Python that - are also insecure in more serious ways. - - Certain string operations would crash Python < 3.3.3 and < 2.7.6 with a - SystemError: http://bugs.python.org/issue18183 - - ftfy's solution was to remove all characters from Supplementary Private - Use Area B, using a regex that is known not to crash given those - characters. In retrospect, it would have been a better idea to indicate - that they were replaced, using the Unicode replacement character \ufffd. - - >>> print(remove_unsafe_private_use('\U0001F4A9\U00100000')) - 💩 - - ftfy no longer applies this fix by default, and it will be removed in - ftfy 3.5. - """ - warnings.warn( - "remove_unsafe_private_use is deprecated and will be removed in " - "ftfy 3.5", - DeprecationWarning - ) - return UNSAFE_PRIVATE_USE_RE.sub('', text) - - # Define a regex to match valid escape sequences in Python string literals. ESCAPE_SEQUENCE_RE = re.compile(r''' ( \\U........ # 8-digit hex escapes @@ -495,7 +481,7 @@ def decode_escapes(text): escapes and literal Unicode -- you're looking at one right now -- the "unicode-escape" codec doesn't work on literal Unicode. (See http://stackoverflow.com/a/24519338/773754 for more details.) - + Instead, this function searches for just the parts of a string that represent escape sequences, and decodes them, leaving the rest alone. All valid escape sequences are made of ASCII characters, and this allows From cfc65354c0a2f3c35ff2660c3943ca0bd8b32e23 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 20 Mar 2015 15:21:10 -0400 Subject: [PATCH 191/502] change default normalization to NFC --- ftfy/__init__.py | 113 ++++++++++++++++++++------------------ setup.py | 19 +++++++ tests/test_real_tweets.py | 12 ++-- 3 files changed, 86 insertions(+), 58 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index c3fcbd90..b82b674c 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -20,11 +20,11 @@ def fix_text(text, - remove_unsafe_private_use=False, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, - normalization='NFKC', + normalization='NFC', + fix_latin_ligatures=True, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, @@ -40,7 +40,8 @@ def fix_text(text, >>> print(fix_text('ünicode')) ünicode - >>> print(fix_text('Broken text… it’s flubberific!')) + >>> print(fix_text('Broken text… it’s flubberific!'), + ... normalization='NFKC') Broken text... it's flubberific! >>> print(fix_text('HTML entities <3')) @@ -50,7 +51,7 @@ def fix_text(text, HTML entities <3 >>> print(fix_text('\001\033[36;44mI’m blue, da ba dee da ba ' - ... 'doo…\033[0m')) + ... 'doo…\033[0m', normalization='NFKC')) I'm blue, da ba dee da ba doo... >>> # This example string starts with a byte-order mark, even if @@ -80,20 +81,26 @@ def fix_text(text, reasonably fixable. See `fix_text_encoding` for details. - If `normalization` is not None, apply the specified form of Unicode normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. - The default, 'NFKC', applies the following relevant transformations: - - - C: Combine characters and diacritics that are written using separate - code points, such as converting "e" plus an acute accent modifier - into "é", or converting "ka" (か) plus a dakuten into the - single character "ga" (が). - - K: Replace characters that are functionally equivalent with the most - common form. For example, half-width katakana will be replaced with - full-width versions, full-width Roman characters will be replaced with - ASCII characters, ellipsis characters will be replaced with three - periods, and the ligature 'fl' will be replaced with 'fl'. + + - The default normalization, NFC, combines characters and diacritics that + are written using separate code points, such as converting "e" plus an + acute accent modifier into "é", or converting "ka" (か) plus a dakuten + into the single character "ga" (が). Unicode can be converted to NFC + form without any change in its meaning. + + - If you ask for NFKC normalization, it will apply additional + normalizations that can change the meanings of characters. For example, + ellipsis characters will be replaced with three periods, all ligatures + will be replaced with the individual characters that make them up, + and characters that differ in font style will be converted to the same + character. - If `uncurl_quotes` is True, replace various curly quotation marks with plain-ASCII straight quotes. + - If `fix_latin_ligatures` is True, then ligatures made of Latin letters, + such as `fi`, will be separated into individual letters. These ligatures are + usually not meaningful outside of font rendering, and often represent + copy-and-paste errors. - If `fix_line_breaks` is true, convert all line breaks to Unix style (CRLF and CR line breaks become LF line breaks). - If `fix_surrogates` is true, ensure that there are no UTF-16 surrogates @@ -107,12 +114,6 @@ def fix_text(text, - If anything was changed, repeat all the steps, so that the function is idempotent. "&amp;" will become "&", for example, not "&". - There is one deprecated option: - - - If `remove_unsafe_private_use` is True, it issues a DeprecationWarning - and applies a workaround for an old Python bug. This option will go away - in ftfy 3.5. - `fix_text` will work one line at a time, with the possibility that some lines are in different encodings. When it encounters lines longer than `max_decode_length`, it will not run the `fix_encoding` step, to avoid @@ -144,12 +145,12 @@ def fix_text(text, out.append( fix_text_segment( substring, - remove_unsafe_private_use=remove_unsafe_private_use, fix_entities=fix_entities, remove_terminal_escapes=remove_terminal_escapes, fix_encoding=fix_encoding_this_time, normalization=normalization, uncurl_quotes=uncurl_quotes, + fix_latin_ligatures=fix_latin_ligatures, fix_line_breaks=fix_line_breaks, fix_surrogates=fix_surrogates, remove_control_chars=remove_control_chars, @@ -163,12 +164,17 @@ def fix_text(text, ftfy = fix_text +def fix_encoding(text): + return fixes.fix_encoding(text) + + def fix_file(input_file, - remove_unsafe_private_use=False, + encoding=None, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, - normalization='NFKC', + normalization='NFC', + fix_latin_ligatures=True, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, @@ -187,16 +193,19 @@ def fix_file(input_file, entities = fix_entities for line in input_file: if isinstance(line, bytes): - line, encoding = guess_bytes(line) + if encoding is None: + line, encoding = guess_bytes(line) + else: + line = line.decode(encoding) if fix_entities == 'auto' and '<' in line and '>' in line: entities = False yield fix_text_segment( line, - remove_unsafe_private_use=remove_unsafe_private_use, fix_entities=entities, remove_terminal_escapes=remove_terminal_escapes, fix_encoding=fix_encoding, normalization=normalization, + fix_latin_ligatures=fix_latin_ligatures, uncurl_quotes=uncurl_quotes, fix_line_breaks=fix_line_breaks, fix_surrogates=fix_surrogates, @@ -206,11 +215,11 @@ def fix_file(input_file, def fix_text_segment(text, - remove_unsafe_private_use=False, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, - normalization='NFKC', + normalization='NFC', + fix_latin_ligatures=True, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, @@ -230,8 +239,6 @@ def fix_text_segment(text, fix_entities = False while True: origtext = text - if remove_unsafe_private_use: - text = fixes.remove_unsafe_private_use(text) if fix_entities: text = fixes.unescape_html(text) if remove_terminal_escapes: @@ -240,6 +247,8 @@ def fix_text_segment(text, text = fixes.fix_text_encoding(text) if normalization is not None: text = unicodedata.normalize(normalization, text) + if fix_latin_ligatures: + text = fixes.fix_latin_ligatures(text) if uncurl_quotes: text = fixes.uncurl_quotes(text) if fix_line_breaks: @@ -256,20 +265,31 @@ def fix_text_segment(text, def guess_bytes(bstring): """ - If you have some bytes in an unknown encoding, here's a reasonable - strategy for decoding them, by trying a few common encodings that - can be distinguished from each other. - - This is not a magic bullet. If the bytes are coming from some MySQL - database with the "character set" set to ISO Elbonian, this won't figure - it out. Perhaps more relevantly, this currently doesn't try East Asian - encodings. - - The encodings we try are: + NOTE: Using `guess_bytes` is not the recommended way of using ftfy. It's + not the primary purpose of ftfy. + + In the unfortunate situation that you have some bytes in an unknown + encoding, ftfy can guess a reasonable strategy for decoding them, by trying + a few common encodings that can be distinguished from each other. + + Unlike the rest of ftfy, this may not be accurate, and it may *create* + Unicode problems instead of solving them! + + It doesn't try East Asian encodings at all, and if you have East Asian text + that you don't know how to decode, you are somewhat out of luck. East + Asian encodings require some serious statistics to distinguish from each + other, so we can't support them without decreasing the accuracy of ftfy. + + If you don't know which encoding you have at all, I recommend + trying the 'chardet' module, but being appropriately skeptical about its + results. + + The encodings we try here are: - UTF-16 with a byte order mark, because a UTF-16 byte order mark looks like nothing else - - UTF-8, because it's the global de facto standard + - UTF-8, because it's the global standard, which has been used by a + majority of the Web since 2008 - "utf-8-variants", because it's what people actually implement when they think they're doing UTF-8 - MacRoman, because Microsoft Office thinks it's still a thing, and it @@ -351,14 +371,3 @@ def explain_unicode(text): category=unicodedata.category(char), name=unicodedata.name(char, '') )) - - -def fix_bad_encoding(text): - """ - Kept for compatibility with previous versions of ftfy. - """ - warnings.warn( - 'fix_bad_encoding is now known as fix_text_encoding', - DeprecationWarning - ) - return fix_text_encoding(text) diff --git a/setup.py b/setup.py index 876d3501..b8674e45 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ from setuptools import setup +import sys setup( name="ftfy", @@ -31,3 +32,21 @@ ] } ) + + +if sys.version_info[0] < 3: + sys.stderr.write(""" + Heads up! You're now running ftfy 4.x on Python 2.x. This is fine, but + here's an advance warning that you won't be able to upgrade ftfy + forever without upgrading Python. + + The next major version of ftfy, version 5.0, will probably only work + on Python 3, making it easier to develop and ensure its consistency. + + It's fine if you're happy with Python 2 and ftfy 4. Save yourself a + headache later and be sure to pin a version in your dependencies. + Instead of just asking for 'ftfy', you should ask for: + + 'ftfy >= 4, < 5' + """) + diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index c1a748ac..e5b467ed 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -10,10 +10,10 @@ ("Le Schtroumpf Docteur conseille g√¢teaux et baies schtroumpfantes pour un r√©gime √©quilibr√©.", "Le Schtroumpf Docteur conseille gâteaux et baies schtroumpfantes pour un régime équilibré."), ("✔ No problems", "✔ No problems"), - ('4288×…', '4288×...'), - ('RETWEET SE VOCÊ…', 'RETWEET SE VOCÊ...'), - ('PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…', 'PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ...'), - ('TEM QUE SEGUIR, SDV SÓ…', 'TEM QUE SEGUIR, SDV SÓ...'), + ('4288×…', '4288×…'), + ('RETWEET SE VOCÊ…', 'RETWEET SE VOCÊ…'), + ('PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…', 'PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…'), + ('TEM QUE SEGUIR, SDV SÓ…', 'TEM QUE SEGUIR, SDV SÓ…'), ('Join ZZAJÉ’s Official Fan List and receive news, events, and more!', "Join ZZAJÉ's Official Fan List and receive news, events, and more!"), ('L’épisode 8 est trop fou ouahh', "L'épisode 8 est trop fou ouahh"), ("РґРѕСЂРѕРіРµ Из-РїРѕРґ #футбол", @@ -26,7 +26,7 @@ ("Beta Haber: Hırsızı Büyü Korkuttu", "Beta Haber: Hırsızı Büyü Korkuttu"), ("Ôôô VIDA MINHA", "Ôôô VIDA MINHA"), - ('[x]\xa0©', '[x] ©'), + ('[x]\xa0©', '[x]\xa0©'), ('2012—∞', '2012—∞'), ('Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.', 'Con il corpo e lo spirito ammaccato, è come se nel cuore avessi un vetro conficcato.'), @@ -34,7 +34,7 @@ ('Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New “ Romance”.', 'Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New " Romance".'), ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí \u0301 \u0301"), - ('HUHLL Õ…', 'HUHLL Õ...'), + ('HUHLL Õ…', 'HUHLL Õ…'), ('Iggy Pop (né Jim Osterberg)', 'Iggy Pop (né Jim Osterberg)'), ## Current false positives: From 2700d622b358542d3413ba1a9e8aad23c7258fc6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 26 Mar 2015 18:36:27 -0400 Subject: [PATCH 192/502] change tests to not assume NFKC normalization --- ftfy/__init__.py | 4 ++-- tests/test_real_tweets.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index b82b674c..783b6050 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -40,8 +40,8 @@ def fix_text(text, >>> print(fix_text('ünicode')) ünicode - >>> print(fix_text('Broken text… it’s flubberific!'), - ... normalization='NFKC') + >>> print(fix_text('Broken text… it’s flubberific!', + ... normalization='NFKC')) Broken text... it's flubberific! >>> print(fix_text('HTML entities <3')) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index e5b467ed..2e252e47 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -29,11 +29,11 @@ ('[x]\xa0©', '[x]\xa0©'), ('2012—∞', '2012—∞'), ('Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.', - 'Con il corpo e lo spirito ammaccato, è come se nel cuore avessi un vetro conficcato.'), + 'Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.'), ('приятности. РІСњВ¤', 'приятности. ❤'), ('Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New “ Romance”.', 'Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New " Romance".'), - ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí \u0301 \u0301"), + ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí´´"), ('HUHLL Õ…', 'HUHLL Õ…'), ('Iggy Pop (né Jim Osterberg)', 'Iggy Pop (né Jim Osterberg)'), From f73ad8bdc52687aaaef70489521107fdbc198a63 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 26 Mar 2015 18:51:25 -0400 Subject: [PATCH 193/502] fix some docstrings --- ftfy/fixes.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 3cb9fb3b..6ef15211 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -64,17 +64,17 @@ def fix_encoding(text): ftfy decodes text that looks like it was decoded incorrectly. It leaves alone text that doesn't. - >>> print(fix_text_encoding('único')) + >>> print(fix_encoding('único')) único - >>> print(fix_text_encoding('This text is fine already :þ')) + >>> print(fix_encoding('This text is fine already :þ')) This text is fine already :þ Because these characters often come from Microsoft products, we allow for the possibility that we get not just Unicode characters 128-255, but also Windows's conflicting idea of what characters 128-160 are. - >>> print(fix_text_encoding('This — should be an em dash')) + >>> print(fix_encoding('This — should be an em dash')) This — should be an em dash We might have to deal with both Windows characters and raw control @@ -82,26 +82,26 @@ def fix_encoding(text): 0x81 that have no mapping in Windows. This is a string that Python's standard `.encode` and `.decode` methods cannot correct. - >>> print(fix_text_encoding('This text is sad .â\x81”.')) + >>> print(fix_encoding('This text is sad .â\x81”.')) This text is sad .⁔. However, it has safeguards against fixing sequences of letters and punctuation that can occur in valid text: - >>> print(fix_text_encoding('not such a fan of Charlotte Brontë…”')) + >>> print(fix_encoding('not such a fan of Charlotte Brontë…”')) not such a fan of Charlotte Brontë…” Cases of genuine ambiguity can sometimes be addressed by finding other characters that are not double-encoded, and expecting the encoding to be consistent: - >>> print(fix_text_encoding('AHÅ™, the new sofa from IKEA®')) + >>> print(fix_encoding('AHÅ™, the new sofa from IKEA®')) AHÅ™, the new sofa from IKEA® Finally, we handle the case where the text is in a single-byte encoding that was intended as Windows-1252 all along but read as Latin-1: - >>> print(fix_text_encoding('This text was never UTF-8 at all\x85')) + >>> print(fix_encoding('This text was never UTF-8 at all\x85')) This text was never UTF-8 at all… The best version of the text is found using @@ -158,8 +158,7 @@ def fix_one_step_and_explain(text): """ Performs a single step of re-decoding text that's been decoded incorrectly. - Returns the decoded text, plus a "plan" for how to reproduce what it - did. + Returns the decoded text, plus a "plan" for how to reproduce what it did. """ if isinstance(text, bytes): raise UnicodeError(BYTES_ERROR_TEXT) @@ -224,9 +223,9 @@ def fix_one_step_and_explain(text): # The cases that remain are mixups between two different single-byte # encodings, and not the common case of Latin-1 vs. Windows-1252. # - # Those cases are somewhat rare, and impossible to solve without false - # positives. If you're in one of these situations, you should try using - # the `ftfy.guess_bytes` function. + # Those cases are somewhat rare, and may be unsolvable without adding false + # positives. If you're in one of these situations, you should try using the + # `ftfy.guess_bytes` function. # Return the text unchanged; the plan is empty. return text, [] @@ -324,6 +323,19 @@ def uncurl_quotes(text): def fix_latin_ligatures(text): + """ + Replace single-character ligatures of Latin letters, such as 'fi', with the + characters that they contain, as in 'fi'. Latin ligatures are usually not + intended in text, and are a result of copy-and-paste glitches. + + We leave ligatures in other scripts alone to be safe. They may be fully + intended for the text to render correctly, and removing them may lose + information. If you want to take apart nearly all ligatures, use NFKC + normalization. + + >>> print(fix_latin_ligatures("fluffiest")) + fluffiest + """ return text.translate(LIGATURES) From c18b11ec19d438e28f083dad293f331454b28029 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 26 Mar 2015 19:18:22 -0400 Subject: [PATCH 194/502] add fix_character_width --- ftfy/__init__.py | 19 ++++++++++++++----- ftfy/chardata.py | 30 ++++++++++++++++++++++++------ ftfy/fixes.py | 24 ++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 783b6050..9e305e5e 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -25,6 +25,7 @@ def fix_text(text, fix_encoding=True, normalization='NFC', fix_latin_ligatures=True, + fix_character_width=False, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, @@ -60,6 +61,9 @@ def fix_text(text, Party like it's 1999! + >>> print(fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ', fix_character_width=True)) + LOUD NOISES + >>> len(fix_text('fi' * 100000)) 200000 @@ -78,7 +82,7 @@ def fix_text(text, in different colors. - If `fix_encoding` is True, look for common mistakes that come from encoding or decoding Unicode text incorrectly, and fix them if they are - reasonably fixable. See `fix_text_encoding` for details. + reasonably fixable. See `fixes.fix_encoding` for details. - If `normalization` is not None, apply the specified form of Unicode normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. @@ -101,6 +105,8 @@ def fix_text(text, such as `fi`, will be separated into individual letters. These ligatures are usually not meaningful outside of font rendering, and often represent copy-and-paste errors. + - If `fix_character_width` is True, half-width and full-width characters + will be replaced by their standard-width form. - If `fix_line_breaks` is true, convert all line breaks to Unix style (CRLF and CR line breaks become LF line breaks). - If `fix_surrogates` is true, ensure that there are no UTF-16 surrogates @@ -151,6 +157,7 @@ def fix_text(text, normalization=normalization, uncurl_quotes=uncurl_quotes, fix_latin_ligatures=fix_latin_ligatures, + fix_character_width=fix_character_width, fix_line_breaks=fix_line_breaks, fix_surrogates=fix_surrogates, remove_control_chars=remove_control_chars, @@ -162,10 +169,7 @@ def fix_text(text, return ''.join(out) ftfy = fix_text - - -def fix_encoding(text): - return fixes.fix_encoding(text) +fix_encoding = fixes.fix_encoding def fix_file(input_file, @@ -175,6 +179,7 @@ def fix_file(input_file, fix_encoding=True, normalization='NFC', fix_latin_ligatures=True, + fix_character_width=False, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, @@ -206,6 +211,7 @@ def fix_file(input_file, fix_encoding=fix_encoding, normalization=normalization, fix_latin_ligatures=fix_latin_ligatures, + fix_character_width=fix_character_width, uncurl_quotes=uncurl_quotes, fix_line_breaks=fix_line_breaks, fix_surrogates=fix_surrogates, @@ -220,6 +226,7 @@ def fix_text_segment(text, fix_encoding=True, normalization='NFC', fix_latin_ligatures=True, + fix_character_width=False, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, @@ -249,6 +256,8 @@ def fix_text_segment(text, text = unicodedata.normalize(normalization, text) if fix_latin_ligatures: text = fixes.fix_latin_ligatures(text) + if fix_character_width: + text = fixes.fix_character_width(text) if uncurl_quotes: text = fixes.uncurl_quotes(text) if fix_line_breaks: diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 0d875fea..e5e2a170 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -7,6 +7,7 @@ from __future__ import unicode_literals import re import zlib +import unicodedata from pkg_resources import resource_string from ftfy.compatibility import unichr @@ -72,13 +73,17 @@ def chars_to_classes(string): # A translate mapping that will strip all C0 control characters except # those that represent whitespace. -CONTROL_CHARS = {} -for i in range(32): - CONTROL_CHARS[i] = None +def _build_control_char_mapping(): + control_chars = {} + for i in range(32): + control_chars[i] = None + + # Map whitespace control characters to themselves. + for char in '\t\n\f\r': + del control_chars[ord(char)] + return control_chars +CONTROL_CHARS = _build_control_char_mapping() -# Map whitespace control characters to themselves. -for char in '\t\n\f\r': - del CONTROL_CHARS[ord(char)] # A translate mapping that breaks ligatures made of Latin letters. While # ligatures may be important to the representation of other languages, in @@ -98,3 +103,16 @@ def chars_to_classes(string): ord('st'): 'st' } + +# A translate mapping that replaces halfwidth and fullwidth forms with their +# standard-width forms. +def _build_width_map(): + width_map = {0x3000: ' '} + for i in range(0xff01, 0xfff0): + char = unichr(i) + alternate = unicodedata.normalize('NFKC', char) + if alternate != char: + width_map[i] = alternate + return width_map +WIDTH_MAP = _build_width_map() + diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 6ef15211..e451cb9d 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -5,8 +5,8 @@ """ from __future__ import unicode_literals -from ftfy.chardata import (possible_encoding, - CHARMAP_ENCODINGS, CONTROL_CHARS, LIGATURES) +from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS, + CONTROL_CHARS, LIGATURES, WIDTH_MAP) from ftfy.badness import text_cost from ftfy.compatibility import htmlentitydefs, unichr, UNSAFE_PRIVATE_USE_RE import re @@ -339,6 +339,26 @@ def fix_latin_ligatures(text): return text.translate(LIGATURES) +def fix_character_width(text): + """ + The ASCII characters, katakana, and Hangul characters have alternate + "halfwidth" or "fullwidth" forms that help text line up in a grid. + + If you don't need these width properties, you probably want to replace + these characters with their standard form, which is what this function + does. + + Note that this replaces the ideographic space, U+3000, with the ASCII + space, U+20. + + >>> print(fix_character_width("ＬＯＵＤ　ＮＯＩＳＥＳ")) + LOUD NOISES + >>> print(fix_character_width("Ｕﾀｰﾝ")) # this means "U-turn" + Uターン + """ + return text.translate(WIDTH_MAP) + + def fix_line_breaks(text): r""" Convert all line breaks to Unix style. From 3c8795fca06f8362ef7a54b8f30bd4a985883977 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 27 Mar 2015 16:36:59 -0400 Subject: [PATCH 195/502] `fix_character_width` is on by default for consistency This lets us continue documenting that all fixes will be on by default. --- ftfy/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 9e305e5e..c8eabd9c 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -25,7 +25,7 @@ def fix_text(text, fix_encoding=True, normalization='NFC', fix_latin_ligatures=True, - fix_character_width=False, + fix_character_width=True, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, @@ -61,7 +61,7 @@ def fix_text(text, Party like it's 1999! - >>> print(fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ', fix_character_width=True)) + >>> print(fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ')) LOUD NOISES >>> len(fix_text('fi' * 100000)) @@ -179,7 +179,7 @@ def fix_file(input_file, fix_encoding=True, normalization='NFC', fix_latin_ligatures=True, - fix_character_width=False, + fix_character_width=True, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, @@ -226,7 +226,7 @@ def fix_text_segment(text, fix_encoding=True, normalization='NFC', fix_latin_ligatures=True, - fix_character_width=False, + fix_character_width=True, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, From a000af911fa0d42bcb178f1fc2c84a24ce82ce52 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 27 Mar 2015 16:51:33 -0400 Subject: [PATCH 196/502] Some updates to the main function - Perform normalization last (it makes more sense than in the middle) - Decode up to 1 million codepoints per step - Revisions to docstrings --- ftfy/__init__.py | 90 ++++++++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index c8eabd9c..65b1ef60 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -23,7 +23,6 @@ def fix_text(text, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, - normalization='NFC', fix_latin_ligatures=True, fix_character_width=True, uncurl_quotes=True, @@ -31,10 +30,11 @@ def fix_text(text, fix_surrogates=True, remove_control_chars=True, remove_bom=True, - max_decode_length=2**16): + normalization='NFC', + max_decode_length=10**6): r""" - Given Unicode text as input, make its representation consistent and - possibly less broken. + Given Unicode text as input, fix inconsistencies and glitches in it, + such as mojibake. Let's start with some examples: @@ -77,57 +77,72 @@ def fix_text(text, entities, but don't do so in text where you have seen a pair of actual angle brackets (that's probably actually HTML and you shouldn't mess with the entities). + - If `remove_terminal_escapes` is True, remove sequences of bytes that are instructions for Unix terminals, such as the codes that make text appear in different colors. + - If `fix_encoding` is True, look for common mistakes that come from encoding or decoding Unicode text incorrectly, and fix them if they are reasonably fixable. See `fixes.fix_encoding` for details. - - If `normalization` is not None, apply the specified form of Unicode - normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. - - - The default normalization, NFC, combines characters and diacritics that - are written using separate code points, such as converting "e" plus an - acute accent modifier into "é", or converting "ka" (か) plus a dakuten - into the single character "ga" (が). Unicode can be converted to NFC - form without any change in its meaning. - - - If you ask for NFKC normalization, it will apply additional - normalizations that can change the meanings of characters. For example, - ellipsis characters will be replaced with three periods, all ligatures - will be replaced with the individual characters that make them up, - and characters that differ in font style will be converted to the same - character. - If `uncurl_quotes` is True, replace various curly quotation marks with plain-ASCII straight quotes. + - If `fix_latin_ligatures` is True, then ligatures made of Latin letters, such as `fi`, will be separated into individual letters. These ligatures are usually not meaningful outside of font rendering, and often represent copy-and-paste errors. + - If `fix_character_width` is True, half-width and full-width characters will be replaced by their standard-width form. + - If `fix_line_breaks` is true, convert all line breaks to Unix style (CRLF and CR line breaks become LF line breaks). + - If `fix_surrogates` is true, ensure that there are no UTF-16 surrogates in the resulting string, by converting them to the correct characters when they're appropriately paired, or replacing them with \ufffd otherwise. + - If `fix_control_characters` is true, remove all C0 control characters except the common useful ones: TAB, CR, LF, and FF. (CR characters may have already been removed by the `fix_line_breaks` step.) + - If `remove_bom` is True, remove the Byte-Order Mark if it exists. + (This is a decoded Unicode string. It doesn't have a "byte order".) + + - If `normalization` is not None, apply the specified form of Unicode + normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. + + - The default normalization, NFC, combines characters and diacritics that + are written using separate code points, such as converting "e" plus an + acute accent modifier into "é", or converting "ka" (か) plus a dakuten + into the single character "ga" (が). Unicode can be converted to NFC + form without any change in its meaning. + + - If you ask for NFKC normalization, it will apply additional + normalizations that can change the meanings of characters. For example, + ellipsis characters will be replaced with three periods, all ligatures + will be replaced with the individual characters that make them up, + and characters that differ in font style will be converted to the same + character. + - If anything was changed, repeat all the steps, so that the function is idempotent. "&amp;" will become "&", for example, not "&". `fix_text` will work one line at a time, with the possibility that some - lines are in different encodings. When it encounters lines longer than - `max_decode_length`, it will not run the `fix_encoding` step, to avoid + lines are in different encodings, allowing it to fix text that has been + concatenated together from different sources. + + When it encounters lines longer than `max_decode_length` (1 million + codepoints by default), it will not run the `fix_encoding` step, to avoid unbounded slowdowns. - If you are certain your entire text is in the same encoding (though that - encoding is possibly flawed), and do not mind performing operations on - the whole text at once, use `fix_text_segment`. + If you're certain that any decoding errors in the text would have affected + the entire text in the same way, and you don't mind operations that scale + with the length of the text, you can use `fix_text_segment` directly to + fix the whole string in one batch. """ if isinstance(text, bytes): raise UnicodeError(fixes.BYTES_ERROR_TEXT) @@ -154,14 +169,14 @@ def fix_text(text, fix_entities=fix_entities, remove_terminal_escapes=remove_terminal_escapes, fix_encoding=fix_encoding_this_time, - normalization=normalization, uncurl_quotes=uncurl_quotes, fix_latin_ligatures=fix_latin_ligatures, fix_character_width=fix_character_width, fix_line_breaks=fix_line_breaks, fix_surrogates=fix_surrogates, remove_control_chars=remove_control_chars, - remove_bom=remove_bom + remove_bom=remove_bom, + normalization=normalization ) ) pos = textbreak @@ -177,14 +192,15 @@ def fix_file(input_file, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, - normalization='NFC', fix_latin_ligatures=True, fix_character_width=True, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, remove_control_chars=True, - remove_bom=True): + remove_bom=True, + normalization='NFC' + ): """ Fix text that is found in a file. @@ -209,14 +225,14 @@ def fix_file(input_file, fix_entities=entities, remove_terminal_escapes=remove_terminal_escapes, fix_encoding=fix_encoding, - normalization=normalization, fix_latin_ligatures=fix_latin_ligatures, fix_character_width=fix_character_width, uncurl_quotes=uncurl_quotes, fix_line_breaks=fix_line_breaks, fix_surrogates=fix_surrogates, remove_control_chars=remove_control_chars, - remove_bom=remove_bom + remove_bom=remove_bom, + normalization=normalization ) @@ -224,18 +240,18 @@ def fix_text_segment(text, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, - normalization='NFC', fix_latin_ligatures=True, fix_character_width=True, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, remove_control_chars=True, - remove_bom=True): + remove_bom=True, + normalization='NFC'): """ Apply fixes to text in a single chunk. This could be a line of text within a larger run of `fix_text`, or it could be a larger amount - of text that you are certain is all in the same encoding. + of text that you are certain is in a consistent encoding. See `fix_text` for a description of the parameters. """ @@ -252,8 +268,6 @@ def fix_text_segment(text, text = fixes.remove_terminal_escapes(text) if fix_encoding: text = fixes.fix_text_encoding(text) - if normalization is not None: - text = unicodedata.normalize(normalization, text) if fix_latin_ligatures: text = fixes.fix_latin_ligatures(text) if fix_character_width: @@ -268,6 +282,8 @@ def fix_text_segment(text, text = fixes.remove_control_chars(text) if remove_bom: text = fixes.remove_bom(text) + if normalization is not None: + text = unicodedata.normalize(normalization, text) if text == origtext: return text @@ -283,12 +299,12 @@ def guess_bytes(bstring): Unlike the rest of ftfy, this may not be accurate, and it may *create* Unicode problems instead of solving them! - + It doesn't try East Asian encodings at all, and if you have East Asian text that you don't know how to decode, you are somewhat out of luck. East Asian encodings require some serious statistics to distinguish from each other, so we can't support them without decreasing the accuracy of ftfy. - + If you don't know which encoding you have at all, I recommend trying the 'chardet' module, but being appropriately skeptical about its results. From aa9fc7614f1687e26b77ff31bae3e3957b6d8a8e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 27 Mar 2015 18:29:29 -0400 Subject: [PATCH 197/502] Documentation updates --- README.md | 218 +++++++------------------------ docs/conf.py | 11 +- docs/index.rst | 181 +++++++++++++++++++------ ftfy/bad_codecs/utf8_variants.py | 2 +- ftfy/fixes.py | 32 ++--- 5 files changed, 207 insertions(+), 237 deletions(-) diff --git a/README.md b/README.md index b2d5e611..f91de30a 100644 --- a/README.md +++ b/README.md @@ -35,190 +35,74 @@ page](http://www.luminoso.com/careers.html). ## What it does -ftfy makes Unicode text less broken and more consistent. It works in -Python 2.7, Python 3.2, or later. +`ftfy` fixes Unicode that's broken in various ways. It works in Python 2.7, +Python 3.2, or later. -The most interesting kind of brokenness that this resolves is when someone -has encoded Unicode with one standard and decoded it with a different one. -This often shows up as characters that turn into nonsense sequences: +The goal of `ftfy` is to **take in bad Unicode and output good Unicode**, for use +in your Unicode-aware code. This is different from taking in non-Unicode and +outputting Unicode, which is not a goal of ftfy. It also isn't designed to +protect you from having to write Unicode-aware code. ftfy helps those who help +themselves. -- The word `schön` might appear as `schön`. -- An em dash (`—`) might appear as `—`. -- Text that was meant to be enclosed in quotation marks might end up - instead enclosed in `“` and `â€<9d>` (where `<9d>` is an unprintable - codepoint). +Of course you're better off if your input is decoded properly and has no +glitches. But you often don't have any control over your input; it's someone +else's mistake, but it's your problem now. -This is called "mojibake", and it happens very often to real text. Fortunately, -the nonsense sequences usually contain all the information you need to -reconstruct what character was supposed to be there. +`ftfy` will do everything it can to fix the problem. -Any given text string might have other irritating properties, possibly even -interacting with the erroneous decoding: +## Mojibake -- The text could contain HTML entities such as `&` in place of certain - characters, when you would rather see what the characters actually are. -- For that matter, it could contain instructions for a text terminal to - do something like change colors, but you are not sending the text to a - terminal, so those instructions are just going to look like `^[[30m;` - or something in the middle of the text. -- The text could write words in non-standard ways for display purposes, - such as using the three characters `fl` `o` `p` for the word "flop". - This can happen when you copy text out of a PDF, for example. +The most interesting kind of brokenness that ftfy will fix is when someone has +encoded Unicode with one standard and decoded it with a different one. This +often shows up as characters that turn into nonsense sequences (called +"mojibake"): -Of course you're better off if all the text you take as input is decoded -properly and written in standard ways. But often, your input is something you -have no control over. Somebody else's minor mistake becomes your problem. +- The word ``schön`` might appear as ``schön``. +- An em dash (``—``) might appear as ``—``. +- Text that was meant to be enclosed in quotation marks might end up + instead enclosed in ``“`` and ``â€<9d>``, where ``<9d>`` represents an + unprintable character. -ftfy will do everything it can to fix the problem. +ftfy uses heuristics to detect and undo this kind of mojibake, with a very +low rate of false positives. ## Examples -In these examples, `unicode_literals` are turned on. ftfy always expects -Unicode strings as input. - - >>> from __future__ import unicode_literals - >>> from ftfy import fix_text - - >>> print(fix_text('This — should be an em dash')) - This — should be an em dash - - >>> print(fix_text('ünicode')) - ünicode - - >>> print(fix_text('Broken text… it’s flubberific!')) - Broken text... it's flubberific! - - >>> print(fix_text('HTML entities <3')) - HTML entities <3 - -If any HTML tags appear in your input, ftfy will make sure to leave the HTML -entities alone: - - >>> print(fix_text('HTML entities <3')) - HTML entities <3 - -ftfy repeats its process until it reaches a result that it won't change: - - >>> wtf = '\xc3\xa0\xc2\xb2\xc2\xa0_\xc3\xa0\xc2\xb2\xc2\xa0' - >>> print(fix_text(wtf)) - ಠ_ಠ +`fix_text` is the main function of ftfy. This section is meant to give you a +taste of the things it can do. Please read [the +documentation](http://ftfy.readthedocs.org) for more information on what ftfy +does, and how to configure it for your needs. -## Using ftfy - -The main function, `fix_text`, will run text through a sequence of fixes. If -the text changed, it will run them through again, so that you can be sure -the output ends up in a standard form that will be unchanged by `fix_text`. -All the fixes are on by default, but you can pass options to turn them off. - -- If `fix_entities` is True, replace HTML entities with their equivalent - characters. If it's "auto" (the default), then consider replacing HTML - entities, but don't do so in text where you have seen a pair of actual - angle brackets (that's probably actually HTML and you shouldn't mess - with the entities). -- If `remove_terminal_escapes` is True, remove sequences of bytes that are - instructions for Unix terminals, such as the codes that make text appear - in different colors. -- If `fix_encoding` is True, look for common mistakes that come from - encoding or decoding Unicode text incorrectly, and fix them if they are - reasonably fixable. See `fix_text_encoding` for details. -- If `normalization` is not None, apply the specified form of Unicode - normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. - The default, 'NFKC', applies the following relevant transformations: +```python - - C: Combine characters and diacritics that are written using separate - code points, such as converting "e" plus an acute accent modifier - into "é", or converting "ka" (か) plus a dakuten into the - single character "ga" (が). - - K: Replace characters that are functionally equivalent with the most - common form. For example, half-width katakana will be replaced with - full-width versions, full-width Roman characters will be replaced with - ASCII characters, ellipsis characters will be replaced with three - periods, and the ligature 'fl' will be replaced with 'fl'. +>>> print(fix_text('This text should be in “quotesâ€\x9d.')) +This text should be in "quotes". -- If `uncurl_quotes` is True, replace various curly quotation marks with - plain-ASCII straight quotes. -- If `fix_line_breaks` is true, convert all line breaks to Unix style - (CRLF and CR line breaks become LF line breaks). -- If `fix_control_characters` is true, remove all C0 control characters - except the common useful ones: TAB, CR, LF, and FF. (CR characters - may have already been removed by the `fix_line_breaks` step.) -- If `remove_bom` is True, remove the Byte-Order Mark if it exists. -- If anything was changed, repeat all the steps, so that the function is - idempotent. "&amp;amp;" will become "&", for example, not "&amp;". +>>> print(fix_text('ünicode')) +ünicode +>>> print(fix_text('Broken text… it’s flubberific!', +... normalization='NFKC')) +Broken text... it's flubberific! -### Encodings ftfy can handle +>>> print(fix_text('HTML entities <3')) +HTML entities <3 -ftfy can understand text that was decoded as any of these single-byte -encodings: +>>> print(fix_text('HTML entities in HTML <3')) +HTML entities in HTML <3 -- Latin-1 (ISO-8859-1) -- Windows-1252 (cp1252 -- used in Microsoft products) -- Windows-1251 (cp1251 -- the Russian version of cp1252) -- MacRoman (used on Mac OS 9 and earlier) -- cp437 (used in MS-DOS) +>>> print(fix_text('\001\033[36;44mI’m blue, da ba dee da ba ' +... 'doo…\033[0m', normalization='NFKC')) +I'm blue, da ba dee da ba doo... -when it was actually intended to be decoded as one of these variable-length -encodings: +>>> print(fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ')) +LOUD NOISES -- UTF-8 -- CESU-8 (what some people think is UTF-8) +>>> print(fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ', fix_character_width=False)) +ＬＯＵＤ　ＮＯＩＳＥＳ +``` -It can also understand text that was intended as Windows-1252 but decoded as -Latin-1. That's the very common case where things like smart-quotes and -bullets turn into single weird control characters. - -However, ftfy cannot understand other mixups between single-byte encodings, -because it is extremely difficult to detect which mixup in particular is the -one that happened. - -## Non-Unicode strings - -When first using ftfy, you might be confused to find that you can't give it a -bytestring (the type of object called `str` in Python 2). - -ftfy fixes text. Treating bytestrings as text is exactly the kind of thing that -causes the Unicode problems that ftfy has to fix. So if you don't give it a -Unicode string, ftfy will point you to the [Python Unicode -HOWTO](http://docs.python.org/3/howto/unicode.html). - -Reasonable ways that you might exchange data, such as JSON or XML, already have -perfectly good ways of expressing Unicode strings. Given a Unicode string, ftfy -can apply fixes that are very likely to work without false positives. - -### A note on encoding detection - -If your input is a mess of unmarked bytes, you might want a tool that can just -statistically analyze those bytes and predict what encoding they're in. - -ftfy is not that tool. The `guess_bytes` function will do this in very limited -cases, but to support more encodings from around the world, something more is -needed. - -You may have heard of chardet. Chardet is admirable, but it doesn't completely -do the job either. Its heuristics are designed for multi-byte encodings, such -as UTF-8 and the language-specific encodings used in East Asian languages. It -works badly on single-byte encodings, to the point where it will output wrong -answers with high confidence. - -ftfy's `guess_bytes` doesn't even try the East Asian encodings, so the ideal thing -would combine the simple heuristic of `guess_bytes` with the multibyte character -set detection of `chardet`. This ideal thing doesn't exist yet. - -## Command-line usage - -ftfy installs itself as a command line tool that reads a file and applies -`fix_text` to it. - -This has exactly the problem described above: a file on a disk is made of bytes -in an unspecified encoding. It has to guess the encoding of the bytes in the -file. But if it guesses wrong, it might be able to fix it anyway, because -that's what ftfy does. - -You can type `ftfy FILENAME`, and it will read in FILENAME, guess its encoding, -fix everything that `fix_text` fixes, and write the result to standard out as -UTF-8. ## Who maintains ftfy? @@ -230,11 +114,3 @@ Luminoso provides ftfy as free, open source software under the extremely permissive MIT license. You can report bugs regarding ftfy on GitHub and we'll handle them. - -## More details - -To learn how the pieces of ftfy work: - -- You could just read the code. I recommend it. There are copious comments and - everything has a docstring. -- You can read nicely-formatted documentation at http://ftfy.readthedocs.org. diff --git a/docs/conf.py b/docs/conf.py index 3638da41..9c8fbf27 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ # General information about the project. project = u'ftfy' -copyright = u'2014, Rob Speer' +copyright = u'2015, Rob Speer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '3.3' +version = '4.0' # The full version, including alpha/beta/rc tags. -release = '3.3.0' +release = '4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -94,7 +94,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'nature' +#html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -127,7 +128,7 @@ # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. diff --git a/docs/index.rst b/docs/index.rst index 1450d81c..a50bb3d2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,10 +6,39 @@ ftfy: fixes text for you ======================== -This is a module for making text less broken and more consistent. It works in -Python 2.7, Python 3.2, or later. +`ftfy` fixes Unicode that's broken in various ways. It works in Python 2.7, +Python 3.2, or later. -The most interesting kind of brokenness that this resolves is when someone has +The goal of ftfy is to **take in bad Unicode and output good Unicode**, for use +in your Unicode-aware code. This is different from taking in non-Unicode and +outputting Unicode, which is not a goal of ftfy. It also isn't designed to +protect you from having to write Unicode-aware code. ftfy helps those who help +themselves. + +Of course you're better off if your input is decoded properly and has no +glitches. But you often don't have any control over your input; it's someone +else's mistake, but it's your problem now. + +`ftfy` will do everything it can to fix the problem. + +.. note:: + + Time is marching on. ftfy 4.x has full support for Python 2.7, but given + the sweeping changes to Unicode in Python, it's getting inconvenient to add + new features to ftfy that work the same on both versions. + + ftfy 5.0, when it is released, will probably only support Python 3.3 and + later. + + If you're running on Python 2, ftfy 4.x will keep working for you. It's + fine. You can save yourself a headache by adding `ftfy < 5` to your + requirements, making sure you stay on version 4. + + +Mojibake +-------- + +The most interesting kind of brokenness that ftfy will fix is when someone has encoded Unicode with one standard and decoded it with a different one. This often shows up as characters that turn into nonsense sequences (called "mojibake"): @@ -17,43 +46,86 @@ often shows up as characters that turn into nonsense sequences (called - The word ``schön`` might appear as ``schön``. - An em dash (``—``) might appear as ``—``. - Text that was meant to be enclosed in quotation marks might end up - instead enclosed in ``“`` and ``”`` (and that last character - probably won't even display as anything meaningful). + instead enclosed in ``“`` and ``â€<9d>``, where ``<9d>`` represents an + unprintable character. + +This causes your Unicode-aware code to end up with garbage text because someone +else (or maybe "someone else") made a mistake. -This happens very often to real text. Fortunately, the nonsense sequences -usually contain all the information you need to reconstruct what character was -supposed to be there. +This happens very often to real text. It's often the fault of software that +makes it difficult to use UTF-8 correctly, such as Microsoft Office and some +programming languages. The :func:`ftfy.fix_encoding` function will look for +evidence of mojibake and, when possible, it will undo the process that produced +it to get back the text that was supposed to be there. + +Does this sound impossible? It's really not. UTF-8 is a well-designed encoding +that makes it obvious when it's being misused, and a string of mojibake usually +contains all the information we need to recover the original string. + +When ftfy is tested on multilingual data from Twitter, it has a false positive +rate of less than 1 per million tweets. + + +Other fixes +----------- Any given text string might have other irritating properties, possibly even -interacting with the erroneous decoding: +interacting with the erroneous decoding. The main function of ftfy, +:func:`ftfy.fix_text`, will fix other problems along the way, such as: - The text could contain HTML entities such as ``&`` in place of certain characters, when you would rather see what the characters actually are. + - For that matter, it could contain instructions for a text terminal to do something like change colors, but you are not sending the text to a terminal, so those instructions are just going to look like ``^[[30m;`` or something in the middle of the text. + - The text could write words in non-standard ways for display purposes, such as using the three characters ``fl`` ``o`` ``p`` for the word "flop". This can happen when you copy text out of a PDF, for example. -Of course you're better off if all the text you take as input is decoded -properly and written in standard ways. But often, your input is something you -have no control over. Somebody else's minor mistake becomes your problem. +- It might not be in *NFC normalized* form. You generally want your text to be + NFC-normalized, to avoid situations where unequal sequences of codepoints + can represent exactly the same text. You can also opt for ftfy to use the + more aggressive *NFKC normalization*. + + +.. note:: + + Before version 4.0, ftfy used NFKC normalization by default. This covered a + lot of helpful fixes at once, such as expanding ligatures and replacing + "fullwidth" characters with their standard form. However, it also performed + transformations that lose information, such as converting `™` to `TM` and + `H₂O` to `H2O`. -ftfy will do everything it can to fix the problem. + The default, starting in ftfy 4.0, is to use NFC normalization unless told + to use NFKC normalization (or no normalization at all). The more helpful + parts of NFKC are implemented as separate, limited fixes. + + +There are other interesting things that `ftfy` can do that aren't part of +the :func:`ftfy.fix_text` pipeline, such as: + +* :func:`ftfy.explain_unicode`: show you what's going on in a string, + for debugging purposes +* :func:`ftfy.fixes.decode_escapes`: do what everyone thinks the built-in + `unicode_escape` codec does, but this one doesn't *cause* mojibake Encodings ftfy can handle ------------------------- -ftfy can understand text that was decoded as any of these single-byte +`ftfy` can't fix all possible mix-ups. Its goal is to cover the most common +encoding mix-ups while keeping false positives to a very low rate. + +`ftfy` can understand text that was decoded as any of these single-byte encodings: - Latin-1 (ISO-8859-1) - Windows-1252 (cp1252 -- used in Microsoft products) - Windows-1251 (cp1251 -- the Russian version of cp1252) - MacRoman (used on Mac OS 9 and earlier) -- cp437 (used in MS-DOS) +- cp437 (used in MS-DOS and some versions of the Windows command prompt) when it was actually intended to be decoded as one of these variable-length encodings: @@ -69,22 +141,36 @@ However, ftfy cannot understand other mixups between single-byte encodings, because it is extremely difficult to detect which mixup in particular is the one that happened. +We also can't handle the non-UTF encodings used for Chinese, Japanese, and +Korean, such as ``shift-jis`` and ``gb18030``. See `issue #34 +`_ for why this is so +hard. + +But remember that the input to `ftfy` is Unicode, so it handles actual +CJK *text* just fine. It just can't discover that a CJK *encoding* introduced +mojibake into the text. + Using ftfy ---------- -The main function, `fix_text`, will run text through a sequence of fixes. If +The main function, :func:`ftfy.fix_text`, will run text through a sequence of fixes. If the text changed, it will run them through again, so that you can be sure -the output ends up in a standard form that will be unchanged by `fix_text`. +the output ends up in a standard form that will be unchanged by :func:`ftfy.fix_text`. All the fixes are on by default, but you can pass options to turn them off. +.. versionchanged:: 4.0 + The default normalization was changed from `'NFKC'` to `'NFC'`. The new options + *fix_latin_ligatures* and *fix_character_width* were added to implement some + of the less lossy parts of NFKC normalization on top of NFC. + .. autofunction:: ftfy.fix_text .. autofunction:: ftfy.fix_text_segment -.. autofunction:: ftfy.fix_file +.. autofunction:: ftfy.fix_encoding -.. autofunction:: ftfy.guess_bytes +.. autofunction:: ftfy.fix_file .. autofunction:: ftfy.explain_unicode @@ -110,19 +196,20 @@ A note on encoding detection If your input is a mess of unmarked bytes, you might want a tool that can just statistically analyze those bytes and predict what encoding they're in. -ftfy is not that tool. The `guess_bytes` function will do this in very limited -cases, but to support more encodings from around the world, something more is -needed. +`ftfy` is not that tool. The :func:`ftfy.guess_bytes` function it contains will +do this in very limited cases, but to support more encodings from around the +world, something more is needed. -You may have heard of chardet. Chardet is admirable, but it doesn't completely -do the job either. Its heuristics are designed for multi-byte encodings, such -as UTF-8 and the language-specific encodings used in East Asian languages. It -works badly on single-byte encodings, to the point where it will output wrong -answers with high confidence. +You may have heard of `chardet`. Chardet is admirable, but it doesn't +completely do the job either. Its heuristics are designed for multi-byte +encodings, such as UTF-8 and the language-specific encodings used in East Asian +languages. It works badly on single-byte encodings, to the point where it will +output wrong answers with high confidence. -ftfy's `guess_bytes` doesn't even try the East Asian encodings, so the ideal thing -would combine the simple heuristic of `guess_bytes` with the multibyte character -set detection of `chardet`. This ideal thing doesn't exist yet. +:func:`ftfy.guess_bytes` doesn't even try the East Asian encodings, so the +ideal thing would combine the simple heuristic of :func:`ftfy.guess_bytes` with +the multibyte character set detection of `chardet`. This ideal thing doesn't +exist yet. Accuracy @@ -132,31 +219,41 @@ Accuracy Module documentation ==================== -`ftfy.fixes`: how individual fixes are implemented +*ftfy.fixes*: how individual fixes are implemented -------------------------------------------------- .. automodule:: ftfy.fixes - :members: + :members: unescape_html, remove_terminal_escapes, uncurl_quotes, + fix_latin_ligatures, fix_character_width, fix_line_breaks, + fix_surrogates, remove_control_chars, remove_bom, decode_escapes, + fix_one_step_and_explain, apply_plan -`ftfy.bad_codecs`: decode text from encodings Python doesn't like ------------------------------------------------------------------ +*ftfy.badness*: measures the "badness" of text +---------------------------------------------- -.. automodule:: ftfy.bad_codecs +.. automodule:: ftfy.badness :members: -`ftfy.badness`: measures the "badness" of text ----------------------------------------------- +*ftfy.bad_codecs*: support some encodings Python doesn't like +------------------------------------------------------------- -.. automodule:: ftfy.badness - :members: +.. automodule:: ftfy.bad_codecs + +"Sloppy" encodings +~~~~~~~~~~~~~~~~~~ +.. automodule:: ftfy.bad_codecs.sloppy + +Variants of UTF-8 +~~~~~~~~~~~~~~~~~ +.. automodule:: ftfy.bad_codecs.utf8_variants -`ftfy.chardata` and `ftfy.build_data`: trivia about characters +*ftfy.chardata* and *ftfy.build_data*: trivia about characters -------------------------------------------------------------- -These files load information about the character properties in Unicode 6.1. -Yes, even if your version of Python doesn't support Unicode 6.1. This ensures +These files load information about the character properties in Unicode 7.0. +Yes, even if your version of Python doesn't support Unicode 7.0. This ensures that ftfy's behavior is consistent across versions. .. automodule:: ftfy.chardata diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 565cb2b4..0fbc026d 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -6,7 +6,7 @@ codepoint 0. This is particularly relevant in Python 3, which provides no other way of -decoding CESU-8 or Java's encoding. [1] +decoding CESU-8. The easiest way to use the codec is to simply import `ftfy.bad_codecs`: diff --git a/ftfy/fixes.py b/ftfy/fixes.py index e451cb9d..61762b40 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -40,18 +40,11 @@ def fix_encoding(text): r""" Fix text with incorrectly-decoded garbage ("mojibake") whenever possible. - Something you will find all over the place, in real-world text, is text - that's mistakenly encoded as utf-8, decoded in some ugly format like - latin-1 or even Windows codepage 1252, and encoded as utf-8 again. - - This causes your perfectly good Unicode-aware code to end up with garbage - text because someone else (or maybe "someone else") made a mistake. - - This function looks for the evidence of that having happened and fixes it. - It determines whether it should replace nonsense sequences of single-byte - characters that were really meant to be UTF-8 characters, and if so, turns - them into the correctly-encoded Unicode character that they were meant to - represent. + This function looks for the evidence of mojibake, formulates a plan to fix + it, and applies the plan. It determines whether it should replace nonsense + sequences of single-byte characters that were really meant to be UTF-8 + characters, and if so, turns them into the correctly-encoded Unicode + character that they were meant to represent. The input to the function must be Unicode. If you don't have Unicode text, you're not using the right tool to solve your problem. @@ -86,7 +79,9 @@ def fix_encoding(text): This text is sad .⁔. However, it has safeguards against fixing sequences of letters and - punctuation that can occur in valid text: + punctuation that can occur in valid text. In the following example, + the last three characters are not replaced with a Korean character, + even though they could be. >>> print(fix_encoding('not such a fan of Charlotte Brontë…”')) not such a fan of Charlotte Brontë…” @@ -326,12 +321,13 @@ def fix_latin_ligatures(text): """ Replace single-character ligatures of Latin letters, such as 'fi', with the characters that they contain, as in 'fi'. Latin ligatures are usually not - intended in text, and are a result of copy-and-paste glitches. + intended in text strings (though they're lovely in *rendered* text). If + you have such a ligature in your string, it is probably a result of a + copy-and-paste glitch. - We leave ligatures in other scripts alone to be safe. They may be fully - intended for the text to render correctly, and removing them may lose - information. If you want to take apart nearly all ligatures, use NFKC - normalization. + We leave ligatures in other scripts alone to be safe. They may be intended, + and removing them may lose information. If you want to take apart nearly + all ligatures, use NFKC normalization. >>> print(fix_latin_ligatures("fluffiest")) fluffiest From c5ee60e3839ad753329574f229dab417d1624cf5 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 30 Mar 2015 18:20:36 -0400 Subject: [PATCH 198/502] WIP: Documentation updates --- docs/accuracy.rst | 17 ++++------------- docs/index.rst | 41 ++++++++++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/docs/accuracy.rst b/docs/accuracy.rst index 4e93f2a3..8e22f55f 100644 --- a/docs/accuracy.rst +++ b/docs/accuracy.rst @@ -1,15 +1,6 @@ -This version of ftfy was evaluated on 12,200,000 tweets received from Twitter's streaming API, in early 2014. +This version of ftfy was evaluated on 8,140,000 tweets received from Twitter's streaming API, in early 2015. +None of them were false positives. -1627 (0.013%) of them were detected by ftfy as containing mojibake. These tweets fall into three categories: +The last time ftfy was evaluated, in 2014, it had two false positives in 12,200 +examples, for an overall false positive rate of 0.16 per million. -- True positives: ftfy turns incorrect text into correct text. -- False positives: ftfy turns correct text into incorrect text. -- Confusions: ftfy turns incorrect text into different incorrect text. - -The results for the 1627 tweets that ftfy found were: - -- True positives: 1587 (97.54%) -- False positives: 2 (0.12%) -- Confusions: 38 (2.34%) - -Two false positives in 12,200,000 examples corresponds to an overall false positive rate of 0.16 per million. diff --git a/docs/index.rst b/docs/index.rst index a50bb3d2..194fed55 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,16 +23,14 @@ else's mistake, but it's your problem now. .. note:: - Time is marching on. ftfy 4.x has full support for Python 2.7, but given - the sweeping changes to Unicode in Python, it's getting inconvenient to add - new features to ftfy that work the same on both versions. - - ftfy 5.0, when it is released, will probably only support Python 3.3 and - later. + Time is marching on. ftfy 4.x supports Python 2.7 and 3.x, but when + ftfy 5.0 is released, it will probably only support Python 3. + + If you're running on Python 2, ftfy 4.x will keep working for you. You + don't have to upgrade to 5.0. You can save yourself a headache by adding + `ftfy < 5` to your requirements, making sure you stay on version 4. - If you're running on Python 2, ftfy 4.x will keep working for you. It's - fine. You can save yourself a headache by adding `ftfy < 5` to your - requirements, making sure you stay on version 4. + See `Future versions of ftfy`_ for why this needs to happen. Mojibake @@ -261,3 +259,28 @@ that ftfy's behavior is consistent across versions. .. autofunction:: ftfy.build_data.make_char_data_file + +Future versions of ftfy +======================= + +ftfy has full support for Python 2.7, even including a backport of Unicode 7 +character classes to Python 2. But given the sweeping changes to Unicode in +Python, it's getting inconvenient to add new features to ftfy that work the +same on both versions. + +ftfy 5.0, when it is released, will probably only support Python 3. + +If you want to see examples of why ftfy is particularly difficult to maintain +on two versions of Python (which is more like three versions because of Python +2's "wide" and "narrow" builds), take a look at functions such as +:func:`ftfy.bad_codecs.utf8_variants.mangle_surrogates` and +:func:`ftfy.compatibility._narrow_unichr_workaround`. + +We're following the lead of venerable projects such as jQuery. jQuery dropped +support for IE 6-8 in version 2.0, and in doing so, it became much simpler and +faster for people who didn't need to support old versions of IE in their web +applications. Meanwhile, jQuery 1.9 remained there for those who needed it. + +Similarly, ftfy 5.0 will reduce the size and complexity of the code greatly, +but ftfy 4.x will remain there. If you're running on Python 2, please make sure +that `ftfy < 5` is in your requirements list, not just `ftfy`. From afc5df6360607810bb957c3368f11dd29856a6ee Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 30 Mar 2015 19:37:50 -0400 Subject: [PATCH 199/502] sloppy-utf-8 is now a thing, apparently also, update accuracy.rst based on more results (no false positives in 12M tweets) --- docs/accuracy.rst | 4 +- ftfy/bad_codecs/__init__.py | 7 ++ ftfy/bad_codecs/sloppy_utf8.py | 126 +++++++++++++++++++++++++++++++ ftfy/bad_codecs/utf8_variants.py | 13 +++- ftfy/fixes.py | 8 +- tests/test_encodings.py | 9 +++ tests/test_unicode.py | 37 +++++---- 7 files changed, 182 insertions(+), 22 deletions(-) create mode 100644 ftfy/bad_codecs/sloppy_utf8.py diff --git a/docs/accuracy.rst b/docs/accuracy.rst index 8e22f55f..b8163cfe 100644 --- a/docs/accuracy.rst +++ b/docs/accuracy.rst @@ -1,6 +1,6 @@ -This version of ftfy was evaluated on 8,140,000 tweets received from Twitter's streaming API, in early 2015. +This version of ftfy was evaluated on 12,000,000 tweets received from Twitter's streaming API, in early 2015. None of them were false positives. -The last time ftfy was evaluated, in 2014, it had two false positives in 12,200 +The last time ftfy was evaluated, in 2014, it had two false positives in 12,200,000 examples, for an overall false positive rate of 0.16 per million. diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py index 0984bd52..91d38e93 100644 --- a/ftfy/bad_codecs/__init__.py +++ b/ftfy/bad_codecs/__init__.py @@ -45,6 +45,10 @@ 'java_utf_8', 'java_utf8' ) +SLOPPY_UTF8_NAMES = ( + 'sloppy_utf_8', 'sloppy_utf8' +) + def search_function(encoding): """ @@ -68,6 +72,9 @@ def search_function(encoding): if norm_encoding in UTF8_VAR_NAMES: from ftfy.bad_codecs.utf8_variants import CODEC_INFO codec = CODEC_INFO + elif norm_encoding in SLOPPY_UTF8_NAMES: + from ftfy.bad_codecs.sloppy_utf8 import CODEC_INFO + codec = CODEC_INFO elif norm_encoding.startswith('sloppy_'): from ftfy.bad_codecs.sloppy import CODECS codec = CODECS.get(norm_encoding) diff --git a/ftfy/bad_codecs/sloppy_utf8.py b/ftfy/bad_codecs/sloppy_utf8.py new file mode 100644 index 00000000..388c2e14 --- /dev/null +++ b/ftfy/bad_codecs/sloppy_utf8.py @@ -0,0 +1,126 @@ +r""" +This file defines a codec called "sloppy-utf-8", which handles two surprisingly +common ways to mangle UTF-8: + +- Replace all 0xa0 bytes with 0x20 (because in Windows-1252 it looks like a + non-breaking space, and that's the same as a space, right?) +- Sprinkle in punctuation from Windows-1252 as single bytes, such as byte 0x85 + for an ellipsis. (The Mac OS Terminal has a similar fix, where 0x85 on its + own will be displayed as a slightly ugly ellipsis instead of a replacement + character.) + +Aside from these cases, it acts the same as the "utf-8-variants" decoder. +Encoding with "sloppy-utf-8" is the same as encoding with "utf-8". +""" +from __future__ import unicode_literals +import codecs +from ftfy.bad_codecs.utf8_variants import ( + IncrementalEncoder, IncrementalDecoder, + UTF8IncrementalDecoder +) +NAME = 'sloppy-utf-8' + + +class SloppyIncrementalDecoder(IncrementalDecoder): + def _buffer_decode_step(self, input, errors, final): + """ + There are three possibilities for each decoding step: + + - Decode as much apparently-real UTF-8 as possible. + - Decode a six-byte CESU-8 sequence at the current position. + - Decode a Java-style null at the current position. + + When decoding "apparently-real UTF-8", we might get an error, + and that's where the sloppiness kicks in. If the error is something + we recognize and can fix, we'll fix it. + """ + # Get a reference to the superclass method that we'll be using for + # most of the real work. + sup = UTF8IncrementalDecoder._buffer_decode + + # Find the next byte position that indicates a variant of UTF-8. + # CESU-8 sequences always start with 0xed, and Java nulls always + # start with 0xc0, both of which are conveniently impossible in + # real UTF-8. + cutoff1 = input.find(b'\xed') + cutoff2 = input.find(b'\xc0') + + # Set `cutoff` to whichever cutoff comes first. + if cutoff1 != -1 and cutoff2 != -1: + cutoff = min(cutoff1, cutoff2) + elif cutoff1 != -1: + cutoff = cutoff1 + elif cutoff2 != -1: + cutoff = cutoff2 + else: + # Decode the entire input at once. + try: + return sup(input, 'strict', final) + except UnicodeDecodeError as e: + return self._handle_errors(input, errors, final, e) + + if cutoff1 == 0: + # Decode a possible six-byte sequence starting with 0xed. + return self._buffer_decode_surrogates(sup, input, errors, final) + elif cutoff2 == 0: + # Decode a possible two-byte sequence, 0xc0 0x80. + return self._buffer_decode_null(sup, input, errors, final) + else: + # Decode the bytes up until the next weird thing as UTF-8. + # Set final=True because 0xc0 and 0xed don't make sense in the + # middle of a sequence, in any variant. + try: + return sup(input[:cutoff], 'strict', True) + except UnicodeDecodeError as e: + return self._handle_errors(input[:cutoff], errors, final, e) + + def _handle_errors(self, input, errors, final, err): + """ + """ + sup = UTF8IncrementalDecoder._buffer_decode + + if err.start > 0: + return sup(input[:err.start], errors, final) + else: + if err.reason == 'invalid start byte': + byte = input[0:1] + + # We'll be able to do this with a set of integers when we're on Py3 only. + if byte in b'\x80\x82\x84\x85\x8b\x91\x92\x93\x94\x95\x96\x97\x9b': + return byte.decode('windows-1252'), 1 + + elif err.reason == 'invalid continuation byte': + cause = input[err.end] + if cause == ' ': + input = input[:err.end] + b'\xa0' + input[err.end + 1:] + + return sup(input, errors, final) + + +# Everything below here is boilerplate that matches the modules in the +# built-in `encodings` package. +def encode(input, errors='strict'): + return IncrementalEncoder(errors).encode(input, final=True), len(input) + + +def decode(input, errors='strict'): + return SloppyIncrementalDecoder(errors).decode(input, final=True), len(input) + + +class StreamWriter(codecs.StreamWriter): + encode = encode + + +class StreamReader(codecs.StreamReader): + decode = decode + + +CODEC_INFO = codecs.CodecInfo( + name=NAME, + encode=encode, + decode=decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=SloppyIncrementalDecoder, + streamreader=StreamReader, + streamwriter=StreamWriter, +) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 0fbc026d..6f267476 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -6,7 +6,7 @@ codepoint 0. This is particularly relevant in Python 3, which provides no other way of -decoding CESU-8. +decoding CESU-8 [1]_. The easiest way to use the codec is to simply import `ftfy.bad_codecs`: @@ -132,8 +132,8 @@ def _buffer_decode_step(self, input, errors, final): elif cutoff2 != -1: cutoff = cutoff2 else: - # The entire input can be decoded as UTF-8, so just do so. - return sup(input, errors, final) + # Decode the entire input at once. + return sup(input, errors, True) if cutoff1 == 0: # Decode a possible six-byte sequence starting with 0xed. @@ -146,6 +146,13 @@ def _buffer_decode_step(self, input, errors, final): # Set final=True because 0xc0 and 0xed don't make sense in the # middle of a sequence, in any variant. return sup(input[:cutoff], errors, True) + + def _handle_errors(self, input, errors, final, err): + """ + A subclass can override this to run special code upon encountering + a UTF-8 decoding error. + """ + return UTF8IncrementalDecoder._buffer_decode(input, errors, final) @staticmethod def _buffer_decode_null(sup, input, errors, final): diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 61762b40..c296e29d 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -186,7 +186,13 @@ def fix_one_step_and_explain(text): steps = [('encode', encoding), ('decode', decoding)] return fixed, steps except UnicodeDecodeError: - possible_1byte_encodings.append(encoding) + try: + decoding = 'sloppy-utf-8' + fixed = encoded_bytes.decode(decoding) + steps = [('encode', encoding), ('decode', decoding)] + return fixed, steps + except UnicodeDecodeError: + possible_1byte_encodings.append(encoding) # The next most likely case is that this is Latin-1 that was intended to # be read as Windows-1252, because those two encodings in particular are diff --git a/tests/test_encodings.py b/tests/test_encodings.py index 3500ecc2..1170c510 100644 --- a/tests/test_encodings.py +++ b/tests/test_encodings.py @@ -13,6 +13,15 @@ def test_cesu8(): test_text = '\U00077777 is an unassigned character, and \x00 is null' eq_(test_bytes.decode('cesu8'), test_text) + +def test_sloppy_utf8(): + eq_(bad_codecs.search_function('sloppy-utf8').__class__, + bad_codecs.search_function('sloppy-utf-8').__class__) + + eq_(b'\xe2\x80\x9cmismatched quotes\x85\x94'.decode('sloppy-utf-8'), + '“mismatched quotes…”') + + def test_russian_crash(): thebytes = b'\xe8\xed\xe2\xe5\xed\xf2\xe0\xf0\xe8\xe7\xe0\xf6\xe8\xff ' # We don't care what the result is, but this shouldn't crash diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 2d609207..6eee9ff3 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -from ftfy.fixes import fix_text_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, fix_surrogates +from __future__ import unicode_literals +from ftfy.fixes import fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, fix_surrogates from ftfy.badness import ENDING_PUNCT_RE import unicodedata import sys @@ -33,13 +34,13 @@ def test_all_bmp_characters(): phrases = [ - u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", - u"\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", - u"\u2039ALLÍ ESTÁ\u203A", - u"\u2014ALLÍ ESTÁ\u2014", - u"AHÅ™, the new sofa from IKEA®", - u"ВІКІ is Ukrainian for WIKI", - #u"\u2014a radius of 10 Å\u2014", + "\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", + "\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", + "\u2039ALLÍ ESTÁ\u203A", + "\u2014ALLÍ ESTÁ\u2014", + "AHÅ™, the new sofa from IKEA®", + "ВІКІ is Ukrainian for WIKI", + #"\u2014a radius of 10 Å\u2014", ] # These phrases should not be erroneously "fixed" def test_valid_phrases(): @@ -48,12 +49,12 @@ def test_valid_phrases(): def check_phrase(text): - eq_(fix_text_encoding(text), text) - eq_(fix_text_encoding(text.encode('utf-8').decode('latin-1')), text) + eq_(fix_encoding(text), text) + eq_(fix_encoding(text.encode('utf-8').decode('latin-1')), text) # make sure that the opening punctuation is not the only thing that makes # it work - eq_(fix_text_encoding(text[1:]), text[1:]) - eq_(fix_text_encoding(text[1:].encode('utf-8').decode('latin-1')), text[1:]) + eq_(fix_encoding(text[1:]), text[1:]) + eq_(fix_encoding(text[1:].encode('utf-8').decode('latin-1')), text[1:]) def test_possible_encoding(): @@ -63,13 +64,17 @@ def test_possible_encoding(): def test_fix_with_backslash(): - eq_(fix_text_encoding(u"<40\\% vs \xe2\x89\xa540\\%"), u"<40\\% vs ≥40\\%") + eq_(fix_encoding("<40\\% vs \xe2\x89\xa540\\%"), "<40\\% vs ≥40\\%") -def test_surrogates(): - eq_(fix_surrogates(u'\udbff\udfff'), u'\U0010ffff') - eq_(fix_surrogates(u'\ud800\udc00'), u'\U00010000') +def test_sloppy_utf8(): + eq_(fix_encoding('\xe2\x80\x9cmismatched quotes\x85\x94'), '“mismatched quotes…”') + eq_(fix_encoding('“mismatched quotes…”'), '“mismatched quotes…”') + +def test_surrogates(): + eq_(fix_surrogates('\udbff\udfff'), '\U0010ffff') + eq_(fix_surrogates('\ud800\udc00'), '\U00010000') if __name__ == '__main__': From 24f850de13c306ad62ce8e9bff6061fea18dd42e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 30 Mar 2015 20:53:28 -0400 Subject: [PATCH 200/502] fixes to sloppy-utf-8 --- ftfy/bad_codecs/sloppy_utf8.py | 10 +++++++--- ftfy/fixes.py | 15 ++++++++++----- tests/test_real_tweets.py | 2 ++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/ftfy/bad_codecs/sloppy_utf8.py b/ftfy/bad_codecs/sloppy_utf8.py index 388c2e14..8d6732b1 100644 --- a/ftfy/bad_codecs/sloppy_utf8.py +++ b/ftfy/bad_codecs/sloppy_utf8.py @@ -90,9 +90,13 @@ def _handle_errors(self, input, errors, final, err): return byte.decode('windows-1252'), 1 elif err.reason == 'invalid continuation byte': - cause = input[err.end] - if cause == ' ': - input = input[:err.end] + b'\xa0' + input[err.end + 1:] + cause = input[err.end:err.end + 1] + if cause == b' ': + fixed = input[:err.end] + b'\xa0' + input[err.end + 1:] + return self._buffer_decode_step(fixed, errors, final) + elif cause == b'.' and input[err.end:err.end + 3] == b'...': + fixed = input[:err.end] + b'\x85' + input[err.end + 3:] + return self._buffer_decode_step(fixed, errors, final) return sup(input, errors, final) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index c296e29d..8a911910 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -137,8 +137,10 @@ def fix_encoding_and_explain(text): ('encode', 'cp437') in plan_so_far: cost += 2 - # We need pretty solid evidence to decode from Windows-1251 (Cyrillic). - if ('encode', 'sloppy-windows-1251') in plan_so_far: + # We need pretty solid evidence to decode from Windows-1251 (Cyrillic) + # or from sloppy-utf-8. + if ('encode', 'sloppy-windows-1251') in plan_so_far or\ + ('encode', 'sloppy-utf-8') in plan_so_far: cost += 5 if cost < best_cost: @@ -189,10 +191,13 @@ def fix_one_step_and_explain(text): try: decoding = 'sloppy-utf-8' fixed = encoded_bytes.decode(decoding) - steps = [('encode', encoding), ('decode', decoding)] - return fixed, steps + if len(fixed) < len(text): + steps = [('encode', encoding), ('decode', decoding)] + return fixed, steps except UnicodeDecodeError: - possible_1byte_encodings.append(encoding) + pass + + possible_1byte_encodings.append(encoding) # The next most likely case is that this is Latin-1 that was intended to # be read as Windows-1252, because those two encodings in particular are diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 2e252e47..952e83aa 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -36,6 +36,8 @@ ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí´´"), ('HUHLL Õ…', 'HUHLL Õ…'), ('Iggy Pop (né Jim Osterberg)', 'Iggy Pop (né Jim Osterberg)'), + ('eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías', + 'eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías'), ## Current false positives: #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", From 658465fe7c5b409aa2c0a0b4d4f6681eb7e476cd Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 31 Mar 2015 02:53:11 -0400 Subject: [PATCH 201/502] try sloppy-utf-8 when utf-8 looks like it's doing something --- ftfy/fixes.py | 18 +++++++++--------- tests/test_unicode.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 8a911910..6897e52d 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -134,13 +134,13 @@ def fix_encoding_and_explain(text): # is that we won't use these encodings unless they can successfully # replace multiple characters. if ('encode', 'macroman') in plan_so_far or\ - ('encode', 'cp437') in plan_so_far: + ('encode', 'cp437') in plan_so_far or\ + ('decode', 'sloppy-utf-8') in plan_so_far: cost += 2 # We need pretty solid evidence to decode from Windows-1251 (Cyrillic) # or from sloppy-utf-8. - if ('encode', 'sloppy-windows-1251') in plan_so_far or\ - ('encode', 'sloppy-utf-8') in plan_so_far: + if ('encode', 'sloppy-windows-1251') in plan_so_far: cost += 5 if cost < best_cost: @@ -188,14 +188,14 @@ def fix_one_step_and_explain(text): steps = [('encode', encoding), ('decode', decoding)] return fixed, steps except UnicodeDecodeError: - try: - decoding = 'sloppy-utf-8' - fixed = encoded_bytes.decode(decoding) - if len(fixed) < len(text): + if len(encoded_bytes.decode('utf-8-variants', 'replace')) < len(text): + try: + decoding = 'sloppy-utf-8' + fixed = encoded_bytes.decode(decoding) steps = [('encode', encoding), ('decode', decoding)] return fixed, steps - except UnicodeDecodeError: - pass + except UnicodeDecodeError: + pass possible_1byte_encodings.append(encoding) diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 6eee9ff3..e50ea892 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -69,7 +69,7 @@ def test_fix_with_backslash(): def test_sloppy_utf8(): eq_(fix_encoding('\xe2\x80\x9cmismatched quotes\x85\x94'), '“mismatched quotes…”') - eq_(fix_encoding('“mismatched quotes…”'), '“mismatched quotes…”') + eq_(fix_encoding('“mismatched quotes…”'), '“mismatched quotes…”') def test_surrogates(): From e060867975bd7346a8a4d57c79c0c796f3ee7abf Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 31 Mar 2015 04:47:20 -0400 Subject: [PATCH 202/502] add real test for mixed-encoding quotation marks --- tests/test_real_tweets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 952e83aa..113bf066 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -38,6 +38,8 @@ ('Iggy Pop (né Jim Osterberg)', 'Iggy Pop (né Jim Osterberg)'), ('eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías', 'eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías'), + ("Direzione Pd, ok “senza modifiche” all'Italicum.", + "Direzione Pd, ok \"senza modifiche\" all'Italicum."), ## Current false positives: #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", From 611d9740696d46fcf1ea8cf0affa2983bffb39a9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 1 Apr 2015 13:30:22 -0400 Subject: [PATCH 203/502] sloppy-utf-8 was a bad approach We have to be more conservative about where we allow utf-8 and windows-1252 to mix. The sloppy-utf-8 approach was generating a fair number of false positives, including quite thoroughly reinterpreting someone's not-really-ASCII ASCII art. It will probably work better to think of the problem as "windows-1252 with occasional UTF-8 mixed in", instead of "UTF-8 with occasional windows-1252 mixed in". --- ftfy/bad_codecs/__init__.py | 7 -- ftfy/bad_codecs/sloppy_utf8.py | 130 --------------------------------- ftfy/fixes.py | 15 +--- tests/test_encodings.py | 8 -- 4 files changed, 2 insertions(+), 158 deletions(-) delete mode 100644 ftfy/bad_codecs/sloppy_utf8.py diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py index 91d38e93..0984bd52 100644 --- a/ftfy/bad_codecs/__init__.py +++ b/ftfy/bad_codecs/__init__.py @@ -45,10 +45,6 @@ 'java_utf_8', 'java_utf8' ) -SLOPPY_UTF8_NAMES = ( - 'sloppy_utf_8', 'sloppy_utf8' -) - def search_function(encoding): """ @@ -72,9 +68,6 @@ def search_function(encoding): if norm_encoding in UTF8_VAR_NAMES: from ftfy.bad_codecs.utf8_variants import CODEC_INFO codec = CODEC_INFO - elif norm_encoding in SLOPPY_UTF8_NAMES: - from ftfy.bad_codecs.sloppy_utf8 import CODEC_INFO - codec = CODEC_INFO elif norm_encoding.startswith('sloppy_'): from ftfy.bad_codecs.sloppy import CODECS codec = CODECS.get(norm_encoding) diff --git a/ftfy/bad_codecs/sloppy_utf8.py b/ftfy/bad_codecs/sloppy_utf8.py deleted file mode 100644 index 8d6732b1..00000000 --- a/ftfy/bad_codecs/sloppy_utf8.py +++ /dev/null @@ -1,130 +0,0 @@ -r""" -This file defines a codec called "sloppy-utf-8", which handles two surprisingly -common ways to mangle UTF-8: - -- Replace all 0xa0 bytes with 0x20 (because in Windows-1252 it looks like a - non-breaking space, and that's the same as a space, right?) -- Sprinkle in punctuation from Windows-1252 as single bytes, such as byte 0x85 - for an ellipsis. (The Mac OS Terminal has a similar fix, where 0x85 on its - own will be displayed as a slightly ugly ellipsis instead of a replacement - character.) - -Aside from these cases, it acts the same as the "utf-8-variants" decoder. -Encoding with "sloppy-utf-8" is the same as encoding with "utf-8". -""" -from __future__ import unicode_literals -import codecs -from ftfy.bad_codecs.utf8_variants import ( - IncrementalEncoder, IncrementalDecoder, - UTF8IncrementalDecoder -) -NAME = 'sloppy-utf-8' - - -class SloppyIncrementalDecoder(IncrementalDecoder): - def _buffer_decode_step(self, input, errors, final): - """ - There are three possibilities for each decoding step: - - - Decode as much apparently-real UTF-8 as possible. - - Decode a six-byte CESU-8 sequence at the current position. - - Decode a Java-style null at the current position. - - When decoding "apparently-real UTF-8", we might get an error, - and that's where the sloppiness kicks in. If the error is something - we recognize and can fix, we'll fix it. - """ - # Get a reference to the superclass method that we'll be using for - # most of the real work. - sup = UTF8IncrementalDecoder._buffer_decode - - # Find the next byte position that indicates a variant of UTF-8. - # CESU-8 sequences always start with 0xed, and Java nulls always - # start with 0xc0, both of which are conveniently impossible in - # real UTF-8. - cutoff1 = input.find(b'\xed') - cutoff2 = input.find(b'\xc0') - - # Set `cutoff` to whichever cutoff comes first. - if cutoff1 != -1 and cutoff2 != -1: - cutoff = min(cutoff1, cutoff2) - elif cutoff1 != -1: - cutoff = cutoff1 - elif cutoff2 != -1: - cutoff = cutoff2 - else: - # Decode the entire input at once. - try: - return sup(input, 'strict', final) - except UnicodeDecodeError as e: - return self._handle_errors(input, errors, final, e) - - if cutoff1 == 0: - # Decode a possible six-byte sequence starting with 0xed. - return self._buffer_decode_surrogates(sup, input, errors, final) - elif cutoff2 == 0: - # Decode a possible two-byte sequence, 0xc0 0x80. - return self._buffer_decode_null(sup, input, errors, final) - else: - # Decode the bytes up until the next weird thing as UTF-8. - # Set final=True because 0xc0 and 0xed don't make sense in the - # middle of a sequence, in any variant. - try: - return sup(input[:cutoff], 'strict', True) - except UnicodeDecodeError as e: - return self._handle_errors(input[:cutoff], errors, final, e) - - def _handle_errors(self, input, errors, final, err): - """ - """ - sup = UTF8IncrementalDecoder._buffer_decode - - if err.start > 0: - return sup(input[:err.start], errors, final) - else: - if err.reason == 'invalid start byte': - byte = input[0:1] - - # We'll be able to do this with a set of integers when we're on Py3 only. - if byte in b'\x80\x82\x84\x85\x8b\x91\x92\x93\x94\x95\x96\x97\x9b': - return byte.decode('windows-1252'), 1 - - elif err.reason == 'invalid continuation byte': - cause = input[err.end:err.end + 1] - if cause == b' ': - fixed = input[:err.end] + b'\xa0' + input[err.end + 1:] - return self._buffer_decode_step(fixed, errors, final) - elif cause == b'.' and input[err.end:err.end + 3] == b'...': - fixed = input[:err.end] + b'\x85' + input[err.end + 3:] - return self._buffer_decode_step(fixed, errors, final) - - return sup(input, errors, final) - - -# Everything below here is boilerplate that matches the modules in the -# built-in `encodings` package. -def encode(input, errors='strict'): - return IncrementalEncoder(errors).encode(input, final=True), len(input) - - -def decode(input, errors='strict'): - return SloppyIncrementalDecoder(errors).decode(input, final=True), len(input) - - -class StreamWriter(codecs.StreamWriter): - encode = encode - - -class StreamReader(codecs.StreamReader): - decode = decode - - -CODEC_INFO = codecs.CodecInfo( - name=NAME, - encode=encode, - decode=decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=SloppyIncrementalDecoder, - streamreader=StreamReader, - streamwriter=StreamWriter, -) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 6897e52d..61762b40 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -134,12 +134,10 @@ def fix_encoding_and_explain(text): # is that we won't use these encodings unless they can successfully # replace multiple characters. if ('encode', 'macroman') in plan_so_far or\ - ('encode', 'cp437') in plan_so_far or\ - ('decode', 'sloppy-utf-8') in plan_so_far: + ('encode', 'cp437') in plan_so_far: cost += 2 - # We need pretty solid evidence to decode from Windows-1251 (Cyrillic) - # or from sloppy-utf-8. + # We need pretty solid evidence to decode from Windows-1251 (Cyrillic). if ('encode', 'sloppy-windows-1251') in plan_so_far: cost += 5 @@ -188,15 +186,6 @@ def fix_one_step_and_explain(text): steps = [('encode', encoding), ('decode', decoding)] return fixed, steps except UnicodeDecodeError: - if len(encoded_bytes.decode('utf-8-variants', 'replace')) < len(text): - try: - decoding = 'sloppy-utf-8' - fixed = encoded_bytes.decode(decoding) - steps = [('encode', encoding), ('decode', decoding)] - return fixed, steps - except UnicodeDecodeError: - pass - possible_1byte_encodings.append(encoding) # The next most likely case is that this is Latin-1 that was intended to diff --git a/tests/test_encodings.py b/tests/test_encodings.py index 1170c510..f59e08aa 100644 --- a/tests/test_encodings.py +++ b/tests/test_encodings.py @@ -14,14 +14,6 @@ def test_cesu8(): eq_(test_bytes.decode('cesu8'), test_text) -def test_sloppy_utf8(): - eq_(bad_codecs.search_function('sloppy-utf8').__class__, - bad_codecs.search_function('sloppy-utf-8').__class__) - - eq_(b'\xe2\x80\x9cmismatched quotes\x85\x94'.decode('sloppy-utf-8'), - '“mismatched quotes…”') - - def test_russian_crash(): thebytes = b'\xe8\xed\xe2\xe5\xed\xf2\xe0\xf0\xe8\xe7\xe0\xf6\xe8\xff ' # We don't care what the result is, but this shouldn't crash From 409177deff4aa4d8588dc64cfa3f28eb0647c12d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 1 Apr 2015 14:39:13 -0400 Subject: [PATCH 204/502] groundwork for alternate mixed-encoding fix --- ftfy/__init__.py | 23 ++++++++++++++++++----- ftfy/bad_codecs/utf8_variants.py | 3 +-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 65b1ef60..7e161f52 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -19,10 +19,21 @@ import warnings +# These are flags used by the fix_encoding option. BASIC is equivalent to True, +# and it fixes UTF-8 when it can be applied consistently throughout the string. +# +# THOROUGH, on the other hand, will find stubborn UTF-8 sequences that didn't +# decode consistently with the rest of the string, and decode them anyway if +# they're a sufficiently common character. This covers the case where -- for +# whatever reason -- something out there is encoding left curly quotes in UTF-8 +# and right curly quotes in Latin-1. +BASIC, THOROUGH = 1, 2 + + def fix_text(text, fix_entities='auto', remove_terminal_escapes=True, - fix_encoding=True, + fix_encoding=THOROUGH, fix_latin_ligatures=True, fix_character_width=True, uncurl_quotes=True, @@ -82,9 +93,11 @@ def fix_text(text, instructions for Unix terminals, such as the codes that make text appear in different colors. - - If `fix_encoding` is True, look for common mistakes that come from + - If `fix_encoding` is True (or BASIC), look for common mistakes that come from encoding or decoding Unicode text incorrectly, and fix them if they are - reasonably fixable. See `fixes.fix_encoding` for details. + reasonably fixable. If it's set to THOROUGH (the default), it will make + an extra pass to fix even more problems. See `fixes.fix_encoding` for + details. - If `uncurl_quotes` is True, replace various curly quotation marks with plain-ASCII straight quotes. @@ -191,7 +204,7 @@ def fix_file(input_file, encoding=None, fix_entities='auto', remove_terminal_escapes=True, - fix_encoding=True, + fix_encoding=THOROUGH, fix_latin_ligatures=True, fix_character_width=True, uncurl_quotes=True, @@ -239,7 +252,7 @@ def fix_file(input_file, def fix_text_segment(text, fix_entities='auto', remove_terminal_escapes=True, - fix_encoding=True, + fix_encoding=THOROUGH, fix_latin_ligatures=True, fix_character_width=True, uncurl_quotes=True, diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 6f267476..346fbf25 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -119,8 +119,7 @@ def _buffer_decode_step(self, input, errors, final): # Find the next byte position that indicates a variant of UTF-8. # CESU-8 sequences always start with 0xed, and Java nulls always - # start with 0xc0, both of which are conveniently impossible in - # real UTF-8. + # start with 0xc0. cutoff1 = input.find(b'\xed') cutoff2 = input.find(b'\xc0') From 58da81e58e5e73441192effa80df8f8de664d216 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 1 Apr 2015 17:20:54 -0400 Subject: [PATCH 205/502] spacing fix --- ftfy/fixes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 61762b40..0ee92f9c 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -339,7 +339,7 @@ def fix_character_width(text): """ The ASCII characters, katakana, and Hangul characters have alternate "halfwidth" or "fullwidth" forms that help text line up in a grid. - + If you don't need these width properties, you probably want to replace these characters with their standard form, which is what this function does. From 8b9cfd6cec49d87511815747634cd99f0d42abb0 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 2 Apr 2015 16:41:20 -0400 Subject: [PATCH 206/502] refactor costs, add a 0xa0-fixing step A transcoding step, such as the one that tries turning 0x20 into 0xa0, needs to be able to assign a cost based on what it did, so costs are now included in the 'steps' list instead of assigned later. --- ftfy/__init__.py | 23 +++------- ftfy/chardata.py | 1 + ftfy/fixes.py | 113 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 99 insertions(+), 38 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 7e161f52..65b1ef60 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -19,21 +19,10 @@ import warnings -# These are flags used by the fix_encoding option. BASIC is equivalent to True, -# and it fixes UTF-8 when it can be applied consistently throughout the string. -# -# THOROUGH, on the other hand, will find stubborn UTF-8 sequences that didn't -# decode consistently with the rest of the string, and decode them anyway if -# they're a sufficiently common character. This covers the case where -- for -# whatever reason -- something out there is encoding left curly quotes in UTF-8 -# and right curly quotes in Latin-1. -BASIC, THOROUGH = 1, 2 - - def fix_text(text, fix_entities='auto', remove_terminal_escapes=True, - fix_encoding=THOROUGH, + fix_encoding=True, fix_latin_ligatures=True, fix_character_width=True, uncurl_quotes=True, @@ -93,11 +82,9 @@ def fix_text(text, instructions for Unix terminals, such as the codes that make text appear in different colors. - - If `fix_encoding` is True (or BASIC), look for common mistakes that come from + - If `fix_encoding` is True, look for common mistakes that come from encoding or decoding Unicode text incorrectly, and fix them if they are - reasonably fixable. If it's set to THOROUGH (the default), it will make - an extra pass to fix even more problems. See `fixes.fix_encoding` for - details. + reasonably fixable. See `fixes.fix_encoding` for details. - If `uncurl_quotes` is True, replace various curly quotation marks with plain-ASCII straight quotes. @@ -204,7 +191,7 @@ def fix_file(input_file, encoding=None, fix_entities='auto', remove_terminal_escapes=True, - fix_encoding=THOROUGH, + fix_encoding=True, fix_latin_ligatures=True, fix_character_width=True, uncurl_quotes=True, @@ -252,7 +239,7 @@ def fix_file(input_file, def fix_text_segment(text, fix_entities='auto', remove_terminal_escapes=True, - fix_encoding=THOROUGH, + fix_encoding=True, fix_latin_ligatures=True, fix_character_width=True, uncurl_quotes=True, diff --git a/ftfy/chardata.py b/ftfy/chardata.py index e5e2a170..d2231d04 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -116,3 +116,4 @@ def _build_width_map(): return width_map WIDTH_MAP = _build_width_map() + diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 0ee92f9c..732edd83 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -112,6 +112,56 @@ def fix_text_encoding(text): return fix_encoding(text) +# When we support discovering mojibake in more encodings, we run the risk +# of more false positives. We can mitigate false positives by assigning an +# additional cost to using encodings that are rarer than Windows-1252, so +# that these encodings will only be used if they fix multiple problems. +ENCODING_COSTS = { + 'macroman': 2, + 'cp437': 3, + 'sloppy-windows-1251': 5 +} + + +# Recognize UTF-8 mojibake that's so blatant that we can fix it even when the +# rest of the string doesn't decode as UTF-8 -- namely, UTF-8 re-encodings of +# the 'General Punctuation' characters U+2000 to U+2040. +# +# These are recognizable by the distinctive 'â€' ('\xe2\x80') sequence they all +# begin with when decoded as Windows-1252. + +OBVIOUS_UTF8_RE = re.compile(b'\xe2\x80[\x80-\xbf]') + + +# Recognize UTF-8 sequences that would be valid if it weren't for a b'\xa0' +# that some Windows-1252 program converted to a plain space. +# +# The smaller values are included on a case-by-case basis, because we don't want +# to decode likely input sequences to unlikely characters. These are the ones +# that *do* form likely characters before 0xa0: +# +# 0xc2 -> U+A0 NO-BREAK SPACE +# 0xc3 -> U+E0 LATIN SMALL LETTER A WITH GRAVE +# 0xc5 -> U+160 LATIN CAPITAL LETTER S WITH CARON +# 0xce -> U+3A0 GREEK CAPITAL LETTER PI +# 0xd0 -> U+420 CYRILLIC CAPITAL LETTER ER +# +# These still need to come with a cost, so that they only get converted when +# there's evidence that it fixes other things. Any of these could represent +# characters that legitimately appear surrounded by spaces, particularly U+C5 +# (Å), which is a word in multiple languages! +# +# We should consider checking for b'\x85' being converted to ... in the future. +# I've seen it once, but the text still wasn't recoverable. + +ALTERED_UTF8_RE = re.compile(b'[\xc2\xc3\xc5\xce\xd0][ ]' + b'|[\xe0-\xef][ ][\x80-\xbf]' + b'|[\xe0-\xef][\x80-\xbf][ ]' + b'|[\xf0-\xf4][ ][\x80-\xbf][\x80-\xbf]' + b'|[\xf0-\xf4][\x80-\xbf][ ][\x80-\xbf]' + b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]') + + def fix_encoding_and_explain(text): """ Re-decodes text that has been decoded incorrectly, and also return a @@ -129,17 +179,9 @@ def fix_encoding_and_explain(text): text, plan = fix_one_step_and_explain(text) plan_so_far.extend(plan) cost = text_cost(text) - - # Add a penalty if we used a particularly obsolete encoding. The result - # is that we won't use these encodings unless they can successfully - # replace multiple characters. - if ('encode', 'macroman') in plan_so_far or\ - ('encode', 'cp437') in plan_so_far: - cost += 2 - - # We need pretty solid evidence to decode from Windows-1251 (Cyrillic). - if ('encode', 'sloppy-windows-1251') in plan_so_far: - cost += 5 + orig_cost = cost + for _op, _encoding, step_cost in plan_so_far: + cost += step_cost if cost < best_cost: best_cost = cost @@ -180,11 +222,25 @@ def fix_one_step_and_explain(text): # remember the encoding for later. try: decoding = 'utf-8' + # Check encoded_bytes for sequences that would be UTF-8, + # except they have b' ' where b'\xa0' would belong. + if ALTERED_UTF8_RE.search(encoded_bytes): + fixed_bytes, cost = restore_byte_a0(encoded_bytes) + try: + fixed = fixed_bytes.decode('utf-8-variants') + steps = [('transcode', 'restore_byte_a0', cost), + ('decode', 'utf-8-variants', 0)] + return fixed, steps + except UnicodeDecodeError: + pass + if b'\xed' in encoded_bytes or b'\xc0' in encoded_bytes: decoding = 'utf-8-variants' fixed = encoded_bytes.decode(decoding) - steps = [('encode', encoding), ('decode', decoding)] + steps = [('encode', encoding, ENCODING_COSTS.get(encoding, 0)), + ('decode', decoding, 0)] return fixed, steps + except UnicodeDecodeError: possible_1byte_encodings.append(encoding) @@ -207,7 +263,7 @@ def fix_one_step_and_explain(text): fixed = encoded.decode('windows-1252') steps = [] if fixed != text: - steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] + steps = [('encode', 'latin-1', 0), ('decode', 'windows-1252', 1)] return fixed, steps except UnicodeDecodeError: # This text contained characters that don't even make sense @@ -215,6 +271,8 @@ def fix_one_step_and_explain(text): # that case, let's not assume anything. pass + # TODO: look for a-hat-euro sequences that remain, and fix them in isolation. + # The cases that remain are mixups between two different single-byte # encodings, and not the common case of Latin-1 vs. Windows-1252. # @@ -230,20 +288,26 @@ def apply_plan(text, plan): """ Apply a plan for fixing the encoding of text. - The plan is a list of tuples of the form (operation, encoding), where - `operation` is either 'encode' or 'decode', and `encoding` is an encoding - name such as 'utf-8' or 'latin-1'. + The plan is a list of tuples of the form (operation, encoding, cost): - Because only text can be encoded, and only bytes can be decoded, the plan - should alternate 'encode' and 'decode' steps, or else this function will - encounter an error. + - `operation` is 'encode' if it turns a string into bytes, 'decode' if it + turns bytes into a string, and 'transcode' if it turns bytes into bytes. + - `encoding` is the name of the encoding to use, such as 'utf-8' or + 'latin-1', or the function name in the case of 'transcode'. + - `cost` is a penalty to apply to the score of the resulting text, when + performing a dubious operation that requires a lot of evidence. """ obj = text - for operation, encoding in plan: + for operation, encoding, cost in plan: if operation == 'encode': obj = obj.encode(encoding) elif operation == 'decode': obj = obj.decode(encoding) + elif operation == 'transcode': + if encoding == 'restore_byte_a0': + obj = restore_byte_a0(obj) + else: + raise ValueError("Unknown transcode operation: %s" % encoding) else: raise ValueError("Unknown plan step: %s" % operation) @@ -524,3 +588,12 @@ def decode_match(match): return codecs.decode(match.group(0), 'unicode-escape') return ESCAPE_SEQUENCE_RE.sub(decode_match, text) + + +def restore_byte_a0(byts): + def replacement(match): + return match.group(0).replace(b'\x20', b'\xa0') + + fixed = ALTERED_UTF8_RE.sub(replacement, byts) + return fixed, fixed.count(b'\xa0') * 2 + From 1efd76c1adb09463f086361d9f676443aa20df40 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 3 Apr 2015 13:48:56 -0400 Subject: [PATCH 207/502] implement the new mixed-encoding fix --- ftfy/chardata.py | 54 +++++++++++++++++++++++++++++++++ ftfy/fixes.py | 69 ++++++++++++++++--------------------------- tests/test_unicode.py | 2 +- 3 files changed, 81 insertions(+), 44 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index d2231d04..1c997b2c 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -46,6 +46,60 @@ def _build_regexes(): ENCODING_REGEXES = _build_regexes() +def _build_utf8_punct_regex(): + """ + Recognize UTF-8 mojibake that's so blatant that we can fix it even when the + rest of the string doesn't decode as UTF-8 -- namely, UTF-8 sequences for + the 'General Punctuation' characters U+2000 to U+2040, re-encoded in + Windows-1252. + + These are recognizable by the distinctive 'â€' ('\xe2\x80') sequence they all + begin with when decoded as Windows-1252. + """ + obvious_utf8_regexes = {} + + # We're making a regex that has all the literal bytes from 0x80 to 0xbf in + # a range. "Couldn't this have just said [\x80-\xbf]?", you might ask. + # However, when we decode the regex as Windows-1252, the resulting characters + # won't even be remotely contiguous. + # + # Unrelatedly, the expression that generates these bytes will be so much + # prettier when we deprecate Python 2. + continuation_char_list = ''.join(unichr(i) for i in range(0x80, 0xc0)).encode('latin-1') + obvious_utf8 = 'â€[' + continuation_char_list.decode('sloppy-windows-1252') + ']' + return re.compile(obvious_utf8) +PARTIAL_UTF8_PUNCT_RE = _build_utf8_punct_regex() + + +# Recognize UTF-8 sequences that would be valid if it weren't for a b'\xa0' +# that some Windows-1252 program converted to a plain space. +# +# The smaller values are included on a case-by-case basis, because we don't want +# to decode likely input sequences to unlikely characters. These are the ones +# that *do* form likely characters before 0xa0: +# +# 0xc2 -> U+A0 NO-BREAK SPACE +# 0xc3 -> U+E0 LATIN SMALL LETTER A WITH GRAVE +# 0xc5 -> U+160 LATIN CAPITAL LETTER S WITH CARON +# 0xce -> U+3A0 GREEK CAPITAL LETTER PI +# 0xd0 -> U+420 CYRILLIC CAPITAL LETTER ER +# +# These still need to come with a cost, so that they only get converted when +# there's evidence that it fixes other things. Any of these could represent +# characters that legitimately appear surrounded by spaces, particularly U+C5 +# (Å), which is a word in multiple languages! +# +# We should consider checking for b'\x85' being converted to ... in the future. +# I've seen it once, but the text still wasn't recoverable. + +ALTERED_UTF8_RE = re.compile(b'[\xc2\xc3\xc5\xce\xd0][ ]' + b'|[\xe0-\xef][ ][\x80-\xbf]' + b'|[\xe0-\xef][\x80-\xbf][ ]' + b'|[\xf0-\xf4][ ][\x80-\xbf][\x80-\xbf]' + b'|[\xf0-\xf4][\x80-\xbf][ ][\x80-\xbf]' + b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]') + + def possible_encoding(text, encoding): """ Given text and a single-byte encoding, check whether that text could have diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 732edd83..3ad54962 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -6,7 +6,8 @@ from __future__ import unicode_literals from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS, - CONTROL_CHARS, LIGATURES, WIDTH_MAP) + CONTROL_CHARS, LIGATURES, WIDTH_MAP, + PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE) from ftfy.badness import text_cost from ftfy.compatibility import htmlentitydefs, unichr, UNSAFE_PRIVATE_USE_RE import re @@ -123,45 +124,6 @@ def fix_text_encoding(text): } -# Recognize UTF-8 mojibake that's so blatant that we can fix it even when the -# rest of the string doesn't decode as UTF-8 -- namely, UTF-8 re-encodings of -# the 'General Punctuation' characters U+2000 to U+2040. -# -# These are recognizable by the distinctive 'â€' ('\xe2\x80') sequence they all -# begin with when decoded as Windows-1252. - -OBVIOUS_UTF8_RE = re.compile(b'\xe2\x80[\x80-\xbf]') - - -# Recognize UTF-8 sequences that would be valid if it weren't for a b'\xa0' -# that some Windows-1252 program converted to a plain space. -# -# The smaller values are included on a case-by-case basis, because we don't want -# to decode likely input sequences to unlikely characters. These are the ones -# that *do* form likely characters before 0xa0: -# -# 0xc2 -> U+A0 NO-BREAK SPACE -# 0xc3 -> U+E0 LATIN SMALL LETTER A WITH GRAVE -# 0xc5 -> U+160 LATIN CAPITAL LETTER S WITH CARON -# 0xce -> U+3A0 GREEK CAPITAL LETTER PI -# 0xd0 -> U+420 CYRILLIC CAPITAL LETTER ER -# -# These still need to come with a cost, so that they only get converted when -# there's evidence that it fixes other things. Any of these could represent -# characters that legitimately appear surrounded by spaces, particularly U+C5 -# (Å), which is a word in multiple languages! -# -# We should consider checking for b'\x85' being converted to ... in the future. -# I've seen it once, but the text still wasn't recoverable. - -ALTERED_UTF8_RE = re.compile(b'[\xc2\xc3\xc5\xce\xd0][ ]' - b'|[\xe0-\xef][ ][\x80-\xbf]' - b'|[\xe0-\xef][\x80-\xbf][ ]' - b'|[\xf0-\xf4][ ][\x80-\xbf][\x80-\xbf]' - b'|[\xf0-\xf4][\x80-\xbf][ ][\x80-\xbf]' - b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]') - - def fix_encoding_and_explain(text): """ Re-decodes text that has been decoded incorrectly, and also return a @@ -244,6 +206,12 @@ def fix_one_step_and_explain(text): except UnicodeDecodeError: possible_1byte_encodings.append(encoding) + # Look for a-hat-euro sequences that remain, and fix them in isolation. + if PARTIAL_UTF8_PUNCT_RE.search(text): + steps = [('transcode', 'fix_partial_utf8_punct_in_1252', 1)] + fixed = fix_partial_utf8_punct_in_1252(text) + return fixed, steps + # The next most likely case is that this is Latin-1 that was intended to # be read as Windows-1252, because those two encodings in particular are # easily confused. @@ -271,8 +239,6 @@ def fix_one_step_and_explain(text): # that case, let's not assume anything. pass - # TODO: look for a-hat-euro sequences that remain, and fix them in isolation. - # The cases that remain are mixups between two different single-byte # encodings, and not the common case of Latin-1 vs. Windows-1252. # @@ -291,7 +257,7 @@ def apply_plan(text, plan): The plan is a list of tuples of the form (operation, encoding, cost): - `operation` is 'encode' if it turns a string into bytes, 'decode' if it - turns bytes into a string, and 'transcode' if it turns bytes into bytes. + turns bytes into a string, and 'transcode' if it keeps the type the same. - `encoding` is the name of the encoding to use, such as 'utf-8' or 'latin-1', or the function name in the case of 'transcode'. - `cost` is a penalty to apply to the score of the resulting text, when @@ -306,6 +272,8 @@ def apply_plan(text, plan): elif operation == 'transcode': if encoding == 'restore_byte_a0': obj = restore_byte_a0(obj) + elif encoding == 'fix_partial_utf8_punct_in_1252': + obj = fix_partial_utf8_punct_in_1252(obj) else: raise ValueError("Unknown transcode operation: %s" % encoding) else: @@ -597,3 +565,18 @@ def replacement(match): fixed = ALTERED_UTF8_RE.sub(replacement, byts) return fixed, fixed.count(b'\xa0') * 2 + +def fix_partial_utf8_punct_in_1252(text): + """ + Fix particular characters that seem to be found in the wild encoded in + UTF-8 and decoded in Latin-1 or Windows-1252, even when this fix can't be + consistently applied. + + For this function, we assume the text has been decoded in Windows-1252. + If it was decoded in Latin-1, we'll call this right after it goes through + the Latin-1-to-Windows-1252 fixer. + """ + def replacement(match): + return match.group(0).encode('sloppy-windows-1252').decode('utf-8') + return PARTIAL_UTF8_PUNCT_RE.sub(replacement, text) + diff --git a/tests/test_unicode.py b/tests/test_unicode.py index e50ea892..1096530a 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -67,7 +67,7 @@ def test_fix_with_backslash(): eq_(fix_encoding("<40\\% vs \xe2\x89\xa540\\%"), "<40\\% vs ≥40\\%") -def test_sloppy_utf8(): +def test_mixed_utf8(): eq_(fix_encoding('\xe2\x80\x9cmismatched quotes\x85\x94'), '“mismatched quotes…”') eq_(fix_encoding('“mismatched quotes…”'), '“mismatched quotes…”') From fdb8a5d82732f1a5d825312e5d1f7ec2043f55aa Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 6 Apr 2015 14:59:57 -0400 Subject: [PATCH 208/502] Clean up ftfy.fixes and document it more --- ftfy/chardata.py | 9 ++++++++- ftfy/fixes.py | 26 +++++++++++++++++--------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 1c997b2c..da7bb401 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -100,6 +100,11 @@ def _build_utf8_punct_regex(): b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]') +# These regexes match various Unicode variations on single and double quotes. +SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') +DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') + + def possible_encoding(text, encoding): """ Given text and a single-byte encoding, check whether that text could have @@ -161,6 +166,9 @@ def _build_control_char_mapping(): # A translate mapping that replaces halfwidth and fullwidth forms with their # standard-width forms. def _build_width_map(): + # Though it's not listed as a fullwidth character, we'll want to convert + # U+3000 IDEOGRAPHIC SPACE to U+20 SPACE on the same principle, so start + # with that in the dictionary. width_map = {0x3000: ' '} for i in range(0xff01, 0xfff0): char = unichr(i) @@ -170,4 +178,3 @@ def _build_width_map(): return width_map WIDTH_MAP = _build_width_map() - diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 3ad54962..7d7b20e9 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -7,7 +7,8 @@ from __future__ import unicode_literals from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS, CONTROL_CHARS, LIGATURES, WIDTH_MAP, - PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE) + PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE, + SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE) from ftfy.badness import text_cost from ftfy.compatibility import htmlentitydefs, unichr, UNSAFE_PRIVATE_USE_RE import re @@ -87,6 +88,12 @@ def fix_encoding(text): >>> print(fix_encoding('not such a fan of Charlotte Brontë…”')) not such a fan of Charlotte Brontë…” + ftfy can now recover some complex manglings of text, such as when UTF-8 + mojibake has been normalized in a way that replaces U+A0 with a space: + + >>> print(fix_encoding('The more you know 🌠')) + The more you know 🌠 + Cases of genuine ambiguity can sometimes be addressed by finding other characters that are not double-encoded, and expecting the encoding to be consistent: @@ -130,7 +137,7 @@ def fix_encoding_and_explain(text): "plan" indicating all the steps required to fix it. To fix similar text in the same way, without having to detect anything, - you can use the ``apply_plan`` function. + you can use the :func:`ftfy.fixes.apply_plan` function. """ best_version = text best_cost = text_cost(text) @@ -242,9 +249,8 @@ def fix_one_step_and_explain(text): # The cases that remain are mixups between two different single-byte # encodings, and not the common case of Latin-1 vs. Windows-1252. # - # Those cases are somewhat rare, and may be unsolvable without adding false - # positives. If you're in one of these situations, you should try using the - # `ftfy.guess_bytes` function. + # These cases may be unsolvable without adding false positives, though + # I have vague ideas about how to optionally address them in the future. # Return the text unchanged; the plan is empty. return text, [] @@ -336,9 +342,6 @@ def remove_terminal_escapes(text): return ANSI_RE.sub('', text) -SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') -DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') - def uncurl_quotes(text): r""" Replace curly quotation marks with straight equivalents. @@ -559,6 +562,11 @@ def decode_match(match): def restore_byte_a0(byts): + """ + Find sequences that would convincingly decode as UTF-8 if the byte 0x20 + were changed to 0xa0, and fix them. This is used as a step within + `fix_encoding`. + """ def replacement(match): return match.group(0).replace(b'\x20', b'\xa0') @@ -570,7 +578,7 @@ def fix_partial_utf8_punct_in_1252(text): """ Fix particular characters that seem to be found in the wild encoded in UTF-8 and decoded in Latin-1 or Windows-1252, even when this fix can't be - consistently applied. + consistently applied. This is used as a step within `fix_encoding`. For this function, we assume the text has been decoded in Windows-1252. If it was decoded in Latin-1, we'll call this right after it goes through From a7b0470ed43d4e62b3dd93dc0ccc5db55ee1f9b8 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 6 Apr 2015 15:00:49 -0400 Subject: [PATCH 209/502] Tweak some heuristics; new passing test cases --- ftfy/badness.py | 3 ++- ftfy/build_data.py | 7 ++++++- ftfy/char_classes.dat | Bin 3847 -> 3847 bytes tests/test_real_tweets.py | 6 ++++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index f94fc552..333c7db2 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -97,6 +97,7 @@ def _make_weirdness_regex(): '\N{RIGHT SINGLE QUOTATION MARK}\N{RIGHT DOUBLE QUOTATION MARK}' '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' + '\N{ACUTE ACCENT}' ']+' ) @@ -129,7 +130,7 @@ def sequence_weirdness(text): """ text2 = unicodedata.normalize('NFC', text) weirdness = len(WEIRDNESS_RE.findall(chars_to_classes(text2))) - punct_discount = len(ENDING_PUNCT_RE.findall(text2)) + punct_discount = sum(len(x) for x in ENDING_PUNCT_RE.findall(text2)) return weirdness * 2 - punct_discount diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 8e78890e..20743c56 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -5,7 +5,7 @@ classes we care about change, or if a new version of Python supports a new Unicode standard and we want it to affect our string decoding. -The file that we generate is based on Unicode 6.1, as supported by Python 3.3. +The file that we generate is based on Unicode 7.0, as supported by Python 3.5. You can certainly use it in earlier versions. This simply makes sure that we get consistent results from running ftfy on different versions of Python. @@ -104,7 +104,12 @@ def make_char_data_file(do_it_anyway=False): else: cclasses[codepoint] = 'o' + # Mark whitespace control characters as whitespace cclasses[9] = cclasses[10] = cclasses[12] = cclasses[13] = ' ' + + # Tilde is not a "math symbol" the way it's used + cclasses[ord('~')] = 'o' + out = open('char_classes.dat', 'wb') out.write(zlib.compress(''.join(cclasses).encode('ascii'))) out.close() diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index e948607c13bde716ea9ea2b01f53929c7e3c55da..47d8a9abf7fb3ff5cacb69ad9e8d0ff0e9756f73 100644 GIT binary patch delta 2885 zcmYk4dpOgJAIH0#U%#U0MCuo|bX@8vBgrvrCyG>#3R}!2Es440GDGIuIn~i|Y2|(? zid?d68FCq>v|LB5SquvebD7I9yZ`EU{`j5uAD`#_Jnzr*`aI9)c_)GrL64k)@&~u_ zzUa#dUTII$F(8BJCV@U?gPd=}fU!`D4T?cTcUX_j|VbVCL>iz8UUR&YXT%^?k= z`Ds23otV@loN3k!^7E_b75#ResgLz21$Lkl`kjuSS@{&1yZqiT)}y-V>`Rt*r_c}P zBzYVl9?0A{7bpscC2oyr>NH(Pv&#uz6B-wWIv5Z8v?GksJ0 zY*~NJ{i8|LPx{aCWQ+j>nSSNcdrD5o=txDQs{fWYpOxihkFNH-#t1tk*^F-qk3EE) z4Usr^Icw$1fGB>p1@j@|?&{p>q5*Igt>DhbdqwcKcP_87Ei{g zie|nJT$7u>n3-b-{3?55RxLNEM7Kp=@Ac%A?OHl$ZuF8iXa|3z6yYu|oHX5jN%!hV zC{fxTQS9U0QV$$czk$ed9EKk4I1hMu;U)FGI3YU~P zhp}2?Y-~&Pn{2DQ2K~?A_=j7=I%E(=87^Vyx%3B8th7-1dc z^CTQV<%E(-A5N-@X4&J4iCvtlHMaf>RWbNSV>62rmabp}O%8J~VkE=%B#pHhYuWEed(X-xpV~3ljsVRJ%3w7sY+{Fk?KAS>XgiEDqXz~YD z*8}miy>>#2UfIgR()i`(tiiCs&Im}4@xK-}-pvSP9F()D?8=GTlqrw?T@#7xhI_{_boys{MXh1xi43%HYQ z?X26MqiSc0U#1uB$O@`adwq#04XI(?D6pLLRreMt4<}2xm4Invw(T`~mU*x7Kuj*P zrwsq&{QPDwbB_M*v-bUjrKpp_8u zMVwr{`~uQz*}5zpA`t@2yV`R~hcv(tiBMBrq}w-F@Pjq&_K-+_3M1Y=G{#J1)_x3Z z+=&1BlrrWToTz>b)^}ceS~%k|fP*DNLqJEoZPlsyz6P3zfx{r0ZW5sWNTeh;hK8=u z<}Q0@A7ceyXSgEaX?PB&9e=QdLSQDS$0EGTN~94i}P=pHEe%JJLKK zK>^%eVB@eL>WTtNs}tM_bsO7QN0l#Q$Xv~8DZh^u^fY~t#I8lPmAKRtk($+D@9rsi=^==?a z97A_>prb@)csCw6RgYAsj)n-vp)J<461i(4?uxgmN@Uk>Wv(5tiv7f%G1PhWG+?LLrb3@=}%=LH=egNe2`b21!*)m>S~G8|Vt{9=^CL+H+H3G~O&OtWTTAEzglk-K&gM)>_JAwNBNOz(s7o?z(V*nvyujvWrh3EpiVH zQ+MROqPy~Hd!}OoD0zw^(zEm3NC|M$+??8kd@TQ?rA7%R?%B_QWllU*v%SZI#%^;7 zyYdt>MK9hMCD3wRs;ezvsgZkf6*I>_FGazAmiu_E6(>F%RGLEQPYgANzQj7XZ_!4L zo{;bkUC@@bv;esGx>$D-Jh1lKDb(ug_|0!b`p+bXZq$2oa)`j_TEzPY^jRJMkS zH78OsPma)-Iaway3yG;g(-*z7Y1K_QHZTZgth1H!u>~L&O`6Q7o?4@XM<>Mw za#e}{M1H5oXy9}0WB`V>if!4n*mTV%A=8sx*Tm! zEY?kJHe;9{%oEowoLx7RC9Oj!YfTDquqYcqXe+1RYpYsMQf4syQ_%5NXe9q@LwDr< ziaM_uvc~XrNvE!{%ii1Cc%fgZ&Lnnr$=jJkiqsDq>_o*h6c%V9p&>6e$|}VyKyn3m zg(JYfoo^L3&k}&b3S2IByrFATN=DoC@JCPC*SGJkBCM(v5N-A>yHM+Z0R!-rLb$|U(l9U>(u!GQJjt#5yq zqqNO5o~qbdE#G3@h0+Q^Zamlyw$k?9<6UacQwnV;GV3{W*onb<*IdKG>|4KD4q`LXWmvAuW!4O9vtgU>_N(9d<9FVFe4h97yg$$T^E|KTlWvr5bPo>f#eX>f zkIhxNTXpny*y#|9^4NU7+#D}oDon#-t$qWlxcU8Vb`UHC6V>X!g$}2DkBSP)S5#XI;Wnk%1RlQQyB5 z7PPn=e9W+%--X;5jw%G^x5V?8e(MrKXKJhLH4iKIJ)N^UM)U8~>54+jod=_uxD$3E&^%#K%ro2JvNNm07`u^N)oqKk^9dlD zO6l*-cWxd^{?+&-+eC**mgd{|=rIU$6-dNKzlh||v9*s1(CrlJ_G9*M8gAIWnsm;% z)}UR+4jVbVgB1{w`<}s}=H`=5R__h=?QV>YMvUWZZU$M)S63}~m*4?TB_vLP^fA%( zzO&38H07CQ!2xWGw5u@OopeZ%`0H7LPFG+^nFlcD(#Gp^dq%hv3VN|c`LF|n`vcV- z)G83QfgAuj2nNj4XT7PH3#g5u9NbG;7q@ou5U8ges5mhaRcfw|mwEB^hV>CZUW7gm z13EY-gspy)|8RH=i)7eyyr%AHWJ!r>@kI6W>DTuL=DP1VGZAfo zF$u=th2G5}7M9+N|3DpgN2R8+LOxt=;Gi_kp5>WHVri~;GpuND-(_UC;ap7(Vb^I< zfO;Y!2a+l3S@G~Jf&GH*Ef`ki4pAnZY(i@m*c-0jdz3d27+C@tRV!VdCmHO#{d_Yt zx@i!+)O@zqDsYVXr>3oWmvNNgjv1ibg6x*&ADTngyydmCNPm0e={N3|`R}6>(DE2g ztBhKt2*KQHeIcH_u%GawF!qvTR1OkPRXc*S_o|==`>nlf>@7_Y`PNfa{Zk`|Z6rfG z>)w}4OHT)+ce@yuDig*6;yY2$k>a}L^K`cpF;cYnaV36B=0bR+Q*@^hV9zs_ z(#xNgAk&arlXL5WKVGOhLLO$E+fil(kJ&b{6woH6RRr9LJGi%*+7s!^Ss{XH9Tem= zZl5-T`$LI*S14AVv>IE9Iwf=vvRJUf%`&N6h;dt~=&H0CvT=|l(6DX+&pzUGn9g}6 zq*|1#!RH271WqJ+2=veHEL-c{w~*uBDS&<+ONM~wsnFsWGk7Ly8` zA8z;2X6Mf#Jo9DCh#!pu<-6nqm1i0~;yd{!)86z!j~ahQXJ)0uyAAow?-iktSXS(p zXx18?{`AAZ$q(yK+4*UHd_D-;kA&OyJr>udla;^&7AOMF6*PAlQG&bl`P;+k-Qlxr z#>S-Oa?Rw#6mDDWJHVf5Z9rN3F30ZGtEImvGhT@o^TlCH?%(VDk>Juz$XHuv)E{V% zrXCqh9WMYSVSK}sCG;OkYz3BTF!~ z{@_Vv;Is}N(UQ@^e6VNsN&%+EHK)|zZOhbA#_IHfa4VB&ky*EN3U@F_M{6+IgL7Q? zC_~5umm%(T)QI`B;RX?W%KT#KM0+t4EZH^+(Dy7~(fFU99$Zuk(fRB1c5m7Wd~$dC zvc?*|-mk|u__PXM<+Ru9^DcR^=QHqnU~*= z@^Wd%=M%*u3z|jek)rw~2?)!|^;)!~iG~z{b6;LyzGFuS+S>(17|d-G>+VW4asKHT zYvz}_hY7R%>`;8Fh@n~yhTCn6A*SA*i0U+MOs|~V(P=ENhSJr^_bHbtv$G2=k%}l6 zM(Ain$)%c+=9;GIkTv*A)>Xld}yjt;x~JZZtDvN&LZEJ^UOh5!=pz56Qp<8LxCe|JVNVk6wyO`I^F0^Ji+x= zg+q0(-O|+AwZ(RBcg67VE@GQgh9{7qfAzXE{-Fr*hl2Hv{Xzli*s&yK#m2~9+&DH%Az(N0Y|NlOX*MYoob;%zPGqU>b zVxN5N0`uyXmX=U~SjA-uBg7la59Dh@NP`cWn#X72WtKgfs9@1DlfIuL?iqiQGVJKW zqBn?eh<{doYo}+plxy=4W2^V>m zS87zaa$)I4!K}V+&)R5`N$OVYF{R#c3QCP>RCd8g+3IUk**HR+uap{gjSzd0(3~I{ z<2n5}c)?1_(TcJGXL24^WWLH~$f4djV&}wFvMG||lT07+H`A>3`texpKNb7F-;(tZsc<1D`s4z>c^&h>t( z4da!9K|2`%>YKyZn}$5gtVVOp(c5^ZZ0igwc8NrtY~-sV$n+u7#}55Xb4N*|=!0q3 zmB3*mV)5%7v8Z^>b9xlXc&Ye!7OHXa?9BG0ytqyKEK$a9Q_#o1;=U0xSKEx$P1FDW z8_a*`Wh+eteT`mToY1O|+tl>0Jm{wNx|mJFZBu(v;k)z4RW>4W%-1fr^`Ddq&94Ub Wy-I6x-07Rs`sA{4Qe7-HRsR Date: Mon, 6 Apr 2015 15:07:23 -0400 Subject: [PATCH 210/502] update test_unicode for new heuristics --- tests/test_unicode.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 1096530a..21b82a41 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals from ftfy.fixes import fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, fix_surrogates -from ftfy.badness import ENDING_PUNCT_RE +from ftfy.badness import sequence_weirdness import unicodedata import sys from nose.tools import eq_ @@ -19,13 +19,15 @@ def char_names(text): # Most single-character strings which have been misencoded should be restored. -def test_all_bmp_characters(): +def test_bmp_characters(): for index in range(0xa0, 0xfffd): char = unichr(index) # Exclude code points that are not assigned if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): garble = char.encode('utf-8').decode('latin-1') - if not (index < 0x800 and ENDING_PUNCT_RE.search(garble)): + # Exclude characters whose re-encoding is protected by the + # 'sequence_weirdness' metric + if sequence_weirdness(garble) >= 0: garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') for garb in (garble, garble2): fixed, plan = fix_encoding_and_explain(garb) From d418f6f7e68456a23f0d8153f2378aef34a47192 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 6 Apr 2015 15:07:47 -0400 Subject: [PATCH 211/502] Update the command-line tool --- ftfy/__init__.py | 17 ++++++++------- ftfy/cli.py | 54 +++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 65b1ef60..6eba797f 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -18,6 +18,8 @@ import unicodedata import warnings +__version__ = '4.0.0' + def fix_text(text, fix_entities='auto', @@ -199,15 +201,14 @@ def fix_file(input_file, fix_surrogates=True, remove_control_chars=True, remove_bom=True, - normalization='NFC' - ): + normalization='NFC'): """ Fix text that is found in a file. If the file is being read as Unicode text, use that. If it's being read as - bytes, then unfortunately, we have to guess what encoding it is. We'll try - a few common encodings, but we make no promises. See the `guess_bytes` - function for how this is done. + bytes, then we hope an encoding was supplied. If not, unfortunately, we + have to guess what encoding it is. We'll try a few common encodings, but we + make no promises. See the `guess_bytes` function for how this is done. The output is a stream of fixed lines of text. """ @@ -290,8 +291,8 @@ def fix_text_segment(text, def guess_bytes(bstring): """ - NOTE: Using `guess_bytes` is not the recommended way of using ftfy. It's - not the primary purpose of ftfy. + NOTE: Using `guess_bytes` is not the recommended way of using ftfy. ftfy + is not designed to be an encoding detector. In the unfortunate situation that you have some bytes in an unknown encoding, ftfy can guess a reasonable strategy for decoding them, by trying @@ -306,7 +307,7 @@ def guess_bytes(bstring): other, so we can't support them without decreasing the accuracy of ftfy. If you don't know which encoding you have at all, I recommend - trying the 'chardet' module, but being appropriately skeptical about its + trying the 'chardet' module, and being appropriately skeptical about its results. The encodings we try here are: diff --git a/ftfy/cli.py b/ftfy/cli.py index 6ac83706..7827f1a9 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -4,12 +4,22 @@ Because files do not come with their encoding marked, it first runs the file through `ftfy.guess_bytes`, then runs it through `ftfy.fix_text`. """ -from ftfy import fix_file +from ftfy import fix_file, __version__ import sys ENCODE_STDOUT = (sys.hexversion < 0x03000000) +ENCODE_ERROR_TEXT = """ +Unfortunately, this output stream does not support Unicode. + +- If this is a Linux system, your locale may be very old or misconfigured. + You should `export LANG=C.UTF-8`. +- If this is a Windows system, please do not use the Command Prompt (cmd.exe) + to run Python. It does not and cannot support real Unicode. +""" + + def main(): """ Run ftfy as a command-line utility. (Requires Python 2.7 or later, or @@ -17,17 +27,51 @@ def main(): """ import argparse - parser = argparse.ArgumentParser() - parser.add_argument('filename', help='file to transcode') + parser = argparse.ArgumentParser( + description="ftfy (fixes text for you), version %s" % __version__ + ) + parser.add_argument('filename', + help='The file whose Unicode is to be fixed') + parser.add_argument('-g', '--guess', action='store_true', + help="Ask ftfy to guess the encoding of your input. " + "This is risky. Overrides -e.") + parser.add_argument('-e', '--encoding', type=str, default='utf-8', + help='The encoding of the input. Defaults to UTF-8.') + parser.add_argument('-n', '--normalization', type=str, default='NFC', + help='The normalization of Unicode to apply. ' + 'Defaults to NFC. Can be "none".') + parser.add_argument('--preserve-entities', action='store_true', + help="Leave HTML entities as they are. The default " + "is to decode them in non-HTML files.") args = parser.parse_args() file = open(args.filename) - for line in fix_file(file): + encoding = args.encoding + if args.guess: + encoding = None + + normalization = args.normalization + if normalization.lower() == 'none': + normalization = None + + if args.preserve_entities: + fix_entities = False + else: + fix_entities = 'auto' + + for line in fix_file( + file, encoding=encoding, fix_entities=fix_entities, + normalization=normalization + ): if ENCODE_STDOUT: sys.stdout.write(line.encode('utf-8')) else: - sys.stdout.write(line) + try: + sys.stdout.write(line) + except UnicodeEncodeError: + sys.stderr.write(ENCODE_ERROR_TEXT) + sys.exit(1) if __name__ == '__main__': From 75c23096c49c4ebf732308ae8ca86f17d32baa3f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 6 Apr 2015 15:10:32 -0400 Subject: [PATCH 212/502] update version number to 4.0.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b8674e45..2fc3534d 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="ftfy", - version='3.4.0', + version='4.0.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From 1f2f58eeb8833fe233a94ff583789872fdc28f78 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 6 Apr 2015 15:37:10 -0400 Subject: [PATCH 213/502] soup up the command-line tool some more --- ftfy/cli.py | 54 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/ftfy/cli.py b/ftfy/cli.py index 7827f1a9..f3d51d21 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -7,7 +7,7 @@ from ftfy import fix_file, __version__ import sys -ENCODE_STDOUT = (sys.hexversion < 0x03000000) +ENCODE_STDIO = (sys.hexversion < 0x03000000) ENCODE_ERROR_TEXT = """ @@ -19,6 +19,16 @@ to run Python. It does not and cannot support real Unicode. """ +DECODE_ERROR_TEXT = """ +Error: this input couldn't be decoded as %r. We got the following error: + + %s + +ftfy works best when its input is in a known encoding. You can use `ftfy -g` +to guess, if you're desperate. Otherwise, give the encoding name with the +`-e` option, such as `ftfy -e latin-1`. +""" + def main(): """ @@ -30,8 +40,9 @@ def main(): parser = argparse.ArgumentParser( description="ftfy (fixes text for you), version %s" % __version__ ) - parser.add_argument('filename', - help='The file whose Unicode is to be fixed') + parser.add_argument('filename', default='-', nargs='?', + help='The file whose Unicode is to be fixed. Defaults' + 'to -, meaning standard input.') parser.add_argument('-g', '--guess', action='store_true', help="Ask ftfy to guess the encoding of your input. " "This is risky. Overrides -e.") @@ -46,10 +57,17 @@ def main(): args = parser.parse_args() - file = open(args.filename) encoding = args.encoding if args.guess: encoding = None + file_flags = 'rb' + else: + file_flags = 'r' + + if args.filename == '-': + file = sys.stdin + else: + file = open(args.filename, file_flags) normalization = args.normalization if normalization.lower() == 'none': @@ -60,18 +78,22 @@ def main(): else: fix_entities = 'auto' - for line in fix_file( - file, encoding=encoding, fix_entities=fix_entities, - normalization=normalization - ): - if ENCODE_STDOUT: - sys.stdout.write(line.encode('utf-8')) - else: - try: - sys.stdout.write(line) - except UnicodeEncodeError: - sys.stderr.write(ENCODE_ERROR_TEXT) - sys.exit(1) + try: + for line in fix_file( + file, encoding=encoding, fix_entities=fix_entities, + normalization=normalization + ): + if ENCODE_STDIO: + sys.stdout.write(line.encode('utf-8')) + else: + try: + sys.stdout.write(line) + except UnicodeEncodeError: + sys.stderr.write(ENCODE_ERROR_TEXT) + sys.exit(1) + except UnicodeDecodeError as e: + sys.stderr.write(DECODE_ERROR_TEXT % (encoding, e)) + sys.exit(1) if __name__ == '__main__': From 2c7a6dd00ca49e5fab55632789004568c59b8cbe Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 6 Apr 2015 15:38:08 -0400 Subject: [PATCH 214/502] documentation updates --- docs/accuracy.rst | 25 +++++++++++++++++++++---- ftfy/fixes.py | 14 +++++--------- setup.py | 2 +- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/docs/accuracy.rst b/docs/accuracy.rst index b8163cfe..ac12b499 100644 --- a/docs/accuracy.rst +++ b/docs/accuracy.rst @@ -1,6 +1,23 @@ -This version of ftfy was evaluated on 12,000,000 tweets received from Twitter's streaming API, in early 2015. -None of them were false positives. +ftfy uses Twitter's streaming API as an endless source of realistic sample +data. Twitter is massively multilingual, and despite that it's supposed to be +uniformly UTF-8, in practice, any encoding mistake that someone can make will +be made by someone's Twitter client. -The last time ftfy was evaluated, in 2014, it had two false positives in 12,200,000 -examples, for an overall false positive rate of 0.16 per million. +This version of ftfy was evaluated on 13,000,000 tweets received from Twitter's +streaming API in April 2015. None of them were false positives. +ftfy changes about 1 in every 8500 tweets. We sampled 1000 of these changes for +further evaluation, and found: + +- 980 of them correctly restored the text. +- 12 of them incompletely or incorrectly restored the text, when a sufficiently + advanced heuristic might have been able to fully recover the text. +- 8 of them represented text that had lost too much information to be fixed. +- None of them changed correct text to incorrect text (these would be false + positives). + +In all the data we've sampled, including from previous versions of ftfy, there +are only two false positives that remain that we know of:: + + fix_encoding('├┤a┼┐a┼┐a┼┐a┼┐a') == 'ôaſaſaſaſa' + fix_encoding('ESSE CARA AI QUEM É¿') == 'ESSE CARA AI QUEM ɿ' diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 7d7b20e9..c001ef24 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -51,13 +51,8 @@ def fix_encoding(text): The input to the function must be Unicode. If you don't have Unicode text, you're not using the right tool to solve your problem. - .. note:: - The following examples are written using unmarked literal strings, - but they are Unicode text. In Python 2 we have "unicode_literals" - turned on, and in Python 3 this is always the case. - - ftfy decodes text that looks like it was decoded incorrectly. It leaves - alone text that doesn't. + `fix_encoding` decodes text that looks like it was decoded incorrectly. It + leaves alone text that doesn't. >>> print(fix_encoding('único')) único @@ -88,8 +83,9 @@ def fix_encoding(text): >>> print(fix_encoding('not such a fan of Charlotte Brontë…”')) not such a fan of Charlotte Brontë…” - ftfy can now recover some complex manglings of text, such as when UTF-8 - mojibake has been normalized in a way that replaces U+A0 with a space: + This function can now recover some complex manglings of text, such as when + UTF-8 mojibake has been normalized in a way that replaces U+A0 with a + space: >>> print(fix_encoding('The more you know 🌠')) The more you know 🌠 diff --git a/setup.py b/setup.py index 2fc3534d..d4e7a50e 100644 --- a/setup.py +++ b/setup.py @@ -48,5 +48,5 @@ Instead of just asking for 'ftfy', you should ask for: 'ftfy >= 4, < 5' - """) + """.strip(' ')) From 8b724466a0ca0f602e16532901813e95134249dc Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 6 Apr 2015 15:58:36 -0400 Subject: [PATCH 215/502] Document the CLI tool --- docs/index.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 194fed55..b50d328d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -214,6 +214,36 @@ Accuracy -------- .. include:: accuracy.rst + +Command-line tool +----------------- +ftfy can be used from the command line. By default, it takes UTF-8 input and +writes it to UTF-8 output, fixing problems in its Unicode as it goes. + +Here's the usage documentation for the `ftfy` command:: + + usage: ftfy [-h] [-g] [-e ENCODING] [-n NORMALIZATION] [--preserve-entities] + [filename] + + ftfy (fixes text for you), version 4.0.0 + + positional arguments: + filename The file whose Unicode is to be fixed. Defaults to -, + meaning standard input. + + optional arguments: + -h, --help show this help message and exit + -g, --guess Ask ftfy to guess the encoding of your input. This is + risky. Overrides -e. + -e ENCODING, --encoding ENCODING + The encoding of the input. Defaults to UTF-8. + -n NORMALIZATION, --normalization NORMALIZATION + The normalization of Unicode to apply. Defaults to + NFC. Can be "none". + --preserve-entities Leave HTML entities as they are. The default is to + decode them in non-HTML files. + + Module documentation ==================== From 954e25a799a7c5f5f3521824b1e2b53a1e6ab5d4 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 6 Apr 2015 15:58:45 -0400 Subject: [PATCH 216/502] pylint fixes --- ftfy/__init__.py | 6 +++--- ftfy/bad_codecs/sloppy.py | 2 +- ftfy/bad_codecs/utf8_variants.py | 7 ------- ftfy/build_data.py | 2 +- ftfy/chardata.py | 30 ++++++++++++++++++------------ ftfy/cli.py | 15 +++++++-------- ftfy/compatibility.py | 20 -------------------- ftfy/fixes.py | 17 +++++++++++------ 8 files changed, 41 insertions(+), 58 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 6eba797f..fd600fe7 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -92,8 +92,8 @@ def fix_text(text, plain-ASCII straight quotes. - If `fix_latin_ligatures` is True, then ligatures made of Latin letters, - such as `fi`, will be separated into individual letters. These ligatures are - usually not meaningful outside of font rendering, and often represent + such as `fi`, will be separated into individual letters. These ligatures + are usually not meaningful outside of font rendering, and often represent copy-and-paste errors. - If `fix_character_width` is True, half-width and full-width characters @@ -136,7 +136,7 @@ def fix_text(text, `fix_text` will work one line at a time, with the possibility that some lines are in different encodings, allowing it to fix text that has been concatenated together from different sources. - + When it encounters lines longer than `max_decode_length` (1 million codepoints by default), it will not run the `fix_encoding` step, to avoid unbounded slowdowns. diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index 073b1443..d9d6c687 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -75,7 +75,7 @@ import sys REPLACEMENT_CHAR = '\ufffd' -PY26 = sys.version_info[:2] == (2,6) +PY26 = sys.version_info[:2] == (2, 6) def make_sloppy_codec(encoding): """ diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 346fbf25..933ce912 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -146,13 +146,6 @@ def _buffer_decode_step(self, input, errors, final): # middle of a sequence, in any variant. return sup(input[:cutoff], errors, True) - def _handle_errors(self, input, errors, final, err): - """ - A subclass can override this to run special code upon encountering - a UTF-8 decoding error. - """ - return UTF8IncrementalDecoder._buffer_decode(input, errors, final) - @staticmethod def _buffer_decode_null(sup, input, errors, final): """ diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 20743c56..0d64b785 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -47,7 +47,7 @@ def make_char_data_file(do_it_anyway=False): an alpha version at the time of writing this code. It will run in earlier versions, but you won't get the Unicode 7 standard, leading to inconsistent behavior. - + To protect against this, running this in the wrong version of Python will raise an error unless you pass `do_it_anyway=True`. """ diff --git a/ftfy/chardata.py b/ftfy/chardata.py index da7bb401..aeed8fd1 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -53,20 +53,22 @@ def _build_utf8_punct_regex(): the 'General Punctuation' characters U+2000 to U+2040, re-encoded in Windows-1252. - These are recognizable by the distinctive 'â€' ('\xe2\x80') sequence they all - begin with when decoded as Windows-1252. + These are recognizable by the distinctive 'â€' ('\xe2\x80') sequence they + all begin with when decoded as Windows-1252. """ - obvious_utf8_regexes = {} - # We're making a regex that has all the literal bytes from 0x80 to 0xbf in # a range. "Couldn't this have just said [\x80-\xbf]?", you might ask. - # However, when we decode the regex as Windows-1252, the resulting characters - # won't even be remotely contiguous. + # However, when we decode the regex as Windows-1252, the resulting + # characters won't even be remotely contiguous. # # Unrelatedly, the expression that generates these bytes will be so much # prettier when we deprecate Python 2. - continuation_char_list = ''.join(unichr(i) for i in range(0x80, 0xc0)).encode('latin-1') - obvious_utf8 = 'â€[' + continuation_char_list.decode('sloppy-windows-1252') + ']' + continuation_char_list = ''.join( + unichr(i) for i in range(0x80, 0xc0) + ).encode('latin-1') + obvious_utf8 = ('â€[' + + continuation_char_list.decode('sloppy-windows-1252') + + ']') return re.compile(obvious_utf8) PARTIAL_UTF8_PUNCT_RE = _build_utf8_punct_regex() @@ -130,9 +132,11 @@ def chars_to_classes(string): return string.translate(CHAR_CLASS_STRING) -# A translate mapping that will strip all C0 control characters except -# those that represent whitespace. def _build_control_char_mapping(): + """ + Build a translate mapping that strips all C0 control characters, + except those that represent whitespace. + """ control_chars = {} for i in range(32): control_chars[i] = None @@ -163,9 +167,11 @@ def _build_control_char_mapping(): } -# A translate mapping that replaces halfwidth and fullwidth forms with their -# standard-width forms. def _build_width_map(): + """ + Build a translate mapping that replaces halfwidth and fullwidth forms + with their standard-width forms. + """ # Though it's not listed as a fullwidth character, we'll want to convert # U+3000 IDEOGRAPHIC SPACE to U+20 SPACE on the same principle, so start # with that in the dictionary. diff --git a/ftfy/cli.py b/ftfy/cli.py index f3d51d21..fd0b55d5 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -41,7 +41,7 @@ def main(): description="ftfy (fixes text for you), version %s" % __version__ ) parser.add_argument('filename', default='-', nargs='?', - help='The file whose Unicode is to be fixed. Defaults' + help='The file whose Unicode is to be fixed. Defaults ' 'to -, meaning standard input.') parser.add_argument('-g', '--guess', action='store_true', help="Ask ftfy to guess the encoding of your input. " @@ -63,7 +63,7 @@ def main(): file_flags = 'rb' else: file_flags = 'r' - + if args.filename == '-': file = sys.stdin else: @@ -79,10 +79,9 @@ def main(): fix_entities = 'auto' try: - for line in fix_file( - file, encoding=encoding, fix_entities=fix_entities, - normalization=normalization - ): + for line in fix_file(file, encoding=encoding, + fix_entities=fix_entities, + normalization=normalization): if ENCODE_STDIO: sys.stdout.write(line.encode('utf-8')) else: @@ -91,8 +90,8 @@ def main(): except UnicodeEncodeError: sys.stderr.write(ENCODE_ERROR_TEXT) sys.exit(1) - except UnicodeDecodeError as e: - sys.stderr.write(DECODE_ERROR_TEXT % (encoding, e)) + except UnicodeDecodeError as err: + sys.stderr.write(DECODE_ERROR_TEXT % (encoding, err)) sys.exit(1) diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index 1246248c..697f151c 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -34,26 +34,6 @@ def _narrow_unichr_workaround(codepoint): if sys.maxunicode < 0x10000: unichr = _narrow_unichr_workaround - # In a narrow build of Python, we can't write a regex involving astral - # characters. If we want to write the regex: - # - # [\U00100000-\U0010ffff] - # - # The actual string that defines it quietly turns into: - # - # [\udbc0\udc00-\udbff\udfff] - # - # And now the range operator only applies to the middle two characters. - # It looks like a range that's going backwards from \dc00 to \dbff, - # which is an error. - # - # What we can do instead is rewrite the expression to be _about_ the two - # surrogates that make up the astral characters, instead of the characters - # themselves. This would be wrong on a wide build, but it works on a - # narrow build. - UNSAFE_PRIVATE_USE_RE = re.compile('[\udbc0-\udbff][\udc00-\udfff]') -else: - UNSAFE_PRIVATE_USE_RE = re.compile('[\U00100000-\U0010ffff]') def bytes_to_ints(bytestring): diff --git a/ftfy/fixes.py b/ftfy/fixes.py index c001ef24..d73a4a2e 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -10,7 +10,7 @@ PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE) from ftfy.badness import text_cost -from ftfy.compatibility import htmlentitydefs, unichr, UNSAFE_PRIVATE_USE_RE +from ftfy.compatibility import htmlentitydefs, unichr import re import sys import codecs @@ -106,11 +106,14 @@ def fix_encoding(text): The best version of the text is found using :func:`ftfy.badness.text_cost`. """ - text, _plan = fix_encoding_and_explain(text) + text, _ = fix_encoding_and_explain(text) return text def fix_text_encoding(text): + """ + A deprecated name for :func:`ftfy.fixes.fix_encoding`. + """ warnings.warn('fix_text_encoding is now known as fix_encoding', DeprecationWarning) return fix_encoding(text) @@ -144,8 +147,7 @@ def fix_encoding_and_explain(text): text, plan = fix_one_step_and_explain(text) plan_so_far.extend(plan) cost = text_cost(text) - orig_cost = cost - for _op, _encoding, step_cost in plan_so_far: + for _, _, step_cost in plan_so_far: cost += step_cost if cost < best_cost: @@ -234,7 +236,8 @@ def fix_one_step_and_explain(text): fixed = encoded.decode('windows-1252') steps = [] if fixed != text: - steps = [('encode', 'latin-1', 0), ('decode', 'windows-1252', 1)] + steps = [('encode', 'latin-1', 0), + ('decode', 'windows-1252', 1)] return fixed, steps except UnicodeDecodeError: # This text contained characters that don't even make sense @@ -266,7 +269,7 @@ def apply_plan(text, plan): performing a dubious operation that requires a lot of evidence. """ obj = text - for operation, encoding, cost in plan: + for operation, encoding, _ in plan: if operation == 'encode': obj = obj.encode(encoding) elif operation == 'decode': @@ -564,6 +567,7 @@ def restore_byte_a0(byts): `fix_encoding`. """ def replacement(match): + "The function to apply when this regex matches." return match.group(0).replace(b'\x20', b'\xa0') fixed = ALTERED_UTF8_RE.sub(replacement, byts) @@ -581,6 +585,7 @@ def fix_partial_utf8_punct_in_1252(text): the Latin-1-to-Windows-1252 fixer. """ def replacement(match): + "The function to apply when this regex matches." return match.group(0).encode('sloppy-windows-1252').decode('utf-8') return PARTIAL_UTF8_PUNCT_RE.sub(replacement, text) From 4ac57ba2cc0920fcbc1864f6bfbd6cc49535a496 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 9 Apr 2015 16:41:52 -0400 Subject: [PATCH 217/502] fix command-line tool so it can handle non-utf-8 input Also, update the CHANGELOG and accuracy.rst --- CHANGELOG.md | 60 ++++++++++++++++++++++++++++++++++++++++++++--- docs/accuracy.rst | 2 +- ftfy/cli.py | 5 +--- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bcca067..60c5de73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,39 @@ +## Version 4.0.0 (April 10, 2015) + +Breaking changes: + +- The default normalization is now NFC, not NFKC. +- The `fix_text` function has some new options that perform more targeted + operations that are part of NFKC normalization, without requiring hitting all + your text with the huge mallet that is NFKC. + +New features: + +- Fixers for strange new forms of mojibake, including particularly clear cases + of mixed UTF-8 and Windows-1252. +- New heuristics, so that ftfy can fix more stuff, while maintaining + approximately zero false positives. +- The command-line tool trusts you to know what encoding your *input* is in, + and assumes UTF-8 by default. You can still tell it to guess with the `-g` + option. +- The command-line tool can be configured with options, and can be used as a + pipe. +- Recognizes characters that are new in Unicode 7.0. + +Deprecations: + +- `fix_text_encoding` has been renamed to simply `fix_encoding`, and the old + name emits a warning. It will disappear in ftfy 4.2 or 5.0. +- `remove_unsafe_private_use` has been removed entirely, after two versions + of deprecation. + +Pending deprecations: + +- Python 2.6 support is largely coincidental. +- Python 2.7 support is on notice. If you use Python 2, be sure to pin a + version of ftfy less than 5.0 in your requirements. + + ## Version 3.4.0 (January 15, 2015) New features: @@ -10,10 +46,12 @@ Deprecations: - `remove_unsafe_private_use` emits a warning, and will disappear in the next minor or major version. + ## Version 3.3.1 (December 12, 2014) This version restores compatibility with Python 2.6. + ## Version 3.3.0 (August 16, 2014) Heuristic changes: @@ -45,6 +83,7 @@ Deprecations: - `remove_unsafe_private_use` is no longer needed in any current version of Python. This fixer will disappear in a later version of ftfy. + ## Version 3.2.0 (June 27, 2014) - `fix_line_breaks` fixes three additional characters that are considered line @@ -55,15 +94,18 @@ Deprecations: U+2028 LINE SEPARATOR U+2029 PARAGRAPH SEPARATOR + ## Version 3.1.3 (May 15, 2014) - Fix `utf-8-variants` so it never outputs surrogate codepoints, even on Python 2 where that would otherwise be possible. + ## Version 3.1.2 (January 29, 2014) - Fix bug in 3.1.1 where strings with backslashes in them could never be fixed + ## Version 3.1.1 (January 29, 2014) - Add the `ftfy.bad_codecs` package, which registers new codecs that can @@ -76,6 +118,7 @@ Deprecations: - Simplify the code using `ftfy.bad_codecs`. + ## Version 3.0.6 (November 5, 2013) - `fix_entities` can now be True, False, or 'auto'. The new case is True, which @@ -84,29 +127,36 @@ Deprecations: - `build_data.py` will refuse to run on Python < 3.3, to prevent building an inconsistent data file. + ## Version 3.0.5 (November 1, 2013) - Fix the arguments to `fix_file`, because they were totally wrong. + ## Version 3.0.4 (October 1, 2013) - Restore compatibility with Python 2.6. + ## Version 3.0.3 (September 9, 2013) - Fixed an ugly regular expression bug that prevented ftfy from importing on a narrow build of Python. + ## Version 3.0.2 (September 4, 2013) - Fixed some false positives. + - Basically, 3.0.1 was too eager to treat text as MacRoman or cp437 when three consecutive characters coincidentally decoded as UTF-8. Increased the cost of those encodings so that they have to successfully decode multiple UTF-8 characters. - - See tests/test_real_tweets.py for the new test cases that were added as a + + - See `tests/test_real_tweets.py` for the new test cases that were added as a result. + ## Version 3.0.1 (August 30, 2013) - Fix bug in `fix_java_encoding` that led to only the first instance of @@ -114,6 +164,7 @@ Deprecations: - Add a fixer that removes unassigned characters that can break Python 3.3 (http://bugs.python.org/issue18183) + ## Version 3.0 (August 26, 2013) - Generally runs faster @@ -121,27 +172,30 @@ Deprecations: - Simplified decoding logic - Understands more encodings and more kinds of mistakes - Takes options that enable or disable particular normalization steps -- Long line handling: now the time-consuming step (fix_text_encoding) will be +- Long line handling: now the time-consuming step (`fix_text_encoding`) will be consistently skipped on long lines, but all other fixes will apply - Tested on millions of examples from Twitter, ensuring a near-zero rate of false positives + ## Version 2.0.2 (June 20, 2013) - Fix breaking up of long lines, so it can't go into an infinite loop + ## Version 2.0.1 (March 19, 2013) - Restored Python 2.6 support + ## Version 2.0 (January 30, 2013) - Python 3 support - Use fast Python built-ins to speed up fixes - Bugfixes + ## Version 1.0 (August 24, 2012) - Made into its own package with no dependencies, instead of a part of `metanl` - diff --git a/docs/accuracy.rst b/docs/accuracy.rst index ac12b499..98e25b2b 100644 --- a/docs/accuracy.rst +++ b/docs/accuracy.rst @@ -3,7 +3,7 @@ data. Twitter is massively multilingual, and despite that it's supposed to be uniformly UTF-8, in practice, any encoding mistake that someone can make will be made by someone's Twitter client. -This version of ftfy was evaluated on 13,000,000 tweets received from Twitter's +This version of ftfy was evaluated on 17,000,000 tweets received from Twitter's streaming API in April 2015. None of them were false positives. ftfy changes about 1 in every 8500 tweets. We sampled 1000 of these changes for diff --git a/ftfy/cli.py b/ftfy/cli.py index fd0b55d5..323653d9 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -60,14 +60,11 @@ def main(): encoding = args.encoding if args.guess: encoding = None - file_flags = 'rb' - else: - file_flags = 'r' if args.filename == '-': file = sys.stdin else: - file = open(args.filename, file_flags) + file = open(args.filename, 'rb') normalization = args.normalization if normalization.lower() == 'none': From e8a15b51e244d3d43a85c3bc79893ae86ab51892 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 10 Apr 2015 19:33:09 -0400 Subject: [PATCH 218/502] Simplify ENDING_PUNCT_RE It turns out that we do want to give a bonus to every instance of ending punctuation. When two of them are next to each other, that's twice as punctuationy and it should count. This is what fixes the Portuguese Hogwarts false positive. I made a change adding up the lengths of the sequences, but indeed, just removing the + makes each character into a separate match. --- ftfy/badness.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 333c7db2..b684045c 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -88,9 +88,6 @@ def _make_weirdness_regex(): # of a mojibake sequence. It's plausible that such a character could appear # after an accented capital letter, for example, so we'll want to add a # slight preference to leave these characters alone. -# -# The match ends with a + so that we only give the bonus once for a -# consecutive sequence of these characters. ENDING_PUNCT_RE = re.compile( '[' '\N{HORIZONTAL ELLIPSIS}\N{EM DASH}\N{EN DASH}' @@ -98,7 +95,7 @@ def _make_weirdness_regex(): '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' '\N{ACUTE ACCENT}' - ']+' + ']' ) def sequence_weirdness(text): @@ -130,7 +127,7 @@ def sequence_weirdness(text): """ text2 = unicodedata.normalize('NFC', text) weirdness = len(WEIRDNESS_RE.findall(chars_to_classes(text2))) - punct_discount = sum(len(x) for x in ENDING_PUNCT_RE.findall(text2)) + punct_discount = len(ENDING_PUNCT_RE.findall(text2)) return weirdness * 2 - punct_discount From 00dc1c3819f6a644c6fd2178aecfd47e975f11d5 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 10 Apr 2015 19:37:50 -0400 Subject: [PATCH 219/502] Minor code review fixes --- CHANGELOG.md | 4 ++-- ftfy/bad_codecs/utf8_variants.py | 2 +- ftfy/cli.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60c5de73..d9bd1f05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,13 +19,13 @@ New features: - The command-line tool can be configured with options, and can be used as a pipe. - Recognizes characters that are new in Unicode 7.0. +- `remove_unsafe_private_use` has been removed entirely, after two versions + of deprecation. Deprecations: - `fix_text_encoding` has been renamed to simply `fix_encoding`, and the old name emits a warning. It will disappear in ftfy 4.2 or 5.0. -- `remove_unsafe_private_use` has been removed entirely, after two versions - of deprecation. Pending deprecations: diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 933ce912..15677820 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -145,7 +145,7 @@ def _buffer_decode_step(self, input, errors, final): # Set final=True because 0xc0 and 0xed don't make sense in the # middle of a sequence, in any variant. return sup(input[:cutoff], errors, True) - + @staticmethod def _buffer_decode_null(sup, input, errors, final): """ diff --git a/ftfy/cli.py b/ftfy/cli.py index 323653d9..15558aa6 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -53,7 +53,8 @@ def main(): 'Defaults to NFC. Can be "none".') parser.add_argument('--preserve-entities', action='store_true', help="Leave HTML entities as they are. The default " - "is to decode them in non-HTML files.") + "is to decode them, as long as no HTML tags " + "have appeared in the file.") args = parser.parse_args() From d51d5c578d46bf2ac6218340b5ca1d80cecb4ea6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 13 Apr 2015 13:37:13 -0400 Subject: [PATCH 220/502] Fix missing steps in plan The plans output by `fix_one_step_and_explain` were missing the 'encode' steps, so they didn't take into account the cost of using a particular encoding, which actually led to a false positive involving a Ukrainian letter. --- ftfy/fixes.py | 4 +++- tests/test_real_tweets.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index d73a4a2e..b756b52a 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -195,7 +195,9 @@ def fix_one_step_and_explain(text): fixed_bytes, cost = restore_byte_a0(encoded_bytes) try: fixed = fixed_bytes.decode('utf-8-variants') - steps = [('transcode', 'restore_byte_a0', cost), + steps = [('encode', encoding, + ENCODING_COSTS.get(encoding, 0)), + ('transcode', 'restore_byte_a0', cost), ('decode', 'utf-8-variants', 0)] return fixed, steps except UnicodeDecodeError: diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 6cb20921..218ac228 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -44,6 +44,7 @@ "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), ('Engkau masih yg terindah, indah di dalam hatiku♫~', 'Engkau masih yg terindah, indah di dalam hatiku♫~'), + ('SENSЕ - Oleg Tsedryk', 'SENSЕ - Oleg Tsedryk'), # this Е is a Ukrainian letter ## Current false positives: #("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), From ce40a0dcaa7f3b8471e45792bcc727a5042375fd Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 14 Apr 2015 14:03:09 -0400 Subject: [PATCH 221/502] Clarify some docstrings based on code review --- ftfy/fixes.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index b756b52a..03b41661 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -135,8 +135,8 @@ def fix_encoding_and_explain(text): Re-decodes text that has been decoded incorrectly, and also return a "plan" indicating all the steps required to fix it. - To fix similar text in the same way, without having to detect anything, - you can use the :func:`ftfy.fixes.apply_plan` function. + The resulting plan could be used with :func:`ftfy.fixes.apply_plan` + to fix additional strings that are broken in the same way. """ best_version = text best_cost = text_cost(text) @@ -267,8 +267,9 @@ def apply_plan(text, plan): turns bytes into a string, and 'transcode' if it keeps the type the same. - `encoding` is the name of the encoding to use, such as 'utf-8' or 'latin-1', or the function name in the case of 'transcode'. - - `cost` is a penalty to apply to the score of the resulting text, when - performing a dubious operation that requires a lot of evidence. + - The `cost` does not affect how the plan itself works. It's used by other + users of plans, namely `fix_encoding_and_explain`, which has to decide + *which* plan to use. """ obj = text for operation, encoding, _ in plan: From ce54d288416f2838ceec52d474dd52339609a943 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 14 Apr 2015 14:03:27 -0400 Subject: [PATCH 222/502] Support outputting to a file; better errors With these changes, we give more specific advice to Windows users, and in particular we suggest that they output to a file. This way, Python can take care of the UTF-8 encoding, and we don't have to find and recommend a solution to the Unicode problems in cmd.exe. --- ftfy/cli.py | 54 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/ftfy/cli.py b/ftfy/cli.py index 15558aa6..6cdfb600 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -1,26 +1,33 @@ """ -A simple command-line utility for fixing text found in a file. - -Because files do not come with their encoding marked, it first runs the file -through `ftfy.guess_bytes`, then runs it through `ftfy.fix_text`. +A command-line utility for fixing text found in a file. """ from ftfy import fix_file, __version__ import sys +import io ENCODE_STDIO = (sys.hexversion < 0x03000000) -ENCODE_ERROR_TEXT = """ +ENCODE_ERROR_TEXT_UNIX = """ftfy error: +Unfortunately, this output stream does not support Unicode. + +Your system locale may be very old or misconfigured. You should use a locale +that supports UTF-8. One way to do this is to `export LANG=C.UTF-8`. +""" + +ENCODE_ERROR_TEXT_WINDOWS = """ftfy error: Unfortunately, this output stream does not support Unicode. -- If this is a Linux system, your locale may be very old or misconfigured. - You should `export LANG=C.UTF-8`. -- If this is a Windows system, please do not use the Command Prompt (cmd.exe) - to run Python. It does not and cannot support real Unicode. +You might be trying to output to the Windows Command Prompt (cmd.exe), which +does not fully support Unicode for historical reasons. In general, we recommend +finding a way to run Python without using cmd.exe. + +You can work around this problem by using the '-o filename' option in ftfy to +output to a file instead. """ -DECODE_ERROR_TEXT = """ -Error: this input couldn't be decoded as %r. We got the following error: +DECODE_ERROR_TEXT = """ftfy error: +This input couldn't be decoded as %r. We got the following error: %s @@ -32,8 +39,7 @@ def main(): """ - Run ftfy as a command-line utility. (Requires Python 2.7 or later, or - the 'argparse' module.) + Run ftfy as a command-line utility. """ import argparse @@ -43,6 +49,9 @@ def main(): parser.add_argument('filename', default='-', nargs='?', help='The file whose Unicode is to be fixed. Defaults ' 'to -, meaning standard input.') + parser.add_argument('-o', '--output', type=str, default='-', + help='The file to output to. Defaults to -, meaning ' + 'standard output.') parser.add_argument('-g', '--guess', action='store_true', help="Ask ftfy to guess the encoding of your input. " "This is risky. Overrides -e.") @@ -67,6 +76,14 @@ def main(): else: file = open(args.filename, 'rb') + encode_output = ENCODE_STDIO or (args.output != '-') + if args.output == '-': + encode_output = ENCODE_STDIO + outfile = sys.stdout + else: + encode_output = False + outfile = io.open(args.output, 'w', encoding='utf-8') + normalization = args.normalization if normalization.lower() == 'none': normalization = None @@ -80,13 +97,16 @@ def main(): for line in fix_file(file, encoding=encoding, fix_entities=fix_entities, normalization=normalization): - if ENCODE_STDIO: - sys.stdout.write(line.encode('utf-8')) + if encode_output: + outfile.write(line.encode('utf-8')) else: try: - sys.stdout.write(line) + outfile.write(line) except UnicodeEncodeError: - sys.stderr.write(ENCODE_ERROR_TEXT) + if sys.platform == 'win32': + sys.stderr.write(ENCODE_ERROR_TEXT_WINDOWS) + else: + sys.stderr.write(ENCODE_ERROR_TEXT_UNIX) sys.exit(1) except UnicodeDecodeError as err: sys.stderr.write(DECODE_ERROR_TEXT % (encoding, err)) From 97f4374537db0a95af7d7d5073993ad235670eb1 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 14 Apr 2015 14:06:51 -0400 Subject: [PATCH 223/502] update accuracy stats --- docs/accuracy.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/accuracy.rst b/docs/accuracy.rst index 98e25b2b..b0fbf88c 100644 --- a/docs/accuracy.rst +++ b/docs/accuracy.rst @@ -3,8 +3,12 @@ data. Twitter is massively multilingual, and despite that it's supposed to be uniformly UTF-8, in practice, any encoding mistake that someone can make will be made by someone's Twitter client. -This version of ftfy was evaluated on 17,000,000 tweets received from Twitter's -streaming API in April 2015. None of them were false positives. +We check what ftfy's :func:`fix_encoding` heuristic does to this data, and we +aim to have the rate of false positives be indistinguishable from zero. + +A pre-release version of ftfy was evaluated on 30,880,000 tweets received from +Twitter's streaming API in April 2015. There was 1 false positive, and it was +due to a bug that has now been fixed. ftfy changes about 1 in every 8500 tweets. We sampled 1000 of these changes for further evaluation, and found: @@ -13,8 +17,8 @@ further evaluation, and found: - 12 of them incompletely or incorrectly restored the text, when a sufficiently advanced heuristic might have been able to fully recover the text. - 8 of them represented text that had lost too much information to be fixed. -- None of them changed correct text to incorrect text (these would be false - positives). +- None of those 1000 changed correct text to incorrect text (these would be + false positives). In all the data we've sampled, including from previous versions of ftfy, there are only two false positives that remain that we know of:: From d52c4340a62802bcd0fcbd68516c5ac66fb10436 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 14 Apr 2015 14:58:26 -0400 Subject: [PATCH 224/502] Update function name used in the streamtester --- ftfy/streamtester/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/streamtester/__init__.py b/ftfy/streamtester/__init__.py index 4b5c0614..d4a752ff 100644 --- a/ftfy/streamtester/__init__.py +++ b/ftfy/streamtester/__init__.py @@ -3,7 +3,7 @@ in a stream. A concrete implementation of it is found in `twitter_tester.py`. """ from __future__ import print_function, unicode_literals -from ftfy.fixes import fix_text_encoding +from ftfy.fixes import fix_encoding from ftfy.chardata import possible_encoding @@ -24,7 +24,7 @@ def check_ftfy(self, text): """ self.count += 1 if not possible_encoding(text, 'ascii'): - fixed = fix_text_encoding(text) + fixed = fix_encoding(text) if text != fixed: # possibly filter common bots before printing print(u'\nText:\t{text}\nFixed:\t{fixed}\n'.format( From 8c8349b40756a16fc9137f709a5279cfb4179b75 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 15 Apr 2015 17:06:45 -0400 Subject: [PATCH 225/502] streamtester can optionally check the entire `fix_text` process --- ftfy/streamtester/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ftfy/streamtester/__init__.py b/ftfy/streamtester/__init__.py index d4a752ff..5778880e 100644 --- a/ftfy/streamtester/__init__.py +++ b/ftfy/streamtester/__init__.py @@ -3,7 +3,8 @@ in a stream. A concrete implementation of it is found in `twitter_tester.py`. """ from __future__ import print_function, unicode_literals -from ftfy.fixes import fix_encoding +from ftfy import fix_text +from ftfy.fixes import fix_encoding, unescape_html from ftfy.chardata import possible_encoding @@ -17,14 +18,18 @@ def __init__(self): self.num_fixed = 0 self.count = 0 - def check_ftfy(self, text): + def check_ftfy(self, text, encoding_only=True): """ Given a single text input, check whether `ftfy.fix_text_encoding` would change it. If so, display the change. """ self.count += 1 + text = unescape_html(text) if not possible_encoding(text, 'ascii'): - fixed = fix_encoding(text) + if encoding_only: + fixed = fix_encoding(text) + else: + fixed = fix_text(text, fix_character_width=False, uncurl_quotes=False) if text != fixed: # possibly filter common bots before printing print(u'\nText:\t{text}\nFixed:\t{fixed}\n'.format( From 50074e88ddeacb5e69aaecd1183d8245d1821270 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 15 Apr 2015 17:07:18 -0400 Subject: [PATCH 226/502] Revise changelog --- CHANGELOG.md | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9bd1f05..0b9e7647 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,34 +2,50 @@ Breaking changes: -- The default normalization is now NFC, not NFKC. +- The default normalization form is now NFC, not NFKC. NFKC replaces a large + number of characters with 'equivalent' characters, and some of these + replacements are useful, but some are not desirable to do by default. + - The `fix_text` function has some new options that perform more targeted - operations that are part of NFKC normalization, without requiring hitting all - your text with the huge mallet that is NFKC. + operations that are part of NFKC normalization, such as + `fix_character_width`, without requiring hitting all your text with the huge + mallet that is NFKC. + + - If you were already using NFC normalization, or in general if you want to + preserve the *spacing* of CJK text, you should be sure to set + `fix_character_width=False`. + +- The `remove_unsafe_private_use` parameter has been removed entirely, after + two versions of deprecation. The function name `fix_bad_encoding` is also + gone. New features: - Fixers for strange new forms of mojibake, including particularly clear cases of mixed UTF-8 and Windows-1252. + - New heuristics, so that ftfy can fix more stuff, while maintaining approximately zero false positives. + - The command-line tool trusts you to know what encoding your *input* is in, and assumes UTF-8 by default. You can still tell it to guess with the `-g` option. + - The command-line tool can be configured with options, and can be used as a pipe. + - Recognizes characters that are new in Unicode 7.0. -- `remove_unsafe_private_use` has been removed entirely, after two versions - of deprecation. Deprecations: -- `fix_text_encoding` has been renamed to simply `fix_encoding`, and the old - name emits a warning. It will disappear in ftfy 4.2 or 5.0. +- `fix_text_encoding` is being renamed again, for conciseness and consistency. + It's now simply called `fix_encoding`. The name `fix_text_encoding` is + available but emits a warning. Pending deprecations: - Python 2.6 support is largely coincidental. + - Python 2.7 support is on notice. If you use Python 2, be sure to pin a version of ftfy less than 5.0 in your requirements. @@ -89,7 +105,7 @@ Deprecations: - `fix_line_breaks` fixes three additional characters that are considered line breaks in some environments, such as Javascript, and Python's "codecs" library. These are all now replaced with \n: - + U+0085 , with alias "NEXT LINE" U+2028 LINE SEPARATOR U+2029 PARAGRAPH SEPARATOR From 9349b9740f893da95f146c2df972dbf3338b9c21 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 15 Apr 2015 17:08:30 -0400 Subject: [PATCH 227/502] Minor cleanups --- ftfy/__init__.py | 4 ++-- ftfy/cli.py | 1 - ftfy/compatibility.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index fd600fe7..2721bd8b 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -13,10 +13,8 @@ ftfy.bad_codecs.ok() from ftfy import fixes -from ftfy.fixes import fix_text_encoding from ftfy.compatibility import is_printable import unicodedata -import warnings __version__ = '4.0.0' @@ -185,8 +183,10 @@ def fix_text(text, return ''.join(out) +# Some alternate names for the main functions ftfy = fix_text fix_encoding = fixes.fix_encoding +fix_text_encoding = fixes.fix_text_encoding # deprecated def fix_file(input_file, diff --git a/ftfy/cli.py b/ftfy/cli.py index 6cdfb600..4aaccbe3 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -76,7 +76,6 @@ def main(): else: file = open(args.filename, 'rb') - encode_output = ENCODE_STDIO or (args.output != '-') if args.output == '-': encode_output = ENCODE_STDIO outfile = sys.stdout diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index 697f151c..b960c964 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -4,7 +4,6 @@ """ from __future__ import unicode_literals import sys -import re import unicodedata if sys.hexversion >= 0x03000000: From f420192218cfb4ce0baf59ed50986c224a182140 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 15 Apr 2015 17:09:14 -0400 Subject: [PATCH 228/502] Documentation updates, including guidance on choosing options --- docs/accuracy.rst | 12 ++++++++++-- docs/index.rst | 33 +++++++++++++++++++++++++++----- ftfy/bad_codecs/utf8_variants.py | 6 +++--- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/docs/accuracy.rst b/docs/accuracy.rst index b0fbf88c..c3015373 100644 --- a/docs/accuracy.rst +++ b/docs/accuracy.rst @@ -10,8 +10,15 @@ A pre-release version of ftfy was evaluated on 30,880,000 tweets received from Twitter's streaming API in April 2015. There was 1 false positive, and it was due to a bug that has now been fixed. -ftfy changes about 1 in every 8500 tweets. We sampled 1000 of these changes for -further evaluation, and found: +When looking at the changes ftfy makes, we found: + +- :func:`ftfy.fix_text`, with all default options, will change about 1 in 18 tweets. +- With stylistic changes (`fix_character_width` and `uncurl_quotes`) turned off, + :func:`ftfy.fix_text` will change about 1 in every 300 tweets. +- :func:`ftfy.fix_encoding` alone will change about 1 in every 8500 tweets. + +We sampled 1000 of these :func:`ftfy.fix_encoding` changes for further +evaluation, and found: - 980 of them correctly restored the text. - 12 of them incompletely or incorrectly restored the text, when a sufficiently @@ -25,3 +32,4 @@ are only two false positives that remain that we know of:: fix_encoding('├┤a┼┐a┼┐a┼┐a┼┐a') == 'ôaſaſaſaſa' fix_encoding('ESSE CARA AI QUEM É¿') == 'ESSE CARA AI QUEM ɿ' + diff --git a/docs/index.rst b/docs/index.rst index b50d328d..589418f3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -151,11 +151,29 @@ mojibake into the text. Using ftfy ---------- -The main function, :func:`ftfy.fix_text`, will run text through a sequence of fixes. If -the text changed, it will run them through again, so that you can be sure -the output ends up in a standard form that will be unchanged by :func:`ftfy.fix_text`. +The main function, :func:`ftfy.fix_text`, will run text through a sequence of +fixes. If the text changed, it will run them through again, so that you can be +sure the output ends up in a standard form that will be unchanged by +:func:`ftfy.fix_text`. All the fixes are on by default, but you can pass options to turn them off. +Check that the default fixes are appropriate for your use case. For example: + +- You should set `fix_entities` to False if the output is meant to be + interpreted as HTML. + +- You should set `fix_character_width` to False if you want to preserve the + spacing of CJK text. + +- You should set `uncurl_quotes` to False if you want to preserve quotation + marks with nice typography. You could even consider doing quite the opposite + of `uncurl_quotes`, running `smartypants`_ on the result to make all the + punctuation nice. + +.. _smartypants: http://pythonhosted.org/smartypants/ + +If the only fix you need is to detect and repair decoding errors (mojibake), then +you should use :func:`ftfy.fix_encoding` directly. .. versionchanged:: 4.0 The default normalization was changed from `'NFKC'` to `'NFC'`. The new options @@ -222,7 +240,8 @@ writes it to UTF-8 output, fixing problems in its Unicode as it goes. Here's the usage documentation for the `ftfy` command:: - usage: ftfy [-h] [-g] [-e ENCODING] [-n NORMALIZATION] [--preserve-entities] + usage: ftfy [-h] [-o OUTPUT] [-g] [-e ENCODING] [-n NORMALIZATION] + [--preserve-entities] [filename] ftfy (fixes text for you), version 4.0.0 @@ -233,6 +252,9 @@ Here's the usage documentation for the `ftfy` command:: optional arguments: -h, --help show this help message and exit + -o OUTPUT, --output OUTPUT + The file to output to. Defaults to -, meaning standard + output. -g, --guess Ask ftfy to guess the encoding of your input. This is risky. Overrides -e. -e ENCODING, --encoding ENCODING @@ -241,7 +263,8 @@ Here's the usage documentation for the `ftfy` command:: The normalization of Unicode to apply. Defaults to NFC. Can be "none". --preserve-entities Leave HTML entities as they are. The default is to - decode them in non-HTML files. + decode them, as long as no HTML tags have appeared in + the file. Module documentation diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 15677820..57e374cd 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -33,9 +33,9 @@ functionality in the sister module, "Breaks Text For You", coming approximately never. -[1] In a pinch, you can decode CESU-8 in Python 2 using the UTF-8 codec: first -decode the bytes (incorrectly), then encode them, then decode them again, using -UTF-8 as the codec every time. +.. [1] In a pinch, you can decode CESU-8 in Python 2 using the UTF-8 codec: + first decode the bytes (incorrectly), then encode them, then decode them + again, using UTF-8 as the codec every time. """ from __future__ import unicode_literals From 20b6698a7f2cc565bd240121fba586cf6c6f3bc5 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 15 Apr 2015 18:27:28 -0400 Subject: [PATCH 229/502] recognize unknown characters that are probably new emoji --- CHANGELOG.md | 3 ++- ftfy/build_data.py | 5 +++++ ftfy/char_classes.dat | Bin 3847 -> 3760 bytes tests/test_unicode.py | 20 ++++++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b9e7647..cfe663bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,8 @@ New features: - The command-line tool can be configured with options, and can be used as a pipe. -- Recognizes characters that are new in Unicode 7.0. +- Recognizes characters that are new in Unicode 7.0, as well as emoji from + Unicode 8.0+ that may already be in use on iOS. Deprecations: diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 0d64b785..940bb37d 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -91,6 +91,11 @@ def make_char_data_file(do_it_anyway=False): cclasses[codepoint] = '2' elif category == 'So': cclasses[codepoint] = '3' + elif 0x1f000 <= codepoint < 0x1f900: + # This range is rapidly having emoji added to it. Assume that + # an unassigned codepoint in this range is just a symbol we + # don't know yet. + cclasses[codepoint] = '3' elif category == 'Cn': cclasses[codepoint] = '_' elif category == 'Cc': diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index 47d8a9abf7fb3ff5cacb69ad9e8d0ff0e9756f73..cbc2ccda4290b86f390c701a5e2a81b86fb2fa93 100644 GIT binary patch literal 3760 zcmeHHX;c!37IvCRr^T#O?hCD^)?{jCqQX>D=9*>h+o+{a8(bj8g~g_fDJ{z+b0xDh z_Y@U(Qd3M4OGMl;l~7R$5ZMG?Gk@m2Ip_VFU+Y7hH z<+=OG7$6gm8tgyoUq?pc?|o0&@?Z$e%3RKz33+VA zUG_=XNtdK!y&60PMIz%_k*u{q#DVW#h8LV^HbRLwM23rYE{DzVL#`CAoX6AQS zPiHY}c!#e<@t2tA9Pd*u6sqwZhVI*vvyZ*ie7Qne)LWmwIkF$|YbC|eQ~j_DM<}W_ zLgZ!pWll=O+s1sAHFZL3c>t;dQzI#fIi}^1YnUjyQAB*}1`&mo=LPx%JC!kux{D1% zru9kVp7pi6dN=1Us1h#a--7Jbx2Nu@Gg|e3eEV~IIkwF|F67*F%_`>AXtp& zH^G)G6|s^j9;O9Y!Ui{E8yN52R8=W}1(8fQv`O|Feq?RAe5Wu<+-f;lp?LnO7&i2a zE=0_#K-&`gD9K7OEe)~C|kU%WsSykbN zAKj3mYoK07_SY9QxYkiFT6|WWAbvlji?z}uK#5bpOE_{)M1en=_4-^ux~O&FayTqa z_e{HG!y?9e>e{pVV+l1cJbI2E#zk+vh4jq?C^PN1F~iJ%X&#N>^i*gMC|A`zgA{cD z3n`T@q?`N7uH>kCpvdXbmGYGzz`-#aR{*JR&k@1^iR@){_PNI&r-Ayfp1Nk<$4qc- z%SG+Qw#5NvYOfj_!^*+!aG5>T8fDaB#n@(+x|$*~3l5Pw>5%TQ?pt54#GQQe*g5}& z+;^*NQqGtO^{QHDtEq*{KgY!;`pNq_ktYjZ993M36Kw3vk9nN_dIQ{w>s4 zlU^N|yNeZ@MK$X=i4f6rR09HNv+sT^6_=SK;5Atzq`h35(t10CyVN3jGar-`c*3Hv z1};0F>7VxKltQ3=ogngXE(qC-?Otm@s8egFL21GB7C{GaK=qJ{gQW;pbK4%j%Hbh_ zg{N~4PAkZ{0bxS}J6JX>?CGw?JoltKeEbe(>ATsxj3)COkFXr9;F@BpB??2ll+ke9 zJ=?Bv+?N6geAG+MHw3Mu{Tv>Yl<8F}gsfpUSOV$jZ(OxgPj0MJR7ja%+RcDoybD0x zcHfZTxSAnPCBYHxGeujk;=~b_vsG+3A99G@jU2j8DMRut0De1V_H&M^t>bGk>cM)9D1i^FzRcF*s zF;!o`ZFew>Wem~%Aw0}avC^nGp4s3*Fuvyf|ii0;t4>DELxa~)K>c4QU(fz zGC`};ra#bsyD!Ch3G&&iwu@6`EG$v%wY-pcD_NOu74-6CA_Dy` zG$ghmlD4q|N*>l#sSn=5vyK>tJco{n+6hg_Ldl2{OB8OELrcEZx^u8o3p z=_-E;5_M=J)O&le|?S6drUEgH7(8E@+SlR&s*@5FkU zZO2O~$htJB=HvE0jiH=Et9-lVcBy18jYe~8LoV+E5{wzI?b5LXH-o34Z`FNMB^dhY-U8g&@IsTeTkF`pIdw|<@a7K8tE6E z)K<0Occ0_6@yi6P&U9nArD&9au(0p%UXahg(z+8U!pp2EhMA8M>9bhz;2|w^**PTy zPN+wCL)y)xVkRJR$X5$Jx1<(5c!U{L_LL;`s6Dck*6|9Zak}XTz;mzQxrx^Sce({3 z`#_*4!f4&`)*ZP=d)z-;eugz!_&B3I{XR4^x5>aKa&r8q%le{D2aPK-Jhp7F!Z~tX zF$P^wUAtSQ7L~4+{N+m6ouB?vC}?Q~8X!x8erMT~g3gZm*_ar9cJVvyL;Y zo93tYf9KiQ61&Dfr)4DyMRI;qSu&{xM>d3oI|Hu`m&8XY~NKd??$e0-5nLK@^P#Jy`o zI~%}&5b>w=2SNAY;d(OzJ2G%@=!qZKaqXTl<;qG*9R4a*Z-$g0jM@C>;wdSiv2X=u zKk5}k79NA9)DQ(^-x6~269bN7sfJjkEwHYIt+tpV1UYpTSmaQWH(ATes{2s{nzow)-`=$9ry#_e(^@lw5a8JdxIZt^D zHhBEYK-;M)2g9}fi}p5JBjoLOUNUV8TrZY4f!}H(72?pfj$9->OerNhLCDoqYU}Pj zviuh%{x@tpKe{xf{rfOF%7~vOouA1@e<^bp1=_ANY>~|1S#6`1N^e uplj)Spwlu_{;K5bFZw&yQG@y?&Pn)SwoYFuh8(aaiAVbYSU94i%4Trx5jTyjCQM4FbdafxzY zN=tJ|5)w5RR4gU8)Rd4Eh15uK0Tq=U-t^|fyf>fTIrHhAGxx(i=bn4-@80vjH}28G zxN}rV*Lt=0gb&-VM6SR4$`}fjy9f?9N8FMLz(PBtAToiS(Yxcp=d*{q{;pCYWUB;m zo;WLD1PTKMd|U1Yo)Z<@b=cUd>MQPD%l~AgzQiwTZIvSYwj%sOg>Du=DHC82Kd%K^xw_c(t#DB) zLR6y;gqGSH&^!-jW(Dv%Y>pN3R{~3?2j7e>(2_x?2XaW;hMJFz7=P}cK9kM#@AC5q z)4a3hc7oxP9NU-)$Y$}E(Uq)lYD-w-ctN^vpMG3gGR7i%2LAEI)5<{yue67I$%PJv zM~u3m9~lK?xrgG;DB828`P6fUUboO6>@0m4C>hEUpAHs9f|EAHKdIUT*n=XcGic6q zR+apsPHS(kwKK6@emB(Ywpm)Wmqf%XjoP~5_VNj*^fG5dun!LyP+^{6%Wx%CzO4JO z{C>jM%2IG2i84zLy%W4H@OnFX$8?b1sq(?vy9bjgAB>)2Nho6=Jmcc|4sveT_*i9= zM!@=Z-{qwx&z{cwrYHwE$&z;r62A{U8zyz>anUW1$MABj>Gxr`SLRL>4*{~N?@nbY z8!)N8McA{po&D0HG7br39`6Bw2rx=gpFi{x?S#^=WOu9n!iA%Z84E4tAaK^u|gX`t{?r zHWpcYZ}r{qqdWMn{-@(J{ACU?X%1zx+9%$aW5SA-<0yafsY-V%hyE4bd!fLOHpWb*a`qV4*q=`+z+o zm^|hNFxFwwhoiSx?jG$usCs{C3`qxI3)#l#9)>uWJ(hF zt~qW2F9O16d9jLwrnA)=fJC&3Biyeg?&$mA_)2vS$E>u^@|5JU+o%CPlkWe%Q3#e+ zhxprZ_|ht)MxAdXm2-p_6sCS>h&{E2JvDBvNjVpsx7R8$lBF)^=^yh&HRCwX9LD;lH|nK+cw z|Bl6+aDquiYbMvuc}pR`KFKaRZS(c?HR@Y{At6Q^jXU7_uGd216|WCHb`MSaC1I(P z#WYQXccp)GiOoXoLa73VtN ztM+d@abrxx6qs-RE&4CthL8FrNvT)N;sdREI&;fLv;jb=P)AE-Ffdp6 zH(TnB5s}dslw{Kgo1R3kdmr2+PWb$o%ytV+(mDhlIHNZ$obepOfKxzWh6jA?wI~IC z#yYT}{f1P7WSrImk&4t59==MQyWo>^h!J|F&kYVq$Ff)r*7>Ppnzf6c_YQ|um*Gnb z!Anhj?dS%lv&kt>ztm(vsa_92ILuC*ct1bpq7p={=idx>XN%Vm6-y`*N2f-{8(@Sy z&Y0|G^-6!v#LE+yTgitJa)n`x(28`P8_)o&@eKVR0nWi6ho_%g{af^!HFhY3<- z0eh>uYkSumXTnR>+%JZeLc=$AA!9_ASa&WUZ7;E23kl>;fLd*5r3$ws%tapy_2`}- z%iX%bmAeV=*oZS)>AT9i=Ajvb5A8Ufv7VVp-v>@}WjI~qqA^4)r)rg6^~qndv5YZ< z$J1>j;!V!Jxo!gR$jQc(?&DX?c<%=E1`I`PtYV3lAF`hf7Hw)_WkN%>+!%*4q065B zaVp_HWcTI1J{Fp}!ir58c*St&^s0+SM0ECv3$CR|!s-`7DRxzS!ydi6zcfLxkV550 z17v)hUUCaB(p!Rb5y0g^OEx3B62r&O%}2E6earQa!Y!84{6W(-arHtGyQw( zBPpp#a}0-sM52JRZ44&G3hcTdlP!0Aa^& z7C3A2v8MeUE+~GZYsAIJs4uh<@i?BE=UP)^4Ni;Rk*AzF{Bbb`{GGzrd$lC#{;q$dVFFf^s0;+Txd?Wcpy^}sN~frw~IH+L41uhrbaUckj8 ziBkoXy_fv+>o=ERHZ@A19&?si zr%t~cF`Y$kx$g@>9RwgY&^J zTCqoZSp5%u*OD+C|C@sO@fPhrWc~{OQsDnZ0kJMQjpn+R?_f?*1h9QLe{SzK(W=&; YBdE!>-%z6eB%=+p?Tu<(YmF)RUq&l3mH+?% diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 21b82a41..455973be 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -74,6 +74,26 @@ def test_mixed_utf8(): eq_(fix_encoding('“mismatched quotes…”'), '“mismatched quotes…”') +def test_unknown_emoji(): + # Make a string with two burritos in it. Python doesn't know about Unicode + # burritos, but ftfy can guess they're probably emoji anyway. + emoji_text = 'dos burritos: \U0001f32f\U0001f32f' + + # Mangle the burritos into a mess of Russian characters. (It would have + # been great if we could have decoded them in cp437 instead, to turn them + # into "DOS burritos", but the resulting string is one ftfy could already + # fix.) + emojibake = emoji_text.encode('utf-8').decode('windows-1251') + + # Restore the original text. + eq_(fix_encoding(emojibake), emoji_text) + + # This doesn't happen if we replace the burritos with arbitrary unassigned + # characters. The mangled text passes through as is. + not_emoji = 'dos burritos: \U0003f32f\U0003f32f'.encode('utf-8').decode('windows-1251') + eq_(fix_encoding(not_emoji), not_emoji) + + def test_surrogates(): eq_(fix_surrogates('\udbff\udfff'), '\U0010ffff') eq_(fix_surrogates('\ud800\udc00'), '\U00010000') From 159805e131616bc723c476af1ba17fec51336fd3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Apr 2015 11:29:48 -0400 Subject: [PATCH 230/502] from e-mail feedback: warn when passing non-bytes to guess_bytes --- ftfy/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 2721bd8b..683a6201 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -324,6 +324,12 @@ def guess_bytes(bstring): - "sloppy-windows-1252", the Latin-1-like encoding that is the most common single-byte encoding """ + if type(bstring) == type(''): + raise UnicodeError( + "This string was already decoded as Unicode. You should pass " + "bytes to guess_bytes, not Unicode." + ) + if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): return bstring.decode('utf-16'), 'utf-16' From cd37c735f54ab0fab955d886484d680ef3e3971b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 21 Apr 2015 13:46:23 -0400 Subject: [PATCH 231/502] begin the README with a fix_encoding example --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f91de30a..1fb4eee6 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,11 @@ [![Version](https://pypip.in/v/ftfy/badge.png)](https://crate.io/packages/ftfy) [![Docs](https://readthedocs.org/projects/ftfy/badge/?version=latest)](http://ftfy.readthedocs.org/en/latest/) +```python +>>> print(fix_encoding("(ง'̀⌣'Ì\x81)ง")) +(ง'̀⌣'́)ง +``` + Full documentation: **http://ftfy.readthedocs.org** ## Testimonials @@ -69,9 +74,11 @@ low rate of false positives. ## Examples `fix_text` is the main function of ftfy. This section is meant to give you a -taste of the things it can do. Please read [the -documentation](http://ftfy.readthedocs.org) for more information on what ftfy -does, and how to configure it for your needs. +taste of the things it can do. `fix_encoding` is the more specific function +that only fixes mojibake. + +Please read [the documentation](http://ftfy.readthedocs.org) for more +information on what ftfy does, and how to configure it for your needs. ```python From e656bc88ffacbd153eeb1e50772174690de602b1 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 23 Apr 2015 19:31:30 -0400 Subject: [PATCH 232/502] revert: ACUTE ACCENT can't be ending punctuation without breaking things --- ftfy/badness.py | 1 - tests/test_real_tweets.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index b684045c..0ac55f04 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -94,7 +94,6 @@ def _make_weirdness_regex(): '\N{RIGHT SINGLE QUOTATION MARK}\N{RIGHT DOUBLE QUOTATION MARK}' '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' - '\N{ACUTE ACCENT}' ']' ) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 218ac228..452c1289 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -40,8 +40,6 @@ 'eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías'), ("Direzione Pd, ok “senza modifiche” all'Italicum.", "Direzione Pd, ok \"senza modifiche\" all'Italicum."), - ("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", - "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), ('Engkau masih yg terindah, indah di dalam hatiku♫~', 'Engkau masih yg terindah, indah di dalam hatiku♫~'), ('SENSЕ - Oleg Tsedryk', 'SENSЕ - Oleg Tsedryk'), # this Е is a Ukrainian letter @@ -49,6 +47,7 @@ ## Current false positives: #("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), #("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿") + #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), ## This kind of tweet can't be fixed without a full-blown encoding detector. #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", From 72620f9ba370a671d88ca4f2b140c28a05254ebb Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 24 Apr 2015 17:45:54 -0400 Subject: [PATCH 233/502] Fix link to Luminoso careers page --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b2d5e611..99348b42 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ is making sure it has the correct characters in it! Luminoso is growing fast and hiring in all areas. If you're interested in joining us, take a look at [our careers -page](http://www.luminoso.com/careers.html). +page](http://www.luminoso.com/career.html). ## What it does From c76ac27135dae24f82c53ccfee0796001118ca5d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 8 May 2015 14:16:29 -0400 Subject: [PATCH 234/502] revert a left-over change from the 'sloppy-utf-8' approach At one point, in the utf-8-variants decoder, I stopped passing on the 'final' argument and simply overrode it to True. I think the point of this was to avoid an infinite loop when I was trying to reuse the decoder in something called 'sloppy-utf-8', but this turned out to just be a bad idea in many ways. I really should be passing on final here, in case the input is broken in the middle of a standard UTF-8 sequence. --- ftfy/bad_codecs/utf8_variants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 57e374cd..65ffbc15 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -131,8 +131,8 @@ def _buffer_decode_step(self, input, errors, final): elif cutoff2 != -1: cutoff = cutoff2 else: - # Decode the entire input at once. - return sup(input, errors, True) + # Decode the input as standard UTF-8. + return sup(input, errors, final) if cutoff1 == 0: # Decode a possible six-byte sequence starting with 0xed. From f53a2f0c22c3e218479f53402e893cd0d677046e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 11 May 2015 14:54:41 -0400 Subject: [PATCH 235/502] update docs --- docs/accuracy.rst | 5 ++++- docs/conf.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/accuracy.rst b/docs/accuracy.rst index c3015373..6d4c9f1c 100644 --- a/docs/accuracy.rst +++ b/docs/accuracy.rst @@ -28,8 +28,11 @@ evaluation, and found: false positives). In all the data we've sampled, including from previous versions of ftfy, there -are only two false positives that remain that we know of:: +are only three false positives that remain that we know of:: fix_encoding('├┤a┼┐a┼┐a┼┐a┼┐a') == 'ôaſaſaſaſa' fix_encoding('ESSE CARA AI QUEM É¿') == 'ESSE CARA AI QUEM ɿ' + fix_encoding('``hogwarts nao existe, voce nao vai pegar o trem pra lá´´') + == '``hogwarts nao existe, voce nao vai pegar o trem pra lᴴ' + diff --git a/docs/conf.py b/docs/conf.py index 9c8fbf27..f5188f43 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -94,7 +94,6 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -#html_theme = 'alabaster' html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme From 9dd86088baa3602835adb3db9361cacce183eed7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 28 May 2015 04:56:52 -0400 Subject: [PATCH 236/502] README fixes: pypip.in is gone Also make the leading example into one that renders consistently, despite that it's not the one I found in the wild. Apparently systems are inconsistent about how they render apostrophes with combining accents. --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9b95ddeb..347cc42d 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,11 @@ # ftfy: fixes text for you -[![Downloads](https://pypip.in/d/ftfy/badge.png)](https://crate.io/packages/ftfy) -[![Version](https://pypip.in/v/ftfy/badge.png)](https://crate.io/packages/ftfy) +[![PyPI package](https://badge.fury.io/py/ftfy.svg)](http://badge.fury.io/py/ftfy) [![Docs](https://readthedocs.org/projects/ftfy/badge/?version=latest)](http://ftfy.readthedocs.org/en/latest/) ```python ->>> print(fix_encoding("(ง'̀⌣'Ì\x81)ง")) -(ง'̀⌣'́)ง +>>> print(fix_encoding("(ง'⌣')ง")) +(ง'⌣')ง ``` Full documentation: **http://ftfy.readthedocs.org** From 9414fe435bd649af72b820ec67ef361cc4267c8e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 29 May 2015 11:29:39 -0400 Subject: [PATCH 237/502] Explain `restore_byte_a0` better. Fixes #38. This is a change to a docstring only. --- ftfy/fixes.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 03b41661..bd964a33 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -565,9 +565,17 @@ def decode_match(match): def restore_byte_a0(byts): """ - Find sequences that would convincingly decode as UTF-8 if the byte 0x20 - were changed to 0xa0, and fix them. This is used as a step within - `fix_encoding`. + Some mojibake has been additionally altered by a process that said "hmm, + \\xa0, that's basically a space!" and replaced it with an ASCII space. + When the \\xa0 is part of a sequence that we intend to decode as UTF-8, + changing byte A0 to 20 would make it fail to decode. + + This process finds sequences that would convincingly decode as UTF-8 if + byte 20 were changed to A0, and puts back the A0. For the purpose of + deciding whether this is a good idea, this step gets a cost of twice + the number of bytes that are changed. + + This is used as a step within `fix_encoding`. """ def replacement(match): "The function to apply when this regex matches." From 7ac9204dfcc76a37cfdebc61a38338be3ebb9d55 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 22 Jun 2015 01:21:04 -0400 Subject: [PATCH 238/502] a utility to justify Unicode text for display --- ftfy/__init__.py | 9 +-- ftfy/formatting.py | 141 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 ftfy/formatting.py diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 683a6201..ab7e7b53 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -13,10 +13,11 @@ ftfy.bad_codecs.ok() from ftfy import fixes +from ftfy.formatting import display_ljust from ftfy.compatibility import is_printable import unicodedata -__version__ = '4.0.0' +__version__ = '4.1.0' def fix_text(text, @@ -386,7 +387,7 @@ def explain_unicode(text): U+00B0 ° [So] DEGREE SIGN U+0029 ) [Pe] RIGHT PARENTHESIS U+256F ╯ [So] BOX DRAWINGS LIGHT ARC UP AND LEFT - U+FE35 ︵ [Ps] PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS + U+FE35 ︵ [Ps] PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS U+0020 [Zs] SPACE U+253B ┻ [So] BOX DRAWINGS HEAVY UP AND HORIZONTAL U+2501 ━ [So] BOX DRAWINGS HEAVY HORIZONTAL @@ -397,8 +398,8 @@ def explain_unicode(text): display = char else: display = char.encode('unicode-escape').decode('ascii') - print('U+{code:04X} {display:<7} [{category}] {name}'.format( - display=display, + print('U+{code:04X} {display} [{category}] {name}'.format( + display=display_ljust(display, 7), code=ord(char), category=unicodedata.category(char), name=unicodedata.name(char, '') diff --git a/ftfy/formatting.py b/ftfy/formatting.py new file mode 100644 index 00000000..d7370a15 --- /dev/null +++ b/ftfy/formatting.py @@ -0,0 +1,141 @@ +from unicodedata import east_asian_width, combining, category, normalize + + +def character_width(char): + """ + Determine the width that a character is likely to be displayed as in + a monospaced terminal. The width will always be 0, 1, or 2. + + We are assuming that the character will appear in a modern, Unicode-aware + terminal emulator, where CJK characters -- plus characters designated + "fullwidth" by CJK encodings -- will span two character cells. (Maybe + it's an over-simplification to call these displays "monospaced".) + + When a character width is "Ambiguous", we assume it to span one character + cell, because we assume that the monospaced display you're using is + designed mostly to display Western characters with CJK characters as the + exception. This assumption will go wrong, for example, if your display is + actually on a Japanese flip-phone... but let's assume it isn't. + + If this character is a control code, the idea of it having a "width" + probably doesn't even apply, because it will probably have some other + effect on your terminal. For example, there's no sensible width for a + line break. We return the default of 1 in the absence of anything sensible + to do there. + + >>> character_width('車') + 2 + >>> character_width('A') + 1 + >>> character_width('\N{SOFT HYPHEN}') + 0 + """ + if combining(char) != 0: + # Combining characters don't advance the cursor; they modify the + # previous character instead. + return 0 + elif east_asian_width(char) in 'FW': + # Characters designated Wide (W) or Fullwidth (F) will take up two + # columns in many Unicode-aware terminal emulators. + return 2 + elif category(char) == 'Cf': + # Characters in category Cf are un-printable formatting characters + # that do not advance the cursor, such as zero-width spaces or + # right-to-left marks. + return 0 + else: + return 1 + + +def monospaced_width(text): + """ + Return the number of character cells that this string is likely to occupy + when displayed in a monospaced, modern, Unicode-aware terminal emulator. + We refer to this as the "display width" of the string. + + This can be useful for formatting text that may contain non-spacing + characters, or CJK characters that take up two character cells. + + >>> monospaced_width('ちゃぶ台返し') + 12 + >>> len('ちゃぶ台返し') + 6 + """ + # NFC-normalize the text first, so that we don't need special cases for + # Hangul jamo. + text = normalize('NFC', text) + return sum([character_width(char) for char in text]) + + +def display_ljust(text, width, fillchar=' '): + """ + Return `text` left-justified in a Unicode string whose display width, + in a monospaced terminal, should be at least `width` character cells. + The rest of the string will be padded with `fillchar`, which must be + a width-1 character. + + "Left" here means toward the beginning of the string, which may actually + appear on the right in an RTL context. This is similar to the use of the + word "left" in "left parenthesis". + + >>> lines = ['Table flip', '(╯°□°)╯︵ ┻━┻', 'ちゃぶ台返し'] + >>> for line in lines: + ... print(display_ljust(line, 20, '▒')) + Table flip▒▒▒▒▒▒▒▒▒▒ + (╯°□°)╯︵ ┻━┻▒▒▒▒▒▒▒ + ちゃぶ台返し▒▒▒▒▒▒▒▒ + """ + if character_width(fillchar) != 1: + raise ValueError("The padding character must have display width 1") + text_width = monospaced_width(text) + padding = max(0, width - text_width) + return text + fillchar * padding + + +def display_rjust(text, width, fillchar=' '): + """ + Return `text` right-justified in a Unicode string whose display width, + in a monospaced terminal, should be at least `width` character cells. + The rest of the string will be padded with `fillchar`, which must be + a width-1 character. + + "Right" here means toward the end of the string, which may actually be on + the left in an RTL context. This is similar to the use of the word "right" + in "right parenthesis". + + >>> lines = ['Table flip', '(╯°□°)╯︵ ┻━┻', 'ちゃぶ台返し'] + >>> for line in lines: + ... print(display_rjust(line, 20, '▒')) + ▒▒▒▒▒▒▒▒▒▒Table flip + ▒▒▒▒▒▒▒(╯°□°)╯︵ ┻━┻ + ▒▒▒▒▒▒▒▒ちゃぶ台返し + """ + if character_width(fillchar) != 1: + raise ValueError("The padding character must have display width 1") + text_width = monospaced_width(text) + padding = max(0, width - text_width) + return fillchar * padding + text + + +def display_center(text, width, fillchar=' '): + """ + Return `text` centered in a Unicode string whose display width, in a + monospaced terminal, should be at least `width` character cells. The rest + of the string will be padded with `fillchar`, which must be a width-1 + character. + + >>> lines = ['Table flip', '(╯°□°)╯︵ ┻━┻', 'ちゃぶ台返し'] + >>> for line in lines: + ... print(display_center(line, 20, '▒')) + ▒▒▒▒▒Table flip▒▒▒▒▒ + ▒▒▒(╯°□°)╯︵ ┻━┻▒▒▒▒ + ▒▒▒▒ちゃぶ台返し▒▒▒▒ + """ + if character_width(fillchar) != 1: + raise ValueError("The padding character must have display width 1") + text_width = monospaced_width(text) + padding = max(0, width - text_width) + left_padding = padding // 2 + right_padding = padding - left_padding + return fillchar * left_padding + text + fillchar * right_padding + From 30267b825869cb340da60ff78c1466f6e00338b0 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 22 Jun 2015 11:59:55 -0400 Subject: [PATCH 239/502] don't prematurely bump the version --- ftfy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index ab7e7b53..5b3f1e3c 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -17,7 +17,7 @@ from ftfy.compatibility import is_printable import unicodedata -__version__ = '4.1.0' +__version__ = '4.0.0' def fix_text(text, From f7fd15c62913bbeb1a22bfe25e85f073f04069ab Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 10 Aug 2015 14:09:23 -0400 Subject: [PATCH 240/502] show safer representations in streamtester --- ftfy/streamtester/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/streamtester/__init__.py b/ftfy/streamtester/__init__.py index 5778880e..61fd14a6 100644 --- a/ftfy/streamtester/__init__.py +++ b/ftfy/streamtester/__init__.py @@ -29,10 +29,10 @@ def check_ftfy(self, text, encoding_only=True): if encoding_only: fixed = fix_encoding(text) else: - fixed = fix_text(text, fix_character_width=False, uncurl_quotes=False) + fixed = fix_text(text, uncurl_quotes=False, fix_character_width=False) if text != fixed: # possibly filter common bots before printing - print(u'\nText:\t{text}\nFixed:\t{fixed}\n'.format( + print(u'\nText:\t{text!r}\nFixed:\t{fixed!r}\n'.format( text=text, fixed=fixed )) self.num_fixed += 1 From 3ec0cb5bb67751b7df0e531645827a2a88bff8e3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 12 Aug 2015 14:48:29 -0400 Subject: [PATCH 241/502] Split the vaguely-named 'test_unicode' The two new test modules are 'test_characters', for testing what happens to individual codepoints or characters, and 'test_synthetic', for testing particular made-up examples. --- tests/test_characters.py | 39 ++++++++++++++++++++ tests/{test_unicode.py => test_synthetic.py} | 0 2 files changed, 39 insertions(+) create mode 100644 tests/test_characters.py rename tests/{test_unicode.py => test_synthetic.py} (100%) diff --git a/tests/test_characters.py b/tests/test_characters.py new file mode 100644 index 00000000..00a18314 --- /dev/null +++ b/tests/test_characters.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from ftfy.fixes import fix_encoding_and_explain, apply_plan, possible_encoding, fix_surrogates +from ftfy.badness import sequence_weirdness +import unicodedata +import sys +from nose.tools import eq_ + +if sys.hexversion >= 0x03000000: + unichr = chr + + +# Most single-character strings which have been misencoded should be restored. +def test_bmp_characters(): + for index in range(0xa0, 0xfffd): + char = unichr(index) + # Exclude code points that are not assigned + if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): + garble = char.encode('utf-8').decode('latin-1') + # Exclude characters whose re-encoding is protected by the + # 'sequence_weirdness' metric + if sequence_weirdness(garble) >= 0: + garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') + for garb in (garble, garble2): + fixed, plan = fix_encoding_and_explain(garb) + eq_(fixed, char) + eq_(apply_plan(garb, plan), char) + + +def test_possible_encoding(): + for codept in range(256): + char = chr(codept) + assert possible_encoding(char, 'latin-1') + + +def test_surrogates(): + eq_(fix_surrogates('\udbff\udfff'), '\U0010ffff') + eq_(fix_surrogates('\ud800\udc00'), '\U00010000') + diff --git a/tests/test_unicode.py b/tests/test_synthetic.py similarity index 100% rename from tests/test_unicode.py rename to tests/test_synthetic.py From 69e4c6cec7df1bca9c923662569f524001581e11 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 12 Aug 2015 14:49:48 -0400 Subject: [PATCH 242/502] sloppy codecs allow '\ufffd' to be encoded as b'\x1a' --- ftfy/bad_codecs/sloppy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index d9d6c687..ce5860a9 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -107,6 +107,10 @@ def make_sloppy_codec(encoding): if char != REPLACEMENT_CHAR: sloppy_chars[i] = char + # For ftfy's own purposes, we're going to allow byte 1A, the "Substitute" + # control code, to encode the Unicode replacement character U+FFFD. + sloppy_chars[0x1a] = REPLACEMENT_CHAR + # Create the data structures that tell the charmap methods how to encode # and decode in this sloppy encoding. decoding_table = ''.join(sloppy_chars) From e896ddf389f57a8fc10f7d42af0f086751b989e9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 12 Aug 2015 14:50:38 -0400 Subject: [PATCH 243/502] Use a regex to match 'utf-8-variants' sequences more cleanly --- ftfy/bad_codecs/utf8_variants.py | 92 +++++++++++++------------------- 1 file changed, 38 insertions(+), 54 deletions(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 65ffbc15..7fbeae0f 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -46,8 +46,29 @@ import codecs NAME = 'utf-8-variants' -# This regular expression matches all possible six-byte CESU-8 sequences. -CESU8_RE = re.compile(b'\xed[\xa0-\xaf][\x80-\xbf]\xed[\xb0-\xbf][\x80-\xbf]') + +# This regular expression matches all possible six-byte CESU-8 sequences, +# plus truncations of them at the end of the string. +CESU8_EXPR = ( + b'(\xed[\xa0-\xaf][\x80-\xbf]\xed[\xb0-\xbf][\x80-\xbf]' + b'|\xed([\xa0-\xaf]([\x80-\xbf](\xed[\xb0-\xbf]?)?)?)?$' + b')' +) +CESU8_RE = re.compile(CESU8_EXPR) + +# This expression matches isolated surrogate characters that aren't +# CESU-8, which have to be handled carefully on Python 2. +SURROGATE_EXPR = (b'(\xed[\xa0-\xbf][\x80-\xbf])') + +# This expression matches the Java encoding of U+0. We don't need to check +# for its truncated version, because that will get passed on to the standard +# UTF-8 decoder, which will agree that we need to see more bytes. +NULL_EXPR = b'(\xc0\x80)' + + +# This regex matches cases that we need to decode differently from +# standard UTF-8. +SPECIAL_BYTES_RE = re.compile(b'|'.join([NULL_EXPR, CESU8_EXPR, SURROGATE_EXPR])) class IncrementalDecoder(UTF8IncrementalDecoder): @@ -118,58 +139,22 @@ def _buffer_decode_step(self, input, errors, final): sup = UTF8IncrementalDecoder._buffer_decode # Find the next byte position that indicates a variant of UTF-8. - # CESU-8 sequences always start with 0xed, and Java nulls always - # start with 0xc0. - cutoff1 = input.find(b'\xed') - cutoff2 = input.find(b'\xc0') - - # Set `cutoff` to whichever cutoff comes first. - if cutoff1 != -1 and cutoff2 != -1: - cutoff = min(cutoff1, cutoff2) - elif cutoff1 != -1: - cutoff = cutoff1 - elif cutoff2 != -1: - cutoff = cutoff2 - else: - # Decode the input as standard UTF-8. + match = SPECIAL_BYTES_RE.search(input) + if match is None: return sup(input, errors, final) - if cutoff1 == 0: - # Decode a possible six-byte sequence starting with 0xed. - return self._buffer_decode_surrogates(sup, input, errors, final) - elif cutoff2 == 0: - # Decode a possible two-byte sequence, 0xc0 0x80. - return self._buffer_decode_null(sup, input, errors, final) - else: - # Decode the bytes up until the next weird thing as UTF-8. - # Set final=True because 0xc0 and 0xed don't make sense in the - # middle of a sequence, in any variant. + cutoff = match.start() + if cutoff > 0: return sup(input[:cutoff], errors, True) - @staticmethod - def _buffer_decode_null(sup, input, errors, final): - """ - Decode the bytes 0xc0 0x80 as U+0000, like Java does. - """ - nextbyte = input[1:2] - if nextbyte == b'': - if final: - # We found 0xc0 at the end of the stream, which is an error. - # Delegate to the superclass method to handle that error. - return sup(input, errors, final) - else: - # We found 0xc0 and we don't know what comes next, so consume - # no bytes and wait. - return '', 0 - elif nextbyte == b'\x80': - # We found the usual 0xc0 0x80 sequence, so decode it and consume - # two bytes. + # Some byte sequence that we intend to handle specially matches + # at the beginning of the input. + if input.startswith(b'\xc0'): + # Decode the two-byte sequence 0xc0 0x80. return '\u0000', 2 else: - # We found 0xc0 followed by something else, which is an error. - # Whatever should happen is equivalent to what happens when the - # superclass is given just the byte 0xc0, with final=True. - return sup(b'\xc0', errors, True) + # Decode a possible six-byte sequence starting with 0xed. + return self._buffer_decode_surrogates(sup, input, errors, final) @staticmethod def _buffer_decode_surrogates(sup, input, errors, final): @@ -190,21 +175,20 @@ def _buffer_decode_surrogates(sup, input, errors, final): if final: # We found 0xed near the end of the stream, and there aren't # six bytes to decode. Delegate to the superclass method to - # handle it as normal UTF-8. It might be a Hangul character - # or an error. + # handle it as an error. if PYTHON2 and len(input) >= 3: # We can't trust Python 2 to raise an error when it's # asked to decode a surrogate, so let's force the issue. input = mangle_surrogates(input) return sup(input, errors, final) else: - # We found 0xed, the stream isn't over yet, and we don't know - # enough of the following bytes to decode anything, so consume - # zero bytes and wait. + # We found a surrogate, the stream isn't over yet, and we don't + # know enough of the following bytes to decode anything, so + # consume zero bytes and wait. return '', 0 else: if CESU8_RE.match(input): - # If this is a CESU-8 sequence, do some math to pull out + # Given this is a CESU-8 sequence, do some math to pull out # the intended 20-bit value, and consume six bytes. bytenums = bytes_to_ints(input[:6]) codepoint = ( From 54c8c7d7537190ef4558108612aa0b37fa52fcb4 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 12 Aug 2015 14:51:55 -0400 Subject: [PATCH 244/502] Enable sorta-decoding UTF-8 when bytes have been lost --- ftfy/chardata.py | 20 +++++++++++++-- ftfy/fixes.py | 67 ++++++++++++++++++++++++++++++++---------------- 2 files changed, 63 insertions(+), 24 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index aeed8fd1..91f9e345 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -33,14 +33,14 @@ def _build_regexes(): encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')} for encoding in CHARMAP_ENCODINGS: - latin1table = ''.join(unichr(i) for i in range(128, 256)) + latin1table = ''.join(unichr(i) for i in range(128, 256)) + '\x1a' charlist = latin1table.encode('latin-1').decode(encoding) # Build a regex from the ASCII range, followed by the decodings of # bytes 0x80-0xff in this character set. (This uses the fact that all # regex special characters are ASCII, and therefore won't appear in the # string.) - regex = '^[\x00-\x7f{0}]*$'.format(charlist) + regex = '^[\x00-\x19\x1b-\x7f{0}]*$'.format(charlist) encoding_regexes[encoding] = re.compile(regex) return encoding_regexes ENCODING_REGEXES = _build_regexes() @@ -101,6 +101,22 @@ def _build_utf8_punct_regex(): b'|[\xf0-\xf4][\x80-\xbf][ ][\x80-\xbf]' b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]') +# This expression matches UTF-8 and CESU-8 sequences where some of the +# continuation bytes have been lost. The byte 0x1a (sometimes written as ^Z) is +# used within ftfy to represent a byte that produced the replacement character +# \ufffd. We don't know which byte it was, but we can at least decode the UTF-8 +# sequence as \ufffd instead of failing to re-decode it at all. +LOSSY_UTF8_RE = re.compile( + b'([\xc2-\xdf][\x1a]' + b'|\xed[\xa0-\xaf][\x1a]\xed[\xb0-\xbf][\x1a\x80-\xbf]' + b'|\xed[\xa0-\xaf][\x1a\x80-\xbf]\xed[\xb0-\xbf][\x1a]' + b'|[\xe0-\xef][\x1a][\x1a\x80-\xbf]' + b'|[\xe0-\xef][\x1a\x80-\xbf][\x1a]' + b'|[\xf0-\xf4][\x1a][\x1a\x80-\xbf][\x1a\x80-\xbf]' + b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a][\x1a\x80-\xbf]' + b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a\x80-\xbf][\x1a]' + b'|\x1a)' +) # These regexes match various Unicode variations on single and double quotes. SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') diff --git a/ftfy/fixes.py b/ftfy/fixes.py index bd964a33..e82049d9 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -8,7 +8,7 @@ from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS, CONTROL_CHARS, LIGATURES, WIDTH_MAP, PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE, - SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE) + LOSSY_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE) from ftfy.badness import text_cost from ftfy.compatibility import htmlentitydefs, unichr import re @@ -184,6 +184,8 @@ def fix_one_step_and_explain(text): for encoding in CHARMAP_ENCODINGS: if possible_encoding(text, encoding): encoded_bytes = text.encode(encoding) + encode_step = ('encode', encoding, ENCODING_COSTS.get(encoding, 0)) + transcode_steps = [] # Now, find out if it's UTF-8 (or close enough). Otherwise, # remember the encoding for later. @@ -192,22 +194,21 @@ def fix_one_step_and_explain(text): # Check encoded_bytes for sequences that would be UTF-8, # except they have b' ' where b'\xa0' would belong. if ALTERED_UTF8_RE.search(encoded_bytes): - fixed_bytes, cost = restore_byte_a0(encoded_bytes) - try: - fixed = fixed_bytes.decode('utf-8-variants') - steps = [('encode', encoding, - ENCODING_COSTS.get(encoding, 0)), - ('transcode', 'restore_byte_a0', cost), - ('decode', 'utf-8-variants', 0)] - return fixed, steps - except UnicodeDecodeError: - pass + encoded_bytes, cost = _restore_byte_a0_with_cost(encoded_bytes) + transcode_steps.append(('transcode', 'restore_byte_a0', cost)) + + # Check for the byte 0x1a, which indicates where one of our + # 'sloppy' codecs found a replacement character. + if b'\x1a' in encoded_bytes: + encoded_bytes = replace_lossy_sequences(encoded_bytes) + transcode_steps.append(('transcode', 'replace_lossy_sequences', 0)) if b'\xed' in encoded_bytes or b'\xc0' in encoded_bytes: decoding = 'utf-8-variants' + + decode_step = ('decode', decoding, 0) + steps = [encode_step] + transcode_steps + [decode_step] fixed = encoded_bytes.decode(decoding) - steps = [('encode', encoding, ENCODING_COSTS.get(encoding, 0)), - ('decode', decoding, 0)] return fixed, steps except UnicodeDecodeError: @@ -278,10 +279,8 @@ def apply_plan(text, plan): elif operation == 'decode': obj = obj.decode(encoding) elif operation == 'transcode': - if encoding == 'restore_byte_a0': - obj = restore_byte_a0(obj) - elif encoding == 'fix_partial_utf8_punct_in_1252': - obj = fix_partial_utf8_punct_in_1252(obj) + if encoding in TRANSCODERS: + obj = TRANSCODERS[encoding](obj) else: raise ValueError("Unknown transcode operation: %s" % encoding) else: @@ -563,26 +562,43 @@ def decode_match(match): return ESCAPE_SEQUENCE_RE.sub(decode_match, text) +def _restore_byte_a0_with_cost(byts): + """ + See `restore_byte_a0`. This version of the function additionally + returns a cost, which is twice the number of bytes being replaced. + """ + def replacement(match): + "The function to apply when this regex matches." + return match.group(0).replace(b'\x20', b'\xa0') + + fixed = ALTERED_UTF8_RE.sub(replacement, byts) + return fixed, fixed.count(b'\xa0') * 2 + + def restore_byte_a0(byts): """ Some mojibake has been additionally altered by a process that said "hmm, \\xa0, that's basically a space!" and replaced it with an ASCII space. When the \\xa0 is part of a sequence that we intend to decode as UTF-8, changing byte A0 to 20 would make it fail to decode. - + This process finds sequences that would convincingly decode as UTF-8 if byte 20 were changed to A0, and puts back the A0. For the purpose of deciding whether this is a good idea, this step gets a cost of twice the number of bytes that are changed. - + This is used as a step within `fix_encoding`. """ + return _restore_byte_a0_with_cost(byts)[0] + + +def replace_lossy_sequences(byts): def replacement(match): "The function to apply when this regex matches." - return match.group(0).replace(b'\x20', b'\xa0') + return '\ufffd'.encode('utf-8') - fixed = ALTERED_UTF8_RE.sub(replacement, byts) - return fixed, fixed.count(b'\xa0') * 2 + fixed = LOSSY_UTF8_RE.sub(replacement, byts) + return fixed def fix_partial_utf8_punct_in_1252(text): @@ -600,3 +616,10 @@ def replacement(match): return match.group(0).encode('sloppy-windows-1252').decode('utf-8') return PARTIAL_UTF8_PUNCT_RE.sub(replacement, text) + +TRANSCODERS = { + 'restore_byte_a0': restore_byte_a0, + 'replace_lossy_sequences': replace_lossy_sequences, + 'fix_partial_utf8_punct_in_1252': fix_partial_utf8_punct_in_1252 +} + From 90718bbd5050454c0816a90b9b713b35e5b6b65b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 12 Aug 2015 14:53:08 -0400 Subject: [PATCH 245/502] update tests for lossy decoding --- tests/test_bytes.py | 3 +++ tests/test_real_tweets.py | 17 +++++++++++-- tests/test_synthetic.py | 50 ++++++--------------------------------- 3 files changed, 25 insertions(+), 45 deletions(-) diff --git a/tests/test_bytes.py b/tests/test_bytes.py index 204c786c..70954c9d 100644 --- a/tests/test_bytes.py +++ b/tests/test_bytes.py @@ -41,6 +41,9 @@ def test_guess_bytes(): def test_mangle_surrogates(): + eq_(b'Eric the half a bee \xed\xa0\x80'.decode('utf-8-variants', 'replace'), + 'Eric the half a bee ���') + if PYTHON2: # These are the encodings of a surrogate character, plus a similar-looking # Korean character. Only the surrogate character's bytes should get mangled. diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 452c1289..50e79e9f 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals from ftfy import fix_text +from ftfy.fixes import fix_encoding_and_explain, apply_plan from nose.tools import eq_ + TEST_CASES = [ ## These are excerpts from tweets actually seen on the public Twitter ## stream. Usernames and links have been removed. @@ -44,20 +46,26 @@ 'Engkau masih yg terindah, indah di dalam hatiku♫~'), ('SENSЕ - Oleg Tsedryk', 'SENSЕ - Oleg Tsedryk'), # this Е is a Ukrainian letter + # Test that we can mostly decode this face when the nonprintable + # character \x9d is lost + ('ââ€\x9d’(⌣˛⌣)ââ€\x9dŽ', '┒(⌣˛⌣)┎'), + ('ââ€�’(⌣˛⌣)ââ€�Ž', '�(⌣˛⌣)�'), + ## Current false positives: #("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), #("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿") #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), - + ## This kind of tweet can't be fixed without a full-blown encoding detector. #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", # "Deja dos heridos hundimiento de barco turístico en Acapulco."), - + ## The heuristics aren't confident enough to fix this text and its weird encoding. #("Blog Traffic Tip 2 – Broadcast Email Your Blog", # "Blog Traffic Tip 2 – Broadcast Email Your Blog"), ] + def test_real_tweets(): """ Test with text actually found on Twitter. @@ -73,6 +81,11 @@ def test_real_tweets(): text. """ for orig, target in TEST_CASES: + # make sure that the fix_encoding step outputs a plan that we can + # successfully run to reproduce its result + encoding_fix, plan = fix_encoding_and_explain(orig) + eq_(apply_plan(orig, plan), encoding_fix) + # make sure we can decode the text as intended eq_(fix_text(orig), target) diff --git a/tests/test_synthetic.py b/tests/test_synthetic.py index 455973be..74a83de5 100644 --- a/tests/test_synthetic.py +++ b/tests/test_synthetic.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from ftfy.fixes import fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, fix_surrogates -from ftfy.badness import sequence_weirdness -import unicodedata +from ftfy.fixes import fix_encoding import sys from nose.tools import eq_ @@ -10,31 +8,6 @@ unichr = chr -def char_names(text): - """ - Show the names of the characters involved. Helpful for debugging when - characters themselves are not visually distinguishable. - """ - return [unicodedata.name(c) for c in text] - - -# Most single-character strings which have been misencoded should be restored. -def test_bmp_characters(): - for index in range(0xa0, 0xfffd): - char = unichr(index) - # Exclude code points that are not assigned - if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): - garble = char.encode('utf-8').decode('latin-1') - # Exclude characters whose re-encoding is protected by the - # 'sequence_weirdness' metric - if sequence_weirdness(garble) >= 0: - garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') - for garb in (garble, garble2): - fixed, plan = fix_encoding_and_explain(garb) - eq_(fixed, char) - eq_(apply_plan(garb, plan), char) - - phrases = [ "\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", "\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", @@ -51,20 +24,16 @@ def test_valid_phrases(): def check_phrase(text): + # Check each valid phrase above, making sure that it doesn't get changed eq_(fix_encoding(text), text) eq_(fix_encoding(text.encode('utf-8').decode('latin-1')), text) + # make sure that the opening punctuation is not the only thing that makes # it work eq_(fix_encoding(text[1:]), text[1:]) eq_(fix_encoding(text[1:].encode('utf-8').decode('latin-1')), text[1:]) -def test_possible_encoding(): - for codept in range(256): - char = chr(codept) - assert possible_encoding(char, 'latin-1') - - def test_fix_with_backslash(): eq_(fix_encoding("<40\\% vs \xe2\x89\xa540\\%"), "<40\\% vs ≥40\\%") @@ -74,6 +43,10 @@ def test_mixed_utf8(): eq_(fix_encoding('“mismatched quotes…”'), '“mismatched quotes…”') +def test_lossy_utf8(): + eq_(fix_encoding('“lossy decodingâ€�'), '“lossy decoding�') + + def test_unknown_emoji(): # Make a string with two burritos in it. Python doesn't know about Unicode # burritos, but ftfy can guess they're probably emoji anyway. @@ -92,12 +65,3 @@ def test_unknown_emoji(): # characters. The mangled text passes through as is. not_emoji = 'dos burritos: \U0003f32f\U0003f32f'.encode('utf-8').decode('windows-1251') eq_(fix_encoding(not_emoji), not_emoji) - - -def test_surrogates(): - eq_(fix_surrogates('\udbff\udfff'), '\U0010ffff') - eq_(fix_surrogates('\ud800\udc00'), '\U00010000') - - -if __name__ == '__main__': - test_all_bmp_characters() From 01ecc8b1f2b86f3b197df7ac61ccbcb3d3da10f3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 13 Aug 2015 14:53:59 -0400 Subject: [PATCH 246/502] add to test_real_tweets -- one of them fails at the moment --- tests/test_real_tweets.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 50e79e9f..08776992 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -45,6 +45,14 @@ ('Engkau masih yg terindah, indah di dalam hatiku♫~', 'Engkau masih yg terindah, indah di dalam hatiku♫~'), ('SENSЕ - Oleg Tsedryk', 'SENSЕ - Oleg Tsedryk'), # this Е is a Ukrainian letter + ('OK??:( `¬´ ):', 'OK??:( `¬´ ):'), + ("selamat berpuasa sob (Ã\xa0¸‡'̀⌣'ÃŒÂ\x81)Ã\xa0¸‡", + "selamat berpuasa sob (ง'̀⌣'́)ง"), + + # This one has two differently-broken layers of Windows-1252 <=> UTF-8, + # and it's kind of amazing that we solve it. + ('Arsenal v Wolfsburg: pre-season friendly â\x80â\x80\x9c live!', + 'Arsenal v Wolfsburg: pre-season friendly – live!'), # Test that we can mostly decode this face when the nonprintable # character \x9d is lost From 506f2f830f2d4b56fb27d8fe27b9ccbb39e23a67 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 13 Aug 2015 15:03:46 -0400 Subject: [PATCH 247/502] Fix false positive by generalizing the 'tilde exception' --- ftfy/build_data.py | 8 ++++++-- ftfy/char_classes.dat | Bin 3760 -> 3757 bytes 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 940bb37d..841920ff 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -112,8 +112,12 @@ def make_char_data_file(do_it_anyway=False): # Mark whitespace control characters as whitespace cclasses[9] = cclasses[10] = cclasses[12] = cclasses[13] = ' ' - # Tilde is not a "math symbol" the way it's used - cclasses[ord('~')] = 'o' + # Some other exceptions for characters that are more commonly used as + # punctuation or decoration than for their ostensible purpose. + # For example, tilde is not usually a "math symbol", and the accents + # `´ are much more like quotation marks than modifiers. + for char in "^~`´˝＾｀": + cclasses[ord(char)] = 'o' out = open('char_classes.dat', 'wb') out.write(zlib.compress(''.join(cclasses).encode('ascii'))) diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index cbc2ccda4290b86f390c701a5e2a81b86fb2fa93..c66dd25b28c9664f5b33c0f37086e91b3d0a185f 100644 GIT binary patch delta 1107 zcmV-Z1g!h89jzUZM}H~<{Wq(KaGc;M{c|qKF3BwqpInmD$4B;=WtWdz_K|;_N&8g2 z&!v5;Km0!UTA#7)U$M<89^NYJ9)qXWO2Wec`9p4aNTE zn>kV*M)wDwTO7u7zLz?8`;N&%IX2J#cZ94SV`l;sySt_3!)oh|+0QTkP?Ofe_#0P$ zcza$c|4PL?-mi*XQKs?J^v$7Yp(6vAloV(B5^>e`G11GOsb8 z!=1@u`m@@C&MjZ)|Dnmu|2baP6r}LZA@5EI-dgy5V_w!QBsWXxFo@Z1hu(H*Sqn|IrxWOOm zdq+G_S8@GOs}&2HqKEklf3Te=0aX_fjXr*uy4t3_1~+MH0>*`DKPf5lCp z-Jrhudcm_DD}Lv?{Db9vxc;;5Xtuxpk0yOIJ${x)uS>T5WAd2OY0D3{cl>I+JL3Rv zc6j^*zRrub|LnDbQ#Akpc%OHl<^Vs&yN4?P00000000000002^f2+LvKNRspZRnL-b+veeQku*9kJYNnTi49UxJ%k_&hQ>;-GxbYpT)OT z4gdfE0Q_*tUuk#we;=!;_^5~a{a!yHrhW!QJ^#KQ$=-xt(f&oYzOH|Mwa3dpsfqt% z z$%BR7tT?m!FB`Y=*R2fGhHvJ@=}t1C7fC2!Kh#r+|^==;wfqxaS9 zyE6j-yu`bYzX0GJHeO)|lfeooF5or7R`1Rz0002+P5m|34>n&$0RR910000000000 z000009`Nov4Dg-3yLSS!Aq%hqlh6ta5Z^+!viOU=rjYaQf0=jJH4uE9xk{T#?2=0*nl^2mxsZfASEQ z>@t7<&!Xg`d&};-wSKx5Jj~y~v|M^VpDxMy9Fn{+r_Oz{i*gr}y~q~QnVvVCJ?y>v z`s0h$)lj^B>=8P*Dc`U?jFq=g9>`~f{{7F3j3u)*H*MEl)9>r?eYLd~I@YSK`*J(3 z#$Sh<&2O`9XY1u@wD(Y6Da@w(f75s4>5ko7%nxsUTW@;2t^8Y!?^pb6+jFun{M5Rk z*x!6JN6N$K{@`LO;i-^@&dGhXsLDZ`6AC-qbNuYD zxCyiy)K_0Gc(!B3?_8IEu)Gh~f7Tt%_SgT>q>rY@&+_PX$+mw?9&^X}6e;Kz9Pa0LJW0000000000e*ge~m3RM#B7UfC z{bz80&ac1q%UY((dzK#W|J}Q;7LQO$b6NkfT6KBr+fB|_-`srQCH8Lr^BZ#&-_#AN z=MA${%uBmoSQP*O0000000000000000Kl8L;|Kr%{5_s>|Fa!;NgK`?-b1asFsbgd z__oRc0001hA1?VTf9)>+V-*!2^-#ax>j%Wt&w!}s-`6AAoA4{zzsT0t_0O;Nc=;za z@qdiGv9J@avp#37t`iQdGnG!ByF7=Xe9MLII-yCXo7}PTl(9F_*B>UzFf zI!E$moxFwT`{`+RrPE4^f7+U#-KURj_3rTp0002^dawH9Ro^ewxy(BATx9?T;_$gN zv=%>ku+WNhb6n=|?*l*2@frrR6)FPEV@1Kg1V(|M_F| zzM6e^W&nVfc=z!a0KCJ-E9{dY3n?wUM%e1z83h0U0KTce=K8_r%P0T<0000000000 z000000Kfy@eTMDSvmpzv0+Y}R4H4j5$W|7AvDXxG-u*B0?)nCkAqzeOzOR3i bAqzhS;GYoQodc603qJ|IgAo1?fx^pY8)s7( From 01f050261d05c8afddc5a938ebd769575927e1f2 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 13 Aug 2015 15:29:52 -0400 Subject: [PATCH 248/502] add docstrings and Sphinx docs for newer 'transcoding' fixes --- docs/index.rst | 5 ++++- ftfy/fixes.py | 40 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 589418f3..2b03929c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -277,7 +277,10 @@ Module documentation :members: unescape_html, remove_terminal_escapes, uncurl_quotes, fix_latin_ligatures, fix_character_width, fix_line_breaks, fix_surrogates, remove_control_chars, remove_bom, decode_escapes, - fix_one_step_and_explain, apply_plan + fix_one_step_and_explain, apply_plan, restore_byte_a0, + replace_lossy_sequences, fix_partial_utf8_punct_in_1252 + + *ftfy.badness*: measures the "badness" of text diff --git a/ftfy/fixes.py b/ftfy/fixes.py index e82049d9..c2b88e77 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -578,8 +578,8 @@ def replacement(match): def restore_byte_a0(byts): """ Some mojibake has been additionally altered by a process that said "hmm, - \\xa0, that's basically a space!" and replaced it with an ASCII space. - When the \\xa0 is part of a sequence that we intend to decode as UTF-8, + byte A0, that's basically a space!" and replaced it with an ASCII space. + When the A0 is part of a sequence that we intend to decode as UTF-8, changing byte A0 to 20 would make it fail to decode. This process finds sequences that would convincingly decode as UTF-8 if @@ -593,6 +593,40 @@ def restore_byte_a0(byts): def replace_lossy_sequences(byts): + """ + This function identifies sequences where information has been lost, + indicated by byte 1A, and if they would otherwise look like a UTF-8 + sequence, it replaces them with the UTF-8 sequence for U+FFFD. + + A further explanation: + + ftfy can now fix text in a few cases that it would previously fix + incompletely, because of the fact that it can't successfully apply the fix + to the entire string. A very common case of this is when characters have + been erroneously decoded as windows-1252, but instead of the "sloppy" + windows-1252 that passes through unassigned bytes, the unassigned bytes get + turned into U+FFFD (�), so we can't tell what they were. + + This most commonly happens with curly quotation marks that appear + ``“ like this â€�``. + + We can do better by building on ftfy's "sloppy codecs" to let them handle + less-sloppy but more-lossy text. When they encounter the character ``�``, + instead of refusing to encode it, they encode it as byte 1A -- an + ASCII control code called SUBSTITUTE that once was meant for about the same + purpose. We can then apply a fixer that looks for UTF-8 sequences where + some continuation bytes have been replaced by byte 1A, and decode the whole + sequence as �; if that doesn't work, it'll just turn the byte back into � + itself. + + As a result, the above text ``“ like this â€�`` will decode as + ``“ like this �``. + + This is used as a step within `fix_encoding`. + + One consequence of this transformation is that `fix_encoding` will no + longer pass through byte 1A if you were using it for something else. + """ def replacement(match): "The function to apply when this regex matches." return '\ufffd'.encode('utf-8') @@ -610,6 +644,8 @@ def fix_partial_utf8_punct_in_1252(text): For this function, we assume the text has been decoded in Windows-1252. If it was decoded in Latin-1, we'll call this right after it goes through the Latin-1-to-Windows-1252 fixer. + + This is used as a step within `fix_encoding`. """ def replacement(match): "The function to apply when this regex matches." From b58212430db79ced338605ceb861326afd32530f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 13 Aug 2015 16:18:01 -0400 Subject: [PATCH 249/502] explain control codes better in `character_width` --- ftfy/formatting.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ftfy/formatting.py b/ftfy/formatting.py index d7370a15..3fe8e1b0 100644 --- a/ftfy/formatting.py +++ b/ftfy/formatting.py @@ -17,11 +17,15 @@ def character_width(char): exception. This assumption will go wrong, for example, if your display is actually on a Japanese flip-phone... but let's assume it isn't. - If this character is a control code, the idea of it having a "width" - probably doesn't even apply, because it will probably have some other - effect on your terminal. For example, there's no sensible width for a - line break. We return the default of 1 in the absence of anything sensible - to do there. + Combining marks and formatting codepoints do not advance the cursor, + so their width is 0. + + If the character is a particular kind of control code -- the kind + represented by the lowest bytes of ASCII, with Unicode category Cc -- the + idea of it having a "width" probably doesn't even apply, because it will + probably have some other effect on your terminal. For example, there's no + sensible width for a line break. We return the default of 1 in the absence + of anything sensible to do there. >>> character_width('車') 2 From 91fbc0e670e39fc0ea45a180ec5cc109b54dd69a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 13 Aug 2015 16:19:08 -0400 Subject: [PATCH 250/502] remove intermediate assignments to `text_width` --- ftfy/formatting.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ftfy/formatting.py b/ftfy/formatting.py index 3fe8e1b0..a5a6c49a 100644 --- a/ftfy/formatting.py +++ b/ftfy/formatting.py @@ -91,8 +91,7 @@ def display_ljust(text, width, fillchar=' '): """ if character_width(fillchar) != 1: raise ValueError("The padding character must have display width 1") - text_width = monospaced_width(text) - padding = max(0, width - text_width) + padding = max(0, width - monospaced_width(text)) return text + fillchar * padding @@ -116,8 +115,7 @@ def display_rjust(text, width, fillchar=' '): """ if character_width(fillchar) != 1: raise ValueError("The padding character must have display width 1") - text_width = monospaced_width(text) - padding = max(0, width - text_width) + padding = max(0, width - monospaced_width(text)) return fillchar * padding + text @@ -137,8 +135,7 @@ def display_center(text, width, fillchar=' '): """ if character_width(fillchar) != 1: raise ValueError("The padding character must have display width 1") - text_width = monospaced_width(text) - padding = max(0, width - text_width) + padding = max(0, width - monospaced_width(text)) left_padding = padding // 2 right_padding = padding - left_padding return fillchar * left_padding + text + fillchar * right_padding From a8c91d89d73521ff0195fe962781f24604396294 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 13 Aug 2015 16:24:55 -0400 Subject: [PATCH 251/502] formatting.py: Python 2 compatibility fixes --- ftfy/formatting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ftfy/formatting.py b/ftfy/formatting.py index a5a6c49a..11a64f3a 100644 --- a/ftfy/formatting.py +++ b/ftfy/formatting.py @@ -1,3 +1,5 @@ +# coding: utf-8 +from __future__ import unicode_literals, division from unicodedata import east_asian_width, combining, category, normalize From bc13431e334a93b5bbf6501a9d26c16cede34b20 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 13 Aug 2015 16:41:36 -0400 Subject: [PATCH 252/502] an example that's too good to leave out of the tests --- tests/test_real_tweets.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 08776992..33fb451f 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -59,6 +59,10 @@ ('ââ€\x9d’(⌣˛⌣)ââ€\x9dŽ', '┒(⌣˛⌣)┎'), ('ââ€�’(⌣˛⌣)ââ€�Ž', '�(⌣˛⌣)�'), + # You tried + ('I just figured out how to tweet emojis! â\x9a½í\xa0½í¸\x80í\xa0½í¸\x81í\xa0½í¸\x82í\xa0½í¸\x86í\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8e', + 'I just figured out how to tweet emojis! ⚽😀😁😂😆😎😎😎😎') + ## Current false positives: #("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), #("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿") From 67dc138a1edb206a7b08049a9509fa37e50f0df4 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 13 Aug 2015 16:47:23 -0400 Subject: [PATCH 253/502] fix redundancy in docstring --- ftfy/fixes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index c2b88e77..bc328ac0 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -639,7 +639,7 @@ def fix_partial_utf8_punct_in_1252(text): """ Fix particular characters that seem to be found in the wild encoded in UTF-8 and decoded in Latin-1 or Windows-1252, even when this fix can't be - consistently applied. This is used as a step within `fix_encoding`. + consistently applied. For this function, we assume the text has been decoded in Windows-1252. If it was decoded in Latin-1, we'll call this right after it goes through From edd3af9de56ba4dcbbee2ccdebfa3f0593f74903 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 13 Aug 2015 18:55:26 -0400 Subject: [PATCH 254/502] Don't mess with actual uses of U+1A, if those exist We're talking about a codepoint that apparently never appears in Twitter or in the Common Crawl. Perhaps that just means it's not allowed in tweets or in HTML, though. --- ftfy/fixes.py | 15 ++++++++------- tests/test_synthetic.py | 4 +++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index bc328ac0..2a701ad6 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -199,7 +199,7 @@ def fix_one_step_and_explain(text): # Check for the byte 0x1a, which indicates where one of our # 'sloppy' codecs found a replacement character. - if b'\x1a' in encoded_bytes: + if encoding.startswith('sloppy') and b'\x1a' in encoded_bytes: encoded_bytes = replace_lossy_sequences(encoded_bytes) transcode_steps.append(('transcode', 'replace_lossy_sequences', 0)) @@ -594,9 +594,9 @@ def restore_byte_a0(byts): def replace_lossy_sequences(byts): """ - This function identifies sequences where information has been lost, - indicated by byte 1A, and if they would otherwise look like a UTF-8 - sequence, it replaces them with the UTF-8 sequence for U+FFFD. + This function identifies sequences where information has been lost in + a "sloppy" codec, indicated by byte 1A, and if they would otherwise look + like a UTF-8 sequence, it replaces them with the UTF-8 sequence for U+FFFD. A further explanation: @@ -622,10 +622,11 @@ def replace_lossy_sequences(byts): As a result, the above text ``“ like this â€�`` will decode as ``“ like this �``. - This is used as a step within `fix_encoding`. + If U+1A was actually in the original string, then the sloppy codecs will + not be used, and this function will not be run, so your weird control + character will be left alone but wacky fixes like this won't be possible. - One consequence of this transformation is that `fix_encoding` will no - longer pass through byte 1A if you were using it for something else. + This is used as a step within `fix_encoding`. """ def replacement(match): "The function to apply when this regex matches." diff --git a/tests/test_synthetic.py b/tests/test_synthetic.py index 74a83de5..bf75ff24 100644 --- a/tests/test_synthetic.py +++ b/tests/test_synthetic.py @@ -15,7 +15,9 @@ "\u2014ALLÍ ESTÁ\u2014", "AHÅ™, the new sofa from IKEA®", "ВІКІ is Ukrainian for WIKI", - #"\u2014a radius of 10 Å\u2014", + 'These control characters \x1a are apparently intentional \x81', + "I don't know what this is \x1a but this is the euro sign €", + "\u2014a radius of 10 Å\u2014", ] # These phrases should not be erroneously "fixed" def test_valid_phrases(): From 667aa475b262a059c03f0d69cd15fa4b4c2f26e4 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 17 Aug 2015 17:50:44 -0400 Subject: [PATCH 255/502] add some more test examples --- tests/test_real_tweets.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 33fb451f..e90f7d4b 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -61,12 +61,14 @@ # You tried ('I just figured out how to tweet emojis! â\x9a½í\xa0½í¸\x80í\xa0½í¸\x81í\xa0½í¸\x82í\xa0½í¸\x86í\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8e', - 'I just figured out how to tweet emojis! ⚽😀😁😂😆😎😎😎😎') + 'I just figured out how to tweet emojis! ⚽😀😁😂😆😎😎😎😎'), + ('CÃ\xa0nan nan GÃ\xa0idheal', 'Cànan nan Gàidheal'), ## Current false positives: #("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), - #("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿") + #("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿"), #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), + #("SELKÄ\xa0EDELLÄ\xa0MAAHAN via @YouTube", "SELKÄ EDELLÄ MAAHAN via @YouTube"), ## This kind of tweet can't be fixed without a full-blown encoding detector. #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", From 5b7b26221beb195f6376b95472f3aee3a64cfedc Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 11 Sep 2015 14:24:35 -0400 Subject: [PATCH 256/502] minor fixes based on PR review --- docs/index.rst | 2 -- ftfy/chardata.py | 12 +++++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 2b03929c..5f774491 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -281,8 +281,6 @@ Module documentation replace_lossy_sequences, fix_partial_utf8_punct_in_1252 - - *ftfy.badness*: measures the "badness" of text ---------------------------------------------- diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 91f9e345..d6641b48 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -37,9 +37,11 @@ def _build_regexes(): charlist = latin1table.encode('latin-1').decode(encoding) # Build a regex from the ASCII range, followed by the decodings of - # bytes 0x80-0xff in this character set. (This uses the fact that all - # regex special characters are ASCII, and therefore won't appear in the - # string.) + # bytes 0x80-0xff in this character set, followed by the decoding of + # byte 0x1a in case we made that one special. + # + # All regex special characters are in the \x1b-\x7f range, so we don't + # need to worry about escaping them in the regex. regex = '^[\x00-\x19\x1b-\x7f{0}]*$'.format(charlist) encoding_regexes[encoding] = re.compile(regex) return encoding_regexes @@ -107,7 +109,7 @@ def _build_utf8_punct_regex(): # \ufffd. We don't know which byte it was, but we can at least decode the UTF-8 # sequence as \ufffd instead of failing to re-decode it at all. LOSSY_UTF8_RE = re.compile( - b'([\xc2-\xdf][\x1a]' + b'[\xc2-\xdf][\x1a]' b'|\xed[\xa0-\xaf][\x1a]\xed[\xb0-\xbf][\x1a\x80-\xbf]' b'|\xed[\xa0-\xaf][\x1a\x80-\xbf]\xed[\xb0-\xbf][\x1a]' b'|[\xe0-\xef][\x1a][\x1a\x80-\xbf]' @@ -115,7 +117,7 @@ def _build_utf8_punct_regex(): b'|[\xf0-\xf4][\x1a][\x1a\x80-\xbf][\x1a\x80-\xbf]' b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a][\x1a\x80-\xbf]' b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a\x80-\xbf][\x1a]' - b'|\x1a)' + b'|\x1a' ) # These regexes match various Unicode variations on single and double quotes. From d517a3c342ff5e6fd9c1414c5a09b3da2567779f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 11 Sep 2015 15:21:57 -0400 Subject: [PATCH 257/502] Simplify utf8_variants expressions; test incomplete decoding --- ftfy/bad_codecs/utf8_variants.py | 29 +++++++++++++++++++++++------ tests/test_bytes.py | 21 +++++++++++++++++++-- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 7fbeae0f..e4aea494 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -48,12 +48,20 @@ NAME = 'utf-8-variants' # This regular expression matches all possible six-byte CESU-8 sequences, -# plus truncations of them at the end of the string. +# plus truncations of them at the end of the string. (If any of the +# subgroups matches $, then all the subgroups after it also have to match $, +# as there are no more characters to match.) CESU8_EXPR = ( - b'(\xed[\xa0-\xaf][\x80-\xbf]\xed[\xb0-\xbf][\x80-\xbf]' - b'|\xed([\xa0-\xaf]([\x80-\xbf](\xed[\xb0-\xbf]?)?)?)?$' + b'(' + b'\xed' + b'([\xa0-\xaf]|$)' + b'([\x80-\xbf]|$)' + b'(\xed|$)' + b'([\xb0-\xbf]|$)' + b'([\x80-\xbf]|$)' b')' ) + CESU8_RE = re.compile(CESU8_EXPR) # This expression matches isolated surrogate characters that aren't @@ -63,7 +71,7 @@ # This expression matches the Java encoding of U+0. We don't need to check # for its truncated version, because that will get passed on to the standard # UTF-8 decoder, which will agree that we need to see more bytes. -NULL_EXPR = b'(\xc0\x80)' +NULL_EXPR = b'(\xc0(\x80|$))' # This regex matches cases that we need to decode differently from @@ -150,8 +158,17 @@ def _buffer_decode_step(self, input, errors, final): # Some byte sequence that we intend to handle specially matches # at the beginning of the input. if input.startswith(b'\xc0'): - # Decode the two-byte sequence 0xc0 0x80. - return '\u0000', 2 + if len(input) > 1: + # Decode the two-byte sequence 0xc0 0x80. + return '\u0000', 2 + else: + if final: + # We hit the end of the stream. Let the superclass method + # handle it. + return sup(input, errors, True) + else: + # Wait to see another byte. + return '', 0 else: # Decode a possible six-byte sequence starting with 0xed. return self._buffer_decode_surrogates(sup, input, errors, final) diff --git a/tests/test_bytes.py b/tests/test_bytes.py index 70954c9d..e283869f 100644 --- a/tests/test_bytes.py +++ b/tests/test_bytes.py @@ -1,7 +1,7 @@ # coding: utf-8 from __future__ import unicode_literals from ftfy import guess_bytes -from ftfy.bad_codecs.utf8_variants import mangle_surrogates +from ftfy.bad_codecs.utf8_variants import mangle_surrogates, IncrementalDecoder from nose.tools import eq_ import sys @@ -54,10 +54,27 @@ def test_mangle_surrogates(): eq_(mangle_surrogates(b'\xed\xa0\xbd\xed\xb8\xb9test\xed\xb4\xb4'), b'\xff\xff\xff\xff\xff\xfftest\xed\xb4\xb4') eq_(mangle_surrogates(b'test\xed\xb4\xb4'), b'test\xed\xb4\xb4') - + # Handle short bytestrings correctly. eq_(mangle_surrogates(b'\xed'), b'\xed') eq_(mangle_surrogates(b''), b'') else: # Make sure mangle_surrogates doesn't do anything eq_(mangle_surrogates(b'\xed\xa0\x80\xed\x9e\x99'), b'\xed\xa0\x80\xed\x9e\x99') + + +def test_incomplete_sequences(): + test_bytes = b'surrogates: \xed\xa0\x80\xed\xb0\x80 / null: \xc0\x80' + test_string = 'surrogates: \U00010000 / null: \x00' + + # Test that we can feed this string to decode() in multiple pieces, and no + # matter where the break between those pieces is, we get the same result. + for split_point in range(len(test_string) + 1): + left = test_bytes[:split_point] + right = test_bytes[split_point:] + + decoder = IncrementalDecoder() + got = decoder.decode(left, final=False) + got += decoder.decode(right) + eq_(got, test_string) + From 0e1162ac872018cc4c8db2265a22a75fb9f2ab66 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 30 Sep 2015 16:02:28 -0400 Subject: [PATCH 258/502] remove _restore_byte_a0_with_cost, compute cost afterward --- ftfy/fixes.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 2a701ad6..0814db31 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -194,7 +194,8 @@ def fix_one_step_and_explain(text): # Check encoded_bytes for sequences that would be UTF-8, # except they have b' ' where b'\xa0' would belong. if ALTERED_UTF8_RE.search(encoded_bytes): - encoded_bytes, cost = _restore_byte_a0_with_cost(encoded_bytes) + encoded_bytes = restore_byte_a0(encoded_bytes) + cost = encoded_bytes.count(b'\xa0') * 2 transcode_steps.append(('transcode', 'restore_byte_a0', cost)) # Check for the byte 0x1a, which indicates where one of our @@ -562,19 +563,6 @@ def decode_match(match): return ESCAPE_SEQUENCE_RE.sub(decode_match, text) -def _restore_byte_a0_with_cost(byts): - """ - See `restore_byte_a0`. This version of the function additionally - returns a cost, which is twice the number of bytes being replaced. - """ - def replacement(match): - "The function to apply when this regex matches." - return match.group(0).replace(b'\x20', b'\xa0') - - fixed = ALTERED_UTF8_RE.sub(replacement, byts) - return fixed, fixed.count(b'\xa0') * 2 - - def restore_byte_a0(byts): """ Some mojibake has been additionally altered by a process that said "hmm, @@ -589,7 +577,11 @@ def restore_byte_a0(byts): This is used as a step within `fix_encoding`. """ - return _restore_byte_a0_with_cost(byts)[0] + def replacement(match): + "The function to apply when this regex matches." + return match.group(0).replace(b'\x20', b'\xa0') + + return ALTERED_UTF8_RE.sub(replacement, byts) def replace_lossy_sequences(byts): From afed27ab8f25716bbf4db28bcafcd1a25c850281 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 30 Sep 2015 17:33:57 -0400 Subject: [PATCH 259/502] fix comment about truncated encoding of U+0 --- ftfy/bad_codecs/utf8_variants.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index e4aea494..5b602f7a 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -68,9 +68,8 @@ # CESU-8, which have to be handled carefully on Python 2. SURROGATE_EXPR = (b'(\xed[\xa0-\xbf][\x80-\xbf])') -# This expression matches the Java encoding of U+0. We don't need to check -# for its truncated version, because that will get passed on to the standard -# UTF-8 decoder, which will agree that we need to see more bytes. +# This expression matches the Java encoding of U+0, including if it's +# truncated and we need more bytes. NULL_EXPR = b'(\xc0(\x80|$))' From 9529a02cf6e24e685a1a70da5e442b7746695bec Mon Sep 17 00:00:00 2001 From: Nicolas Rosset Date: Mon, 21 Dec 2015 23:15:31 +0100 Subject: [PATCH 260/502] fix_text_encoding is depreciated According to fixes.py (lines 113-119), fix_text_encoding is depreciated for fix_encoding --- ftfy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 5b3f1e3c..e85b8f28 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -269,7 +269,7 @@ def fix_text_segment(text, if remove_terminal_escapes: text = fixes.remove_terminal_escapes(text) if fix_encoding: - text = fixes.fix_text_encoding(text) + text = fixes.fix_encoding(text) if fix_latin_ligatures: text = fixes.fix_latin_ligatures(text) if fix_character_width: From e80bdc1d2092a836abf1e08b5c46563b254a0c00 Mon Sep 17 00:00:00 2001 From: Sara Jewett Date: Wed, 23 Dec 2015 15:40:37 -0500 Subject: [PATCH 261/502] Specify encodings when dealing with files --- ftfy/streamtester/twitter_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/streamtester/twitter_tester.py b/ftfy/streamtester/twitter_tester.py index 6ad125ee..a6542e5c 100644 --- a/ftfy/streamtester/twitter_tester.py +++ b/ftfy/streamtester/twitter_tester.py @@ -48,7 +48,7 @@ def save_files(self): for lang, lines in self.lines_by_lang.items(): filename = 'tweets.{}.txt'.format(lang) fullname = os.path.join(self.OUTPUT_DIR, filename) - langfile = open(fullname, 'a') + langfile = open(fullname, 'a', encoding='utf-8') for line in lines: print(line.replace('\n', ' '), file=langfile) langfile.close() From d01dec7e723bc45e8e06ecd0f50b183f510e8e0b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 22 Jan 2016 11:50:56 -0500 Subject: [PATCH 262/502] Bump version number to 4.1 --- ftfy/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index e85b8f28..e4951be3 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -17,7 +17,7 @@ from ftfy.compatibility import is_printable import unicodedata -__version__ = '4.0.0' +__version__ = '4.1.0' def fix_text(text, diff --git a/setup.py b/setup.py index d4e7a50e..11d7eebe 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="ftfy", - version='4.0.0', + version='4.1.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From a38652aa53eb95e2952e1b8dd7abd4dd6fc7f878 Mon Sep 17 00:00:00 2001 From: Andrew Lin Date: Fri, 12 Feb 2016 15:05:38 -0500 Subject: [PATCH 263/502] Simplify replace_lossy_sequences(). --- ftfy/fixes.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 0814db31..9080cbbf 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -620,12 +620,7 @@ def replace_lossy_sequences(byts): This is used as a step within `fix_encoding`. """ - def replacement(match): - "The function to apply when this regex matches." - return '\ufffd'.encode('utf-8') - - fixed = LOSSY_UTF8_RE.sub(replacement, byts) - return fixed + return LOSSY_UTF8_RE.sub('\ufffd'.encode('utf-8'), byts) def fix_partial_utf8_punct_in_1252(text): From 4e1a9e0c7778a402558db83af07b8368c70a14c9 Mon Sep 17 00:00:00 2001 From: Andrew Lin Date: Fri, 12 Feb 2016 15:06:41 -0500 Subject: [PATCH 264/502] Remove an extra newline. --- ftfy/bad_codecs/utf8_variants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 5b602f7a..f9c1b5a2 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -72,7 +72,6 @@ # truncated and we need more bytes. NULL_EXPR = b'(\xc0(\x80|$))' - # This regex matches cases that we need to decode differently from # standard UTF-8. SPECIAL_BYTES_RE = re.compile(b'|'.join([NULL_EXPR, CESU8_EXPR, SURROGATE_EXPR])) From 1f50d57c6691c92659063c0ea6e8e809182eed6e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Feb 2016 00:49:35 -0500 Subject: [PATCH 265/502] Handle cases where the buffer ends in the middle of a surrogate --- ftfy/bad_codecs/utf8_variants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index f9c1b5a2..0b541260 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -66,7 +66,7 @@ # This expression matches isolated surrogate characters that aren't # CESU-8, which have to be handled carefully on Python 2. -SURROGATE_EXPR = (b'(\xed[\xa0-\xbf][\x80-\xbf])') +SURROGATE_EXPR = (b'(\xed([\xa0-\xbf]|$)([\x80-\xbf]|$))') # This expression matches the Java encoding of U+0, including if it's # truncated and we need more bytes. From 377686cdd038bff1d73bf6381cdb3885d31ee520 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Feb 2016 00:57:14 -0500 Subject: [PATCH 266/502] update changelog --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfe663bb..1171ee50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +## Version 4.1.0 (February 25, 2016) + +Heuristic changes: + +- ftfy can now deal with "lossy" mojibake. If your text has been run through + a strict Windows-1252 decoder, such as the one in Python, it may contain + the replacement character � (U+FFFD) where there were bytes that are + unassigned in Windows-1252. + + Although ftfy won't recover the lost information, it can now detect this + situation, replace the entire lossy character as �, and decode the rest of + the characters. Previous versions would be unable to fix any string that + contained U+FFFD. + + As an example, text in curly quotes that gets corrupted `“ like this â€�` + now gets fixed to be `“ like this �`. + +- Heuristics now count characters such as `~` and `^` as punctuation instead + of wacky math symbols, improving the detection of mojibake in some edge cases. + +Additionally, the `utf-8-variants` codec was simplified and optimized. + + ## Version 4.0.0 (April 10, 2015) Breaking changes: From 102c1381e4b01a8b465f07a66e6ca142396ea204 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Feb 2016 12:59:44 -0500 Subject: [PATCH 267/502] describe ftfy.formatting in changelog --- CHANGELOG.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1171ee50..2bb169d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Heuristic changes: unassigned in Windows-1252. Although ftfy won't recover the lost information, it can now detect this - situation, replace the entire lossy character as �, and decode the rest of + situation, replace the entire lossy character with �, and decode the rest of the characters. Previous versions would be unable to fix any string that contained U+FFFD. @@ -18,7 +18,13 @@ Heuristic changes: - Heuristics now count characters such as `~` and `^` as punctuation instead of wacky math symbols, improving the detection of mojibake in some edge cases. -Additionally, the `utf-8-variants` codec was simplified and optimized. +New features: + +- A new module, `ftfy.formatting`, can be used to justify Unicode text in a + monospaced terminal. It takes into account that each character can take up + anywhere from 0 to 2 character cells. + +- Internally, the `utf-8-variants` codec was simplified and optimized. ## Version 4.0.0 (April 10, 2015) From fe3f251efb71452f3647dc2ec530829e06664cf7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Feb 2016 14:39:56 -0500 Subject: [PATCH 268/502] update char_classes to the finalized Unicode 8.0 --- CHANGELOG.md | 4 ++++ ftfy/char_classes.dat | Bin 3757 -> 3896 bytes 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bb169d3..b9668c21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ Heuristic changes: As an example, text in curly quotes that gets corrupted `“ like this â€�` now gets fixed to be `“ like this �`. +- Updated the data file of Unicode character categories to Unicode 8.0, as used + in Python 3.5.0. (No matter what version of Python you're on, ftfy uses the + same data.) + - Heuristics now count characters such as `~` and `^` as punctuation instead of wacky math symbols, improving the detection of mojibake in some edge cases. diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index c66dd25b28c9664f5b33c0f37086e91b3d0a185f..5a92ba547aba78224af6cdf75b7a6eb0aee6d9ef 100644 GIT binary patch delta 2991 zcmXw%c~sJg7RPO-=9NxbIc4R-G&#BCQkq&R@?P^S&C-h0Tp9OpBNsB4_|;T4Wok)= zxiOVVsmQpJAX4FyD~btjh#M*jA|fjLtC@Mn-b?o367deKR#2G}+?Ej^~?=9i$xKHH^A~y$ypG7!AxnUNg`hF6W>h{u?mi?$p*b zC%*UdslhWxR^)zZ1a(X^; zILJRaUYf0C8?(YM4D4ILwmNxy#QDM*iJyu*=8jd+9#bBU@V83HySzS@4 z+r@5-Y^R=v-WEM}0de{S|I;^?E*l!~yu3P#Ucq%r8!=_FuqhyShS39a#n>Dl*jt+i zV7<*Tef?(iPS-f1u@+Fsvp#xRw>c;Z=W}@|Aj)_C8lnFk(WN?Rhk&0e%{~~veTr}R zEiw2Q@&F*7^00Gz17PZNcUPXZunx6|476pvTsn2`k5TvR_8e3F=+2RC=kZl;R9VL^ zGbmwzaH*>13K!=FVV(o~AsN=XzIiC3;^;vanX(wv&Mn+|Wu{J&=&GTB8OGY6&jD2}Ndc-73fZs(>W**VJaj7FXt9-ABbI2V}vozW|*#Ef{6oS;d9S3-Q zaQxZwaD=mze__6OAeV1rO7=6=Mx(k~J<@t$lB>+V0U_Nq4*U{0=&J8px)WdK+LJC?F;IJ8WO0ZRfhd zUD%W2KwN(hml|}P#Y$Sp6d|o>G7LhRz`)b)P1ccu*esQJQZCqsu-OBg|F8EmL@Vwr zI!*6l_w~<}C*{i6kVe_DdJt#BMeyBVIInri3xhwewMw`@P*%P;A%H4E7ASX_YjuqV zKH)I^t{Vf}oi!~W0YMn*6uBVJ!OWIKdHrr?zaYB$$8fmctL0}xn+y&(mXWpipN<~G zpn-yBHDBnyMlrWO_ZGMusFzGY%SSP^`-!505$XscP(k5du);cf;-G0KtXi*x#&vR2 z)iEc^hVJT6_uqL@fgf|)KYxUB`7Q#R7dL3#NhDC)$7Qf|)DK1DX1X-N9i>g@(-4;PdInGXJM?};NT^QW?{ z+w`E8D8q7WX+Qz;_h8H~7vC-zBYu0k-|}0F&kNVJ1afR~Xf9Hr?Mw&@|e? zOUQTBm3{{pF<^T<>jS$Y*>-c>@eap2CcUUazUU2vUl^b09wTIT#NB$>wJ?=O&2jG+ zcw0b}Mua!{irA*p66DmtiZp;_X*oG)9(X12YxSHZm+YKy*j@1fDdnW4f>N3E$(=;|Bt zl@_6i_+^aZQn33_;Hh{lqpm+af+^u#m#-i69YG4b;D%At_59h2@%C}*&4FovVzL&l z@t3^Q#@1Ovt8M5Hn!3PyDqN|^d`0F26$Rc>(rb%B^wTZya|oU;PEWZ$gO;zc>})?| zW4#)BTGAgtKQ%;vntSdwv#X%ugNUaM5!C_@zx4gd6-6GD#!_KXrp)`iSzz~lcBS<$ zQg`VSPikN{OX=GiOo1`gkSD)M4F=A5smAv*C2MhfscnGV^GskW zDvD&0q^HB1ytb8B{d@H%J_V>>{qaCcw^7BTwJ@p2fVlLhzSUPJ#JM^pmSjP{jQDyY zShlIGl{qcCb~d~HaoD{RbtULxj3bQyUwd@#21&Jm60(^fSY>&Jg4Mp~g(QDvcRTHZ zXe+lQ#%tbL?L@y~o_V@kdqWV_~R8Ak;C?$`hZGv2rTd z8;or>^gOYdU=MCWf|pZT^uNdYD6TFh)>tZrnh!uY=||H#X%o^3Xd&%d59<;)Zq&}# zv~8BBHaviD8+xE2<;BINQAbXOuC1+Pg*87)bQ(U1e=c%pi(eNtT~G~w7LV!B;e}S= z&MVN1rgWFVfX%?94e%f+Ba12%FhEt2S|ZU=0Oj95?$sA&Rq_+zC{~H?h>@WSe)EXM z^3*SSyXV)uT^h7VwQ2&MN*0_01d=oEk!nNep{HriIA@;UUetIo1%zN3TVOsIIN^hl zmCsEGQx2YhZAnBZJbhsl@pg`lGYe4@TP~%^32C zX?nDuriRAS`cib{1Xi>(`K8bpnq-4amo1yf`rrQZ$%-pN6>X>L;-bt`C@rD`=8Kog zG9#ZY3Dp8z)hyJMZ373A{qW^YgVTp_6^JPC@-F`zHSbR2BLsib;DBtngT)>Ry0ZHp zyZ@*6g*T?QYG^17bW&8q8MsmjYZd!~IiCNN?wjt<{W|p?VkLcIk5M;2d zA00H2s-3#_Re=RuT)$5q$Fo2eRq2rgKbklu}*J(Tnl@ARlXGxPh5MuEPewodsw#8f@j9sm~; zA~?nSZWKe8*v~a?U^*6G_yhs$S~V@@~ delta 2850 zcmYjRc~sH~7j@d4#vGe(GRuvc>2qr_H8fG7sWEfS4Ko)^EltfdH(dF(Xik|~nwbld znVEZvf{H?l=op112`UPe5-M(>2!g;@^UpW;k9Y2Q=bd}cednFmuHT^dKofA-a9uL* zin%FUf*x40J@a_JRy^IZ<$!g;j<_C@(g!6sL7hfjI-f*)=4BZR;bKukCY$9}b$`W{ zeKL~A9~nF{cvu4ykWU|QSS!9%l;^+p5GZq$d3O(0Wz@nppb|e!v#^#?3*7~?*Che? zUzPH!H|339Mp_gNrCdw^j8Cm#OR6E|sZwc&65ff(!G!dSAOUMnSiLO?)A>4X=UO5i z!NrJ9HD|Hb!8xPu0NANBE5jF=4+sH`=MmZafx@=2UrWyF)Gqq9Qo*PvF$6xZCXygu z!*Nq%oFL%%PhcZaI=SVEmSTv!3SUqiN{6B$hNArI!}@=TAu2)yDj`X=#Tx z3qf>8?(Wo7?${p35`z>;LItVMJz5f0TZ%Xz>|DdGAXgfMu=NJUeOjA;>EBv5qmK6| zyA{1p&w;)RXfaw0D9HTUU0c`{kQieBOQ>0CXGc+@R>&qfA`S}tx$f%)ce~6_be4+F zmY*q1EkL2;&=d$PTG|{|@}af}RrvRdRT(#LuQ468c0YI>yV4#%U$49-ofKwwLSFD; z9ffV2!PgB9N(efV>kgO{tzsFg^R+w0SJw&P37}5->~m%O@D-itbzU7{OZuDkRLb`b zeMA=p@n$OLGC!Y?PDw%4K~+#U;5h1vEAX4clu(Q()d;N`FdrW#$bhD}a|eHGMYp-N z(9T+XRcDcY9M&nc(!$4)CLtaeYHl?&(S&A4Q`vw_MN=+wDky( z-1HnyIb?{5`RP`MUp{CX*FlXNW`3n(G-8QTxA)z)hL)$%6+}omt=@Iu#{L@bVs$u* zdM~D4v2Yj`9J}ES%IdSnhk;V~^O}77f)8wn-peO$`MKFFfo-j%yVABY&`jec4i{Dn z>qg2Q=+-DBq7_HYENk()#4I>O1~_XE+~(z4N7Z92Ul+KPJy-bsB%e@lCjS7lPMH3g za{U^~bP+Uly1P9+NvW+7w)=5RYm)DSqLy}P0?}1BMZ693-p0a&p_|2dfIda|Fv%rx zer7V$=jsHapUJngV<-v3P2|GQtx$v=LqT5UpQ`PZoPm{JtS@n_HR0R_>YT7qMIMKe zK6V`^D=aMngIx9cwe&|C$Z`rF-*X}DO-ZO=ZeKTJlG5}7dggM}S^qcJ0@oUvU92W& zJTD|(OYv8{S&FnQe|~62{8;!Ps-=l?)4OW+UT+$ax(5)qI>QzP5C#u(L%+rxgj~D~{1UA*JcKUbd70iwmXe#2EMR^laemXxO=izasXm1o= z0(U`&Y%L_Owxw&*o7vEu;AxAX-!Tx)kh%la>2Bt>6#x2DwBLg5yL@~{ z_*j3)BYwI*N#P4eLfSu>AqCaj-OqwDf^780V^PC}Zf{x4&)arO#ZrUV_qbJ{C0Z2d zqGdSaI4;*F0rkUf-tzyJKN&c>?w6wLVD;A}C(504!2kBhHbluH?^9Nw3Go_&J#j(Gk=uyf&v-6dcYs>4+MXD|Ph zq|s$DFh^qybj`l{^?$ay$uD5d!S8SoC%za2N%!jyfaabzKFlo>je|eA=}}w zCW|ik>_h)eLfwT77YuyeA#{|%k~M3IWA)g2iR+3QxlKK7Pp0*yYP#`7o_W7HHjgn} zFbz=Z>YOat9UE3`DH`<2i4-|CJYHi zNVBn=EBn#2DdaISo6rO!`Ua{~SOfYM@ufm_7TGsNje8^AXNIHf`rFssKc1d636>M} zIp_SNDnc8^J5BLZksFEaV;h#NBpiS!Kb@w*E$qG03Cpl5kNlB+1(nQarP8OOxtz#! z(wA^nc|peKYKA&R@wL?2Q~$JB72~hlA5ScTAZ7OlPg+|JDUq4=`k~>Qb`c0wTyi8o zcWlU1>0ZF-mFS!UlG6~zN+*gfyXkJA$ICT)5%$O4==Gi5Kk0C8FPQ5Bjaq5~L30LpqYSLi^Q15i%vfTSF60*AUF>7Y8 zEjl={wm3ZR3lgRXgS=ymOOCxA>|a$a>XBBjlnyWT!R=CKF3ZI$o^NX`9*WAty$xQq zjO12gOncjVK>RcZTfqI?HMvKfrdr9f>~*zQXu}`iT4rQ9qc1=a)l{9}wpeEm|9QO3 zeRQmRDjcA=KAn6@l&1Dz^$wzVog25f8SM(};$<93k!O%`%ntR~p~)F14;4P4M^H8J z$c@M@n<*RPvRn~^dM=SiL9#Z`B^m?!8E z#JH%p_TQ4m6MGBMHpyuI7`|_Y14AT>j(Xhx41r5<23kz(d@KI+VO!4Gv)F@ZAI}XsV`Cng zaitDjGD)QpaN z0{8kX(Tu)1ExXFWObyGR(?5KV{V(V*FWT%_Mt2jCo3IsX9h?56qQW_eYmwx1zj&~+ m2oxikR^OzcTb48Ao4*m*R%Pb$qV9LLeOWJa8L++hcI1DK!4KvD From f0421d4c00d7ccef86ea2020fb5161add08b2ae3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Feb 2016 14:49:00 -0500 Subject: [PATCH 269/502] Clarify comments in chardata._build_regexes --- ftfy/chardata.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index d6641b48..40a36265 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -33,15 +33,17 @@ def _build_regexes(): encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')} for encoding in CHARMAP_ENCODINGS: + # Make a sequence of characters that bytes \x80 to \xFF decode to + # in each encoding, as well as byte \x1A, which is used to represent + # the replacement character � in the sloppy-* encodings. latin1table = ''.join(unichr(i) for i in range(128, 256)) + '\x1a' charlist = latin1table.encode('latin-1').decode(encoding) - # Build a regex from the ASCII range, followed by the decodings of - # bytes 0x80-0xff in this character set, followed by the decoding of - # byte 0x1a in case we made that one special. - # - # All regex special characters are in the \x1b-\x7f range, so we don't - # need to worry about escaping them in the regex. + # The rest of the ASCII bytes -- bytes \x00 to \x19 and \x1B + # to \x7F -- will decode as those ASCII characters in any encoding we + # support, so we can just include them as ranges. This also lets us + # not worry about escaping regex special characters, because all of + # them are in the \x1B to \x7F range. regex = '^[\x00-\x19\x1b-\x7f{0}]*$'.format(charlist) encoding_regexes[encoding] = re.compile(regex) return encoding_regexes From 527a931a8a3f8f780024c7c473b287595521d93c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Feb 2016 14:50:18 -0500 Subject: [PATCH 270/502] update doc configuration for 4.1 --- docs/conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f5188f43..4047e44b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ # General information about the project. project = u'ftfy' -copyright = u'2015, Rob Speer' +copyright = u'2016, Rob Speer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '4.0' +version = '4.1' # The full version, including alpha/beta/rc tags. -release = '4.0' +release = '4.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From ba9bcef6d1e4caecc3777f8a882e39b38be13953 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Feb 2016 15:03:52 -0500 Subject: [PATCH 271/502] describe installation in README.md --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 347cc42d..ea01b50e 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,16 @@ LOUD NOISES ``` +## Installing + +ftfy is a package that can be installed using `pip`: + + pip install ftfy + +You can also clone this Git repository and install it with +`python setup.py install`. + + ## Who maintains ftfy? I'm Rob Speer (rob@luminoso.com). I develop this tool as part of my From 07edb54b3757d7233aa0cc9907075d54a1815d2f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Feb 2016 15:14:46 -0500 Subject: [PATCH 272/502] add docs for ftfy.formatting --- docs/index.rst | 6 ++++++ ftfy/formatting.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 5f774491..75f5c41e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -302,6 +302,12 @@ Variants of UTF-8 .. automodule:: ftfy.bad_codecs.utf8_variants +*ftfy.formatting*: justify Unicode text in a monospaced terminal +---------------------------------------------------------------- +.. automodule:: ftfy.formatting + :members: + + *ftfy.chardata* and *ftfy.build_data*: trivia about characters -------------------------------------------------------------- These files load information about the character properties in Unicode 7.0. diff --git a/ftfy/formatting.py b/ftfy/formatting.py index 11a64f3a..17ad9630 100644 --- a/ftfy/formatting.py +++ b/ftfy/formatting.py @@ -90,6 +90,10 @@ def display_ljust(text, width, fillchar=' '): Table flip▒▒▒▒▒▒▒▒▒▒ (╯°□°)╯︵ ┻━┻▒▒▒▒▒▒▒ ちゃぶ台返し▒▒▒▒▒▒▒▒ + + This example, and the similar ones that follow, should come out justified + correctly when viewed in a monospaced terminal. It will probably not look + correct if you're viewing this code or documentation in a Web browser. """ if character_width(fillchar) != 1: raise ValueError("The padding character must have display width 1") From adda049461ff03c32ca7a16b71663ca998bfc03d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Feb 2016 15:23:54 -0500 Subject: [PATCH 273/502] Refer to Unicode 8, not Unicode 7, in docs --- docs/index.rst | 6 +++--- ftfy/build_data.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 75f5c41e..73452433 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -310,8 +310,8 @@ Variants of UTF-8 *ftfy.chardata* and *ftfy.build_data*: trivia about characters -------------------------------------------------------------- -These files load information about the character properties in Unicode 7.0. -Yes, even if your version of Python doesn't support Unicode 7.0. This ensures +These files load information about the character properties in Unicode 8.0. +Yes, even if your version of Python doesn't support Unicode 8.0. This ensures that ftfy's behavior is consistent across versions. .. automodule:: ftfy.chardata @@ -323,7 +323,7 @@ that ftfy's behavior is consistent across versions. Future versions of ftfy ======================= -ftfy has full support for Python 2.7, even including a backport of Unicode 7 +ftfy has full support for Python 2.7, even including a backport of Unicode 8 character classes to Python 2. But given the sweeping changes to Unicode in Python, it's getting inconvenient to add new features to ftfy that work the same on both versions. diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 841920ff..e5e41a8f 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -5,7 +5,7 @@ classes we care about change, or if a new version of Python supports a new Unicode standard and we want it to affect our string decoding. -The file that we generate is based on Unicode 7.0, as supported by Python 3.5. +The file that we generate is based on Unicode 8.0, as supported by Python 3.5. You can certainly use it in earlier versions. This simply makes sure that we get consistent results from running ftfy on different versions of Python. @@ -43,9 +43,8 @@ def make_char_data_file(do_it_anyway=False): Build the compressed data file 'char_classes.dat' and write it to the current directory. - If you run this, run it in Python 3.5 or later, even though that requires - an alpha version at the time of writing this code. It will run in earlier - versions, but you won't get the Unicode 7 standard, leading to inconsistent + If you run this, run it in Python 3.5 or later. It will run in earlier + versions, but you won't get the Unicode 8 standard, leading to inconsistent behavior. To protect against this, running this in the wrong version of Python will From 3e10f8a52b00b20b87b9adc5f24d3980adf6590d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 13 Apr 2016 15:23:08 -0400 Subject: [PATCH 274/502] Fix stdin decoding on Python 3; bump version to 4.1.1 --- CHANGELOG.md | 6 ++++++ ftfy/__init__.py | 2 +- ftfy/cli.py | 13 ++++++++++--- setup.py | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9668c21..07c486be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## Version 4.1.1 (April 13, 2016) + +- Bug fix: in the command-line interface, the `-e` option had no effect on + Python 3 when using standard input. Now, it correctly lets you specify + a different encoding for standard input. + ## Version 4.1.0 (February 25, 2016) Heuristic changes: diff --git a/ftfy/__init__.py b/ftfy/__init__.py index e4951be3..df76f61f 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -17,7 +17,7 @@ from ftfy.compatibility import is_printable import unicodedata -__version__ = '4.1.0' +__version__ = '4.1.1' def fix_text(text, diff --git a/ftfy/cli.py b/ftfy/cli.py index 4aaccbe3..45000bce 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -5,7 +5,8 @@ import sys import io -ENCODE_STDIO = (sys.hexversion < 0x03000000) +import codecs +PY2 = (sys.hexversion < 0x03000000) ENCODE_ERROR_TEXT_UNIX = """ftfy error: @@ -72,12 +73,18 @@ def main(): encoding = None if args.filename == '-': - file = sys.stdin + # Get a standard input stream made of bytes, so we can decode it as + # whatever encoding is necessary. + if PY2: + file = sys.stdin + else: + file = sys.stdin.buffer + else: file = open(args.filename, 'rb') if args.output == '-': - encode_output = ENCODE_STDIO + encode_output = PY2 outfile = sys.stdout else: encode_output = False diff --git a/setup.py b/setup.py index 11d7eebe..73a473b7 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="ftfy", - version='4.1.0', + version='4.1.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From 522facfe2b84fc4cfa46aa3c6b25e93bb0b8c272 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 13 Apr 2016 17:49:47 -0400 Subject: [PATCH 275/502] cli: use the version check from ftfy.compatibility --- ftfy/cli.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ftfy/cli.py b/ftfy/cli.py index 45000bce..f4a99839 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -6,7 +6,7 @@ import sys import io import codecs -PY2 = (sys.hexversion < 0x03000000) +from ftfy.compatibility import PYTHON2 ENCODE_ERROR_TEXT_UNIX = """ftfy error: @@ -75,16 +75,15 @@ def main(): if args.filename == '-': # Get a standard input stream made of bytes, so we can decode it as # whatever encoding is necessary. - if PY2: + if PYTHON2: file = sys.stdin else: file = sys.stdin.buffer - else: file = open(args.filename, 'rb') if args.output == '-': - encode_output = PY2 + encode_output = PYTHON2 outfile = sys.stdout else: encode_output = False From d819245ec7674e0c0c791afa371b61659ea18531 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 1 Aug 2016 18:32:12 -0400 Subject: [PATCH 276/502] Heuristic updates - Math next to currency is okay - "Ending punctuation" wasn't a general enough case, expand to other common symbols - There are even more emoji now than we expected --- ftfy/badness.py | 23 +++++++++++++---------- ftfy/build_data.py | 9 +++------ ftfy/char_classes.dat | Bin 3896 -> 3879 bytes 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 0ac55f04..70d9a3ae 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -21,8 +21,7 @@ # m = Letter modifier (Lm) # M = Mark (Mc, Me, Mn) # N = Miscellaneous numbers (No) -# 0 = Math symbol (Sm) -# 1 = Currency symbol (Sc) +# 1 = Math symbol (Sm) or currency symbol (Sc) # 2 = Symbol modifier (Sk) # 3 = Other symbol (So) # S = UTF-16 surrogate @@ -71,9 +70,9 @@ def _make_weirdness_regex(): # - Modifier marks (M) # - Letter modifiers (m) # - Miscellaneous numbers (N) - # - Symbols (0123) + # - Symbols (123) - exclusive_categories = 'MmN0123' + exclusive_categories = 'MmN123' for cat1 in exclusive_categories: others_range = ''.join(c for c in exclusive_categories if c != cat1) groups.append('{cat1}[{others_range}]'.format( @@ -84,16 +83,20 @@ def _make_weirdness_regex(): WEIRDNESS_RE = _make_weirdness_regex() -# A few characters are common ending punctuation that can show up at the end -# of a mojibake sequence. It's plausible that such a character could appear -# after an accented capital letter, for example, so we'll want to add a -# slight preference to leave these characters alone. -ENDING_PUNCT_RE = re.compile( +# These characters appear in mojibake but also appear commonly on their own. +# We have a slight preference to leave them alone. +COMMON_SYMBOL_RE = re.compile( '[' '\N{HORIZONTAL ELLIPSIS}\N{EM DASH}\N{EN DASH}' + '\N{LEFT SINGLE QUOTATION MARK}\N{LEFT DOUBLE QUOTATION MARK}' '\N{RIGHT SINGLE QUOTATION MARK}\N{RIGHT DOUBLE QUOTATION MARK}' + '\N{INVERTED EXCLAMATION MARK}\N{INVERTED QUESTION MARK}' + '\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}' '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' + '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}' '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' + '\N{NO-BREAK SPACE}' + '\N{ACUTE ACCENT}\N{MULTIPLICATION SIGN}\N{LATIN SMALL LETTER SHARP S}' ']' ) @@ -126,7 +129,7 @@ def sequence_weirdness(text): """ text2 = unicodedata.normalize('NFC', text) weirdness = len(WEIRDNESS_RE.findall(chars_to_classes(text2))) - punct_discount = len(ENDING_PUNCT_RE.findall(text2)) + punct_discount = len(COMMON_SYMBOL_RE.findall(text2)) return weirdness * 2 - punct_discount diff --git a/ftfy/build_data.py b/ftfy/build_data.py index e5e41a8f..ca0987e4 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -28,8 +28,7 @@ # M = Mark (Mc, Me, Mn) # N = Miscellaneous numbers (No) # P = Private use (Co) -# 0 = Math symbol (Sm) -# 1 = Currency symbol (Sc) +# 1 = Math symbol (Sm) or currency symbol (Sc) # 2 = Symbol modifier (Sk) # 3 = Other symbol (So) # S = UTF-16 surrogate @@ -82,15 +81,13 @@ def make_char_data_file(do_it_anyway=False): cclasses[codepoint] = 'M' elif category == 'No': cclasses[codepoint] = 'N' - elif category == 'Sm': - cclasses[codepoint] = '0' - elif category == 'Sc': + elif category == 'Sm' or category == 'Sc': cclasses[codepoint] = '1' elif category == 'Sk': cclasses[codepoint] = '2' elif category == 'So': cclasses[codepoint] = '3' - elif 0x1f000 <= codepoint < 0x1f900: + elif 0x1f000 <= codepoint <= 0x1ffff: # This range is rapidly having emoji added to it. Assume that # an unassigned codepoint in this range is just a symbol we # don't know yet. diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index 5a92ba547aba78224af6cdf75b7a6eb0aee6d9ef..4cd9591cc3e9a4f455a3ec783f9c3536ed552797 100644 GIT binary patch literal 3879 zcmeHJdr%VS7Pqa|Ez`Cwdz#+L*65*`mZm19tFHOZSAxN&CaZ#^f*?Mqwd|#F%Y34e znV1-oqN1VF%hV`IN%0Yo8WJMn14&Wfw!3$BXYOBj_WpHe_WW_?eCM3sncw%D^UZwN zA{pO3p7{DJoexzXS6^NKpe`%45=X?lC7Bq079RV#Fv&GL98WUxIL@^yGg7g%_n@p- zq+8IW60@Xlg&_A+uT{sYRx2<;TZV0)`l(#@x)+jqJk~iLEAMP~(-ep+2p$^0lp=Tg zYx0CBv$-omU8vM52ogt~FI@tR&a0Wl^NY+?N~U)yXoKUDdXWimo~0L7*p=W^OHJCq zqa-#d{^f#0Xd;RItahlQ@?g=;nMPA@fj9qhj`T(j8f9M$q)!Ch`5p+n%#{>vLgyRO zC`XcO$G903`#f2^fyvm=Y6`=WnG8Ngvssd2V(em|9Bdt_9aGvXAun z2d#M1W8IlJM{dRj0z*&*!4?+&(J1A|HlM=8U7Q=`debWz%2AX-^7Z2}>l}|mV7m!z zP5f%!RL%YrVxC7KM76-4sb6Dm{W44>tjK<~TPn$cg_|{8RPZu`V-(e?FlStCa9<%~FMuN~}>uEXwq&Fvt->L9bAT+2K3%7U0c zOvsOSg&wy@)=&Z_V!~t*2x_c5O9S}sTBVJI2jGP~oaK^DqR9i`xgNqb_(>L2^Uqs* z)n|p9bAdkNE7wbq7Ftg=<8on@R&lB~t>PRpXL-+#~C zb)FwtNpHHWUn(dZRM36-fp@`YrXcR&ZTroj@YU(zAW+fQhA_R}$V2d^w^1}v)c{c+ zbne8EQ!3h_IE2`>oNj%+;7wRsRVMPGMs@&r_Pi#5bor7?-q_|OiIJhyCA%A9xBWqp zH#e(dXxNUj)Kcb9=4^?%3I<#Ck_1dmi^K?MC?v`+=hsdeOou;vq0Bo%k&yBohGG;q*!LFjhvZ-yg37-x=W-jsA!>U4`)WfZUsQxJhwa9xB-B@c+ISu+z2+rr!qP!jrukz^^e^7%a3CmQp>$Ae%L6G33bjf~i zlA}+BKgQ1U@@ePT5fnYma{DRxa~KO$b|yGD)M3k>0utpJhH#RhGBQmOqhdI2&rp5R z_bw)0xn6P(kD0d1iz6y&oS)kg%~%Tb-d*|&g|BGB<#0;{j%^P6)6L%JNkrD6tD9`~ zcvb$__3n<>J;vpbo(G&~26BB^FRS*Fmo`^pou59BwdL4sECY#W=hf|}`HZLfgeD@; z>jF;oX>m~WHba_SJ@le^!GQ79NedCKFc{3$u;Ad-Jg#UZ4kt~^5(0yx%);=iHjVA@ ztO;7z-mM=NlC%S=nO2If34L;11{TVnl{~r<(ThGg=)*H*z|06lFBMkq^`KZ> z%jDoplGREthE`V6pSMisSL>gM6GB-`)87vN;k0DgsD*cRT5??CvYS9oViNwyjy2&p zHp#a66RX1LJ;s){eO03BK`qjB%ww}tgOzFu0Z_;ii?^91RWw^&N8sO%aO=P_)Yi=1 zsp8Y`7`PuL(GC-dX*qAt?Z}Fsmqj;Ea%?l^;#{=aOmsq));s~d9YhgyjQa&=qlC&M z@GQGsZXm4nRkZ)Z=|}HI*U^X$p)($stOJ0qjKVJOb?UGaVVcRkY4|;pVAhf)i>JI4 z>no7SZ2U*_yjcJkhD)OK=4a1zjxKr_f2UW~7I=dpo#>41FV*+(eGS0uey1&f+P z>8X%~=B5D{RGDI6@^%v}5D(`j9Y5PcaL{y_Pm9qRguTVs$sHCwj_nG+9K3Y~qNT+o zd0`E2nCr(YAr;cts%-ZXQ}>`|k(UZuej_rZ1AYpIy4!&%A;AJKj+wj($xTtf2T}=r zEb2s%G=@ri-{eqt{_8sqsa~R8}0fiRB60@^-Jg#;u|C#A(o1w>Nvg- zz-zLYiy6Jf{P1QBgG^nUVmxgyDmDZTDvKucfc3FK#>m$EU81v!l&Pk+0c;oVlsHy+ zG$cqy6l?9BAo6h7S9h*3%SF$lr84^GJmk(<%PvU-v2LT1wc{p}AaT`&7)KrwT?19;N659*l4bojp3|E_^j9<_|1R8|Xia-;(u| zJ3#e5JUbDxG&zhUbKz7ODdXrszUC(dQ6pL+E9~P$Nlb==uU6fBC&9`5)mBLs>`vaS z88O_FHS4n3W?rqPxmy!YGObeK&4u0hF6(7@Sx;Vj6Tlp}i zpt>Xk*Q~{m9)k=@QTsls&(@F6^bv?fwAQADCms`gjo4;vv+&(=iddiS-^E0?Jlgx` zd-yH7U^D$-E*n$KJ_# zxC{BXHCJ&O^s(FaGgEa+3;NvcYSo6%dH7T7we20-^z4TnqzQfFb^{mDW}2#!QTq}e<_f%YT?z0S+ds-@fmzRhG+A?v7bV#fB(Y#ZD_k4HGd~@zeDB!c4 zVqUhcSL-SJxI;gB{T=+*rybl^)RV#C%4;3T=iN6?5+=NHvA{H+m@!1wQ0f6c<&s#$<9r9sbhx5$9<3# zsHP5e&nWRSpAbCjQ7HU!k@vlsMaB3?dU~TK6d843j zzsu60mW)0L!A%%EG3zVkC!VQgI2L^_a-wJ0b>X`kwODzC$>Y4%^`;JMYS5gV8k27R zHF7f{aXi1?Cu@q{1#v`K9P8Utor6TX8>4!9jcDzTQ3U<1$b7E(kxQBleqk7oO9MV( zp0ihRy>AH)m2unoyevt^f#}aCc)Fhu{Ex!-izZyGoL(bQwOPB$&zP78nuPjVGG8p5 zJp1FYb4E+1p>}xt(B^a43MY!RZKn|!*N3}UQFWPvaRM>U0=?jLb4|}21VMh}fP+-g z&})uNDz|=EOlgp7irHU;l`6BHFUYW4-X0Z0UV#^MO5>ZJMlt3pe0NU_nITE3wqmqR zO5EAGTQ7}NabgYT%plhdnG?2yb4B5CTxw|RKk!)jlkzUtBi+uC0!k`Bhtxk!U5d}7 zym(LKGO_4Ok85N|OpMl{uZ|Yho~deg15MQGc?YEgi*Yc?4)Q!^&R9Q+_;IGydA|H|pk&M*~5{h=a<*{9{cbBe|p@<#cs}!3;1!}@u1k?my(YOaVa4LgR?5=~hjq3Ga z6+MkF^q1h*=><(46f=Ehbx{V@Z)^-+OF#Um`Qq)nw|=3tOyQ43Ec--6x|XHou!k4X zr1i`*q%^Mnp-*D`r|6zK%l1{B+mOeFzL?%F4#n>plNmRcCWM<&r6`yrhK{A)8LuJw zv6xEHxQxFSX0iL+Kir?fnlNXwleI2%Ui(;nLZ*lesFNP81+do~_@DL%a~meyP}p-@ zmvMLdN=oO)_+WX!9Qih5rKV2DBN(FHk1-<64{C zozw^OijiqH-f!-RLd$a}GR&K`z@`Y@QgpFT9{ip^>bnbX=Ja7dyxC{^iOI*gYg_p> zxid{`GvA>U$ecEj;9_oNCXnIf-3|A9+}xpaYBVzK`#?z2BM;1KaNJ#ai&!>rPpENh zx~99^wk1mt)?zf-+ljk!+jK##?3;;E0wvo2U`+@xK%VV1euJ-x#iudqBLrY z_8c5Ololvhcf_3Xn5IgQEj0=ZL@y%b7yX?Fd{0KB={3D6Aq+A5nr!ue=MbFl2GtFl ztmRFYjkb(ZuJ=tM$p$OIYQKhi^2jPvV7B=;_HX$e1*(vzy(F>y3Vd%UXw`)P+Nnn9 zSr}Inqor7#%9gD#tt{VXpxy9YjVbrRt@sc?L)V=KRs~pe0QLkQQptDmO4%1*R^URe zD;5-_N!`yG`F7rAm7DJ*b{0Q&rTBI-6`tMxWC%kA>LAuVRr!|8oxop$KXR+C(94c& z@upanKIB2i-|I8rm8pFfHA3qvL|2{6`9MbTvAtn-ww}w?8;wNq9zqg0noZ=Gi;^SF zW|KI+Y=ov20xHf=f!iu7y-Z`XK#3{IZGi#Ngcv4|3114ZQNNj;U9{q}<(#qyAUqWK<6=*Gz1+s99nIu|O5>*L?$$esbox%aDM$A( z#4AxeiKUOs^|WsiB8+Gfr=`Iizq*NAd9SjSN3LD|a(`o|UfIKyAc;$#sQ9a%<(J1r zSsF#ABz~`y@M_Fox~`;&F)6%yCZpw1(4FHoMcIWYTL|wT*4f=_MCBYpz+wO(<)x`| zX3Os9;@qWOE!6YEO`M_#w^@6Y9qqD(XY8ak&`LqnVON{h_B6Oh`pgVRA5j{Q1R>@F zZGFvLv5BcmC;i=l=muTa;~Q|+z4FbFNw&w>bG`N;MhnRR z7*pQ_^klt~W`LWpaYb4dg#W}Qw9qIF7EWsRB>iSi zJa_;GUZUXoHyr8L+1@XmkI|2x8i(@Hq4Qt+ij5Y`Rj*GdGjH1LzSfvXpDAv-{vqRf zGrcVa#X0;1#o{^H^>sGXfPk}e;K7&s^B>17uNqNptd5gtMfHOQ;p1FHQtk-%r$-a< z!=h@SY{1KPpK9%NM1Dmn9>zS;>!+6$#7iCx_@UhI3&F9|>($g!KEp{T*_g2FopaVZ zzArqM()~GB;=0=;&b=vzqpFHFQr3OppSLI`z8yJpFzzevsbOtK+P2g>zP7HqM(JDF zL@nAH2`t2gunYGl6oMC6&(so7ZS&7P{E)0_6*Xr4JJ+fu&1?VR*MFye2g=0@4}We< z4e~KlA|E56G=3?*vFfS7r2HuZeITib**G4Htoi5R-{@}#{;wQZ+eyFE<+_r0(uJfv nEU19}Z_SU*OaHay{^~=>lL_(ae^vauZf(oSHITkF;>AA!CUjLj From d477687703c043543e8dffa85d516ff253c828b6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 1 Aug 2016 18:33:31 -0400 Subject: [PATCH 277/502] update tests based on new heuristic --- tests/test_real_tweets.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index e90f7d4b..995780b2 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -48,6 +48,8 @@ ('OK??:( `¬´ ):', 'OK??:( `¬´ ):'), ("selamat berpuasa sob (Ã\xa0¸‡'̀⌣'ÃŒÂ\x81)Ã\xa0¸‡", "selamat berpuasa sob (ง'̀⌣'́)ง"), + ("The Mona Lisa doesn’t have eyebrows.", + "The Mona Lisa doesn't have eyebrows."), # This one has two differently-broken layers of Windows-1252 <=> UTF-8, # and it's kind of amazing that we solve it. @@ -62,18 +64,23 @@ # You tried ('I just figured out how to tweet emojis! â\x9a½í\xa0½í¸\x80í\xa0½í¸\x81í\xa0½í¸\x82í\xa0½í¸\x86í\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8e', 'I just figured out how to tweet emojis! ⚽😀😁😂😆😎😎😎😎'), - ('CÃ\xa0nan nan GÃ\xa0idheal', 'Cànan nan Gàidheal'), - ## Current false positives: - #("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), - #("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿"), - #("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), - #("SELKÄ\xa0EDELLÄ\xa0MAAHAN via @YouTube", "SELKÄ EDELLÄ MAAHAN via @YouTube"), + # Former false positives + ("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿"), + ("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), + ("SELKÄ\xa0EDELLÄ\xa0MAAHAN via @YouTube", "SELKÄ\xa0EDELLÄ\xa0MAAHAN via @YouTube"), + ("Offering 5×£35 pin ups", "Offering 5×£35 pin ups"), + + ## This remains a false positive + # ("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), ## This kind of tweet can't be fixed without a full-blown encoding detector. #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", # "Deja dos heridos hundimiento de barco turístico en Acapulco."), + ## The original text looks too plausible + # ('CÃ\xa0nan nan GÃ\xa0idheal', 'Cànan nan Gàidheal'), + ## The heuristics aren't confident enough to fix this text and its weird encoding. #("Blog Traffic Tip 2 – Broadcast Email Your Blog", # "Blog Traffic Tip 2 – Broadcast Email Your Blog"), From ed5fed1dcad726e598f133dba61d035caac0556a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 2 Aug 2016 14:48:15 -0400 Subject: [PATCH 278/502] The degree sign is also a reasonably common character --- ftfy/badness.py | 2 +- tests/test_real_tweets.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 70d9a3ae..f2011d7f 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -90,7 +90,7 @@ def _make_weirdness_regex(): '\N{HORIZONTAL ELLIPSIS}\N{EM DASH}\N{EN DASH}' '\N{LEFT SINGLE QUOTATION MARK}\N{LEFT DOUBLE QUOTATION MARK}' '\N{RIGHT SINGLE QUOTATION MARK}\N{RIGHT DOUBLE QUOTATION MARK}' - '\N{INVERTED EXCLAMATION MARK}\N{INVERTED QUESTION MARK}' + '\N{INVERTED EXCLAMATION MARK}\N{INVERTED QUESTION MARK}\N{DEGREE SIGN}' '\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}' '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}' diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 995780b2..5f5fa0d5 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -50,6 +50,8 @@ "selamat berpuasa sob (ง'̀⌣'́)ง"), ("The Mona Lisa doesn’t have eyebrows.", "The Mona Lisa doesn't have eyebrows."), + ("#╨┐╤Ç╨░╨▓╨╕╨╗╤î╨╜╨╛╨╡╨┐╨╕╤é╨░╨╜╨╕╨╡", "#правильноепитание"), + ('∆°', '∆°'), # This one has two differently-broken layers of Windows-1252 <=> UTF-8, # and it's kind of amazing that we solve it. From 21b42a3efcb10b8656f6efd467d81705713a4b46 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 2 Aug 2016 15:23:03 -0400 Subject: [PATCH 279/502] Add test for expanded emoji range, bump version --- setup.py | 3 ++- tests/test_synthetic.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 73a473b7..fb439fb5 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="ftfy", - version='4.1.1', + version='4.1.2', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", @@ -20,6 +20,7 @@ "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", diff --git a/tests/test_synthetic.py b/tests/test_synthetic.py index bf75ff24..5a1cb097 100644 --- a/tests/test_synthetic.py +++ b/tests/test_synthetic.py @@ -67,3 +67,20 @@ def test_unknown_emoji(): # characters. The mangled text passes through as is. not_emoji = 'dos burritos: \U0003f32f\U0003f32f'.encode('utf-8').decode('windows-1251') eq_(fix_encoding(not_emoji), not_emoji) + + # The range we accept as emoji has gotten larger. Let's make sure we can + # decode the futuristic emoji U+1F960, which will probably be a picture of + # a fortune cookie in Unicode 10.0: + emoji_text = "\U0001f960 I see emoji in your future" + emojibake = emoji_text.encode('utf-8').decode('windows-1252') + eq_(fix_encoding(emojibake), emoji_text) + + # We believe enough in the future of this codepoint that we'll even + # recognize it with a mangled byte A0 + emojibake = emojibake.replace('\xa0', ' ') + + # Increment the first byte to get a very similar test case, but a + # codepoint that will definitely not exist anytime soon. In this case, + # we consider the existing text, "ñŸ¥\xa0", to be more probable. + not_emoji = "\U0005f960 I see mojibake in your present" + eq_(fix_encoding(not_emoji), not_emoji) From 08f094f3ca05090f73137df439fd508704c26dfa Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 2 Aug 2016 15:42:42 -0400 Subject: [PATCH 280/502] Add a bonus for decoding a byte-order mark --- ftfy/badness.py | 1 + tests/test_characters.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index f2011d7f..3095fb34 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -97,6 +97,7 @@ def _make_weirdness_regex(): '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' '\N{NO-BREAK SPACE}' '\N{ACUTE ACCENT}\N{MULTIPLICATION SIGN}\N{LATIN SMALL LETTER SHARP S}' + '\ufeff' # The byte-order mark, whose encoding '' looks common ']' ) diff --git a/tests/test_characters.py b/tests/test_characters.py index 00a18314..49fc1a72 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from ftfy.fixes import fix_encoding_and_explain, apply_plan, possible_encoding, fix_surrogates +from ftfy.fixes import fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, fix_surrogates from ftfy.badness import sequence_weirdness import unicodedata import sys @@ -25,6 +25,8 @@ def test_bmp_characters(): fixed, plan = fix_encoding_and_explain(garb) eq_(fixed, char) eq_(apply_plan(garb, plan), char) + else: + print("%r <- %r" % (char, garble)) def test_possible_encoding(): @@ -33,6 +35,10 @@ def test_possible_encoding(): assert possible_encoding(char, 'latin-1') +def test_byte_order_mark(): + eq_(fix_encoding(''), '\ufeff') + + def test_surrogates(): eq_(fix_surrogates('\udbff\udfff'), '\U0010ffff') eq_(fix_surrogates('\ud800\udc00'), '\U00010000') From 1e3f2b6eee695e2b399a1780b01f94c35ea8c178 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 2 Aug 2016 15:54:24 -0400 Subject: [PATCH 281/502] Fix the heuristic for variation selectors. They're not "marks". --- ftfy/build_data.py | 4 ++++ ftfy/char_classes.dat | Bin 3879 -> 3876 bytes tests/test_characters.py | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index ca0987e4..2d052718 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -114,6 +114,10 @@ def make_char_data_file(do_it_anyway=False): # `´ are much more like quotation marks than modifiers. for char in "^~`´˝＾｀": cclasses[ord(char)] = 'o' + + # Variation selectors often go with symbols + for codept in range(0xfe00, 0xfe10): + cclasses[codept] = '3' out = open('char_classes.dat', 'wb') out.write(zlib.compress(''.join(cclasses).encode('ascii'))) diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index 4cd9591cc3e9a4f455a3ec783f9c3536ed552797..f2b01569b581e355e218a68e296201b39179b77c 100644 GIT binary patch delta 1025 zcmV+c1pfP{9;6#6D2_4uA` zt%Z)YYU{e(j;rz4;b!yOY}@&Ic^d6AlvfI~>G|~Cc;YU2v-#ay-vggVyu!9gUJXKypK8FXB#q?*j1)W>I(Eqb0Gn30Z71=H5!T!Is{_b`# z{jKL89liAr44Dwud`^ARw|}zxd3W}HyhDSqTt;K?baty#%{+S+zkB$b=`>qw`=?<} zHl{xq{ic68zbXArTzTH^Ef3WHcsQewi?T*~;~C}U{K@86%rqb?A z9~-Y`G-lC6`(HaJ>GL;WJ0k(bo0!@4(!^Z-yM}Y(5N(^;xG3hJKYw}>JKJ+S_E*>h z+70U2_X{5FSn)ge5uC_@9xd^*ZWBuc6k2@Jke{m|C!6jXQ|TrX8WJlv=jgU-0$7z&cHw8 z-McFQ007{Cci&LpTYq?WHwFLz000000Qj-q{f8s`qqg;*!Sgwv|5VnsOjpk=J>LJj zPhBmCP`VZJ{;*ngdF#hb&R5^ueBdSaZvXQebBu550oC({*(vAUx?Wfn0000000000 z000000002Mn|R^~008_P4|)FCj;Evz=M0~r)?Jv!=PbUha(@5-007`0m;Ok*%YR=* zg`pnm_iH^sOg#ofJ^#8M$sKukTKkvTdP!G(iu=nysfqt%hE1hbI~w*r$;3j`3pgnVW3 vlf9;p^R7S4yXza1kqbTp_`3d+kqbWtz@IF8cMg-03qJ|?3bO3~PQy<+7-%03 delta 1026 zcmV+d1pWJ@9;Y6#T?YcwQIlT>V1JJ(->^N5RgX{}C}w5-=bx7uOJ{3t+ODUjU)SS% zwzU>I)~c=RayzcZUx%B`Z?kRZ>*Z;*&rn_|%%;LYZDZ+#DZ9`O$2=M_KN z_MGesKecXF9&f&wBjsWAeDJx&*?7*^Qs-{pIayZio9F*KLSFA13O<7WTg(j=Q(#mGUp9Ugmy$OxtxT^HCm(vUn`2$0Eos%ctluYT9i_O64bOYu~B4fz}%Ocv9h)fRMa`9lBCn#@ct?^I;Bpa=W^*802K z!SuJDe{}TLKQLrMT=O~gNq^tU?&sav`|%D9!g3jn!PD8TPBruFS^VzdZ>H01t?i$N zIoX*0VDy{*<@~1fJ8|WCySF@0|NXp_b3!i48tIK^l$Y}-n`1G*^F^E zyEA=kyqeLNMHB6R?VO~~-+=9m1Qcgv-bM@~U&WS^`ZD!-5n16#dCzN)!=XmU| zunDvq)U)pwJle71ckav2++wVI-hH}{u6kj*KGeYmygdpz3%b3Xs6tZSLBo>_Xl z|979dS`MLfE9CuQwd(TLkDHvYzPb6pOYGhL=QrjU-_!%D=MA${&bxKJuqprm00000 z00000000000Dw2~#1Q}h_&FZ({IeZTNgK`?K0~d$FpbYye1BWz00000z&|eik#?8= zzKRM%J=E{ldVrXE42XLEbv=?h^6<3wFSGTMuKE=Bmw!?d|HsH13p?RF>vPuXI@y7B zrqU^LSL86O-U?ZFovcZwo7}PTRIxYF*B>UzFfB~^ znFz2=*nKYTTFajzSk{{rX2$=rajAY?sxWQ%W?q=?B$M?b=|?-*tydY$mX`07BoC^% zZ{oFgo4=mkSF`WV47|>%^zKXm006JE@eVrx000000F##sBQmd%ZT0Sq0ssI2U(}De zfAHPQ2LJ#70Koff{!=~x0000000000fFI%Af8EJf_U_&Zvylt60+T=s1`vD+`O4xa wdrcwdU4NK&*Ef@q3qAz!b^QU8kqbTsfInIG?i`bm3qK0*6=d1}0SMku+BnN0YXATM diff --git a/tests/test_characters.py b/tests/test_characters.py index 49fc1a72..3b02cd9f 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -39,6 +39,12 @@ def test_byte_order_mark(): eq_(fix_encoding(''), '\ufeff') +def test_emoji_variation_selector(): + # The hearts here are explicitly marked as emoji using the variation + # selector U+FE0F. This is not weird. + eq_(sequence_weirdness('❤️❤️❤️❤️❤️❤️❤️❤️❤️'), 0) + + def test_surrogates(): eq_(fix_surrogates('\udbff\udfff'), '\U0010ffff') eq_(fix_surrogates('\ud800\udc00'), '\U00010000') From 1e9509c9c87302d808f7d1daf641f1e1eeec25f8 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 2 Aug 2016 15:56:40 -0400 Subject: [PATCH 282/502] Fix not_emoji example --- tests/test_synthetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_synthetic.py b/tests/test_synthetic.py index 5a1cb097..d919f3c5 100644 --- a/tests/test_synthetic.py +++ b/tests/test_synthetic.py @@ -82,5 +82,5 @@ def test_unknown_emoji(): # Increment the first byte to get a very similar test case, but a # codepoint that will definitely not exist anytime soon. In this case, # we consider the existing text, "ñŸ¥\xa0", to be more probable. - not_emoji = "\U0005f960 I see mojibake in your present" + not_emoji = "\U0005f960 I see mojibake in your present".encode('utf-8').decode('windows-1252') eq_(fix_encoding(not_emoji), not_emoji) From eadbf3e18016442ac9a9ddb89f03b056d2cc9a4c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 2 Aug 2016 15:56:53 -0400 Subject: [PATCH 283/502] remove debugging output from test --- tests/test_characters.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_characters.py b/tests/test_characters.py index 3b02cd9f..b3b6ea27 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -25,8 +25,6 @@ def test_bmp_characters(): fixed, plan = fix_encoding_and_explain(garb) eq_(fixed, char) eq_(apply_plan(garb, plan), char) - else: - print("%r <- %r" % (char, garble)) def test_possible_encoding(): From 24c5097779c0661eb837cae360d87091c71830ea Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 2 Aug 2016 15:57:57 -0400 Subject: [PATCH 284/502] Add a missing check in test_unknown_emoji --- tests/test_synthetic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_synthetic.py b/tests/test_synthetic.py index d919f3c5..865ec4c8 100644 --- a/tests/test_synthetic.py +++ b/tests/test_synthetic.py @@ -78,6 +78,7 @@ def test_unknown_emoji(): # We believe enough in the future of this codepoint that we'll even # recognize it with a mangled byte A0 emojibake = emojibake.replace('\xa0', ' ') + eq_(fix_encoding(emojibake), emoji_text) # Increment the first byte to get a very similar test case, but a # codepoint that will definitely not exist anytime soon. In this case, From 6a20caa8ae0614f39e870937dd5095f499edf4a9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 2 Aug 2016 17:48:30 -0400 Subject: [PATCH 285/502] Make the heart-emoji test clearer --- tests/test_characters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_characters.py b/tests/test_characters.py index b3b6ea27..a7318b95 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -40,7 +40,7 @@ def test_byte_order_mark(): def test_emoji_variation_selector(): # The hearts here are explicitly marked as emoji using the variation # selector U+FE0F. This is not weird. - eq_(sequence_weirdness('❤️❤️❤️❤️❤️❤️❤️❤️❤️'), 0) + eq_(sequence_weirdness('❤\ufe0f' * 10), 0) def test_surrogates(): From 117839155b20d3e4abfed36acbe07a3023eedc65 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 3 Aug 2016 12:46:42 -0400 Subject: [PATCH 286/502] Fix build_data cases involving emoji modifiers --- ftfy/build_data.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 2d052718..e6ddf3dc 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -77,6 +77,10 @@ def make_char_data_file(do_it_anyway=False): cclasses[codepoint] = 'm' else: raise ValueError('got some weird kind of letter') + elif 0xfe00 <= codepoint <= 0xfe0f or 0x1f3fb <= codepoint <= 0x1f3ff: + # Variation selectors and skin-tone modifiers have the category + # of non-spacing marks, but they act like symbols + cclasses[codepoint] = '3' elif category.startswith('M'): # marks cclasses[codepoint] = 'M' elif category == 'No': @@ -87,13 +91,6 @@ def make_char_data_file(do_it_anyway=False): cclasses[codepoint] = '2' elif category == 'So': cclasses[codepoint] = '3' - elif 0x1f000 <= codepoint <= 0x1ffff: - # This range is rapidly having emoji added to it. Assume that - # an unassigned codepoint in this range is just a symbol we - # don't know yet. - cclasses[codepoint] = '3' - elif category == 'Cn': - cclasses[codepoint] = '_' elif category == 'Cc': cclasses[codepoint] = 'X' elif category == 'Cs': @@ -102,6 +99,13 @@ def make_char_data_file(do_it_anyway=False): cclasses[codepoint] = 'P' elif category.startswith('Z'): cclasses[codepoint] = ' ' + elif 0x1f000 <= codepoint <= 0x1ffff: + # This range is rapidly having emoji added to it. Assume that + # an unassigned codepoint in this range is just a symbol we + # don't know yet. + cclasses[codepoint] = '3' + elif category == 'Cn': + cclasses[codepoint] = '_' else: cclasses[codepoint] = 'o' @@ -115,10 +119,6 @@ def make_char_data_file(do_it_anyway=False): for char in "^~`´˝＾｀": cclasses[ord(char)] = 'o' - # Variation selectors often go with symbols - for codept in range(0xfe00, 0xfe10): - cclasses[codept] = '3' - out = open('char_classes.dat', 'wb') out.write(zlib.compress(''.join(cclasses).encode('ascii'))) out.close() From d51b41cd1de3d1c845bb36a7b22eee2ec5fac6f1 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 3 Aug 2016 12:47:19 -0400 Subject: [PATCH 287/502] We don't need to be so picky about Latin letters anymore --- ftfy/build_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index e6ddf3dc..1bb726bf 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -61,12 +61,12 @@ def make_char_data_file(do_it_anyway=False): if category.startswith('L'): # letters is_latin = unicodedata.name(char).startswith('LATIN') - if is_latin and codepoint < 0x200: + if is_latin: if category == 'Lu': cclasses[codepoint] = 'L' else: cclasses[codepoint] = 'l' - else: # non-Latin letter, or close enough + else: if category == 'Lu' or category == 'Lt': cclasses[codepoint] = 'A' elif category == 'Ll': From 174f32d8473095163a433019d479c94ca154ed2b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 3 Aug 2016 12:47:46 -0400 Subject: [PATCH 288/502] Update char_classes with new data --- ftfy/char_classes.dat | Bin 3876 -> 3891 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index f2b01569b581e355e218a68e296201b39179b77c..cd6688dc1366835930e3b4ca3be8323157a803b0 100644 GIT binary patch literal 3891 zcmeH{eK-?}7sqw`b#<>>dLea*TY0A_iNsckNZ#KvW>>KFJx*v?2t=+H$-Kt z`Ky=^zbq^t@1nx42JG{`L)xdh+hyS*En#W(afKozu|hL#rN6aAw4Y>DFwkXO>``Z|=_CxdZbKjcCifCMjG z0BRst(>S3pdNNDhmHq+gXp>vGu4F`4JHzV1FGrJf9FauEhp5am1wA8cD3>K)Qk-!( zkWt4DdXvn(@2jeB*x%qx8{PNhfqP0_;O(l=osIVGXZAZS7>!iCxL%?YJ}A|>BhH#W zhp|l01XmhRA%hSPqVbJvHG^8DO}tEvEd5-K;$1?O)@)c7wcWAq^Z7;jiMbzzKlzd; z1|=DP&tkLpkaL1$nGr_!jMRf`mxMGYcIFR1bne~nnAZ`%8eQ6y^@6Z4=LKwf^Kr<7 zYO5#7OG|*L_@DXGs_R!_xtFfp^9At4r6gR*PbD@Sqdk`ZHi~b}`}F8UHap@52Q&!w z&^3+FaR=NroKAEGpKzv=yIw%YqAc`?a#0yBx5x zde~!XkzM!Q+~qcEvzr|JBAX8+dDXIT$xgmFkRMsPl0gDs>l{8N^? z1M+AR!JX-B4v^B;>=5>Ytd}6W)BbYjbb!GAnT7)zB&RLsr&&9q#}j^e1;mtuuRYj+ z)V8MZ@czJ?hk1DUIrdGT3;pIpx)fa?tjvNbvwL!`N+V!=R2<2(JLqjj(`}e&J#@kI z@e=EpQxl_de0N^cYbT%qj`)4nscl+WHE*AZr$ykP=*y|1AU6Q09D!NPHd3FfjjL@M zf#ND=>iXRFy25?>EKMKYhzZ45HG6k##Q{;&-u-DsdF~DkGyR%JOqxorRzpvXI#q2^ zrQO!B96#1u@(3N96&9(_*(qiYx$G#DyhX#U$PXqHL`{1<`-QuJVO_K6+qhAn^7CTF zg2JJ&0?JDGR_o7Qj{Kt9KE)J0-0N7ya>UQ3+ue7VzNHUN1vhwB9fMetvhU?$*>~zv zw#6Hy;#|LkgihPcpfX!?-QRPxQD+f558B7($dRW5Ak^3J4vYOMX%FD-(bg4hY4bG) zsx-%Bf(i0iwKZk9W|tqBl#IKSdc)Vf;?zK+6?qO8L}z)$d3fJ^!{rIqY@g9NCtDF0@p!8=mDMs_ownAd{IQf(XZ*wGL=2J(AC zIN20vIqP3`z2>BJ-r&MU<={d>#SutM_my$HPYg_6DBMycxo!%sdpKZ*td-zW5(k#j1M6(71f>%Q<3% z5#gyn$sXX%DIPbxoz?r0To#AJf=;8cs-6Xl*hOzOXbJxWOzI1%0jpty@U~x|{FlT3 za54ue4wnx~Pu0;v3oWJ~%3!?wr&Kbl(ljc^=KTF^vo53lxLhcQM%s?3D^-k-L9#mD zQPBZJ6N{^BThq;(N<^)s=`yQZpW|xQ+*`LEX6%d#I}B{G9i?(!AfOxN>X8AO2OBZW zO>D3@E^xv3Tk&^2{EXpOnHlR3DhF}vrO<@CIY}t%4GEH+@eS^g0U$7ybs&42330+$ z$=JTOXKqzYCJ<;}wq=y20%6Ia>oXfeJhzrLM~IafyrFg5m2lP=vE^J4Pfq>FCq6so z(l=EjH<#V2=yx8ND{bU^rBLXdQIQw0KgGWJG^cd8} zENBkjEG0jUsQl5&5N124L7mqW16DTY9?Z)p3Wu||DEw~QSL%gTEbyMM$^W`!wCAcw zEzH*y@*JEzC8-7rdqcA_jyqTD0RopR36W;fkKSeR!f>?79_T(w;2=gW98 zytU6~LYm5kcWQx9c{!(C$}*~C0)b%atuF|;PDs*3STtXP9!(3#L=E)z?F4@8UB4uF z9B|A|HD|EVi_Xrzj`+ZNjdO2=qt)SQx<@Y7EleL8g1vv0QCbLCBy_jw5SQuO9d92W zKp~c9e3d-7!nc#kTI~fpsSvJYbjOT6dj~5?$vlZyn|~Dnls14WqddmrQD_SFQ|BPx zTZ9)({*0gP>(KekRbgBLdig11B(0P4@qEZcr_0ODXhEh@0uxpL#P9~FGj2nJf&MqE z-|-}ev}u`}e0y3&C|5{^a814%y20ykS9u@Q_JAgS_mHMW0N0Hw1XxRR2J*YTxNEaqX_tX`|?;7VA@3A5KYy z{TDDVyCMdzMQiuQW*4Y=-^qq|`(>db;UkIoAy@oeJ*tIiWi}_ovOQkPmD{-s0V8$| zsg!fqjCftW>b;tHX5!ppd){?@CP3nVBs#aqOuZTBNUyvhJ?chi_3%Fe*S`oe@J ztjipMB^Lvufp}=DiUir&UZ3qT&}32V&{C z`>UG)K+?7dp)Ry>pDUJrqG;+K<@$Om;qI}ckZ`$Ue(ngn>Cvp(^3NN}mCeM1mx0ln zU(Bw^TaN7In-_T&{Q!#w3VWDUu``E^b0}tUNCtmAG*F{mn6%lA$}6apg+Ir%EMIhO zK2X@BSjtK@q7s-(XQi%MVP2f#@~5q$#~-fBW;s{7g^{~osc*_w=(>4brHp*QN@E_y zEH$)FUTfh1At{U{>~TxnZPpMt`V1$3%YIh>xQ-gpRNm9Nz;m1(FQkQ`vBa}@Tbm7J zYso>QBq3q*qfW;wwvDAUTpA{2Mxdoy&g)R*1WylR0!-myFV;5YgpKf+f=ub0xfy4( zjvlhT8ZYMynF`G#6SBQ=#q?RZQdvf?N2#Q3y@(!RprBCCB9ePrfKv>H>TkDstA%Ky zzi_0Hcwp<-OI1+V1PNsm+vVl&R?G-Mu1MHP!1DNorQ|WR{4K8iNLGKqDO81zCzFG~ zPzUiRwc;K&ZOE5;0>~Dz_xK@Gvnrx~lA6!9|ET>D7Ifz=zfn+VCfm-V>!yx{Po17h zC>N`Bh!gj*d`|z~wUCjDE@iNrtBN}>+ME>9Ia`i*oBfoxMxS$-|J+8aLqDy|IXPHX z-@I8px(d`|UKu{XwZq%x3$zu7a-H;g!G&!vkyIUNOy;TFq)2k7XFJoLg?eT{jw`T2 z`C@Pa4N88@($U6omy&jDaheKxG!HMMkjNISeM6%v@jURXZf#uI)%MI-p%mVk{ z+J4PnH>WEnA=kc+#1;KowDjMnv-!PH7Nf@+A^Pk6nd0B|zX|-G5?DBTO+B?}*8F|| mLK?yZv;NweEfTTXUsF(!=~9SMe^H|)_KCJ!@nZ0;d;bCmAw^pN literal 3876 zcmeH|X;2f^7RRwJ)CCnSiWNwGu8~ay0omdPia?@52qZ|VAgSyMWMhdf0xC;X1cU&Q zqCvobfDj-+R7AiCC=o({5CH>*kg$aqvOe4Qrk&}``_xZwruV~{bI!fz%zu7!?!DN2 z5!dx9>FqaKAIm?jxv}~2vvj{wJc)1;2hjV%bNnd}=aLyfARD+HVObU%$Qhbnk&HLw zJ4jsNUff0=-1XdBxnubn3C4TdsO^iha_9Z7dE{O)7fER&tqrj_B%oz{HSPnft14C?Wz;@rClt(^A_%am+C7(tO?P6ny^riZ*(rq+}txbTKcICl84^ello}w+)A2sJbHcXts^12PDlJvyNPWL z+zR$=l}Q{a$1TrazQmlbU1w(X`XZlKlKE!0P>_WR*hNFYuQN;~IUCD#cRs}rMV$c zjPI!jJh!`J>&m>QLoSK}!>Hk|3^j1m%~Bfy8_4#3vcRGMf+-xtVlVM#=xGL0ef!RS z)dik@HV87cawGYX)l#upgesum!e`KYa znY`idGmh=xd=E^6O~@@cy2vi|J`3(A6ciH&R@W>S5QT*N1a3#MBqf$e6+k}yS_GlM6n;0i+Iz^`U5 zGHy_5O+%7S(MaB)q~;74`T%i$*55Ut&15e!bWLIa9G<^Q54E;0=wN8WyUUgQat=uc ze(C7YvG^E=0$)=1@*1mKxm_0%%F`mAs70or3z2Fsa?*9@oC$rLz(7yvocZ40&hT+Q zn3Y~KG-_9-Y%X>vK5l`zfk0UHkw@t2W`O}nk4KtGx?e3`KNs-wl{Ck?OtkQPx17Rc zVf%8aNbBc`?tTx|@HO!tC4m-;QmwvxeaSJez4~n7*qlY^hg2qtSUDOPbYaOQq-3Qt zkV%JhUK)E#r!92iVJ`f<;@R!hQJ)Vq(v@uHs5}o6f2d{T^1!Twnh#dQ)K}Y6dtk&` z%~jcj*yPkSsO6ou`LOzOD>_=Q?czz(vZw)G5JgBYZIw|V!I;MoH=AZy1nhcNTVl$uH&gWcP)Om?#s^0HMzR@G^t|+~z_s$`M|S+Y>5!rys;ml{ zXA5TzzUDcKdWpiLou*;_3xh41=^3|-=3sMoU|CP9VzRT9FC49jTFgT*{38&`M1cvI z>;x(C#Mrqf`5X@)i>^tqFgzF9iekWv&% zU?sCCK*I}^%II5yrs~?g5zK}z<6dlT;25jrTeS3%HZBH_bRpzu>JtFpTPT;c-|c?C z^s(g=-7c~N`%F;ti}r~|qlxYlEkCD7a{7*k@#JSQ%qzIa*>1aqtr>l-I1=OFjjgt8 z+2x+M);^ee)`-s{w|Yg=p)3gFb@_gZQojOw{6%ZHZI8{CV)*2OOx5ngrM=K0HjqFb z*YNVsib4LnWMO8-(5t4U^|TkqH2%C5>j$&dj8H^;4vW7Mj~6DS^FXl6doL1JZ=KlT zUgfRZ{$A&zFK!1?JtyVg;iiAA&Qd-b29$`<1=|6*k|xVrVT5;MtY-*O^!I7I z<0pMS(D0`O!4A{tgsgX$cBEgGiGrKNJ+{eU0RdQmOjpFr3rMlh5X47T)o z_?Z!r$i{O#2h0GXFf4Le-~G(_&T)mC(WbTKZO~MjaJn;mph(B<%v&JFn0uzk$YrLv zO@y67Xt$VOFF0iYYN-lPdDMCNay^qpQPGN(e^QxP)I7#ncNAeY+FD9m!hbeeauw=> ziShnRO${6rQX01&@NO##N(g1)j$G&^I;gwN<|bp3iTew%;vMF_PVJJjSp=)(u;yk! z?9w{+XyQF~Avv#-DbI8*+~w-s#P^URAEgHQc7&coML+1k6p~TUpCVJwUwUV7cj$s^SXO9kvjSz#D8DWf)6*f^X~KF1ynS{Mo)TEgSL7g^qe zt*H?#Nx0ZCGi=Qg7r|RjKH1bN!?h9X9QP_-La3yyc9QwTDV)9|ATIohb>TwohlU{x zI2)>%Skh$NW-A7AwfLjqmzSOn;uX^>KB*%vEy|9whnl2{xG^pk;T|bdOLH0k02aF7 zP29bdUrmfsr>efKX3hig>$Z<1iw+K-H{UWg9`X6^n(avm>plVkZ zAX?sOYW7%WGVS#^rk`J^8MSIL`hbc$2{3G4+^{ZpBRdWp-%G1j;4spJrsIiE{agZO zqtWG_T@JBZ7_l;vzzPwfD7gT(rhxQS6vAN!PzCnGjfhP*e9&NRi*Kh^yWq#S@%hTm z37@w!ak{T{5(t`*O&5!mwm$i}Za~6AF`Kjp1QDYm! zNSyk86AwW6P4+`k4pP{_ib0(z&X=`DBA+??FML$EMZe?Bi?2T=Xhz&6WozC)BvNRO zs!$34B&a#kYbZ80yFlD*&Izl$8P#nbi%braiW!Y0o97TTRsv_X04KSP{yutt z-iF`3=Opadw^~|CgAF(}C!^${&g43BQUCe-uP04fs8@eG@)3}uK1F8|kj_a(MpIux z9FMp=t~%kUX*B9I#9`rz91h+U7Jo~@1eM;@1bj+QDu^sxJRA~JF#X)lQ}A7b@zwj# zh@>ZUaQ`lrhu8;KHbD9;+_0IiU$MEBcBrAg`%aT@$xow833+cT?mc-Z Date: Wed, 3 Aug 2016 12:58:36 -0400 Subject: [PATCH 289/502] Add tests for emoji skin-tone selectors --- tests/test_characters.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_characters.py b/tests/test_characters.py index a7318b95..49b9aba8 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -43,6 +43,12 @@ def test_emoji_variation_selector(): eq_(sequence_weirdness('❤\ufe0f' * 10), 0) +def test_emoji_skintone_selector(): + # Dear heuristic, you can't call skin-tone selectors weird anymore. + # We welcome Santa Clauses of all colors. + eq_(sequence_weirdness('🎅🏿🎅🏽🎅🏼🎅🏻'), 0) + + def test_surrogates(): eq_(fix_surrogates('\udbff\udfff'), '\U0010ffff') eq_(fix_surrogates('\ud800\udc00'), '\U00010000') From bb354227aa631a492984999cbe93ded2080ad91e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 4 Aug 2016 15:12:16 -0400 Subject: [PATCH 290/502] Remove unnecessary variable from build_data --- ftfy/build_data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 1bb726bf..346113d6 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -60,8 +60,7 @@ def make_char_data_file(do_it_anyway=False): category = unicodedata.category(char) if category.startswith('L'): # letters - is_latin = unicodedata.name(char).startswith('LATIN') - if is_latin: + if unicodedata.name(char).startswith('LATIN'): if category == 'Lu': cclasses[codepoint] = 'L' else: @@ -118,7 +117,7 @@ def make_char_data_file(do_it_anyway=False): # `´ are much more like quotation marks than modifiers. for char in "^~`´˝＾｀": cclasses[ord(char)] = 'o' - + out = open('char_classes.dat', 'wb') out.write(zlib.compress(''.join(cclasses).encode('ascii'))) out.close() From 44fa5fd433218b7d9f90ac411be3b256e073c02e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 28 Sep 2016 11:39:11 -0400 Subject: [PATCH 291/502] Show some likely false negatives in the stream-tester --- ftfy/streamtester/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ftfy/streamtester/__init__.py b/ftfy/streamtester/__init__.py index 61fd14a6..e097bf6b 100644 --- a/ftfy/streamtester/__init__.py +++ b/ftfy/streamtester/__init__.py @@ -32,10 +32,12 @@ def check_ftfy(self, text, encoding_only=True): fixed = fix_text(text, uncurl_quotes=False, fix_character_width=False) if text != fixed: # possibly filter common bots before printing - print(u'\nText:\t{text!r}\nFixed:\t{fixed!r}\n'.format( + print('\nText:\t{text!r}\nFixed:\t{fixed!r}\n'.format( text=text, fixed=fixed )) self.num_fixed += 1 + elif 'â€' in text or '\x80' in text: + print('\nNot fixed:\t{text!r}'.format(text=text)) # Print status updates once in a while if self.count % 100 == 0: From b7fa77c724d052b7041bffc60f4a9f9bcf32daeb Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 28 Sep 2016 11:40:55 -0400 Subject: [PATCH 292/502] Bump version to 4.2 --- ftfy/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index df76f61f..59f029b7 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -17,7 +17,7 @@ from ftfy.compatibility import is_printable import unicodedata -__version__ = '4.1.1' +__version__ = '4.2.0' def fix_text(text, diff --git a/setup.py b/setup.py index fb439fb5..ac702cd3 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="ftfy", - version='4.1.2', + version='4.2.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From e8d6b41a4572f8958a49efd3e56be4ffa9da2d97 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 28 Sep 2016 12:26:52 -0400 Subject: [PATCH 293/502] Update the changelog for 4.2 --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07c486be..003f8e3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,40 @@ +## Version 4.2.0 (September 28, 2016) + +Heuristic changes: + +- Math symbols next to currency symbols are no longer considered 'weird' by the + heuristic. This fixes a false positive where text that involved the + multiplication sign and British pounds or euros (as in '5×£35') could turn + into Hebrew letters. + +- A heuristic that used to be a bonus for certain punctuation now also gives a + bonus to successfully decoding other common codepoints, such as the + non-breaking space, the degree sign, and the byte order mark. + +- In version 4.0, we tried to "future-proof" the categorization of emoji (as a + kind of symbol) to include codepoints that would likely be assigned to emoji + later. The future happened, and there are even more emoji than we expected. + We have expanded the range to include those emoji, too. + + ftfy is still mostly based on information from Unicode 8 (as Python 3.5 is), + but this expanded range should include the emoji from Unicode 9 and 10. + +- Emoji are increasingly being modified by variation selectors and skin-tone + modifiers. Those codepoints are now grouped with 'symbols' in ftfy, so they + fit right in with emoji, instead of being considered 'marks' as their Unicode + category would suggest. + + This enables fixing mojibake that involves iOS's new diverse emoji. + +- An old heuristic that wasn't necessary anymore considered Latin text with + high-numbered codepoints to be 'weird', but this is normal in languages such + as Vietnamese and Azerbaijani. This does not seem to have caused any false + positives, but it caused ftfy to be too reluctant to fix some cases of broken + text in those languages. + + The heuristic has been changed, and all languages that use Latin letters + should be on even footing now. + ## Version 4.1.1 (April 13, 2016) - Bug fix: in the command-line interface, the `-e` option had no effect on From 21c40433f20244aca4707e84df6ea6a2eaabfc0b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 20 Dec 2016 15:28:33 -0500 Subject: [PATCH 294/502] Remove more kinds of nonsense control characters. Fixes #47. --- docs/index.rst | 4 ++-- ftfy/__init__.py | 17 ++++++++++------ ftfy/chardata.py | 20 +++++++++++++------ ftfy/fixes.py | 43 +++++++++++++++++++++++----------------- tests/test_characters.py | 16 ++++++++++++--- 5 files changed, 65 insertions(+), 35 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 73452433..b263a23b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -169,7 +169,7 @@ Check that the default fixes are appropriate for your use case. For example: marks with nice typography. You could even consider doing quite the opposite of `uncurl_quotes`, running `smartypants`_ on the result to make all the punctuation nice. - + .. _smartypants: http://pythonhosted.org/smartypants/ If the only fix you need is to detect and repair decoding errors (mojibake), then @@ -322,7 +322,7 @@ that ftfy's behavior is consistent across versions. Future versions of ftfy ======================= - + ftfy has full support for Python 2.7, even including a backport of Unicode 8 character classes to Python 2. But given the sweeping changes to Unicode in Python, it's getting inconvenient to add new features to ftfy that work the diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 59f029b7..90142b6d 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -106,12 +106,15 @@ def fix_text(text, when they're appropriately paired, or replacing them with \ufffd otherwise. - - If `fix_control_characters` is true, remove all C0 control characters - except the common useful ones: TAB, CR, LF, and FF. (CR characters - may have already been removed by the `fix_line_breaks` step.) + - If `fix_control_characters` is true, remove control characters that are + not suitable for use in text. This includes most of the ASCII control + characters, plus some Unicode controls such as the byte order mark + (U+FEFF). Useful control characters such as Tab and Line Feed are left + as they are. - - If `remove_bom` is True, remove the Byte-Order Mark if it exists. - (This is a decoded Unicode string. It doesn't have a "byte order".) + - If `remove_bom` is True, remove the Byte-Order Mark at the start of the + string if it exists. It may be that you don't want to remove all + control characters (as above) but you still want to remove the BOM. - If `normalization` is not None, apply the specified form of Unicode normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. @@ -282,7 +285,9 @@ def fix_text_segment(text, text = fixes.fix_surrogates(text) if remove_control_chars: text = fixes.remove_control_chars(text) - if remove_bom: + if remove_bom and not remove_control_chars: + # Skip this step if we've already done `remove_control_chars`, + # because it would be redundant. text = fixes.remove_bom(text) if normalization is not None: text = unicodedata.normalize(normalization, text) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 40a36265..1b76f95f 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -8,6 +8,7 @@ import re import zlib import unicodedata +import itertools from pkg_resources import resource_string from ftfy.compatibility import unichr @@ -154,16 +155,23 @@ def chars_to_classes(string): def _build_control_char_mapping(): """ - Build a translate mapping that strips all C0 control characters, - except those that represent whitespace. + Build a translate mapping that strips likely-unintended control characters. + See :func:`ftfy.fixes.remove_control_chars` for a description of these + codepoint ranges and why they should be removed. """ control_chars = {} - for i in range(32): + + for i in itertools.chain( + range(0x00, 0x09), [0x0b], + range(0x0d, 0x20), [0x7f], + range(0x206a, 0x2070), + [0xfeff], + range(0xfff9, 0xfffd), + range(0x1d173, 0x1d17b), + range(0xe0000, 0xe0080) + ): control_chars[i] = None - # Map whitespace control characters to themselves. - for char in '\t\n\f\r': - del control_chars[ord(char)] return control_chars CONTROL_CHARS = _build_control_char_mapping() diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 9080cbbf..7da80ac2 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -482,24 +482,31 @@ def fix_surrogates(text): def remove_control_chars(text): """ - Remove all ASCII control characters except for the important ones. - - This removes characters in these ranges: - - - U+0000 to U+0008 - - U+000B - - U+000E to U+001F - - U+007F - - It leaves alone these characters that are commonly used for formatting: - - - TAB (U+0009) - - LF (U+000A) - - FF (U+000C) - - CR (U+000D) - - Feel free to object that FF isn't "commonly" used for formatting. I've at - least seen it used. + Remove various control characters that you probably didn't intend to be in + your text. Many of these characters appear in the table of "Characters not + suitable for use with markup" at + http://www.unicode.org/reports/tr20/tr20-9.html. + + This includes: + + - ASCII control characters, except for the important whitespace characters + (U+00 to U+08, U+0B, U+0E to U+1F, U+7F) + - Deprecated Arabic control characters (U+206A to U+206F) + - Interlinear annotation characters (U+FFF9 to U+FFFB) + - The Object Replacement Character (U+FFFC) + - The byte order mark (U+FEFF) + - Musical notation control characters (U+1D173 to U+1D17A) + - Tag characters (U+E0000 to U+E007F) + + However, these similar characters are left alone: + + - Control characters that produce whitespace (U+09, U+0A, U+0C, U+0D, + U+2028, and U+2029) + - C1 control characters (U+80 to U+9F) -- even though they are basically + never used intentionally, they are important clues about what mojibake + has happened + - Control characters that affect glyph rendering, such as joiners and + right-to-left marks (U+200C to U+200F, U+202A to U+202E) """ return text.translate(CONTROL_CHARS) diff --git a/tests/test_characters.py b/tests/test_characters.py index 49b9aba8..6616ba90 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -1,6 +1,7 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals -from ftfy.fixes import fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, fix_surrogates +from ftfy.fixes import ( + fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, + remove_control_chars, fix_surrogates +) from ftfy.badness import sequence_weirdness import unicodedata import sys @@ -37,6 +38,15 @@ def test_byte_order_mark(): eq_(fix_encoding(''), '\ufeff') +def test_control_chars(): + text = ( + "\ufeffSometimes, \ufffcbad ideas \x7f\ufffalike these characters\ufffb " + "\u206aget standardized\U000E0065\U000E006E.\n" + ) + fixed = "Sometimes, bad ideas like these characters get standardized.\n" + eq_(remove_control_chars(text), fixed) + + def test_emoji_variation_selector(): # The hearts here are explicitly marked as emoji using the variation # selector U+FE0F. This is not weird. From 6d1fab72773da860c9a88bf584bb75c727282450 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 20 Dec 2016 15:28:33 -0500 Subject: [PATCH 295/502] Remove more kinds of nonsense control characters. Fixes #47. --- docs/index.rst | 4 ++-- ftfy/__init__.py | 17 ++++++++++------ ftfy/chardata.py | 20 +++++++++++++------ ftfy/fixes.py | 43 +++++++++++++++++++++++----------------- tests/test_characters.py | 14 ++++++++++++- 5 files changed, 65 insertions(+), 33 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 73452433..b263a23b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -169,7 +169,7 @@ Check that the default fixes are appropriate for your use case. For example: marks with nice typography. You could even consider doing quite the opposite of `uncurl_quotes`, running `smartypants`_ on the result to make all the punctuation nice. - + .. _smartypants: http://pythonhosted.org/smartypants/ If the only fix you need is to detect and repair decoding errors (mojibake), then @@ -322,7 +322,7 @@ that ftfy's behavior is consistent across versions. Future versions of ftfy ======================= - + ftfy has full support for Python 2.7, even including a backport of Unicode 8 character classes to Python 2. But given the sweeping changes to Unicode in Python, it's getting inconvenient to add new features to ftfy that work the diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 59f029b7..90142b6d 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -106,12 +106,15 @@ def fix_text(text, when they're appropriately paired, or replacing them with \ufffd otherwise. - - If `fix_control_characters` is true, remove all C0 control characters - except the common useful ones: TAB, CR, LF, and FF. (CR characters - may have already been removed by the `fix_line_breaks` step.) + - If `fix_control_characters` is true, remove control characters that are + not suitable for use in text. This includes most of the ASCII control + characters, plus some Unicode controls such as the byte order mark + (U+FEFF). Useful control characters such as Tab and Line Feed are left + as they are. - - If `remove_bom` is True, remove the Byte-Order Mark if it exists. - (This is a decoded Unicode string. It doesn't have a "byte order".) + - If `remove_bom` is True, remove the Byte-Order Mark at the start of the + string if it exists. It may be that you don't want to remove all + control characters (as above) but you still want to remove the BOM. - If `normalization` is not None, apply the specified form of Unicode normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. @@ -282,7 +285,9 @@ def fix_text_segment(text, text = fixes.fix_surrogates(text) if remove_control_chars: text = fixes.remove_control_chars(text) - if remove_bom: + if remove_bom and not remove_control_chars: + # Skip this step if we've already done `remove_control_chars`, + # because it would be redundant. text = fixes.remove_bom(text) if normalization is not None: text = unicodedata.normalize(normalization, text) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 40a36265..1b76f95f 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -8,6 +8,7 @@ import re import zlib import unicodedata +import itertools from pkg_resources import resource_string from ftfy.compatibility import unichr @@ -154,16 +155,23 @@ def chars_to_classes(string): def _build_control_char_mapping(): """ - Build a translate mapping that strips all C0 control characters, - except those that represent whitespace. + Build a translate mapping that strips likely-unintended control characters. + See :func:`ftfy.fixes.remove_control_chars` for a description of these + codepoint ranges and why they should be removed. """ control_chars = {} - for i in range(32): + + for i in itertools.chain( + range(0x00, 0x09), [0x0b], + range(0x0d, 0x20), [0x7f], + range(0x206a, 0x2070), + [0xfeff], + range(0xfff9, 0xfffd), + range(0x1d173, 0x1d17b), + range(0xe0000, 0xe0080) + ): control_chars[i] = None - # Map whitespace control characters to themselves. - for char in '\t\n\f\r': - del control_chars[ord(char)] return control_chars CONTROL_CHARS = _build_control_char_mapping() diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 9080cbbf..7da80ac2 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -482,24 +482,31 @@ def fix_surrogates(text): def remove_control_chars(text): """ - Remove all ASCII control characters except for the important ones. - - This removes characters in these ranges: - - - U+0000 to U+0008 - - U+000B - - U+000E to U+001F - - U+007F - - It leaves alone these characters that are commonly used for formatting: - - - TAB (U+0009) - - LF (U+000A) - - FF (U+000C) - - CR (U+000D) - - Feel free to object that FF isn't "commonly" used for formatting. I've at - least seen it used. + Remove various control characters that you probably didn't intend to be in + your text. Many of these characters appear in the table of "Characters not + suitable for use with markup" at + http://www.unicode.org/reports/tr20/tr20-9.html. + + This includes: + + - ASCII control characters, except for the important whitespace characters + (U+00 to U+08, U+0B, U+0E to U+1F, U+7F) + - Deprecated Arabic control characters (U+206A to U+206F) + - Interlinear annotation characters (U+FFF9 to U+FFFB) + - The Object Replacement Character (U+FFFC) + - The byte order mark (U+FEFF) + - Musical notation control characters (U+1D173 to U+1D17A) + - Tag characters (U+E0000 to U+E007F) + + However, these similar characters are left alone: + + - Control characters that produce whitespace (U+09, U+0A, U+0C, U+0D, + U+2028, and U+2029) + - C1 control characters (U+80 to U+9F) -- even though they are basically + never used intentionally, they are important clues about what mojibake + has happened + - Control characters that affect glyph rendering, such as joiners and + right-to-left marks (U+200C to U+200F, U+202A to U+202E) """ return text.translate(CONTROL_CHARS) diff --git a/tests/test_characters.py b/tests/test_characters.py index 49b9aba8..f985533d 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from ftfy.fixes import fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, fix_surrogates +from ftfy.fixes import ( + fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, + remove_control_chars, fix_surrogates +) from ftfy.badness import sequence_weirdness import unicodedata import sys @@ -37,6 +40,15 @@ def test_byte_order_mark(): eq_(fix_encoding(''), '\ufeff') +def test_control_chars(): + text = ( + "\ufeffSometimes, \ufffcbad ideas \x7f\ufffalike these characters\ufffb " + "\u206aget standardized\U000E0065\U000E006E.\n" + ) + fixed = "Sometimes, bad ideas like these characters get standardized.\n" + eq_(remove_control_chars(text), fixed) + + def test_emoji_variation_selector(): # The hearts here are explicitly marked as emoji using the variation # selector U+FE0F. This is not weird. From 3572cc692e68c5f195c5570ee6006066a6fb84f3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 20 Dec 2016 15:44:25 -0500 Subject: [PATCH 296/502] documentation: ftfy 5 is probably happening soon --- docs/index.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b263a23b..20258804 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -336,11 +336,11 @@ on two versions of Python (which is more like three versions because of Python :func:`ftfy.bad_codecs.utf8_variants.mangle_surrogates` and :func:`ftfy.compatibility._narrow_unichr_workaround`. -We're following the lead of venerable projects such as jQuery. jQuery dropped -support for IE 6-8 in version 2.0, and in doing so, it became much simpler and -faster for people who didn't need to support old versions of IE in their web -applications. Meanwhile, jQuery 1.9 remained there for those who needed it. - -Similarly, ftfy 5.0 will reduce the size and complexity of the code greatly, -but ftfy 4.x will remain there. If you're running on Python 2, please make sure -that `ftfy < 5` is in your requirements list, not just `ftfy`. +This will happen soon, and we'll follow the plan that jQuery used years ago +when it dropped support for IE 6-8. We'll release the last Python 2 version and +the first Python-3-only version with the same feature set. ftfy 5.0 will +reduce the size and complexity of the code greatly, but ftfy 4.x will remain +there for those who need it. + +If you're running on Python 2, please make sure that `ftfy < 5` is in your +requirements list, not just `ftfy`. From 3e27d5763378c4d0c63f417acacfdb01a1ef003f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 20 Dec 2016 15:47:00 -0500 Subject: [PATCH 297/502] Documentation: Change description of GB18030. Fixes #57. --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 20258804..290eb3a9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -139,7 +139,7 @@ However, ftfy cannot understand other mixups between single-byte encodings, because it is extremely difficult to detect which mixup in particular is the one that happened. -We also can't handle the non-UTF encodings used for Chinese, Japanese, and +We also can't handle the legacy encodings used for Chinese, Japanese, and Korean, such as ``shift-jis`` and ``gb18030``. See `issue #34 `_ for why this is so hard. From c960156c92cd80d442743b4cc87f0a669ef28430 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 20 Dec 2016 16:03:39 -0500 Subject: [PATCH 298/502] Get HTML entity definitions from html5lib. Fixes #50. --- ftfy/compatibility.py | 3 --- ftfy/fixes.py | 5 +++-- setup.py | 5 +++++ tests/test_entities.py | 3 +++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py index b960c964..ad5c1097 100644 --- a/ftfy/compatibility.py +++ b/ftfy/compatibility.py @@ -7,16 +7,13 @@ import unicodedata if sys.hexversion >= 0x03000000: - from html import entities unichr = chr xrange = range PYTHON2 = False else: - import htmlentitydefs as entities unichr = unichr xrange = xrange PYTHON2 = True -htmlentitydefs = entities PYTHON34_OR_LATER = (sys.hexversion >= 0x03040000) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 7da80ac2..5a407736 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -10,7 +10,8 @@ PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE, LOSSY_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE) from ftfy.badness import text_cost -from ftfy.compatibility import htmlentitydefs, unichr +from ftfy.compatibility import unichr +from html5lib.constants import entities import re import sys import codecs @@ -322,7 +323,7 @@ def fixup(match): else: # named entity try: - text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) + text = entities[text[1:]] except KeyError: pass return text # leave as is diff --git a/setup.py b/setup.py index ac702cd3..20db4836 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,11 @@ description="Fixes some problems with Unicode text after the fact", packages=['ftfy', 'ftfy.bad_codecs'], package_data={'ftfy': ['char_classes.dat']}, + + # Oh no, we grew a dependency! We could actually go back to having no + # dependencies if we drop support for Python <= 3.4, because the feature + # we need from html5lib is now in the standard library, as html.unescape. + install_requires=['html5lib'], classifiers=[ "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", diff --git a/tests/test_entities.py b/tests/test_entities.py index f23abffc..24c4d446 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -2,6 +2,7 @@ from ftfy import fix_text, fix_text_segment from nose.tools import eq_ + def test_entities(): example = '&\n\n&' eq_(fix_text(example), '&\n\n&') @@ -16,3 +17,5 @@ def test_entities(): eq_(fix_text_segment('<>', fix_entities=False), '<>') eq_(fix_text_segment('<>', fix_entities=True), '<>') eq_(fix_text_segment('<>'), '<>') + eq_(fix_text_segment('jednocześnie'), 'jednocześnie') + eq_(fix_text_segment('JEDNOCZEŚNIE'), 'JEDNOCZEŚNIE') From 5986bd46f437aa3c468eeadc1c676e3b4d97d5c6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 20 Dec 2016 16:25:50 -0500 Subject: [PATCH 299/502] Luminoso blurb in README: We're hiring but not "in all areas" now --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ea01b50e..e9f7a14a 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,8 @@ and it's built on Python and open-source technologies. We use ftfy every day at Luminoso, because the first step in understanding text is making sure it has the correct characters in it! -Luminoso is growing fast and hiring in all areas. If you're interested in -joining us, take a look at [our careers -page](http://www.luminoso.com/career.html). +Luminoso is growing fast and hiring. If you're interested in joining us, take a +look at [our careers page](http://www.luminoso.com/career.html). ## What it does From b45d39724edb8ec64d8aca9b82c5fe5f2ddd7703 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 20 Dec 2016 16:26:54 -0500 Subject: [PATCH 300/502] Bump version to 4.3 --- ftfy/__init__.py | 12 ++++++------ setup.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 90142b6d..6418a97c 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -7,17 +7,17 @@ """ from __future__ import unicode_literals - -# See the docstring for ftfy.bad_codecs to see what we're doing here. +import unicodedata import ftfy.bad_codecs -ftfy.bad_codecs.ok() - from ftfy import fixes from ftfy.formatting import display_ljust from ftfy.compatibility import is_printable -import unicodedata -__version__ = '4.2.0' +__version__ = '4.3.0' + + +# See the docstring for ftfy.bad_codecs to see what we're doing here. +ftfy.bad_codecs.ok() def fix_text(text, diff --git a/setup.py b/setup.py index 20db4836..6b0c81a6 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="ftfy", - version='4.2.0', + version='4.3.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From c0f375f8bce3b4dc1c007714b7d3bc3181cc7a32 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 20 Dec 2016 16:27:54 -0500 Subject: [PATCH 301/502] Some cleanups to make pylint happy --- ftfy/bad_codecs/utf8_variants.py | 6 +++--- ftfy/badness.py | 2 +- ftfy/chardata.py | 1 - ftfy/cli.py | 2 +- ftfy/fixes.py | 9 ++++----- ftfy/formatting.py | 7 +++++-- ftfy/streamtester/oauth.py | 1 - ftfy/streamtester/twitter_tester.py | 1 - 8 files changed, 14 insertions(+), 15 deletions(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 0b541260..4cdb2239 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -39,11 +39,11 @@ """ from __future__ import unicode_literals -from ftfy.compatibility import bytes_to_ints, unichr, PYTHON2 -from encodings.utf_8 import (IncrementalDecoder as UTF8IncrementalDecoder, - IncrementalEncoder as UTF8IncrementalEncoder) import re import codecs +from encodings.utf_8 import (IncrementalDecoder as UTF8IncrementalDecoder, + IncrementalEncoder as UTF8IncrementalEncoder) +from ftfy.compatibility import bytes_to_ints, unichr, PYTHON2 NAME = 'utf-8-variants' diff --git a/ftfy/badness.py b/ftfy/badness.py index 3095fb34..c08b7a1e 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -5,9 +5,9 @@ """ from __future__ import unicode_literals -from ftfy.chardata import chars_to_classes import re import unicodedata +from ftfy.chardata import chars_to_classes # The following regex uses the mapping of character classes to ASCII # characters defined in chardata.py and build_data.py: diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 1b76f95f..40260d43 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -211,4 +211,3 @@ def _build_width_map(): width_map[i] = alternate return width_map WIDTH_MAP = _build_width_map() - diff --git a/ftfy/cli.py b/ftfy/cli.py index f4a99839..802a46c8 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -1,11 +1,11 @@ """ A command-line utility for fixing text found in a file. """ -from ftfy import fix_file, __version__ import sys import io import codecs +from ftfy import fix_file, __version__ from ftfy.compatibility import PYTHON2 diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 5a407736..13fa16cb 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -5,6 +5,10 @@ """ from __future__ import unicode_literals +import re +import sys +import codecs +import warnings from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS, CONTROL_CHARS, LIGATURES, WIDTH_MAP, PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE, @@ -12,10 +16,6 @@ from ftfy.badness import text_cost from ftfy.compatibility import unichr from html5lib.constants import entities -import re -import sys -import codecs -import warnings BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. @@ -654,4 +654,3 @@ def replacement(match): 'replace_lossy_sequences': replace_lossy_sequences, 'fix_partial_utf8_punct_in_1252': fix_partial_utf8_punct_in_1252 } - diff --git a/ftfy/formatting.py b/ftfy/formatting.py index 17ad9630..78051999 100644 --- a/ftfy/formatting.py +++ b/ftfy/formatting.py @@ -1,4 +1,8 @@ # coding: utf-8 +""" +This module provides functions for justifying Unicode text in a monospaced +display such as a terminal. +""" from __future__ import unicode_literals, division from unicodedata import east_asian_width, combining, category, normalize @@ -90,7 +94,7 @@ def display_ljust(text, width, fillchar=' '): Table flip▒▒▒▒▒▒▒▒▒▒ (╯°□°)╯︵ ┻━┻▒▒▒▒▒▒▒ ちゃぶ台返し▒▒▒▒▒▒▒▒ - + This example, and the similar ones that follow, should come out justified correctly when viewed in a monospaced terminal. It will probably not look correct if you're viewing this code or documentation in a Web browser. @@ -145,4 +149,3 @@ def display_center(text, width, fillchar=' '): left_padding = padding // 2 right_padding = padding - left_padding return fillchar * left_padding + text + fillchar * right_padding - diff --git a/ftfy/streamtester/oauth.py b/ftfy/streamtester/oauth.py index 8e300ed7..a948459c 100644 --- a/ftfy/streamtester/oauth.py +++ b/ftfy/streamtester/oauth.py @@ -70,4 +70,3 @@ def unhide(secret): consumer_key=consumer_key, consumer_secret=unhide(fig_leaf) ) - diff --git a/ftfy/streamtester/twitter_tester.py b/ftfy/streamtester/twitter_tester.py index a6542e5c..561bcf20 100644 --- a/ftfy/streamtester/twitter_tester.py +++ b/ftfy/streamtester/twitter_tester.py @@ -86,4 +86,3 @@ def main(): if __name__ == '__main__': main() - From 23a2d9eec23be517320b111f0a6ab93e64bd48cf Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 20 Dec 2016 16:47:57 -0500 Subject: [PATCH 302/502] Update ftfy.formatting to use wcwidth --- ftfy/formatting.py | 91 ++++++++++++++++++++++++---------------------- setup.py | 8 ++-- 2 files changed, 51 insertions(+), 48 deletions(-) diff --git a/ftfy/formatting.py b/ftfy/formatting.py index 78051999..793cbb28 100644 --- a/ftfy/formatting.py +++ b/ftfy/formatting.py @@ -2,59 +2,34 @@ """ This module provides functions for justifying Unicode text in a monospaced display such as a terminal. + +We used to have our own implementation here, but now we mostly rely on +the 'wcwidth' library. """ from __future__ import unicode_literals, division -from unicodedata import east_asian_width, combining, category, normalize +from unicodedata import normalize +from wcwidth import wcwidth, wcswidth def character_width(char): - """ + r""" Determine the width that a character is likely to be displayed as in - a monospaced terminal. The width will always be 0, 1, or 2. - - We are assuming that the character will appear in a modern, Unicode-aware - terminal emulator, where CJK characters -- plus characters designated - "fullwidth" by CJK encodings -- will span two character cells. (Maybe - it's an over-simplification to call these displays "monospaced".) - - When a character width is "Ambiguous", we assume it to span one character - cell, because we assume that the monospaced display you're using is - designed mostly to display Western characters with CJK characters as the - exception. This assumption will go wrong, for example, if your display is - actually on a Japanese flip-phone... but let's assume it isn't. + a monospaced terminal. The width for a printable character will + always be 0, 1, or 2. - Combining marks and formatting codepoints do not advance the cursor, - so their width is 0. - - If the character is a particular kind of control code -- the kind - represented by the lowest bytes of ASCII, with Unicode category Cc -- the - idea of it having a "width" probably doesn't even apply, because it will - probably have some other effect on your terminal. For example, there's no - sensible width for a line break. We return the default of 1 in the absence - of anything sensible to do there. + Nonprintable or control characters will return -1, a convention that comes + from wcwidth. >>> character_width('車') 2 >>> character_width('A') 1 - >>> character_width('\N{SOFT HYPHEN}') + >>> character_width('\N{ZERO WIDTH JOINER}') 0 + >>> character_width('\n') + -1 """ - if combining(char) != 0: - # Combining characters don't advance the cursor; they modify the - # previous character instead. - return 0 - elif east_asian_width(char) in 'FW': - # Characters designated Wide (W) or Fullwidth (F) will take up two - # columns in many Unicode-aware terminal emulators. - return 2 - elif category(char) == 'Cf': - # Characters in category Cf are un-printable formatting characters - # that do not advance the cursor, such as zero-width spaces or - # right-to-left marks. - return 0 - else: - return 1 + return wcwidth(char) def monospaced_width(text): @@ -66,15 +41,27 @@ def monospaced_width(text): This can be useful for formatting text that may contain non-spacing characters, or CJK characters that take up two character cells. + Returns -1 if the string contains a non-printable or control character. + >>> monospaced_width('ちゃぶ台返し') 12 >>> len('ちゃぶ台返し') 6 + >>> monospaced_width('owl\N{SOFT HYPHEN}flavored') + 12 + >>> monospaced_width('example\x80') + -1 + + # The Korean word 'ibnida' can be written with 3 characters or 7 jamo. + # Either way, it *looks* the same and takes up 6 character cells. + >>> monospaced_width('입니다') + 6 + >>> monospaced_width('\u110b\u1175\u11b8\u1102\u1175\u1103\u1161') + 6 """ # NFC-normalize the text first, so that we don't need special cases for # Hangul jamo. - text = normalize('NFC', text) - return sum([character_width(char) for char in text]) + return wcswidth(normalize('NFC', text)) def display_ljust(text, width, fillchar=' '): @@ -101,7 +88,13 @@ def display_ljust(text, width, fillchar=' '): """ if character_width(fillchar) != 1: raise ValueError("The padding character must have display width 1") - padding = max(0, width - monospaced_width(text)) + + text_width = monospaced_width(text) + if text_width == -1: + # There's a control character here, so just don't add padding + return text + + padding = max(0, width - text_width) return text + fillchar * padding @@ -125,7 +118,12 @@ def display_rjust(text, width, fillchar=' '): """ if character_width(fillchar) != 1: raise ValueError("The padding character must have display width 1") - padding = max(0, width - monospaced_width(text)) + + text_width = monospaced_width(text) + if text_width == -1: + return text + + padding = max(0, width - text_width) return fillchar * padding + text @@ -145,7 +143,12 @@ def display_center(text, width, fillchar=' '): """ if character_width(fillchar) != 1: raise ValueError("The padding character must have display width 1") - padding = max(0, width - monospaced_width(text)) + + text_width = monospaced_width(text) + if text_width == -1: + return text + + padding = max(0, width - text_width) left_padding = padding // 2 right_padding = padding - left_padding return fillchar * left_padding + text + fillchar * right_padding diff --git a/setup.py b/setup.py index 6b0c81a6..ccff46eb 100644 --- a/setup.py +++ b/setup.py @@ -13,10 +13,10 @@ packages=['ftfy', 'ftfy.bad_codecs'], package_data={'ftfy': ['char_classes.dat']}, - # Oh no, we grew a dependency! We could actually go back to having no - # dependencies if we drop support for Python <= 3.4, because the feature - # we need from html5lib is now in the standard library, as html.unescape. - install_requires=['html5lib'], + # We could drop the html5lib dependency if we drop support for Python <= + # 3.4, because the feature we need from html5lib is now in the standard + # library, as html.unescape. + install_requires=['html5lib', 'wcwidth'], classifiers=[ "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", From 58c76ee6dc552ba7b9c091353a9b019de5432f4d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 21 Dec 2016 12:52:58 -0500 Subject: [PATCH 303/502] Update char_classes to Unicode 9 --- ftfy/build_data.py | 12 ++++++------ ftfy/char_classes.dat | Bin 3891 -> 3996 bytes tests/test_synthetic.py | 22 ++++++++++++++++++---- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 346113d6..85b84140 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -5,7 +5,7 @@ classes we care about change, or if a new version of Python supports a new Unicode standard and we want it to affect our string decoding. -The file that we generate is based on Unicode 8.0, as supported by Python 3.5. +The file that we generate is based on Unicode 9.0, as supported by Python 3.6. You can certainly use it in earlier versions. This simply makes sure that we get consistent results from running ftfy on different versions of Python. @@ -42,16 +42,16 @@ def make_char_data_file(do_it_anyway=False): Build the compressed data file 'char_classes.dat' and write it to the current directory. - If you run this, run it in Python 3.5 or later. It will run in earlier - versions, but you won't get the Unicode 8 standard, leading to inconsistent + If you run this, run it in Python 3.6 or later. It will run in earlier + versions, but you won't get the Unicode 9 standard, leading to inconsistent behavior. To protect against this, running this in the wrong version of Python will raise an error unless you pass `do_it_anyway=True`. """ - if sys.hexversion < 0x03050000 and not do_it_anyway: + if sys.hexversion < 0x03060000 and not do_it_anyway: raise RuntimeError( - "This function should be run in Python 3.5 or later." + "This function should be run in Python 3.6 or later." ) cclasses = [None] * 0x110000 @@ -60,7 +60,7 @@ def make_char_data_file(do_it_anyway=False): category = unicodedata.category(char) if category.startswith('L'): # letters - if unicodedata.name(char).startswith('LATIN'): + if unicodedata.name(char, '').startswith('LATIN'): if category == 'Lu': cclasses[codepoint] = 'L' else: diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index cd6688dc1366835930e3b4ca3be8323157a803b0..c03f661f044f874afc0aac53f13053671f9e36ef 100644 GIT binary patch literal 3996 zcmeHIX;2ebwssl87F)U%aa1(g_yk0j24shn_=E-oM2N^5MMcnt$i5TBtyK^KNt!^2 zvJ;lb5&{WH5X7(}OTrS82naDGAtoUK5|T`J&rEgIRK2RHnqRMK&W~H?+_T(U-#K5M zq+&k9HKwe;bj!#33GMF;FPdU?tmeI#Ss!QrB1fgmNUkscr#*<>dsE}5CSBLP<(Vo4 zM#%QsFc<5Mi1jfR&=haqLD?0Mx-0m95(a1dC zZN^)iXvD8h7e|Y*!0T*k9ks&%+HdXXD0_;@F`jZMaw1lzh$WKIf;aN%A)mt|(=pK1 z8e02$xS2zM)P8!f!W0?xc_=T;jIjFvEy^T;2o_8)rg)9>GRW-7mb>jiVy*);~7(L;R2A0bm}dTNK5_o+P;rvSvqUr4CqI1PIf=7-GhXh#ooir} zSsATU@g%?}q+=P?l3RBN4mXHbb4V8MqjdL*LpjcA1$9tDad++yA449pjzOC9ta|#I z==CXq+N2Tpy_7y{XaouVE<3x?e9g;BYrfn;^t;LXce|1m274ne3C18kQO#3JRbFItVFT_^1?$A#SIV|c zkn{3tl}BiR(*E{=)8&8=Wr0kffxWDXzBH+Py)BQVq{kr&#-+i;8^zd$3adwL-wpul z=DZc~!=$}E4AK}JN|dq11H21pvIq7>Orw&nlWv)9YLlz%P_j9=!^tpZ%lJ+Y=y4aZ zNH2PP72>gx7a>ETg8S%~X0`J!OlqF&CRS)rqePa#V1PA4dGn<=Mki?YgORJ6w~J)3 z&Lb-A?#@x|G8yAm#9IQ$&WT_p;Z%VV{nYzsGIQK!d7?0Mmo^2Q|>u0I=o7-uL?}b6Ql^@1kCZQHc@i}4H2J;t#BMHBtG_&vW#Fr&m$)aGa`TE2*!`bs1IvQSY%{QO%lqb<|bUUEZ3eQMjRn>sTL%G1!50 z`9Wd!;(h7k9p?=OYQ4OhOhrfI9XVeHtAk>-9go>n?I|l0Jt@v)<&8_QC9z6$#3%0WyPr7iqolicJfp-}AR!0!m#rd!w|cl`MAzF`{FeaV+Y{qe4-EPgx1VhYs@Un!2-7EVMWlI3JTgY09N>Wl&l@- z965-CrSxK_Xy{Cfii4|N*HE@p|6-_At5iO5d((zin1k1@TL~D^f6g%Z|U4Y^79u+^YNDb{0W2b%#4RJ%X7J8jd9uhR-n? zckXGefQXh?GqR%Ls&qbS<`?P#>(<>bhV|rmtmXH|7YIfjXikjGc+YicJZn65rfc)pX|o6u*VPI!#WkI=7XrcW!!qR&o+|rz~9{ z*rxZWLS>~T3LBr#KL65>WLzxyEsnzQX>e06rL{=L_$#%SJayO0@^9asm$&Pi)>R4W75iysT z$(u=iweQhX1n4;sM|~GXedFPNanOKJaIG4BGz{wUT+Xh6KB~3NU1kuyjCsl8IVW^o zq`^2pHbJ%HPhe=-RUErmD~UDtlvB;4^yLbh%#ciR%D&qD^#mu5W^3n zY#REc_Gr{vw3LXj0AkFEAWm1hW^%65v62PaxOIK!9sa`9Kx&&Oq`V_hHIW9kEy)t; zKO6gP=~k$G5aEc+h1oM%o?08jVlLYDqpaR$Aa7*FD3wQ(O^1AqQ>ax+#!7d?+p1Ob zV6AUg@`GW7Vx9A?xx=Ck4ljc1KsySqtA8vSVSmrLiu+~?u(TYg@8#Rjhn9#2P~SSbdFh#gQ*ANA~&4nUWCEZ#sJ^X zZ(r-tq2US#-VNECg`B=Q;yzpBLLKRNs}lfipTLg;3Wk|L!Nj>MdDG@eYK-Nvu#y+^ zPx=7vP@5rUh{|Hf^{Ontik0QXelP}jU%*a>B$q1LIp*jXZ(8q{WanfXXmsq)ONqpt z;%}AfJ02R3#u@P6t+unNgC+hgZxxUSXJxV-Q}6QcA)f!rG2(o@BiMc;PQ}u-;rwMF5tAgzA%vRg>E43u7-IOYbV!-OQgu zRJBJIg{zd{uWl(UKNW}9ofDlx_2z$B^xu=RivN5!)hNpb%8NhPrNbvbCRja?Y1H;L z81h~a^GS)V)=Jt4tM7-sGZ$6>Nu3vhO8w~@ODML+&y_&b4I23Tj#WIrwA_e32gEvv;kkh` zet96q{MRTWp#C#b&3wkw$&g58vNK?aWM}>x0AEks_nZ2?IscX>dLea*TY0A_iNsckNZ#KvW>>KFJx*v?2t=+H$-Kt z`Ky=^zbq^t@1nx42JG{`L)xdh+hyS*En#W(afKozu|hL#rN6aAw4Y>DFwkXO>``Z|=_CxdZbKjcCifCMjG z0BRst(>S3pdNNDhmHq+gXp>vGu4F`4JHzV1FGrJf9FauEhp5am1wA8cD3>K)Qk-!( zkWt4DdXvn(@2jeB*x%qx8{PNhfqP0_;O(l=osIVGXZAZS7>!iCxL%?YJ}A|>BhH#W zhp|l01XmhRA%hSPqVbJvHG^8DO}tEvEd5-K;$1?O)@)c7wcWAq^Z7;jiMbzzKlzd; z1|=DP&tkLpkaL1$nGr_!jMRf`mxMGYcIFR1bne~nnAZ`%8eQ6y^@6Z4=LKwf^Kr<7 zYO5#7OG|*L_@DXGs_R!_xtFfp^9At4r6gR*PbD@Sqdk`ZHi~b}`}F8UHap@52Q&!w z&^3+FaR=NroKAEGpKzv=yIw%YqAc`?a#0yBx5x zde~!XkzM!Q+~qcEvzr|JBAX8+dDXIT$xgmFkRMsPl0gDs>l{8N^? z1M+AR!JX-B4v^B;>=5>Ytd}6W)BbYjbb!GAnT7)zB&RLsr&&9q#}j^e1;mtuuRYj+ z)V8MZ@czJ?hk1DUIrdGT3;pIpx)fa?tjvNbvwL!`N+V!=R2<2(JLqjj(`}e&J#@kI z@e=EpQxl_de0N^cYbT%qj`)4nscl+WHE*AZr$ykP=*y|1AU6Q09D!NPHd3FfjjL@M zf#ND=>iXRFy25?>EKMKYhzZ45HG6k##Q{;&-u-DsdF~DkGyR%JOqxorRzpvXI#q2^ zrQO!B96#1u@(3N96&9(_*(qiYx$G#DyhX#U$PXqHL`{1<`-QuJVO_K6+qhAn^7CTF zg2JJ&0?JDGR_o7Qj{Kt9KE)J0-0N7ya>UQ3+ue7VzNHUN1vhwB9fMetvhU?$*>~zv zw#6Hy;#|LkgihPcpfX!?-QRPxQD+f558B7($dRW5Ak^3J4vYOMX%FD-(bg4hY4bG) zsx-%Bf(i0iwKZk9W|tqBl#IKSdc)Vf;?zK+6?qO8L}z)$d3fJ^!{rIqY@g9NCtDF0@p!8=mDMs_ownAd{IQf(XZ*wGL=2J(AC zIN20vIqP3`z2>BJ-r&MU<={d>#SutM_my$HPYg_6DBMycxo!%sdpKZ*td-zW5(k#j1M6(71f>%Q<3% z5#gyn$sXX%DIPbxoz?r0To#AJf=;8cs-6Xl*hOzOXbJxWOzI1%0jpty@U~x|{FlT3 za54ue4wnx~Pu0;v3oWJ~%3!?wr&Kbl(ljc^=KTF^vo53lxLhcQM%s?3D^-k-L9#mD zQPBZJ6N{^BThq;(N<^)s=`yQZpW|xQ+*`LEX6%d#I}B{G9i?(!AfOxN>X8AO2OBZW zO>D3@E^xv3Tk&^2{EXpOnHlR3DhF}vrO<@CIY}t%4GEH+@eS^g0U$7ybs&42330+$ z$=JTOXKqzYCJ<;}wq=y20%6Ia>oXfeJhzrLM~IafyrFg5m2lP=vE^J4Pfq>FCq6so z(l=EjH<#V2=yx8ND{bU^rBLXdQIQw0KgGWJG^cd8} zENBkjEG0jUsQl5&5N124L7mqW16DTY9?Z)p3Wu||DEw~QSL%gTEbyMM$^W`!wCAcw zEzH*y@*JEzC8-7rdqcA_jyqTD0RopR36W;fkKSeR!f>?79_T(w;2=gW98 zytU6~LYm5kcWQx9c{!(C$}*~C0)b%atuF|;PDs*3STtXP9!(3#L=E)z?F4@8UB4uF z9B|A|HD|EVi_Xrzj`+ZNjdO2=qt)SQx<@Y7EleL8g1vv0QCbLCBy_jw5SQuO9d92W zKp~c9e3d-7!nc#kTI~fpsSvJYbjOT6dj~5?$vlZyn|~Dnls14WqddmrQD_SFQ|BPx zTZ9)({*0gP>(KekRbgBLdig11B(0P4@qEZcr_0ODXhEh@0uxpL#P9~FGj2nJf&MqE z-|-}ev}u`}e0y3&C|5{^a814%y20ykS9u@Q_JAgS_mHMW0N0Hw1XxRR2J*YTxNEaqX_tX`|?;7VA@3A5KYy z{TDDVyCMdzMQiuQW*4Y=-^qq|`(>db;UkIoAy@oeJ*tIiWi}_ovOQkPmD{-s0V8$| zsg!fqjCftW>b;tHX5!ppd){?@CP3nVBs#aqOuZTBNUyvhJ?chi_3%Fe*S`oe@J ztjipMB^Lvufp}=DiUir&UZ3qT&}32V&{C z`>UG)K+?7dp)Ry>pDUJrqG;+K<@$Om;qI}ckZ`$Ue(ngn>Cvp(^3NN}mCeM1mx0ln zU(Bw^TaN7In-_T&{Q!#w3VWDUu``E^b0}tUNCtmAG*F{mn6%lA$}6apg+Ir%EMIhO zK2X@BSjtK@q7s-(XQi%MVP2f#@~5q$#~-fBW;s{7g^{~osc*_w=(>4brHp*QN@E_y zEH$)FUTfh1At{U{>~TxnZPpMt`V1$3%YIh>xQ-gpRNm9Nz;m1(FQkQ`vBa}@Tbm7J zYso>QBq3q*qfW;wwvDAUTpA{2Mxdoy&g)R*1WylR0!-myFV;5YgpKf+f=ub0xfy4( zjvlhT8ZYMynF`G#6SBQ=#q?RZQdvf?N2#Q3y@(!RprBCCB9ePrfKv>H>TkDstA%Ky zzi_0Hcwp<-OI1+V1PNsm+vVl&R?G-Mu1MHP!1DNorQ|WR{4K8iNLGKqDO81zCzFG~ zPzUiRwc;K&ZOE5;0>~Dz_xK@Gvnrx~lA6!9|ET>D7Ifz=zfn+VCfm-V>!yx{Po17h zC>N`Bh!gj*d`|z~wUCjDE@iNrtBN}>+ME>9Ia`i*oBfoxMxS$-|J+8aLqDy|IXPHX z-@I8px(d`|UKu{XwZq%x3$zu7a-H;g!G&!vkyIUNOy;TFq)2k7XFJoLg?eT{jw`T2 z`C@Pa4N88@($U6omy&jDaheKxG!HMMkjNISeM6%v@jURXZf#uI)%MI-p%mVk{ z+J4PnH>WEnA=kc+#1;KowDjMnv-!PH7Nf@+A^Pk6nd0B|zX|-G5?DBTO+B?}*8F|| mLK?yZv;NweEfTTXUsF(!=~9SMe^H|)_KCJ!@nZ0;d;bCmAw^pN diff --git a/tests/test_synthetic.py b/tests/test_synthetic.py index 865ec4c8..d70830ad 100644 --- a/tests/test_synthetic.py +++ b/tests/test_synthetic.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals from ftfy.fixes import fix_encoding +from ftfy.badness import sequence_weirdness import sys from nose.tools import eq_ @@ -49,15 +50,15 @@ def test_lossy_utf8(): eq_(fix_encoding('“lossy decodingâ€�'), '“lossy decoding�') -def test_unknown_emoji(): +def test_burritos(): # Make a string with two burritos in it. Python doesn't know about Unicode # burritos, but ftfy can guess they're probably emoji anyway. emoji_text = 'dos burritos: \U0001f32f\U0001f32f' - # Mangle the burritos into a mess of Russian characters. (It would have + # Mangle the burritos into a mess of Cyrillic characters. (It would have # been great if we could have decoded them in cp437 instead, to turn them - # into "DOS burritos", but the resulting string is one ftfy could already - # fix.) + # into "DOS burritos", but the resulting string is one that ftfy has been + # able to fix even without knowledge of Unicode 9.) emojibake = emoji_text.encode('utf-8').decode('windows-1251') # Restore the original text. @@ -68,6 +69,8 @@ def test_unknown_emoji(): not_emoji = 'dos burritos: \U0003f32f\U0003f32f'.encode('utf-8').decode('windows-1251') eq_(fix_encoding(not_emoji), not_emoji) + +def test_unknown_emoji(): # The range we accept as emoji has gotten larger. Let's make sure we can # decode the futuristic emoji U+1F960, which will probably be a picture of # a fortune cookie in Unicode 10.0: @@ -85,3 +88,14 @@ def test_unknown_emoji(): # we consider the existing text, "ñŸ¥\xa0", to be more probable. not_emoji = "\U0005f960 I see mojibake in your present".encode('utf-8').decode('windows-1252') eq_(fix_encoding(not_emoji), not_emoji) + + +def test_unicode_9(): + # This string is 'bɪg'.upper() in Python 3.6 or later, containing the + # new codepoint U+A7AE LATIN CAPITAL LETTER SMALL CAPITAL I. + eq_(sequence_weirdness("BꞮG"), 0) + + # That should be less weird than having a definitely-unassigned character + # in the string. + eq_(sequence_weirdness("B\U00090000G"), 2) + From bb0484838b0db453515df54a83887278cda252bd Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 21 Dec 2016 13:45:13 -0500 Subject: [PATCH 304/502] update docs to describe Unicode 9 --- docs/index.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 290eb3a9..ec71473c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -148,6 +148,7 @@ But remember that the input to `ftfy` is Unicode, so it handles actual CJK *text* just fine. It just can't discover that a CJK *encoding* introduced mojibake into the text. + Using ftfy ---------- @@ -190,6 +191,7 @@ you should use :func:`ftfy.fix_encoding` directly. .. autofunction:: ftfy.explain_unicode + Non-Unicode strings ------------------- @@ -206,6 +208,7 @@ Reasonable ways that you might exchange data, such as JSON or XML, already have perfectly good ways of expressing Unicode strings. Given a Unicode string, ftfy can apply fixes that are very likely to work without false positives. + A note on encoding detection ---------------------------- @@ -310,8 +313,8 @@ Variants of UTF-8 *ftfy.chardata* and *ftfy.build_data*: trivia about characters -------------------------------------------------------------- -These files load information about the character properties in Unicode 8.0. -Yes, even if your version of Python doesn't support Unicode 8.0. This ensures +These files load information about the character properties in Unicode 9.0. +Yes, even if your version of Python doesn't support Unicode 9.0. This ensures that ftfy's behavior is consistent across versions. .. automodule:: ftfy.chardata @@ -323,7 +326,7 @@ that ftfy's behavior is consistent across versions. Future versions of ftfy ======================= -ftfy has full support for Python 2.7, even including a backport of Unicode 8 +ftfy has full support for Python 2.7, even including a backport of Unicode 9 character classes to Python 2. But given the sweeping changes to Unicode in Python, it's getting inconvenient to add new features to ftfy that work the same on both versions. From b9243c6402c2535094154fa817dd019716c36881 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 29 Dec 2016 15:49:30 -0500 Subject: [PATCH 305/502] Update the changelog --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 003f8e3a..85dee930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,44 @@ +## Version 4.3.0 (December 29, 2016) + +ftfy has gotten by for four years without dependencies on other Python +libraries, but now we can spare ourselves some code and some maintenance burden +by delegating certain tasks to other libraries that already solve them well. +This version now depends on the `html5lib` and `wcwidth` libraries. + +Feature changes: + +- The `remove_control_chars` fixer will now remove some non-ASCII control + characters as well, such as deprecated Arabic control characters and + byte-order marks. Bidirectional controls are still left as is. + + This should have no impact on well-formed text, while cleaning up many + characters that the Unicode Consortium deems "not suitable for markup" + (see Unicode Technical Report #20). + +- The `unescape_html` fixer uses a more thorough list of HTML entities, + which it imports from `html5lib`. + +- `ftfy.formatting` now uses `wcwidth` to compute the width that a string + will occupy in a text console. + +Heuristic changes: + +- Updated the data file of Unicode character categories to Unicode 9, as used + in Python 3.6.0. (No matter what version of Python you're on, ftfy uses the + same data.) + +Pending deprecations: + +- The `remove_bom` option will become deprecated in 5.0, because it has been + superseded by `remove_control_chars`. + +- ftfy 5.0 will remove the previously deprecated name `fix_text_encoding`. It + was renamed to `fix_encoding` in 4.0. + +- ftfy 5.0 will require Python 3.2 or later, as planned. Python 2 users, please + specify `ftfy < 5` in your dependencies if you haven't already. + + ## Version 4.2.0 (September 28, 2016) Heuristic changes: @@ -35,12 +76,14 @@ Heuristic changes: The heuristic has been changed, and all languages that use Latin letters should be on even footing now. + ## Version 4.1.1 (April 13, 2016) - Bug fix: in the command-line interface, the `-e` option had no effect on Python 3 when using standard input. Now, it correctly lets you specify a different encoding for standard input. + ## Version 4.1.0 (February 25, 2016) Heuristic changes: From 3e1f9ff8bbb6b91feb1af54800e6d69690691474 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 29 Dec 2016 16:56:23 -0500 Subject: [PATCH 306/502] Describe pending deprecations consistently --- ftfy/__init__.py | 13 +++++++------ ftfy/bad_codecs/utf8_variants.py | 3 +++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 6418a97c..af36d445 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -106,15 +106,16 @@ def fix_text(text, when they're appropriately paired, or replacing them with \ufffd otherwise. - - If `fix_control_characters` is true, remove control characters that are - not suitable for use in text. This includes most of the ASCII control + - If `remove_control_characters` is true, remove control characters that + are not suitable for use in text. This includes most of the ASCII control characters, plus some Unicode controls such as the byte order mark - (U+FEFF). Useful control characters such as Tab and Line Feed are left - as they are. + (U+FEFF). Useful control characters, such as Tab, Line Feed, and + bidirectional marks, are left as they are. - If `remove_bom` is True, remove the Byte-Order Mark at the start of the - string if it exists. It may be that you don't want to remove all - control characters (as above) but you still want to remove the BOM. + string if it exists. (This is largely redundant, because it's a special + case of `remove_control_characters`. This option will become deprecated + in a later version.) - If `normalization` is not None, apply the specified form of Unicode normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 4cdb2239..cd89be69 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -235,6 +235,9 @@ def mangle_surrogates(bytestring): applications from malformed strings, we turn surrogate sequences at the start of the string into the bytes `ff ff ff`, which we're *sure* won't decode, and which turn into three replacement characters in 'replace' mode. + + This function does nothing in Python 3, and it will be deprecated in ftfy + 5.0. """ if PYTHON2: if bytestring.startswith(b'\xed') and len(bytestring) >= 3: From 3a17430dc91186567c6873f7fdf7d18fd907efe7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 12 Jan 2017 13:39:47 -0500 Subject: [PATCH 307/502] v4.3.1: Fix 'remove_control_chars' to leave '\r' alone. --- CHANGELOG.md | 8 ++++++++ ftfy/__init__.py | 4 ++-- ftfy/chardata.py | 2 +- setup.py | 2 +- tests/test_characters.py | 4 ++-- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85dee930..2d6ed894 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Version 4.3.1 (January 12, 2017) + +Bug fix: + +' `remove_control_chars` was removing U+0D ('\r') prematurely. That's the + job of `fix_line_breaks`. + + ## Version 4.3.0 (December 29, 2016) ftfy has gotten by for four years without dependencies on other Python diff --git a/ftfy/__init__.py b/ftfy/__init__.py index af36d445..c39f1ec5 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -13,7 +13,7 @@ from ftfy.formatting import display_ljust from ftfy.compatibility import is_printable -__version__ = '4.3.0' +__version__ = '4.3.1' # See the docstring for ftfy.bad_codecs to see what we're doing here. @@ -106,7 +106,7 @@ def fix_text(text, when they're appropriately paired, or replacing them with \ufffd otherwise. - - If `remove_control_characters` is true, remove control characters that + - If `remove_control_chars` is true, remove control characters that are not suitable for use in text. This includes most of the ASCII control characters, plus some Unicode controls such as the byte order mark (U+FEFF). Useful control characters, such as Tab, Line Feed, and diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 40260d43..2990c539 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -163,7 +163,7 @@ def _build_control_char_mapping(): for i in itertools.chain( range(0x00, 0x09), [0x0b], - range(0x0d, 0x20), [0x7f], + range(0x0e, 0x20), [0x7f], range(0x206a, 0x2070), [0xfeff], range(0xfff9, 0xfffd), diff --git a/setup.py b/setup.py index ccff46eb..068b41ca 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="ftfy", - version='4.3.0', + version='4.3.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", diff --git a/tests/test_characters.py b/tests/test_characters.py index f985533d..a571630c 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -43,9 +43,9 @@ def test_byte_order_mark(): def test_control_chars(): text = ( "\ufeffSometimes, \ufffcbad ideas \x7f\ufffalike these characters\ufffb " - "\u206aget standardized\U000E0065\U000E006E.\n" + "\u206aget standardized\U000E0065\U000E006E.\r\n" ) - fixed = "Sometimes, bad ideas like these characters get standardized.\n" + fixed = "Sometimes, bad ideas like these characters get standardized.\r\n" eq_(remove_control_chars(text), fixed) From df2fa69f0c6806b0655e18be82c26f282b250115 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 13 Feb 2017 13:11:19 -0500 Subject: [PATCH 308/502] Add testimonial (quoted from e-mail with permission) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e9f7a14a..f8df1cbf 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ Full documentation: **http://ftfy.readthedocs.org** — [@planarrowspace](http://twitter.com/planarrowspace) - “A handy piece of magic” — [@simonw](http://twitter.com/simonw) +- “ftfy did the right thing right away, with no faffing about. Excellent work, solving a very tricky real-world (whole-world!) problem.” + — Brennan Young - “Hat mir die Tage geholfen. Im Übrigen bin ich der Meinung, dass wir keine komplexen Maschinen mit Computern bauen sollten solange wir nicht einmal Umlaute sicher verarbeiten können. :D” — [Bruno Ranieri](http://yrrsinn.de/2012/09/17/gelesen-kw37/) - “I have no idea when I’m gonna need this, but I’m definitely bookmarking it.” From cd60fdb9c2360bf12c8c490e1eafae1eb0f1bd0d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 14 Feb 2017 19:25:28 -0500 Subject: [PATCH 309/502] Adjust heuristics for common cases in Windows-1250 mojibake --- ftfy/badness.py | 11 +++++++++-- ftfy/chardata.py | 6 ++++-- ftfy/fixes.py | 4 +++- tests/test_characters.py | 2 +- tests/test_real_tweets.py | 3 +++ 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index c08b7a1e..ad3dfd06 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -58,6 +58,13 @@ def _make_weirdness_regex(): groups.append('[Ll][AaC]') groups.append('[AaC][Ll]') + # Match non-combining diacritics. We've already set aside the common ones + # like ^ (the CIRCUMFLEX ACCENT, repurposed as a caret, exponent sign, + # or happy eye) and assigned them to category 'o'. The remaining ones, + # like the diaeresis (¨), are pretty weird to see on their own instead + # of combined with a letter. + groups.append('2') + # Match C1 control characters, which are almost always the result of # decoding Latin-1 that was meant to be Windows-1252. groups.append('X') @@ -70,9 +77,9 @@ def _make_weirdness_regex(): # - Modifier marks (M) # - Letter modifiers (m) # - Miscellaneous numbers (N) - # - Symbols (123) + # - Symbols (1 or 3, because 2 is already weird on its own) - exclusive_categories = 'MmN123' + exclusive_categories = 'MmN13' for cat1 in exclusive_categories: others_range = ''.join(c for c in exclusive_categories if c != cat1) groups.append('{cat1}[{others_range}]'.format( diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 2990c539..a6bf1514 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -12,14 +12,16 @@ from pkg_resources import resource_string from ftfy.compatibility import unichr -# These are the five encodings we will try to fix in ftfy, in the +# These are the encodings we will try to fix in ftfy, in the # order that they should be tried. CHARMAP_ENCODINGS = [ 'latin-1', 'sloppy-windows-1252', + 'sloppy-windows-1250', + 'iso-8859-2', + 'sloppy-windows-1251', 'macroman', 'cp437', - 'sloppy-windows-1251', ] diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 13fa16cb..89e3d2c6 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -126,8 +126,10 @@ def fix_text_encoding(text): # that these encodings will only be used if they fix multiple problems. ENCODING_COSTS = { 'macroman': 2, + 'iso-8859-2': 2, + 'sloppy-windows-1250': 2, + 'sloppy-windows-1251': 3, 'cp437': 3, - 'sloppy-windows-1251': 5 } diff --git a/tests/test_characters.py b/tests/test_characters.py index a571630c..5700323b 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -18,7 +18,7 @@ def test_bmp_characters(): for index in range(0xa0, 0xfffd): char = unichr(index) # Exclude code points that are not assigned - if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn'): + if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn', 'Sk'): garble = char.encode('utf-8').decode('latin-1') # Exclude characters whose re-encoding is protected by the # 'sequence_weirdness' metric diff --git a/tests/test_real_tweets.py b/tests/test_real_tweets.py index 5f5fa0d5..bd177d0e 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_tweets.py @@ -53,6 +53,9 @@ ("#╨┐╤Ç╨░╨▓╨╕╨╗╤î╨╜╨╛╨╡╨┐╨╕╤é╨░╨╜╨╕╨╡", "#правильноепитание"), ('∆°', '∆°'), + # Test Windows-1250 mixups + ("Liège Avenue de l'HĂ´pital", "Liège Avenue de l'Hôpital"), + # This one has two differently-broken layers of Windows-1252 <=> UTF-8, # and it's kind of amazing that we solve it. ('Arsenal v Wolfsburg: pre-season friendly â\x80â\x80\x9c live!', From 088b2d14f26db1ae542cf9ed53d4dee6b3a0e045 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 15 Feb 2017 16:38:41 -0500 Subject: [PATCH 310/502] Rename test_real_tweets; include some non-tweet Windows-1250 examples --- ...{test_real_tweets.py => test_real_text.py} | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) rename tests/{test_real_tweets.py => test_real_text.py} (84%) diff --git a/tests/test_real_tweets.py b/tests/test_real_text.py similarity index 84% rename from tests/test_real_tweets.py rename to tests/test_real_text.py index bd177d0e..71388463 100644 --- a/tests/test_real_tweets.py +++ b/tests/test_real_text.py @@ -6,8 +6,8 @@ TEST_CASES = [ - ## These are excerpts from tweets actually seen on the public Twitter - ## stream. Usernames and links have been removed. + ## These are excerpts from text actually seen in the wild, mostly on + ## Twitter. Usernames and links have been removed. ("He's Justin❤", "He's Justin❤"), ("Le Schtroumpf Docteur conseille g√¢teaux et baies schtroumpfantes pour un r√©gime √©quilibr√©.", "Le Schtroumpf Docteur conseille gâteaux et baies schtroumpfantes pour un régime équilibré."), @@ -55,6 +55,8 @@ # Test Windows-1250 mixups ("Liège Avenue de l'HĂ´pital", "Liège Avenue de l'Hôpital"), + ("It was named „scars´ stones“ after the rock-climbers who got hurt while climbing on it.", + "It was named\xa0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it."), # This one has two differently-broken layers of Windows-1252 <=> UTF-8, # and it's kind of amazing that we solve it. @@ -92,19 +94,26 @@ ] -def test_real_tweets(): +def test_real_text(): """ - Test with text actually found on Twitter. + Test with text actually found in the wild (mostly on Twitter). - I collected these test cases by listening to the Twitter streaming API for + I collected test cases by listening to the Twitter streaming API for a million or so tweets, picking out examples with high weirdness according to ftfy version 2, and seeing what ftfy decoded them to. There are some impressive things that can happen to text, even in an ecosystem that is supposedly entirely UTF-8. - The tweets that appear in TEST_CASES are the most interesting examples of - these, with some trickiness of how to decode them into the actually intended - text. + TEST_CASES contains the most interesting examples of these, often with some + trickiness of how to decode them into the actually intended text. + + For some reason, sampling Twitter gives no examples of text being + accidentally decoded as Windows-1250, even though it's one of the more + common encodings and this mojibake has been spotted in the wild. It may be + that Windows-1250 is used in places that culturally don't use Twitter much + (Central and Eastern Europe), and therefore nobody designs a Twitter app or + bot to use Windows-1250. I've collected a couple of examples of + Windows-1250 mojibake from elsewhere. """ for orig, target in TEST_CASES: # make sure that the fix_encoding step outputs a plan that we can From c06f814bd0e44c906f9281be8b5be454e6caa326 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 15 Feb 2017 16:43:26 -0500 Subject: [PATCH 311/502] Add a version of the windows-1250 test for iso-8859-2 --- tests/test_real_text.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_real_text.py b/tests/test_real_text.py index 71388463..5781a7e7 100644 --- a/tests/test_real_text.py +++ b/tests/test_real_text.py @@ -58,6 +58,10 @@ ("It was named „scars´ stones“ after the rock-climbers who got hurt while climbing on it.", "It was named\xa0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it."), + # The second test is different in iso-8859-2 + ("It was namedÂ\xa0â\x80\x9escars´ stonesâ\x80\x9c after the rock-climbers who got hurt while climbing on it.", + "It was named\xa0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it."), + # This one has two differently-broken layers of Windows-1252 <=> UTF-8, # and it's kind of amazing that we solve it. ('Arsenal v Wolfsburg: pre-season friendly â\x80â\x80\x9c live!', @@ -106,7 +110,7 @@ def test_real_text(): TEST_CASES contains the most interesting examples of these, often with some trickiness of how to decode them into the actually intended text. - + For some reason, sampling Twitter gives no examples of text being accidentally decoded as Windows-1250, even though it's one of the more common encodings and this mojibake has been spotted in the wild. It may be From 76359a3f3cc54ba519d992c5a0b4cce4dbcd8910 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 16 Feb 2017 16:44:14 -0500 Subject: [PATCH 312/502] found some more test cases for windows-1250 --- tests/test_real_text.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_real_text.py b/tests/test_real_text.py index 5781a7e7..bb71a191 100644 --- a/tests/test_real_text.py +++ b/tests/test_real_text.py @@ -57,6 +57,8 @@ ("Liège Avenue de l'HĂ´pital", "Liège Avenue de l'Hôpital"), ("It was named „scars´ stones“ after the rock-climbers who got hurt while climbing on it.", "It was named\xa0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it."), + ("vedere Ă®nceĹŁoĹźatÄ\x83", "vedere înceţoşată"), + ("NapĂ\xadšte nám !", "Napíšte nám !"), # The second test is different in iso-8859-2 ("It was namedÂ\xa0â\x80\x9escars´ stonesâ\x80\x9c after the rock-climbers who got hurt while climbing on it.", From 957027cb1d04be40c0665da69784e7fb1e8072aa Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 16 Feb 2017 13:11:55 -0500 Subject: [PATCH 313/502] `fix_entities` works better after `fix_encoding` Applying `fix_entities` can introduce characters that make `fix_encoding` impossible, but not the other way around. Thus, I'm changing the order of operations to put `fix_entities` after `fix_encoding`. --- ftfy/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index c39f1ec5..7082155e 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -73,12 +73,6 @@ def fix_text(text, Based on the options you provide, ftfy applies these steps in order: - - If `fix_entities` is True, replace HTML entities with their equivalent - characters. If it's "auto" (the default), then consider replacing HTML - entities, but don't do so in text where you have seen a pair of actual - angle brackets (that's probably actually HTML and you shouldn't mess - with the entities). - - If `remove_terminal_escapes` is True, remove sequences of bytes that are instructions for Unix terminals, such as the codes that make text appear in different colors. @@ -87,6 +81,12 @@ def fix_text(text, encoding or decoding Unicode text incorrectly, and fix them if they are reasonably fixable. See `fixes.fix_encoding` for details. + - If `fix_entities` is True, replace HTML entities with their equivalent + characters. If it's "auto" (the default), then consider replacing HTML + entities, but don't do so in text where you have seen a pair of actual + angle brackets (that's probably actually HTML and you shouldn't mess + with the entities). + - If `uncurl_quotes` is True, replace various curly quotation marks with plain-ASCII straight quotes. @@ -268,12 +268,12 @@ def fix_text_segment(text, fix_entities = False while True: origtext = text - if fix_entities: - text = fixes.unescape_html(text) if remove_terminal_escapes: text = fixes.remove_terminal_escapes(text) if fix_encoding: text = fixes.fix_encoding(text) + if fix_entities: + text = fixes.unescape_html(text) if fix_latin_ligatures: text = fixes.fix_latin_ligatures(text) if fix_character_width: From 0e879842643b6ced252ced072d026ff177d7920e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 16 Feb 2017 13:21:06 -0500 Subject: [PATCH 314/502] Replace synthetic example with a real one This attempt to tweet the shrug emoticon is corrupted in a way that depends on HTML entities being fixed after encodings. --- ftfy/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 7082155e..75967aff 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -52,9 +52,8 @@ def fix_text(text, >>> print(fix_text('HTML entities <3')) HTML entities <3 - >>> print(fix_text('\001\033[36;44mI’m blue, da ba dee da ba ' - ... 'doo…\033[0m', normalization='NFKC')) - I'm blue, da ba dee da ba doo... + >>> print(fix_text("¯\\_(ã\x83\x84)_/¯")) + ¯\_(ツ)_/¯ >>> # This example string starts with a byte-order mark, even if >>> # you can't see it on the Web. From 1ab560eaddfed1be51a81e63bcf04648720b213d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 15:07:58 -0500 Subject: [PATCH 315/502] set version to 4.4, remove warning about ftfy 5 This version is being released at the same time as 5.0, so if you got it, you already heeded the warning. --- ftfy/__init__.py | 2 +- setup.py | 20 +------------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 75967aff..3764cf50 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -13,7 +13,7 @@ from ftfy.formatting import display_ljust from ftfy.compatibility import is_printable -__version__ = '4.3.1' +__version__ = '4.4' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/setup.py b/setup.py index 068b41ca..947676a8 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="ftfy", - version='4.3.1', + version='4.4', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", @@ -38,21 +38,3 @@ ] } ) - - -if sys.version_info[0] < 3: - sys.stderr.write(""" - Heads up! You're now running ftfy 4.x on Python 2.x. This is fine, but - here's an advance warning that you won't be able to upgrade ftfy - forever without upgrading Python. - - The next major version of ftfy, version 5.0, will probably only work - on Python 3, making it easier to develop and ensure its consistency. - - It's fine if you're happy with Python 2 and ftfy 4. Save yourself a - headache later and be sure to pin a version in your dependencies. - Instead of just asking for 'ftfy', you should ask for: - - 'ftfy >= 4, < 5' - """.strip(' ')) - From 957aff8cfd7154606deda16b96f4250fbe71b8fb Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 15:14:45 -0500 Subject: [PATCH 316/502] update the changelog for 4.4 --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d6ed894..ce978472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +## Version 4.4.0 (February 17, 2017) + +Heuristic changes: + +- ftfy can now fix mojibake involving the Windows-1250 or ISO-8859-2 encodings. + +- With a few exceptions for commonly-used characters such as `^`, it is now + considered "weird" whenever a diacritic appears in non-combining form, + such as the diaeresis character `¨`. + +- These changes to the heuristics, and others we've made in recent versions, + let us lower the "cost" for fixing mojibake in some encodings, causing them + to be fixed in more cases. + + ## Version 4.3.1 (January 12, 2017) Bug fix: From ec36d2645f28833da50d7734fc66c3c27351e87c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 15:16:37 -0500 Subject: [PATCH 317/502] Update docs to reflect new encodings that we fix --- docs/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index ec71473c..416b4f6c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -122,6 +122,8 @@ encodings: - Latin-1 (ISO-8859-1) - Windows-1252 (cp1252 -- used in Microsoft products) - Windows-1251 (cp1251 -- the Russian version of cp1252) +- Windows-1250 (cp1250 -- the Eastern European version of cp1252) +- ISO-8859-2 (which is not quite the same as Windows-1250) - MacRoman (used on Mac OS 9 and earlier) - cp437 (used in MS-DOS and some versions of the Windows command prompt) From 4efeb5ecb155192696d7597ab2cbd0f56195ff44 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 16:23:44 -0500 Subject: [PATCH 318/502] Mention the change in the order of fixers in the CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce978472..ede41df2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ Heuristic changes: - ftfy can now fix mojibake involving the Windows-1250 or ISO-8859-2 encodings. +- The `fix_entities` fixer is now applied after `fix_encoding`. This makes + more situations resolvable when both fixes are needed. + - With a few exceptions for commonly-used characters such as `^`, it is now considered "weird" whenever a diacritic appears in non-combining form, such as the diaeresis character `¨`. From f3545de091fdfd5a227dc6b82e16b5e9bf6275c7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 21 Feb 2017 00:54:17 -0500 Subject: [PATCH 319/502] Adjusted heuristics to fix a class of false positives I believe that these false positives, such as a word with an accent followed by a registered sign, have always been there. Added the trade mark and registered signs as characters we should be forgiving of, which unfortunately makes one test case into a false negative. --- ftfy/badness.py | 2 ++ tests/test_real_text.py | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index ad3dfd06..d5d55518 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -98,6 +98,8 @@ def _make_weirdness_regex(): '\N{LEFT SINGLE QUOTATION MARK}\N{LEFT DOUBLE QUOTATION MARK}' '\N{RIGHT SINGLE QUOTATION MARK}\N{RIGHT DOUBLE QUOTATION MARK}' '\N{INVERTED EXCLAMATION MARK}\N{INVERTED QUESTION MARK}\N{DEGREE SIGN}' + '\N{TRADE MARK SIGN}' + '\N{REGISTERED SIGN}' '\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}' '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}' diff --git a/tests/test_real_text.py b/tests/test_real_text.py index bb71a191..f8275359 100644 --- a/tests/test_real_text.py +++ b/tests/test_real_text.py @@ -42,8 +42,6 @@ 'eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías'), ("Direzione Pd, ok “senza modifiche” all'Italicum.", "Direzione Pd, ok \"senza modifiche\" all'Italicum."), - ('Engkau masih yg terindah, indah di dalam hatiku♫~', - 'Engkau masih yg terindah, indah di dalam hatiku♫~'), ('SENSЕ - Oleg Tsedryk', 'SENSЕ - Oleg Tsedryk'), # this Е is a Ukrainian letter ('OK??:( `¬´ ):', 'OK??:( `¬´ ):'), ("selamat berpuasa sob (Ã\xa0¸‡'̀⌣'ÃŒÂ\x81)Ã\xa0¸‡", @@ -83,6 +81,8 @@ ("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), ("SELKÄ\xa0EDELLÄ\xa0MAAHAN via @YouTube", "SELKÄ\xa0EDELLÄ\xa0MAAHAN via @YouTube"), ("Offering 5×£35 pin ups", "Offering 5×£35 pin ups"), + ("NESTLÉ® requiere contratar personal para diferentes areas a nivel nacional e internacional", + "NESTLÉ® requiere contratar personal para diferentes areas a nivel nacional e internacional"), ## This remains a false positive # ("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), @@ -94,6 +94,11 @@ ## The original text looks too plausible # ('CÃ\xa0nan nan GÃ\xa0idheal', 'Cànan nan Gàidheal'), + ## Turning sorta-plausible text into musical notes isn't convincing enough + ## under the current heuristics + # ('Engkau masih yg terindah, indah di dalam hatiku♫~', + # 'Engkau masih yg terindah, indah di dalam hatiku♫~'), + ## The heuristics aren't confident enough to fix this text and its weird encoding. #("Blog Traffic Tip 2 – Broadcast Email Your Blog", # "Blog Traffic Tip 2 – Broadcast Email Your Blog"), From d0ec45d40efee18019cb77f2e6f6b2550a253150 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 23 Feb 2017 14:29:33 -0500 Subject: [PATCH 320/502] Judge IPA letters differently from typical Latin letters --- ftfy/badness.py | 8 ++++++++ ftfy/build_data.py | 8 +++++++- ftfy/char_classes.dat | Bin 3996 -> 3989 bytes tests/test_synthetic.py | 3 +++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index d5d55518..b00d4e88 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -58,6 +58,14 @@ def _make_weirdness_regex(): groups.append('[Ll][AaC]') groups.append('[AaC][Ll]') + # Match IPA letters next to capital letters. + # + # IPA uses lowercase letters only. Some accented capital letters next to + # punctuation can accidentally decode as IPA letters, and an IPA letter + # appearing next to a capital letter is a good sign that this happened. + groups.append('[LA]i') + groups.append('i[LA]') + # Match non-combining diacritics. We've already set aside the common ones # like ^ (the CIRCUMFLEX ACCENT, repurposed as a caret, exponent sign, # or happy eye) and assigned them to category 'o'. The remaining ones, diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 85b84140..8269d2ee 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -59,7 +59,13 @@ def make_char_data_file(do_it_anyway=False): char = unichr(codepoint) category = unicodedata.category(char) - if category.startswith('L'): # letters + if (0x250 <= codepoint < 0x300) and char != 'ə': + # IPA symbols and modifiers. + # + # This category excludes the schwa (ə), which is used as a normal + # Latin letter in some languages. + cclasses[codepoint] = 'i' + elif category.startswith('L'): # letters if unicodedata.name(char, '').startswith('LATIN'): if category == 'Lu': cclasses[codepoint] = 'L' diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index c03f661f044f874afc0aac53f13053671f9e36ef..e963e65681ee7d340a6208690db66cfa0ea5ed35 100644 GIT binary patch literal 3989 zcmeHIX;c$d7RD)+)&-{r0Yyz}T>w!6$RdlN!=NIYC%&$48zvjohANQSi?)|>|&Ue0Z z@7;U}-wRRH`6r!^)hk9{Sp4Afq{xENfj_v5xVvRD0_WDs`tS(v+Y{$xeQ7YO(CqEL zO$}8s>!&y=FEh(1E~j9(dS32HhH!M%*RFb;^1Jg2lY8;G5lB?BgUBrISVzZMWZ+i4 zmS9Y%r*o8+RDzF3j`=c`5exJY%7_EU&#;}xvszjj(nHT*jx^#RU0ui2HY;wlB=Mvp zL`pr#IckiIE#+~w&ds!0bLE3z;DSP#%!b2|L9YY|>ZF|ogHAUhlgUAy;kUH=h?U0!;?H9&wBBnua{WgwgB#^oTmpajSPAT7HqI@|^|MbOu^@0}zTJ1SvZj^1ZM=)sJMjEy zYbi#$>@<)X+KcF}HoQ|{7aShvtGu5A@VS&fBLL^tc|k0~9BC~T-*8){7SkhxP_U^L z;)thXTE&Z0A4t&frCe)KqFdNEvEo!Uv=eGX+t?xE+6}CNi&l7lE$j#4SUnfAH|ssU z+3>^sw1+EGY}RG1d%=*{jbTdx27CxB*IMFH!YVZP_}++Wq$u_v>87gL1i-;kCZat@ zWESo!ZY9>0)!9?5rUkqn$US2LH_HX@l0w>q2jmr{Rf+QcdIH1Zbnbfxa7`z}f1XSpCN^Di3<5sx^y)Pv?`zP9 z+;9R|eqSMdke_R2;v&vHU@3II8>Im zIN@(DYx`Oe;j6L&yeVs64;j9Y9n03J5CgzsQ8dQ@$3l=L=L@N~kG-)d7nQEcSFX#Shbzblj4&F zZSUAxItq?#QE%T5Nh^|fFsI9$;(lI>VHc(O{Sfcgg@3f6L+pnY>Qf$Q=54G45p8;c zi3>~&iC%B3KJ-AT8=>MwSJqOMnbqBus?tbo8ji(CRjew+pwI0MuP_*3!N~>Ce!3nP z*pS&*^{q2jQ&TNXS$?xH(n@qa1nsEMNV$SAdz@1j@GaY;=A-eXcV3d<0JSx7UBfaoKL| zMr$iat`}iEzM+YsuGf982=SIrZU=(Yt8V~OXS{>Umnc>F-xfT(aO$sWe;P=aSAR^x zK+&$tXFV2o^Og=}!e0sTI2<)=QL(gy;#weqseIR6=p4b}`j_X9preTe2e9A4SDsP# zz7MWAMygDzCe5GVNJ9Bn0<%{>ZQAoqpyzsGCRA6dXrE?wl@QV;@;m016DHQ?r0=hH z`MfOXPODeZD%Ym;;%29osVav4XN{goQ&hp<9tO_C)4#r%YA-A6t9N`Ob-~c56K*MP zODcrKdGQUZ^keA}v)KUt#XS|I^kG5YyXTWijlJ~p~=iU;8ZwchHd}T&TZLo z(-PT2|E5dN3bgnmT->>Ijz(V4agCt5;2w6Je#CWzmrn!=tI(*m0{Q2RIPB#zY zW@1!f5IDkKWaH#NLoy40>*qB7(OcM@XheaFf3Jxo>uOuSq5={>M-p*0 zYs`@`BPZ%rJ+4IZZXebdVaL)ip$oV&hf(*WGAsYpvRzYAwDHJ!KJSz1Sk~G-l_n!( z`TE1mj_3`gNgn6@5LV-7qBO49;c-cqp4~j_*P#8R zrtvyfLfV1Z&1euR*v4@o0{U?5YSO}p2t&X#;IcvYYnF8pi@JPqe>makWb<@Z2`hi* z1{s?-bJ#}HF@S6>d?bFYVHKsW3aXT$i=?~4) z!KT$#<$`a*<5RHxS0|hGDI)_ZS7tMS!8jrevZJlUdZ$FH1~rYvz_gVIgi9;!(8Y_% z#6a!vLUUGJvcg|7Ck)bWTx|?}$1rNk7bqKd#64BQC!{`?uyJ<7i4Qe3iT$gV3&^6l z*)fAe9O0AW#XF;#&k|4Zv44WkS+8q5MN@$tN1Go4`?N0MUR!iNXb7iKTQ^O5rKDdx zeb0c9diFkWVz_(H@75pZCM%84uX(jnh#eY7)+VV6or{nLB{MbYiO_05!3$i*QV6AS z8Rfu$keL+K2lxGaEjMw-o+;!7l5)e#@^IE)uIu?2ivu-@|GscyU+v(h31R8g)fQk9DV87mFHG+JlJ$T`^#+S zK*CX_t6JLq7st~1`ibAbT}ho|Di3ILJ|b8;7k;HbGPfbbrdl-)=SdZRpolN!xEXHAmnXlI&2k3L+68~X|Ch4y*J?ZNE=;oOSIAw$ z9C`WQ^K4^V?{b_V*w8@IBf5 gZ+rcZfPcfRnX(D^e@(VO$isYae0|p$*N8)Z2bFw`e*gdg literal 3996 zcmeHIX;2ebwssl87F)U%aa1(g_yk0j24shn_=E-oM2N^5MMcnt$i5TBtyK^KNt!^2 zvJ;lb5&{WH5X7(}OTrS82naDGAtoUK5|T`J&rEgIRK2RHnqRMK&W~H?+_T(U-#K5M zq+&k9HKwe;bj!#33GMF;FPdU?tmeI#Ss!QrB1fgmNUkscr#*<>dsE}5CSBLP<(Vo4 zM#%QsFc<5Mi1jfR&=haqLD?0Mx-0m95(a1dC zZN^)iXvD8h7e|Y*!0T*k9ks&%+HdXXD0_;@F`jZMaw1lzh$WKIf;aN%A)mt|(=pK1 z8e02$xS2zM)P8!f!W0?xc_=T;jIjFvEy^T;2o_8)rg)9>GRW-7mb>jiVy*);~7(L;R2A0bm}dTNK5_o+P;rvSvqUr4CqI1PIf=7-GhXh#ooir} zSsATU@g%?}q+=P?l3RBN4mXHbb4V8MqjdL*LpjcA1$9tDad++yA449pjzOC9ta|#I z==CXq+N2Tpy_7y{XaouVE<3x?e9g;BYrfn;^t;LXce|1m274ne3C18kQO#3JRbFItVFT_^1?$A#SIV|c zkn{3tl}BiR(*E{=)8&8=Wr0kffxWDXzBH+Py)BQVq{kr&#-+i;8^zd$3adwL-wpul z=DZc~!=$}E4AK}JN|dq11H21pvIq7>Orw&nlWv)9YLlz%P_j9=!^tpZ%lJ+Y=y4aZ zNH2PP72>gx7a>ETg8S%~X0`J!OlqF&CRS)rqePa#V1PA4dGn<=Mki?YgORJ6w~J)3 z&Lb-A?#@x|G8yAm#9IQ$&WT_p;Z%VV{nYzsGIQK!d7?0Mmo^2Q|>u0I=o7-uL?}b6Ql^@1kCZQHc@i}4H2J;t#BMHBtG_&vW#Fr&m$)aGa`TE2*!`bs1IvQSY%{QO%lqb<|bUUEZ3eQMjRn>sTL%G1!50 z`9Wd!;(h7k9p?=OYQ4OhOhrfI9XVeHtAk>-9go>n?I|l0Jt@v)<&8_QC9z6$#3%0WyPr7iqolicJfp-}AR!0!m#rd!w|cl`MAzF`{FeaV+Y{qe4-EPgx1VhYs@Un!2-7EVMWlI3JTgY09N>Wl&l@- z965-CrSxK_Xy{Cfii4|N*HE@p|6-_At5iO5d((zin1k1@TL~D^f6g%Z|U4Y^79u+^YNDb{0W2b%#4RJ%X7J8jd9uhR-n? zckXGefQXh?GqR%Ls&qbS<`?P#>(<>bhV|rmtmXH|7YIfjXikjGc+YicJZn65rfc)pX|o6u*VPI!#WkI=7XrcW!!qR&o+|rz~9{ z*rxZWLS>~T3LBr#KL65>WLzxyEsnzQX>e06rL{=L_$#%SJayO0@^9asm$&Pi)>R4W75iysT z$(u=iweQhX1n4;sM|~GXedFPNanOKJaIG4BGz{wUT+Xh6KB~3NU1kuyjCsl8IVW^o zq`^2pHbJ%HPhe=-RUErmD~UDtlvB;4^yLbh%#ciR%D&qD^#mu5W^3n zY#REc_Gr{vw3LXj0AkFEAWm1hW^%65v62PaxOIK!9sa`9Kx&&Oq`V_hHIW9kEy)t; zKO6gP=~k$G5aEc+h1oM%o?08jVlLYDqpaR$Aa7*FD3wQ(O^1AqQ>ax+#!7d?+p1Ob zV6AUg@`GW7Vx9A?xx=Ck4ljc1KsySqtA8vSVSmrLiu+~?u(TYg@8#Rjhn9#2P~SSbdFh#gQ*ANA~&4nUWCEZ#sJ^X zZ(r-tq2US#-VNECg`B=Q;yzpBLLKRNs}lfipTLg;3Wk|L!Nj>MdDG@eYK-Nvu#y+^ zPx=7vP@5rUh{|Hf^{Ontik0QXelP}jU%*a>B$q1LIp*jXZ(8q{WanfXXmsq)ONqpt z;%}AfJ02R3#u@P6t+unNgC+hgZxxUSXJxV-Q}6QcA)f!rG2(o@BiMc;PQ}u-;rwMF5tAgzA%vRg>E43u7-IOYbV!-OQgu zRJBJIg{zd{uWl(UKNW}9ofDlx_2z$B^xu=RivN5!)hNpb%8NhPrNbvbCRja?Y1H;L z81h~a^GS)V)=Jt4tM7-sGZ$6>Nu3vhO8w~@ODML+&y_&b4I23Tj#WIrwA_e32gEvv;kkh` zet96q{MRTWp#C#b&3wkw$&g58vNK?aWM}>x0AEks_nZ2?IscX Date: Thu, 23 Feb 2017 14:43:58 -0500 Subject: [PATCH 321/502] describe IPA heuristic in changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ede41df2..525f542a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ Heuristic changes: considered "weird" whenever a diacritic appears in non-combining form, such as the diaeresis character `¨`. +- It is also now weird when IPA phonetic letters, besides `ə`, appear next to + capital letters. + - These changes to the heuristics, and others we've made in recent versions, let us lower the "cost" for fixing mojibake in some encodings, causing them to be fixed in more cases. From b6615795e817e8dd27bbbf81ec0a36d4bbe2251a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 27 May 2014 14:17:01 -0400 Subject: [PATCH 322/502] Simplify code by removing compatibility with Python <= 3.3. Don't panic. You can still use code from before this commit if your requirements have you stuck on Python 2. But Python 3 really does have much better Unicode support *and* better semantics for bytestrings, which I can't take advantage of while still developing for Python 2 and 3. So this is the version where I would like new features to go in the future. --- ftfy/__init__.py | 4 +-- ftfy/bad_codecs/__init__.py | 1 - ftfy/bad_codecs/sloppy.py | 5 ++- ftfy/bad_codecs/utf8_variants.py | 49 +++++----------------------- ftfy/badness.py | 1 - ftfy/build_data.py | 5 +-- ftfy/chardata.py | 4 +-- ftfy/cli.py | 7 +--- ftfy/compatibility.py | 55 -------------------------------- ftfy/fixes.py | 16 +++++----- setup.py | 26 ++++++++++++--- tests/test_bytes.py | 11 ++++--- tests/test_encodings.py | 1 - tests/test_entities.py | 1 - 14 files changed, 50 insertions(+), 136 deletions(-) delete mode 100644 ftfy/compatibility.py diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 3764cf50..e7d43474 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -6,12 +6,10 @@ for more information. """ -from __future__ import unicode_literals import unicodedata import ftfy.bad_codecs from ftfy import fixes from ftfy.formatting import display_ljust -from ftfy.compatibility import is_printable __version__ = '4.4' @@ -399,7 +397,7 @@ def explain_unicode(text): U+253B ┻ [So] BOX DRAWINGS HEAVY UP AND HORIZONTAL """ for char in text: - if is_printable(char): + if char.isprintable(): display = char else: display = char.encode('unicode-escape').decode('ascii') diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py index 0984bd52..6eaab3cb 100644 --- a/ftfy/bad_codecs/__init__.py +++ b/ftfy/bad_codecs/__init__.py @@ -29,7 +29,6 @@ >>> print(b'\xed\xa0\xbd\xed\xb8\x8d'.decode('utf-8-variants')) 😍 """ -from __future__ import unicode_literals from encodings import normalize_encoding import codecs diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index ce5860a9..dcfec3f7 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -69,7 +69,6 @@ U+0081 \x81 [Cc] U+201A ‚ [Ps] SINGLE LOW-9 QUOTATION MARK """ -from __future__ import unicode_literals import codecs from encodings import normalize_encoding import sys @@ -87,8 +86,8 @@ def make_sloppy_codec(encoding): `codecs.charmap_decode` and `charmap_encode`. This function, given an encoding name, *defines* those boilerplate classes. """ - # Make an array of all 256 possible bytes. - all_bytes = bytearray(range(256)) + # Make a bytestring of all 256 possible bytes. + all_bytes = bytes(range(256)) # Get a list of what they would decode to in Latin-1. sloppy_chars = list(all_bytes.decode('latin-1')) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index cd89be69..fbfb7130 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -38,7 +38,6 @@ again, using UTF-8 as the codec every time. """ -from __future__ import unicode_literals import re import codecs from encodings.utf_8 import (IncrementalDecoder as UTF8IncrementalDecoder, @@ -190,11 +189,8 @@ def _buffer_decode_surrogates(sup, input, errors, final): if final: # We found 0xed near the end of the stream, and there aren't # six bytes to decode. Delegate to the superclass method to - # handle it as an error. - if PYTHON2 and len(input) >= 3: - # We can't trust Python 2 to raise an error when it's - # asked to decode a surrogate, so let's force the issue. - input = mangle_surrogates(input) + # handle it as normal UTF-8. It might be a Hangul character + # or an error. return sup(input, errors, final) else: # We found a surrogate, the stream isn't over yet, and we don't @@ -205,50 +201,21 @@ def _buffer_decode_surrogates(sup, input, errors, final): if CESU8_RE.match(input): # Given this is a CESU-8 sequence, do some math to pull out # the intended 20-bit value, and consume six bytes. - bytenums = bytes_to_ints(input[:6]) codepoint = ( - ((bytenums[1] & 0x0f) << 16) + - ((bytenums[2] & 0x3f) << 10) + - ((bytenums[4] & 0x0f) << 6) + - (bytenums[5] & 0x3f) + + ((input[1] & 0x0f) << 16) + + ((input[2] & 0x3f) << 10) + + ((input[4] & 0x0f) << 6) + + (input[5] & 0x3f) + 0x10000 ) - return unichr(codepoint), 6 + return chr(codepoint), 6 else: # This looked like a CESU-8 sequence, but it wasn't one. # 0xed indicates the start of a three-byte sequence, so give - # three bytes to the superclass to decode as usual -- except - # for working around the Python 2 discrepancy as before. - if PYTHON2: - input = mangle_surrogates(input) + # three bytes to the superclass to decode as usual. return sup(input[:3], errors, False) -def mangle_surrogates(bytestring): - """ - When Python 3 sees the UTF-8 encoding of a surrogate codepoint, it treats - it as an error (which it is). In 'replace' mode, it will decode as three - replacement characters. But Python 2 will just output the surrogate - codepoint. - - To ensure consistency between Python 2 and Python 3, and protect downstream - applications from malformed strings, we turn surrogate sequences at the - start of the string into the bytes `ff ff ff`, which we're *sure* won't - decode, and which turn into three replacement characters in 'replace' mode. - - This function does nothing in Python 3, and it will be deprecated in ftfy - 5.0. - """ - if PYTHON2: - if bytestring.startswith(b'\xed') and len(bytestring) >= 3: - decoded = bytestring[:3].decode('utf-8', 'replace') - if '\ud800' <= decoded <= '\udfff': - return b'\xff\xff\xff' + mangle_surrogates(bytestring[3:]) - return bytestring - else: - # On Python 3, nothing needs to be done. - return bytestring - # The encoder is identical to UTF-8. IncrementalEncoder = UTF8IncrementalEncoder diff --git a/ftfy/badness.py b/ftfy/badness.py index b00d4e88..f2922507 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -4,7 +4,6 @@ more reasonable. """ -from __future__ import unicode_literals import re import unicodedata from ftfy.chardata import chars_to_classes diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 8269d2ee..ee28b58c 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -11,12 +11,9 @@ The file will be written to the current directory. """ -from __future__ import unicode_literals import unicodedata import sys import zlib -if sys.hexversion >= 0x03000000: - unichr = chr # L = Latin capital letter # l = Latin lowercase letter @@ -56,7 +53,7 @@ def make_char_data_file(do_it_anyway=False): cclasses = [None] * 0x110000 for codepoint in range(0x0, 0x110000): - char = unichr(codepoint) + char = chr(codepoint) category = unicodedata.category(char) if (0x250 <= codepoint < 0x300) and char != 'ə': diff --git a/ftfy/chardata.py b/ftfy/chardata.py index a6bf1514..6384a691 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -4,13 +4,11 @@ encodings that use them. """ -from __future__ import unicode_literals import re import zlib import unicodedata import itertools from pkg_resources import resource_string -from ftfy.compatibility import unichr # These are the encodings we will try to fix in ftfy, in the # order that they should be tried. @@ -39,7 +37,7 @@ def _build_regexes(): # Make a sequence of characters that bytes \x80 to \xFF decode to # in each encoding, as well as byte \x1A, which is used to represent # the replacement character � in the sloppy-* encodings. - latin1table = ''.join(unichr(i) for i in range(128, 256)) + '\x1a' + latin1table = ''.join(chr(i) for i in range(128, 256)) + '\x1a' charlist = latin1table.encode('latin-1').decode(encoding) # The rest of the ASCII bytes -- bytes \x00 to \x19 and \x1B diff --git a/ftfy/cli.py b/ftfy/cli.py index 802a46c8..bcd69ca0 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -1,12 +1,10 @@ """ A command-line utility for fixing text found in a file. """ - import sys import io import codecs from ftfy import fix_file, __version__ -from ftfy.compatibility import PYTHON2 ENCODE_ERROR_TEXT_UNIX = """ftfy error: @@ -75,10 +73,7 @@ def main(): if args.filename == '-': # Get a standard input stream made of bytes, so we can decode it as # whatever encoding is necessary. - if PYTHON2: - file = sys.stdin - else: - file = sys.stdin.buffer + file = sys.stdin.buffer else: file = open(args.filename, 'rb') diff --git a/ftfy/compatibility.py b/ftfy/compatibility.py deleted file mode 100644 index ad5c1097..00000000 --- a/ftfy/compatibility.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Makes some function names and behavior consistent between Python 2 and -Python 3, and also between narrow and wide builds. -""" -from __future__ import unicode_literals -import sys -import unicodedata - -if sys.hexversion >= 0x03000000: - unichr = chr - xrange = range - PYTHON2 = False -else: - unichr = unichr - xrange = xrange - PYTHON2 = True - -PYTHON34_OR_LATER = (sys.hexversion >= 0x03040000) - - -def _narrow_unichr_workaround(codepoint): - """ - A replacement for unichr() on narrow builds of Python. This will get - us the narrow representation of an astral character, which will be - a string of length two, containing two UTF-16 surrogates. - """ - escaped = b'\\U%08x' % codepoint - return escaped.decode('unicode-escape') - - -if sys.maxunicode < 0x10000: - unichr = _narrow_unichr_workaround - - -def bytes_to_ints(bytestring): - """ - No matter what version of Python this is, make a sequence of integers from - a bytestring. On Python 3, this is easy, because a 'bytes' object _is_ a - sequence of integers. - """ - if PYTHON2: - return [ord(b) for b in bytestring] - else: - return bytestring - - -def is_printable(char): - """ - str.isprintable() is new in Python 3. It's useful in `explain_unicode`, so - let's make a crude approximation in Python 2. - """ - if PYTHON2: - return not unicodedata.category(char).startswith('C') - else: - return char.isprintable() diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 89e3d2c6..122c469c 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -4,7 +4,6 @@ can perform. """ -from __future__ import unicode_literals import re import sys import codecs @@ -35,8 +34,8 @@ If you're confused by this, please read the Python Unicode HOWTO: - http://docs.python.org/%d/howto/unicode.html -""" % sys.version_info[0] + http://docs.python.org/3/howto/unicode.html +""" def fix_encoding(text): @@ -295,7 +294,6 @@ def apply_plan(text, plan): HTML_ENTITY_RE = re.compile(r"&#?\w{0,8};") - def unescape_html(text): """ Decode all three types of HTML entities/character references. @@ -317,9 +315,9 @@ def fixup(match): # character reference try: if text[:3] == "&#x": - return unichr(int(text[3:-1], 16)) + return chr(int(text[3:-1], 16)) else: - return unichr(int(text[2:-1])) + return chr(int(text[2:-1])) except ValueError: pass else: @@ -519,11 +517,13 @@ def remove_bom(text): Remove a byte-order mark that was accidentally decoded as if it were part of the text. - >>> print(remove_bom("\ufeffWhere do you want to go today?")) + >>> print(remove_bom(chr(0xfeff) + "Where do you want to go today?")) Where do you want to go today? """ - return text.lstrip(unichr(0xfeff)) + return text.lstrip(chr(0xfeff)) + +UNSAFE_PRIVATE_USE_RE = re.compile('[\U00100000-\U0010ffff]') # Define a regex to match valid escape sequences in Python string literals. ESCAPE_SEQUENCE_RE = re.compile(r''' diff --git a/setup.py b/setup.py index 947676a8..b6845795 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,30 @@ from setuptools import setup import sys + +PY2_MESSAGE = """ +Sorry, this version of ftfy is no longer written for Python 2. + +It is much easier to provide consistent handling of Unicode when developing +only for Python 3. ftfy is exactly the kind of library that Python 3 is +designed for, and now there is enough Python 3 usage that we can take advantage +of it and write better, simpler code. + +The older version of ftfy, version 4.4, is still available and can run on +Python 2. Try this: + + pip install ftfy==4.4 +""" + + +if sys.version_info[0] < 3: + print(PY2_MESSAGE) + sys.exit(1) + + setup( name="ftfy", - version='4.4', + version='5.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", @@ -18,9 +39,6 @@ # library, as html.unescape. install_requires=['html5lib', 'wcwidth'], classifiers=[ - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", diff --git a/tests/test_bytes.py b/tests/test_bytes.py index e283869f..3ab9eec5 100644 --- a/tests/test_bytes.py +++ b/tests/test_bytes.py @@ -1,13 +1,9 @@ # coding: utf-8 -from __future__ import unicode_literals from ftfy import guess_bytes -from ftfy.bad_codecs.utf8_variants import mangle_surrogates, IncrementalDecoder +from ftfy.bad_codecs.utf8_variants import IncrementalDecoder from nose.tools import eq_ -import sys -PYTHON2 = sys.hexversion < 0x03000000 - TEST_ENCODINGS = [ 'utf-16', 'utf-8', 'sloppy-windows-1252' ] @@ -36,6 +32,7 @@ def test_guess_bytes(): bowdlerized_null = b'null\xc0\x80separated' result_str, result_encoding = guess_bytes(bowdlerized_null) +<<<<<<< HEAD eq_(result_str, u'null\x00separated') eq_(result_encoding, u'utf-8-variants') @@ -77,4 +74,8 @@ def test_incomplete_sequences(): got = decoder.decode(left, final=False) got += decoder.decode(right) eq_(got, test_string) +======= + eq_(result_str, 'null\x00separated') + eq_(result_encoding, 'utf-8-variants') +>>>>>>> 0485ed5... Simplify code by removing compatibility with Python <= 3.3. diff --git a/tests/test_encodings.py b/tests/test_encodings.py index f59e08aa..fde38631 100644 --- a/tests/test_encodings.py +++ b/tests/test_encodings.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from ftfy import bad_codecs, guess_bytes from nose.tools import eq_ diff --git a/tests/test_entities.py b/tests/test_entities.py index 24c4d446..f51b3818 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals from ftfy import fix_text, fix_text_segment from nose.tools import eq_ From d0bccfd5bb2ea82955d8274e06c13bd86bbcfe06 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 14:39:12 -0500 Subject: [PATCH 323/502] fix some stuff that was incorrectly merged --- ftfy/bad_codecs/utf8_variants.py | 1 - ftfy/chardata.py | 4 ++-- ftfy/cli.py | 21 ++++++++------------- ftfy/fixes.py | 7 +++---- tests/test_bytes.py | 28 ---------------------------- 5 files changed, 13 insertions(+), 48 deletions(-) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index fbfb7130..597f777f 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -42,7 +42,6 @@ import codecs from encodings.utf_8 import (IncrementalDecoder as UTF8IncrementalDecoder, IncrementalEncoder as UTF8IncrementalEncoder) -from ftfy.compatibility import bytes_to_ints, unichr, PYTHON2 NAME = 'utf-8-variants' diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 6384a691..53795ac4 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -69,7 +69,7 @@ def _build_utf8_punct_regex(): # Unrelatedly, the expression that generates these bytes will be so much # prettier when we deprecate Python 2. continuation_char_list = ''.join( - unichr(i) for i in range(0x80, 0xc0) + chr(i) for i in range(0x80, 0xc0) ).encode('latin-1') obvious_utf8 = ('â€[' + continuation_char_list.decode('sloppy-windows-1252') @@ -205,7 +205,7 @@ def _build_width_map(): # with that in the dictionary. width_map = {0x3000: ' '} for i in range(0xff01, 0xfff0): - char = unichr(i) + char = chr(i) alternate = unicodedata.normalize('NFKC', char) if alternate != char: width_map[i] = alternate diff --git a/ftfy/cli.py b/ftfy/cli.py index bcd69ca0..1b22a2d8 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -78,10 +78,8 @@ def main(): file = open(args.filename, 'rb') if args.output == '-': - encode_output = PYTHON2 outfile = sys.stdout else: - encode_output = False outfile = io.open(args.output, 'w', encoding='utf-8') normalization = args.normalization @@ -97,17 +95,14 @@ def main(): for line in fix_file(file, encoding=encoding, fix_entities=fix_entities, normalization=normalization): - if encode_output: - outfile.write(line.encode('utf-8')) - else: - try: - outfile.write(line) - except UnicodeEncodeError: - if sys.platform == 'win32': - sys.stderr.write(ENCODE_ERROR_TEXT_WINDOWS) - else: - sys.stderr.write(ENCODE_ERROR_TEXT_UNIX) - sys.exit(1) + try: + outfile.write(line) + except UnicodeEncodeError: + if sys.platform == 'win32': + sys.stderr.write(ENCODE_ERROR_TEXT_WINDOWS) + else: + sys.stderr.write(ENCODE_ERROR_TEXT_UNIX) + sys.exit(1) except UnicodeDecodeError as err: sys.stderr.write(DECODE_ERROR_TEXT % (encoding, err)) sys.exit(1) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 122c469c..dedfbec1 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -13,7 +13,6 @@ PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE, LOSSY_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE) from ftfy.badness import text_cost -from ftfy.compatibility import unichr from html5lib.constants import entities @@ -455,7 +454,7 @@ def convert_surrogate_pair(match): """ pair = match.group(0) codept = 0x10000 + (ord(pair[0]) - 0xd800) * 0x400 + (ord(pair[1]) - 0xdc00) - return unichr(codept) + return chr(codept) def fix_surrogates(text): @@ -463,8 +462,8 @@ def fix_surrogates(text): Replace 16-bit surrogate codepoints with the characters they represent (when properly paired), or with \ufffd otherwise. - >>> high_surrogate = unichr(0xd83d) - >>> low_surrogate = unichr(0xdca9) + >>> high_surrogate = chr(0xd83d) + >>> low_surrogate = chr(0xdca9) >>> print(fix_surrogates(high_surrogate + low_surrogate)) 💩 >>> print(fix_surrogates(low_surrogate + high_surrogate)) diff --git a/tests/test_bytes.py b/tests/test_bytes.py index 3ab9eec5..e552ca41 100644 --- a/tests/test_bytes.py +++ b/tests/test_bytes.py @@ -32,34 +32,10 @@ def test_guess_bytes(): bowdlerized_null = b'null\xc0\x80separated' result_str, result_encoding = guess_bytes(bowdlerized_null) -<<<<<<< HEAD eq_(result_str, u'null\x00separated') eq_(result_encoding, u'utf-8-variants') -def test_mangle_surrogates(): - eq_(b'Eric the half a bee \xed\xa0\x80'.decode('utf-8-variants', 'replace'), - 'Eric the half a bee ���') - - if PYTHON2: - # These are the encodings of a surrogate character, plus a similar-looking - # Korean character. Only the surrogate character's bytes should get mangled. - eq_(mangle_surrogates(b'\xed\xa0\x80\xed\x9e\x99'), b'\xff\xff\xff\xed\x9e\x99') - - # Mangle sequences of surrogates, but don't mangle surrogates later in - # the string (there's no need to in our decoders). - eq_(mangle_surrogates(b'\xed\xa0\xbd\xed\xb8\xb9test\xed\xb4\xb4'), - b'\xff\xff\xff\xff\xff\xfftest\xed\xb4\xb4') - eq_(mangle_surrogates(b'test\xed\xb4\xb4'), b'test\xed\xb4\xb4') - - # Handle short bytestrings correctly. - eq_(mangle_surrogates(b'\xed'), b'\xed') - eq_(mangle_surrogates(b''), b'') - else: - # Make sure mangle_surrogates doesn't do anything - eq_(mangle_surrogates(b'\xed\xa0\x80\xed\x9e\x99'), b'\xed\xa0\x80\xed\x9e\x99') - - def test_incomplete_sequences(): test_bytes = b'surrogates: \xed\xa0\x80\xed\xb0\x80 / null: \xc0\x80' test_string = 'surrogates: \U00010000 / null: \x00' @@ -74,8 +50,4 @@ def test_incomplete_sequences(): got = decoder.decode(left, final=False) got += decoder.decode(right) eq_(got, test_string) -======= - eq_(result_str, 'null\x00separated') - eq_(result_encoding, 'utf-8-variants') ->>>>>>> 0485ed5... Simplify code by removing compatibility with Python <= 3.3. From ab7eb68d09cb77350f5734fd1f5350fbaa4b3e4a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 14:46:12 -0500 Subject: [PATCH 324/502] Remove some compatibility crutches --- ftfy/__init__.py | 1 - ftfy/bad_codecs/__init__.py | 1 - ftfy/bad_codecs/sloppy.py | 1 - ftfy/badness.py | 1 - ftfy/chardata.py | 1 - ftfy/fixes.py | 1 - ftfy/formatting.py | 2 -- ftfy/streamtester/__init__.py | 1 - ftfy/streamtester/oauth.py | 2 -- ftfy/streamtester/twitter_tester.py | 4 ---- tests/test_bytes.py | 1 - tests/test_characters.py | 7 +------ tests/test_encodings.py | 1 - tests/test_real_text.py | 2 -- tests/test_synthetic.py | 5 ----- 15 files changed, 1 insertion(+), 30 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index e7d43474..1ca03af9 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ ftfy: fixes text for you diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py index 6eaab3cb..473c890a 100644 --- a/ftfy/bad_codecs/__init__.py +++ b/ftfy/bad_codecs/__init__.py @@ -1,4 +1,3 @@ -# coding: utf-8 r""" Give Python the ability to decode some common, flawed encodings. diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index dcfec3f7..0d3bc1b2 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -1,4 +1,3 @@ -# coding: utf-8 r""" Decodes single-byte encodings, filling their "holes" in the same messy way that everyone else does. diff --git a/ftfy/badness.py b/ftfy/badness.py index f2922507..bddabdcd 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Heuristics to determine whether re-encoding text is actually making it more reasonable. diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 53795ac4..478cf56a 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ This gives other modules access to the gritty details about characters and the encodings that use them. diff --git a/ftfy/fixes.py b/ftfy/fixes.py index dedfbec1..f6b829df 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ This module contains the individual fixes that the main fix_text function can perform. diff --git a/ftfy/formatting.py b/ftfy/formatting.py index 793cbb28..ea19f0c5 100644 --- a/ftfy/formatting.py +++ b/ftfy/formatting.py @@ -1,4 +1,3 @@ -# coding: utf-8 """ This module provides functions for justifying Unicode text in a monospaced display such as a terminal. @@ -6,7 +5,6 @@ We used to have our own implementation here, but now we mostly rely on the 'wcwidth' library. """ -from __future__ import unicode_literals, division from unicodedata import normalize from wcwidth import wcwidth, wcswidth diff --git a/ftfy/streamtester/__init__.py b/ftfy/streamtester/__init__.py index e097bf6b..9e0a06b2 100644 --- a/ftfy/streamtester/__init__.py +++ b/ftfy/streamtester/__init__.py @@ -2,7 +2,6 @@ This file defines a general method for evaluating ftfy using data that arrives in a stream. A concrete implementation of it is found in `twitter_tester.py`. """ -from __future__ import print_function, unicode_literals from ftfy import fix_text from ftfy.fixes import fix_encoding, unescape_html from ftfy.chardata import possible_encoding diff --git a/ftfy/streamtester/oauth.py b/ftfy/streamtester/oauth.py index a948459c..9eef8f1c 100644 --- a/ftfy/streamtester/oauth.py +++ b/ftfy/streamtester/oauth.py @@ -1,9 +1,7 @@ -# coding: utf-8 """ Do what is necessary to authenticate this tester as a Twitter "app", using somebody's Twitter account. """ -from __future__ import unicode_literals import os diff --git a/ftfy/streamtester/twitter_tester.py b/ftfy/streamtester/twitter_tester.py index 561bcf20..dab62e73 100644 --- a/ftfy/streamtester/twitter_tester.py +++ b/ftfy/streamtester/twitter_tester.py @@ -1,11 +1,7 @@ """ Implements a StreamTester that runs over Twitter data. See the class docstring. - -This module is written for Python 3 only. The __future__ imports you see here -are just to let Python 2 scan the file without crashing with a SyntaxError. """ -from __future__ import print_function, unicode_literals import os from collections import defaultdict from ftfy.streamtester import StreamTester diff --git a/tests/test_bytes.py b/tests/test_bytes.py index e552ca41..fd5649c7 100644 --- a/tests/test_bytes.py +++ b/tests/test_bytes.py @@ -1,4 +1,3 @@ -# coding: utf-8 from ftfy import guess_bytes from ftfy.bad_codecs.utf8_variants import IncrementalDecoder from nose.tools import eq_ diff --git a/tests/test_characters.py b/tests/test_characters.py index 5700323b..592134dc 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals from ftfy.fixes import ( fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, remove_control_chars, fix_surrogates @@ -9,14 +7,11 @@ import sys from nose.tools import eq_ -if sys.hexversion >= 0x03000000: - unichr = chr - # Most single-character strings which have been misencoded should be restored. def test_bmp_characters(): for index in range(0xa0, 0xfffd): - char = unichr(index) + char = chr(index) # Exclude code points that are not assigned if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn', 'Sk'): garble = char.encode('utf-8').decode('latin-1') diff --git a/tests/test_encodings.py b/tests/test_encodings.py index fde38631..ac24591d 100644 --- a/tests/test_encodings.py +++ b/tests/test_encodings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from ftfy import bad_codecs, guess_bytes from nose.tools import eq_ diff --git a/tests/test_real_text.py b/tests/test_real_text.py index f8275359..5a027882 100644 --- a/tests/test_real_text.py +++ b/tests/test_real_text.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals from ftfy import fix_text from ftfy.fixes import fix_encoding_and_explain, apply_plan from nose.tools import eq_ diff --git a/tests/test_synthetic.py b/tests/test_synthetic.py index 58ff2ca3..7ff4e274 100644 --- a/tests/test_synthetic.py +++ b/tests/test_synthetic.py @@ -1,13 +1,8 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals from ftfy.fixes import fix_encoding from ftfy.badness import sequence_weirdness import sys from nose.tools import eq_ -if sys.hexversion >= 0x03000000: - unichr = chr - phrases = [ "\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", From 39df63a34fe0d60b228f6906d62dc5f116948297 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 15:04:02 -0500 Subject: [PATCH 325/502] Use Python 3 semantics for iterating bytes --- ftfy/chardata.py | 10 ++++------ ftfy/fixes.py | 6 +++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 478cf56a..ad57113f 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -36,8 +36,8 @@ def _build_regexes(): # Make a sequence of characters that bytes \x80 to \xFF decode to # in each encoding, as well as byte \x1A, which is used to represent # the replacement character � in the sloppy-* encodings. - latin1table = ''.join(chr(i) for i in range(128, 256)) + '\x1a' - charlist = latin1table.encode('latin-1').decode(encoding) + byte_range = bytes(list(range(0x80, 0x100)) + [0x1a]) + charlist = byte_range.decode(encoding) # The rest of the ASCII bytes -- bytes \x00 to \x19 and \x1B # to \x7F -- will decode as those ASCII characters in any encoding we @@ -67,11 +67,9 @@ def _build_utf8_punct_regex(): # # Unrelatedly, the expression that generates these bytes will be so much # prettier when we deprecate Python 2. - continuation_char_list = ''.join( - chr(i) for i in range(0x80, 0xc0) - ).encode('latin-1') + continuation_bytes = bytes(range(0x80, 0xc0)) obvious_utf8 = ('â€[' - + continuation_char_list.decode('sloppy-windows-1252') + + continuation_bytes.decode('sloppy-windows-1252') + ']') return re.compile(obvious_utf8) PARTIAL_UTF8_PUNCT_RE = _build_utf8_punct_regex() diff --git a/ftfy/fixes.py b/ftfy/fixes.py index f6b829df..fec25ab0 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -195,16 +195,16 @@ def fix_one_step_and_explain(text): # except they have b' ' where b'\xa0' would belong. if ALTERED_UTF8_RE.search(encoded_bytes): encoded_bytes = restore_byte_a0(encoded_bytes) - cost = encoded_bytes.count(b'\xa0') * 2 + cost = encoded_bytes.count(0xa0) * 2 transcode_steps.append(('transcode', 'restore_byte_a0', cost)) # Check for the byte 0x1a, which indicates where one of our # 'sloppy' codecs found a replacement character. - if encoding.startswith('sloppy') and b'\x1a' in encoded_bytes: + if encoding.startswith('sloppy') and 0x1a in encoded_bytes: encoded_bytes = replace_lossy_sequences(encoded_bytes) transcode_steps.append(('transcode', 'replace_lossy_sequences', 0)) - if b'\xed' in encoded_bytes or b'\xc0' in encoded_bytes: + if 0xed in encoded_bytes or 0xc0 in encoded_bytes: decoding = 'utf-8-variants' decode_step = ('decode', decoding, 0) From 7cafc18e13c8e88e090fe5af1d9a1ae492aea541 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 15:05:32 -0500 Subject: [PATCH 326/502] Use keyword-only arguments where appropriate --- ftfy/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 1ca03af9..42013315 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -18,6 +18,7 @@ def fix_text(text, + *, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, @@ -192,6 +193,7 @@ def fix_text(text, def fix_file(input_file, encoding=None, + *, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, @@ -239,6 +241,7 @@ def fix_file(input_file, def fix_text_segment(text, + *, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, From d7739578f57b7be1bc173fb8e00e22252e436a1a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 15:06:23 -0500 Subject: [PATCH 327/502] increment version to 5.0 --- ftfy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 42013315..903e57ed 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -10,7 +10,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '4.4' +__version__ = '5.0' # See the docstring for ftfy.bad_codecs to see what we're doing here. From 24ecc1406fe9d0592238fe58722da645391923c6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 15 Feb 2017 19:51:35 -0500 Subject: [PATCH 328/502] Add tests for the CLI --- tests/halibote.txt | 1 + tests/test_cli.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tests/halibote.txt create mode 100644 tests/test_cli.py diff --git a/tests/halibote.txt b/tests/halibote.txt new file mode 100644 index 00000000..0be33306 --- /dev/null +++ b/tests/halibote.txt @@ -0,0 +1 @@ +ã€æ›´æ–°ã€‘ã€Šå“ˆåˆ©æ³¢ç‰¹ã€‹çŸ³å §åœæ‰æ–°å©šå¨¶åˆæˆ€ä»Šç— é€ diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..6f686760 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,56 @@ +import subprocess +import os +from nose.tools import eq_ + + +# Get the filename of 'halibote.txt', which contains some mojibake about +# Harry Potter in Chinese +THIS_DIR = os.path.dirname(__file__) +TEST_FILENAME = os.path.join(THIS_DIR, 'halibote.txt') +CORRECT_OUTPUT = '【更新】《哈利波特》石堧卜才新婚娶初戀今痠逝\n' +FAILED_OUTPUT = '''ftfy error: +This input couldn't be decoded as 'windows-1252'. We got the following error: + + 'charmap' codec can't decode byte 0x90 in position 5: character maps to + +ftfy works best when its input is in a known encoding. You can use `ftfy -g` +to guess, if you're desperate. Otherwise, give the encoding name with the +`-e` option, such as `ftfy -e latin-1`. +''' + + +def get_command_output(args, stdin=None): + return subprocess.check_output(args, stdin=stdin, stderr=subprocess.STDOUT, timeout=5).decode('utf-8') + + +def test_basic(): + output = get_command_output(['ftfy', TEST_FILENAME]) + eq_(output, CORRECT_OUTPUT) + + +def test_guess_bytes(): + output = get_command_output(['ftfy', '-g', TEST_FILENAME]) + eq_(output, CORRECT_OUTPUT) + + +def test_alternate_encoding(): + # The file isn't really in Windows-1252. But that's a problem ftfy + # can fix, if it's allowed to be sloppy when reading the file. + output = get_command_output(['ftfy', '-e', 'sloppy-windows-1252', TEST_FILENAME]) + eq_(output, CORRECT_OUTPUT) + + +def test_wrong_encoding(): + # It's more of a problem when the file doesn't actually decode. + try: + get_command_output(['ftfy', '-e', 'windows-1252', TEST_FILENAME]) + assert False, "Should have raised a CalledProcessError" + except subprocess.CalledProcessError as e: + eq_(e.output.decode('utf-8'), FAILED_OUTPUT) + + +def test_stdin(): + with open(TEST_FILENAME, 'rb') as infile: + output = get_command_output(['ftfy'], stdin=infile) + eq_(output, CORRECT_OUTPUT) + From b8a9f49b4a90e951c1e11d97381b3cd612198cfe Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 15:21:05 -0500 Subject: [PATCH 329/502] update changelog for 5.0 --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 525f542a..33956f50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## Version 5.0 (February 17, 2017) + +Breaking changes: + +- Dropped support for Python 2. If you need Python 2 support, you should get + version 4.4, which has the same features as this version. + +- The top-level functions require their arguments to be given as keyword + arguments. + +Version 5.0 also now has tests for the command-line invocation of ftfy. + + ## Version 4.4.0 (February 17, 2017) Heuristic changes: From dc29c8d6874b928b49f8e8bee00ae61645faa11c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 16:56:32 -0500 Subject: [PATCH 330/502] flake8 fixes --- ftfy/__init__.py | 2 +- ftfy/bad_codecs/sloppy.py | 1 + ftfy/badness.py | 1 + ftfy/chardata.py | 17 ++++++++++------- ftfy/cli.py | 1 - ftfy/fixes.py | 3 ++- ftfy/streamtester/oauth.py | 1 + 7 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 903e57ed..e83c2f9b 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -330,7 +330,7 @@ def guess_bytes(bstring): - "sloppy-windows-1252", the Latin-1-like encoding that is the most common single-byte encoding """ - if type(bstring) == type(''): + if isinstance(bstring, str): raise UnicodeError( "This string was already decoded as Unicode. You should pass " "bytes to guess_bytes, not Unicode." diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index 0d3bc1b2..aa2cfaac 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -75,6 +75,7 @@ REPLACEMENT_CHAR = '\ufffd' PY26 = sys.version_info[:2] == (2, 6) + def make_sloppy_codec(encoding): """ Take a codec name, and return a 'sloppy' version of that codec that can diff --git a/ftfy/badness.py b/ftfy/badness.py index bddabdcd..4fcadf32 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -116,6 +116,7 @@ def _make_weirdness_regex(): ']' ) + def sequence_weirdness(text): """ Determine how often a text has unexpected characters or sequences of diff --git a/ftfy/chardata.py b/ftfy/chardata.py index ad57113f..0c851655 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -140,6 +140,7 @@ def possible_encoding(text, encoding): resource_string(__name__, 'char_classes.dat') ).decode('ascii') + def chars_to_classes(string): """ Convert each Unicode character to a letter indicating which of many @@ -159,13 +160,15 @@ def _build_control_char_mapping(): control_chars = {} for i in itertools.chain( - range(0x00, 0x09), [0x0b], - range(0x0e, 0x20), [0x7f], - range(0x206a, 0x2070), - [0xfeff], - range(0xfff9, 0xfffd), - range(0x1d173, 0x1d17b), - range(0xe0000, 0xe0080) + range(0x00, 0x09), + [0x0b], + range(0x0e, 0x20), + [0x7f], + range(0x206a, 0x2070), + [0xfeff], + range(0xfff9, 0xfffd), + range(0x1d173, 0x1d17b), + range(0xe0000, 0xe0080) ): control_chars[i] = None diff --git a/ftfy/cli.py b/ftfy/cli.py index 1b22a2d8..39990285 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -3,7 +3,6 @@ """ import sys import io -import codecs from ftfy import fix_file, __version__ diff --git a/ftfy/fixes.py b/ftfy/fixes.py index fec25ab0..8522bdbd 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -4,7 +4,6 @@ """ import re -import sys import codecs import warnings from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS, @@ -292,6 +291,7 @@ def apply_plan(text, plan): HTML_ENTITY_RE = re.compile(r"&#?\w{0,8};") + def unescape_html(text): """ Decode all three types of HTML entities/character references. @@ -330,6 +330,7 @@ def fixup(match): ANSI_RE = re.compile('\033\\[((?:\\d|;)*)([a-zA-Z])') + def remove_terminal_escapes(text): r""" Strip out "ANSI" terminal escape sequences, such as those that produce diff --git a/ftfy/streamtester/oauth.py b/ftfy/streamtester/oauth.py index 9eef8f1c..2efbd152 100644 --- a/ftfy/streamtester/oauth.py +++ b/ftfy/streamtester/oauth.py @@ -7,6 +7,7 @@ AUTH_TOKEN_PATH = os.path.expanduser('~/.cache/oauth/twitter_ftfy.auth') + def get_auth(): """ Twitter has some bizarre requirements about how to authorize an "app" to From 15b5e650f2e7febf1e31451447a9718996b20aaf Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 17 Feb 2017 17:19:04 -0500 Subject: [PATCH 331/502] simplify how bytes are identified in guess_bytes --- ftfy/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index e83c2f9b..6ac8115a 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -340,10 +340,8 @@ def guess_bytes(bstring): return bstring.decode('utf-16'), 'utf-16' byteset = set(bytes(bstring)) - byte_ed, byte_c0, byte_CR, byte_LF = b'\xed\xc0\r\n' - try: - if byte_ed in byteset or byte_c0 in byteset: + if 0xed in byteset or 0xc0 in byteset: # Byte 0xed can be used to encode a range of codepoints that # are UTF-16 surrogates. UTF-8 does not use UTF-16 surrogates, # so when we see 0xed, it's very likely we're being asked to @@ -370,7 +368,8 @@ def guess_bytes(bstring): except UnicodeDecodeError: pass - if byte_CR in bstring and byte_LF not in bstring: + if 0x0d in byteset and 0x0a not in byteset: + # Files that contain CR and not LF are likely to be MacRoman. return bstring.decode('macroman'), 'macroman' else: return bstring.decode('sloppy-windows-1252'), 'sloppy-windows-1252' From a7dd0610af97d2ca9336e60cbf56e259f3ea7b0e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 23 Feb 2017 14:46:09 -0500 Subject: [PATCH 332/502] Code review fixes --- ftfy/__init__.py | 2 +- ftfy/fixes.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 6ac8115a..ada14d10 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -339,7 +339,7 @@ def guess_bytes(bstring): if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): return bstring.decode('utf-16'), 'utf-16' - byteset = set(bytes(bstring)) + byteset = set(bstring) try: if 0xed in byteset or 0xc0 in byteset: # Byte 0xed can be used to encode a range of codepoints that diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 8522bdbd..963794f7 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -522,8 +522,6 @@ def remove_bom(text): return text.lstrip(chr(0xfeff)) -UNSAFE_PRIVATE_USE_RE = re.compile('[\U00100000-\U0010ffff]') - # Define a regex to match valid escape sequences in Python string literals. ESCAPE_SEQUENCE_RE = re.compile(r''' ( \\U........ # 8-digit hex escapes From 8ea2c1380afe1e671781ef8d95af3485ddefe2fb Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 9 Mar 2017 16:59:47 -0500 Subject: [PATCH 333/502] chardata: Remove obsolete comment, simplify obvious_utf8 expression --- ftfy/chardata.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 0c851655..ccb615a5 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -64,12 +64,8 @@ def _build_utf8_punct_regex(): # a range. "Couldn't this have just said [\x80-\xbf]?", you might ask. # However, when we decode the regex as Windows-1252, the resulting # characters won't even be remotely contiguous. - # - # Unrelatedly, the expression that generates these bytes will be so much - # prettier when we deprecate Python 2. - continuation_bytes = bytes(range(0x80, 0xc0)) obvious_utf8 = ('â€[' - + continuation_bytes.decode('sloppy-windows-1252') + + bytes(range(0x80, 0xc0)).decode('sloppy-windows-1252') + ']') return re.compile(obvious_utf8) PARTIAL_UTF8_PUNCT_RE = _build_utf8_punct_regex() From d95a3b4b131fa30c3b2f578dca0b9eee5c9da8e8 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 9 Mar 2017 17:16:40 -0500 Subject: [PATCH 334/502] Simplify documentation to not discuss Python 2 vs. 3 Also, only write `ftfy` in monospace when referring to the CLI tool. --- docs/index.rst | 84 +++++++++++++------------------------------------- 1 file changed, 22 insertions(+), 62 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 416b4f6c..618d32cb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,8 +6,7 @@ ftfy: fixes text for you ======================== -`ftfy` fixes Unicode that's broken in various ways. It works in Python 2.7, -Python 3.2, or later. +**ftfy** fixes Unicode that's broken in various ways. The goal of ftfy is to **take in bad Unicode and output good Unicode**, for use in your Unicode-aware code. This is different from taking in non-Unicode and @@ -19,18 +18,15 @@ Of course you're better off if your input is decoded properly and has no glitches. But you often don't have any control over your input; it's someone else's mistake, but it's your problem now. -`ftfy` will do everything it can to fix the problem. +ftfy will do everything it can to fix the problem. .. note:: - Time is marching on. ftfy 4.x supports Python 2.7 and 3.x, but when - ftfy 5.0 is released, it will probably only support Python 3. + This documentation is for ftfy 5, which runs on Python 3 only, following + the plan to drop Python 2 support that was announced in ftfy 3.3. - If you're running on Python 2, ftfy 4.x will keep working for you. You - don't have to upgrade to 5.0. You can save yourself a headache by adding - `ftfy < 5` to your requirements, making sure you stay on version 4. - - See `Future versions of ftfy`_ for why this needs to happen. + If you're running on Python 2, ftfy 4.x will keep working for you. In that + case, you should add `ftfy < 5` to your requirements. Mojibake @@ -102,7 +98,7 @@ interacting with the erroneous decoding. The main function of ftfy, parts of NFKC are implemented as separate, limited fixes. -There are other interesting things that `ftfy` can do that aren't part of +There are other interesting things that ftfy can do that aren't part of the :func:`ftfy.fix_text` pipeline, such as: * :func:`ftfy.explain_unicode`: show you what's going on in a string, @@ -113,10 +109,10 @@ the :func:`ftfy.fix_text` pipeline, such as: Encodings ftfy can handle ------------------------- -`ftfy` can't fix all possible mix-ups. Its goal is to cover the most common +ftfy can't fix all possible mix-ups. Its goal is to cover the most common encoding mix-ups while keeping false positives to a very low rate. -`ftfy` can understand text that was decoded as any of these single-byte +ftfy can understand text that was decoded as any of these single-byte encodings: - Latin-1 (ISO-8859-1) @@ -146,7 +142,7 @@ Korean, such as ``shift-jis`` and ``gb18030``. See `issue #34 `_ for why this is so hard. -But remember that the input to `ftfy` is Unicode, so it handles actual +But remember that the input to ftfy is Unicode, so it handles actual CJK *text* just fine. It just can't discover that a CJK *encoding* introduced mojibake into the text. @@ -179,7 +175,7 @@ If the only fix you need is to detect and repair decoding errors (mojibake), the you should use :func:`ftfy.fix_encoding` directly. .. versionchanged:: 4.0 - The default normalization was changed from `'NFKC'` to `'NFC'`. The new options + The default normalization was changed from `'NFKC'` to `'NFC'`. The options *fix_latin_ligatures* and *fix_character_width* were added to implement some of the less lossy parts of NFKC normalization on top of NFC. @@ -194,30 +190,20 @@ you should use :func:`ftfy.fix_encoding` directly. .. autofunction:: ftfy.explain_unicode -Non-Unicode strings -------------------- - -When first using ftfy, you might be confused to find that you can't give it a -bytestring (the type of object called `str` in Python 2). - -ftfy fixes text. Treating bytestrings as text is exactly the kind of thing that -causes the Unicode problems that ftfy has to fix. So if you don't give it a -Unicode string, ftfy will point you to the `Python Unicode HOWTO`_. - -.. _`Python Unicode Howto`: http://docs.python.org/3/howto/unicode.html - -Reasonable ways that you might exchange data, such as JSON or XML, already have -perfectly good ways of expressing Unicode strings. Given a Unicode string, ftfy -can apply fixes that are very likely to work without false positives. - - A note on encoding detection ---------------------------- -If your input is a mess of unmarked bytes, you might want a tool that can just -statistically analyze those bytes and predict what encoding they're in. +:func:`ftfy.fix_text` expects its input to be a Python 3 `str` (a Unicode +string). If you pass in `bytes` instead, ftfy will point you to the `Python +Unicode HOWTO`_. + +.. _`Python Unicode HOWTO`: http://docs.python.org/3/howto/unicode.html -`ftfy` is not that tool. The :func:`ftfy.guess_bytes` function it contains will +Now, you may know that your input is a mess of bytes in an unknown encoding, +and you might want a tool that can just statistically analyze those bytes and +predict what encoding they're in. + +ftfy is not that tool. The :func:`ftfy.guess_bytes` function it contains will do this in very limited cases, but to support more encodings from around the world, something more is needed. @@ -249,7 +235,7 @@ Here's the usage documentation for the `ftfy` command:: [--preserve-entities] [filename] - ftfy (fixes text for you), version 4.0.0 + ftfy (fixes text for you), version 5.0 positional arguments: filename The file whose Unicode is to be fixed. Defaults to -, @@ -323,29 +309,3 @@ that ftfy's behavior is consistent across versions. :members: .. autofunction:: ftfy.build_data.make_char_data_file - - -Future versions of ftfy -======================= - -ftfy has full support for Python 2.7, even including a backport of Unicode 9 -character classes to Python 2. But given the sweeping changes to Unicode in -Python, it's getting inconvenient to add new features to ftfy that work the -same on both versions. - -ftfy 5.0, when it is released, will probably only support Python 3. - -If you want to see examples of why ftfy is particularly difficult to maintain -on two versions of Python (which is more like three versions because of Python -2's "wide" and "narrow" builds), take a look at functions such as -:func:`ftfy.bad_codecs.utf8_variants.mangle_surrogates` and -:func:`ftfy.compatibility._narrow_unichr_workaround`. - -This will happen soon, and we'll follow the plan that jQuery used years ago -when it dropped support for IE 6-8. We'll release the last Python 2 version and -the first Python-3-only version with the same feature set. ftfy 5.0 will -reduce the size and complexity of the code greatly, but ftfy 4.x will remain -there for those who need it. - -If you're running on Python 2, please make sure that `ftfy < 5` is in your -requirements list, not just `ftfy`. From 0ba0aa87086538266f281bcf44d17871728c465c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 10 Mar 2017 13:36:50 -0500 Subject: [PATCH 335/502] Fix regression involving the entity … in unescape_html --- CHANGELOG.md | 14 +++++++++++++- ftfy/__init__.py | 2 +- ftfy/fixes.py | 10 ++++++++-- setup.py | 4 ++-- tests/test_entities.py | 5 +++++ 5 files changed, 29 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33956f50..ea7b1ef6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## Version 5.0.1 and 4.4.1 (March 10, 2017) + +Bug fix: + +- The `unescape_html` fixer will decode entities between `€` and `Ÿ` + as what they would be in Windows-1252, even without the help of + `fix_encoding`. + + This better matches what Web browsers do, and fixes a regression that version + 4.4 introduced in an example that uses `…` as an ellipsis. + + ## Version 5.0 (February 17, 2017) Breaking changes: @@ -36,7 +48,7 @@ Heuristic changes: Bug fix: -' `remove_control_chars` was removing U+0D ('\r') prematurely. That's the +- `remove_control_chars` was removing U+0D ('\r') prematurely. That's the job of `fix_line_breaks`. diff --git a/ftfy/__init__.py b/ftfy/__init__.py index ada14d10..025f6e44 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -10,7 +10,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.0' +__version__ = '5.0.1' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 963794f7..2e0a6e37 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -313,9 +313,15 @@ def fixup(match): # character reference try: if text[:3] == "&#x": - return chr(int(text[3:-1], 16)) + codept = int(text[3:-1], 16) else: - return chr(int(text[2:-1])) + codept = int(text[2:-1]) + if 0x80 <= codept < 0xa0: + # Decode this range of characters as Windows-1252, as Web + # browsers do in practice. + return bytes([codept]).decode('sloppy-windows-1252') + else: + return chr(codept) except ValueError: pass else: diff --git a/setup.py b/setup.py index b6845795..b02d3294 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ The older version of ftfy, version 4.4, is still available and can run on Python 2. Try this: - pip install ftfy==4.4 + pip install ftfy==4.4.1 """ @@ -24,7 +24,7 @@ setup( name="ftfy", - version='5.0', + version='5.0.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", diff --git a/tests/test_entities.py b/tests/test_entities.py index f51b3818..3693ae11 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -1,4 +1,5 @@ from ftfy import fix_text, fix_text_segment +from ftfy.fixes import unescape_html from nose.tools import eq_ @@ -18,3 +19,7 @@ def test_entities(): eq_(fix_text_segment('<>'), '<>') eq_(fix_text_segment('jednocześnie'), 'jednocześnie') eq_(fix_text_segment('JEDNOCZEŚNIE'), 'JEDNOCZEŚNIE') + eq_(fix_text_segment('ellipsis…', normalization='NFKC'), 'ellipsis...') + eq_(fix_text_segment('ellipsis…', normalization='NFKC'), 'ellipsis...') + eq_(fix_text_segment('broken'), 'broken\x81') + eq_(unescape_html('euro €'), 'euro €') From 594cfb938aefd87f020746fd5ce1765278fa64bd Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 10 Mar 2017 13:59:55 -0500 Subject: [PATCH 336/502] Correct the state of Python 2.7 support in the README --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f8df1cbf..a2176b74 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,7 @@ look at [our careers page](http://www.luminoso.com/career.html). ## What it does -`ftfy` fixes Unicode that's broken in various ways. It works in Python 2.7, -Python 3.2, or later. +`ftfy` fixes Unicode that's broken in various ways. The goal of `ftfy` is to **take in bad Unicode and output good Unicode**, for use in your Unicode-aware code. This is different from taking in non-Unicode and @@ -113,10 +112,17 @@ LOUD NOISES ## Installing -ftfy is a package that can be installed using `pip`: +ftfy is a Python 3 package that can be installed using `pip`: pip install ftfy +(Or use `pip3 install ftfy` on systems where Python 2 and 3 are both globally +installed and `pip` refers to Python 2.) + +If you're on Python 2.7, you can install an older version: + + pip install 'ftfy<5' + You can also clone this Git repository and install it with `python setup.py install`. From f018eaa820e731c91a42133b9da5a1dae23fb170 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Mon, 20 Mar 2017 21:47:35 -0300 Subject: [PATCH 337/502] Add MANIFEST.in file. This includes the readme, changelog, license and tests files into the source distribution published on PyPI. --- MANIFEST.in | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..c90ff95a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +graft tests + +include *.md +include *.txt + +global-exclude __pycache__ *.py[cod] From b589770fad5ed8f9643a77ba35c432e35fe76aca Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 21 Mar 2017 15:39:05 -0400 Subject: [PATCH 338/502] Bump version to 5.0.2 --- CHANGELOG.md | 5 +++++ setup.py | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea7b1ef6..b9c6d25b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## Version 5.0.2 and 4.4.2 (March 21, 2017) + +Added a `MANIFEST.in` that puts files such as the license file and this +changelog inside the source distribution. + ## Version 5.0.1 and 4.4.1 (March 10, 2017) Bug fix: diff --git a/setup.py b/setup.py index b02d3294..2b57e2a5 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ The older version of ftfy, version 4.4, is still available and can run on Python 2. Try this: - pip install ftfy==4.4.1 + pip install ftfy==4.4.2 """ @@ -24,7 +24,7 @@ setup( name="ftfy", - version='5.0.1', + version='5.0.2', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", @@ -44,6 +44,7 @@ "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", From 8a907103527e6078e2456a5220cfa641af2ffc13 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 24 Mar 2017 11:26:08 -0400 Subject: [PATCH 339/502] accuracy.rst was out of date, update it a bit --- docs/accuracy.rst | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/accuracy.rst b/docs/accuracy.rst index 6d4c9f1c..f4d1714e 100644 --- a/docs/accuracy.rst +++ b/docs/accuracy.rst @@ -6,9 +6,9 @@ be made by someone's Twitter client. We check what ftfy's :func:`fix_encoding` heuristic does to this data, and we aim to have the rate of false positives be indistinguishable from zero. -A pre-release version of ftfy was evaluated on 30,880,000 tweets received from -Twitter's streaming API in April 2015. There was 1 false positive, and it was -due to a bug that has now been fixed. +A pre-release version of ftfy 4.0 was evaluated on 30,880,000 tweets received +from Twitter's streaming API in April 2015. There was 1 false positive, and it +was due to a bug that has now been fixed. When looking at the changes ftfy makes, we found: @@ -27,12 +27,8 @@ evaluation, and found: - None of those 1000 changed correct text to incorrect text (these would be false positives). -In all the data we've sampled, including from previous versions of ftfy, there -are only three false positives that remain that we know of:: +There is one string of real-world text that we have encountered that remains a +false positive in the current version: fix_encoding('├┤a┼┐a┼┐a┼┐a┼┐a') == 'ôaſaſaſaſa' - fix_encoding('ESSE CARA AI QUEM É¿') == 'ESSE CARA AI QUEM ɿ' - fix_encoding('``hogwarts nao existe, voce nao vai pegar o trem pra lá´´') - == '``hogwarts nao existe, voce nao vai pegar o trem pra lᴴ' - From c9dc6fedbe762b1bff22b9face2e1a2dc959d858 Mon Sep 17 00:00:00 2001 From: Andrew Lin Date: Fri, 24 Mar 2017 11:36:32 -0400 Subject: [PATCH 340/502] Fix __version__. --- ftfy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 025f6e44..24791f8c 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -10,7 +10,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.0.1' +__version__ = '5.0.2' # See the docstring for ftfy.bad_codecs to see what we're doing here. From bb3f3493a012d75025240dbf2fa40d6f4b6c3316 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Mar 2017 17:26:06 -0400 Subject: [PATCH 341/502] Use Python 3's dict of HTML entities instead of html5lib --- ftfy/__init__.py | 2 +- ftfy/fixes.py | 10 +++++----- setup.py | 9 ++------- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 24791f8c..960e74dc 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -10,7 +10,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.0.2' +__version__ = '5.1' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 2e0a6e37..620fa6b4 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -11,7 +11,7 @@ PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE, LOSSY_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE) from ftfy.badness import text_cost -from html5lib.constants import entities +from html import entities BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. @@ -325,12 +325,12 @@ def fixup(match): except ValueError: pass else: - # named entity + # This is a named entity; if it's a known HTML5 entity, replace + # it with the appropriate character. try: - text = entities[text[1:]] + return entities.html5[text[1:]] except KeyError: - pass - return text # leave as is + return text return HTML_ENTITY_RE.sub(fixup, text) diff --git a/setup.py b/setup.py index 2b57e2a5..f38cbe91 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( name="ftfy", - version='5.0.2', + version='5.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", @@ -33,14 +33,9 @@ description="Fixes some problems with Unicode text after the fact", packages=['ftfy', 'ftfy.bad_codecs'], package_data={'ftfy': ['char_classes.dat']}, - - # We could drop the html5lib dependency if we drop support for Python <= - # 3.4, because the feature we need from html5lib is now in the standard - # library, as html.unescape. - install_requires=['html5lib', 'wcwidth'], + install_requires=['wcwidth'], classifiers=[ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", From 968c0ba3504bfc87bffca8ead5bdb8a8eeca39d3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Mar 2017 17:50:59 -0400 Subject: [PATCH 342/502] Clean up and fix the inner function used by `unescape_html` --- ftfy/fixes.py | 60 ++++++++++++++++++++++-------------------- tests/test_entities.py | 1 + 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 620fa6b4..8f081e6b 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -292,6 +292,36 @@ def apply_plan(text, plan): HTML_ENTITY_RE = re.compile(r"&#?\w{0,8};") +def _unescape_fixup(match): + """ + Replace one matched HTML entity with the character it represents, + if possible. + """ + text = match.group(0) + if text[:2] == "&#": + # character reference + try: + if text[:3] == "&#x": + codept = int(text[3:-1], 16) + else: + codept = int(text[2:-1]) + if 0x80 <= codept < 0xa0: + # Decode this range of characters as Windows-1252, as Web + # browsers do in practice. + return bytes([codept]).decode('sloppy-windows-1252') + else: + return chr(codept) + except ValueError: + return text + else: + # This is a named entity; if it's a known HTML5 entity, replace + # it with the appropriate character. + try: + return entities.html5[text[1:]] + except KeyError: + return text + + def unescape_html(text): """ Decode all three types of HTML entities/character references. @@ -303,35 +333,7 @@ def unescape_html(text): >>> print(unescape_html('<tag>')) """ - def fixup(match): - """ - Replace one matched HTML entity with the character it represents, - if possible. - """ - text = match.group(0) - if text[:2] == "&#": - # character reference - try: - if text[:3] == "&#x": - codept = int(text[3:-1], 16) - else: - codept = int(text[2:-1]) - if 0x80 <= codept < 0xa0: - # Decode this range of characters as Windows-1252, as Web - # browsers do in practice. - return bytes([codept]).decode('sloppy-windows-1252') - else: - return chr(codept) - except ValueError: - pass - else: - # This is a named entity; if it's a known HTML5 entity, replace - # it with the appropriate character. - try: - return entities.html5[text[1:]] - except KeyError: - return text - return HTML_ENTITY_RE.sub(fixup, text) + return HTML_ENTITY_RE.sub(_unescape_fixup, text) ANSI_RE = re.compile('\033\\[((?:\\d|;)*)([a-zA-Z])') diff --git a/tests/test_entities.py b/tests/test_entities.py index 3693ae11..f67ead2d 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -23,3 +23,4 @@ def test_entities(): eq_(fix_text_segment('ellipsis…', normalization='NFKC'), 'ellipsis...') eq_(fix_text_segment('broken'), 'broken\x81') eq_(unescape_html('euro €'), 'euro €') + eq_(unescape_html('not an entity x6;'), 'not an entity x6;') From 5998567befec99a94c0fc79b726d5c1193b95ef2 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 29 Mar 2017 17:59:58 -0400 Subject: [PATCH 343/502] Describe HTML dependency change in the CHANGELOG --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9c6d25b..7bd51da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Version 5.1 (April ??, 2017) + +Removed the dependency on `html5lib` by dropping support for Python 3.2. + +We previously used the dictionary `html5lib.constants.entities` to decode HTML +entities. In Python 3.3 and later, that exact dictionary is now in the +standard library as `html.entities.html5`. + ## Version 5.0.2 and 4.4.2 (March 21, 2017) Added a `MANIFEST.in` that puts files such as the license file and this From 248ef102b4f61749d2042ada633eb149baa19b44 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 21 Mar 2017 15:42:11 -0400 Subject: [PATCH 344/502] Move the tests into a JSON file for those who want to port ftfy --- tests/test_cases.json | 409 ++++++++++++++++++++++++++++++++++++++++ tests/test_real_text.py | 199 +++++++------------ 2 files changed, 478 insertions(+), 130 deletions(-) create mode 100644 tests/test_cases.json diff --git a/tests/test_cases.json b/tests/test_cases.json new file mode 100644 index 00000000..2e310450 --- /dev/null +++ b/tests/test_cases.json @@ -0,0 +1,409 @@ +[ + { + "label": "Low-codepoint emoji", + "original": "He's Justinâ\u009d¤", + "fixed": "He's Justin❤", + "enabled": true + }, + { + "label": "UTF-8 / MacRoman mix-up about smurfs", + "original": "Le Schtroumpf Docteur conseille g√¢teaux et baies schtroumpfantes pour un r√©gime √©quilibr√©.", + "fixed": "Le Schtroumpf Docteur conseille gâteaux et baies schtroumpfantes pour un régime équilibré.", + "enabled": true + }, + { + "label": "Checkmark that almost looks okay as mojibake", + "original": "✔ No problems", + "fixed": "✔ No problems", + "enabled": true + }, + { + "label": "UTF-8 / Windows-1251 Russian mixup about futbol", + "original": "РґРѕСЂРѕРіРµ Р\u0098Р·-РїРѕРґ #футбол", + "fixed": "дороге Из-под #футбол", + "enabled": true + }, + { + "label": "Latin-1 / Windows-1252 mixup in German", + "original": "\u0084Handwerk bringt dich überall hin\u0093: Von der YOU bis nach Monaco", + "fixed-encoding": "„Handwerk bringt dich überall hin“: Von der YOU bis nach Monaco", + "fixed": "\"Handwerk bringt dich überall hin\": Von der YOU bis nach Monaco", + "enabled": true + }, + { + "label": "CESU-8 / Windows-1252 emoji", + "original": "Hi guys í ½í¸\u008d", + "fixed": "Hi guys 😍", + "enabled": true + }, + { + "label": "CESU-8 / Latin-1 emoji", + "original": "hihi RT username: â\u0098ºí ½í¸\u0098", + "fixed": "hihi RT username: ☺😘", + "enabled": true + }, + { + "label": "Latin-1 / Windows-1252 mixup in Turkish", + "original": "Beta Haber: Hırsızı Büyü Korkuttu", + "fixed": "Beta Haber: Hırsızı Büyü Korkuttu", + "enabled": true + }, + { + "label": "Negative: 'è' preceded by a non-breaking space is not a small capital Y", + "original": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", + "fixed": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", + "enabled": true + }, + { + "label": "UTF-8 / Windows-1251 mixed up twice in Russian", + "original": "приятности. РІСњВ¤", + "fixed": "приятности. ❤", + "enabled": true + }, + { + "label": "UTF-8 / Windows-1252 mixed up twice in Malay", + "original": "Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New “ Romanceâ€Â\u009d.", + "fixed-encoding": "Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New “ Romance”.", + "fixed": "Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New \" Romance\".", + "enabled": true + }, + { + "label": "Negative: accents are sometimes used as quotes", + "original": "``toda produzida pronta pra assa aí´´", + "fixed": "``toda produzida pronta pra assa aí´´", + "enabled": true + }, + { + "label": "Negative: 'Õ' followed by an ellipsis", + "comment": "Should not turn into the Armenian letter Յ", + "original": "HUHLL Õ…", + "fixed": "HUHLL Õ…", + "enabled": true + }, + { + "label": "UTF-8 / Windows-1252 mixed up twice in naming Iggy Pop", + "original": "Iggy Pop (né Jim Osterberg)", + "fixed": "Iggy Pop (né Jim Osterberg)", + "enabled": true + }, + { + "label": "Left quote is UTF-8, right quote is Latin-1, both encoded in Windows-1252", + "original": "Direzione Pd, ok â\u0080\u009csenza modifiche\u0094 all'Italicum.", + "fixed-encoding": "Direzione Pd, ok “senza modifiche” all'Italicum.", + "fixed": "Direzione Pd, ok \"senza modifiche\" all'Italicum.", + "enabled": true + }, + { + "label": "UTF-8 / sloppy Windows-1252 mixed up twice in a triumphant emoticon", + "original": "selamat berpuasa sob (Ã\u00a0¸‡'̀⌣'ÃŒÂ\u0081)Ã\u00a0¸‡", + "fixed": "selamat berpuasa sob (ง'̀⌣'́)ง", + "enabled": true + }, + { + "label": "UTF-8 / Windows-1252 mixed up three times", + "original": "The Mona Lisa doesn’t have eyebrows.", + "fixed-encoding": "The Mona Lisa doesn’t have eyebrows.", + "fixed": "The Mona Lisa doesn't have eyebrows.", + "enabled": true + }, + { + "label": "UTF-8 / Codepage 437 mixup in Russian", + "original": "#╨┐╤Ç╨░╨▓╨╕╨╗╤î╨╜╨╛╨╡╨┐╨╕╤é╨░╨╜╨╕╨╡", + "fixed": "#правильноепитание", + "enabled": true + }, + { + "label": "UTF-8 / Windows-1250 mixup in French", + "original": "Liège Avenue de l'HĂ´pital", + "fixed": "Liège Avenue de l'Hôpital", + "enabled": true + }, + { + "label": "UTF-8 / sloppy Windows-1250 mixup in English", + "original": "It was named „scars´ stones“ after the rock-climbers who got hurt while climbing on it.", + "fixed-encoding": "It was named\u00a0„scars´ stones“ after the rock-climbers who got hurt while climbing on it.", + "fixed": "It was named\u00a0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it.", + "enabled": true + }, + { + "label": "The same text as above, but as a UTF-8 / ISO-8859-2 mixup", + "original": "It was namedÂ\u00a0â\u0080\u009escars´ stonesâ\u0080\u009c after the rock-climbers who got hurt while climbing on it.", + "fixed-encoding": "It was named\u00a0„scars´ stones“ after the rock-climbers who got hurt while climbing on it.", + "fixed": "It was named\u00a0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it.", + "enabled": true + }, + { + "label": "UTF-8 / sloppy Windows-1250 mixup in Romanian", + "original": "vedere Ă®nceĹŁoĹźatÄ\u0083", + "fixed": "vedere înceţoşată", + "enabled": true + }, + { + "label": "UTF-8 / Windows-1250 mixup in Slovak", + "original": "NapĂ\u00adšte nám !", + "fixed": "Napíšte nám !", + "enabled": true + }, + { + "label": "A face with UTF-8 / sloppy Windows-1252 mixed up twice", + "original": "ââ€\u009d’(⌣˛⌣)ââ€\u009dŽ", + "fixed": "┒(⌣˛⌣)┎", + "enabled": true + }, + { + "label": "We can mostly decode the face above when we lose the character U+009D", + "original": "ââ€�’(⌣˛⌣)ââ€�Ž", + "fixed": "�(⌣˛⌣)�", + "enabled": true + }, + { + "label": "CESU-8 / Latin-1 mixup over several emoji", + "comment": "You tried", + "original": "I just figured out how to tweet emojis! â\u009a½í\u00a0½í¸\u0080í\u00a0½í¸\u0081í\u00a0½í¸\u0082í\u00a0½í¸\u0086í\u00a0½í¸\u008eí\u00a0½í¸\u008eí\u00a0½í¸\u008eí\u00a0½í¸\u008e", + "fixed": "I just figured out how to tweet emojis! ⚽😀😁😂😆😎😎😎😎", + "enabled": true + }, + { + "label": "Two levels of inconsistent mojibake", + "comment": "The en-dash was mojibaked in UTF-8 / Windows-1252 as three characters, two of which were mojibaked again as Windows-1252 / Latin-1, and the third of which was mojibaked as UTF-8 / Latin-1", + "original": "Arsenal v Wolfsburg: pre-season friendly â\u0080â\u0080\u009c live!", + "fixed": "Arsenal v Wolfsburg: pre-season friendly – live!", + "enabled": true + }, + { + "label": "Negative: multiplication sign and ellipsis", + "comment": "Should not turn into a dot below", + "original": "4288×…", + "fixed": "4288×…", + "enabled": true + }, + { + "label": "Negative: 'Ê' followed by an ellipsis", + "comment": "Should not turn into a squat reversed esh", + "original": "RETWEET SE VOCÊ…", + "fixed": "RETWEET SE VOCÊ…", + "enabled": true + }, + { + "label": "Negative: 'É' followed by an ellipsis", + "comment": "Should not turn into 'MARQUɅ'", + "original": "PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…", + "fixed": "PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…", + "enabled": true + }, + { + "label": "Negative: 'Ó' followed by an ellipsis", + "comment": "Should not turn into 'SӅ'", + "original": "TEM QUE SEGUIR, SDV SÓ…", + "fixed": "TEM QUE SEGUIR, SDV SÓ…", + "enabled": true + }, + { + "label": "Negative: 'É' followed by a curly apostrophe", + "comment": "Should not turn into 'ZZAJɒs'", + "original": "Join ZZAJÉ’s Official Fan List and receive news, events, and more!", + "fixed-encoding": "Join ZZAJÉ’s Official Fan List and receive news, events, and more!", + "fixed": "Join ZZAJÉ's Official Fan List and receive news, events, and more!", + "enabled": true + }, + { + "label": "Negative: 'é' preceded by curly apostrophe", + "comment": "Should not turn into 'LՎpisode'", + "original": "L’épisode 8 est trop fou ouahh", + "fixed-encoding": "L’épisode 8 est trop fou ouahh", + "fixed": "L'épisode 8 est trop fou ouahh", + "enabled": true + }, + { + "label": "Negative: three raised eyebrows or something?", + "comment": "Should not turn into private use character U+F659", + "original": "Ôôô VIDA MINHA", + "fixed": "Ôôô VIDA MINHA", + "enabled": true + }, + { + "label": "Negative: copyright sign preceded by non-breaking space", + "comment": "Should not turn into 'ʩ'", + "original": "[x]\u00a0©", + "fixed": "[x]\u00a0©", + "enabled": true + }, + { + "label": "Negative: en dash and infinity sign", + "comment": "Should not turn into '2012Ѱ'", + "original": "2012—∞", + "fixed": "2012—∞", + "enabled": true + }, + { + "label": "Negative: This Е is a Ukrainian letter, but nothing else is wrong", + "original": "SENSЕ - Oleg Tsedryk", + "fixed": "SENSЕ - Oleg Tsedryk", + "enabled": true + }, + { + "label": "Negative: angry face", + "comment": "The face should not turn into '`«'", + "original": "OK??:( `¬´ ):", + "fixed": "OK??:( `¬´ ):", + "enabled": true + }, + { + "label": "Negative: triangle and degree sign", + "comment": "I'm not really sure what it *is* supposed to be, but it's not 'ơ'", + "original": "∆°", + "fixed": "∆°", + "enabled": true + }, + { + "label": "Negative: Portuguese with inverted question mark", + "comment": "Former false positive - it should not turn into 'QUEM ɿ'", + "original": "ESSE CARA AI QUEM É¿", + "fixed": "ESSE CARA AI QUEM É¿", + "enabled": true + }, + { + "label": "Negative: Portuguese with acute accents as quotation marks", + "comment": "Former false positive - the end should not turn into a superscript H", + "original": "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", + "fixed": "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", + "enabled": true + }, + { + "label": "Negative: Finnish Ä followed by a non-breaking space", + "comment": "Former false positive - should not become a G with a dot", + "original": "SELKÄ\u00a0EDELLÄ\u00a0MAAHAN via @YouTube", + "fixed": "SELKÄ\u00a0EDELLÄ\u00a0MAAHAN via @YouTube", + "enabled": true + }, + { + "label": "Negative: multiplying by currency", + "comment": "Former false positive - should not become the Hebrew letter 'final pe'", + "original": "Offering 5×£35 pin ups", + "fixed": "Offering 5×£35 pin ups", + "enabled": true + }, + { + "label": "Negative: registered chocolate brand name", + "comment": "Former false positive - should not become the IPA letter 'lezh'", + "original": "NESTLÉ® requiere contratar personal para diferentes areas a nivel nacional e internacional", + "fixed": "NESTLÉ® requiere contratar personal para diferentes areas a nivel nacional e internacional", + "enabled": true + }, + + { + "label": "Punctuation pile-up should actually be musical notes", + "comment": "Unfortunately, the difference doesn't look convincing to ftfy", + "original": "Engkau masih yg terindah, indah di dalam hatiku♫~", + "fixed": "Engkau masih yg terindah, indah di dalam hatiku♫~", + "enabled": false + }, + { + "label": "UTF-8 / Windows-1252 mixup in Gaelic involving non-breaking spaces", + "comment": "Too similar to the Finnish negative example with non-breaking spaces", + "original": "CÃ\u00a0nan nan GÃ\u00a0idheal", + "fixed": "Cànan nan Gàidheal", + "enabled": false + }, + { + "label": "Windows-1252 / MacRoman mixup in Spanish", + "comment": "Requires something like encoding detection", + "original": "Deja dos heridos hundimiento de barco tur\u0092stico en Acapulco.", + "fixed": "Deja dos heridos hundimiento de barco turístico en Acapulco.", + "enabled": false + }, + { + "label": "UTF-8 / Windows-1251 mixup in tweet spam", + "comment": "The Windows-1251-fixing plan is too conservative to fix this", + "original": "Blog Traffic Tip 2 – Broadcast Email Your Blog", + "fixed": "Blog Traffic Tip 2 – Broadcast Email Your Blog", + "enabled": false + }, + { + "label": "Negative: Leet line-art", + "comment": "Current false positive: it concidentally all decodes as UTF-8/CP437 mojibake and becomes 'ôaſaſaſaſa'", + "original": "├┤a┼┐a┼┐a┼┐a┼┐a", + "fixed": "├┤a┼┐a┼┐a┼┐a┼┐a", + "enabled": false + }, + + { + "label": "Synthetic, negative: Brontë's name does not end with a Korean syllable", + "comment": "The original example of why ftfy needs heuristics", + "original": "I'm not such a fan of Charlotte Brontë…”", + "fixed-encoding": "I'm not such a fan of Charlotte Brontë…”", + "fixed": "I'm not such a fan of Charlotte Brontë…\"", + "enabled": true + }, + { + "label": "Synthetic, negative: hypothetical Swedish product name", + "comment": "This used to be a constructed example of a false positive, until you added another symbol", + "original": "AHÅ™, the new sofa from IKEA", + "fixed": "AHÅ™, the new sofa from IKEA", + "enabled": true + }, + { + "label": "Synthetic, negative: Ukrainian capital letters", + "comment": "We need to fix Windows-1251 conservatively, or else this decodes as '²ʲ'", + "original": "ВІКІ is Ukrainian for WIKI", + "fixed": "ВІКІ is Ukrainian for WIKI", + "enabled": true + }, + { + "label": "Synthetic, negative: don't leak our internal use of byte 0x1A", + "comment": "We use byte 0x1A internally as an encoding of U+FFFD, but literal occurrences of U+1A are just ASCII control characters", + "original": "These control characters \u001a are apparently intentional \u0081", + "fixed-encoding": "These control characters \u001a are apparently intentional \u0081", + "fixed": "These control characters are apparently intentional \u0081", + "enabled": true + }, + { + "label": "Synthetic, negative: U+1A on its own", + "comment": "We use byte 0x1A internally as an encoding of U+FFFD, but literal occurrences of U+1A are just ASCII control characters", + "original": "Here's a control character: \u001a", + "fixed-encoding": "Here's a control character: \u001a", + "fixed": "Here's a control character: ", + "enabled": true + }, + { + "label": "Synthetic, negative: A-with-circle as an Angstrom sign", + "comment": "Should not turn into '10 ŗ'", + "original": "a radius of 10 Å—", + "fixed": "a radius of 10 Å—", + "enabled": true + }, + { + "label": "Synthetic, negative: Spanish with exclamation points on the wrong sides", + "original": "!YO SÉ¡", + "fixed": "!YO SÉ¡", + "enabled": true + }, + { + "label": "Synthetic: fix text with backslashes in it", + "comment": "Tests for a regression on a long-ago bug", + "original": "<40\\% vs \u00e2\u0089\u00a540\\%", + "fixed": "<40\\% vs ≥40\\%", + "enabled": true + }, + { + "label": "Synthetic: curly quotes with mismatched encoding glitches in Latin-1", + "original": "\u00e2\u0080\u009cmismatched quotes\u0085\u0094", + "fixed-encoding": "“mismatched quotes…”", + "fixed": "\"mismatched quotes…\"", + "enabled": true + }, + { + "label": "Synthetic: curly quotes with mismatched encoding glitches in Windows-1252", + "original": "“mismatched quotes…”", + "fixed-encoding": "“mismatched quotes…”", + "fixed": "\"mismatched quotes…\"", + "enabled": true + }, + { + "label": "Synthetic: lossy decoding in sloppy-windows-1252", + "original": "“lossy decodingâ€�", + "fixed-encoding": "“lossy decoding�", + "fixed": "\"lossy decoding�", + "enabled": true + } +] diff --git a/tests/test_real_text.py b/tests/test_real_text.py index 5a027882..ca3d9aff 100644 --- a/tests/test_real_text.py +++ b/tests/test_real_text.py @@ -1,138 +1,77 @@ +""" +Test with text actually found in the wild (mostly on Twitter). + +I collected test cases by listening to the Twitter streaming API for +a million or so tweets, picking out examples with high weirdness according +to ftfy version 2, and seeing what ftfy decoded them to. There are some +impressive things that can happen to text, even in an ecosystem that is +supposedly entirely UTF-8. + +TEST_CASES contains the most interesting examples of these, often with some +trickiness of how to decode them into the actually intended text. + +For some reason, sampling Twitter gives no examples of text being +accidentally decoded as Windows-1250, even though it's one of the more +common encodings and this mojibake has been spotted in the wild. It may be +that Windows-1250 is used in places that culturally don't use Twitter much +(Central and Eastern Europe), and therefore nobody designs a Twitter app or +bot to use Windows-1250. I've collected a couple of examples of +Windows-1250 mojibake from elsewhere. +""" from ftfy import fix_text from ftfy.fixes import fix_encoding_and_explain, apply_plan from nose.tools import eq_ +import json +import os -TEST_CASES = [ - ## These are excerpts from text actually seen in the wild, mostly on - ## Twitter. Usernames and links have been removed. - ("He's Justin❤", "He's Justin❤"), - ("Le Schtroumpf Docteur conseille g√¢teaux et baies schtroumpfantes pour un r√©gime √©quilibr√©.", - "Le Schtroumpf Docteur conseille gâteaux et baies schtroumpfantes pour un régime équilibré."), - ("✔ No problems", "✔ No problems"), - ('4288×…', '4288×…'), - ('RETWEET SE VOCÊ…', 'RETWEET SE VOCÊ…'), - ('PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…', 'PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…'), - ('TEM QUE SEGUIR, SDV SÓ…', 'TEM QUE SEGUIR, SDV SÓ…'), - ('Join ZZAJÉ’s Official Fan List and receive news, events, and more!', "Join ZZAJÉ's Official Fan List and receive news, events, and more!"), - ('L’épisode 8 est trop fou ouahh', "L'épisode 8 est trop fou ouahh"), - ("РґРѕСЂРѕРіРµ Из-РїРѕРґ #футбол", - "дороге Из-под #футбол"), - ("\x84Handwerk bringt dich \xfcberall hin\x93: Von der YOU bis nach Monaco", - '"Handwerk bringt dich überall hin": Von der YOU bis nach Monaco'), - ("Hi guys 😍", "Hi guys 😍"), - ("hihi RT username: ☺😘", - "hihi RT username: ☺😘"), - ("Beta Haber: Hırsızı Büyü Korkuttu", - "Beta Haber: Hırsızı Büyü Korkuttu"), - ("Ôôô VIDA MINHA", "Ôôô VIDA MINHA"), - ('[x]\xa0©', '[x]\xa0©'), - ('2012—∞', '2012—∞'), - ('Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.', - 'Con il corpo e lo spirito ammaccato,\xa0è come se nel cuore avessi un vetro conficcato.'), - ('приятности. РІСњВ¤', 'приятности. ❤'), - ('Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New “ Romance”.', - 'Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New " Romance".'), - ("``toda produzida pronta pra assa aí´´", "``toda produzida pronta pra assa aí´´"), - ('HUHLL Õ…', 'HUHLL Õ…'), - ('Iggy Pop (né Jim Osterberg)', 'Iggy Pop (né Jim Osterberg)'), - ('eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías', - 'eres mía, mía, mía, no te hagas la loca eso muy bien ya lo sabías'), - ("Direzione Pd, ok “senza modifiche” all'Italicum.", - "Direzione Pd, ok \"senza modifiche\" all'Italicum."), - ('SENSЕ - Oleg Tsedryk', 'SENSЕ - Oleg Tsedryk'), # this Е is a Ukrainian letter - ('OK??:( `¬´ ):', 'OK??:( `¬´ ):'), - ("selamat berpuasa sob (Ã\xa0¸‡'̀⌣'ÃŒÂ\x81)Ã\xa0¸‡", - "selamat berpuasa sob (ง'̀⌣'́)ง"), - ("The Mona Lisa doesn’t have eyebrows.", - "The Mona Lisa doesn't have eyebrows."), - ("#╨┐╤Ç╨░╨▓╨╕╨╗╤î╨╜╨╛╨╡╨┐╨╕╤é╨░╨╜╨╕╨╡", "#правильноепитание"), - ('∆°', '∆°'), - - # Test Windows-1250 mixups - ("Liège Avenue de l'HĂ´pital", "Liège Avenue de l'Hôpital"), - ("It was named „scars´ stones“ after the rock-climbers who got hurt while climbing on it.", - "It was named\xa0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it."), - ("vedere Ă®nceĹŁoĹźatÄ\x83", "vedere înceţoşată"), - ("NapĂ\xadšte nám !", "Napíšte nám !"), - - # The second test is different in iso-8859-2 - ("It was namedÂ\xa0â\x80\x9escars´ stonesâ\x80\x9c after the rock-climbers who got hurt while climbing on it.", - "It was named\xa0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it."), - - # This one has two differently-broken layers of Windows-1252 <=> UTF-8, - # and it's kind of amazing that we solve it. - ('Arsenal v Wolfsburg: pre-season friendly â\x80â\x80\x9c live!', - 'Arsenal v Wolfsburg: pre-season friendly – live!'), - - # Test that we can mostly decode this face when the nonprintable - # character \x9d is lost - ('ââ€\x9d’(⌣˛⌣)ââ€\x9dŽ', '┒(⌣˛⌣)┎'), - ('ââ€�’(⌣˛⌣)ââ€�Ž', '�(⌣˛⌣)�'), - - # You tried - ('I just figured out how to tweet emojis! â\x9a½í\xa0½í¸\x80í\xa0½í¸\x81í\xa0½í¸\x82í\xa0½í¸\x86í\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8eí\xa0½í¸\x8e', - 'I just figured out how to tweet emojis! ⚽😀😁😂😆😎😎😎😎'), - - # Former false positives - ("ESSE CARA AI QUEM É¿", "ESSE CARA AI QUEM É¿"), - ("``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´"), - ("SELKÄ\xa0EDELLÄ\xa0MAAHAN via @YouTube", "SELKÄ\xa0EDELLÄ\xa0MAAHAN via @YouTube"), - ("Offering 5×£35 pin ups", "Offering 5×£35 pin ups"), - ("NESTLÉ® requiere contratar personal para diferentes areas a nivel nacional e internacional", - "NESTLÉ® requiere contratar personal para diferentes areas a nivel nacional e internacional"), - - ## This remains a false positive - # ("├┤a┼┐a┼┐a┼┐a┼┐a", "├┤a┼┐a┼┐a┼┐a┼┐a"), - - ## This kind of tweet can't be fixed without a full-blown encoding detector. - #("Deja dos heridos hundimiento de barco tur\x92stico en Acapulco.", - # "Deja dos heridos hundimiento de barco turístico en Acapulco."), - - ## The original text looks too plausible - # ('CÃ\xa0nan nan GÃ\xa0idheal', 'Cànan nan Gàidheal'), - - ## Turning sorta-plausible text into musical notes isn't convincing enough - ## under the current heuristics - # ('Engkau masih yg terindah, indah di dalam hatiku♫~', - # 'Engkau masih yg terindah, indah di dalam hatiku♫~'), - - ## The heuristics aren't confident enough to fix this text and its weird encoding. - #("Blog Traffic Tip 2 – Broadcast Email Your Blog", - # "Blog Traffic Tip 2 – Broadcast Email Your Blog"), -] +THIS_DIR = os.path.dirname(__file__) +TEST_FILENAME = os.path.join(THIS_DIR, 'test_cases.json') +TEST_DATA = json.load(open(TEST_FILENAME, encoding='utf-8')) + + +# We set up the tests as a dictionary where they can be looked up by label, +# even though we already have them in a list, so we can call 'check_example' by +# the test label. This will make the test label show up in the name of the test +# in nose. + +def _make_tests(): + available_tests = {} + for val in TEST_DATA: + label = val['label'] + available_tests[label] = { + 'original': val['original'], + 'fixed': val['fixed'], + 'fixed-encoding': val.get('fixed-encoding', val['fixed']) + } + return available_tests + + +AVAILABLE_TESTS = _make_tests() + + +def check_example(label): + # Run the test with the given label in the data file + val = AVAILABLE_TESTS[label] + orig = val['original'] + fixed = val['fixed'] + + # Make sure that the fix_encoding step outputs a plan that we can + # successfully run to reproduce its result + encoding_fix, plan = fix_encoding_and_explain(orig) + eq_(apply_plan(orig, plan), encoding_fix) + + # Make sure we can decode the text as intended + eq_(fix_text(orig), fixed) + eq_(encoding_fix, val['fixed-encoding']) + + # Make sure we can decode as intended even with an extra layer of badness + extra_bad = orig.encode('utf-8').decode('latin-1') + eq_(fix_text(extra_bad), fixed) def test_real_text(): - """ - Test with text actually found in the wild (mostly on Twitter). - - I collected test cases by listening to the Twitter streaming API for - a million or so tweets, picking out examples with high weirdness according - to ftfy version 2, and seeing what ftfy decoded them to. There are some - impressive things that can happen to text, even in an ecosystem that is - supposedly entirely UTF-8. - - TEST_CASES contains the most interesting examples of these, often with some - trickiness of how to decode them into the actually intended text. - - For some reason, sampling Twitter gives no examples of text being - accidentally decoded as Windows-1250, even though it's one of the more - common encodings and this mojibake has been spotted in the wild. It may be - that Windows-1250 is used in places that culturally don't use Twitter much - (Central and Eastern Europe), and therefore nobody designs a Twitter app or - bot to use Windows-1250. I've collected a couple of examples of - Windows-1250 mojibake from elsewhere. - """ - for orig, target in TEST_CASES: - # make sure that the fix_encoding step outputs a plan that we can - # successfully run to reproduce its result - encoding_fix, plan = fix_encoding_and_explain(orig) - eq_(apply_plan(orig, plan), encoding_fix) - - # make sure we can decode the text as intended - eq_(fix_text(orig), target) - - # make sure we can decode as intended even with an extra layer of badness - extra_bad = orig.encode('utf-8').decode('latin-1') - eq_(fix_text(extra_bad), target) + for test_case in TEST_DATA: + if test_case['enabled']: + label = test_case['label'] + yield check_example, label From e2a46258783f14f830016ca73b117f5d9066686b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 31 Mar 2017 13:34:11 -0400 Subject: [PATCH 345/502] Revise test_real_text.py to test_examples_in_json.py --- ..._real_text.py => test_examples_in_json.py} | 44 +++++++++++-------- ...hetic.py => test_futuristic_codepoints.py} | 44 ------------------- 2 files changed, 25 insertions(+), 63 deletions(-) rename tests/{test_real_text.py => test_examples_in_json.py} (60%) rename tests/{test_synthetic.py => test_futuristic_codepoints.py} (60%) diff --git a/tests/test_real_text.py b/tests/test_examples_in_json.py similarity index 60% rename from tests/test_real_text.py rename to tests/test_examples_in_json.py index ca3d9aff..8a988483 100644 --- a/tests/test_real_text.py +++ b/tests/test_examples_in_json.py @@ -1,22 +1,27 @@ """ -Test with text actually found in the wild (mostly on Twitter). - -I collected test cases by listening to the Twitter streaming API for -a million or so tweets, picking out examples with high weirdness according -to ftfy version 2, and seeing what ftfy decoded them to. There are some -impressive things that can happen to text, even in an ecosystem that is -supposedly entirely UTF-8. - -TEST_CASES contains the most interesting examples of these, often with some -trickiness of how to decode them into the actually intended text. - -For some reason, sampling Twitter gives no examples of text being -accidentally decoded as Windows-1250, even though it's one of the more -common encodings and this mojibake has been spotted in the wild. It may be -that Windows-1250 is used in places that culturally don't use Twitter much -(Central and Eastern Europe), and therefore nobody designs a Twitter app or -bot to use Windows-1250. I've collected a couple of examples of -Windows-1250 mojibake from elsewhere. +Test ftfy's fixes using the data in `test_cases.json`. + +I collected many test cases by listening to the Twitter streaming API for +millions of tweets, picking out examples with high weirdness, and seeing what +ftfy decoded them to. There are some impressive things that can happen to text, +even in an ecosystem that is supposedly entirely UTF-8. + +Some examples come from the Common Crawl (particularly those involving +Windows-1250 mojibake, which is more common on arbitrary Web pages than on +Twitter), and some examples marked as 'synthetic' are contrived to test +particular features of ftfy. + +Each test case is a dictionary containing the following items: + +- "label": a label that will identify the test case in nosetests +- "original": the text to be ftfy'd +- "fixed": what the result of ftfy.fix_text should be on this text + +There are also two optional fields: + +- "fixed-encoding": what the result of just ftfy.fix_encoding should be. + If missing, it will be considered to be the same as "fixed". +- "comment": possibly-enlightening commentary on the test case. """ from ftfy import fix_text from ftfy.fixes import fix_encoding_and_explain, apply_plan @@ -70,7 +75,8 @@ def check_example(label): eq_(fix_text(extra_bad), fixed) -def test_real_text(): +def test_cases(): + # Run all the test cases in `test_cases.json` for test_case in TEST_DATA: if test_case['enabled']: label = test_case['label'] diff --git a/tests/test_synthetic.py b/tests/test_futuristic_codepoints.py similarity index 60% rename from tests/test_synthetic.py rename to tests/test_futuristic_codepoints.py index 7ff4e274..b2cdfba7 100644 --- a/tests/test_synthetic.py +++ b/tests/test_futuristic_codepoints.py @@ -4,50 +4,6 @@ from nose.tools import eq_ -phrases = [ - "\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", - "\u201CI'm not such a fan of Charlotte Brontë\u2026\u201D", - "\u2039ALLÍ ESTÁ\u203A", - "\u2014ALLÍ ESTÁ\u2014", - "AHÅ™, the new sofa from IKEA®", - "ВІКІ is Ukrainian for WIKI", - 'These control characters \x1a are apparently intentional \x81', - "I don't know what this is \x1a but this is the euro sign €", - "\u2014a radius of 10 Å\u2014", - # You can point out that these exclamation points are backwards. I know! - "!YO SÉ¡" - -] -# These phrases should not be erroneously "fixed" -def test_valid_phrases(): - for phrase in phrases: - yield check_phrase, phrase - - -def check_phrase(text): - # Check each valid phrase above, making sure that it doesn't get changed - eq_(fix_encoding(text), text) - eq_(fix_encoding(text.encode('utf-8').decode('latin-1')), text) - - # make sure that the opening punctuation is not the only thing that makes - # it work - eq_(fix_encoding(text[1:]), text[1:]) - eq_(fix_encoding(text[1:].encode('utf-8').decode('latin-1')), text[1:]) - - -def test_fix_with_backslash(): - eq_(fix_encoding("<40\\% vs \xe2\x89\xa540\\%"), "<40\\% vs ≥40\\%") - - -def test_mixed_utf8(): - eq_(fix_encoding('\xe2\x80\x9cmismatched quotes\x85\x94'), '“mismatched quotes…”') - eq_(fix_encoding('“mismatched quotes…”'), '“mismatched quotes…”') - - -def test_lossy_utf8(): - eq_(fix_encoding('“lossy decodingâ€�'), '“lossy decoding�') - - def test_burritos(): # Make a string with two burritos in it. Python doesn't know about Unicode # burritos, but ftfy can guess they're probably emoji anyway. From 73e2e0cbba4e7bfd341d535a39a98bf824fa6126 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 4 Apr 2017 12:21:28 -0400 Subject: [PATCH 346/502] Simplify the JSON tests to not require building a separate dict --- tests/test_examples_in_json.py | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/tests/test_examples_in_json.py b/tests/test_examples_in_json.py index 8a988483..37533508 100644 --- a/tests/test_examples_in_json.py +++ b/tests/test_examples_in_json.py @@ -35,29 +35,8 @@ TEST_DATA = json.load(open(TEST_FILENAME, encoding='utf-8')) -# We set up the tests as a dictionary where they can be looked up by label, -# even though we already have them in a list, so we can call 'check_example' by -# the test label. This will make the test label show up in the name of the test -# in nose. - -def _make_tests(): - available_tests = {} - for val in TEST_DATA: - label = val['label'] - available_tests[label] = { - 'original': val['original'], - 'fixed': val['fixed'], - 'fixed-encoding': val.get('fixed-encoding', val['fixed']) - } - return available_tests - - -AVAILABLE_TESTS = _make_tests() - - -def check_example(label): - # Run the test with the given label in the data file - val = AVAILABLE_TESTS[label] +def check_example(val): + # Run one example from the data file orig = val['original'] fixed = val['fixed'] @@ -68,7 +47,7 @@ def check_example(label): # Make sure we can decode the text as intended eq_(fix_text(orig), fixed) - eq_(encoding_fix, val['fixed-encoding']) + eq_(encoding_fix, val.get('fixed-encoding', val['fixed'])) # Make sure we can decode as intended even with an extra layer of badness extra_bad = orig.encode('utf-8').decode('latin-1') @@ -79,5 +58,5 @@ def test_cases(): # Run all the test cases in `test_cases.json` for test_case in TEST_DATA: if test_case['enabled']: - label = test_case['label'] - yield check_example, label + check_example.description = "test_examples_in_json.check_example({!r})".format(test_case['label']) + yield check_example, test_case From bb50b3fa83aad15f312ed6e7b8da5e6483a601c2 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 4 Apr 2017 12:21:52 -0400 Subject: [PATCH 347/502] Organize JSON test cases a bit more --- tests/test_cases.json | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/test_cases.json b/tests/test_cases.json index 2e310450..83eccbd7 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -48,12 +48,6 @@ "fixed": "Beta Haber: Hırsızı Büyü Korkuttu", "enabled": true }, - { - "label": "Negative: 'è' preceded by a non-breaking space is not a small capital Y", - "original": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", - "fixed": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", - "enabled": true - }, { "label": "UTF-8 / Windows-1251 mixed up twice in Russian", "original": "приятности. РІСњВ¤", @@ -67,19 +61,6 @@ "fixed": "Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New \" Romance\".", "enabled": true }, - { - "label": "Negative: accents are sometimes used as quotes", - "original": "``toda produzida pronta pra assa aí´´", - "fixed": "``toda produzida pronta pra assa aí´´", - "enabled": true - }, - { - "label": "Negative: 'Õ' followed by an ellipsis", - "comment": "Should not turn into the Armenian letter Յ", - "original": "HUHLL Õ…", - "fixed": "HUHLL Õ…", - "enabled": true - }, { "label": "UTF-8 / Windows-1252 mixed up twice in naming Iggy Pop", "original": "Iggy Pop (né Jim Osterberg)", @@ -170,6 +151,12 @@ "fixed": "Arsenal v Wolfsburg: pre-season friendly – live!", "enabled": true }, + { + "label": "Negative: 'è' preceded by a non-breaking space is not a small capital Y", + "original": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", + "fixed": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", + "enabled": true + }, { "label": "Negative: multiplication sign and ellipsis", "comment": "Should not turn into a dot below", @@ -177,6 +164,20 @@ "fixed": "4288×…", "enabled": true }, + { + "label": "Negative: accents are sometimes used as quotes", + "comment": "The CESU-8 decoder will try to decode this, and then fail when it hits the end of the string", + "original": "``toda produzida pronta pra assa aí´´", + "fixed": "``toda produzida pronta pra assa aí´´", + "enabled": true + }, + { + "label": "Negative: 'Õ' followed by an ellipsis", + "comment": "Should not turn into the Armenian letter Յ", + "original": "HUHLL Õ…", + "fixed": "HUHLL Õ…", + "enabled": true + }, { "label": "Negative: 'Ê' followed by an ellipsis", "comment": "Should not turn into a squat reversed esh", From becb306530dfcf246dcd4c294e65cbb3f59a9c48 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 4 Apr 2017 12:22:43 -0400 Subject: [PATCH 348/502] Burritos are no longer futuristic The test with the burrito emoji is no different from any other emoji test now that we're on Unicode 9. --- tests/test_futuristic_codepoints.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/test_futuristic_codepoints.py b/tests/test_futuristic_codepoints.py index b2cdfba7..3fa10fcc 100644 --- a/tests/test_futuristic_codepoints.py +++ b/tests/test_futuristic_codepoints.py @@ -4,26 +4,6 @@ from nose.tools import eq_ -def test_burritos(): - # Make a string with two burritos in it. Python doesn't know about Unicode - # burritos, but ftfy can guess they're probably emoji anyway. - emoji_text = 'dos burritos: \U0001f32f\U0001f32f' - - # Mangle the burritos into a mess of Cyrillic characters. (It would have - # been great if we could have decoded them in cp437 instead, to turn them - # into "DOS burritos", but the resulting string is one that ftfy has been - # able to fix even without knowledge of Unicode 9.) - emojibake = emoji_text.encode('utf-8').decode('windows-1251') - - # Restore the original text. - eq_(fix_encoding(emojibake), emoji_text) - - # This doesn't happen if we replace the burritos with arbitrary unassigned - # characters. The mangled text passes through as is. - not_emoji = 'dos burritos: \U0003f32f\U0003f32f'.encode('utf-8').decode('windows-1251') - eq_(fix_encoding(not_emoji), not_emoji) - - def test_unknown_emoji(): # The range we accept as emoji has gotten larger. Let's make sure we can # decode the futuristic emoji U+1F960, which will probably be a picture of From 74e3b2b2a59d635464e1abd14f0e7026fed1d0d2 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 4 Apr 2017 13:03:01 -0400 Subject: [PATCH 349/502] test_examples_in_json: minor cleanups --- tests/test_examples_in_json.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_examples_in_json.py b/tests/test_examples_in_json.py index 37533508..69169162 100644 --- a/tests/test_examples_in_json.py +++ b/tests/test_examples_in_json.py @@ -47,7 +47,7 @@ def check_example(val): # Make sure we can decode the text as intended eq_(fix_text(orig), fixed) - eq_(encoding_fix, val.get('fixed-encoding', val['fixed'])) + eq_(encoding_fix, val.get('fixed-encoding', fixed)) # Make sure we can decode as intended even with an extra layer of badness extra_bad = orig.encode('utf-8').decode('latin-1') @@ -58,5 +58,8 @@ def test_cases(): # Run all the test cases in `test_cases.json` for test_case in TEST_DATA: if test_case['enabled']: - check_example.description = "test_examples_in_json.check_example({!r})".format(test_case['label']) + desc = "test_examples_in_json.check_example({!r})".format( + test_case['label'] + ) + check_example.description = desc yield check_example, test_case From eb03d1fd901d7a7d642f8a310f49af154069a4a6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 7 Apr 2017 11:25:18 -0400 Subject: [PATCH 350/502] Use assert_raises in test_cli --- tests/test_cli.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 6f686760..701b15d3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,6 @@ import subprocess import os -from nose.tools import eq_ +from nose.tools import eq_, assert_raises # Get the filename of 'halibote.txt', which contains some mojibake about @@ -42,11 +42,10 @@ def test_alternate_encoding(): def test_wrong_encoding(): # It's more of a problem when the file doesn't actually decode. - try: + with assert_raises(subprocess.CalledProcessError) as context: get_command_output(['ftfy', '-e', 'windows-1252', TEST_FILENAME]) - assert False, "Should have raised a CalledProcessError" - except subprocess.CalledProcessError as e: - eq_(e.output.decode('utf-8'), FAILED_OUTPUT) + e = context.exception + eq_(e.output.decode('utf-8'), FAILED_OUTPUT) def test_stdin(): From 7c0c912d0eeb518f4b0c069068a0c953a602c67a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 7 Apr 2017 11:47:48 -0400 Subject: [PATCH 351/502] Update CHANGELOG for 5.1 --- CHANGELOG.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bd51da6..074d2ee6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,23 @@ -## Version 5.1 (April ??, 2017) +## Version 5.1 (April 7, 2017) -Removed the dependency on `html5lib` by dropping support for Python 3.2. +- Removed the dependency on `html5lib` by dropping support for Python 3.2. + + We previously used the dictionary `html5lib.constants.entities` to decode + HTML entities. In Python 3.3 and later, that exact dictionary is now in the + standard library as `html.entities.html5`. + +- Moved many test cases about how particular text should be fixed into + `test_cases.json`, which may ease porting to other languages. + +The functionality of this version remains the same as 5.0.2 and 4.4.2. -We previously used the dictionary `html5lib.constants.entities` to decode HTML -entities. In Python 3.3 and later, that exact dictionary is now in the -standard library as `html.entities.html5`. ## Version 5.0.2 and 4.4.2 (March 21, 2017) Added a `MANIFEST.in` that puts files such as the license file and this changelog inside the source distribution. + ## Version 5.0.1 and 4.4.1 (March 10, 2017) Bug fix: From 898a6c17d8b6dac9a9e8851c8c6a27f594677af9 Mon Sep 17 00:00:00 2001 From: Jakub Wilk Date: Sat, 22 Apr 2017 19:48:44 +0200 Subject: [PATCH 352/502] Fix typo --- tests/test_cases.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cases.json b/tests/test_cases.json index 83eccbd7..30241b9c 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -322,7 +322,7 @@ }, { "label": "Negative: Leet line-art", - "comment": "Current false positive: it concidentally all decodes as UTF-8/CP437 mojibake and becomes 'ôaſaſaſaſa'", + "comment": "Current false positive: it coincidentally all decodes as UTF-8/CP437 mojibake and becomes 'ôaſaſaſaſa'", "original": "├┤a┼┐a┼┐a┼┐a┼┐a", "fixed": "├┤a┼┐a┼┐a┼┐a┼┐a", "enabled": false From 62abc5146096df5874ae48ed41c5033197f3280a Mon Sep 17 00:00:00 2001 From: "Rolando (Max) Espinoza" Date: Mon, 15 May 2017 17:33:11 -0300 Subject: [PATCH 353/502] Make string comparison cross-platform by using os.linesep This fixes Windows failures when running CLI tests due to different line separator (\n != \r\n). Worth noting that windows requires PYTHONIOENCODING=utf8 environment variable to make it compatible with the CMD terminal IO. --- tests/test_cli.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 701b15d3..bd887f22 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,16 +7,18 @@ # Harry Potter in Chinese THIS_DIR = os.path.dirname(__file__) TEST_FILENAME = os.path.join(THIS_DIR, 'halibote.txt') -CORRECT_OUTPUT = '【更新】《哈利波特》石堧卜才新婚娶初戀今痠逝\n' -FAILED_OUTPUT = '''ftfy error: -This input couldn't be decoded as 'windows-1252'. We got the following error: - - 'charmap' codec can't decode byte 0x90 in position 5: character maps to - -ftfy works best when its input is in a known encoding. You can use `ftfy -g` -to guess, if you're desperate. Otherwise, give the encoding name with the -`-e` option, such as `ftfy -e latin-1`. -''' +CORRECT_OUTPUT = os.linesep.join(['【更新】《哈利波特》石堧卜才新婚娶初戀今痠逝', '']) +FAILED_OUTPUT = os.linesep.join([ + "ftfy error:", + "This input couldn't be decoded as 'windows-1252'. We got the following error:", + "", + " 'charmap' codec can't decode byte 0x90 in position 5: character maps to ", + "", + "ftfy works best when its input is in a known encoding. You can use `ftfy -g`", + "to guess, if you're desperate. Otherwise, give the encoding name with the", + "`-e` option, such as `ftfy -e latin-1`.", + "", +]) def get_command_output(args, stdin=None): From 832651c0170fbb6e07d5f8030a340d0111b4cf1d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 15 May 2017 17:14:03 -0400 Subject: [PATCH 354/502] bump version to 5.1.1 and update the CHANGELOG --- CHANGELOG.md | 10 ++++++++++ ftfy/__init__.py | 2 +- setup.py | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 074d2ee6..29a340d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## Version 5.1.1 and 4.4.3 (May 15, 2017) + +These releases fix two unrelated problems with the tests, one in each version. + +- v5.1.1: fixed the CLI tests (which are new in v5) so that they pass + on Windows, as long as the Python output encoding is UTF-8. + +- v4.4.3: added the `# coding: utf-8` declaration to two files that were + missing it, so that tests can run on Python 2. + ## Version 5.1 (April 7, 2017) - Removed the dependency on `html5lib` by dropping support for Python 3.2. diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 960e74dc..293502dd 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -10,7 +10,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.1' +__version__ = '5.1.1' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/setup.py b/setup.py index f38cbe91..9ea786f7 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ The older version of ftfy, version 4.4, is still available and can run on Python 2. Try this: - pip install ftfy==4.4.2 + pip install ftfy==4.4.3 """ @@ -24,7 +24,7 @@ setup( name="ftfy", - version='5.1', + version='5.1.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From 11f7a2a14ce682de30d6e8588a910135246aeadb Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 22 Nov 2017 13:11:01 -0500 Subject: [PATCH 355/502] Prevent clearing a file by writing it while reading it (fixes #78) --- ftfy/cli.py | 10 ++++++++-- tests/test_cli.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ftfy/cli.py b/ftfy/cli.py index 39990285..d91a68fd 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -2,7 +2,7 @@ A command-line utility for fixing text found in a file. """ import sys -import io +import os from ftfy import fix_file, __version__ @@ -34,6 +34,9 @@ `-e` option, such as `ftfy -e latin-1`. """ +SAME_FILE_ERROR_TEXT = """ftfy error: +Can't read and write the same file. Please output to a new file instead. +""" def main(): """ @@ -79,7 +82,10 @@ def main(): if args.output == '-': outfile = sys.stdout else: - outfile = io.open(args.output, 'w', encoding='utf-8') + if os.path.abspath(args.output) == os.path.abspath(args.filename): + sys.stderr.write(SAME_FILE_ERROR_TEXT) + sys.exit(1) + outfile = open(args.output, 'w', encoding='utf-8') normalization = args.normalization if normalization.lower() == 'none': diff --git a/tests/test_cli.py b/tests/test_cli.py index bd887f22..dc829769 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -50,6 +50,13 @@ def test_wrong_encoding(): eq_(e.output.decode('utf-8'), FAILED_OUTPUT) +def test_same_file(): + with assert_raises(subprocess.CalledProcessError) as context: + get_command_output(['ftfy', TEST_FILENAME, '-o', TEST_FILENAME]) + e = context.exception + assert e.output.decode('utf-8').startswith("ftfy error:\nCan't read and write the same file.") + + def test_stdin(): with open(TEST_FILENAME, 'rb') as infile: output = get_command_output(['ftfy'], stdin=infile) From 6dcc0bb0d1c18b3ffd3d9a0e897cb891d69f55e6 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 22 Nov 2017 13:12:03 -0500 Subject: [PATCH 356/502] Add MODIFIER LETTER APOSTROPHE as a kind of single quote (fixes #73) --- ftfy/chardata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index ccb615a5..3a7460b6 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -117,7 +117,7 @@ def _build_utf8_punct_regex(): ) # These regexes match various Unicode variations on single and double quotes. -SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]') +SINGLE_QUOTE_RE = re.compile('[\u02bc\u2018-\u201b]') DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') From 223dc633338e0044b280210bc67ddf85e9c23745 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Wed, 22 Nov 2017 13:30:26 -0500 Subject: [PATCH 357/502] Replace MODIFIER LETTER APOSTROPHE with a single quote (including in a character that includes MODIFIER LETTER APOSTROPHE in its NFKC normalization) --- ftfy/chardata.py | 3 ++- tests/test_cases.json | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 3a7460b6..1111389a 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -187,7 +187,8 @@ def _build_control_char_mapping(): ord('ffi'): 'ffi', ord('ffl'): 'ffl', ord('ſt'): 'ſt', - ord('st'): 'st' + ord('st'): 'st', + ord('ʼn'): "ʼn", } diff --git a/tests/test_cases.json b/tests/test_cases.json index 30241b9c..9ccf5189 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -151,6 +151,13 @@ "fixed": "Arsenal v Wolfsburg: pre-season friendly – live!", "enabled": true }, + { + "label": "Handle Afrikaans 'n character", + "original": "ʼn Chloroplas is ʼn organel wat in fotosinterende plante voorkom.", + "fixed-encoding": "ʼn Chloroplas is ʼn organel wat in fotosinterende plante voorkom.", + "fixed": "'n Chloroplas is 'n organel wat in fotosinterende plante voorkom.", + "enabled": true + }, { "label": "Negative: 'è' preceded by a non-breaking space is not a small capital Y", "original": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", From 5b7b510995dfa8c7a030911daa112d7634524c73 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 27 Nov 2017 13:58:20 -0500 Subject: [PATCH 358/502] fix Serbian/Croatian digraphs along with ligatures --- ftfy/chardata.py | 31 ++++++++++++++++++++++++------- tests/test_cases.json | 7 +++++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 1111389a..8c70992d 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -173,22 +173,39 @@ def _build_control_char_mapping(): # A translate mapping that breaks ligatures made of Latin letters. While -# ligatures may be important to the representation of other languages, in -# Latin letters they tend to represent a copy/paste error. +# ligatures may be important to the representation of other languages, in Latin +# letters they tend to represent a copy/paste error. It omits ligatures such +# as æ that are frequently used intentionally. # -# Ligatures may also be separated by NFKC normalization, but that is sometimes -# more normalization than you want. +# This list additionally includes some Latin digraphs that represent two +# characters for legacy encoding reasons, not for typographical reasons. +# +# Ligatures and digraphs may also be separated by NFKC normalization, but that +# is sometimes more normalization than you want. + LIGATURES = { - ord('IJ'): 'IJ', + ord('IJ'): 'IJ', # Dutch ligatures ord('ij'): 'ij', - ord('ff'): 'ff', + ord('ʼn'): "ʼn", # Afrikaans digraph meant to avoid auto-curled quote + ord('DZ'): 'DZ', # Serbian/Croatian digraphs for Cyrillic conversion + ord('Dz'): 'Dz', + ord('dz'): 'dz', + ord('DŽ'): 'DŽ', + ord('Dž'): 'Dž', + ord('dž'): 'dž', + ord('LJ'): 'LJ', + ord('Lj'): 'Lj', + ord('lj'): 'lj', + ord('NJ'): 'NJ', + ord('Nj'): 'Nj', + ord('nj'): "nj", + ord('ff'): 'ff', # Latin typographical ligatures ord('fi'): 'fi', ord('fl'): 'fl', ord('ffi'): 'ffi', ord('ffl'): 'ffl', ord('ſt'): 'ſt', ord('st'): 'st', - ord('ʼn'): "ʼn", } diff --git a/tests/test_cases.json b/tests/test_cases.json index 9ccf5189..19e1aede 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -158,6 +158,13 @@ "fixed": "'n Chloroplas is 'n organel wat in fotosinterende plante voorkom.", "enabled": true }, + { + "label": "Handle Croatian single-codepoint digraphs", + "original": "izum „bootstrap load“ koji je korištenjem polisilicijskog sloja proizveo dovoljno dobre kondenzatore na čipu", + "fixed-encoding": "izum „bootstrap load“ koji je korištenjem polisilicijskog sloja proizveo dovoljno dobre kondenzatore na čipu", + "fixed": "izum \"bootstrap load\" koji je korištenjem polisilicijskog sloja proizveo dovoljno dobre kondenzatore na čipu", + "enabled": true + }, { "label": "Negative: 'è' preceded by a non-breaking space is not a small capital Y", "original": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", From 2de4b2aaf81aed1f3550bfa24aae3b2f37facf70 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 27 Nov 2017 16:58:33 -0500 Subject: [PATCH 359/502] Documentation fixes --- docs/conf.py | 6 +++--- ftfy/__init__.py | 2 +- ftfy/formatting.py | 8 +++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 4047e44b..f6d4b33a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ # General information about the project. project = u'ftfy' -copyright = u'2016, Rob Speer' +copyright = u'2017, Rob Speer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '4.1' +version = '5.2' # The full version, including alpha/beta/rc tags. -release = '4.1' +release = '5.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 293502dd..a4886151 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -10,7 +10,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.1.1' +__version__ = '5.2.0' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/ftfy/formatting.py b/ftfy/formatting.py index ea19f0c5..20d4c65f 100644 --- a/ftfy/formatting.py +++ b/ftfy/formatting.py @@ -31,7 +31,7 @@ def character_width(char): def monospaced_width(text): - """ + r""" Return the number of character cells that this string is likely to occupy when displayed in a monospaced, modern, Unicode-aware terminal emulator. We refer to this as the "display width" of the string. @@ -50,8 +50,10 @@ def monospaced_width(text): >>> monospaced_width('example\x80') -1 - # The Korean word 'ibnida' can be written with 3 characters or 7 jamo. - # Either way, it *looks* the same and takes up 6 character cells. + A more complex example: The Korean word 'ibnida' can be written with 3 + pre-composed characters or 7 jamo. Either way, it *looks* the same and + takes up 6 character cells. + >>> monospaced_width('입니다') 6 >>> monospaced_width('\u110b\u1175\u11b8\u1102\u1175\u1103\u1161') From a315b73f527e794f2f45c6c32aac5c0d7a6c9f5c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 27 Nov 2017 16:59:14 -0500 Subject: [PATCH 360/502] Bump version to 5.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9ea786f7..3fa81c66 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( name="ftfy", - version='5.1.1', + version='5.2.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From 680b4f86b026f37838223dbc09a635ea4e828f6e Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 1 Dec 2017 11:30:01 -0500 Subject: [PATCH 361/502] Update the changelog --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29a340d8..88f2e826 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +## Version 5.2 (November 27, 2017) + +- The command-line tool will not accept the same filename as its input + and output. (Previously, this would write a zero-length file.) + +- The `uncurl_quotes` fixer, which replaces curly quotes with straight quotes, + now also replaces MODIFIER LETTER APOSTROPHE. + +- Codepoints that contain two Latin characters crammed together for legacy + encoding reasons are replaced by those two separate characters, even in NFC + mode. We formerly did this just with ligatures such as `fi` and `IJ`, but now + this includes the Afrikaans digraph `ʼn` and Serbian/Croatian digraphs such as + `dž`. + + ## Version 5.1.1 and 4.4.3 (May 15, 2017) These releases fix two unrelated problems with the tests, one in each version. From 27e78b169034dd89455aa97c1ab26871bd393cfe Mon Sep 17 00:00:00 2001 From: Andrew Lin Date: Mon, 8 Jan 2018 11:37:14 -0500 Subject: [PATCH 362/502] Use os.path.realpath, not os.path.abspath, to detect output filename collision. This strengthens the protections introduced in 11f7a2 against symbolic links. --- ftfy/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/cli.py b/ftfy/cli.py index d91a68fd..3193466a 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -82,7 +82,7 @@ def main(): if args.output == '-': outfile = sys.stdout else: - if os.path.abspath(args.output) == os.path.abspath(args.filename): + if os.path.realpath(args.output) == os.path.realpath(args.filename): sys.stderr.write(SAME_FILE_ERROR_TEXT) sys.exit(1) outfile = open(args.output, 'w', encoding='utf-8') From 6c630ed3e022aebfb6e7ccb926aca0fd3f081c08 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Jan 2018 11:44:27 -0500 Subject: [PATCH 363/502] Add another heuristic to detect clear cases of mojibake --- ftfy/badness.py | 32 ++++++++++++++++++++++++++------ tests/test_cases.json | 12 ++++++++---- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 4fcadf32..ab3e1c9d 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -35,9 +35,6 @@ def _make_weirdness_regex(): """ groups = [] - # Match lowercase letters that are followed by non-ASCII uppercase letters - groups.append('lA') - # Match diacritical marks, except when they modify a non-cased letter or # another mark. # @@ -91,7 +88,7 @@ def _make_weirdness_regex(): groups.append('{cat1}[{others_range}]'.format( cat1=cat1, others_range=others_range )) - regex = '|'.join('({0})'.format(group) for group in groups) + regex = '|'.join(groups) return re.compile(regex) WEIRDNESS_RE = _make_weirdness_regex() @@ -116,6 +113,26 @@ def _make_weirdness_regex(): ']' ) +# These are sequences that are common mojibake, resulting from common encodings +# that are mixed up with UTF-8 on characters from their own character map. +# +# This helps to strengthen evidence that text should be fixed in a way that's +# separate from the character classes above, or to counter COMMON_SYMBOL_RE's +# fondness for characters such as inverted exclamation marks and multiplication +# signs in contexts where they really do look like mojibake. + +MOJIBAKE_SYMBOL_RE = re.compile( + # Mojibake of low-numbered characters from and ISO-8859-1, ISO-8859-2. + # This also covers some cases from related encodings such as Windows-1252 + # and Windows-1250. + '[ÂÃĂ][\x80-\xbf\u0100-\u02ff]|' + # ISO-8859-1 or Windows-1252 mojibake of characters U+10000 to U+1FFFF + 'ðŸ|' + # Windows-1252 or Windows-1250 mojibake of Windows punctuation characters + 'â€|' + # Windows-1251 mojibake of Windows punctuation characters + 'вЂ' +) def sequence_weirdness(text): """ @@ -146,8 +163,11 @@ def sequence_weirdness(text): """ text2 = unicodedata.normalize('NFC', text) weirdness = len(WEIRDNESS_RE.findall(chars_to_classes(text2))) - punct_discount = len(COMMON_SYMBOL_RE.findall(text2)) - return weirdness * 2 - punct_discount + adjustment = ( + len(MOJIBAKE_SYMBOL_RE.findall(text2)) * 2 - + len(COMMON_SYMBOL_RE.findall(text2)) + ) + return weirdness * 2 + adjustment def text_cost(text): diff --git a/tests/test_cases.json b/tests/test_cases.json index 19e1aede..951d2fe9 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -165,6 +165,12 @@ "fixed": "izum \"bootstrap load\" koji je korištenjem polisilicijskog sloja proizveo dovoljno dobre kondenzatore na čipu", "enabled": true }, + { + "label": "A with an accute accent, in isolation", + "original": "Nicolás", + "fixed": "Nicolás", + "enabled": true + }, { "label": "Negative: 'è' preceded by a non-breaking space is not a small capital Y", "original": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", @@ -315,10 +321,9 @@ }, { "label": "UTF-8 / Windows-1252 mixup in Gaelic involving non-breaking spaces", - "comment": "Too similar to the Finnish negative example with non-breaking spaces", "original": "CÃ\u00a0nan nan GÃ\u00a0idheal", "fixed": "Cànan nan Gàidheal", - "enabled": false + "enabled": true }, { "label": "Windows-1252 / MacRoman mixup in Spanish", @@ -329,10 +334,9 @@ }, { "label": "UTF-8 / Windows-1251 mixup in tweet spam", - "comment": "The Windows-1251-fixing plan is too conservative to fix this", "original": "Blog Traffic Tip 2 – Broadcast Email Your Blog", "fixed": "Blog Traffic Tip 2 – Broadcast Email Your Blog", - "enabled": false + "enabled": true }, { "label": "Negative: Leet line-art", From e286d061bc547b457ac08e77756dec0b33b5aa03 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Jan 2018 11:46:10 -0500 Subject: [PATCH 364/502] README updates --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index a2176b74..eac1a2e3 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ Full documentation: **http://ftfy.readthedocs.org** — [@planarrowspace](http://twitter.com/planarrowspace) - “A handy piece of magic” — [@simonw](http://twitter.com/simonw) +- “Saved me a large amount of frustrating dev work” + — [@iancal](http://twitter.com/iancal) - “ftfy did the right thing right away, with no faffing about. Excellent work, solving a very tricky real-world (whole-world!) problem.” — Brennan Young - “Hat mir die Tage geholfen. Im Übrigen bin ich der Meinung, dass wir keine komplexen Maschinen mit Computern bauen sollten solange wir nicht einmal Umlaute sicher verarbeiten können. :D” @@ -70,6 +72,10 @@ often shows up as characters that turn into nonsense sequences (called ftfy uses heuristics to detect and undo this kind of mojibake, with a very low rate of false positives. +This part of ftfy now has an unofficial Web implementation by simonw: +https://ftfy.now.sh/ + + ## Examples `fix_text` is the main function of ftfy. This section is meant to give you a From 51fb7c16a3448d90c446cdeb657983b60fa48796 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Jan 2018 11:51:34 -0500 Subject: [PATCH 365/502] Update version number and changelog for 5.3 --- CHANGELOG.md | 8 ++++++++ docs/conf.py | 4 ++-- ftfy/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88f2e826..7ddf2df6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Version 5.3 (January 25, 2018) + +- A heuristic has been too conservative since version 4.2, causing a regression + compared to previous versions: ftfy would fail to fix mojibake of common + characters such as `á` when seen in isolation. A new heuristic now makes it + possible to fix more of these common cases with less evidence. + + ## Version 5.2 (November 27, 2017) - The command-line tool will not accept the same filename as its input diff --git a/docs/conf.py b/docs/conf.py index f6d4b33a..2ec2868d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ # built documents. # # The short X.Y version. -version = '5.2' +version = '5.3' # The full version, including alpha/beta/rc tags. -release = '5.2' +release = '5.3.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/ftfy/__init__.py b/ftfy/__init__.py index a4886151..79f30991 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -10,7 +10,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.2.0' +__version__ = '5.3.0' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/setup.py b/setup.py index 3fa81c66..44ff7495 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( name="ftfy", - version='5.2.0', + version='5.3.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From d83e9e953b8e8c131040036b99e08065d2a8fbe9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Jan 2018 12:03:03 -0500 Subject: [PATCH 366/502] Tune the heuristic to be more careful and match its comments better --- ftfy/badness.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index ab3e1c9d..3b2d4529 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -122,12 +122,14 @@ def _make_weirdness_regex(): # signs in contexts where they really do look like mojibake. MOJIBAKE_SYMBOL_RE = re.compile( - # Mojibake of low-numbered characters from and ISO-8859-1, ISO-8859-2. - # This also covers some cases from related encodings such as Windows-1252 - # and Windows-1250. - '[ÂÃĂ][\x80-\xbf\u0100-\u02ff]|' - # ISO-8859-1 or Windows-1252 mojibake of characters U+10000 to U+1FFFF - 'ðŸ|' + # Mojibake of low-numbered characters from ISO-8859-1 and, in some cases, + # ISO-8859-2. This also covers some cases from related encodings such as + # Windows-1252 and Windows-1250. + '[ÂÃĂ][\x80-\xbf]|' + # ISO-8859-1, ISO-8859-2, or Windows-1252 mojibake of characters U+10000 + # to U+1FFFF. (The Windows-1250 and Windows-1251 versions might be too + # plausible.) + '[ðđ][Ÿ\x9f]|' # Windows-1252 or Windows-1250 mojibake of Windows punctuation characters 'â€|' # Windows-1251 mojibake of Windows punctuation characters From 83e46920bd651031ed890541044a954b53e6c3b8 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Jan 2018 12:22:55 -0500 Subject: [PATCH 367/502] Change the new heuristic to explicitly list characters to fix Also, add some new synthetic tests for borderline cases. The new potential false positive I've noticed isn't actually changed by the new heuristic, but I should take note anyway. --- ftfy/badness.py | 2 +- tests/test_cases.json | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 3b2d4529..2642c448 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -125,7 +125,7 @@ def _make_weirdness_regex(): # Mojibake of low-numbered characters from ISO-8859-1 and, in some cases, # ISO-8859-2. This also covers some cases from related encodings such as # Windows-1252 and Windows-1250. - '[ÂÃĂ][\x80-\xbf]|' + '[ÂÃĂ][\x80-\x9f€ƒ†‡ˆ‰Œ˜™œŸ¡¢£¤¥¦§¨ª¬¯°±²³µ¶·¸¹º¼½¾¿ˇ˘˝]|' # ISO-8859-1, ISO-8859-2, or Windows-1252 mojibake of characters U+10000 # to U+1FFFF. (The Windows-1250 and Windows-1251 versions might be too # plausible.) diff --git a/tests/test_cases.json b/tests/test_cases.json index 951d2fe9..a9f1d203 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -321,9 +321,10 @@ }, { "label": "UTF-8 / Windows-1252 mixup in Gaelic involving non-breaking spaces", + "comment": "Too similar to the Finnish negative example with non-breaking spaces", "original": "CÃ\u00a0nan nan GÃ\u00a0idheal", "fixed": "Cànan nan Gàidheal", - "enabled": true + "enabled": false }, { "label": "Windows-1252 / MacRoman mixup in Spanish", @@ -424,5 +425,18 @@ "fixed-encoding": "“lossy decoding�", "fixed": "\"lossy decoding�", "enabled": true + }, + { + "label": "Synthetic, negative: Romanian word before a non-breaking space", + "original": "NICIODATĂ\u00a0", + "fixed": "NICIODATĂ\u00a0", + "enabled": true + }, + { + "label": "Synthetic, false positive: the Sami name of a town in Lapland, in capital letters, followed by a non-breaking space", + "comment": "Related testing tells me I should worry about cases like this, though I haven't seen a real example", + "original": "SUOVÂKUOŠKÂ\u00a0", + "fixed": "SUOVÂKUOŠKÂ\u00a0", + "enabled": false } ] From 68800e7ce59806177a651a26a50d57486df2e475 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Jan 2018 14:08:59 -0500 Subject: [PATCH 368/502] be more careful in the Windows-1251 case; add borderline tests --- ftfy/badness.py | 4 ++-- tests/test_cases.json | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 2642c448..69ba9ba4 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -132,8 +132,8 @@ def _make_weirdness_regex(): '[ðđ][Ÿ\x9f]|' # Windows-1252 or Windows-1250 mojibake of Windows punctuation characters 'â€|' - # Windows-1251 mojibake of Windows punctuation characters - 'вЂ' + # Windows-1251 mojibake of some Windows punctuation characters + 'вЂ[љћ¦°№™ќ“”]' ) def sequence_weirdness(text): diff --git a/tests/test_cases.json b/tests/test_cases.json index a9f1d203..6a38cfd8 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -339,6 +339,13 @@ "fixed": "Blog Traffic Tip 2 – Broadcast Email Your Blog", "enabled": true }, + { + "label": "UTF-8 / Windows-1251 mixup", + "original": "S&P Confirms Ukrsotsbank’s “B-“ Rating", + "fixed-encoding": "S&P Confirms Ukrsotsbank’s “B-“ Rating", + "fixed": "S&P Confirms Ukrsotsbank's \"B-\" Rating", + "enabled": true + }, { "label": "Negative: Leet line-art", "comment": "Current false positive: it coincidentally all decodes as UTF-8/CP437 mojibake and becomes 'ôaſaſaſaſa'", @@ -438,5 +445,12 @@ "original": "SUOVÂKUOŠKÂ\u00a0", "fixed": "SUOVÂKUOŠKÂ\u00a0", "enabled": false + }, + { + "label": "Synthetic, negative: Camel-cased Serbian that looks like a UTF-8 / Windows-1251 mixup", + "comment": "I made this text up, but it seems like it means 'HelloDevil'. Could be a username or something.", + "original": "ПоздравЂаво", + "fixed": "ПоздравЂаво", + "enabled": true } ] From a6bf161091fb55da76ee58522341ca239ba04a41 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Jan 2018 14:10:29 -0500 Subject: [PATCH 369/502] spelling fix in test cases --- tests/test_cases.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cases.json b/tests/test_cases.json index 6a38cfd8..4ca97c40 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -166,7 +166,7 @@ "enabled": true }, { - "label": "A with an accute accent, in isolation", + "label": "A with an acute accent, in isolation", "original": "Nicolás", "fixed": "Nicolás", "enabled": true From 1d3c29ee11bb39252396f11bea77d05d8d88ac0c Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 25 Jan 2018 14:57:03 -0500 Subject: [PATCH 370/502] The test case about a false positive wasn't a real false positive Just to be clear, this has no real effect because the test case is disabled anyway. But the thing the test case warned would happen didn't happen on that text, and we need some slightly less probable text to make it happen. It would have an effect if disabled test cases said what they currently turned into, as Andrew recommended. --- tests/test_cases.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_cases.json b/tests/test_cases.json index 4ca97c40..5f152627 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -440,10 +440,10 @@ "enabled": true }, { - "label": "Synthetic, false positive: the Sami name of a town in Lapland, in capital letters, followed by a non-breaking space", - "comment": "Related testing tells me I should worry about cases like this, though I haven't seen a real example", - "original": "SUOVÂKUOŠKÂ\u00a0", - "fixed": "SUOVÂKUOŠKÂ\u00a0", + "label": "Synthetic, false positive: the title of a manga, in weird capitalized romaji, with a non-breaking space", + "comment": "Testing tells me I should worry about cases like this, though I haven't seen a real example. Searching for similar real text yields a lot of examples that actually come out fine.", + "original": "MISUTÂ\u00a0AJIKKO", + "fixed": "MISUTÂ\u00a0AJIKKO", "enabled": false }, { From ea8952d58c284650818c0effc411627083bd2c31 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 24 Apr 2018 10:42:53 -0400 Subject: [PATCH 371/502] Packaging updates for the new PyPI --- CHANGELOG.md | 5 +++++ setup.py | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ddf2df6..1fdfdd56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## Version 5.3.1 (April 24, 2018) + +Packaging updates; provides better metadata for the new PyPI. + + ## Version 5.3 (January 25, 2018) - A heuristic has been too conservative since version 4.2, causing a regression diff --git a/setup.py b/setup.py index 44ff7495..b5359aed 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,8 @@ pip install ftfy==4.4.3 """ +DESCRIPTION = open('README.md').read() + if sys.version_info[0] < 3: print(PY2_MESSAGE) @@ -24,16 +26,19 @@ setup( name="ftfy", - version='5.3.0', + version='5.3.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", url='http://github.com/LuminosoInsight/python-ftfy', platforms=["any"], description="Fixes some problems with Unicode text after the fact", + long_description=DESCRIPTION, + long_description_content_type='text/markdown', packages=['ftfy', 'ftfy.bad_codecs'], package_data={'ftfy': ['char_classes.dat']}, install_requires=['wcwidth'], + python_requires='>=3.3', classifiers=[ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", @@ -50,5 +55,8 @@ 'console_scripts': [ 'ftfy = ftfy.cli:main' ] + }, + project_urls={ + 'Documentation': 'http://ftfy.readthedocs.io', } ) From dac57a7c672df262ee81b8102d9a1aeb440568f0 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 3 May 2018 12:27:15 -0400 Subject: [PATCH 372/502] Port test cases to pytest --- .gitignore | 2 ++ pytest.ini | 2 ++ setup.cfg | 2 ++ setup.py | 2 ++ tests/test_bytes.py | 22 ++++++++-------- tests/test_characters.py | 17 ++++++------ tests/test_cli.py | 21 ++++++++------- tests/test_encodings.py | 8 +++--- tests/test_entities.py | 33 ++++++++++++------------ tests/test_examples_in_json.py | 40 ++++++++++++----------------- tests/test_futuristic_codepoints.py | 11 ++++---- 11 files changed, 77 insertions(+), 83 deletions(-) create mode 100644 pytest.ini create mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore index 6dc3e66a..934704b6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ dist build _build twitterlogs +.eggs +.pytest_cache diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..df3eb518 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --doctest-modules diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..b7e47898 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test=pytest diff --git a/setup.py b/setup.py index 44ff7495..14e6a5f6 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,8 @@ packages=['ftfy', 'ftfy.bad_codecs'], package_data={'ftfy': ['char_classes.dat']}, install_requires=['wcwidth'], + setup_requires=['pytest-runner'], + tests_require=['pytest'], classifiers=[ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", diff --git a/tests/test_bytes.py b/tests/test_bytes.py index fd5649c7..e5679c89 100644 --- a/tests/test_bytes.py +++ b/tests/test_bytes.py @@ -1,6 +1,6 @@ from ftfy import guess_bytes from ftfy.bad_codecs.utf8_variants import IncrementalDecoder -from nose.tools import eq_ +import pytest TEST_ENCODINGS = [ @@ -13,26 +13,24 @@ ] -def check_bytes_decoding(string): +@pytest.mark.parametrize("string", TEST_STRINGS) +def test_guess_bytes(string): for encoding in TEST_ENCODINGS: result_str, result_encoding = guess_bytes(string.encode(encoding)) - eq_(result_str, string) - eq_(result_encoding, encoding) + assert result_str == string + assert result_encoding == encoding if '\n' in string: old_mac_bytes = string.replace('\n', '\r').encode('macroman') result_str, result_encoding = guess_bytes(old_mac_bytes) - eq_(result_str, string.replace('\n', '\r')) + assert result_str == string.replace('\n', '\r') -def test_guess_bytes(): - for string in TEST_STRINGS: - yield check_bytes_decoding, string - +def test_guess_bytes_null(): bowdlerized_null = b'null\xc0\x80separated' result_str, result_encoding = guess_bytes(bowdlerized_null) - eq_(result_str, u'null\x00separated') - eq_(result_encoding, u'utf-8-variants') + assert result_str == u'null\x00separated' + assert result_encoding == u'utf-8-variants' def test_incomplete_sequences(): @@ -48,5 +46,5 @@ def test_incomplete_sequences(): decoder = IncrementalDecoder() got = decoder.decode(left, final=False) got += decoder.decode(right) - eq_(got, test_string) + assert got == test_string diff --git a/tests/test_characters.py b/tests/test_characters.py index 592134dc..5e868c7e 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -5,7 +5,6 @@ from ftfy.badness import sequence_weirdness import unicodedata import sys -from nose.tools import eq_ # Most single-character strings which have been misencoded should be restored. @@ -21,8 +20,8 @@ def test_bmp_characters(): garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') for garb in (garble, garble2): fixed, plan = fix_encoding_and_explain(garb) - eq_(fixed, char) - eq_(apply_plan(garb, plan), char) + assert fixed == char + assert apply_plan(garb, plan) == char def test_possible_encoding(): @@ -32,7 +31,7 @@ def test_possible_encoding(): def test_byte_order_mark(): - eq_(fix_encoding(''), '\ufeff') + assert fix_encoding('') == '\ufeff' def test_control_chars(): @@ -41,22 +40,22 @@ def test_control_chars(): "\u206aget standardized\U000E0065\U000E006E.\r\n" ) fixed = "Sometimes, bad ideas like these characters get standardized.\r\n" - eq_(remove_control_chars(text), fixed) + assert remove_control_chars(text) == fixed def test_emoji_variation_selector(): # The hearts here are explicitly marked as emoji using the variation # selector U+FE0F. This is not weird. - eq_(sequence_weirdness('❤\ufe0f' * 10), 0) + assert sequence_weirdness('❤\ufe0f' * 10) == 0 def test_emoji_skintone_selector(): # Dear heuristic, you can't call skin-tone selectors weird anymore. # We welcome Santa Clauses of all colors. - eq_(sequence_weirdness('🎅🏿🎅🏽🎅🏼🎅🏻'), 0) + assert sequence_weirdness('🎅🏿🎅🏽🎅🏼🎅🏻') == 0 def test_surrogates(): - eq_(fix_surrogates('\udbff\udfff'), '\U0010ffff') - eq_(fix_surrogates('\ud800\udc00'), '\U00010000') + assert fix_surrogates('\udbff\udfff') == '\U0010ffff' + assert fix_surrogates('\ud800\udc00') == '\U00010000' diff --git a/tests/test_cli.py b/tests/test_cli.py index dc829769..d1a0d2c6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,6 @@ import subprocess import os -from nose.tools import eq_, assert_raises +import pytest # Get the filename of 'halibote.txt', which contains some mojibake about @@ -27,38 +27,37 @@ def get_command_output(args, stdin=None): def test_basic(): output = get_command_output(['ftfy', TEST_FILENAME]) - eq_(output, CORRECT_OUTPUT) + assert output == CORRECT_OUTPUT def test_guess_bytes(): output = get_command_output(['ftfy', '-g', TEST_FILENAME]) - eq_(output, CORRECT_OUTPUT) + assert output == CORRECT_OUTPUT def test_alternate_encoding(): # The file isn't really in Windows-1252. But that's a problem ftfy # can fix, if it's allowed to be sloppy when reading the file. output = get_command_output(['ftfy', '-e', 'sloppy-windows-1252', TEST_FILENAME]) - eq_(output, CORRECT_OUTPUT) + assert output == CORRECT_OUTPUT def test_wrong_encoding(): # It's more of a problem when the file doesn't actually decode. - with assert_raises(subprocess.CalledProcessError) as context: + with pytest.raises(subprocess.CalledProcessError) as exception: get_command_output(['ftfy', '-e', 'windows-1252', TEST_FILENAME]) - e = context.exception - eq_(e.output.decode('utf-8'), FAILED_OUTPUT) + assert exception.value.output.decode('utf-8') == FAILED_OUTPUT def test_same_file(): - with assert_raises(subprocess.CalledProcessError) as context: + with pytest.raises(subprocess.CalledProcessError) as exception: get_command_output(['ftfy', TEST_FILENAME, '-o', TEST_FILENAME]) - e = context.exception - assert e.output.decode('utf-8').startswith("ftfy error:\nCan't read and write the same file.") + error = exception.value.output.decode('utf-8') + assert error.startswith("ftfy error:\nCan't read and write the same file.") def test_stdin(): with open(TEST_FILENAME, 'rb') as infile: output = get_command_output(['ftfy'], stdin=infile) - eq_(output, CORRECT_OUTPUT) + assert output == CORRECT_OUTPUT diff --git a/tests/test_encodings.py b/tests/test_encodings.py index ac24591d..2af573c7 100644 --- a/tests/test_encodings.py +++ b/tests/test_encodings.py @@ -1,15 +1,15 @@ from ftfy import bad_codecs, guess_bytes -from nose.tools import eq_ def test_cesu8(): - eq_(bad_codecs.search_function('cesu8').__class__, - bad_codecs.search_function('cesu-8').__class__) + cls1 = bad_codecs.search_function('cesu8').__class__ + cls2 = bad_codecs.search_function('cesu-8').__class__ + assert cls1 == cls2 test_bytes = (b'\xed\xa6\x9d\xed\xbd\xb7 is an unassigned character, ' b'and \xc0\x80 is null') test_text = '\U00077777 is an unassigned character, and \x00 is null' - eq_(test_bytes.decode('cesu8'), test_text) + assert test_bytes.decode('cesu8') == test_text def test_russian_crash(): diff --git a/tests/test_entities.py b/tests/test_entities.py index f67ead2d..0a2892b2 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -1,26 +1,25 @@ from ftfy import fix_text, fix_text_segment from ftfy.fixes import unescape_html -from nose.tools import eq_ def test_entities(): example = '&\n\n&' - eq_(fix_text(example), '&\n\n&') - eq_(fix_text_segment(example), '&\n\n&') + assert fix_text(example) == '&\n\n&' + assert fix_text_segment(example) == '&\n\n&' - eq_(fix_text(example, fix_entities=True), '&\n\n&') - eq_(fix_text_segment(example, fix_entities=True), '&\n\n&') + assert fix_text(example, fix_entities=True) == '&\n\n&' + assert fix_text_segment(example, fix_entities=True) == '&\n\n&' - eq_(fix_text(example, fix_entities=False), '&\n\n&') - eq_(fix_text_segment(example, fix_entities=False), '&\n\n&') + assert fix_text(example, fix_entities=False) == '&\n\n&' + assert fix_text_segment(example, fix_entities=False) == '&\n\n&' - eq_(fix_text_segment('<>', fix_entities=False), '<>') - eq_(fix_text_segment('<>', fix_entities=True), '<>') - eq_(fix_text_segment('<>'), '<>') - eq_(fix_text_segment('jednocześnie'), 'jednocześnie') - eq_(fix_text_segment('JEDNOCZEŚNIE'), 'JEDNOCZEŚNIE') - eq_(fix_text_segment('ellipsis…', normalization='NFKC'), 'ellipsis...') - eq_(fix_text_segment('ellipsis…', normalization='NFKC'), 'ellipsis...') - eq_(fix_text_segment('broken'), 'broken\x81') - eq_(unescape_html('euro €'), 'euro €') - eq_(unescape_html('not an entity x6;'), 'not an entity x6;') + assert fix_text_segment('<>', fix_entities=False) == '<>' + assert fix_text_segment('<>', fix_entities=True) == '<>' + assert fix_text_segment('<>') == '<>' + assert fix_text_segment('jednocześnie') == 'jednocześnie' + assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE' + assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...' + assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...' + assert fix_text_segment('broken') == 'broken\x81' + assert unescape_html('euro €') == 'euro €' + assert unescape_html('not an entity x6;') == 'not an entity x6;' diff --git a/tests/test_examples_in_json.py b/tests/test_examples_in_json.py index 69169162..6690a0ee 100644 --- a/tests/test_examples_in_json.py +++ b/tests/test_examples_in_json.py @@ -25,9 +25,9 @@ """ from ftfy import fix_text from ftfy.fixes import fix_encoding_and_explain, apply_plan -from nose.tools import eq_ import json import os +import pytest THIS_DIR = os.path.dirname(__file__) @@ -35,31 +35,23 @@ TEST_DATA = json.load(open(TEST_FILENAME, encoding='utf-8')) -def check_example(val): +@pytest.mark.parametrize("test_case", TEST_DATA) +def test_json_example(test_case): # Run one example from the data file - orig = val['original'] - fixed = val['fixed'] + if test_case['enabled']: + orig = test_case['original'] + fixed = test_case['fixed'] - # Make sure that the fix_encoding step outputs a plan that we can - # successfully run to reproduce its result - encoding_fix, plan = fix_encoding_and_explain(orig) - eq_(apply_plan(orig, plan), encoding_fix) + # Make sure that the fix_encoding step outputs a plan that we can + # successfully run to reproduce its result + encoding_fix, plan = fix_encoding_and_explain(orig) + assert apply_plan(orig, plan) == encoding_fix - # Make sure we can decode the text as intended - eq_(fix_text(orig), fixed) - eq_(encoding_fix, val.get('fixed-encoding', fixed)) + # Make sure we can decode the text as intended + assert fix_text(orig) == fixed + assert encoding_fix == test_case.get('fixed-encoding', fixed) - # Make sure we can decode as intended even with an extra layer of badness - extra_bad = orig.encode('utf-8').decode('latin-1') - eq_(fix_text(extra_bad), fixed) + # Make sure we can decode as intended even with an extra layer of badness + extra_bad = orig.encode('utf-8').decode('latin-1') + assert fix_text(extra_bad) == fixed - -def test_cases(): - # Run all the test cases in `test_cases.json` - for test_case in TEST_DATA: - if test_case['enabled']: - desc = "test_examples_in_json.check_example({!r})".format( - test_case['label'] - ) - check_example.description = desc - yield check_example, test_case diff --git a/tests/test_futuristic_codepoints.py b/tests/test_futuristic_codepoints.py index 3fa10fcc..e822d356 100644 --- a/tests/test_futuristic_codepoints.py +++ b/tests/test_futuristic_codepoints.py @@ -1,7 +1,6 @@ from ftfy.fixes import fix_encoding from ftfy.badness import sequence_weirdness import sys -from nose.tools import eq_ def test_unknown_emoji(): @@ -10,26 +9,26 @@ def test_unknown_emoji(): # a fortune cookie in Unicode 10.0: emoji_text = "\U0001f960 I see emoji in your future" emojibake = emoji_text.encode('utf-8').decode('windows-1252') - eq_(fix_encoding(emojibake), emoji_text) + assert fix_encoding(emojibake) == emoji_text # We believe enough in the future of this codepoint that we'll even # recognize it with a mangled byte A0 emojibake = emojibake.replace('\xa0', ' ') - eq_(fix_encoding(emojibake), emoji_text) + assert fix_encoding(emojibake) == emoji_text # Increment the first byte to get a very similar test case, but a # codepoint that will definitely not exist anytime soon. In this case, # we consider the existing text, "ñŸ¥\xa0", to be more probable. not_emoji = "\U0005f960 I see mojibake in your present".encode('utf-8').decode('windows-1252') - eq_(fix_encoding(not_emoji), not_emoji) + assert fix_encoding(not_emoji) == not_emoji def test_unicode_9(): # This string is 'bɪg'.upper() in Python 3.6 or later, containing the # new codepoint U+A7AE LATIN CAPITAL LETTER SMALL CAPITAL I. - eq_(sequence_weirdness("BꞮG"), 0) + assert sequence_weirdness("BꞮG") == 0 # That should be less weird than having a definitely-unassigned character # in the string. - eq_(sequence_weirdness("B\U00090000G"), 2) + assert sequence_weirdness("B\U00090000G") == 2 From 8154a57c2f6f17316f6068082dd63679ddf4e810 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 3 May 2018 13:23:05 -0400 Subject: [PATCH 373/502] fix obsolete unicode literals --- tests/test_bytes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_bytes.py b/tests/test_bytes.py index e5679c89..f141846d 100644 --- a/tests/test_bytes.py +++ b/tests/test_bytes.py @@ -29,8 +29,8 @@ def test_guess_bytes(string): def test_guess_bytes_null(): bowdlerized_null = b'null\xc0\x80separated' result_str, result_encoding = guess_bytes(bowdlerized_null) - assert result_str == u'null\x00separated' - assert result_encoding == u'utf-8-variants' + assert result_str == 'null\x00separated' + assert result_encoding == 'utf-8-variants' def test_incomplete_sequences(): From 0b0d4d4518d84a530a7dab491a05dc60c409a6bd Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 3 May 2018 16:32:25 -0400 Subject: [PATCH 374/502] replace enabled=False test cases with expected failures Now we can find out if something changes to make a test case unexpectedly pass. --- tests/test_cases.json | 134 ++++++++++++++++----------------- tests/test_examples_in_json.py | 51 +++++++++---- 2 files changed, 102 insertions(+), 83 deletions(-) diff --git a/tests/test_cases.json b/tests/test_cases.json index 5f152627..5e29caf7 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -3,221 +3,221 @@ "label": "Low-codepoint emoji", "original": "He's Justinâ\u009d¤", "fixed": "He's Justin❤", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / MacRoman mix-up about smurfs", "original": "Le Schtroumpf Docteur conseille g√¢teaux et baies schtroumpfantes pour un r√©gime √©quilibr√©.", "fixed": "Le Schtroumpf Docteur conseille gâteaux et baies schtroumpfantes pour un régime équilibré.", - "enabled": true + "expect": "pass" }, { "label": "Checkmark that almost looks okay as mojibake", "original": "✔ No problems", "fixed": "✔ No problems", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / Windows-1251 Russian mixup about futbol", "original": "РґРѕСЂРѕРіРµ Р\u0098Р·-РїРѕРґ #футбол", "fixed": "дороге Из-под #футбол", - "enabled": true + "expect": "pass" }, { "label": "Latin-1 / Windows-1252 mixup in German", "original": "\u0084Handwerk bringt dich überall hin\u0093: Von der YOU bis nach Monaco", "fixed-encoding": "„Handwerk bringt dich überall hin“: Von der YOU bis nach Monaco", "fixed": "\"Handwerk bringt dich überall hin\": Von der YOU bis nach Monaco", - "enabled": true + "expect": "pass" }, { "label": "CESU-8 / Windows-1252 emoji", "original": "Hi guys í ½í¸\u008d", "fixed": "Hi guys 😍", - "enabled": true + "expect": "pass" }, { "label": "CESU-8 / Latin-1 emoji", "original": "hihi RT username: â\u0098ºí ½í¸\u0098", "fixed": "hihi RT username: ☺😘", - "enabled": true + "expect": "pass" }, { "label": "Latin-1 / Windows-1252 mixup in Turkish", "original": "Beta Haber: Hırsızı Büyü Korkuttu", "fixed": "Beta Haber: Hırsızı Büyü Korkuttu", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / Windows-1251 mixed up twice in Russian", "original": "приятности. РІСњВ¤", "fixed": "приятности. ❤", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / Windows-1252 mixed up twice in Malay", "original": "Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New “ Romanceâ€Â\u009d.", "fixed-encoding": "Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New “ Romance”.", "fixed": "Kayanya laptopku error deh, soalnya tiap mau ngetik deket-deket kamu font yg keluar selalu Times New \" Romance\".", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / Windows-1252 mixed up twice in naming Iggy Pop", "original": "Iggy Pop (né Jim Osterberg)", "fixed": "Iggy Pop (né Jim Osterberg)", - "enabled": true + "expect": "pass" }, { "label": "Left quote is UTF-8, right quote is Latin-1, both encoded in Windows-1252", "original": "Direzione Pd, ok â\u0080\u009csenza modifiche\u0094 all'Italicum.", "fixed-encoding": "Direzione Pd, ok “senza modifiche” all'Italicum.", "fixed": "Direzione Pd, ok \"senza modifiche\" all'Italicum.", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / sloppy Windows-1252 mixed up twice in a triumphant emoticon", "original": "selamat berpuasa sob (Ã\u00a0¸‡'̀⌣'ÃŒÂ\u0081)Ã\u00a0¸‡", "fixed": "selamat berpuasa sob (ง'̀⌣'́)ง", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / Windows-1252 mixed up three times", "original": "The Mona Lisa doesn’t have eyebrows.", "fixed-encoding": "The Mona Lisa doesn’t have eyebrows.", "fixed": "The Mona Lisa doesn't have eyebrows.", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / Codepage 437 mixup in Russian", "original": "#╨┐╤Ç╨░╨▓╨╕╨╗╤î╨╜╨╛╨╡╨┐╨╕╤é╨░╨╜╨╕╨╡", "fixed": "#правильноепитание", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / Windows-1250 mixup in French", "original": "Liège Avenue de l'HĂ´pital", "fixed": "Liège Avenue de l'Hôpital", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / sloppy Windows-1250 mixup in English", "original": "It was named „scars´ stones“ after the rock-climbers who got hurt while climbing on it.", "fixed-encoding": "It was named\u00a0„scars´ stones“ after the rock-climbers who got hurt while climbing on it.", "fixed": "It was named\u00a0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it.", - "enabled": true + "expect": "pass" }, { "label": "The same text as above, but as a UTF-8 / ISO-8859-2 mixup", "original": "It was namedÂ\u00a0â\u0080\u009escars´ stonesâ\u0080\u009c after the rock-climbers who got hurt while climbing on it.", "fixed-encoding": "It was named\u00a0„scars´ stones“ after the rock-climbers who got hurt while climbing on it.", "fixed": "It was named\u00a0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it.", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / sloppy Windows-1250 mixup in Romanian", "original": "vedere Ă®nceĹŁoĹźatÄ\u0083", "fixed": "vedere înceţoşată", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / Windows-1250 mixup in Slovak", "original": "NapĂ\u00adšte nám !", "fixed": "Napíšte nám !", - "enabled": true + "expect": "pass" }, { "label": "A face with UTF-8 / sloppy Windows-1252 mixed up twice", "original": "ââ€\u009d’(⌣˛⌣)ââ€\u009dŽ", "fixed": "┒(⌣˛⌣)┎", - "enabled": true + "expect": "pass" }, { "label": "We can mostly decode the face above when we lose the character U+009D", "original": "ââ€�’(⌣˛⌣)ââ€�Ž", "fixed": "�(⌣˛⌣)�", - "enabled": true + "expect": "pass" }, { "label": "CESU-8 / Latin-1 mixup over several emoji", "comment": "You tried", "original": "I just figured out how to tweet emojis! â\u009a½í\u00a0½í¸\u0080í\u00a0½í¸\u0081í\u00a0½í¸\u0082í\u00a0½í¸\u0086í\u00a0½í¸\u008eí\u00a0½í¸\u008eí\u00a0½í¸\u008eí\u00a0½í¸\u008e", "fixed": "I just figured out how to tweet emojis! ⚽😀😁😂😆😎😎😎😎", - "enabled": true + "expect": "pass" }, { "label": "Two levels of inconsistent mojibake", "comment": "The en-dash was mojibaked in UTF-8 / Windows-1252 as three characters, two of which were mojibaked again as Windows-1252 / Latin-1, and the third of which was mojibaked as UTF-8 / Latin-1", "original": "Arsenal v Wolfsburg: pre-season friendly â\u0080â\u0080\u009c live!", "fixed": "Arsenal v Wolfsburg: pre-season friendly – live!", - "enabled": true + "expect": "pass" }, { "label": "Handle Afrikaans 'n character", "original": "ʼn Chloroplas is ʼn organel wat in fotosinterende plante voorkom.", "fixed-encoding": "ʼn Chloroplas is ʼn organel wat in fotosinterende plante voorkom.", "fixed": "'n Chloroplas is 'n organel wat in fotosinterende plante voorkom.", - "enabled": true + "expect": "pass" }, { "label": "Handle Croatian single-codepoint digraphs", "original": "izum „bootstrap load“ koji je korištenjem polisilicijskog sloja proizveo dovoljno dobre kondenzatore na čipu", "fixed-encoding": "izum „bootstrap load“ koji je korištenjem polisilicijskog sloja proizveo dovoljno dobre kondenzatore na čipu", "fixed": "izum \"bootstrap load\" koji je korištenjem polisilicijskog sloja proizveo dovoljno dobre kondenzatore na čipu", - "enabled": true + "expect": "pass" }, { "label": "A with an acute accent, in isolation", "original": "Nicolás", "fixed": "Nicolás", - "enabled": true + "expect": "pass" }, { "label": "Negative: 'è' preceded by a non-breaking space is not a small capital Y", "original": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", "fixed": "Con il corpo e lo spirito ammaccato,\u00a0è come se nel cuore avessi un vetro conficcato.", - "enabled": true + "expect": "pass" }, { "label": "Negative: multiplication sign and ellipsis", "comment": "Should not turn into a dot below", "original": "4288×…", "fixed": "4288×…", - "enabled": true + "expect": "pass" }, { "label": "Negative: accents are sometimes used as quotes", "comment": "The CESU-8 decoder will try to decode this, and then fail when it hits the end of the string", "original": "``toda produzida pronta pra assa aí´´", "fixed": "``toda produzida pronta pra assa aí´´", - "enabled": true + "expect": "pass" }, { "label": "Negative: 'Õ' followed by an ellipsis", "comment": "Should not turn into the Armenian letter Յ", "original": "HUHLL Õ…", "fixed": "HUHLL Õ…", - "enabled": true + "expect": "pass" }, { "label": "Negative: 'Ê' followed by an ellipsis", "comment": "Should not turn into a squat reversed esh", "original": "RETWEET SE VOCÊ…", "fixed": "RETWEET SE VOCÊ…", - "enabled": true + "expect": "pass" }, { "label": "Negative: 'É' followed by an ellipsis", "comment": "Should not turn into 'MARQUɅ'", "original": "PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…", "fixed": "PARCE QUE SUR LEURS PLAQUES IL Y MARQUÉ…", - "enabled": true + "expect": "pass" }, { "label": "Negative: 'Ó' followed by an ellipsis", "comment": "Should not turn into 'SӅ'", "original": "TEM QUE SEGUIR, SDV SÓ…", "fixed": "TEM QUE SEGUIR, SDV SÓ…", - "enabled": true + "expect": "pass" }, { "label": "Negative: 'É' followed by a curly apostrophe", @@ -225,7 +225,7 @@ "original": "Join ZZAJÉ’s Official Fan List and receive news, events, and more!", "fixed-encoding": "Join ZZAJÉ’s Official Fan List and receive news, events, and more!", "fixed": "Join ZZAJÉ's Official Fan List and receive news, events, and more!", - "enabled": true + "expect": "pass" }, { "label": "Negative: 'é' preceded by curly apostrophe", @@ -233,83 +233,83 @@ "original": "L’épisode 8 est trop fou ouahh", "fixed-encoding": "L’épisode 8 est trop fou ouahh", "fixed": "L'épisode 8 est trop fou ouahh", - "enabled": true + "expect": "pass" }, { "label": "Negative: three raised eyebrows or something?", "comment": "Should not turn into private use character U+F659", "original": "Ôôô VIDA MINHA", "fixed": "Ôôô VIDA MINHA", - "enabled": true + "expect": "pass" }, { "label": "Negative: copyright sign preceded by non-breaking space", "comment": "Should not turn into 'ʩ'", "original": "[x]\u00a0©", "fixed": "[x]\u00a0©", - "enabled": true + "expect": "pass" }, { "label": "Negative: en dash and infinity sign", "comment": "Should not turn into '2012Ѱ'", "original": "2012—∞", "fixed": "2012—∞", - "enabled": true + "expect": "pass" }, { "label": "Negative: This Е is a Ukrainian letter, but nothing else is wrong", "original": "SENSЕ - Oleg Tsedryk", "fixed": "SENSЕ - Oleg Tsedryk", - "enabled": true + "expect": "pass" }, { "label": "Negative: angry face", "comment": "The face should not turn into '`«'", "original": "OK??:( `¬´ ):", "fixed": "OK??:( `¬´ ):", - "enabled": true + "expect": "pass" }, { "label": "Negative: triangle and degree sign", "comment": "I'm not really sure what it *is* supposed to be, but it's not 'ơ'", "original": "∆°", "fixed": "∆°", - "enabled": true + "expect": "pass" }, { "label": "Negative: Portuguese with inverted question mark", "comment": "Former false positive - it should not turn into 'QUEM ɿ'", "original": "ESSE CARA AI QUEM É¿", "fixed": "ESSE CARA AI QUEM É¿", - "enabled": true + "expect": "pass" }, { "label": "Negative: Portuguese with acute accents as quotation marks", "comment": "Former false positive - the end should not turn into a superscript H", "original": "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", "fixed": "``hogwarts nao existe, voce nao vai pegar o trem pra lá´´", - "enabled": true + "expect": "pass" }, { "label": "Negative: Finnish Ä followed by a non-breaking space", "comment": "Former false positive - should not become a G with a dot", "original": "SELKÄ\u00a0EDELLÄ\u00a0MAAHAN via @YouTube", "fixed": "SELKÄ\u00a0EDELLÄ\u00a0MAAHAN via @YouTube", - "enabled": true + "expect": "pass" }, { "label": "Negative: multiplying by currency", "comment": "Former false positive - should not become the Hebrew letter 'final pe'", "original": "Offering 5×£35 pin ups", "fixed": "Offering 5×£35 pin ups", - "enabled": true + "expect": "pass" }, { "label": "Negative: registered chocolate brand name", "comment": "Former false positive - should not become the IPA letter 'lezh'", "original": "NESTLÉ® requiere contratar personal para diferentes areas a nivel nacional e internacional", "fixed": "NESTLÉ® requiere contratar personal para diferentes areas a nivel nacional e internacional", - "enabled": true + "expect": "pass" }, { @@ -317,41 +317,41 @@ "comment": "Unfortunately, the difference doesn't look convincing to ftfy", "original": "Engkau masih yg terindah, indah di dalam hatiku♫~", "fixed": "Engkau masih yg terindah, indah di dalam hatiku♫~", - "enabled": false + "expect": "fail" }, { "label": "UTF-8 / Windows-1252 mixup in Gaelic involving non-breaking spaces", "comment": "Too similar to the Finnish negative example with non-breaking spaces", "original": "CÃ\u00a0nan nan GÃ\u00a0idheal", "fixed": "Cànan nan Gàidheal", - "enabled": false + "expect": "fail" }, { "label": "Windows-1252 / MacRoman mixup in Spanish", "comment": "Requires something like encoding detection", "original": "Deja dos heridos hundimiento de barco tur\u0092stico en Acapulco.", "fixed": "Deja dos heridos hundimiento de barco turístico en Acapulco.", - "enabled": false + "expect": "fail" }, { "label": "UTF-8 / Windows-1251 mixup in tweet spam", "original": "Blog Traffic Tip 2 – Broadcast Email Your Blog", "fixed": "Blog Traffic Tip 2 – Broadcast Email Your Blog", - "enabled": true + "expect": "pass" }, { "label": "UTF-8 / Windows-1251 mixup", "original": "S&P Confirms Ukrsotsbank’s “B-“ Rating", "fixed-encoding": "S&P Confirms Ukrsotsbank’s “B-“ Rating", "fixed": "S&P Confirms Ukrsotsbank's \"B-\" Rating", - "enabled": true + "expect": "pass" }, { "label": "Negative: Leet line-art", "comment": "Current false positive: it coincidentally all decodes as UTF-8/CP437 mojibake and becomes 'ôaſaſaſaſa'", "original": "├┤a┼┐a┼┐a┼┐a┼┐a", "fixed": "├┤a┼┐a┼┐a┼┐a┼┐a", - "enabled": false + "expect": "fail" }, { @@ -360,21 +360,21 @@ "original": "I'm not such a fan of Charlotte Brontë…”", "fixed-encoding": "I'm not such a fan of Charlotte Brontë…”", "fixed": "I'm not such a fan of Charlotte Brontë…\"", - "enabled": true + "expect": "pass" }, { "label": "Synthetic, negative: hypothetical Swedish product name", "comment": "This used to be a constructed example of a false positive, until you added another symbol", "original": "AHÅ™, the new sofa from IKEA", "fixed": "AHÅ™, the new sofa from IKEA", - "enabled": true + "expect": "pass" }, { "label": "Synthetic, negative: Ukrainian capital letters", "comment": "We need to fix Windows-1251 conservatively, or else this decodes as '²ʲ'", "original": "ВІКІ is Ukrainian for WIKI", "fixed": "ВІКІ is Ukrainian for WIKI", - "enabled": true + "expect": "pass" }, { "label": "Synthetic, negative: don't leak our internal use of byte 0x1A", @@ -382,7 +382,7 @@ "original": "These control characters \u001a are apparently intentional \u0081", "fixed-encoding": "These control characters \u001a are apparently intentional \u0081", "fixed": "These control characters are apparently intentional \u0081", - "enabled": true + "expect": "pass" }, { "label": "Synthetic, negative: U+1A on its own", @@ -390,67 +390,67 @@ "original": "Here's a control character: \u001a", "fixed-encoding": "Here's a control character: \u001a", "fixed": "Here's a control character: ", - "enabled": true + "expect": "pass" }, { "label": "Synthetic, negative: A-with-circle as an Angstrom sign", "comment": "Should not turn into '10 ŗ'", "original": "a radius of 10 Å—", "fixed": "a radius of 10 Å—", - "enabled": true + "expect": "pass" }, { "label": "Synthetic, negative: Spanish with exclamation points on the wrong sides", "original": "!YO SÉ¡", "fixed": "!YO SÉ¡", - "enabled": true + "expect": "pass" }, { "label": "Synthetic: fix text with backslashes in it", "comment": "Tests for a regression on a long-ago bug", "original": "<40\\% vs \u00e2\u0089\u00a540\\%", "fixed": "<40\\% vs ≥40\\%", - "enabled": true + "expect": "pass" }, { "label": "Synthetic: curly quotes with mismatched encoding glitches in Latin-1", "original": "\u00e2\u0080\u009cmismatched quotes\u0085\u0094", "fixed-encoding": "“mismatched quotes…”", "fixed": "\"mismatched quotes…\"", - "enabled": true + "expect": "pass" }, { "label": "Synthetic: curly quotes with mismatched encoding glitches in Windows-1252", "original": "“mismatched quotes…”", "fixed-encoding": "“mismatched quotes…”", "fixed": "\"mismatched quotes…\"", - "enabled": true + "expect": "pass" }, { "label": "Synthetic: lossy decoding in sloppy-windows-1252", "original": "“lossy decodingâ€�", "fixed-encoding": "“lossy decoding�", "fixed": "\"lossy decoding�", - "enabled": true + "expect": "pass" }, { "label": "Synthetic, negative: Romanian word before a non-breaking space", "original": "NICIODATĂ\u00a0", "fixed": "NICIODATĂ\u00a0", - "enabled": true + "expect": "pass" }, { "label": "Synthetic, false positive: the title of a manga, in weird capitalized romaji, with a non-breaking space", "comment": "Testing tells me I should worry about cases like this, though I haven't seen a real example. Searching for similar real text yields a lot of examples that actually come out fine.", "original": "MISUTÂ\u00a0AJIKKO", "fixed": "MISUTÂ\u00a0AJIKKO", - "enabled": false + "expect": "fail" }, { "label": "Synthetic, negative: Camel-cased Serbian that looks like a UTF-8 / Windows-1251 mixup", "comment": "I made this text up, but it seems like it means 'HelloDevil'. Could be a username or something.", "original": "ПоздравЂаво", "fixed": "ПоздравЂаво", - "enabled": true + "expect": "pass" } ] diff --git a/tests/test_examples_in_json.py b/tests/test_examples_in_json.py index 6690a0ee..0a5a104f 100644 --- a/tests/test_examples_in_json.py +++ b/tests/test_examples_in_json.py @@ -34,24 +34,43 @@ TEST_FILENAME = os.path.join(THIS_DIR, 'test_cases.json') TEST_DATA = json.load(open(TEST_FILENAME, encoding='utf-8')) +TESTS_THAT_PASS = [test for test in TEST_DATA if test['expect'] == 'pass'] +TESTS_THAT_FAIL = [test for test in TEST_DATA if test['expect'] == 'fail'] + @pytest.mark.parametrize("test_case", TEST_DATA) +def test_well_formed_example(test_case): + assert test_case['expect'] in ('pass', 'fail') + + +@pytest.mark.parametrize("test_case", TESTS_THAT_PASS) def test_json_example(test_case): # Run one example from the data file - if test_case['enabled']: - orig = test_case['original'] - fixed = test_case['fixed'] - - # Make sure that the fix_encoding step outputs a plan that we can - # successfully run to reproduce its result - encoding_fix, plan = fix_encoding_and_explain(orig) - assert apply_plan(orig, plan) == encoding_fix - - # Make sure we can decode the text as intended - assert fix_text(orig) == fixed - assert encoding_fix == test_case.get('fixed-encoding', fixed) - - # Make sure we can decode as intended even with an extra layer of badness - extra_bad = orig.encode('utf-8').decode('latin-1') - assert fix_text(extra_bad) == fixed + orig = test_case['original'] + fixed = test_case['fixed'] + + # Make sure that the fix_encoding step outputs a plan that we can + # successfully run to reproduce its result + encoding_fix, plan = fix_encoding_and_explain(orig) + assert apply_plan(orig, plan) == encoding_fix + + # Make sure we can decode the text as intended + assert fix_text(orig) == fixed + assert encoding_fix == test_case.get('fixed-encoding', fixed) + + # Make sure we can decode as intended even with an extra layer of badness + extra_bad = orig.encode('utf-8').decode('latin-1') + assert fix_text(extra_bad) == fixed + + +@pytest.mark.parametrize("test_case", TESTS_THAT_FAIL) +@pytest.mark.xfail(strict=True) +def test_failing_json_example(test_case): + # Run an example from the data file that we believe will fail, due to + # ftfy's heuristic being insufficient + orig = test_case['original'] + fixed = test_case['fixed'] + + encoding_fix, plan = fix_encoding_and_explain(orig) + assert encoding_fix == test_case.get('fixed-encoding', fixed) From 7bc9a701051efdc722a39f5abc2c07746673b180 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 31 May 2018 16:45:00 -0400 Subject: [PATCH 375/502] Don't be quite so afraid of changing quotation marks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixing two-character mojibake sequences is a careful balance between false positives and false negatives. This balance is maintained in badness.py by `COMMON_SYMBOL_RE`, which lowers the badness of text containing certain symbols, and `MOJIBAKE_SYMBOL_RE`, which raises the badness in sequences that are very likely to be mojibake. The badness was never raised again on sequences containing quotation marks and a few other symbols, because I was overly cautious when adding this rule. But issue #97 shows that this prevents fixing the most common mojibake of words containing 'ë' or 'û'. I've added these characters to the list of characters that are probably mojibake if they appear after Â, Ã, or Ă: U+201A ‚ SINGLE LOW-9 QUOTATION MARK U+201E „ DOUBLE LOW-9 QUOTATION MARK U+2039 ‹ SINGLE LEFT-POINTING ANGLE QUOTATION MARK U+2018 ‘ LEFT SINGLE QUOTATION MARK U+201C “ LEFT DOUBLE QUOTATION MARK U+201D ” RIGHT DOUBLE QUOTATION MARK U+2022 • BULLET U+203A › SINGLE RIGHT-POINTING ANGLE QUOTATION MARK U+00AB « LEFT-POINTING DOUBLE ANGLE QUOTATION MARK U+00BB » RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK U+00A9 © COPYRIGHT SIGN But I'm still leaving out these, which are too much risk for too little gain: U+2019 ’ RIGHT SINGLE QUOTATION MARK U+00AE ® REGISTERED SIGN Some characters have also been moved into a rule that insists on a word-like character (`\w`) afterward, to make extra sure it's not messing with normal punctuation. See the new test cases for some cases that fall near the boundary of what should be fixed. --- ftfy/badness.py | 5 +++- tests/test_cases.json | 58 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 69ba9ba4..3b1ca69d 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -125,7 +125,10 @@ def _make_weirdness_regex(): # Mojibake of low-numbered characters from ISO-8859-1 and, in some cases, # ISO-8859-2. This also covers some cases from related encodings such as # Windows-1252 and Windows-1250. - '[ÂÃĂ][\x80-\x9f€ƒ†‡ˆ‰Œ˜™œŸ¡¢£¤¥¦§¨ª¬¯°±²³µ¶·¸¹º¼½¾¿ˇ˘˝]|' + '[ÂÃĂ][\x80-\x9f€ƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¢£¤¥¦§¨ª«¬¯°±²³µ¶·¸¹º¼½¾¿ˇ˘˝]|' + # Characters we have to be a little more cautious about if they're at + # the end of a word, but totally okay to fix in the middle + '[ÂÃĂ][›»‘”©™]\w|' # ISO-8859-1, ISO-8859-2, or Windows-1252 mojibake of characters U+10000 # to U+1FFFF. (The Windows-1250 and Windows-1251 versions might be too # plausible.) diff --git a/tests/test_cases.json b/tests/test_cases.json index 5e29caf7..f25614de 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -346,6 +346,20 @@ "fixed": "S&P Confirms Ukrsotsbank's \"B-\" Rating", "expect": "pass" }, + { + "label": "Dutch example with ë", + "comment": "from issue reported by MicroJackson", + "original": "ongeëvenaard", + "fixed-encoding": "ongeëvenaard", + "fixed": "ongeëvenaard", + "expect": "pass" + }, + { + "label": "Negative: Indonesian leetspeak", + "original": "MÄ£ÄM ÌÑÌ Q £ÄGÌ GÄLÄW ÑÍCH SÖÄ£ ÑÝÄ $ÚÄMÌ Q £ÄGÌ GÄK ÉÑÄK BÄDÄÑ....?????????, ......JÄDÍ...", + "fixed": "MÄ£ÄM ÌÑÌ Q £ÄGÌ GÄLÄW ÑÍCH SÖÄ£ ÑÝÄ $ÚÄMÌ Q £ÄGÌ GÄK ÉÑÄK BÄDÄÑ....?????????, ......JÄDÍ...", + "expect": "pass" + }, { "label": "Negative: Leet line-art", "comment": "Current false positive: it coincidentally all decodes as UTF-8/CP437 mojibake and becomes 'ôaſaſaſaſa'", @@ -433,12 +447,49 @@ "fixed": "\"lossy decoding�", "expect": "pass" }, + { + "label": "Synthetic: French word for August in windows-1252", + "original": "août", + "fixed-encoding": "août", + "fixed": "août", + "expect": "pass" + }, + { + "label": "Synthetic: French word for hotel in all-caps windows-1252", + "original": "HÔTEL", + "fixed-encoding": "HÔTEL", + "fixed": "HÔTEL", + "expect": "pass" + }, + { + "label": "Synthetic: Scottish Gaelic word for 'subject' in all-caps windows-1252", + "original": "CÙIS", + "fixed-encoding": "CÙIS", + "fixed": "CÙIS", + "expect": "pass" + }, { "label": "Synthetic, negative: Romanian word before a non-breaking space", + "comment": "The word literally means 'not even once', which might be a good recommendation about fixing Romanian mojibake", "original": "NICIODATĂ\u00a0", "fixed": "NICIODATĂ\u00a0", "expect": "pass" }, + { + "label": "Synthetic, negative: Be careful around curly apostrophes", + "comment": "It shouldn't end up saying 'a lot of Òs'", + "original": "There are a lot of Ã’s in mojibake text", + "fixed-encoding": "There are a lot of Ã’s in mojibake text", + "fixed": "There are a lot of Ã's in mojibake text", + "expect": "pass" + }, + { + "label": "Synthetic, negative: Romanian word before a trademark sign", + "comment": "We would change 'DATÙ' to 'DATÙ', but getting windows-1250 involved makes it better to leave alone", + "original": "NICIODATĂ™", + "fixed": "NICIODATĂ™", + "expect": "pass" + }, { "label": "Synthetic, false positive: the title of a manga, in weird capitalized romaji, with a non-breaking space", "comment": "Testing tells me I should worry about cases like this, though I haven't seen a real example. Searching for similar real text yields a lot of examples that actually come out fine.", @@ -452,5 +503,12 @@ "original": "ПоздравЂаво", "fixed": "ПоздравЂаво", "expect": "pass" + }, + { + "label": "Synthetic, false negative: mojibake with trademark sign at the end of a word", + "comment": "I recall the correct version of this text from a sign in the movie Amélie. Unfortunately, we can't help her twin Amélie, who makes mojibaked signs.", + "original": "OÙ ET QUAND?", + "fixed": "OÙ ET QUAND?", + "expect": "fail" } ] From 95fd6c54b5a80866a1a71672d37fb56b86f19d45 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 1 Jun 2018 10:59:08 -0400 Subject: [PATCH 376/502] update the changelog and absorb the previous un-merged PR --- CHANGELOG.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fdfdd56..e6ca387f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ -## Version 5.3.1 (April 24, 2018) +## Version 5.4 (June 1, 2018) -Packaging updates; provides better metadata for the new PyPI. +- ftfy was still too conservative about fixing short mojibake sequences, + such as "août" -> "août", when the broken version contained punctuation + such as curly or angle quotation marks. + + The new heuristic observes in some cases that, even if quotation marks are + expected to appear next to letters, it is strange to have an accented capital + A before the quotation mark and more letters after the quotation mark. + +- Provides better metadata for the new PyPI. ## Version 5.3 (January 25, 2018) From 8db042895b4ea0c47e1f81bc84255625d0adfca0 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 14 Jun 2018 11:58:39 -0400 Subject: [PATCH 377/502] Read the package description in UTF-8. Fixes #100. Also, remembered to bump the `__version__` constant this time. --- CHANGELOG.md | 9 +++++++++ ftfy/__init__.py | 2 +- setup.py | 4 ++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6ca387f..7a39a119 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## Version 5.4.1 (June 14, 2018) + +- Fixed a bug in the `setup.py` metadata. + + This bug was causing ftfy, a package that fixes encoding mismatches, to not + install in some environments due to an encoding mismatch. (We were really + putting the "meta" in "metadata" here.) + + ## Version 5.4 (June 1, 2018) - ftfy was still too conservative about fixing short mojibake sequences, diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 79f30991..eb92908d 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -10,7 +10,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.3.0' +__version__ = '5.4.1' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/setup.py b/setup.py index c37154f5..05ffc059 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ pip install ftfy==4.4.3 """ -DESCRIPTION = open('README.md').read() +DESCRIPTION = open('README.md', encoding='utf-8').read() if sys.version_info[0] < 3: @@ -26,7 +26,7 @@ setup( name="ftfy", - version='5.4.0', + version='5.4.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From cad062bb4ac5db5b8f1a2f6b9c3a7436cc5b82cf Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 14 Jun 2018 15:27:20 -0400 Subject: [PATCH 378/502] Add travis integration --- .travis.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..495bc943 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +language: python +sudo: false + +cache: + apt: true + directories: + - $HOME/.cache/pip + - $HOME/download +python: + - "3.3" + - "3.4" + - "3.5" + - "3.6" + - "3.7" +before_install: + - pip install pip -U + - pip install pytest -U +install: + - travis_retry pip install -e . +script: + - travis_wait pytest + +after_success: + - codecov +cache: + apt: true + directories: + - $HOME/.pip-cache + From adb69ce275be6767f04d74698bd87aca96e4d412 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 14 Jun 2018 15:38:41 -0400 Subject: [PATCH 379/502] add travis badge Maybe this commit will also get travis's attention to run the first build. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index eac1a2e3..6d08bd8d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # ftfy: fixes text for you +[![Travis](https://img.shields.io/travis/LuminosoInsight/python-ftfy/master.svg?label=Travis%20CI)](https://travis-ci.org/LuminosoInsight/python-ftfy) [![PyPI package](https://badge.fury.io/py/ftfy.svg)](http://badge.fury.io/py/ftfy) [![Docs](https://readthedocs.org/projects/ftfy/badge/?version=latest)](http://ftfy.readthedocs.org/en/latest/) From d401227916d280ab93889741fbe9087082e12059 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 14 Jun 2018 15:40:21 -0400 Subject: [PATCH 380/502] travis: 3.7 is 3.7-dev for now --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 495bc943..37935fdb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ python: - "3.4" - "3.5" - "3.6" - - "3.7" + - "3.7-dev" before_install: - pip install pip -U - pip install pytest -U From 4d62970ee9dfa8591c5e699a73bf59ff2487b3ec Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 14 Jun 2018 15:45:29 -0400 Subject: [PATCH 381/502] ignore setup.py when collecting tests --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index df3eb518..6afcb4c3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --doctest-modules +addopts = --doctest-modules --ignore=setup.py From d38ef78fe1a5051191c4c73b23c28e0910343622 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 14 Jun 2018 15:58:28 -0400 Subject: [PATCH 382/502] Passing tests on 3.7 is going to require a data change Disabled the 3.7-dev test for now. But that was useful to know. The tests fail on newly-introduced Unicode characters such as U+09FC BENGALI LETTER VEDIC ANUSVARA. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 37935fdb..4fb78b2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ python: - "3.4" - "3.5" - "3.6" - - "3.7-dev" before_install: - pip install pip -U - pip install pytest -U From e426374c74ba09dda4c255acd463d930163ab361 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 26 Jun 2018 17:12:31 -0400 Subject: [PATCH 383/502] Tweak the heuristic to be better at MacRoman --- ftfy/badness.py | 13 +++++++++++++ tests/test_cases.json | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 3b1ca69d..18279b9d 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -129,6 +129,19 @@ def _make_weirdness_regex(): # Characters we have to be a little more cautious about if they're at # the end of a word, but totally okay to fix in the middle '[ÂÃĂ][›»‘”©™]\w|' + # Similar mojibake of low-numbered characters in MacRoman. Leaving out + # most mathy characters because of false positives, but cautiously catching + # "√±" (mojibake for "ñ") and "√∂" (mojibake for "ö") in the middle of a + # word. + # + # I guess you could almost have "a√±b" in math, except that's not where + # you'd want the ±. Complex numbers don't quite work that way. "√±" appears + # unattested in equations in my Common Crawl sample. + # + # Also left out eye-like letters, including accented o's, for when ¬ is + # the nose of a kaomoji. + '[¬√][ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñúùûü†¢£§¶ß®©™≠ÆØ¥ªæø≤≥]|' + '\w√[±∂]\w|' # ISO-8859-1, ISO-8859-2, or Windows-1252 mojibake of characters U+10000 # to U+1FFFF. (The Windows-1250 and Windows-1251 versions might be too # plausible.) diff --git a/tests/test_cases.json b/tests/test_cases.json index f25614de..7625a58e 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -360,6 +360,39 @@ "fixed": "MÄ£ÄM ÌÑÌ Q £ÄGÌ GÄLÄW ÑÍCH SÖÄ£ ÑÝÄ $ÚÄMÌ Q £ÄGÌ GÄK ÉÑÄK BÄDÄÑ....?????????, ......JÄDÍ...", "expect": "pass" }, + { + "label": "Three layers of UTF-8 / MacRoman mixup in French", + "comment": "You're welcome", + "original": "Merci de t‚Äö√†√∂¬¨¬©l‚Äö√†√∂¬¨¬©charger le plug-in Flash Player 8", + "fixed": "Merci de télécharger le plug-in Flash Player 8", + "expect": "pass" + }, + { + "label": "UTF-8 / MacRoman mixup in French", + "original": "Merci de bien vouloir activiter le Javascript dans votre navigateur web afin d'en profiter‚Ķ", + "fixed": "Merci de bien vouloir activiter le Javascript dans votre navigateur web afin d'en profiter…", + "expect": "pass" + }, + { + "label": "Italian UTF-8 / MacRoman example with ò", + "original": "Le Vigne di Zam√≤", + "fixed": "Le Vigne di Zamò", + "expect": "pass" + }, + { + "label": "Italian UTF-8 / MacRoman mojibake that looks like math", + "comment": "False negative: Dangerous to fix because of the following example", + "original": "Sarai ricontattato dal nostro Esperto al pi√π presto.", + "fixed": "Sarai ricontattato dal nostro Esperto al più presto.", + "expect": "fail" + }, + { + "label": "Negative: math in Unicode", + "comment": "This isn't mojibake, it's an actual equation", + "original": "(-1/2)! = √π", + "fixed": "(-1/2)! = √π", + "expect": "pass" + }, { "label": "Negative: Leet line-art", "comment": "Current false positive: it coincidentally all decodes as UTF-8/CP437 mojibake and becomes 'ôaſaſaſaſa'", @@ -367,7 +400,6 @@ "fixed": "├┤a┼┐a┼┐a┼┐a┼┐a", "expect": "fail" }, - { "label": "Synthetic, negative: Brontë's name does not end with a Korean syllable", "comment": "The original example of why ftfy needs heuristics", From a9df3b5d0b126e35252b9f42e2c0eb1a1c8b79a3 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 26 Jun 2018 17:18:53 -0400 Subject: [PATCH 384/502] test another kaomoji --- tests/test_cases.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_cases.json b/tests/test_cases.json index 7625a58e..b79df377 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -269,6 +269,12 @@ "fixed": "OK??:( `¬´ ):", "expect": "pass" }, + { + "label": "Negative, synthetic: face with glasses and a raised eyebrow", + "original": "( o¬ô )", + "fixed": "( o¬ô )", + "expect": "pass" + }, { "label": "Negative: triangle and degree sign", "comment": "I'm not really sure what it *is* supposed to be, but it's not 'ơ'", From 76c50101b8905d6f763ce2f80133133fb5a58936 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 9 Jul 2018 12:31:18 -0400 Subject: [PATCH 385/502] Update char_classes to Unicode 10 so tests pass on Python 3.7 --- .travis.yml | 1 + ftfy/char_classes.dat | Bin 3989 -> 4046 bytes tests/test_futuristic_codepoints.py | 8 ++++++++ 3 files changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4fb78b2d..37935fdb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ python: - "3.4" - "3.5" - "3.6" + - "3.7-dev" before_install: - pip install pip -U - pip install pytest -U diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index e963e65681ee7d340a6208690db66cfa0ea5ed35..c5636fb5f18b4a595b4541af195ae6a4d0c077ae 100644 GIT binary patch literal 4046 zcmeH}X;c$d7RS}Gq7`MTB4UBGqh(W;in0b$s~BY!kVMv$McGLyn8+GNMN}YIw#Z%y zix42iKo9~62;u-Gu`Ee3kQfj!CM1M}gph5VwsU6Y)0{K&Wj@UN@ZNdn{{Q#B^E>aJ z_ik3L&R+Xioba&Lap(yaHjI0HcJGs0*+1RV zsJlfp&=PGJlv)E$ZIDmp)rHen>KtFm>P|h6xm7I5yHaPbxmWzB4CNO z6X$9(TW(moRC?IqHCzNNVWGko)+e4fsm3{pFpnAFOR(w>^>SS9OvU zitE0{QS=>8s?ru?mV@+zWAbOlDBcgIF z$8KybdWHw9fw6i!@a8!{8M#q@Ln#_r@luA%VDmaiXCHXjoCOHO3u9w=QZA)0y8Lf)#7s^tUUC-**>RXyuOVB zHrFhp9c+3>^75`EP*W70+l?ZZk8jj=y)wCLf@_v(JxTn=dbf{`bvt8bF`~;4ZwUjE zA-Bs*mz)yz?S0@|2Cc1&y#ChtUB={KxlhmXaV45x<(R*JjIwe_Sy9!FUz)aC{8W2r zWT^W)M9c?K6M}-fJ~uqj7F|aSWf<#3!yd(B0iRTxslkd$RoO z>h{$iCBdTCVR7soi=uB?&>MO{;4r+=#VuM`Ml4$-^3x;A;k}~`1Ir8RWaYI|pwKG2 z)@Cm7{pSom5u#gf=J{&W3I)_}Av~(~ZD;p$BMDsX+x(8sxCES?D&0jZgT-TcN69F? zwC1C$BlZR+g&26Qx79{mj0(}8DJpE7y1un26^TCag8KQl`X^R!6IJ28ZXG(c>Z!_< zkjK|W@jpbTw(JKOl|Dsi`(`eeY%G2soAwNJSP|73Mg+m;`MC?5zS7|rcxHqAq6FgQ z>i^NqyrdzkbjrVfek}1Vq}9j6zx+wy7w88LqRm6_F5fteuqFJx4cVN$wIksQtT7?0 zB9F+cGv^~r0~4xCJb%1`^~^2^bWX%p+lu@2HIHB64i{2_!j>)5`5Omw|Up?4c^V8HXEcC-j1NP?hJ;f}K_hWrkF?(PH%x zeZ!aBU&glKTN-41!~oFqeDKLmC{;(ZAYRhUpL|hk`IS?qR7mH(LW_$p6Z@i*2ti#> zGvUG=CadlmZz7&N&RyT>OnqJ`%8e+^s+$dM!Fl;V3UB)pn&33tyMA81c|7{J4d5*8qoS=q93)2AI(o6F#6Q?y>>C2Lm=Q#-`a!Xqzo=~!3v7=?1>W2`#Iq%G#ZQ{T zNWkN%2iiMKUc9(W=rObD418#eLh7P6wFU84)y5P?)>n4sLbbFKzxJIf4sg7cwxzPK zFg97Lep!g=b88K33+UQRt_Cn50iQ~R346=t-Hx=hDK{=wy{t^o6tt5E~%Ew;92^yf*Cz( zivR~j%^E*D8+JOsPhM?2LF47a{T-9p=R)f#gD55Ic&pRWy%q$jwMDf8~d&n+9F#h`}mSky3S9lhN}ckD9@QDY)bb zpwWxmQGh$jWr79KoTiRaC17ugehpRV2D*2GreLj~S9t@Xt}NBHem^MAU;j=WpWM2P z?AblFi=x@_H742-gRssNq@F2`Iy}*-Idi1m3YNXEX6JP9um}Equrp}yHr4gx!wSRLjwbnIsxIi6C5TKslJ72yk47?3M$+q3 zTQI(9VH1ddOK&ngSk+Xl1G>xg2X|pHQ4a4|yrQu5iF^kX|3*1~60vxLq$~&XbJm)SFJT9aN+n2yMJ>9o&8&I*F5xdk!go5@Dmp}U; zEA#B1VVwEOFMn^60T^!{X28+uf-Cgrbynp_SLRqsX565di9#hmUzZ0YRq3vCBD!9$ z6Yctg+Q|*_0e6FW4Ff7+C){NZ%zguv6BU41TFndIRLO!pD?u4Wa|1mzkjSsPXPRJ z%H=5!rAc%Yoq z!6vIkdy1W)HI7otE$M`Z!wBy*M^OJeT|~3JL@aD=>w#}vF%w{VoKK`InUPeu9+-8Tm z{OV!7lx_TOHYW-1?D!Bz2wj{xCh0Qzo`0M5|APM$_}?U;+$|qjFiH3P9225BC2WWP kcc*-_vh`2jg#Rd)+H(rI@*g?>Val%k%u}NiAelD#D+ZyRiU0rr literal 3989 zcmeHIX;c$d7RD)+)&-{r0Yyz}T>w!6$RdlN!=NIYC%&$48zvjohANQSi?)|>|&Ue0Z z@7;U}-wRRH`6r!^)hk9{Sp4Afq{xENfj_v5xVvRD0_WDs`tS(v+Y{$xeQ7YO(CqEL zO$}8s>!&y=FEh(1E~j9(dS32HhH!M%*RFb;^1Jg2lY8;G5lB?BgUBrISVzZMWZ+i4 zmS9Y%r*o8+RDzF3j`=c`5exJY%7_EU&#;}xvszjj(nHT*jx^#RU0ui2HY;wlB=Mvp zL`pr#IckiIE#+~w&ds!0bLE3z;DSP#%!b2|L9YY|>ZF|ogHAUhlgUAy;kUH=h?U0!;?H9&wBBnua{WgwgB#^oTmpajSPAT7HqI@|^|MbOu^@0}zTJ1SvZj^1ZM=)sJMjEy zYbi#$>@<)X+KcF}HoQ|{7aShvtGu5A@VS&fBLL^tc|k0~9BC~T-*8){7SkhxP_U^L z;)thXTE&Z0A4t&frCe)KqFdNEvEo!Uv=eGX+t?xE+6}CNi&l7lE$j#4SUnfAH|ssU z+3>^sw1+EGY}RG1d%=*{jbTdx27CxB*IMFH!YVZP_}++Wq$u_v>87gL1i-;kCZat@ zWESo!ZY9>0)!9?5rUkqn$US2LH_HX@l0w>q2jmr{Rf+QcdIH1Zbnbfxa7`z}f1XSpCN^Di3<5sx^y)Pv?`zP9 z+;9R|eqSMdke_R2;v&vHU@3II8>Im zIN@(DYx`Oe;j6L&yeVs64;j9Y9n03J5CgzsQ8dQ@$3l=L=L@N~kG-)d7nQEcSFX#Shbzblj4&F zZSUAxItq?#QE%T5Nh^|fFsI9$;(lI>VHc(O{Sfcgg@3f6L+pnY>Qf$Q=54G45p8;c zi3>~&iC%B3KJ-AT8=>MwSJqOMnbqBus?tbo8ji(CRjew+pwI0MuP_*3!N~>Ce!3nP z*pS&*^{q2jQ&TNXS$?xH(n@qa1nsEMNV$SAdz@1j@GaY;=A-eXcV3d<0JSx7UBfaoKL| zMr$iat`}iEzM+YsuGf982=SIrZU=(Yt8V~OXS{>Umnc>F-xfT(aO$sWe;P=aSAR^x zK+&$tXFV2o^Og=}!e0sTI2<)=QL(gy;#weqseIR6=p4b}`j_X9preTe2e9A4SDsP# zz7MWAMygDzCe5GVNJ9Bn0<%{>ZQAoqpyzsGCRA6dXrE?wl@QV;@;m016DHQ?r0=hH z`MfOXPODeZD%Ym;;%29osVav4XN{goQ&hp<9tO_C)4#r%YA-A6t9N`Ob-~c56K*MP zODcrKdGQUZ^keA}v)KUt#XS|I^kG5YyXTWijlJ~p~=iU;8ZwchHd}T&TZLo z(-PT2|E5dN3bgnmT->>Ijz(V4agCt5;2w6Je#CWzmrn!=tI(*m0{Q2RIPB#zY zW@1!f5IDkKWaH#NLoy40>*qB7(OcM@XheaFf3Jxo>uOuSq5={>M-p*0 zYs`@`BPZ%rJ+4IZZXebdVaL)ip$oV&hf(*WGAsYpvRzYAwDHJ!KJSz1Sk~G-l_n!( z`TE1mj_3`gNgn6@5LV-7qBO49;c-cqp4~j_*P#8R zrtvyfLfV1Z&1euR*v4@o0{U?5YSO}p2t&X#;IcvYYnF8pi@JPqe>makWb<@Z2`hi* z1{s?-bJ#}HF@S6>d?bFYVHKsW3aXT$i=?~4) z!KT$#<$`a*<5RHxS0|hGDI)_ZS7tMS!8jrevZJlUdZ$FH1~rYvz_gVIgi9;!(8Y_% z#6a!vLUUGJvcg|7Ck)bWTx|?}$1rNk7bqKd#64BQC!{`?uyJ<7i4Qe3iT$gV3&^6l z*)fAe9O0AW#XF;#&k|4Zv44WkS+8q5MN@$tN1Go4`?N0MUR!iNXb7iKTQ^O5rKDdx zeb0c9diFkWVz_(H@75pZCM%84uX(jnh#eY7)+VV6or{nLB{MbYiO_05!3$i*QV6AS z8Rfu$keL+K2lxGaEjMw-o+;!7l5)e#@^IE)uIu?2ivu-@|GscyU+v(h31R8g)fQk9DV87mFHG+JlJ$T`^#+S zK*CX_t6JLq7st~1`ibAbT}ho|Di3ILJ|b8;7k;HbGPfbbrdl-)=SdZRpolN!xEXHAmnXlI&2k3L+68~X|Ch4y*J?ZNE=;oOSIAw$ z9C`WQ^K4^V?{b_V*w8@IBf5 gZ+rcZfPcfRnX(D^e@(VO$isYae0|p$*N8)Z2bFw`e*gdg diff --git a/tests/test_futuristic_codepoints.py b/tests/test_futuristic_codepoints.py index e822d356..da031429 100644 --- a/tests/test_futuristic_codepoints.py +++ b/tests/test_futuristic_codepoints.py @@ -32,3 +32,11 @@ def test_unicode_9(): # in the string. assert sequence_weirdness("B\U00090000G") == 2 + +def test_unicode_10(): + # This string is the word "thalīṃ" in the Zanabazar Square Script, + # a script added in Unicode 10. These characters are recognized as being + # assigned by Python 3.7, and therefore ftfy should recognize them on + # all versions for consistency. + thalim = "\U00011A1A\U00011A2C\U00011A01\U00011A38" + assert sequence_weirdness(thalim) == 0 From 034c13eaef2fa28ea9b013c8b31274aff660f7e8 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 30 Aug 2018 12:49:41 -0400 Subject: [PATCH 386/502] New test cases, one we can't handle without changing rules --- tests/test_cases.json | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_cases.json b/tests/test_cases.json index f25614de..fe066085 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -151,6 +151,14 @@ "fixed": "Arsenal v Wolfsburg: pre-season friendly – live!", "expect": "pass" }, + { + "label": "Inconsistent UTF-8 / Latin-1 mojibake", + "comment": "Oh like it was never political before", + "original": "Ecuadorâ\u0080\u0099s â\u0080\u0098purely political decision on Assangeâ\u0080\u0099 is likely result of â\u0080\u0098US pressureâ\u0080\u0099…", + "fixed-encoding": "Ecuador’s ‘purely political decision on Assange’ is likely result of ‘US pressure’", + "fixed": "Ecuador's 'purely political decision on Assange' is likely result of 'US pressure'", + "expect": "pass" + }, { "label": "Handle Afrikaans 'n character", "original": "ʼn Chloroplas is ʼn organel wat in fotosinterende plante voorkom.", @@ -327,12 +335,19 @@ "expect": "fail" }, { - "label": "Windows-1252 / MacRoman mixup in Spanish", + "label": "Latin-1 / MacRoman mixup in Spanish", "comment": "Requires something like encoding detection", "original": "Deja dos heridos hundimiento de barco tur\u0092stico en Acapulco.", "fixed": "Deja dos heridos hundimiento de barco turístico en Acapulco.", "expect": "fail" }, + { + "label": "Latin-1 / MacRoman mixup in Spanish, 2 characters", + "comment": "Requires something like encoding detection", + "original": "Habitantes de Coatl\u0087n conf\u0092an en proyecto de edil electo independiente", + "fixed": "Habitantes de Coatlán confían en proyecto de edil electo independiente", + "expect": "fail" + }, { "label": "UTF-8 / Windows-1251 mixup in tweet spam", "original": "Blog Traffic Tip 2 – Broadcast Email Your Blog", From 40b2743a6cf9b5508e25018ea0eda3c48e4aa16b Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 30 Aug 2018 12:56:03 -0400 Subject: [PATCH 387/502] Remove pytest-runner --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 05ffc059..4927b272 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,6 @@ packages=['ftfy', 'ftfy.bad_codecs'], package_data={'ftfy': ['char_classes.dat']}, install_requires=['wcwidth'], - setup_requires=['pytest-runner'], tests_require=['pytest'], python_requires='>=3.3', classifiers=[ From d1cd44b449f9067bbc46fb7efeb97edb45b7b24a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 30 Aug 2018 13:18:30 -0400 Subject: [PATCH 388/502] Remove Python 3.3 testing (no longer supported by Travis) --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 37935fdb..30310d5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ cache: - $HOME/.cache/pip - $HOME/download python: - - "3.3" - "3.4" - "3.5" - "3.6" From 94640116ec12663a7df86c5ef184a7683f07ee04 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 6 Sep 2018 14:37:25 -0400 Subject: [PATCH 389/502] Adapt CLI test to work with Windows line breaks --- tests/test_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index d1a0d2c6..3b27075b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -53,7 +53,8 @@ def test_same_file(): with pytest.raises(subprocess.CalledProcessError) as exception: get_command_output(['ftfy', TEST_FILENAME, '-o', TEST_FILENAME]) error = exception.value.output.decode('utf-8') - assert error.startswith("ftfy error:\nCan't read and write the same file.") + assert error.startswith("ftfy error:") + assert "Can't read and write the same file" in error def test_stdin(): From cc83dc05265f6f2497a8e9a42748522111765224 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 6 Sep 2018 14:55:57 -0400 Subject: [PATCH 390/502] bump version to 5.5 --- ftfy/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index eb92908d..c4340b82 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -10,7 +10,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.4.1' +__version__ = '5.5.0' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/setup.py b/setup.py index 4927b272..7af2d25b 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup( name="ftfy", - version='5.4.1', + version='5.5.0', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From 2862fe7edc2cadb5b0f0948bf9713ea04255cb6f Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 6 Sep 2018 15:31:58 -0400 Subject: [PATCH 391/502] Make the partial mojibake fixer actually work on Latin-1 The rule that locally fixes obvious Windows-1252/UTF-8 mixups (even if they are only mixed up in part of the string) had comments that implied it would work on Latin-1 as well. The code assumed that the Latin-1 case was automatically handled because this fixer would run after the encoding fixer tried replacing Latin-1 with Windows-1252. However, that step only works if it can apply consistently across the whole string, while this fixer is meant for fixing inconsistent mojibake. So the typical outcome is that the encoding fixer would give up and this step wouldn't run at all. A new test case I found in the wild revealed how this wasn't working. Now, the local fixer for inconsistent mojibake takes responsibility for translating Latin-1 characters itself, and we can pass the test. --- ftfy/chardata.py | 13 ++++++++----- ftfy/fixes.py | 24 ++++++++++++++++-------- tests/test_cases.json | 12 +++++++++--- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 8c70992d..be97478a 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -60,11 +60,10 @@ def _build_utf8_punct_regex(): These are recognizable by the distinctive 'â€' ('\xe2\x80') sequence they all begin with when decoded as Windows-1252. """ - # We're making a regex that has all the literal bytes from 0x80 to 0xbf in - # a range. "Couldn't this have just said [\x80-\xbf]?", you might ask. - # However, when we decode the regex as Windows-1252, the resulting - # characters won't even be remotely contiguous. - obvious_utf8 = ('â€[' + # We need to recognize the Latin-1 decodings of bytes 0x80 to 0xbf, which + # are a contiguous range, as well as the Windows-1252 decodings, which are + # not contiguous at all. + obvious_utf8 = ('â[€\x80][\x80-\xbf' + bytes(range(0x80, 0xc0)).decode('sloppy-windows-1252') + ']') return re.compile(obvious_utf8) @@ -120,6 +119,10 @@ def _build_utf8_punct_regex(): SINGLE_QUOTE_RE = re.compile('[\u02bc\u2018-\u201b]') DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') +# This regex matches C1 control characters, which occupy some of the positions +# in the Latin-1 character map that Windows assigns to other characters instead. +C1_CONTROL_RE = re.compile(r'[\x80-\x9f]') + def possible_encoding(text, encoding): """ diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 8f081e6b..e65a635b 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -9,7 +9,8 @@ from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS, CONTROL_CHARS, LIGATURES, WIDTH_MAP, PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE, - LOSSY_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE) + LOSSY_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE, + C1_CONTROL_RE) from ftfy.badness import text_cost from html import entities @@ -633,7 +634,7 @@ def replace_lossy_sequences(byts): not be used, and this function will not be run, so your weird control character will be left alone but wacky fixes like this won't be possible. - This is used as a step within `fix_encoding`. + This is used as a transcoder within `fix_encoding`. """ return LOSSY_UTF8_RE.sub('\ufffd'.encode('utf-8'), byts) @@ -644,16 +645,23 @@ def fix_partial_utf8_punct_in_1252(text): UTF-8 and decoded in Latin-1 or Windows-1252, even when this fix can't be consistently applied. - For this function, we assume the text has been decoded in Windows-1252. - If it was decoded in Latin-1, we'll call this right after it goes through - the Latin-1-to-Windows-1252 fixer. + One form of inconsistency we need to deal with is that some character might + be from the Latin-1 C1 control character set, while others are from the + set of characters that take their place in Windows-1252. So we first replace + those characters, then apply a fix that only works on Windows-1252 characters. - This is used as a step within `fix_encoding`. + This is used as a transcoder within `fix_encoding`. """ - def replacement(match): + def latin1_to_w1252(match): + "The function to apply when this regex matches." + return match.group(0).encode('latin-1').decode('sloppy-windows-1252') + + def w1252_to_utf8(match): "The function to apply when this regex matches." return match.group(0).encode('sloppy-windows-1252').decode('utf-8') - return PARTIAL_UTF8_PUNCT_RE.sub(replacement, text) + + text = C1_CONTROL_RE.sub(latin1_to_w1252, text) + return PARTIAL_UTF8_PUNCT_RE.sub(w1252_to_utf8, text) TRANSCODERS = { diff --git a/tests/test_cases.json b/tests/test_cases.json index fe066085..902d9b7a 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -153,10 +153,16 @@ }, { "label": "Inconsistent UTF-8 / Latin-1 mojibake", - "comment": "Oh like it was never political before", + "original": "Ecuadorâ\u0080\u0099s â\u0080\u0098purely political decision on Assangeâ\u0080\u0099 is likely result of â\u0080\u0098US pressureâ\u0080\u0099\u0085", + "fixed-encoding": "Ecuador’s ‘purely political decision on Assange’ is likely result of ‘US pressure’…", + "fixed": "Ecuador's 'purely political decision on Assange' is likely result of 'US pressure'…", + "expect": "pass" + }, + { + "label": "Inconsistent UTF-8 / Latin-1 mojibake with an ellipsis from the Windows-1252 character set", "original": "Ecuadorâ\u0080\u0099s â\u0080\u0098purely political decision on Assangeâ\u0080\u0099 is likely result of â\u0080\u0098US pressureâ\u0080\u0099…", - "fixed-encoding": "Ecuador’s ‘purely political decision on Assange’ is likely result of ‘US pressure’", - "fixed": "Ecuador's 'purely political decision on Assange' is likely result of 'US pressure'", + "fixed-encoding": "Ecuador’s ‘purely political decision on Assange’ is likely result of ‘US pressure’…", + "fixed": "Ecuador's 'purely political decision on Assange' is likely result of 'US pressure'…", "expect": "pass" }, { From 24766965990cbb96ce372f140c676abe1ae868ea Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 6 Sep 2018 15:44:35 -0400 Subject: [PATCH 392/502] remove redundancy from regex range --- ftfy/chardata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index be97478a..32420969 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -61,10 +61,11 @@ def _build_utf8_punct_regex(): all begin with when decoded as Windows-1252. """ # We need to recognize the Latin-1 decodings of bytes 0x80 to 0xbf, which - # are a contiguous range, as well as the Windows-1252 decodings, which are - # not contiguous at all. + # are a contiguous range, as well as the different Windows-1252 decodings + # of 0x80 to 0x9f, which are not contiguous at all. (Latin-1 and + # Windows-1252 agree on bytes 0xa0 and up.) obvious_utf8 = ('â[€\x80][\x80-\xbf' - + bytes(range(0x80, 0xc0)).decode('sloppy-windows-1252') + + bytes(range(0x80, 0xa0)).decode('sloppy-windows-1252') + ']') return re.compile(obvious_utf8) PARTIAL_UTF8_PUNCT_RE = _build_utf8_punct_regex() From 0dc253d50490a2968b741eb55b2b1898eeb4ba72 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Thu, 6 Sep 2018 16:13:27 -0400 Subject: [PATCH 393/502] Update the changelog --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a39a119..fd8b481a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +## Version 5.5 (September 6, 2018) + +- Recent versions have emphasized making a reasonable attempt to fix short, + common mojibake sequences, such as `û`. In this version, we've expanded the + heuristics to recognize these sequences in MacRoman as well as Windows-125x + encodings. + +- A related rule for fixing isolated Windows-1252/UTF-8 mixups, even when they + were inconsistent with the rest of the string, claimed to work on Latin-1/UTF-8 + mixups as well, but in practice it didn't. We've made the rule more robust. + +- Fixed a failure when testing the CLI on Windows. + +- Removed the `pytest-runner` invocation from setup.py, as it created complex + dependencies that would stop setup.py from working in some environments. + The `pytest` command still works fine. `pytest-runner` is just too clever. + + ## Version 5.4.1 (June 14, 2018) - Fixed a bug in the `setup.py` metadata. @@ -19,6 +37,8 @@ - Provides better metadata for the new PyPI. +- Switched from nosetests to pytest. + ## Version 5.3 (January 25, 2018) From fce841f631b0720a181facd74bbf6b8bf89e6c0a Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 14 Sep 2018 13:54:27 -0400 Subject: [PATCH 394/502] Cope with different Python versions better in setup.py This provides a more helpful message when you attempt to install ftfy on Python 2 (there used to be a message, but in recent versions it was erroring before it got to print the message). Because some instances of running Python 2 are now people who are trying to run Python 3 and messed up their environment, the message now shows details about what version of Python is running and where it came from. This change also sets the Trove classifier that says that we support Python 3.7. --- setup.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 7af2d25b..01032a83 100644 --- a/setup.py +++ b/setup.py @@ -1,29 +1,30 @@ -from setuptools import setup import sys +# Before we get to the rest of setup, with dependencies on setuptools and the +# Python 3 standard library, let's make sure we're not on Python 2 and provide +# a helpful message if we are. PY2_MESSAGE = """ Sorry, this version of ftfy is no longer written for Python 2. -It is much easier to provide consistent handling of Unicode when developing -only for Python 3. ftfy is exactly the kind of library that Python 3 is -designed for, and now there is enough Python 3 usage that we can take advantage -of it and write better, simpler code. - The older version of ftfy, version 4.4, is still available and can run on Python 2. Try this: pip install ftfy==4.4.3 """ -DESCRIPTION = open('README.md', encoding='utf-8').read() - - if sys.version_info[0] < 3: print(PY2_MESSAGE) + readable_version = sys.version.split(' ')[0] + print("The version of Python you're running is: %s" % readable_version) + print("Python is running from: %r" % sys.executable) sys.exit(1) +from setuptools import setup + +DESCRIPTION = open('README.md', encoding='utf-8').read() + setup( name="ftfy", version='5.5.0', @@ -46,6 +47,7 @@ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", From 360f39be2282a1d8d9c90c0b7af40037424d37c0 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 14 Sep 2018 14:06:26 -0400 Subject: [PATCH 395/502] Fix warning about \w outside of a raw string It gets interpreted correctly as a backslash and a 'w', which becomes the appropriate regex atom that matches word-like characters, but Python makes no promises about this. It should just be in a raw string. --- ftfy/badness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 18279b9d..678c40f8 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -141,7 +141,7 @@ def _make_weirdness_regex(): # Also left out eye-like letters, including accented o's, for when ¬ is # the nose of a kaomoji. '[¬√][ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñúùûü†¢£§¶ß®©™≠ÆØ¥ªæø≤≥]|' - '\w√[±∂]\w|' + r'\w√[±∂]\w|' # ISO-8859-1, ISO-8859-2, or Windows-1252 mojibake of characters U+10000 # to U+1FFFF. (The Windows-1250 and Windows-1251 versions might be too # plausible.) From 896822fc443a98bc348d43c712874f87bc4483a7 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 14 Sep 2018 15:53:35 -0400 Subject: [PATCH 396/502] Update char_classes for Unicode 11 and test the result --- ftfy/char_classes.dat | Bin 4046 -> 4121 bytes tests/test_futuristic_codepoints.py | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index c5636fb5f18b4a595b4541af195ae6a4d0c077ae..830e65499affef481e315709c09c413196088051 100644 GIT binary patch literal 4121 zcmeH`Ycw0^y2rJh))+0Pon30Q>|VyDE~&cJHOHvIph1ALLx{=?AbGO)>-G%UTg0!`@?=eJnwqf^ZftMyMAl^ z(<)clZgI5(2NixHY^r^3@X+>AtpT+k`|N1-U*5>0(7;~$&xP2!kwojCJiLPSzF4W3 z%ShrSkl0j&)61)S(T+Ug>P-`O;FUJJ@OcSU^M7AWjF}5x6flXZ3=|}-85Pxw>HSOL_A`@ zRT5Hhuf+hR$W44TKYj82s_s44@k_#*AoB5GBkd~VrG&bpZR!hW{sQ*R>$n0#P^UnR zJ*u#-y5)#Udeija{I_8-zVg`nKE|xKFKnMY`3MWh9wDw&T;My1&SvaibZ=>RNthV; zWkI-7(A!BI(u>-CTg9#jG??$1N+|EOVkrt?W)6uJ$iRqU<5cD62ZGR)mdDZ8 zq5jyP(+pe(j{AJ=VeCZoweX2_m$E_SE|~Oc1Ud=bYYcRhrVIt;dqqdoUac9^M$?B~ zisi}0WuieF{!*5}W+6nC6x36Zq0 z-abZ5uD4$}lUn+aS4{^_ESS5{4G@@D{xYaldc0-N#k_Ow=g-MtvXSw4O<0w@ZoIQ_ z26)Ui5BCRcB#tEQeRoJqY3`U7MwUaYAcfRh!t1 zcKfds_}*K}$3DfJQ(zMw$FcN2n=p5rOug*_Y@|?hJm*ek_>6Ugb`%4kS^XwPMmY}L zThCY?Y5{9@R9_U$64N%2{Vc1GoOYj7)Axh_%pFrqyCn9qmcMMni2nU$1n=9k;u<$( zLHB3Qy>X+VZRqb0;UT!&9}_6{C_F;@in%`C^yMLLlpBR4Zs}`5H6x>~mvTNhi>Sx1 ztED~^D64JJ+n6iWy4OZ)uYoY&|5*VDJUt(_CKO4d3^sEY`XhO-58$;=6?|iMsDPg$ z+PNv1zzq7KTEyQ+hQaEnArAfnJl^#r|5GrOb#+W~qz}PLS^EigStGXc<(>&{o;kQI z4e-bS(nc@ESZ6dsaEsROm#3e zw$Dq`FgV8tWkP0J^zH1R{Mi8T;|?n-4ES5RLjFFNu zRCZEOfk5mdQfCawv#+hg=W-En` zkdUt>eqJrpct>vbq@2;<%%ZLW@qVsg2MWrZ(dtO40-H}nZePneHw^klGAx`-41Ke+ zVXT=v@vU}=9Hi#QndI|6vm1Ji+F*p0EI3i96Mk6UC2l~(=_u&K*vZ_zK{trbhWUmc zzUH*tbAeA=WL@P{;*Q3)owP{ZJvNK{s(%juQ|*}j#H*iH7Q6(RBOJuCTw9<_ict9d z)UX92eiZt%KVJW@&XdZA6+2JqArtBqhB+gt6kwnyD6B#~_zLyck?S67Hkt8dN~wsI z7pF0~5^vC1&3E>d@ma$n{>U~E++Dq!S;+ak8isi?_#FR%T*2g#;~L_0`H^6cv9R6QHgj@8vM*~qlODlh6g6ihHmkt3O zSEY-;BR#YGSqwdc%5IQ1gplPku@Xu%9K>*x|e!617CzIi)IV{;LrdpH;N<$k%8GYgY zw5~VSWdFawr7u_#I6#i!n%gEU_5ohVQmnNIBMpnC&I3FuoqbMIhX5%$~wp zRGj?bhP+Sb-WtOlH?3o+18pHXY;mfYKrqW7J=|KmUmv((1|tyt-BliAmRr91g4~f2 zwySMH>*YE)WgoWdk};+@L)YrHZRBOA=iomr7WIb!`4`RVZw;CjR2r@aN8)y2saso8 z9Kw>Vq8;?tWt;M-(u;Eqt~ZT|R%!P$LmSHdwkuwba}05;ixx9P<;(r9;nHGOd>>AP z-pIH@Ni2ze@4dg+?IZ9he-VAbFc=mYP?8kYf^nupeuO>XAPzhOnFW4lla?7rt6)_+ zDX$`r9a-k9fB&HT-b{WOlm8smC4mwcZ&KPEBWImrRkABHR2BJmA&rZKOP=6BHeh+x z)h}1oW&&hmUu}R$0Ml+xeMW;zMN&Y|TnuD?(c?q?CIh_wT{^UBpOvG%xXG)X866~Y zhL6SWKek($;unR$jS8&jt10EqN?(cRBNLS1)%0#O|gl=xpVX?Yj zai-riX`?PzZHqM8c{jd~{anMwS*eh+4=(}A-A-pp#SuhAa2Xm{0MwB-mw8l1FBzOB zdXM6XCrP7+P~tKflDoOl7(VXEB1TKJw8p&^6cQ*=DC`f~i!AoZ-y)LGLj>rp8`#24 zkNlaWooKt={PTOBiL)ag9&$qd&$q}v^Fx`pQspy8FWpKvoJ;AMx@|2P;A*smGdj4Ds7F*PNvG#6uUM(JZO zLnBm(pFsM`%zkNj&&S~sTrL$wgSUDXE?dX$JkzmcS`}OJpBi=3usV$fU)$)*JIsMS zEJkRi*FHlvF@}tv>$wc&pci#ap`U^x{`B(GmyOBwXI^*IMv)$*uivLUS zAKmjNtv5%8_5QB&keM@85pc=xk~A$2^UxpS7fu|d!@ogZt;$UAfo`o3zWhg~|LOmS zz`rSh9ko1cZ}5sqRuEJ+XiK3T_?P~9Pr%pymkfT3mF{2__RpmK(^j+xhP$f30G!v~ E0KFTw-v9sr literal 4046 zcmeH}X;c$d7RS}Gq7`MTB4UBGqh(W;in0b$s~BY!kVMv$McGLyn8+GNMN}YIw#Z%y zix42iKo9~62;u-Gu`Ee3kQfj!CM1M}gph5VwsU6Y)0{K&Wj@UN@ZNdn{{Q#B^E>aJ z_ik3L&R+Xioba&Lap(yaHjI0HcJGs0*+1RV zsJlfp&=PGJlv)E$ZIDmp)rHen>KtFm>P|h6xm7I5yHaPbxmWzB4CNO z6X$9(TW(moRC?IqHCzNNVWGko)+e4fsm3{pFpnAFOR(w>^>SS9OvU zitE0{QS=>8s?ru?mV@+zWAbOlDBcgIF z$8KybdWHw9fw6i!@a8!{8M#q@Ln#_r@luA%VDmaiXCHXjoCOHO3u9w=QZA)0y8Lf)#7s^tUUC-**>RXyuOVB zHrFhp9c+3>^75`EP*W70+l?ZZk8jj=y)wCLf@_v(JxTn=dbf{`bvt8bF`~;4ZwUjE zA-Bs*mz)yz?S0@|2Cc1&y#ChtUB={KxlhmXaV45x<(R*JjIwe_Sy9!FUz)aC{8W2r zWT^W)M9c?K6M}-fJ~uqj7F|aSWf<#3!yd(B0iRTxslkd$RoO z>h{$iCBdTCVR7soi=uB?&>MO{;4r+=#VuM`Ml4$-^3x;A;k}~`1Ir8RWaYI|pwKG2 z)@Cm7{pSom5u#gf=J{&W3I)_}Av~(~ZD;p$BMDsX+x(8sxCES?D&0jZgT-TcN69F? zwC1C$BlZR+g&26Qx79{mj0(}8DJpE7y1un26^TCag8KQl`X^R!6IJ28ZXG(c>Z!_< zkjK|W@jpbTw(JKOl|Dsi`(`eeY%G2soAwNJSP|73Mg+m;`MC?5zS7|rcxHqAq6FgQ z>i^NqyrdzkbjrVfek}1Vq}9j6zx+wy7w88LqRm6_F5fteuqFJx4cVN$wIksQtT7?0 zB9F+cGv^~r0~4xCJb%1`^~^2^bWX%p+lu@2HIHB64i{2_!j>)5`5Omw|Up?4c^V8HXEcC-j1NP?hJ;f}K_hWrkF?(PH%x zeZ!aBU&glKTN-41!~oFqeDKLmC{;(ZAYRhUpL|hk`IS?qR7mH(LW_$p6Z@i*2ti#> zGvUG=CadlmZz7&N&RyT>OnqJ`%8e+^s+$dM!Fl;V3UB)pn&33tyMA81c|7{J4d5*8qoS=q93)2AI(o6F#6Q?y>>C2Lm=Q#-`a!Xqzo=~!3v7=?1>W2`#Iq%G#ZQ{T zNWkN%2iiMKUc9(W=rObD418#eLh7P6wFU84)y5P?)>n4sLbbFKzxJIf4sg7cwxzPK zFg97Lep!g=b88K33+UQRt_Cn50iQ~R346=t-Hx=hDK{=wy{t^o6tt5E~%Ew;92^yf*Cz( zivR~j%^E*D8+JOsPhM?2LF47a{T-9p=R)f#gD55Ic&pRWy%q$jwMDf8~d&n+9F#h`}mSky3S9lhN}ckD9@QDY)bb zpwWxmQGh$jWr79KoTiRaC17ugehpRV2D*2GreLj~S9t@Xt}NBHem^MAU;j=WpWM2P z?AblFi=x@_H742-gRssNq@F2`Iy}*-Idi1m3YNXEX6JP9um}Equrp}yHr4gx!wSRLjwbnIsxIi6C5TKslJ72yk47?3M$+q3 zTQI(9VH1ddOK&ngSk+Xl1G>xg2X|pHQ4a4|yrQu5iF^kX|3*1~60vxLq$~&XbJm)SFJT9aN+n2yMJ>9o&8&I*F5xdk!go5@Dmp}U; zEA#B1VVwEOFMn^60T^!{X28+uf-Cgrbynp_SLRqsX565di9#hmUzZ0YRq3vCBD!9$ z6Yctg+Q|*_0e6FW4Ff7+C){NZ%zguv6BU41TFndIRLO!pD?u4Wa|1mzkjSsPXPRJ z%H=5!rAc%Yoq z!6vIkdy1W)HI7otE$M`Z!wBy*M^OJeT|~3JL@aD=>w#}vF%w{VoKK`InUPeu9+-8Tm z{OV!7lx_TOHYW-1?D!Bz2wj{xCh0Qzo`0M5|APM$_}?U;+$|qjFiH3P9225BC2WWP kcc*-_vh`2jg#Rd)+H(rI@*g?>Val%k%u}NiAelD#D+ZyRiU0rr diff --git a/tests/test_futuristic_codepoints.py b/tests/test_futuristic_codepoints.py index da031429..4c76c4c1 100644 --- a/tests/test_futuristic_codepoints.py +++ b/tests/test_futuristic_codepoints.py @@ -40,3 +40,22 @@ def test_unicode_10(): # all versions for consistency. thalim = "\U00011A1A\U00011A2C\U00011A01\U00011A38" assert sequence_weirdness(thalim) == 0 + + +def test_unicode_11(): + # Unicode 11 has implemented the mtavruli form of the Georgian script. + # They are analogous to capital letters in that they can be used to + # emphasize text or write a headline. + # + # Python will convert to that form when running .upper() on Georgian text, + # starting in version 3.7.0. We want to recognize the result as reasonable + # text on all versions. + # + # This text is the mtavruli form of "ქართული ენა", meaning "Georgian + # language". + + georgian_mtavruli_text = 'ᲥᲐᲠᲗᲣᲚᲘ ᲔᲜᲐ' + assert sequence_weirdness(georgian_mtavruli_text) == 0 + + mojibake = georgian_mtavruli_text.encode('utf-8').decode('sloppy-windows-1252') + assert fix_encoding(mojibake) == georgian_mtavruli_text From 7abb30c7c71656081d1ed074a785ee6f7afa1ef4 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 14 Sep 2018 16:13:17 -0400 Subject: [PATCH 397/502] Bump version and update changelog for 5.5.1 --- CHANGELOG.md | 9 +++++++++ ftfy/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd8b481a..da75f769 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## Version 5.5.1 (September 14, 2018) + +- Added Python 3.7 support. + +- Updated the data file of Unicode character categories to Unicode 11, as used + in Python 3.7.0. (No matter what version of Python you're on, ftfy uses the + same data.) + + ## Version 5.5 (September 6, 2018) - Recent versions have emphasized making a reasonable attempt to fix short, diff --git a/ftfy/__init__.py b/ftfy/__init__.py index c4340b82..dacbea82 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -10,7 +10,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.5.0' +__version__ = '5.5.1' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/setup.py b/setup.py index 7af2d25b..03e2c75c 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup( name="ftfy", - version='5.5.0', + version='5.5.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From b8b9f30e71be3d055028b62df9422c33f4cc4e77 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Fri, 21 Sep 2018 15:57:25 -0400 Subject: [PATCH 398/502] Update references to Python versions and Unicode versions --- docs/index.rst | 4 ++-- ftfy/build_data.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 618d32cb..a5520a03 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -301,8 +301,8 @@ Variants of UTF-8 *ftfy.chardata* and *ftfy.build_data*: trivia about characters -------------------------------------------------------------- -These files load information about the character properties in Unicode 9.0. -Yes, even if your version of Python doesn't support Unicode 9.0. This ensures +These files load information about the character properties in Unicode 11.0. +Yes, even if your version of Python doesn't support Unicode 11.0. This ensures that ftfy's behavior is consistent across versions. .. automodule:: ftfy.chardata diff --git a/ftfy/build_data.py b/ftfy/build_data.py index ee28b58c..d45e2327 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -5,7 +5,7 @@ classes we care about change, or if a new version of Python supports a new Unicode standard and we want it to affect our string decoding. -The file that we generate is based on Unicode 9.0, as supported by Python 3.6. +The file that we generate is based on Unicode 11.0, as supported by Python 3.7. You can certainly use it in earlier versions. This simply makes sure that we get consistent results from running ftfy on different versions of Python. @@ -39,16 +39,16 @@ def make_char_data_file(do_it_anyway=False): Build the compressed data file 'char_classes.dat' and write it to the current directory. - If you run this, run it in Python 3.6 or later. It will run in earlier - versions, but you won't get the Unicode 9 standard, leading to inconsistent - behavior. + If you run this, run it in Python 3.7.0 or later. It will run in earlier + versions, but you won't get the Unicode 11 standard, leading to inconsistent + behavior. Pre-releases of Python 3.7 won't work (Unicode 11 wasn't out yet). To protect against this, running this in the wrong version of Python will raise an error unless you pass `do_it_anyway=True`. """ - if sys.hexversion < 0x03060000 and not do_it_anyway: + if sys.hexversion < 0x030700f0 and not do_it_anyway: raise RuntimeError( - "This function should be run in Python 3.6 or later." + "This function should be run in Python 3.7.0 or later." ) cclasses = [None] * 0x110000 From 51b59dd13983b617b6c21c7770d5426e53e88e63 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Mon, 24 Sep 2018 11:07:42 -0400 Subject: [PATCH 399/502] Fix another non-raw \w --- ftfy/badness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 678c40f8..bd092946 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -128,7 +128,7 @@ def _make_weirdness_regex(): '[ÂÃĂ][\x80-\x9f€ƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¢£¤¥¦§¨ª«¬¯°±²³µ¶·¸¹º¼½¾¿ˇ˘˝]|' # Characters we have to be a little more cautious about if they're at # the end of a word, but totally okay to fix in the middle - '[ÂÃĂ][›»‘”©™]\w|' + r'[ÂÃĂ][›»‘”©™]\w|' # Similar mojibake of low-numbered characters in MacRoman. Leaving out # most mathy characters because of false positives, but cautiously catching # "√±" (mojibake for "ñ") and "√∂" (mojibake for "ö") in the middle of a From ce312989a802c14c3ae8fc534f4f7743f658a882 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 27 Sep 2018 16:32:56 -0400 Subject: [PATCH 400/502] Update my name Also remove Sphinx configuration for forms of documentation that will never be built, like texinfo, and update the version number on the docs. --- README.md | 2 +- docs/conf.py | 20 ++++++-------------- ftfy/fixes.py | 4 ++-- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 6d08bd8d..e5d6f6ae 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ You can also clone this Git repository and install it with ## Who maintains ftfy? -I'm Rob Speer (rob@luminoso.com). I develop this tool as part of my +I'm Robyn Speer (rspeer@luminoso.com). I develop this tool as part of my text-understanding company, [Luminoso](http://luminoso.com), where it has proven essential. diff --git a/docs/conf.py b/docs/conf.py index 2ec2868d..790afe7c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ # General information about the project. project = u'ftfy' -copyright = u'2017, Rob Speer' +copyright = u'2018, Robyn Speer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '5.3' +version = '5.5' # The full version, including alpha/beta/rc tags. -release = '5.3.0' +release = '5.5.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -185,10 +185,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'ftfy.tex', u'ftfy Documentation', - u'Rob Speer', 'manual'), -] +latex_documents = [] # The name of an image file (relative to this directory) to place at the top of # the title page. @@ -216,8 +213,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'ftfy', u'ftfy Documentation', - [u'Rob Speer'], 1) + ('index', 'ftfy', 'ftfy Documentation', ['Robyn Speer'], 1) ] # If true, show URL addresses after external links. @@ -229,11 +225,7 @@ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) -texinfo_documents = [ - ('index', 'ftfy', u'ftfy Documentation', - u'Rob Speer', 'ftfy', 'One line description of project.', - 'Miscellaneous'), -] +texinfo_documents = [] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] diff --git a/ftfy/fixes.py b/ftfy/fixes.py index e65a635b..decde551 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -327,7 +327,7 @@ def unescape_html(text): """ Decode all three types of HTML entities/character references. - Code by Fredrik Lundh of effbot.org. Rob Speer made a slight change + Code by Fredrik Lundh of effbot.org. Robyn Speer made a slight change to it for efficiency: it won't match entities longer than 8 characters, because there are no valid entities like that. @@ -659,7 +659,7 @@ def latin1_to_w1252(match): def w1252_to_utf8(match): "The function to apply when this regex matches." return match.group(0).encode('sloppy-windows-1252').decode('utf-8') - + text = C1_CONTROL_RE.sub(latin1_to_w1252, text) return PARTIAL_UTF8_PUNCT_RE.sub(w1252_to_utf8, text) From 55c2307614c717aacf9e6c286dc6f523b8707057 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 27 Sep 2018 16:33:59 -0400 Subject: [PATCH 401/502] flake8 fixes --- ftfy/__init__.py | 1 + ftfy/bad_codecs/sloppy.py | 1 + ftfy/badness.py | 2 ++ ftfy/build_data.py | 1 + ftfy/chardata.py | 8 ++++++++ ftfy/cli.py | 1 + 6 files changed, 14 insertions(+) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index dacbea82..dd925c8b 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -185,6 +185,7 @@ def fix_text(text, return ''.join(out) + # Some alternate names for the main functions ftfy = fix_text fix_encoding = fixes.fix_encoding diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index aa2cfaac..bd6f78c0 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -149,6 +149,7 @@ class StreamReader(Codec, codecs.StreamReader): streamwriter=StreamWriter, ) + # Define a codec for each incomplete encoding. The resulting CODECS dictionary # can be used by the main module of ftfy.bad_codecs. CODECS = {} diff --git a/ftfy/badness.py b/ftfy/badness.py index bd092946..d73b291c 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -91,6 +91,7 @@ def _make_weirdness_regex(): regex = '|'.join(groups) return re.compile(regex) + WEIRDNESS_RE = _make_weirdness_regex() # These characters appear in mojibake but also appear commonly on their own. @@ -152,6 +153,7 @@ def _make_weirdness_regex(): 'вЂ[љћ¦°№™ќ“”]' ) + def sequence_weirdness(text): """ Determine how often a text has unexpected characters or sequences of diff --git a/ftfy/build_data.py b/ftfy/build_data.py index d45e2327..3387e444 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -125,5 +125,6 @@ def make_char_data_file(do_it_anyway=False): out.write(zlib.compress(''.join(cclasses).encode('ascii'))) out.close() + if __name__ == '__main__': make_char_data_file() diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 32420969..c36e8081 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -47,6 +47,8 @@ def _build_regexes(): regex = '^[\x00-\x19\x1b-\x7f{0}]*$'.format(charlist) encoding_regexes[encoding] = re.compile(regex) return encoding_regexes + + ENCODING_REGEXES = _build_regexes() @@ -68,6 +70,8 @@ def _build_utf8_punct_regex(): + bytes(range(0x80, 0xa0)).decode('sloppy-windows-1252') + ']') return re.compile(obvious_utf8) + + PARTIAL_UTF8_PUNCT_RE = _build_utf8_punct_regex() @@ -173,6 +177,8 @@ def _build_control_char_mapping(): control_chars[i] = None return control_chars + + CONTROL_CHARS = _build_control_char_mapping() @@ -228,4 +234,6 @@ def _build_width_map(): if alternate != char: width_map[i] = alternate return width_map + + WIDTH_MAP = _build_width_map() diff --git a/ftfy/cli.py b/ftfy/cli.py index 3193466a..596333d7 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -38,6 +38,7 @@ Can't read and write the same file. Please output to a new file instead. """ + def main(): """ Run ftfy as a command-line utility. From 1197ae4746a6ebabc2152035486584757dbb8e5a Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 3 Oct 2018 11:29:26 -0400 Subject: [PATCH 402/502] Update license file - Update my name - Clearly stuff has happened since 2013 --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 4b80d192..cb5ed208 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (C) 2013 Rob Speer (rob@luminoso.com) +Copyright (C) 2013-2018 Robyn Speer (rspeer@luminoso.com) MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of From 51dfbe758438dbb20e29e6db606046bc4d98d685 Mon Sep 17 00:00:00 2001 From: Jacopo Farina Date: Wed, 10 Oct 2018 09:55:11 +0200 Subject: [PATCH 403/502] Fix career URL and use https --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e5d6f6ae..a72f5baa 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,36 @@ # ftfy: fixes text for you [![Travis](https://img.shields.io/travis/LuminosoInsight/python-ftfy/master.svg?label=Travis%20CI)](https://travis-ci.org/LuminosoInsight/python-ftfy) -[![PyPI package](https://badge.fury.io/py/ftfy.svg)](http://badge.fury.io/py/ftfy) -[![Docs](https://readthedocs.org/projects/ftfy/badge/?version=latest)](http://ftfy.readthedocs.org/en/latest/) +[![PyPI package](https://badge.fury.io/py/ftfy.svg)](https://badge.fury.io/py/ftfy) +[![Docs](https://readthedocs.org/projects/ftfy/badge/?version=latest)](https://ftfy.readthedocs.org/en/latest/) ```python >>> print(fix_encoding("(ง'⌣')ง")) (ง'⌣')ง ``` -Full documentation: **http://ftfy.readthedocs.org** +Full documentation: **https://ftfy.readthedocs.org** ## Testimonials - “My life is livable again!” - — [@planarrowspace](http://twitter.com/planarrowspace) + — [@planarrowspace](https://twitter.com/planarrowspace) - “A handy piece of magic” - — [@simonw](http://twitter.com/simonw) + — [@simonw](https://twitter.com/simonw) - “Saved me a large amount of frustrating dev work” - — [@iancal](http://twitter.com/iancal) + — [@iancal](https://twitter.com/iancal) - “ftfy did the right thing right away, with no faffing about. Excellent work, solving a very tricky real-world (whole-world!) problem.” — Brennan Young - “Hat mir die Tage geholfen. Im Übrigen bin ich der Meinung, dass wir keine komplexen Maschinen mit Computern bauen sollten solange wir nicht einmal Umlaute sicher verarbeiten können. :D” - — [Bruno Ranieri](http://yrrsinn.de/2012/09/17/gelesen-kw37/) + — [Bruno Ranieri](https://yrrsinn.de/2012/09/17/gelesen-kw37/) - “I have no idea when I’m gonna need this, but I’m definitely bookmarking it.” - — [/u/ocrow](http://reddit.com/u/ocrow) + — [/u/ocrow](https://reddit.com/u/ocrow) - “9.2/10” — [pylint](https://bitbucket.org/logilab/pylint/) ## Developed at Luminoso -[Luminoso](http://www.luminoso.com) makes groundbreaking software for text +[Luminoso](https://www.luminoso.com) makes groundbreaking software for text analytics that really understands what words mean, in many languages. Our software is used by enterprise customers such as Sony, Intel, Mars, and Scotts, and it's built on Python and open-source technologies. @@ -39,7 +39,7 @@ We use ftfy every day at Luminoso, because the first step in understanding text is making sure it has the correct characters in it! Luminoso is growing fast and hiring. If you're interested in joining us, take a -look at [our careers page](http://www.luminoso.com/career.html). +look at [our careers page](https://luminoso.com/about/work-here). ## What it does @@ -83,7 +83,7 @@ https://ftfy.now.sh/ taste of the things it can do. `fix_encoding` is the more specific function that only fixes mojibake. -Please read [the documentation](http://ftfy.readthedocs.org) for more +Please read [the documentation](https://ftfy.readthedocs.org) for more information on what ftfy does, and how to configure it for your needs. @@ -137,7 +137,7 @@ You can also clone this Git repository and install it with ## Who maintains ftfy? I'm Robyn Speer (rspeer@luminoso.com). I develop this tool as part of my -text-understanding company, [Luminoso](http://luminoso.com), where it has +text-understanding company, [Luminoso](https://luminoso.com), where it has proven essential. Luminoso provides ftfy as free, open source software under the extremely From 26adb80a874856210232d71623d5bd1afde8ce97 Mon Sep 17 00:00:00 2001 From: Moss Collum Date: Fri, 1 Feb 2019 19:03:06 -0500 Subject: [PATCH 404/502] Add Jenkinsfile to drive internal build scripts --- Jenkinsfile | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..07a1a7f1 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,4 @@ +wheelJob( + test: 'pytest', + upstream: [ 'wheelhouse-init' ] +) From 476acc6ad270bffe07f97d4f7cf2139acdc69633 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 13 Mar 2019 11:24:04 -0400 Subject: [PATCH 405/502] Add citation information to docs --- README.md | 29 +++++++++++++++++++++++++++++ docs/index.rst | 30 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/README.md b/README.md index a72f5baa..6ffb42c5 100644 --- a/README.md +++ b/README.md @@ -144,3 +144,32 @@ Luminoso provides ftfy as free, open source software under the extremely permissive MIT license. You can report bugs regarding ftfy on GitHub and we'll handle them. + + +## Citing ftfy + +ftfy has been used as a crucial data processing step in major NLP research. + +It's important to give credit appropriately to everyone whose work you build on +in research. This includes software, not just high-status contributions such as +mathematical models. All I ask when you use ftfy for research is that you cite +it. + +ftfy has a citable record [on Zenodo](https://zenodo.org/record/2591652). +A citation of ftfy may look like this: + + Robyn Speer. (2019). ftfy (Version 5.5). Zenodo. + http://doi.org/10.5281/zenodo.2591652 + +In BibTeX format, the citation is:: + + @misc{speer-2019-ftfy, + author = {Robyn Speer}, + title = {ftfy}, + note = {Version 5.5}, + year = 2019, + howpublished = {Zenodo}, + doi = {10.5281/zenodo.2591652}, + url = {https://doi.org/10.5281/zenodo.2591652} + } + diff --git a/docs/index.rst b/docs/index.rst index a5520a03..2859fb43 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -147,6 +147,36 @@ CJK *text* just fine. It just can't discover that a CJK *encoding* introduced mojibake into the text. +Citing ftfy +----------- +ftfy has been used as a crucial data processing step in major NLP research. + +It's important to give credit appropriately to everyone whose work you build on +in research. This includes software, not just high-status contributions such as +mathematical models. All I ask when you use ftfy for research is that you cite +it. + +ftfy has a citable record [on Zenodo](https://zenodo.org/record/2591652). +A citation of ftfy may look like this: + + Robyn Speer. (2019). ftfy (Version 5.5). Zenodo. + http://doi.org/10.5281/zenodo.2591652 + +In BibTeX format, the citation is:: + + @misc{speer-2019-ftfy, + author = {Robyn Speer}, + title = {ftfy}, + note = {Version 5.5}, + year = 2019, + howpublished = {Zenodo}, + doi = {10.5281/zenodo.2591652}, + url = {https://doi.org/10.5281/zenodo.2591652} + } + + + + Using ftfy ---------- From b81e60eca430e6aa381af228f6a56432fe00159a Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 29 Jul 2019 16:31:44 -0400 Subject: [PATCH 406/502] (skip review) autoformat with isort --- ftfy/__init__.py | 1 + ftfy/badness.py | 1 + ftfy/build_data.py | 2 +- ftfy/chardata.py | 6 ++++-- ftfy/cli.py | 4 ++-- ftfy/fixes.py | 16 +++++++++------- ftfy/formatting.py | 3 ++- 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index dd925c8b..966e477d 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -6,6 +6,7 @@ """ import unicodedata + import ftfy.bad_codecs from ftfy import fixes from ftfy.formatting import display_ljust diff --git a/ftfy/badness.py b/ftfy/badness.py index d73b291c..7fc09120 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -5,6 +5,7 @@ import re import unicodedata + from ftfy.chardata import chars_to_classes # The following regex uses the mapping of character classes to ASCII diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 3387e444..a1d50772 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -11,8 +11,8 @@ The file will be written to the current directory. """ -import unicodedata import sys +import unicodedata import zlib # L = Latin capital letter diff --git a/ftfy/chardata.py b/ftfy/chardata.py index c36e8081..aa516ae9 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -3,10 +3,12 @@ encodings that use them. """ +import html +import itertools import re -import zlib import unicodedata -import itertools +import zlib + from pkg_resources import resource_string # These are the encodings we will try to fix in ftfy, in the diff --git a/ftfy/cli.py b/ftfy/cli.py index 596333d7..121eac34 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -1,10 +1,10 @@ """ A command-line utility for fixing text found in a file. """ -import sys import os -from ftfy import fix_file, __version__ +import sys +from ftfy import __version__, fix_file ENCODE_ERROR_TEXT_UNIX = """ftfy error: Unfortunately, this output stream does not support Unicode. diff --git a/ftfy/fixes.py b/ftfy/fixes.py index decde551..920e51d3 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -3,17 +3,19 @@ can perform. """ -import re import codecs +import html +import re import warnings -from ftfy.chardata import (possible_encoding, CHARMAP_ENCODINGS, - CONTROL_CHARS, LIGATURES, WIDTH_MAP, - PARTIAL_UTF8_PUNCT_RE, ALTERED_UTF8_RE, - LOSSY_UTF8_RE, SINGLE_QUOTE_RE, DOUBLE_QUOTE_RE, - C1_CONTROL_RE) -from ftfy.badness import text_cost from html import entities +from ftfy.badness import text_cost +from ftfy.chardata import ( + ALTERED_UTF8_RE, C1_CONTROL_RE, CHARMAP_ENCODINGS, CONTROL_CHARS, + DOUBLE_QUOTE_RE, + LIGATURES, LOSSY_UTF8_RE, PARTIAL_UTF8_PUNCT_RE, SINGLE_QUOTE_RE, + WIDTH_MAP, possible_encoding +) BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. diff --git a/ftfy/formatting.py b/ftfy/formatting.py index 20d4c65f..59eaaf60 100644 --- a/ftfy/formatting.py +++ b/ftfy/formatting.py @@ -6,7 +6,8 @@ the 'wcwidth' library. """ from unicodedata import normalize -from wcwidth import wcwidth, wcswidth + +from wcwidth import wcswidth, wcwidth def character_width(char): From 2aa3232058e12f95bf79790166efc8b0941f9dff Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 29 Jul 2019 16:33:04 -0400 Subject: [PATCH 407/502] (skip review) autoformat with black --- ftfy/__init__.py | 106 ++++++++++++++++++++++++--------------------- ftfy/badness.py | 11 +++-- ftfy/build_data.py | 4 +- ftfy/chardata.py | 48 ++++++++++---------- ftfy/cli.py | 70 +++++++++++++++++++++--------- ftfy/fixes.py | 41 ++++++++++++------ 6 files changed, 166 insertions(+), 114 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 966e477d..30a833fc 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -18,20 +18,22 @@ ftfy.bad_codecs.ok() -def fix_text(text, - *, - fix_entities='auto', - remove_terminal_escapes=True, - fix_encoding=True, - fix_latin_ligatures=True, - fix_character_width=True, - uncurl_quotes=True, - fix_line_breaks=True, - fix_surrogates=True, - remove_control_chars=True, - remove_bom=True, - normalization='NFC', - max_decode_length=10**6): +def fix_text( + text, + *, + fix_entities='auto', + remove_terminal_escapes=True, + fix_encoding=True, + fix_latin_ligatures=True, + fix_character_width=True, + uncurl_quotes=True, + fix_line_breaks=True, + fix_surrogates=True, + remove_control_chars=True, + remove_bom=True, + normalization='NFC', + max_decode_length=10 ** 6 +): r""" Given Unicode text as input, fix inconsistencies and glitches in it, such as mojibake. @@ -179,7 +181,7 @@ def fix_text(text, fix_surrogates=fix_surrogates, remove_control_chars=remove_control_chars, remove_bom=remove_bom, - normalization=normalization + normalization=normalization, ) ) pos = textbreak @@ -193,20 +195,22 @@ def fix_text(text, fix_text_encoding = fixes.fix_text_encoding # deprecated -def fix_file(input_file, - encoding=None, - *, - fix_entities='auto', - remove_terminal_escapes=True, - fix_encoding=True, - fix_latin_ligatures=True, - fix_character_width=True, - uncurl_quotes=True, - fix_line_breaks=True, - fix_surrogates=True, - remove_control_chars=True, - remove_bom=True, - normalization='NFC'): +def fix_file( + input_file, + encoding=None, + *, + fix_entities='auto', + remove_terminal_escapes=True, + fix_encoding=True, + fix_latin_ligatures=True, + fix_character_width=True, + uncurl_quotes=True, + fix_line_breaks=True, + fix_surrogates=True, + remove_control_chars=True, + remove_bom=True, + normalization='NFC' +): """ Fix text that is found in a file. @@ -238,23 +242,25 @@ def fix_file(input_file, fix_surrogates=fix_surrogates, remove_control_chars=remove_control_chars, remove_bom=remove_bom, - normalization=normalization + normalization=normalization, ) -def fix_text_segment(text, - *, - fix_entities='auto', - remove_terminal_escapes=True, - fix_encoding=True, - fix_latin_ligatures=True, - fix_character_width=True, - uncurl_quotes=True, - fix_line_breaks=True, - fix_surrogates=True, - remove_control_chars=True, - remove_bom=True, - normalization='NFC'): +def fix_text_segment( + text, + *, + fix_entities='auto', + remove_terminal_escapes=True, + fix_encoding=True, + fix_latin_ligatures=True, + fix_character_width=True, + uncurl_quotes=True, + fix_line_breaks=True, + fix_surrogates=True, + remove_control_chars=True, + remove_bom=True, + normalization='NFC' +): """ Apply fixes to text in a single chunk. This could be a line of text within a larger run of `fix_text`, or it could be a larger amount @@ -404,9 +410,11 @@ def explain_unicode(text): display = char else: display = char.encode('unicode-escape').decode('ascii') - print('U+{code:04X} {display} [{category}] {name}'.format( - display=display_ljust(display, 7), - code=ord(char), - category=unicodedata.category(char), - name=unicodedata.name(char, '') - )) + print( + 'U+{code:04X} {display} [{category}] {name}'.format( + display=display_ljust(display, 7), + code=ord(char), + category=unicodedata.category(char), + name=unicodedata.name(char, ''), + ) + ) diff --git a/ftfy/badness.py b/ftfy/badness.py index 7fc09120..7f75ee17 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -86,9 +86,9 @@ def _make_weirdness_regex(): exclusive_categories = 'MmN13' for cat1 in exclusive_categories: others_range = ''.join(c for c in exclusive_categories if c != cat1) - groups.append('{cat1}[{others_range}]'.format( - cat1=cat1, others_range=others_range - )) + groups.append( + '{cat1}[{others_range}]'.format(cat1=cat1, others_range=others_range) + ) regex = '|'.join(groups) return re.compile(regex) @@ -184,9 +184,8 @@ def sequence_weirdness(text): """ text2 = unicodedata.normalize('NFC', text) weirdness = len(WEIRDNESS_RE.findall(chars_to_classes(text2))) - adjustment = ( - len(MOJIBAKE_SYMBOL_RE.findall(text2)) * 2 - - len(COMMON_SYMBOL_RE.findall(text2)) + adjustment = len(MOJIBAKE_SYMBOL_RE.findall(text2)) * 2 - len( + COMMON_SYMBOL_RE.findall(text2) ) return weirdness * 2 + adjustment diff --git a/ftfy/build_data.py b/ftfy/build_data.py index a1d50772..4f10961e 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -47,9 +47,7 @@ def make_char_data_file(do_it_anyway=False): raise an error unless you pass `do_it_anyway=True`. """ if sys.hexversion < 0x030700f0 and not do_it_anyway: - raise RuntimeError( - "This function should be run in Python 3.7.0 or later." - ) + raise RuntimeError("This function should be run in Python 3.7.0 or later.") cclasses = [None] * 0x110000 for codepoint in range(0x0, 0x110000): diff --git a/ftfy/chardata.py b/ftfy/chardata.py index aa516ae9..3ec4381b 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -68,9 +68,11 @@ def _build_utf8_punct_regex(): # are a contiguous range, as well as the different Windows-1252 decodings # of 0x80 to 0x9f, which are not contiguous at all. (Latin-1 and # Windows-1252 agree on bytes 0xa0 and up.) - obvious_utf8 = ('â[€\x80][\x80-\xbf' - + bytes(range(0x80, 0xa0)).decode('sloppy-windows-1252') - + ']') + obvious_utf8 = ( + 'â[€\x80][\x80-\xbf' + + bytes(range(0x80, 0xa0)).decode('sloppy-windows-1252') + + ']' + ) return re.compile(obvious_utf8) @@ -98,12 +100,14 @@ def _build_utf8_punct_regex(): # We should consider checking for b'\x85' being converted to ... in the future. # I've seen it once, but the text still wasn't recoverable. -ALTERED_UTF8_RE = re.compile(b'[\xc2\xc3\xc5\xce\xd0][ ]' - b'|[\xe0-\xef][ ][\x80-\xbf]' - b'|[\xe0-\xef][\x80-\xbf][ ]' - b'|[\xf0-\xf4][ ][\x80-\xbf][\x80-\xbf]' - b'|[\xf0-\xf4][\x80-\xbf][ ][\x80-\xbf]' - b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]') +ALTERED_UTF8_RE = re.compile( + b'[\xc2\xc3\xc5\xce\xd0][ ]' + b'|[\xe0-\xef][ ][\x80-\xbf]' + b'|[\xe0-\xef][\x80-\xbf][ ]' + b'|[\xf0-\xf4][ ][\x80-\xbf][\x80-\xbf]' + b'|[\xf0-\xf4][\x80-\xbf][ ][\x80-\xbf]' + b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]' +) # This expression matches UTF-8 and CESU-8 sequences where some of the # continuation bytes have been lost. The byte 0x1a (sometimes written as ^Z) is @@ -166,15 +170,15 @@ def _build_control_char_mapping(): control_chars = {} for i in itertools.chain( - range(0x00, 0x09), - [0x0b], - range(0x0e, 0x20), - [0x7f], - range(0x206a, 0x2070), - [0xfeff], - range(0xfff9, 0xfffd), - range(0x1d173, 0x1d17b), - range(0xe0000, 0xe0080) + range(0x00, 0x09), + [0x0b], + range(0x0e, 0x20), + [0x7f], + range(0x206a, 0x2070), + [0xfeff], + range(0xfff9, 0xfffd), + range(0x1d173, 0x1d17b), + range(0xe0000, 0xe0080), ): control_chars[i] = None @@ -196,10 +200,10 @@ def _build_control_char_mapping(): # is sometimes more normalization than you want. LIGATURES = { - ord('IJ'): 'IJ', # Dutch ligatures + ord('IJ'): 'IJ', # Dutch ligatures ord('ij'): 'ij', - ord('ʼn'): "ʼn", # Afrikaans digraph meant to avoid auto-curled quote - ord('DZ'): 'DZ', # Serbian/Croatian digraphs for Cyrillic conversion + ord('ʼn'): "ʼn", # Afrikaans digraph meant to avoid auto-curled quote + ord('DZ'): 'DZ', # Serbian/Croatian digraphs for Cyrillic conversion ord('Dz'): 'Dz', ord('dz'): 'dz', ord('DŽ'): 'DŽ', @@ -211,7 +215,7 @@ def _build_control_char_mapping(): ord('NJ'): 'NJ', ord('Nj'): 'Nj', ord('nj'): "nj", - ord('ff'): 'ff', # Latin typographical ligatures + ord('ff'): 'ff', # Latin typographical ligatures ord('fi'): 'fi', ord('fl'): 'fl', ord('ffi'): 'ffi', diff --git a/ftfy/cli.py b/ftfy/cli.py index 121eac34..41c36718 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -48,24 +48,49 @@ def main(): parser = argparse.ArgumentParser( description="ftfy (fixes text for you), version %s" % __version__ ) - parser.add_argument('filename', default='-', nargs='?', - help='The file whose Unicode is to be fixed. Defaults ' - 'to -, meaning standard input.') - parser.add_argument('-o', '--output', type=str, default='-', - help='The file to output to. Defaults to -, meaning ' - 'standard output.') - parser.add_argument('-g', '--guess', action='store_true', - help="Ask ftfy to guess the encoding of your input. " - "This is risky. Overrides -e.") - parser.add_argument('-e', '--encoding', type=str, default='utf-8', - help='The encoding of the input. Defaults to UTF-8.') - parser.add_argument('-n', '--normalization', type=str, default='NFC', - help='The normalization of Unicode to apply. ' - 'Defaults to NFC. Can be "none".') - parser.add_argument('--preserve-entities', action='store_true', - help="Leave HTML entities as they are. The default " - "is to decode them, as long as no HTML tags " - "have appeared in the file.") + parser.add_argument( + 'filename', + default='-', + nargs='?', + help='The file whose Unicode is to be fixed. Defaults ' + 'to -, meaning standard input.', + ) + parser.add_argument( + '-o', + '--output', + type=str, + default='-', + help='The file to output to. Defaults to -, meaning ' 'standard output.', + ) + parser.add_argument( + '-g', + '--guess', + action='store_true', + help="Ask ftfy to guess the encoding of your input. " + "This is risky. Overrides -e.", + ) + parser.add_argument( + '-e', + '--encoding', + type=str, + default='utf-8', + help='The encoding of the input. Defaults to UTF-8.', + ) + parser.add_argument( + '-n', + '--normalization', + type=str, + default='NFC', + help='The normalization of Unicode to apply. ' + 'Defaults to NFC. Can be "none".', + ) + parser.add_argument( + '--preserve-entities', + action='store_true', + help="Leave HTML entities as they are. The default " + "is to decode them, as long as no HTML tags " + "have appeared in the file.", + ) args = parser.parse_args() @@ -98,9 +123,12 @@ def main(): fix_entities = 'auto' try: - for line in fix_file(file, encoding=encoding, - fix_entities=fix_entities, - normalization=normalization): + for line in fix_file( + file, + encoding=encoding, + fix_entities=fix_entities, + normalization=normalization, + ): try: outfile.write(line) except UnicodeEncodeError: diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 920e51d3..64f337f6 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -11,10 +11,17 @@ from ftfy.badness import text_cost from ftfy.chardata import ( - ALTERED_UTF8_RE, C1_CONTROL_RE, CHARMAP_ENCODINGS, CONTROL_CHARS, + ALTERED_UTF8_RE, + C1_CONTROL_RE, + CHARMAP_ENCODINGS, + CONTROL_CHARS, DOUBLE_QUOTE_RE, - LIGATURES, LOSSY_UTF8_RE, PARTIAL_UTF8_PUNCT_RE, SINGLE_QUOTE_RE, - WIDTH_MAP, possible_encoding + LIGATURES, + LOSSY_UTF8_RE, + PARTIAL_UTF8_PUNCT_RE, + SINGLE_QUOTE_RE, + WIDTH_MAP, + possible_encoding, ) BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. @@ -114,8 +121,7 @@ def fix_text_encoding(text): """ A deprecated name for :func:`ftfy.fixes.fix_encoding`. """ - warnings.warn('fix_text_encoding is now known as fix_encoding', - DeprecationWarning) + warnings.warn('fix_text_encoding is now known as fix_encoding', DeprecationWarning) return fix_encoding(text) @@ -242,8 +248,7 @@ def fix_one_step_and_explain(text): fixed = encoded.decode('windows-1252') steps = [] if fixed != text: - steps = [('encode', 'latin-1', 0), - ('decode', 'windows-1252', 1)] + steps = [('encode', 'latin-1', 0), ('decode', 'windows-1252', 1)] return fixed, steps except UnicodeDecodeError: # This text contained characters that don't even make sense @@ -447,9 +452,13 @@ def fix_line_breaks(text): >>> eprint(fix_line_breaks("What is this \x85 I don't even")) What is this \n I don't even """ - return text.replace('\r\n', '\n').replace('\r', '\n')\ - .replace('\u2028', '\n').replace('\u2029', '\n')\ - .replace('\u0085', '\n') + return ( + text.replace('\r\n', '\n') + .replace('\r', '\n') + .replace('\u2028', '\n') + .replace('\u2029', '\n') + .replace('\u0085', '\n') + ) SURROGATE_RE = re.compile('[\ud800-\udfff]') @@ -534,14 +543,17 @@ def remove_bom(text): # Define a regex to match valid escape sequences in Python string literals. -ESCAPE_SEQUENCE_RE = re.compile(r''' +ESCAPE_SEQUENCE_RE = re.compile( + r''' ( \\U........ # 8-digit hex escapes | \\u.... # 4-digit hex escapes | \\x.. # 2-digit hex escapes | \\[0-7]{1,3} # Octal escapes | \\N\{[^}]+\} # Unicode characters by name | \\[\\'"abfnrtv] # Single-character escapes - )''', re.UNICODE | re.VERBOSE) + )''', + re.UNICODE | re.VERBOSE, +) def decode_escapes(text): @@ -574,6 +586,7 @@ def decode_escapes(text): because escaped text is not necessarily a mistake, and there is no way to distinguish text that's supposed to be escaped from text that isn't. """ + def decode_match(match): "Given a regex match, decode the escape sequence it contains." return codecs.decode(match.group(0), 'unicode-escape') @@ -595,6 +608,7 @@ def restore_byte_a0(byts): This is used as a step within `fix_encoding`. """ + def replacement(match): "The function to apply when this regex matches." return match.group(0).replace(b'\x20', b'\xa0') @@ -654,6 +668,7 @@ def fix_partial_utf8_punct_in_1252(text): This is used as a transcoder within `fix_encoding`. """ + def latin1_to_w1252(match): "The function to apply when this regex matches." return match.group(0).encode('latin-1').decode('sloppy-windows-1252') @@ -669,5 +684,5 @@ def w1252_to_utf8(match): TRANSCODERS = { 'restore_byte_a0': restore_byte_a0, 'replace_lossy_sequences': replace_lossy_sequences, - 'fix_partial_utf8_punct_in_1252': fix_partial_utf8_punct_in_1252 + 'fix_partial_utf8_punct_in_1252': fix_partial_utf8_punct_in_1252, } From 11614f85131c0832e20bc713b6dc45a77ba6c763 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 29 Jul 2019 16:29:50 -0400 Subject: [PATCH 408/502] Update how HTML entities are decoded This now includes the standard library's built-in html.unescape, but with an extra fix-up for nonstandardly uppercased entities. --- ftfy/chardata.py | 15 +++++++ ftfy/fixes.py | 97 ++++++++++++++++++++++-------------------- tests/test_entities.py | 4 +- 3 files changed, 68 insertions(+), 48 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 3ec4381b..505f4e30 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -54,6 +54,21 @@ def _build_regexes(): ENCODING_REGEXES = _build_regexes() +def _build_html_uppercase_entities(): + uppercase_entities = {} + for name, codept in html.entities.name2codepoint.items(): + if codept < 0x300: + name_upper = name.upper() + entity_upper = '&' + name_upper + ';' + if name_upper != name and html.unescape(entity_upper) == entity_upper: + uppercase_entities[entity_upper] = chr(codept).upper() + return uppercase_entities + + +HTML_UPPERCASE_ENTITY_RE = re.compile(r"&[A-Z]{3,6};") +HTML_UPPERCASE_ENTITIES = _build_html_uppercase_entities() + + def _build_utf8_punct_regex(): """ Recognize UTF-8 mojibake that's so blatant that we can fix it even when the diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 64f337f6..4e35d8b4 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -16,6 +16,8 @@ CHARMAP_ENCODINGS, CONTROL_CHARS, DOUBLE_QUOTE_RE, + HTML_UPPERCASE_ENTITIES, + HTML_UPPERCASE_ENTITY_RE, LIGATURES, LOSSY_UTF8_RE, PARTIAL_UTF8_PUNCT_RE, @@ -61,54 +63,54 @@ def fix_encoding(text): `fix_encoding` decodes text that looks like it was decoded incorrectly. It leaves alone text that doesn't. - >>> print(fix_encoding('único')) - único + >>> fix_encoding('único') + 'único' - >>> print(fix_encoding('This text is fine already :þ')) - This text is fine already :þ + >>> fix_encoding('This text is fine already :þ') + 'This text is fine already :þ' Because these characters often come from Microsoft products, we allow for the possibility that we get not just Unicode characters 128-255, but also Windows's conflicting idea of what characters 128-160 are. - >>> print(fix_encoding('This — should be an em dash')) - This — should be an em dash + >>> fix_encoding('This — should be an em dash') + 'This — should be an em dash' We might have to deal with both Windows characters and raw control characters at the same time, especially when dealing with characters like 0x81 that have no mapping in Windows. This is a string that Python's standard `.encode` and `.decode` methods cannot correct. - >>> print(fix_encoding('This text is sad .â\x81”.')) - This text is sad .⁔. + >>> fix_encoding('This text is sad .â\x81”.') + 'This text is sad .⁔.' However, it has safeguards against fixing sequences of letters and punctuation that can occur in valid text. In the following example, the last three characters are not replaced with a Korean character, even though they could be. - >>> print(fix_encoding('not such a fan of Charlotte Brontë…”')) - not such a fan of Charlotte Brontë…” + >>> fix_encoding('not such a fan of Charlotte Brontë…”') + 'not such a fan of Charlotte Brontë…”' This function can now recover some complex manglings of text, such as when UTF-8 mojibake has been normalized in a way that replaces U+A0 with a space: - >>> print(fix_encoding('The more you know 🌠')) - The more you know 🌠 + >>> fix_encoding('The more you know 🌠') + 'The more you know 🌠' Cases of genuine ambiguity can sometimes be addressed by finding other characters that are not double-encoded, and expecting the encoding to be consistent: - >>> print(fix_encoding('AHÅ™, the new sofa from IKEA®')) - AHÅ™, the new sofa from IKEA® + >>> fix_encoding('AHÅ™, the new sofa from IKEA®') + 'AHÅ™, the new sofa from IKEA®' Finally, we handle the case where the text is in a single-byte encoding that was intended as Windows-1252 all along but read as Latin-1: - >>> print(fix_encoding('This text was never UTF-8 at all\x85')) - This text was never UTF-8 at all… + >>> fix_encoding('This text was never UTF-8 at all\x85') + 'This text was never UTF-8 at all…' The best version of the text is found using :func:`ftfy.badness.text_cost`. @@ -297,51 +299,52 @@ def apply_plan(text, plan): return obj -HTML_ENTITY_RE = re.compile(r"&#?\w{0,8};") - - def _unescape_fixup(match): """ Replace one matched HTML entity with the character it represents, if possible. """ text = match.group(0) - if text[:2] == "&#": - # character reference - try: - if text[:3] == "&#x": - codept = int(text[3:-1], 16) - else: - codept = int(text[2:-1]) - if 0x80 <= codept < 0xa0: - # Decode this range of characters as Windows-1252, as Web - # browsers do in practice. - return bytes([codept]).decode('sloppy-windows-1252') - else: - return chr(codept) - except ValueError: - return text + if text in HTML_UPPERCASE_ENTITIES: + return HTML_UPPERCASE_ENTITIES[text] else: - # This is a named entity; if it's a known HTML5 entity, replace - # it with the appropriate character. - try: - return entities.html5[text[1:]] - except KeyError: - return text + return text def unescape_html(text): """ - Decode all three types of HTML entities/character references. + Decode HTML entities and character references, including if they've been + case-folded. + + By now, the process of decoding HTML escapes is _mostly_ taken care of by + `html.unescape` in the standard library. But if you can mess up whether + text was meant to be decoded as HTML, you can also mess up how that + interacts with, say, case-folding. + + If a database contains the name 'P&EACUTE;REZ', we can read that and intuit + that it was supposed to say 'PÉREZ'. But HTML entities are case-sensitive, + often in ways that change which character it is, so the standard library + won't even decode '&EACUTE;'. + + There are cases where case-folded entities would be ambiguous in complicated + ways. But we can avoid the complicated cases if we limit ourselves to decoding + uppercased entities with codepoints below U+300, and decoding those as the + uppercased versions. + + >>> unescape_html('<tag>') + '' + + >>> unescape_html('P&EACUTE;REZ') + 'PÉREZ' - Code by Fredrik Lundh of effbot.org. Robyn Speer made a slight change - to it for efficiency: it won't match entities longer than 8 characters, - because there are no valid entities like that. + >>> unescape_html('BUNDESSTRA&SZLIG;E') + 'BUNDESSTRASSE' - >>> print(unescape_html('<tag>')) - + >>> unescape_html('ñ Ñ &NTILDE; &nTILDE;') + 'ñ Ñ Ñ &nTILDE;' """ - return HTML_ENTITY_RE.sub(_unescape_fixup, text) + text2 = html.unescape(text) + return HTML_UPPERCASE_ENTITY_RE.sub(_unescape_fixup, text2) ANSI_RE = re.compile('\033\\[((?:\\d|;)*)([a-zA-Z])') diff --git a/tests/test_entities.py b/tests/test_entities.py index 0a2892b2..319b8edf 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -22,4 +22,6 @@ def test_entities(): assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...' assert fix_text_segment('broken') == 'broken\x81' assert unescape_html('euro €') == 'euro €' - assert unescape_html('not an entity x6;') == 'not an entity x6;' + assert unescape_html('not an entity &#y20x6;') == 'not an entity &#y20x6;' + assert unescape_html('partial entity x6;') == 'partial entity x6;' + assert unescape_html('&&&') == '&&&' From 89c0a33a9bf2ee5201ac3253546886e171b5b6c6 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 29 Jul 2019 16:34:56 -0400 Subject: [PATCH 409/502] remove unused import --- ftfy/fixes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 4e35d8b4..29d7fffc 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -7,7 +7,6 @@ import html import re import warnings -from html import entities from ftfy.badness import text_cost from ftfy.chardata import ( From 08c2c2545d0398977a4cdd30c365cbe822f7d26b Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 30 Jul 2019 12:51:14 -0400 Subject: [PATCH 410/502] update Python version requirements --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5a1b13ba..9c8cbdd3 100644 --- a/setup.py +++ b/setup.py @@ -40,14 +40,14 @@ package_data={'ftfy': ['char_classes.dat']}, install_requires=['wcwidth'], tests_require=['pytest'], - python_requires='>=3.3', + python_requires='>=3.4', classifiers=[ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", From 67c52512c31602e1cb2d19fe541494a0ed42dcad Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 30 Jul 2019 12:52:30 -0400 Subject: [PATCH 411/502] update version number --- docs/conf.py | 4 ++-- ftfy/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 790afe7c..76e305ef 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ # built documents. # # The short X.Y version. -version = '5.5' +version = '5.6' # The full version, including alpha/beta/rc tags. -release = '5.5.2' +release = '5.6.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 30a833fc..97c796de 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -11,7 +11,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.5.1' +__version__ = '5.6' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/setup.py b/setup.py index 9c8cbdd3..4d70b380 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( name="ftfy", - version='5.5.1', + version='5.6', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From fd010d2a209beba47ffa2fd8a2d12ed0c896140a Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 30 Jul 2019 12:55:33 -0400 Subject: [PATCH 412/502] update changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da75f769..b240752f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## Version 5.6 (July 30, 2019) + +- Make use of `html.unescape` for more robust unescaping, now that it's + available in all current versions of Python 3. + +- On top of what `html.unescape` supports, add the ability to convert HTML + escapes of common Latin capital letters that are (nonstandardly) written + in all caps, such as `&NTILDE;` for `Ñ`. + + ## Version 5.5.1 (September 14, 2018) - Added Python 3.7 support. From 4d727d6be02b41da5b2373ba934093593dd23cb0 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 31 Jul 2019 11:47:12 -0400 Subject: [PATCH 413/502] Refine the limitations on uppercase entity names --- ftfy/chardata.py | 30 +++++++++++++++++++++++------- tests/test_entities.py | 4 ++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 505f4e30..f7f74d6b 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -56,16 +56,32 @@ def _build_regexes(): def _build_html_uppercase_entities(): uppercase_entities = {} - for name, codept in html.entities.name2codepoint.items(): - if codept < 0x300: - name_upper = name.upper() - entity_upper = '&' + name_upper + ';' - if name_upper != name and html.unescape(entity_upper) == entity_upper: - uppercase_entities[entity_upper] = chr(codept).upper() + for name, char in html.entities.html5.items(): + # Restrict the set of characters we can attempt to decode if their + # name has been uppercased. If we tried to handle all entity names, + # the results would be ambiguous and the dictionary would be large. + # The limitations here produce a set of 186 characters. + if ( + name == name.lower() + and name.endswith(';') + and len(name) <= 7 + and len(char) == 1 + ): + codept = ord(char) + # Include common Latin letters, Latin-1 symbols, plus some of + # the extra symbols from Windows-1252 + if codept < 0x300 or char in '€‚„…‘“”•–—™': + name_upper = name.upper() + entity_upper = '&' + name_upper + if html.unescape(entity_upper) == entity_upper: + uppercase_entities[entity_upper] = char.upper() return uppercase_entities -HTML_UPPERCASE_ENTITY_RE = re.compile(r"&[A-Z]{3,6};") +# HTML5 entities have names with 2 to 24 characters. The ones we care about +# have names no longer than 6 characters. Limiting this range makes the regex +# more efficient. +HTML_UPPERCASE_ENTITY_RE = re.compile(r"&[A-Z]{2,6};") HTML_UPPERCASE_ENTITIES = _build_html_uppercase_entities() diff --git a/tests/test_entities.py b/tests/test_entities.py index 319b8edf..093b5a42 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -22,6 +22,10 @@ def test_entities(): assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...' assert fix_text_segment('broken') == 'broken\x81' assert unescape_html('euro €') == 'euro €' + assert unescape_html('EURO &EURO;') == 'EURO €' assert unescape_html('not an entity &#y20x6;') == 'not an entity &#y20x6;' assert unescape_html('partial entity x6;') == 'partial entity x6;' assert unescape_html('&&&') == '&&&' + assert unescape_html('this is terrible') == 'this\xa0is\xa0terrible' + assert unescape_html('JEDNOCZE&SACUTE;NIE') == 'JEDNOCZEŚNIE' + assert unescape_html('V&SCARON;ICHNI') == 'VŠICHNI' From 66a599c4780912a47ab03a69492843c6b2ed1de4 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 31 Jul 2019 17:15:45 -0400 Subject: [PATCH 414/502] Don't rely on html.unescape for everything --- ftfy/chardata.py | 40 ++++++++++++--------------- ftfy/fixes.py | 62 ++++++++++++++++++++++++++++-------------- tests/test_entities.py | 10 ++++--- 3 files changed, 65 insertions(+), 47 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index f7f74d6b..3bb09d86 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -54,35 +54,29 @@ def _build_regexes(): ENCODING_REGEXES = _build_regexes() -def _build_html_uppercase_entities(): - uppercase_entities = {} +def _build_html_entities(): + entities = dict(html.entities.html5) + entities = {} + # Create a dictionary based on the built-in HTML5 entity dictionary. + # Add a limited set of HTML entities that we'll also decode if they've + # been case-folded to uppercase, such as decoding &NTILDE; as "Ñ". for name, char in html.entities.html5.items(): - # Restrict the set of characters we can attempt to decode if their - # name has been uppercased. If we tried to handle all entity names, - # the results would be ambiguous and the dictionary would be large. - # The limitations here produce a set of 186 characters. - if ( - name == name.lower() - and name.endswith(';') - and len(name) <= 7 - and len(char) == 1 - ): - codept = ord(char) - # Include common Latin letters, Latin-1 symbols, plus some of - # the extra symbols from Windows-1252 - if codept < 0x300 or char in '€‚„…‘“”•–—™': + if name.endswith(';'): + entities['&' + name] = char + + # Restrict the set of characters we can attempt to decode if their + # name has been uppercased. If we tried to handle all entity names, + # the results would be ambiguous. + if name == name.lower(): name_upper = name.upper() entity_upper = '&' + name_upper if html.unescape(entity_upper) == entity_upper: - uppercase_entities[entity_upper] = char.upper() - return uppercase_entities + entities[entity_upper] = char.upper() + return entities -# HTML5 entities have names with 2 to 24 characters. The ones we care about -# have names no longer than 6 characters. Limiting this range makes the regex -# more efficient. -HTML_UPPERCASE_ENTITY_RE = re.compile(r"&[A-Z]{2,6};") -HTML_UPPERCASE_ENTITIES = _build_html_uppercase_entities() +HTML_ENTITY_RE = re.compile(r"&#?[0-9A-Za-z]{1,24};") +HTML_ENTITIES = _build_html_entities() def _build_utf8_punct_regex(): diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 29d7fffc..9fdaec98 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -15,8 +15,8 @@ CHARMAP_ENCODINGS, CONTROL_CHARS, DOUBLE_QUOTE_RE, - HTML_UPPERCASE_ENTITIES, - HTML_UPPERCASE_ENTITY_RE, + HTML_ENTITIES, + HTML_ENTITY_RE, LIGATURES, LOSSY_UTF8_RE, PARTIAL_UTF8_PUNCT_RE, @@ -304,35 +304,58 @@ def _unescape_fixup(match): if possible. """ text = match.group(0) - if text in HTML_UPPERCASE_ENTITIES: - return HTML_UPPERCASE_ENTITIES[text] + if text in HTML_ENTITIES: + return HTML_ENTITIES[text] + elif text.startswith('&#'): + unescaped = html.unescape(text) + + # If html.unescape only decoded part of the string, that's not what + # we want. We're expecting the &# sequence to decode as a single + # character. + if len(unescaped) == 1: + return unescaped + else: + return text else: return text def unescape_html(text): """ - Decode HTML entities and character references, including if they've been - case-folded. + Decode HTML entities and character references, including some nonstandard + ones written in all-caps. - By now, the process of decoding HTML escapes is _mostly_ taken care of by - `html.unescape` in the standard library. But if you can mess up whether - text was meant to be decoded as HTML, you can also mess up how that - interacts with, say, case-folding. + Python has a built-in called `html.unescape` that can decode HTML escapes, + including a bunch of messy edge cases such as decoding escapes without + semicolons such as "&". - If a database contains the name 'P&EACUTE;REZ', we can read that and intuit - that it was supposed to say 'PÉREZ'. But HTML entities are case-sensitive, - often in ways that change which character it is, so the standard library - won't even decode '&EACUTE;'. + If you know you've got HTML-escaped text, applying `html.unescape` is the + right way to convert it to plain text. But in ambiguous situations, that + would create false positives. For example, the informally written text + "this¬ that" should not automatically be decoded as "this¬ that". - There are cases where case-folded entities would be ambiguous in complicated - ways. But we can avoid the complicated cases if we limit ourselves to decoding - uppercased entities with codepoints below U+300, and decoding those as the - uppercased versions. + In this function, we decode the escape sequences that appear in the + `html.entities.html5` dictionary, as long as they are the unambiguous ones + that end in semicolons. + + We also decode all-caps versions of Latin letters and common symbols. + If a database contains the name 'P&EACUTE;REZ', we can read that and intuit + that it was supposed to say 'PÉREZ'. This is limited to a smaller set of + entities, because there are many instances where entity names are + case-sensitive in complicated ways. >>> unescape_html('<tag>') '' + >>> unescape_html('𝒥ohn ℋancock') + '𝒥ohn ℋancock' + + >>> unescape_html('✓') + '✓' + + >>> unescape_html('Pérez') + 'Pérez' + >>> unescape_html('P&EACUTE;REZ') 'PÉREZ' @@ -342,8 +365,7 @@ def unescape_html(text): >>> unescape_html('ñ Ñ &NTILDE; &nTILDE;') 'ñ Ñ Ñ &nTILDE;' """ - text2 = html.unescape(text) - return HTML_UPPERCASE_ENTITY_RE.sub(_unescape_fixup, text2) + return HTML_ENTITY_RE.sub(_unescape_fixup, text) ANSI_RE = re.compile('\033\\[((?:\\d|;)*)([a-zA-Z])') diff --git a/tests/test_entities.py b/tests/test_entities.py index 093b5a42..a9ca37ea 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -21,11 +21,13 @@ def test_entities(): assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...' assert fix_text_segment('ellipsis…', normalization='NFKC') == 'ellipsis...' assert fix_text_segment('broken') == 'broken\x81' + assert fix_text_segment('&amp;amp;') == '&' assert unescape_html('euro €') == 'euro €' assert unescape_html('EURO &EURO;') == 'EURO €' - assert unescape_html('not an entity &#y20x6;') == 'not an entity &#y20x6;' - assert unescape_html('partial entity x6;') == 'partial entity x6;' - assert unescape_html('&&&') == '&&&' - assert unescape_html('this is terrible') == 'this\xa0is\xa0terrible' + assert unescape_html('not an entity x6;') == 'not an entity x6;' assert unescape_html('JEDNOCZE&SACUTE;NIE') == 'JEDNOCZEŚNIE' assert unescape_html('V&SCARON;ICHNI') == 'VŠICHNI' + assert ( + fix_text_segment('this is just informal english ¬ html') == + 'this is just informal english ¬ html' + ) From 419fa84ad36247d18c23d7008a621eb5c1b4123c Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 31 Jul 2019 17:22:01 -0400 Subject: [PATCH 415/502] handle some codepoint-number edge cases --- ftfy/fixes.py | 9 ++++----- tests/test_entities.py | 2 ++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 9fdaec98..0bcb48cd 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -310,12 +310,11 @@ def _unescape_fixup(match): unescaped = html.unescape(text) # If html.unescape only decoded part of the string, that's not what - # we want. We're expecting the &# sequence to decode as a single - # character. - if len(unescaped) == 1: - return unescaped - else: + # we want. The semicolon should be consumed. + if ';' in unescaped: return text + else: + return unescaped else: return text diff --git a/tests/test_entities.py b/tests/test_entities.py index a9ca37ea..2d05e275 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -27,6 +27,8 @@ def test_entities(): assert unescape_html('not an entity x6;') == 'not an entity x6;' assert unescape_html('JEDNOCZE&SACUTE;NIE') == 'JEDNOCZEŚNIE' assert unescape_html('V&SCARON;ICHNI') == 'VŠICHNI' + assert unescape_html('￿') == '' + assert unescape_html('�') == '\ufffd' assert ( fix_text_segment('this is just informal english ¬ html') == 'this is just informal english ¬ html' From 32150103daca7781bb89761c4d2bbc4db7956287 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 7 Aug 2019 16:32:36 -0400 Subject: [PATCH 416/502] Fix changelog for the revision to unescaping --- CHANGELOG.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b240752f..f14f27ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,18 @@ -## Version 5.6 (July 30, 2019) +## Version 5.6 (August 7, 2019) -- Make use of `html.unescape` for more robust unescaping, now that it's - available in all current versions of Python 3. +- The `unescape_html` function now supports all the HTML5 entities that appear + in `html.entities.html5`, including those with long names such as + `˝`. -- On top of what `html.unescape` supports, add the ability to convert HTML +- Unescaping of numeric HTML entities now uses the standard library's + `html.unescape`, making edge cases consistent. + + (The reason we don't run `html.unescape` on all text is that it's not always + appropriate to apply, and can lead to false positive fixes. The text + "This&NotThat" should not have "&Not" replaced by a symbol, as + `html.unescape` would do.) + +- On top of Python's support for HTML5 entities, ftfy will also convert HTML escapes of common Latin capital letters that are (nonstandardly) written in all caps, such as `&NTILDE;` for `Ñ`. From f6ec495766a01e35711e6e6b3b251f57356363f5 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 9 Aug 2019 15:37:12 -0400 Subject: [PATCH 417/502] Fix duplicated creation of entities dict --- ftfy/chardata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 3bb09d86..7db1e888 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -55,7 +55,6 @@ def _build_regexes(): def _build_html_entities(): - entities = dict(html.entities.html5) entities = {} # Create a dictionary based on the built-in HTML5 entity dictionary. # Add a limited set of HTML entities that we'll also decode if they've From f59b0e09e13805eb9db6eb73de29c5fb26a3ca6a Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 18 Dec 2019 13:47:55 -0500 Subject: [PATCH 418/502] Add a mailmap --- .mailmap | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..c0a85804 --- /dev/null +++ b/.mailmap @@ -0,0 +1,6 @@ +# Robyn has used different names and e-mail addresses in the course of this project. Map them all to her current name and e-mail. +Robyn Speer +Robyn Speer +Robyn Speer +Robyn Speer + From 82b3501c946df8667eb1809b08645acd70531ede Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 3 Feb 2020 12:43:23 -0500 Subject: [PATCH 419/502] Update to Unicode 12.1 (fixes #131) --- ftfy/build_data.py | 12 ++++++------ ftfy/char_classes.dat | Bin 4121 -> 4187 bytes setup.py | 10 ++-------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 4f10961e..b3e3f257 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -5,7 +5,7 @@ classes we care about change, or if a new version of Python supports a new Unicode standard and we want it to affect our string decoding. -The file that we generate is based on Unicode 11.0, as supported by Python 3.7. +The file that we generate is based on Unicode 12.1, as supported by Python 3.8. You can certainly use it in earlier versions. This simply makes sure that we get consistent results from running ftfy on different versions of Python. @@ -39,15 +39,15 @@ def make_char_data_file(do_it_anyway=False): Build the compressed data file 'char_classes.dat' and write it to the current directory. - If you run this, run it in Python 3.7.0 or later. It will run in earlier - versions, but you won't get the Unicode 11 standard, leading to inconsistent - behavior. Pre-releases of Python 3.7 won't work (Unicode 11 wasn't out yet). + If you run this, run it in Python 3.8 or later. It will run in earlier + versions, but you won't get the Unicode 12 standard, leading to + inconsistent behavior. To protect against this, running this in the wrong version of Python will raise an error unless you pass `do_it_anyway=True`. """ - if sys.hexversion < 0x030700f0 and not do_it_anyway: - raise RuntimeError("This function should be run in Python 3.7.0 or later.") + if sys.version_info < (3, 8) and not do_it_anyway: + raise RuntimeError("This function should be run in Python 3.8 or later.") cclasses = [None] * 0x110000 for codepoint in range(0x0, 0x110000): diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat index 830e65499affef481e315709c09c413196088051..869f346155069e782b0e83814744836e60c39766 100644 GIT binary patch literal 4187 zcmeH|Sx^&L7KX8VlE0uAqm;1$LXo5d77%3mwA}`a8KPj=RfDvSM}G; ztYXi(K%R_#q4KtwcJiBmZ?d$j4dkzp-vj#8Un*kmpsN$tXFNB~jC<@+pPW#iK}bZ5 zVGMAI{1RhFu6aHoTXFO`wHD>0yBzPMwO}#eB5^eIsp=a-9D>UBJ7X~7(`1H&i8zLv z{yn=v+;czuJyC2e{(x@f#!bc1?*l`SN zP~x{13bm+&gewyT9JB#L?VjC?R3LxY+Z#c#=n2P(bAmH-a>xt@gRQ*m+T`Z%>>H z@eO~zis?uWnld)_UHyYIZ_(iP;46OQiYv6d31*8Kyng6yz%Lnh2|=%si!r9iUbkb%B91!Euhu zNm<#$3AB~C`(?S8$D&CmHIky^pSdsUu)mzOc=)Tepbu|WkEjbM%I~~c4r&>l^FM!K zIxP5zDb2OITGhX(#w{FIE-dddkPGY2*Shq&^?7uWB9RyeIX3zp^lk9U!P2l5z*q#R z))V&J<<=O+nVtVLDM|qJ02;wQ1dKrCBZzKc!5^P(2^e*hrm9ZP>XE^wtH^BMn=Tg> zhjo`JCf^bF4avvbY9#f1sf~O77Io1|tLi-r`G_3-gHg?^L5uWSV6!q-;_-tQMPJwA z-dQQG7Fh01>3d14!&of!+@dID>#p>QLzOOOVGfnQdmfo+SPuzU(`+j_^t+f6SFszfgVHx)i_U)!sQ@aL-bs9Sv?ctSona;;;#>U+q5TG>JatvA6zC3oPD-)wAf#@7?FC zt@jNu=FK`Rxt@GdML`@EiO=jB^P42}vjB|yZ zrYJoB=qXwQy8?Y8oC;=#{7lmymtr^a;q}|3_s(Cl=O(67Xcf7MMtl#eE{%)esvJJT zKVQo9K&xGiEQ%Oxe!P)Fg2elueaL&SdDMS<>5jp?9 zI@lQN8bj1Apv=l~8>v{Qo>OYl*UTmA0!1XjcjWcZiQCSUSaW%LZcfhb`X`NsQ-)Tg zG(z5~u*WlSFXh%zc#C&?rlog>oj)8vb`7GcHaet^N)+c$+tc)m<>ntubL=AK*5QlW zy>|OF`&u|8=j!PcL#$zggL~gJE~OhwSgjktU#xk6HS@u{>S&}ca*xpC9#7Hxutkly zZS3~ycO5>t$WC!Am__9^wz75vo;mID_`EO%JO37(ate|W=>mcf6-1+N3ag_S+* z*6FYmde%?cfIpF*+wLD20IzuJq3B}=^mrV?Twuw?-0M*uN$Cz*M&f4ADzgnRF)#^! zo-jyz@X90!!CsngOqUYhplz*0t)j@DV=Or;>dV8w(t0duz8TaK0}_{yXtn` z!>5?wBDNpg_%y@O0!_IRsG+u%99eM|`(1Z8&aQTf~Hhlr#y z$?TCGpI`raZG(rIJH~g7>Qx=T#wWLj0nwy}8>dLTM4j+0)^SylE_uy4ABDDc9dU{z z`ki~Vs|FAg|ES$4NLye0tKEU2bZbv8^?8#(DCBAoi9`oTHi-0gnv^*sm)ShRTdclo zN1p1S1_s}G+G{t@d5&v({c7n#K(ar{?P?g_ucr)O)8fQ!m4j*8V5jkp%{28j%g&Ya z?mTtFCmWaTB79!RRa_>If zxhi5_d02JdOAi>%Gi;5+M^7$#ndMx>wp-c}2`4Y=1-$U~w|7^l?H4f{VeCNNpRgIz zIGiOhmr3Ff>PJ7E1koH~L=pew^YiSc8>#rL=a3YrGLPc}K4bl$S6Yb!Up)nO{E! zFl|Qc2W1u)OH34wKATnRZM3D22iwZiKCwz-?hxovRAvEw(pw?6JG3_)sVg1ylYg~T z8aMSb%*cGdN#I=5IWG^dPNozRax$&OKj(F}Eo7K+Bjim2A(2I0TG=9d(~rGrVNu{* z3;6t81t9TU22_u`gTm6_ypGp`c!i4yfqg-!6OODukJ$}Z@=UWFwkH=Y>@1ZHJhsh}%Hmbd7$R1+0wqf?u?y`1btWneQZxXSW~s`4zrMu zHd7R4I)E7SmdKKjRoDYWif83btkVmgMqnN2xRtOwc5BS>f#o*a>YMJLC|w-|E2+#$ z_hF-lvC6~CwI+;fd!4?BgG3&E{kXVGf$+;+H2mPuPO3htn(G-pI3TH-T1`5!=e{Ta zp&Da@R9LK-ar5AkSQaaF2(U56;;;5+c)@~|S(z5dDP>aGv*YqhNR!%(5eWvmeaanY z^F+7TUYY{AB89JCBG3_gsPT1VahnJfEFaEnD(x;ISX;K}?#g$vf2I}sqQ=YYPmF=v zB-(XQ&WJ;W(|q{etwDhxxuLZC9P`=DXd&TjUuyizxvp<*RlgR(7jr6LN`!Uy))Q*< z9MSY7?0^!zOpTmMrmx9_gph{i2wzlthW`h0#u{0MxG`$1rPL0eh`O!68*ztmL>DuP z4jU6~KB)LYw#hqFg^pp298#cvM6ab~U5w>8kcT3`O|G#MT(ANn@0A(u*C7K0c{ z#&D;ZeO}7ayGRkhz-WYA#B90?fKSGaQ607*U6P7VgMj2Uevszy&deF5seQK+cdW)O zTHm|(Tp5Z7-rtRf4!w&#ecQG||MP%MGm%p4HYX|7{(YWGzgwmd(Hc&xlvFiG3FO%j zYO0D#UY&CWVK{Xd)tYpPW*5VaE&se9FjN+36RMyZVzn0@$&npjzU(*$?BJKN&qH1f z67aCfW_SX&{>@(Fea3ygvi1N9=JMjr@msXEF3;>pR99LNP2-IF8nDnJ)PQ~O>oVIw zw2jbMA|$z=8D6tkkUk6zW4Y6td^
  • Dnh-)WNme)T!q;YVNQF9wWfKzmyI7qeAd* zY4_pQXlWHePcvS9$b7t1rOzklB1-4UpyLY1eoIfj%h&h#-14nKF4)8bdPHWc8%u0k zxWO)W@)~#Or|8%yd$IGDYwyNh4oOdT-?CS-c z=ksuDs|GbpJba2*|5b%rw7)r0*_PZ03tKhK-G+pp);(ONc1X*~)~0D%%YC{+UwwrO zuoFV~Yg!xh-}r9YvY!vC{R8yBi~UixIrg?Szm@eDOX4^5WkTNPKWfvCh*SN$*n3uf zSY1VBo{%G4`Wm-B8)ES9n*Klke**ub1UC2aN`@HgrwT(WWFT}HpxciU3e||VyDE~&cJHOHvIph1ALLx{=?AbGO)>-G%UTg0!`@?=eJnwqf^ZftMyMAl^ z(<)clZgI5(2NixHY^r^3@X+>AtpT+k`|N1-U*5>0(7;~$&xP2!kwojCJiLPSzF4W3 z%ShrSkl0j&)61)S(T+Ug>P-`O;FUJJ@OcSU^M7AWjF}5x6flXZ3=|}-85Pxw>HSOL_A`@ zRT5Hhuf+hR$W44TKYj82s_s44@k_#*AoB5GBkd~VrG&bpZR!hW{sQ*R>$n0#P^UnR zJ*u#-y5)#Udeija{I_8-zVg`nKE|xKFKnMY`3MWh9wDw&T;My1&SvaibZ=>RNthV; zWkI-7(A!BI(u>-CTg9#jG??$1N+|EOVkrt?W)6uJ$iRqU<5cD62ZGR)mdDZ8 zq5jyP(+pe(j{AJ=VeCZoweX2_m$E_SE|~Oc1Ud=bYYcRhrVIt;dqqdoUac9^M$?B~ zisi}0WuieF{!*5}W+6nC6x36Zq0 z-abZ5uD4$}lUn+aS4{^_ESS5{4G@@D{xYaldc0-N#k_Ow=g-MtvXSw4O<0w@ZoIQ_ z26)Ui5BCRcB#tEQeRoJqY3`U7MwUaYAcfRh!t1 zcKfds_}*K}$3DfJQ(zMw$FcN2n=p5rOug*_Y@|?hJm*ek_>6Ugb`%4kS^XwPMmY}L zThCY?Y5{9@R9_U$64N%2{Vc1GoOYj7)Axh_%pFrqyCn9qmcMMni2nU$1n=9k;u<$( zLHB3Qy>X+VZRqb0;UT!&9}_6{C_F;@in%`C^yMLLlpBR4Zs}`5H6x>~mvTNhi>Sx1 ztED~^D64JJ+n6iWy4OZ)uYoY&|5*VDJUt(_CKO4d3^sEY`XhO-58$;=6?|iMsDPg$ z+PNv1zzq7KTEyQ+hQaEnArAfnJl^#r|5GrOb#+W~qz}PLS^EigStGXc<(>&{o;kQI z4e-bS(nc@ESZ6dsaEsROm#3e zw$Dq`FgV8tWkP0J^zH1R{Mi8T;|?n-4ES5RLjFFNu zRCZEOfk5mdQfCawv#+hg=W-En` zkdUt>eqJrpct>vbq@2;<%%ZLW@qVsg2MWrZ(dtO40-H}nZePneHw^klGAx`-41Ke+ zVXT=v@vU}=9Hi#QndI|6vm1Ji+F*p0EI3i96Mk6UC2l~(=_u&K*vZ_zK{trbhWUmc zzUH*tbAeA=WL@P{;*Q3)owP{ZJvNK{s(%juQ|*}j#H*iH7Q6(RBOJuCTw9<_ict9d z)UX92eiZt%KVJW@&XdZA6+2JqArtBqhB+gt6kwnyD6B#~_zLyck?S67Hkt8dN~wsI z7pF0~5^vC1&3E>d@ma$n{>U~E++Dq!S;+ak8isi?_#FR%T*2g#;~L_0`H^6cv9R6QHgj@8vM*~qlODlh6g6ihHmkt3O zSEY-;BR#YGSqwdc%5IQ1gplPku@Xu%9K>*x|e!617CzIi)IV{;LrdpH;N<$k%8GYgY zw5~VSWdFawr7u_#I6#i!n%gEU_5ohVQmnNIBMpnC&I3FuoqbMIhX5%$~wp zRGj?bhP+Sb-WtOlH?3o+18pHXY;mfYKrqW7J=|KmUmv((1|tyt-BliAmRr91g4~f2 zwySMH>*YE)WgoWdk};+@L)YrHZRBOA=iomr7WIb!`4`RVZw;CjR2r@aN8)y2saso8 z9Kw>Vq8;?tWt;M-(u;Eqt~ZT|R%!P$LmSHdwkuwba}05;ixx9P<;(r9;nHGOd>>AP z-pIH@Ni2ze@4dg+?IZ9he-VAbFc=mYP?8kYf^nupeuO>XAPzhOnFW4lla?7rt6)_+ zDX$`r9a-k9fB&HT-b{WOlm8smC4mwcZ&KPEBWImrRkABHR2BJmA&rZKOP=6BHeh+x z)h}1oW&&hmUu}R$0Ml+xeMW;zMN&Y|TnuD?(c?q?CIh_wT{^UBpOvG%xXG)X866~Y zhL6SWKek($;unR$jS8&jt10EqN?(cRBNLS1)%0#O|gl=xpVX?Yj zai-riX`?PzZHqM8c{jd~{anMwS*eh+4=(}A-A-pp#SuhAa2Xm{0MwB-mw8l1FBzOB zdXM6XCrP7+P~tKflDoOl7(VXEB1TKJw8p&^6cQ*=DC`f~i!AoZ-y)LGLj>rp8`#24 zkNlaWooKt={PTOBiL)ag9&$qd&$q}v^Fx`pQspy8FWpKvoJ;AMx@|2P;A*smGdj4Ds7F*PNvG#6uUM(JZO zLnBm(pFsM`%zkNj&&S~sTrL$wgSUDXE?dX$JkzmcS`}OJpBi=3usV$fU)$)*JIsMS zEJkRi*FHlvF@}tv>$wc&pci#ap`U^x{`B(GmyOBwXI^*IMv)$*uivLUS zAKmjNtv5%8_5QB&keM@85pc=xk~A$2^UxpS7fu|d!@ogZt;$UAfo`o3zWhg~|LOmS zz`rSh9ko1cZ}5sqRuEJ+XiK3T_?P~9Pr%pymkfT3mF{2__RpmK(^j+xhP$f30G!v~ E0KFTw-v9sr diff --git a/setup.py b/setup.py index 4d70b380..a3421101 100644 --- a/setup.py +++ b/setup.py @@ -4,14 +4,8 @@ # Python 3 standard library, let's make sure we're not on Python 2 and provide # a helpful message if we are. -PY2_MESSAGE = """ -Sorry, this version of ftfy is no longer written for Python 2. +PY2_MESSAGE = "Python 2 is no longer supported. Please upgrade." -The older version of ftfy, version 4.4, is still available and can run on -Python 2. Try this: - - pip install ftfy==4.4.3 -""" if sys.version_info[0] < 3: print(PY2_MESSAGE) @@ -27,7 +21,7 @@ setup( name="ftfy", - version='5.6', + version='5.7', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From 2e155a10afbb56456348ef4e66e2adc4d98860b2 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 3 Feb 2020 12:46:49 -0500 Subject: [PATCH 420/502] properly update the micro version --- docs/conf.py | 4 ++-- ftfy/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 76e305ef..11954581 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ # built documents. # # The short X.Y version. -version = '5.6' +version = '5.6.1' # The full version, including alpha/beta/rc tags. -release = '5.6.0' +release = '5.6.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 97c796de..33e74bec 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -11,7 +11,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.6' +__version__ = '5.6.1' # See the docstring for ftfy.bad_codecs to see what we're doing here. diff --git a/setup.py b/setup.py index a3421101..387bd299 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name="ftfy", - version='5.7', + version='5.6.1', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From 6d5b6666d93d815057fbefd18015008ffe9569d2 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 3 Feb 2020 12:48:45 -0500 Subject: [PATCH 421/502] update changelog for 5.6.1 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f14f27ee..a9c13f3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## Version 5.6.1 (February 3, 2020) + +- Updated the data file of Unicode character categories to Unicode 12.1, as + used in Python 3.8. (No matter what version of Python you're on, ftfy uses + the same data.) + + ## Version 5.6 (August 7, 2019) - The `unescape_html` function now supports all the HTML5 entities that appear From 00f6f86da1c7825336bf919c6c1e2038f48205db Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 7 Feb 2020 11:59:40 -0500 Subject: [PATCH 422/502] Improve heuristic that was missing the acute accent --- ftfy/badness.py | 7 ++++++- tests/test_cases.json | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 7f75ee17..f5c56eb9 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -128,9 +128,11 @@ def _make_weirdness_regex(): # ISO-8859-2. This also covers some cases from related encodings such as # Windows-1252 and Windows-1250. '[ÂÃĂ][\x80-\x9f€ƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¢£¤¥¦§¨ª«¬¯°±²³µ¶·¸¹º¼½¾¿ˇ˘˝]|' + # Characters we have to be a little more cautious about if they're at # the end of a word, but totally okay to fix in the middle - r'[ÂÃĂ][›»‘”©™]\w|' + r'[ÂÃĂ][›»‘”´©™]\w|' + # Similar mojibake of low-numbered characters in MacRoman. Leaving out # most mathy characters because of false positives, but cautiously catching # "√±" (mojibake for "ñ") and "√∂" (mojibake for "ö") in the middle of a @@ -144,12 +146,15 @@ def _make_weirdness_regex(): # the nose of a kaomoji. '[¬√][ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñúùûü†¢£§¶ß®©™≠ÆØ¥ªæø≤≥]|' r'\w√[±∂]\w|' + # ISO-8859-1, ISO-8859-2, or Windows-1252 mojibake of characters U+10000 # to U+1FFFF. (The Windows-1250 and Windows-1251 versions might be too # plausible.) '[ðđ][Ÿ\x9f]|' + # Windows-1252 or Windows-1250 mojibake of Windows punctuation characters 'â€|' + # Windows-1251 mojibake of some Windows punctuation characters 'вЂ[љћ¦°№™ќ“”]' ) diff --git a/tests/test_cases.json b/tests/test_cases.json index 992b5543..2f4be581 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -93,6 +93,12 @@ "fixed": "#правильноепитание", "expect": "pass" }, + { + "label": "UTF-8 / Windows-1252 mixup in French", + "original": "Hôtel de Police", + "fixed": "Hôtel de Police", + "expect": "pass" + }, { "label": "UTF-8 / Windows-1250 mixup in French", "original": "Liège Avenue de l'HĂ´pital", From 39747dba7676a25a52d4478980c488ab64c442a4 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 12 Feb 2020 11:43:29 -0500 Subject: [PATCH 423/502] Travis is over --- .travis.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 30310d5d..00000000 --- a/.travis.yml +++ /dev/null @@ -1,28 +0,0 @@ -language: python -sudo: false - -cache: - apt: true - directories: - - $HOME/.cache/pip - - $HOME/download -python: - - "3.4" - - "3.5" - - "3.6" - - "3.7-dev" -before_install: - - pip install pip -U - - pip install pytest -U -install: - - travis_retry pip install -e . -script: - - travis_wait pytest - -after_success: - - codecov -cache: - apt: true - directories: - - $HOME/.pip-cache - From 0b4baad53589b3aaf1af9c821111010b74268678 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 13 Feb 2020 12:23:40 -0500 Subject: [PATCH 424/502] update changelog for acute accent in 5.6.1 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9c13f3b..eb30a1b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ used in Python 3.8. (No matter what version of Python you're on, ftfy uses the same data.) +- Corrected an omission where short sequences involving the ACUTE ACCENT + character were not being fixed. ## Version 5.6 (August 7, 2019) From ceb2c3e91632af14d5cbfce81529911b373af06f Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 18 Feb 2020 11:45:37 -0500 Subject: [PATCH 425/502] version 5.7: this should really be a minor bump, not micro --- CHANGELOG.md | 2 +- docs/conf.py | 4 ++-- ftfy/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb30a1b2..271f6116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Version 5.6.1 (February 3, 2020) +## Version 5.7 (February 18, 2020) - Updated the data file of Unicode character categories to Unicode 12.1, as used in Python 3.8. (No matter what version of Python you're on, ftfy uses diff --git a/docs/conf.py b/docs/conf.py index 11954581..5a48fde4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ # built documents. # # The short X.Y version. -version = '5.6.1' +version = '5.7' # The full version, including alpha/beta/rc tags. -release = '5.6.1' +release = '5.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 33e74bec..8656aded 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -11,7 +11,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.6.1' +__version__ = '5.7' # See the docstring for ftfy.bad_codecs to see what we're doing here. From c2186c76bec6b615fecb18d66315d908a1a379c1 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 18 Feb 2020 11:46:41 -0500 Subject: [PATCH 426/502] update version number in setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 387bd299..a3421101 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name="ftfy", - version='5.6.1', + version='5.7', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", From b71cbcf24a3747929bb798a10fd9b01087ab869f Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Wed, 26 Feb 2020 22:49:04 +1100 Subject: [PATCH 427/502] Fix simple typo: obselete -> obsolete Closes #136 --- ftfy/bad_codecs/sloppy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index bd6f78c0..2257a5c1 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -16,7 +16,7 @@ to, and the characters they pick are the Unicode characters with the same numbers: U+0081 and U+008D. This is the same as what Latin-1 does, and the resulting characters tend to fall into a range of Unicode that's set aside for -obselete Latin-1 control characters anyway. +obsolete Latin-1 control characters anyway. These sloppy codecs let Python do the same thing, thus interoperating with other software that works this way. It defines a sloppy version of many From 34f47dd3a1c2c0d29faf45336df0909bf4ff94d7 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 7 Apr 2020 16:46:37 -0400 Subject: [PATCH 428/502] remove obsolete Twitter stream code --- ftfy/streamtester/__init__.py | 45 ---------------- ftfy/streamtester/oauth.py | 71 ------------------------ ftfy/streamtester/twitter_tester.py | 84 ----------------------------- 3 files changed, 200 deletions(-) delete mode 100644 ftfy/streamtester/__init__.py delete mode 100644 ftfy/streamtester/oauth.py delete mode 100644 ftfy/streamtester/twitter_tester.py diff --git a/ftfy/streamtester/__init__.py b/ftfy/streamtester/__init__.py deleted file mode 100644 index 9e0a06b2..00000000 --- a/ftfy/streamtester/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -This file defines a general method for evaluating ftfy using data that arrives -in a stream. A concrete implementation of it is found in `twitter_tester.py`. -""" -from ftfy import fix_text -from ftfy.fixes import fix_encoding, unescape_html -from ftfy.chardata import possible_encoding - - -class StreamTester: - """ - Take in a sequence of texts, and show the ones that will be changed by - ftfy. This will also periodically show updates, such as the proportion of - texts that changed. - """ - def __init__(self): - self.num_fixed = 0 - self.count = 0 - - def check_ftfy(self, text, encoding_only=True): - """ - Given a single text input, check whether `ftfy.fix_text_encoding` - would change it. If so, display the change. - """ - self.count += 1 - text = unescape_html(text) - if not possible_encoding(text, 'ascii'): - if encoding_only: - fixed = fix_encoding(text) - else: - fixed = fix_text(text, uncurl_quotes=False, fix_character_width=False) - if text != fixed: - # possibly filter common bots before printing - print('\nText:\t{text!r}\nFixed:\t{fixed!r}\n'.format( - text=text, fixed=fixed - )) - self.num_fixed += 1 - elif 'â€' in text or '\x80' in text: - print('\nNot fixed:\t{text!r}'.format(text=text)) - - # Print status updates once in a while - if self.count % 100 == 0: - print('.', end='', flush=True) - if self.count % 10000 == 0: - print('\n%d/%d fixed' % (self.num_fixed, self.count)) diff --git a/ftfy/streamtester/oauth.py b/ftfy/streamtester/oauth.py deleted file mode 100644 index 2efbd152..00000000 --- a/ftfy/streamtester/oauth.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Do what is necessary to authenticate this tester as a Twitter "app", using -somebody's Twitter account. -""" -import os - - -AUTH_TOKEN_PATH = os.path.expanduser('~/.cache/oauth/twitter_ftfy.auth') - - -def get_auth(): - """ - Twitter has some bizarre requirements about how to authorize an "app" to - use its API. - - The user of the app has to log in to get a secret token. That's fine. But - the app itself has its own "consumer secret" token. The app has to know it, - and the user of the app has to not know it. - - This is, of course, impossible. It's equivalent to DRM. Your computer can't - *really* make use of secret information while hiding the same information - from you. - - The threat appears to be that, if you have this super-sekrit token, you can - impersonate the app while doing something different. Well, of course you - can do that, because you *have the source code* and you can change it to do - what you want. You still have to log in as a particular user who has a - token that's actually secret, you know. - - Even developers of closed-source applications that use the Twitter API are - unsure what to do, for good reason. These "secrets" are not secret in any - cryptographic sense. A bit of Googling shows that the secret tokens for - every popular Twitter app are already posted on the Web. - - Twitter wants us to pretend this string can be kept secret, and hide this - secret behind a fig leaf like everybody else does. So that's what we've - done. - """ - - from twitter.oauth import OAuth - from twitter import oauth_dance, read_token_file - - def unhide(secret): - """ - Do something mysterious and exactly as secure as every other Twitter - app. - """ - return ''.join([chr(ord(c) - 0x2800) for c in secret]) - - fig_leaf = '⠴⡹⠹⡩⠶⠴⡶⡅⡂⡩⡅⠳⡏⡉⡈⠰⠰⡹⡥⡶⡈⡐⡍⡂⡫⡍⡗⡬⡒⡧⡶⡣⡰⡄⡧⡸⡑⡣⠵⡓⠶⠴⡁' - consumer_key = 'OFhyNd2Zt4Ba6gJGJXfbsw' - - if os.path.exists(AUTH_TOKEN_PATH): - token, token_secret = read_token_file(AUTH_TOKEN_PATH) - else: - authdir = os.path.dirname(AUTH_TOKEN_PATH) - if not os.path.exists(authdir): - os.makedirs(authdir) - token, token_secret = oauth_dance( - app_name='ftfy-tester', - consumer_key=consumer_key, - consumer_secret=unhide(fig_leaf), - token_filename=AUTH_TOKEN_PATH - ) - - return OAuth( - token=token, - token_secret=token_secret, - consumer_key=consumer_key, - consumer_secret=unhide(fig_leaf) - ) diff --git a/ftfy/streamtester/twitter_tester.py b/ftfy/streamtester/twitter_tester.py deleted file mode 100644 index dab62e73..00000000 --- a/ftfy/streamtester/twitter_tester.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Implements a StreamTester that runs over Twitter data. See the class -docstring. -""" -import os -from collections import defaultdict -from ftfy.streamtester import StreamTester - - -class TwitterTester(StreamTester): - """ - This class uses the StreamTester code (defined in `__init__.py`) to - evaluate ftfy's real-world performance, by feeding it live data from - Twitter. - - This is a semi-manual evaluation. It requires a human to look at the - results and determine if they are good. The three possible cases we - can see here are: - - - Success: the process takes in mojibake and outputs correct text. - - False positive: the process takes in correct text, and outputs - mojibake. Every false positive should be considered a bug, and - reported on GitHub if it isn't already. - - Confusion: the process takes in mojibake and outputs different - mojibake. Not a great outcome, but not as dire as a false - positive. - - This tester cannot reveal false negatives. So far, that can only be - done by the unit tests. - """ - OUTPUT_DIR = './twitterlogs' - - def __init__(self): - self.lines_by_lang = defaultdict(list) - super().__init__() - - def save_files(self): - """ - When processing data from live Twitter, save it to log files so that - it can be replayed later. - """ - if not os.path.exists(self.OUTPUT_DIR): - os.makedirs(self.OUTPUT_DIR) - for lang, lines in self.lines_by_lang.items(): - filename = 'tweets.{}.txt'.format(lang) - fullname = os.path.join(self.OUTPUT_DIR, filename) - langfile = open(fullname, 'a', encoding='utf-8') - for line in lines: - print(line.replace('\n', ' '), file=langfile) - langfile.close() - self.lines_by_lang = defaultdict(list) - - def run_sample(self): - """ - Listen to live data from Twitter, and pass on the fully-formed tweets - to `check_ftfy`. This requires the `twitter` Python package as a - dependency. - """ - from twitter import TwitterStream - from ftfy.streamtester.oauth import get_auth - twitter_stream = TwitterStream(auth=get_auth()) - iterator = twitter_stream.statuses.sample() - for tweet in iterator: - if 'text' in tweet: - self.check_ftfy(tweet['text']) - if 'user' in tweet: - lang = tweet['user'].get('lang', 'NONE') - self.lines_by_lang[lang].append(tweet['text']) - if self.count % 10000 == 100: - self.save_files() - - -def main(): - """ - When run from the command line, this script connects to the Twitter stream - and runs the TwitterTester on it forever. Or at least until the stream - drops. - """ - tester = TwitterTester() - tester.run_sample() - - -if __name__ == '__main__': - main() From 5ed0dcac9f49bea2377a4100d8111df78c51c0d9 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 7 Apr 2020 16:47:43 -0400 Subject: [PATCH 429/502] get CHAR_CLASS_STRING from inline Python, not a .dat file --- ftfy/build_data.py | 15 ++++++++++++--- ftfy/char_classes.dat | Bin 4187 -> 0 bytes ftfy/char_classes.py | 6 ++++++ ftfy/chardata.py | 7 +------ setup.py | 1 - 5 files changed, 19 insertions(+), 10 deletions(-) delete mode 100644 ftfy/char_classes.dat create mode 100644 ftfy/char_classes.py diff --git a/ftfy/build_data.py b/ftfy/build_data.py index b3e3f257..73517b59 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -11,6 +11,7 @@ The file will be written to the current directory. """ +import base64 import sys import unicodedata import zlib @@ -119,9 +120,17 @@ def make_char_data_file(do_it_anyway=False): for char in "^~`´˝＾｀": cclasses[ord(char)] = 'o' - out = open('char_classes.dat', 'wb') - out.write(zlib.compress(''.join(cclasses).encode('ascii'))) - out.close() + char_classes_bytes = ''.join(cclasses).encode('ascii') + encoded = base64.b64encode(zlib.compress(char_classes_bytes)) + + script = f""" +import base64 +import zlib +_compressed = {encoded!r} +CHAR_CLASS_STRING = zlib.decompress(base64.b64decode(_compressed)) +""" + with open('char_classes.py', 'w', encoding='utf-8') as out: + print(script, file=out) if __name__ == '__main__': diff --git a/ftfy/char_classes.dat b/ftfy/char_classes.dat deleted file mode 100644 index 869f346155069e782b0e83814744836e60c39766..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4187 zcmeH|Sx^&L7KX8VlE0uAqm;1$LXo5d77%3mwA}`a8KPj=RfDvSM}G; ztYXi(K%R_#q4KtwcJiBmZ?d$j4dkzp-vj#8Un*kmpsN$tXFNB~jC<@+pPW#iK}bZ5 zVGMAI{1RhFu6aHoTXFO`wHD>0yBzPMwO}#eB5^eIsp=a-9D>UBJ7X~7(`1H&i8zLv z{yn=v+;czuJyC2e{(x@f#!bc1?*l`SN zP~x{13bm+&gewyT9JB#L?VjC?R3LxY+Z#c#=n2P(bAmH-a>xt@gRQ*m+T`Z%>>H z@eO~zis?uWnld)_UHyYIZ_(iP;46OQiYv6d31*8Kyng6yz%Lnh2|=%si!r9iUbkb%B91!Euhu zNm<#$3AB~C`(?S8$D&CmHIky^pSdsUu)mzOc=)Tepbu|WkEjbM%I~~c4r&>l^FM!K zIxP5zDb2OITGhX(#w{FIE-dddkPGY2*Shq&^?7uWB9RyeIX3zp^lk9U!P2l5z*q#R z))V&J<<=O+nVtVLDM|qJ02;wQ1dKrCBZzKc!5^P(2^e*hrm9ZP>XE^wtH^BMn=Tg> zhjo`JCf^bF4avvbY9#f1sf~O77Io1|tLi-r`G_3-gHg?^L5uWSV6!q-;_-tQMPJwA z-dQQG7Fh01>3d14!&of!+@dID>#p>QLzOOOVGfnQdmfo+SPuzU(`+j_^t+f6SFszfgVHx)i_U)!sQ@aL-bs9Sv?ctSona;;;#>U+q5TG>JatvA6zC3oPD-)wAf#@7?FC zt@jNu=FK`Rxt@GdML`@EiO=jB^P42}vjB|yZ zrYJoB=qXwQy8?Y8oC;=#{7lmymtr^a;q}|3_s(Cl=O(67Xcf7MMtl#eE{%)esvJJT zKVQo9K&xGiEQ%Oxe!P)Fg2elueaL&SdDMS<>5jp?9 zI@lQN8bj1Apv=l~8>v{Qo>OYl*UTmA0!1XjcjWcZiQCSUSaW%LZcfhb`X`NsQ-)Tg zG(z5~u*WlSFXh%zc#C&?rlog>oj)8vb`7GcHaet^N)+c$+tc)m<>ntubL=AK*5QlW zy>|OF`&u|8=j!PcL#$zggL~gJE~OhwSgjktU#xk6HS@u{>S&}ca*xpC9#7Hxutkly zZS3~ycO5>t$WC!Am__9^wz75vo;mID_`EO%JO37(ate|W=>mcf6-1+N3ag_S+* z*6FYmde%?cfIpF*+wLD20IzuJq3B}=^mrV?Twuw?-0M*uN$Cz*M&f4ADzgnRF)#^! zo-jyz@X90!!CsngOqUYhplz*0t)j@DV=Or;>dV8w(t0duz8TaK0}_{yXtn` z!>5?wBDNpg_%y@O0!_IRsG+u%99eM|`(1Z8&aQTf~Hhlr#y z$?TCGpI`raZG(rIJH~g7>Qx=T#wWLj0nwy}8>dLTM4j+0)^SylE_uy4ABDDc9dU{z z`ki~Vs|FAg|ES$4NLye0tKEU2bZbv8^?8#(DCBAoi9`oTHi-0gnv^*sm)ShRTdclo zN1p1S1_s}G+G{t@d5&v({c7n#K(ar{?P?g_ucr)O)8fQ!m4j*8V5jkp%{28j%g&Ya z?mTtFCmWaTB79!RRa_>If zxhi5_d02JdOAi>%Gi;5+M^7$#ndMx>wp-c}2`4Y=1-$U~w|7^l?H4f{VeCNNpRgIz zIGiOhmr3Ff>PJ7E1koH~L=pew^YiSc8>#rL=a3YrGLPc}K4bl$S6Yb!Up)nO{E! zFl|Qc2W1u)OH34wKATnRZM3D22iwZiKCwz-?hxovRAvEw(pw?6JG3_)sVg1ylYg~T z8aMSb%*cGdN#I=5IWG^dPNozRax$&OKj(F}Eo7K+Bjim2A(2I0TG=9d(~rGrVNu{* z3;6t81t9TU22_u`gTm6_ypGp`c!i4yfqg-!6OODukJ$}Z@=UWFwkH=Y>@1ZHJhsh}%Hmbd7$R1+0wqf?u?y`1btWneQZxXSW~s`4zrMu zHd7R4I)E7SmdKKjRoDYWif83btkVmgMqnN2xRtOwc5BS>f#o*a>YMJLC|w-|E2+#$ z_hF-lvC6~CwI+;fd!4?BgG3&E{kXVGf$+;+H2mPuPO3htn(G-pI3TH-T1`5!=e{Ta zp&Da@R9LK-ar5AkSQaaF2(U56;;;5+c)@~|S(z5dDP>aGv*YqhNR!%(5eWvmeaanY z^F+7TUYY{AB89JCBG3_gsPT1VahnJfEFaEnD(x;ISX;K}?#g$vf2I}sqQ=YYPmF=v zB-(XQ&WJ;W(|q{etwDhxxuLZC9P`=DXd&TjUuyizxvp<*RlgR(7jr6LN`!Uy))Q*< z9MSY7?0^!zOpTmMrmx9_gph{i2wzlthW`h0#u{0MxG`$1rPL0eh`O!68*ztmL>DuP z4jU6~KB)LYw#hqFg^pp298#cvM6ab~U5w>8kcT3`O|G#MT(ANn@0A(u*C7K0c{ z#&D;ZeO}7ayGRkhz-WYA#B90?fKSGaQ607*U6P7VgMj2Uevszy&deF5seQK+cdW)O zTHm|(Tp5Z7-rtRf4!w&#ecQG||MP%MGm%p4HYX|7{(YWGzgwmd(Hc&xlvFiG3FO%j zYO0D#UY&CWVK{Xd)tYpPW*5VaE&se9FjN+36RMyZVzn0@$&npjzU(*$?BJKN&qH1f z67aCfW_SX&{>@(Fea3ygvi1N9=JMjr@msXEF3;>pR99LNP2-IF8nDnJ)PQ~O>oVIw zw2jbMA|$z=8D6tkkUk6zW4Y6td^
  • Dnh-)WNme)T!q;YVNQF9wWfKzmyI7qeAd* zY4_pQXlWHePcvS9$b7t1rOzklB1-4UpyLY1eoIfj%h&h#-14nKF4)8bdPHWc8%u0k zxWO)W@)~#Or|8%yd$IGDYwyNh4oOdT-?CS-c z=ksuDs|GbpJba2*|5b%rw7)r0*_PZ03tKhK-G+pp);(ONc1X*~)~0D%%YC{+UwwrO zuoFV~Yg!xh-}r9YvY!vC{R8yBi~UixIrg?Szm@eDOX4^5WkTNPKWfvCh*SN$*n3uf zSY1VBo{%G4`Wm-B8)ES9n*Klke**ub1UC2aN`@HgrwT(WWFT}HpxciU3e| Date: Tue, 7 Apr 2020 16:56:49 -0400 Subject: [PATCH 430/502] add comments --- ftfy/build_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 73517b59..92ad5721 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -1,5 +1,5 @@ """ -A script to make the char_classes.dat file. +A script to make the char_classes.py file. This never needs to run in normal usage. It needs to be run if the character classes we care about change, or if a new version of Python supports a new @@ -120,10 +120,12 @@ def make_char_data_file(do_it_anyway=False): for char in "^~`´˝＾｀": cclasses[ord(char)] = 'o' + # Compress and encode this string within a Python script. char_classes_bytes = ''.join(cclasses).encode('ascii') encoded = base64.b64encode(zlib.compress(char_classes_bytes)) script = f""" +# This script is automatically generated by build_data.py. import base64 import zlib _compressed = {encoded!r} From 21905e087477fa9c5de9eaed9b262459ca57fb83 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 16 Jul 2020 17:24:39 -0400 Subject: [PATCH 431/502] Better detection of mojibake of low codepoints We have a regex that's designed to say "this looks specifically like mojibake" that's part of the "weirdness" heuristic. It was detecting mojibake of the Latin-1 characters. We can make this work for some more scripts that also have low-numbered codepoints: Greek, Cyrillic, Hebrew, and Arabic. We had a rule that was supposed to catch when Hebrew turns into multiplication signs -- but it wasn't working, because it wasn't separated from the previous rule by `|`. Simply fixing that rule caused a test case to fail, so we need a larger heuristic. --- ftfy/badness.py | 50 ++++++++++++++++++++++++++++++++++++++----- tests/test_cases.json | 37 ++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index f5c56eb9..154a4961 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -118,20 +118,56 @@ def _make_weirdness_regex(): # These are sequences that are common mojibake, resulting from common encodings # that are mixed up with UTF-8 on characters from their own character map. # +# Sequences that match this regex will increase the `sequence_weirdness` of a +# string. +# # This helps to strengthen evidence that text should be fixed in a way that's # separate from the character classes above, or to counter COMMON_SYMBOL_RE's # fondness for characters such as inverted exclamation marks and multiplication # signs in contexts where they really do look like mojibake. MOJIBAKE_SYMBOL_RE = re.compile( - # Mojibake of low-numbered characters from ISO-8859-1 and, in some cases, - # ISO-8859-2. This also covers some cases from related encodings such as - # Windows-1252 and Windows-1250. - '[ÂÃĂ][\x80-\x9f€ƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¢£¤¥¦§¨ª«¬¯°±²³µ¶·¸¹º¼½¾¿ˇ˘˝]|' + # If the first character is in [ÂÃĂ], this covers mojibake of low-numbered + # characters from ISO-8859-1 and, in some cases, ISO-8859-2. This also + # covers some cases from related encodings such as Windows-1252 and + # Windows-1250. + # + # The codepoints of decoded characters these correspond to are as follows: + # + # Initial char. Codepoints What's in this range + # ------------- ---------- -------------------- + #  80..BF Latin-1 control characters and symbols + # Ã, Ă C0..FF Latin-1 accented letters + # Î 380..3BF Greek letters + # Ï, Ď 3C0..3FF Greek letters + # Ð, Đ 400..43F Cyrillic letters + # Ñ, Ń 440..47F Cyrillic letters + # Ø, Ř 600..63F Arabic letters + # Ù, Ů 640..67F Arabic letters + # + # Here, we leave out Hebrew letters (which have a separate heuristic), and + # the rarer letters from each alphabet. This regex doesn't have to contain + # every character we want to decode -- just the sequences that we want to + # specifically detect as 'this looks like mojibake'. + # + # We avoid ranges whose mojibake starts with characters like Ó -- which + # maps to Armenian and some more Cyrillic letters -- because those could be + # the 'eyes' of kaomoji faces. + # + # The set of possible second characters covers symbols that are unlikely to + # have a real meaning when following one of these capital letters, and appear + # as the second character in Latin-1, Windows-1250, or Windows-1252 mojibake. + '[ÂÃÎÏÐÑØÙĂĎĐŃŘŮ][\x80-\x9f€ƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¢£¤¥¦§¨ª«¬¯°±²³µ¶·¸¹º¼½¾¿ˇ˘˝]|' # Characters we have to be a little more cautious about if they're at # the end of a word, but totally okay to fix in the middle - r'[ÂÃĂ][›»‘”´©™]\w|' + r'[ÂÃÎÏÐÑØÙĂĎĐŃŘŮ][›»‘”´©™]\w|' + + # Most Hebrew letters get mojibaked to two-character sequences that start with + # the multiplication sign. In the list of following characters, we exclude + # currency symbols and numbers, which might actually be intended to be + # multiplied. + r'×[\x80-\x9fƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¦§¨ª«¬¯°²³µ¶·¸¿ˇ˘˝›»‘”´©™]|' # Similar mojibake of low-numbered characters in MacRoman. Leaving out # most mathy characters because of false positives, but cautiously catching @@ -146,6 +182,10 @@ def _make_weirdness_regex(): # the nose of a kaomoji. '[¬√][ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñúùûü†¢£§¶ß®©™≠ÆØ¥ªæø≤≥]|' r'\w√[±∂]\w|' + + # MacRoman mojibake of Hebrew involves a diamond character that is + # uncommon in intended text. + '◊|' # ISO-8859-1, ISO-8859-2, or Windows-1252 mojibake of characters U+10000 # to U+1FFFF. (The Windows-1250 and Windows-1251 versions might be too diff --git a/tests/test_cases.json b/tests/test_cases.json index 2f4be581..1fee88d3 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -419,6 +419,43 @@ "fixed": "Sarai ricontattato dal nostro Esperto al più presto.", "expect": "fail" }, + { + "label": "Hebrew UTF-8 / Windows-1252 mojibake", + "comment": "reported by SuperIRabbit as issue #158", + "original": "בהודעה", + "fixed": "בהודעה", + "expect": "pass" + }, + { + "label": "Synthetic: Hebrew UTF-8 / Windows-1250 mojibake", + "original": "בהודעה", + "fixed": "בהודעה", + "expect": "pass" + }, + { + "label": "Synthetic: Hebrew UTF-8 / MacRoman mojibake", + "original": "◊ë◊î◊ï◊ì◊¢◊î", + "fixed": "בהודעה", + "expect": "pass" + }, + { + "label": "Synthetic: Arabic UTF-8 / Windows-1252 mojibake", + "original": "رسالة", + "fixed": "رسالة", + "expect": "pass" + }, + { + "label": "Synthetic: Arabic UTF-8 / Windows-1250 mojibake", + "original": "رسالة", + "fixed": "رسالة", + "expect": "pass" + }, + { + "label": "Synthetic: Arabic UTF-8 / MacRoman mojibake", + "original": "ÿ±ÿ≥ÿߟÑÿ©", + "fixed": "رسالة", + "expect": "pass" + }, { "label": "Negative: math in Unicode", "comment": "This isn't mojibake, it's an actual equation", From 0115c803e587811fe196bf522c60bb040bd17648 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 17 Jul 2020 16:33:38 -0400 Subject: [PATCH 432/502] fix an unintended raw string, add a test case --- CHANGELOG.md | 8 ++++++++ ftfy/badness.py | 2 +- setup.py | 5 ++--- tests/test_cases.json | 7 +++++++ 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 271f6116..49cbcc75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Version 5.8 (July 17, 2020) + +- Improved detection of UTF-8 mojibake of Greek, Cyrillic, Hebrew, and Arabic + scripts. + +- Fixed the undeclared dependency on setuptools by removing the use of + `pkg_resources`. + ## Version 5.7 (February 18, 2020) - Updated the data file of Unicode character categories to Unicode 12.1, as diff --git a/ftfy/badness.py b/ftfy/badness.py index 154a4961..00bf20a4 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -167,7 +167,7 @@ def _make_weirdness_regex(): # the multiplication sign. In the list of following characters, we exclude # currency symbols and numbers, which might actually be intended to be # multiplied. - r'×[\x80-\x9fƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¦§¨ª«¬¯°²³µ¶·¸¿ˇ˘˝›»‘”´©™]|' + '×[\x80-\x9fƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¦§¨ª«¬¯°²³µ¶·¸¿ˇ˘˝›»‘”´©™]|' # Similar mojibake of low-numbered characters in MacRoman. Leaving out # most mathy characters because of false positives, but cautiously catching diff --git a/setup.py b/setup.py index a3421101..4a16e1a3 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name="ftfy", - version='5.7', + version='5.8', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", @@ -34,10 +34,9 @@ package_data={'ftfy': ['char_classes.dat']}, install_requires=['wcwidth'], tests_require=['pytest'], - python_requires='>=3.4', + python_requires='>=3.5', classifiers=[ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", diff --git a/tests/test_cases.json b/tests/test_cases.json index 1fee88d3..fc420889 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -438,6 +438,13 @@ "fixed": "בהודעה", "expect": "pass" }, + { + "label": "Synthetic: Hebrew UTF-8 / Latin-1 mojibake", + "comment": "This example uses low-numbered codepoints to spell 'ABBA' in Hebrew, so that it falls into the range where Latin-1 is different from Windows-1252. As a bonus, this example looks right even if your RTL text rendering isn't working.", + "original": "×\u0090×\u0091×\u0091×\u0090", + "fixed": "אבבא", + "expect": "pass" + }, { "label": "Synthetic: Arabic UTF-8 / Windows-1252 mojibake", "original": "رسالة", From c3edb62a163a52651f9309ad4e712885ed3e6ee8 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 17 Jul 2020 16:47:08 -0400 Subject: [PATCH 433/502] Update comments, remove a few unnecessary cases from the Hebrew regex --- ftfy/badness.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 00bf20a4..5c4209ad 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -127,11 +127,6 @@ def _make_weirdness_regex(): # signs in contexts where they really do look like mojibake. MOJIBAKE_SYMBOL_RE = re.compile( - # If the first character is in [ÂÃĂ], this covers mojibake of low-numbered - # characters from ISO-8859-1 and, in some cases, ISO-8859-2. This also - # covers some cases from related encodings such as Windows-1252 and - # Windows-1250. - # # The codepoints of decoded characters these correspond to are as follows: # # Initial char. Codepoints What's in this range @@ -154,20 +149,24 @@ def _make_weirdness_regex(): # maps to Armenian and some more Cyrillic letters -- because those could be # the 'eyes' of kaomoji faces. # - # The set of possible second characters covers symbols that are unlikely to - # have a real meaning when following one of these capital letters, and appear - # as the second character in Latin-1, Windows-1250, or Windows-1252 mojibake. + # The set of possible second characters covers most of the possible symbols + # that would be the second byte of UTF-8 mojibake. It is limited to the + # ones that are unlikely to have a real meaning when following one of these + # capital letters, and appear as the second character in Latin-1, + # Windows-1250, or Windows-1252 mojibake. '[ÂÃÎÏÐÑØÙĂĎĐŃŘŮ][\x80-\x9f€ƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¢£¤¥¦§¨ª«¬¯°±²³µ¶·¸¹º¼½¾¿ˇ˘˝]|' - # Characters we have to be a little more cautious about if they're at - # the end of a word, but totally okay to fix in the middle + # Character sequences we have to be a little more cautious about if they're + # at the end of a word, but are totally okay to fix in the middle r'[ÂÃÎÏÐÑØÙĂĎĐŃŘŮ][›»‘”´©™]\w|' # Most Hebrew letters get mojibaked to two-character sequences that start with # the multiplication sign. In the list of following characters, we exclude # currency symbols and numbers, which might actually be intended to be - # multiplied. - '×[\x80-\x9fƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¦§¨ª«¬¯°²³µ¶·¸¿ˇ˘˝›»‘”´©™]|' + # multiplied. We also exclude characters like ¶ which, although they don't + # make sense after a multiplication sign, wouldn't decode to an existing + # Hebrew letter if they were mojibake. + '×[\x80-\x9fƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¦§¨ª«¬¯°²³ˇ˘›‘”´©™]|' # Similar mojibake of low-numbered characters in MacRoman. Leaving out # most mathy characters because of false positives, but cautiously catching From 3cc7a5aa06a60e1a9d559a9274019cf24891cc49 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 17 Jul 2020 16:49:25 -0400 Subject: [PATCH 434/502] re-run script to add comment --- ftfy/char_classes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ftfy/char_classes.py b/ftfy/char_classes.py index a108b2d9..baf1eaa5 100644 --- a/ftfy/char_classes.py +++ b/ftfy/char_classes.py @@ -1,4 +1,5 @@ +# This script is automatically generated by build_data.py. import base64 import zlib _compressed = b'eJzt3Uluq8oCANCt3C0QVhB5GqIn/cmbIQ+/FMT+Z4+eom/s2E7uOc5NDBTV0hTYl/r338afP+XPxJ88z6O80vzpRVEx62NRHehrURnhNMFJ+kUqUZy/xZcifPwWR5+f+TXP3z4v+WfxdiHtqPm7lHbUvvk4+dqesfFq1mrelD9lHB8f/cTXRxfso02hXlS9+fr66MrQ/WzmqY3nbLk/+lTrXHw0mW5e/2/1754n3w6SPNj79f2avb1f0zS7Xq/5e1p4e3vP39/f0+JVLB5Lu3fXbUWYOuD7wqsOFJXpXKdpbduRh1GGtl5xUDMnXqPinYxl9yvdrKG60dKsPvztrqg8T9M4jtJDW1NevOqftHFZ1i4uU+oER/LiJ47DyPNi6Up8lax/O5/DgSLQVoR3kLdZyZviZFmSVG/Lwvd5ucTxZZi/NN/K4EpjrFb+bYJULunAoDw7I8myuAifpWkSRctBsz58NxWmV1fYWmpF6HB5esZs3OlqlJN9ZMtmzS0VcGZruEziHO0D2TSaoDj1v7Uyp5emVoqCtomVZa7K3VVIUtddWiUw3EqiogdViqNLXq+V9pW5J/32p0ikSb1Lv025zWKbnz79qmaSfNBa6SCRXekPy19lpE5/vEmMy59H7YJgzWP1P00/LH/aVsd8/ceXuvaDCrj09d+m06yctu/qJcOsdZt6U4lJWP4kHejTLxu+EsX9zrKj+qfz+zpMgvTb4jcZnNR/m6HPZiMso8jDNHakP63/afpNI0/THx00kkH7r6Q/lUzTv8TNpp805buM0//87EreliKZSTJdPpmEe0sXqm3yNj/dQXDc/mWQ8Px/+JyVXC5B1M35I2uTG54c9mi288tKzafhoiQ4kbXlT7O24IOSNhWznnyxK4zOaXHRDSp6C/VuMtNyn3FSvrqj2XrmBwVJJ2enutaaeIJZu/qBadxoMps2+a0zvrP697f8jFH/qe2FVIsu3TnyEpwV20Wz0V6CDaicjuOd3e3qOqr8feDSJC+vwA6UnzXN3tZNDaePrn9rOue74kfzvRJTGh4QP2fM7H9xp94/T1yi77o8rXeB4nh5h4LCLvF0a/uzHLo6g90x9fAE1XUMBiGqqSRJVs+P1U69unzx/Fr1L1eW77TVLShKWd59ii4zPe/BwScdXBmVa07C78jO+Ip2n7D8ey7Bk8vp+lpMfDu+QTds3H0rp+NhHW4cTtv+6rn2Xw8/bs+ygeNNC5E15dtxV2lx7TqK1Q21qM7x9rYwnbXzJjHcqO2Wnite223tW2DcH877qu6Ch6f4zdRHCe3d0tZy3JZ9byzzTTe5FVhtnHvzl1WC++NNng51Yopfw62l7nLNbmtVQZry1BcmVfjljydDdRazU8r1uzedbDadYZA9FvetTfWmf8vngN/3uvXT2f2v7nOqRrsxDvvJ4+Xb4dP3+rW0frvevvhH02kz/XZ9e3sLp6upKtYqN810t34xXaXYh0//9CaHrlCzeDZQtDxVhw/PSlnbB6k+c8r6/kg1Pdwfi/0z2jB7PF/d7Ifi+L08Ehe/ru/NR6LXciqqKqg+TL/Xr3K6qrfqpsG1WKOYqjJRtUocFXPi8SXeaqXW3r+qiqmv+Kr46hu0UX2btvlpbtqui6qw26FotXVSNsFSjTYBNmr1uI122szzZvyHQq9a7D9thZm77bHXXXLeeUQaDxPNvDsWwcFt6U4end7LGB7y77fdV41YfYFjTXB2Pnks+a1OnDLubb5XcIdIt8PcmMgvcJft5eCpdvu+zJ47N/ex92K7ceTztepS5KP5Ymv9qr8p2l3gZR8f3/19vepLgU1Ri7dJ+T3D7ubH597v56X1Otfz9x+b7lA2+lR26eZPF6CdPvt363Jox/VJKNsKsNhR3O4/jnzfLrAv/Z9sX32X0+UFepyFH+MP+yr9RLO9VHedqjDFWkUcZ3eIdeWF/NtbViTxPfGXyqIc/lDh+77jWUdfX6f0bbQWdu/2G7b/OJKtVff2SrtQB/uyh3u9v+P67WV847YMAAAAAADAbvf7/PrZJQEAAAAAAAAAAAAAAAAAAAAAAAAA+G3G4wHOBsoenCmA88+zXT+ezaiGMc3ybxxe9OcpB/cdz+tHgp2eO0rHRmK+VEPIBqMPZ2uhsyxZGh/5oDwYxXY4TvHbvKz19vbxFb6+vj5uemXN0NfNeNj9/DqhtTTq0bObMbTbV1qPsL084O+8r6zJyOVSVnFVzUt1XbVUu5O1o65G8bH9bdwabUbOtPB0FOs2tulI1NOwXR7KsYWD8a/3pLgYajY3ZYZWC9Lk76biD0ffzoblb/Kb9B26UXn3t998hZfrJ32QsAXq9+NompyUTx4/XvIqsWq1ZtXiTzJ7WKqzNjy+ty1YHlbCUFU9TP4OFvfjqX/tVx1EqnfXr682vn3jze+3p8raLWXcPmf3XwAAAAAAAAAAAABg2fBLzYtBTsX7PwDgr/MPAAAAAADAL3PbNzM45pHPm1t7/GWveVZfOKt6rl79ZZvgAYXRMPLmd3ppfh5n/KjSmece8iD9c0zvVflnY9i3tQ9jL5/hOn6Q8tbzU/Nt6XAyyqMoSos/eZf+q2yiaVplNmozOlLkO8/fF9WBlh/NWZZ88mzZY4/Wz7aCp8sPMK2XFmV4i4v6L/5FUfCc7LLc5eQgspV05ueNZ4+33mPb44O11VD++TwiDXabHfvDVP0I5ZFq1mB/LDMWL1fdrENPTU+24/tuWy10rlV7yXo7bsbfhlvMbLdv59vl6fTPUN4ozK7n04fxNIfg3ZYPb0N3fB7xAdPnFe/JaxluT173t1e/WZ0rRedAyzzGmfIweT740uJ0Oue1tLk71idL62Pz8n9HyNtj7EKA8nS3tnxcbYNj9vHyzWS+iXM+2S6tleNvFcfN+/NG+VdTL3M/Pacd6sp8lqeitDkXtWMRpJetLnoVtlpl3DCjw/V6NJ+f6+e3He3dBo3X4qnzWmUzH50gV9Nf1Bd0afnK9t9vfLPLg4oMC3py+99bfzc7mf7e/sfo9N7Z2y8ZnfbTtkL3FG1+hJGTBf7LbBx+np29reNTt5uuhw9ukHQHmWeWqre1t54ahOUyGYMn2Cfmmnlxrzk8/M18mo18pT3bIKOc7Ex/NPrN8HL+MhwPZzOuYo1b6n+Yl3JOmYmqRsLyFVemZSk39rpp9MtFmDb47TcQukTTubSD6XPj1UzGLJpp/yTopC/fgA1GAmpuYibJpYu5/pdUfeIq2j6zzaImB31OXsPB+pzW6nArDJT3VpcHZzqb/jTpMvF+h/pp5+dzxQ/uy+fz6sB3T39a/3l4NJvW/3CTuCn9YZsvpHfH+p98+DFMrzqrHb09+3JuaPuH7GG33pcM22umz/my92Va/cZ4rJG6jbc4Nq4dBoJGPbgZXMKjTzIT38GOzdyBZXvHPpZG4F4N9MP19XGokbpD47iJZva3Qb9za8TduRE7Bwn2h+J7ln//lzi69KvbZt2+MC1/GP+e6Nti1nHe6/z2t1uo6F9TnbN3VdrPd+vP/JeGyi4Z/xpe0mOOHwuXj98lHZ/Knl3L9/CYlgIAgNs9u+8M/BiT73cCAAAAr+/ZNxT4FZ69GQMAAAAAAAAA/LWe/cURfqh928/o+WXf/9/GtjI9fnLl0WL3Dycsx8+K46yQD4fVasvbPvRuufqW5sMPcevzWRfGBXjY/zA9uv+vKKNLZh9QvUOTnfJtFrq1fMXRaTWOO5YfAAAAAAB4Mvf/+Uvd+pEaAAAAAAAAAADAS3j2l7Fe3Jn6nK1j9f16bttzAAB+pkd3t8Zzxp2xcGHcPGfxaZUDAAAAAD9c/Fc6XP7bKqoa4aH9m7QDxLRDP7Tzk+1IbsjDt7V/ma2HbKqnzA0uEi6/vfyzVtOfZOKJzozcsrlOemKduXSK3BU5TOs/hTRcJ+2SCZM7V5469i769s1MOcbrNOH7mlxd50zerFO08PJ2Gi2uFG2k+NvWvIajj+31Tcc/foiTw1WdW7VLdJSHOJ77nLceF2sUxTPOkQC/UXtYnRyrw0Xp7z30Hv0GU1Mp3eiMw/FtLy98Jfw7HWm+suGG7ZVGT84+wNMcOv+9quA269HLxMX7LZ3F+y6dtk9Qxvfo8e7hR1n8QGRbXPyL7v3ZyeEcvOCHODeoDp9rh9bypwiWDmZVM5v/snGpf13qMH34bl41HSSTDiIZRjs6rNcZTNeWHxTp8Q889m7z6xnu3gAAAPA77bzDDgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAY6R7PTujAAAAwIv6a+4f/PbyAQAAAADwPL6vBwAAAAAAAFDZ/SQUAADgt3v25Qmv7NlbJwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPAS8o3F3+wxhVyW/GWeXd8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMDr+gcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD+SdNn5wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBXkKb/AbgHhX8=' From 70ab8d26aa565c98b75606110bdcee69eccbe063 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 17 Jul 2020 16:52:04 -0400 Subject: [PATCH 435/502] update version numbers --- docs/conf.py | 2 +- ftfy/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 5a48fde4..9d113f0f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,7 +48,7 @@ # built documents. # # The short X.Y version. -version = '5.7' +version = '5.8' # The full version, including alpha/beta/rc tags. release = '5.7' diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 8656aded..1626c9df 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -11,7 +11,7 @@ from ftfy import fixes from ftfy.formatting import display_ljust -__version__ = '5.7' +__version__ = '5.8' # See the docstring for ftfy.bad_codecs to see what we're doing here. From be33831355b7e21fa79eae426bd4057038c0843d Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 17 Jul 2020 16:53:10 -0400 Subject: [PATCH 436/502] fix another version number --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 9d113f0f..9427e2e2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,7 +50,7 @@ # The short X.Y version. version = '5.8' # The full version, including alpha/beta/rc tags. -release = '5.7' +release = '5.8' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 35fe5c122832fa1a24e312b824be800aa8ed7496 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 17 Jul 2020 17:19:48 -0400 Subject: [PATCH 437/502] CHAR_CLASS_STRING should be a str --- ftfy/build_data.py | 2 +- ftfy/char_classes.py | 2 +- tests/test_characters.py | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 92ad5721..8b64c577 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -129,7 +129,7 @@ def make_char_data_file(do_it_anyway=False): import base64 import zlib _compressed = {encoded!r} -CHAR_CLASS_STRING = zlib.decompress(base64.b64decode(_compressed)) +CHAR_CLASS_STRING = zlib.decompress(base64.b64decode(_compressed)).decode('ascii') """ with open('char_classes.py', 'w', encoding='utf-8') as out: print(script, file=out) diff --git a/ftfy/char_classes.py b/ftfy/char_classes.py index baf1eaa5..5f7b0fb1 100644 --- a/ftfy/char_classes.py +++ b/ftfy/char_classes.py @@ -3,5 +3,5 @@ import base64 import zlib _compressed = b'eJzt3Uluq8oCANCt3C0QVhB5GqIn/cmbIQ+/FMT+Z4+eom/s2E7uOc5NDBTV0hTYl/r338afP+XPxJ88z6O80vzpRVEx62NRHehrURnhNMFJ+kUqUZy/xZcifPwWR5+f+TXP3z4v+WfxdiHtqPm7lHbUvvk4+dqesfFq1mrelD9lHB8f/cTXRxfso02hXlS9+fr66MrQ/WzmqY3nbLk/+lTrXHw0mW5e/2/1754n3w6SPNj79f2avb1f0zS7Xq/5e1p4e3vP39/f0+JVLB5Lu3fXbUWYOuD7wqsOFJXpXKdpbduRh1GGtl5xUDMnXqPinYxl9yvdrKG60dKsPvztrqg8T9M4jtJDW1NevOqftHFZ1i4uU+oER/LiJ47DyPNi6Up8lax/O5/DgSLQVoR3kLdZyZviZFmSVG/Lwvd5ucTxZZi/NN/K4EpjrFb+bYJULunAoDw7I8myuAifpWkSRctBsz58NxWmV1fYWmpF6HB5esZs3OlqlJN9ZMtmzS0VcGZruEziHO0D2TSaoDj1v7Uyp5emVoqCtomVZa7K3VVIUtddWiUw3EqiogdViqNLXq+V9pW5J/32p0ikSb1Lv025zWKbnz79qmaSfNBa6SCRXekPy19lpE5/vEmMy59H7YJgzWP1P00/LH/aVsd8/ceXuvaDCrj09d+m06yctu/qJcOsdZt6U4lJWP4kHejTLxu+EsX9zrKj+qfz+zpMgvTb4jcZnNR/m6HPZiMso8jDNHakP63/afpNI0/THx00kkH7r6Q/lUzTv8TNpp805buM0//87EreliKZSTJdPpmEe0sXqm3yNj/dQXDc/mWQ8Px/+JyVXC5B1M35I2uTG54c9mi288tKzafhoiQ4kbXlT7O24IOSNhWznnyxK4zOaXHRDSp6C/VuMtNyn3FSvrqj2XrmBwVJJ2enutaaeIJZu/qBadxoMps2+a0zvrP697f8jFH/qe2FVIsu3TnyEpwV20Wz0V6CDaicjuOd3e3qOqr8feDSJC+vwA6UnzXN3tZNDaePrn9rOue74kfzvRJTGh4QP2fM7H9xp94/T1yi77o8rXeB4nh5h4LCLvF0a/uzHLo6g90x9fAE1XUMBiGqqSRJVs+P1U69unzx/Fr1L1eW77TVLShKWd59ii4zPe/BwScdXBmVa07C78jO+Ip2n7D8ey7Bk8vp+lpMfDu+QTds3H0rp+NhHW4cTtv+6rn2Xw8/bs+ygeNNC5E15dtxV2lx7TqK1Q21qM7x9rYwnbXzJjHcqO2Wnite223tW2DcH877qu6Ch6f4zdRHCe3d0tZy3JZ9byzzTTe5FVhtnHvzl1WC++NNng51Yopfw62l7nLNbmtVQZry1BcmVfjljydDdRazU8r1uzedbDadYZA9FvetTfWmf8vngN/3uvXT2f2v7nOqRrsxDvvJ4+Xb4dP3+rW0frvevvhH02kz/XZ9e3sLp6upKtYqN810t34xXaXYh0//9CaHrlCzeDZQtDxVhw/PSlnbB6k+c8r6/kg1Pdwfi/0z2jB7PF/d7Ifi+L08Ehe/ru/NR6LXciqqKqg+TL/Xr3K6qrfqpsG1WKOYqjJRtUocFXPi8SXeaqXW3r+qiqmv+Kr46hu0UX2btvlpbtqui6qw26FotXVSNsFSjTYBNmr1uI122szzZvyHQq9a7D9thZm77bHXXXLeeUQaDxPNvDsWwcFt6U4end7LGB7y77fdV41YfYFjTXB2Pnks+a1OnDLubb5XcIdIt8PcmMgvcJft5eCpdvu+zJ47N/ex92K7ceTztepS5KP5Ymv9qr8p2l3gZR8f3/19vepLgU1Ri7dJ+T3D7ubH597v56X1Otfz9x+b7lA2+lR26eZPF6CdPvt363Jox/VJKNsKsNhR3O4/jnzfLrAv/Z9sX32X0+UFepyFH+MP+yr9RLO9VHedqjDFWkUcZ3eIdeWF/NtbViTxPfGXyqIc/lDh+77jWUdfX6f0bbQWdu/2G7b/OJKtVff2SrtQB/uyh3u9v+P67WV847YMAAAAAADAbvf7/PrZJQEAAAAAAAAAAAAAAAAAAAAAAAAA+G3G4wHOBsoenCmA88+zXT+ezaiGMc3ybxxe9OcpB/cdz+tHgp2eO0rHRmK+VEPIBqMPZ2uhsyxZGh/5oDwYxXY4TvHbvKz19vbxFb6+vj5uemXN0NfNeNj9/DqhtTTq0bObMbTbV1qPsL084O+8r6zJyOVSVnFVzUt1XbVUu5O1o65G8bH9bdwabUbOtPB0FOs2tulI1NOwXR7KsYWD8a/3pLgYajY3ZYZWC9Lk76biD0ffzoblb/Kb9B26UXn3t998hZfrJ32QsAXq9+NompyUTx4/XvIqsWq1ZtXiTzJ7WKqzNjy+ty1YHlbCUFU9TP4OFvfjqX/tVx1EqnfXr682vn3jze+3p8raLWXcPmf3XwAAAAAAAAAAAABg2fBLzYtBTsX7PwDgr/MPAAAAAADAL3PbNzM45pHPm1t7/GWveVZfOKt6rl79ZZvgAYXRMPLmd3ppfh5n/KjSmece8iD9c0zvVflnY9i3tQ9jL5/hOn6Q8tbzU/Nt6XAyyqMoSos/eZf+q2yiaVplNmozOlLkO8/fF9WBlh/NWZZ88mzZY4/Wz7aCp8sPMK2XFmV4i4v6L/5FUfCc7LLc5eQgspV05ueNZ4+33mPb44O11VD++TwiDXabHfvDVP0I5ZFq1mB/LDMWL1fdrENPTU+24/tuWy10rlV7yXo7bsbfhlvMbLdv59vl6fTPUN4ozK7n04fxNIfg3ZYPb0N3fB7xAdPnFe/JaxluT173t1e/WZ0rRedAyzzGmfIweT740uJ0Oue1tLk71idL62Pz8n9HyNtj7EKA8nS3tnxcbYNj9vHyzWS+iXM+2S6tleNvFcfN+/NG+VdTL3M/Pacd6sp8lqeitDkXtWMRpJetLnoVtlpl3DCjw/V6NJ+f6+e3He3dBo3X4qnzWmUzH50gV9Nf1Bd0afnK9t9vfLPLg4oMC3py+99bfzc7mf7e/sfo9N7Z2y8ZnfbTtkL3FG1+hJGTBf7LbBx+np29reNTt5uuhw9ukHQHmWeWqre1t54ahOUyGYMn2Cfmmnlxrzk8/M18mo18pT3bIKOc7Ex/NPrN8HL+MhwPZzOuYo1b6n+Yl3JOmYmqRsLyFVemZSk39rpp9MtFmDb47TcQukTTubSD6XPj1UzGLJpp/yTopC/fgA1GAmpuYibJpYu5/pdUfeIq2j6zzaImB31OXsPB+pzW6nArDJT3VpcHZzqb/jTpMvF+h/pp5+dzxQ/uy+fz6sB3T39a/3l4NJvW/3CTuCn9YZsvpHfH+p98+DFMrzqrHb09+3JuaPuH7GG33pcM22umz/my92Va/cZ4rJG6jbc4Nq4dBoJGPbgZXMKjTzIT38GOzdyBZXvHPpZG4F4N9MP19XGokbpD47iJZva3Qb9za8TduRE7Bwn2h+J7ln//lzi69KvbZt2+MC1/GP+e6Nti1nHe6/z2t1uo6F9TnbN3VdrPd+vP/JeGyi4Z/xpe0mOOHwuXj98lHZ/Knl3L9/CYlgIAgNs9u+8M/BiT73cCAAAAr+/ZNxT4FZ69GQMAAAAAAAAA/LWe/cURfqh928/o+WXf/9/GtjI9fnLl0WL3Dycsx8+K46yQD4fVasvbPvRuufqW5sMPcevzWRfGBXjY/zA9uv+vKKNLZh9QvUOTnfJtFrq1fMXRaTWOO5YfAAAAAAB4Mvf/+Uvd+pEaAAAAAAAAAADAS3j2l7Fe3Jn6nK1j9f16bttzAAB+pkd3t8Zzxp2xcGHcPGfxaZUDAAAAAD9c/Fc6XP7bKqoa4aH9m7QDxLRDP7Tzk+1IbsjDt7V/ma2HbKqnzA0uEi6/vfyzVtOfZOKJzozcsrlOemKduXSK3BU5TOs/hTRcJ+2SCZM7V5469i769s1MOcbrNOH7mlxd50zerFO08PJ2Gi2uFG2k+NvWvIajj+31Tcc/foiTw1WdW7VLdJSHOJ77nLceF2sUxTPOkQC/UXtYnRyrw0Xp7z30Hv0GU1Mp3eiMw/FtLy98Jfw7HWm+suGG7ZVGT84+wNMcOv+9quA269HLxMX7LZ3F+y6dtk9Qxvfo8e7hR1n8QGRbXPyL7v3ZyeEcvOCHODeoDp9rh9bypwiWDmZVM5v/snGpf13qMH34bl41HSSTDiIZRjs6rNcZTNeWHxTp8Q889m7z6xnu3gAAAPA77bzDDgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAY6R7PTujAAAAwIv6a+4f/PbyAQAAAADwPL6vBwAAAAAAAFDZ/SQUAADgt3v25Qmv7NlbJwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPAS8o3F3+wxhVyW/GWeXd8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMDr+gcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD+SdNn5wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBXkKb/AbgHhX8=' -CHAR_CLASS_STRING = zlib.decompress(base64.b64decode(_compressed)) +CHAR_CLASS_STRING = zlib.decompress(base64.b64decode(_compressed)).decode('ascii') diff --git a/tests/test_characters.py b/tests/test_characters.py index 5e868c7e..512804ac 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -5,6 +5,7 @@ from ftfy.badness import sequence_weirdness import unicodedata import sys +from ftfy.char_classes import CHAR_CLASS_STRING # Most single-character strings which have been misencoded should be restored. @@ -59,3 +60,6 @@ def test_surrogates(): assert fix_surrogates('\udbff\udfff') == '\U0010ffff' assert fix_surrogates('\ud800\udc00') == '\U00010000' + +def test_char_class_type(): + assert isinstance(CHAR_CLASS_STRING, str) From 790b4aacbaef36b9032d621dc4e4ede2672a1f51 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 9 Feb 2021 18:41:22 -0500 Subject: [PATCH 438/502] =?UTF-8?q?Add=20an=20exception=20for=20the=20moji?= =?UTF-8?q?bake=20of=20'=C3=A0'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ftfy/badness.py | 5 +++++ tests/test_cases.json | 24 ++++++++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 5c4209ad..2f637efe 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -156,6 +156,11 @@ def _make_weirdness_regex(): # Windows-1250, or Windows-1252 mojibake. '[ÂÃÎÏÐÑØÙĂĎĐŃŘŮ][\x80-\x9f€ƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¢£¤¥¦§¨ª«¬¯°±²³µ¶·¸¹º¼½¾¿ˇ˘˝]|' + # The unlucky mojibake of 'à', which we decode even though it comes from the + # slightly ambiguous sequence of à and a non-breaking space. Fortunately, + # no language appears to have words that end with Ã. + 'Ã\xa0|' + # Character sequences we have to be a little more cautious about if they're # at the end of a word, but are totally okay to fix in the middle r'[ÂÃÎÏÐÑØÙĂĎĐŃŘŮ][›»‘”´©™]\w|' diff --git a/tests/test_cases.json b/tests/test_cases.json index fc420889..3806e431 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -337,19 +337,24 @@ "fixed": "NESTLÉ® requiere contratar personal para diferentes areas a nivel nacional e internacional", "expect": "pass" }, - { - "label": "Punctuation pile-up should actually be musical notes", - "comment": "Unfortunately, the difference doesn't look convincing to ftfy", - "original": "Engkau masih yg terindah, indah di dalam hatiku♫~", - "fixed": "Engkau masih yg terindah, indah di dalam hatiku♫~", - "expect": "fail" + "label": "French example containing non-breaking spaces", + "original": "ART TRIP Ã\u00a0 l'office de tourisme", + "fixed": "ART TRIP à l'office de tourisme", + "expect": "pass" }, { "label": "UTF-8 / Windows-1252 mixup in Gaelic involving non-breaking spaces", "comment": "Too similar to the Finnish negative example with non-breaking spaces", "original": "CÃ\u00a0nan nan GÃ\u00a0idheal", "fixed": "Cànan nan Gàidheal", + "expect": "pass" + }, + { + "label": "Punctuation pile-up should actually be musical notes", + "comment": "Unfortunately, the difference doesn't look convincing to ftfy", + "original": "Engkau masih yg terindah, indah di dalam hatiku♫~", + "fixed": "Engkau masih yg terindah, indah di dalam hatiku♫~", "expect": "fail" }, { @@ -366,6 +371,13 @@ "fixed": "Habitantes de Coatlán confían en proyecto de edil electo independiente", "expect": "fail" }, + { + "label": "A mix of two encodings and two languages", + "comment": "Requires handling of individual anomalous characters", + "original": "À tous mes frères et soeurs dans la syrienneté comme dans l’humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l’égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", + "fixed": "À tous mes frères et soeurs dans la syrienneté comme dans l'humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l'égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", + "expect": "fail" + }, { "label": "UTF-8 / Windows-1251 mixup in tweet spam", "original": "Blog Traffic Tip 2 – Broadcast Email Your Blog", From 746207ae8ff569314b6567dd1342dc57e765d934 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 10 Feb 2021 15:14:52 -0500 Subject: [PATCH 439/502] =?UTF-8?q?Handle=20"=C3=83=20"=20representing=20t?= =?UTF-8?q?he=20complete=20word=20"=C3=A0"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ftfy/fixes.py | 28 +++++++++++++++++++++++++++- tests/test_cases.json | 18 ++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 0bcb48cd..ffacb08a 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -617,6 +617,31 @@ def decode_match(match): return ESCAPE_SEQUENCE_RE.sub(decode_match, text) +# This regex implements an exception to restore_byte_a0, so we can decode the +# very common mojibake of (for example) "à la mode" as "à la mode", not "àla +# mode". +# +# If byte C3 appears with a word break before it and a single space after it -- +# most commonly this shows up as " à " appearing as an entire word -- we'll +# insert \xa0 while keeping the space. Without this change, we would decode "à" +# as the start of the next word, such as "àla". It's almost always intended to +# be a separate word, as in "à la", but when mojibake turns this into "Ã\xa0 +# la", the two kinds of spaces get coalesced into "à la". +# +# We make exceptions for the Portuguese words "às", "àquele", "àquela", +# "àquilo" and their plurals -- these are contractions of, for example, "a +# aquele" and are very common. Note that the final letter is important to +# distinguish this case from French "à quel point". +# +# Other instances in Portuguese, such as "àfrica", seem to be typos (intended +# to be "África" with the accent in the other direction). +# +# Unfortunately, Catalan has many words beginning with "à" that will end up +# with a space inserted in them in this case. We can't do the right thing with +# all of them. The cost is that the mojibake text "à udio" will be interpreted +# as "à udio", not the Catalan word "àudio". +A_GRAVE_WORD_RE = re.compile(b'(? Date: Wed, 10 Feb 2021 15:25:36 -0500 Subject: [PATCH 440/502] Add some tricky test cases --- tests/test_cases.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_cases.json b/tests/test_cases.json index e28a7e6e..fe5e7a44 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -344,7 +344,7 @@ "expect": "pass" }, { - "label": "à remains its own word, even if spaces after it get coalesced into one", + "label": "'à' remains its own word, even if spaces after it get coalesced into one", "original": "à perturber la réflexion des théologiens jusqu'à nos jours", "fixed": "à perturber la réflexion des théologiens jusqu'à nos jours", "expect": "pass" @@ -355,6 +355,18 @@ "fixed": "com especial atenção às crianças", "expect": "pass" }, + { + "label": "This is why we require a space after the 's' in 'às'", + "original": "Troisième édition pour ce festival qui persiste et signe à s'éloigner des grands axes pour prendre les contre-allées en 16 concerts dans 7 villes de 2 pays voisins.", + "fixed": "Troisième édition pour ce festival qui persiste et signe à s'éloigner des grands axes pour prendre les contre-allées en 16 concerts dans 7 villes de 2 pays voisins.", + "expect": "pass" + }, + { + "label": "'à quele' is the Portuguese word 'àquele', not 'à quele'", + "original": "eliminado o antígeno e mantidos os níveis de anticorpos, surgem as condições necessárias ao estabelecimento do granuloma, semelhante à quele observado nas lesões por imunocomplexo em excesso de anticorpos", + "fixed": "eliminado o antígeno e mantidos os níveis de anticorpos, surgem as condições necessárias ao estabelecimento do granuloma, semelhante àquele observado nas lesões por imunocomplexo em excesso de anticorpos", + "expect": "pass" + }, { "label": "A complex, lossy pile-up of mojibake in Portuguese", "original": "â € ðŸ“� Regulamento: â € âš ï¸� As pessoas que marcarem nos comentários perfis empresariais e/ou de marcas, personalidades ou fake serão desclassificadas. âš ï¸� Podem participar pessoas residentes em Petrolina/PE ou Juazeiro/BA, desde que se comprometam a retirar o prêmio em nosso endereço. Funcionários estão vetados. âš ï¸� Serão válidos os comentários postados até 16h, do dia 31/03/2018. E o resultado será divulgado até à s 19h do mesmo dia em uma nova publicação em nosso instagram. â € Boa sorte!!! 😀ðŸ�°", From 82f2c706f61cb8aa938b9eb7fa86df44b4a235d2 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 10 Feb 2021 16:41:46 -0500 Subject: [PATCH 441/502] =?UTF-8?q?tweak=20heuristic=20some=20more,=20hand?= =?UTF-8?q?ling=20words=20ending=20with=20=C3=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ftfy/badness.py | 5 ++++- ftfy/fixes.py | 16 ++++++++-------- tests/test_cases.json | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 2f637efe..5c69400c 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -159,7 +159,10 @@ def _make_weirdness_regex(): # The unlucky mojibake of 'à', which we decode even though it comes from the # slightly ambiguous sequence of à and a non-breaking space. Fortunately, # no language appears to have words that end with Ã. - 'Ã\xa0|' + # + # We also recognize it with a normal space, given the context of a preceding + # ASCII lowercase letter, possibly with a space in between + 'Ã\xa0|[a-z] ?à |' # Character sequences we have to be a little more cautious about if they're # at the end of a word, but are totally okay to fix in the middle diff --git a/ftfy/fixes.py b/ftfy/fixes.py index ffacb08a..4455dd8b 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -204,7 +204,7 @@ def fix_one_step_and_explain(text): # except they have b' ' where b'\xa0' would belong. if ALTERED_UTF8_RE.search(encoded_bytes): encoded_bytes = restore_byte_a0(encoded_bytes) - cost = encoded_bytes.count(0xa0) * 2 + cost = encoded_bytes.count(0xa0) transcode_steps.append(('transcode', 'restore_byte_a0', cost)) # Check for the byte 0x1a, which indicates where one of our @@ -621,12 +621,12 @@ def decode_match(match): # very common mojibake of (for example) "à la mode" as "à la mode", not "àla # mode". # -# If byte C3 appears with a word break before it and a single space after it -- -# most commonly this shows up as " à " appearing as an entire word -- we'll -# insert \xa0 while keeping the space. Without this change, we would decode "à" -# as the start of the next word, such as "àla". It's almost always intended to -# be a separate word, as in "à la", but when mojibake turns this into "Ã\xa0 -# la", the two kinds of spaces get coalesced into "à la". +# If byte C3 appears with a single space after it -- most commonly this shows +# up as " à " appearing as an entire word -- we'll insert \xa0 while keeping +# the space. Without this change, we would decode "à" as the start of the next +# word, such as "àla". It's almost always intended to be a separate word, as in +# "à la", but when mojibake turns this into "Ã\xa0 la", the two kinds of spaces +# get coalesced into "à la". # # We make exceptions for the Portuguese words "às", "àquele", "àquela", # "àquilo" and their plurals -- these are contractions of, for example, "a @@ -640,7 +640,7 @@ def decode_match(match): # with a space inserted in them in this case. We can't do the right thing with # all of them. The cost is that the mojibake text "à udio" will be interpreted # as "à udio", not the Catalan word "àudio". -A_GRAVE_WORD_RE = re.compile(b'(? Date: Wed, 10 Feb 2021 16:45:20 -0500 Subject: [PATCH 442/502] test on python 3.6 through 3.9 --- .gitignore | 2 ++ pytest.ini | 2 +- setup.py | 6 +++--- tox.ini | 6 ++++++ 4 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 934704b6..f9a3c19a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ _build twitterlogs .eggs .pytest_cache +.tox +specimens diff --git a/pytest.ini b/pytest.ini index 6afcb4c3..944fc3ad 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --doctest-modules --ignore=setup.py +addopts = --doctest-modules --ignore=setup.py --ignore=specimens diff --git a/setup.py b/setup.py index 7a5c03ed..de8b43c1 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name="ftfy", - version='5.8', + version='5.9', maintainer='Luminoso Technologies, Inc.', maintainer_email='info@luminoso.com', license="MIT", @@ -33,13 +33,13 @@ packages=['ftfy', 'ftfy.bad_codecs'], install_requires=['wcwidth'], tests_require=['pytest'], - python_requires='>=3.5', + python_requires='>=3.6', classifiers=[ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..ed4c649b --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py36, py37, py38, py39 + +[testenv] +deps = pytest +commands = pytest From 1bfbc86f460e6f0094c443a91c06a6153dabb9a7 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 10 Feb 2021 16:46:27 -0500 Subject: [PATCH 443/502] put my name in as 'maintainer' because really --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index de8b43c1..4bf21f91 100644 --- a/setup.py +++ b/setup.py @@ -22,8 +22,8 @@ setup( name="ftfy", version='5.9', - maintainer='Luminoso Technologies, Inc.', - maintainer_email='info@luminoso.com', + maintainer='Robyn Speer', + maintainer_email='rspeer@luminoso.com', license="MIT", url='http://github.com/LuminosoInsight/python-ftfy', platforms=["any"], From 8c65a8b138f67a4d21145162cf488edb9befbec5 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 10 Feb 2021 16:46:48 -0500 Subject: [PATCH 444/502] update char_classes to Unicode 13 --- ftfy/build_data.py | 4 ++-- ftfy/char_classes.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 8b64c577..76d84903 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -47,8 +47,8 @@ def make_char_data_file(do_it_anyway=False): To protect against this, running this in the wrong version of Python will raise an error unless you pass `do_it_anyway=True`. """ - if sys.version_info < (3, 8) and not do_it_anyway: - raise RuntimeError("This function should be run in Python 3.8 or later.") + if sys.version_info < (3, 9) and not do_it_anyway: + raise RuntimeError("This function should be run in Python 3.9 or later.") cclasses = [None] * 0x110000 for codepoint in range(0x0, 0x110000): diff --git a/ftfy/char_classes.py b/ftfy/char_classes.py index 5f7b0fb1..1b275084 100644 --- a/ftfy/char_classes.py +++ b/ftfy/char_classes.py @@ -2,6 +2,6 @@ # This script is automatically generated by build_data.py. import base64 import zlib -_compressed = b'eJzt3Uluq8oCANCt3C0QVhB5GqIn/cmbIQ+/FMT+Z4+eom/s2E7uOc5NDBTV0hTYl/r338afP+XPxJ88z6O80vzpRVEx62NRHehrURnhNMFJ+kUqUZy/xZcifPwWR5+f+TXP3z4v+WfxdiHtqPm7lHbUvvk4+dqesfFq1mrelD9lHB8f/cTXRxfso02hXlS9+fr66MrQ/WzmqY3nbLk/+lTrXHw0mW5e/2/1754n3w6SPNj79f2avb1f0zS7Xq/5e1p4e3vP39/f0+JVLB5Lu3fXbUWYOuD7wqsOFJXpXKdpbduRh1GGtl5xUDMnXqPinYxl9yvdrKG60dKsPvztrqg8T9M4jtJDW1NevOqftHFZ1i4uU+oER/LiJ47DyPNi6Up8lax/O5/DgSLQVoR3kLdZyZviZFmSVG/Lwvd5ucTxZZi/NN/K4EpjrFb+bYJULunAoDw7I8myuAifpWkSRctBsz58NxWmV1fYWmpF6HB5esZs3OlqlJN9ZMtmzS0VcGZruEziHO0D2TSaoDj1v7Uyp5emVoqCtomVZa7K3VVIUtddWiUw3EqiogdViqNLXq+V9pW5J/32p0ikSb1Lv025zWKbnz79qmaSfNBa6SCRXekPy19lpE5/vEmMy59H7YJgzWP1P00/LH/aVsd8/ceXuvaDCrj09d+m06yctu/qJcOsdZt6U4lJWP4kHejTLxu+EsX9zrKj+qfz+zpMgvTb4jcZnNR/m6HPZiMso8jDNHakP63/afpNI0/THx00kkH7r6Q/lUzTv8TNpp805buM0//87EreliKZSTJdPpmEe0sXqm3yNj/dQXDc/mWQ8Px/+JyVXC5B1M35I2uTG54c9mi288tKzafhoiQ4kbXlT7O24IOSNhWznnyxK4zOaXHRDSp6C/VuMtNyn3FSvrqj2XrmBwVJJ2enutaaeIJZu/qBadxoMps2+a0zvrP697f8jFH/qe2FVIsu3TnyEpwV20Wz0V6CDaicjuOd3e3qOqr8feDSJC+vwA6UnzXN3tZNDaePrn9rOue74kfzvRJTGh4QP2fM7H9xp94/T1yi77o8rXeB4nh5h4LCLvF0a/uzHLo6g90x9fAE1XUMBiGqqSRJVs+P1U69unzx/Fr1L1eW77TVLShKWd59ii4zPe/BwScdXBmVa07C78jO+Ip2n7D8ey7Bk8vp+lpMfDu+QTds3H0rp+NhHW4cTtv+6rn2Xw8/bs+ygeNNC5E15dtxV2lx7TqK1Q21qM7x9rYwnbXzJjHcqO2Wnite223tW2DcH877qu6Ch6f4zdRHCe3d0tZy3JZ9byzzTTe5FVhtnHvzl1WC++NNng51Yopfw62l7nLNbmtVQZry1BcmVfjljydDdRazU8r1uzedbDadYZA9FvetTfWmf8vngN/3uvXT2f2v7nOqRrsxDvvJ4+Xb4dP3+rW0frvevvhH02kz/XZ9e3sLp6upKtYqN810t34xXaXYh0//9CaHrlCzeDZQtDxVhw/PSlnbB6k+c8r6/kg1Pdwfi/0z2jB7PF/d7Ifi+L08Ehe/ru/NR6LXciqqKqg+TL/Xr3K6qrfqpsG1WKOYqjJRtUocFXPi8SXeaqXW3r+qiqmv+Kr46hu0UX2btvlpbtqui6qw26FotXVSNsFSjTYBNmr1uI122szzZvyHQq9a7D9thZm77bHXXXLeeUQaDxPNvDsWwcFt6U4end7LGB7y77fdV41YfYFjTXB2Pnks+a1OnDLubb5XcIdIt8PcmMgvcJft5eCpdvu+zJ47N/ex92K7ceTztepS5KP5Ymv9qr8p2l3gZR8f3/19vepLgU1Ri7dJ+T3D7ubH597v56X1Otfz9x+b7lA2+lR26eZPF6CdPvt363Jox/VJKNsKsNhR3O4/jnzfLrAv/Z9sX32X0+UFepyFH+MP+yr9RLO9VHedqjDFWkUcZ3eIdeWF/NtbViTxPfGXyqIc/lDh+77jWUdfX6f0bbQWdu/2G7b/OJKtVff2SrtQB/uyh3u9v+P67WV847YMAAAAAADAbvf7/PrZJQEAAAAAAAAAAAAAAAAAAAAAAAAA+G3G4wHOBsoenCmA88+zXT+ezaiGMc3ybxxe9OcpB/cdz+tHgp2eO0rHRmK+VEPIBqMPZ2uhsyxZGh/5oDwYxXY4TvHbvKz19vbxFb6+vj5uemXN0NfNeNj9/DqhtTTq0bObMbTbV1qPsL084O+8r6zJyOVSVnFVzUt1XbVUu5O1o65G8bH9bdwabUbOtPB0FOs2tulI1NOwXR7KsYWD8a/3pLgYajY3ZYZWC9Lk76biD0ffzoblb/Kb9B26UXn3t998hZfrJ32QsAXq9+NompyUTx4/XvIqsWq1ZtXiTzJ7WKqzNjy+ty1YHlbCUFU9TP4OFvfjqX/tVx1EqnfXr682vn3jze+3p8raLWXcPmf3XwAAAAAAAAAAAABg2fBLzYtBTsX7PwDgr/MPAAAAAADAL3PbNzM45pHPm1t7/GWveVZfOKt6rl79ZZvgAYXRMPLmd3ppfh5n/KjSmece8iD9c0zvVflnY9i3tQ9jL5/hOn6Q8tbzU/Nt6XAyyqMoSos/eZf+q2yiaVplNmozOlLkO8/fF9WBlh/NWZZ88mzZY4/Wz7aCp8sPMK2XFmV4i4v6L/5FUfCc7LLc5eQgspV05ueNZ4+33mPb44O11VD++TwiDXabHfvDVP0I5ZFq1mB/LDMWL1fdrENPTU+24/tuWy10rlV7yXo7bsbfhlvMbLdv59vl6fTPUN4ozK7n04fxNIfg3ZYPb0N3fB7xAdPnFe/JaxluT173t1e/WZ0rRedAyzzGmfIweT740uJ0Oue1tLk71idL62Pz8n9HyNtj7EKA8nS3tnxcbYNj9vHyzWS+iXM+2S6tleNvFcfN+/NG+VdTL3M/Pacd6sp8lqeitDkXtWMRpJetLnoVtlpl3DCjw/V6NJ+f6+e3He3dBo3X4qnzWmUzH50gV9Nf1Bd0afnK9t9vfLPLg4oMC3py+99bfzc7mf7e/sfo9N7Z2y8ZnfbTtkL3FG1+hJGTBf7LbBx+np29reNTt5uuhw9ukHQHmWeWqre1t54ahOUyGYMn2Cfmmnlxrzk8/M18mo18pT3bIKOc7Ex/NPrN8HL+MhwPZzOuYo1b6n+Yl3JOmYmqRsLyFVemZSk39rpp9MtFmDb47TcQukTTubSD6XPj1UzGLJpp/yTopC/fgA1GAmpuYibJpYu5/pdUfeIq2j6zzaImB31OXsPB+pzW6nArDJT3VpcHZzqb/jTpMvF+h/pp5+dzxQ/uy+fz6sB3T39a/3l4NJvW/3CTuCn9YZsvpHfH+p98+DFMrzqrHb09+3JuaPuH7GG33pcM22umz/my92Va/cZ4rJG6jbc4Nq4dBoJGPbgZXMKjTzIT38GOzdyBZXvHPpZG4F4N9MP19XGokbpD47iJZva3Qb9za8TduRE7Bwn2h+J7ln//lzi69KvbZt2+MC1/GP+e6Nti1nHe6/z2t1uo6F9TnbN3VdrPd+vP/JeGyi4Z/xpe0mOOHwuXj98lHZ/Knl3L9/CYlgIAgNs9u+8M/BiT73cCAAAAr+/ZNxT4FZ69GQMAAAAAAAAA/LWe/cURfqh928/o+WXf/9/GtjI9fnLl0WL3Dycsx8+K46yQD4fVasvbPvRuufqW5sMPcevzWRfGBXjY/zA9uv+vKKNLZh9QvUOTnfJtFrq1fMXRaTWOO5YfAAAAAAB4Mvf/+Uvd+pEaAAAAAAAAAADAS3j2l7Fe3Jn6nK1j9f16bttzAAB+pkd3t8Zzxp2xcGHcPGfxaZUDAAAAAD9c/Fc6XP7bKqoa4aH9m7QDxLRDP7Tzk+1IbsjDt7V/ma2HbKqnzA0uEi6/vfyzVtOfZOKJzozcsrlOemKduXSK3BU5TOs/hTRcJ+2SCZM7V5469i769s1MOcbrNOH7mlxd50zerFO08PJ2Gi2uFG2k+NvWvIajj+31Tcc/foiTw1WdW7VLdJSHOJ77nLceF2sUxTPOkQC/UXtYnRyrw0Xp7z30Hv0GU1Mp3eiMw/FtLy98Jfw7HWm+suGG7ZVGT84+wNMcOv+9quA269HLxMX7LZ3F+y6dtk9Qxvfo8e7hR1n8QGRbXPyL7v3ZyeEcvOCHODeoDp9rh9bypwiWDmZVM5v/snGpf13qMH34bl41HSSTDiIZRjs6rNcZTNeWHxTp8Q889m7z6xnu3gAAAPA77bzDDgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAY6R7PTujAAAAwIv6a+4f/PbyAQAAAADwPL6vBwAAAAAAAFDZ/SQUAADgt3v25Qmv7NlbJwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPAS8o3F3+wxhVyW/GWeXd8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMDr+gcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD+SdNn5wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBXkKb/AbgHhX8=' +_compressed = b'eJzt3Uluq8oCANCt3C1gVhB5GqIn/cmboQy/FMT+Z4+eom/s2E7uOc5NjCmqp8DYl/r338afP+XPxJ88z6O80vzpRVHx0vuiOtDXojLCaYKT9ItUoji/xNcifHyJo4+P/DPPLx/X/KN4upB21PxdSjtqn7yffGy/sPFotmqelD9lHO/v/cLXexfsvU2hXlU9+fp678rQ/WzmqY3nbLnf+1TrXLw3mW4e/2/1z54n3w6SPNjb59tndnn7TNPs8/Mzf0sLl8tb/vb2lhaPYvVY2j373FaEqQO+LTzqQFGZzuc0rW078jDK0NYjDmrmxGNUvJOx7H6kmzVUN1qa1cPf7orK8zSN4yg91Jvy4lH/pI3rsnZ1mVInGMmLnzgOI8+LtSvxVbL+6XwOB4pAWxHeQd5mJW+Kk2VJUj0tC9/n5RrH12H+0nwrgyuNsVr5twlSuaYDg/LsjCTL4iJ8lqZJFC0Hzfrw3VKYXl1ha6kVocP16Rmzcc+8Gmwy2Ue2bNbcUgFnesN1EudoH8im0XTFKYu71IvS7m9TK0VB28TKMlfl7qohqesurRIY9pKoOIMqxdE1r7dK+yrck377UyTSpN6l36bcZrHNT59+VTNJPmitdJDIrvSH5a8yUqc/7mXj8udRuyLY8lj9T9MPy9+WaaH+42td+0EFXPv6b9NpNk7bZ/WaYda6rt4kmITlT9KBPv2y4StR3O8sO6p/+npfh0mQfhNpk99p/bcZ+mg6YRlFHqaxI/1p/U/Tbxp5mv5oHBrsq2vpTyXT9K9x0/WTpnzXcfofH13J6yQH/X9QuZsVkfadtW3yNj/dIDhu/zJIePw/fMxKrtcg6ub4kbXJDQ8OezT9/LpS82m4KgkOZG3506wt+KCkTcWsJ1/sCqNjWlycBhVnC/VuMtNyH3FSPrrRbD3zg4Kkk6NTXWtNPMFLu84D07jRZDZt8ltnfGf172/5GaPzp/YspN2rknofuQZHxXbVbLTXoAOVy3G883S7eh9V/j7w1iQv34EdKD9rmr2tWxouH93+1nTOn4ofzfdKTGk4IH7MmNn/4k69f554i77r7Wm9CxTj5R0KCrvE0972Zzl0dQS7Y+rhAao7MRiEqJa6k6aF42O1U6+uXzy+VueXK+t32jotKEpZXn2KrjNn3oPBJx28Myq3nITfkZ3xO9p9wvLveQueXE/X12Li2/ENTsPGp2/lcjysw43htD1fPdf+6+HH7Vk2cLxpIbKmfDuuKi1uXUex2lGL6hz3t4XlrH1tJo6btKel54rXnrb2LTA+H877qu6Ch4f4zdRHCe3taWs5bsu+N5b5pptcCqw65978ZZXg+niTp0MnMcWvYW+pT7lm+1pVkKY89RuTKvzyx5OhOovZKeX23ZNONpvOMMgeC3vWDnXXv+VzwO973Prp7P5H9zlVo+2Mw/Pk8frt8Olb/Vjavt1uX/yj5bRZvnxeLpdwuVqqYq1y0yx32xfLVYp9+PRPbzJ0hZrVs4Gi5aU6fHhUytpzkOozp6w/H6mWh/tjsX9GG2bH89VuPxTHb+VIXPz6fGs+Ev0sl6Kqguph+q1+lMtVvVUXDT6LLYqlKhNVq8RR8Uo8fou3Wqm1t6+qYup3fFV89QXaqL5M2/w0F23XRVXY7VC02jopm2CpRpsAG7V63EY7beZ5M/5DoVctnj9thZm77LHXXXLeeUQaDxPNPDsWwcG+dCePTu9lDIf8+/X7qhGrL3CsCY7OJ8eS3+rEIePe5s8K7hDpdpgbE/kF7tJfDh5qt6/L7Lhwcyd732w3jny+Vr0VeW++2Fo/6m+Kdm/wsvf37/6+XvWlwKaoxdOk/J5hd/HjY+/389J6m8/z1x+b06Fs9Kns0sWfLkC7fPbv1tuhHe9PQtlWgOJIt3iuuHH+OPJ9+8C+9H+yffVdLpdv0OPsOmzBcKl71vSX6qpTFabYqojj7A6xrnwjf7lkRRLfE3+pLMrhDxW+7zuedfT1+5S+jbY2ONobDm6f7j0r7UIdPJc9fNb7O96/vYzv6cYAAAAAAACc5fMfAAAAAAAAAAAAAAAAAAAAAAAAAIAXtjT1XfbgfACcn491cX7LJdU0pln+jdOL/jzl5L7j1/qZYINJiQPHZmK+VlPIBrMPZ2uhsyxZmh/5oDyYxbb+2+b/Mi9rXS7vX+Hj6+v9pkfWTH3dzIfdv14ntJZGPXt2M4d2+0jTblrt/ZMuF0lkTUau17KKq2pequuqpcqdLCk2bWddjeJj+9u4NdqsnGnh6SzWbWzhzMVt/GNdHsq5hYP5r/ekuBhqNjdlhlYL0uTvpuIPZt9Os2H5m/wm/QndqLz722++wsvtkz5I2AL183E0TU7KO48fL3mVWLVZs2nxJ5kdluqsDcf3tgXLYSUMVdXD5O9gdT+f+td+1SBSPfss57q/XMr49s03v9+eKmt7yrh9zu6/AAAAAAAAAAAAAMCy4ZeaF4Ocivd/AMBf5x8AAAAAAIBf5rZvZnDMI+83t3kXzEpzr77wpeq+evWXbYIbFEbDyJvf6bX5eZzxrUpn7nvIg/T3Mb1X5Z+NYV9vH8Ze3sN1fCPlrfun5tvS4WKUR1GUFn/yLv1X6aJpWmU2ajM6UuQ7z98W1YGWb81Zlnxyb9ljt9bPtoKnyzcwrdcWZbjERf0X/6IouE92We5ycRDZSjrzr41fHvfeY/3xwdpqKP98HJEGu82O/WGqvoXySPXScH8sE9q7YzcO3TU92Y7vu2210LlW7SXr7bgZfxtuMbPdvp1vl6fT30N5ozC77k8fxtMMwbstD29Dd7wf8QHT+xXvyWsZbk9e97dX363OlaJzoGUe40x5mNwffGl1On3ltbS5O3ZOltZj8/J/R8jbMXYhQHm4W1s/rrbBmH28fDOZb+KcT7ZLa2X8reK4eX/eKP9q6mXup8e0Q6cyH+WhKG2ORe1cBOl16xS9ClttMm6Y0XC9Hs3Hx/rxbUd7t0HjtXjqvFbZzEcHyNX0F/UFXVq/0v/7zje7PqjIsKAn+//e+rvZyfT3nn+MDu+dveclo8N+2lbonqLNzzByssB/mY3h50D/K4acPE3vfSRdTTIYJtfDBxdIukHmvvlcTX9lbFiqzNapSViukzl4gn1irpkX95rD09/Mp9nIV9qzDTLKyc70R7PfDN/Ol9PQbNT3IK5ii1vqf5iX8pUyE80ML0Gu8vL12eZY7R/LRZg2+O0XELpE07m0g+Vz89VM5iyaaf8kOElfvgAbzATUXMRMkmsXc/0vqc6Jq2j7zDarmhz0OXkNB+tzWqvDXjiQ5sl1a7y+Peky8X6H+mnH53PFD67LT2q9Py/d09Furv88HM2m9T/sEDelP2zzhfTuWP+TDz+G6VVHtaOXZ1/ODW3/kD3s1uuSYXvNnCb1bR0M/Guf6DY9od0Fq/nows537/EnyOChRuo6bzE2rg0DQaMe7AbXcPRJZuI7eGIzN7BsV+SxNAL3aZ4fr6+PQ43UDY3jJprZ3wbnnVsz7s7N2DlIsB+K71n+/V/i6NKvBoFuX5iWP4x/T/TdwFLHuBAfxyxU9I+tzvG4NXtVpf18t/7Mf2mq7JL5r+ElPWY8WXj7+F3S8aHs2bV8D49pKQAAuN2zz52BH2Py/U4AAADg9T37ggK/wrO7MQAAAAAAAADAX+vZXxzhh9rXf0b3L/v+/za2lenxnSuPFru/OWE5f1YcZ4V8OK1WW97VW5wvJf7d9QP3dOv9WRfmBXjY/zA9uv+vKKNLZm9QvUOTnfJpFrq1fMXolIV3yxzfOvOO5QcAAAAAAJ7M9X/gd9r/+eiJTQAAAAAAAAAA4GU88Us6P8GZ+pytY/X9em7bcwAAfqZHn26NXxmfjIUr4+Y+i0+rHAAAAAD44eK/0uHy31ZR1QwP7d+knSCmnfqhfT3ZjuSGPHxb+5fZekhXPWVucpFw/e3ln7Wa/iQTT3Rm5pbNbdIT28ylU+SuyGFa/ymk4TZpl0yY3Lny1LF30bdPZsox3qYJ39fk6jZn8mabooWX+2m0uFG0keJv2/IznH1sr28a//ghTk5XdW7TLtFRHuJ47nPeel6sURTPOEYC/EbtsDoZq8NV6e8deo9+g6mplG52xuH8ttcXfif8Ox1pvrLhhu2VRk/OPsDTHDr+vargMuvRt4mL11s6i9ddOu05QRnfo+e7hx9l8QORbXHxL7r3ZyeHc/CCH+LcoBo+14bW8qcIlg5eql5s/svGtf51rcP04bvXquUgmXQQyTDa0bBeZzBdW39Q5Ix/4LFXm1/PcPcGAACA32nnFXYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB4q3fTsHAIAAAAv6q+5fvDbywcAAAAAwPP4vh4AAAAAAABAZfsWKAAAwF/i2W9PeGXP7p0AfJdnH2EAAAAA4Id49qU8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAeB35xupv9phCLkv+Ms+ubwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA4HX9AwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP+k6bNzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwCtI0/8AIUUfmw==' CHAR_CLASS_STRING = zlib.decompress(base64.b64decode(_compressed)).decode('ascii') From e211073bcc3cd4b63449ffb6b8894c454fc96dc2 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 10 Feb 2021 16:46:58 -0500 Subject: [PATCH 445/502] Update changelog --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49cbcc75..fbfb9d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +## Version 5.9 (February 10, 2021) + +This version is brought to you by the letter à and the number 0xC3. + +- Tweaked the heuristic to decode, for example, "à" as the letter "à" + more often. + +- This combines with the non-breaking-space fixer to decode "à " as "à" as + well. However, in many cases, the text " à " was intended to be " à ", + preserving the space -- the underlying mojibake had two spaces after it, but + the Web coalesced them into one. We detect this case based on common French + and Portuguese words, and preserve the space when it appears intended. + +Thanks to @zehavoc for bringing to my attention how common this case is. + +- Updated the data file of Unicode character categories to Unicode 13, as + used in Python 3.9. (No matter what version of Python you're on, ftfy uses + the same data.) + ## Version 5.8 (July 17, 2020) - Improved detection of UTF-8 mojibake of Greek, Cyrillic, Hebrew, and Arabic From 3b9e86d1454b893b6598bce52eda4d90b4f4c075 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 10 Feb 2021 17:05:45 -0500 Subject: [PATCH 446/502] hey wait this expected failure is actually a success --- tests/test_cases.json | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_cases.json b/tests/test_cases.json index 8c51ac83..db0d7e58 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -119,6 +119,14 @@ "fixed": "It was named\u00a0\"scars´ stones\" after the rock-climbers who got hurt while climbing on it.", "expect": "pass" }, + { + "label": "UTF-8 / Windows-1252 mixup in mixed French and Arabic", + "comment": "'fixed-encoding' is still mojibaked. The thing that makes this test case work is that the full ftfy pipeline can uncurl the quotes, then fix the mojibake", + "original": "À tous mes frères et soeurs dans la syrienneté comme dans l’humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l’égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", + "fixed-encoding": "À tous mes frères et soeurs dans la syrienneté comme dans l’humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l’égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", + "fixed": "À tous mes frères et soeurs dans la syrienneté comme dans l'humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l'égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", + "expect": "pass" + }, { "label": "UTF-8 / sloppy Windows-1250 mixup in Romanian", "original": "vedere Ă®nceĹŁoĹźatÄ\u0083", @@ -419,13 +427,6 @@ "fixed": "Habitantes de Coatlán confían en proyecto de edil electo independiente", "expect": "fail" }, - { - "label": "A mix of two encodings and two languages", - "comment": "Requires handling of individual anomalous characters", - "original": "À tous mes frères et soeurs dans la syrienneté comme dans l’humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l’égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", - "fixed": "À tous mes frères et soeurs dans la syrienneté comme dans l'humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l'égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", - "expect": "fail" - }, { "label": "UTF-8 / Windows-1251 mixup in tweet spam", "original": "Blog Traffic Tip 2 – Broadcast Email Your Blog", From da172537305b66a674e03bd34967c49b06196c31 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 10 Feb 2021 17:26:41 -0500 Subject: [PATCH 447/502] =?UTF-8?q?include=20examples=20where=20'=D0=93'?= =?UTF-8?q?=20means=20'=C3=A0';=20one=20passes=20already?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_cases.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_cases.json b/tests/test_cases.json index db0d7e58..dfaee008 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -387,6 +387,13 @@ "fixed": "Troisième édition pour ce festival qui persiste et signe à s'éloigner des grands axes pour prendre les contre-allées en 16 concerts dans 7 villes de 2 pays voisins.", "expect": "pass" }, + { + "label": "We can fix 'à' in windows-1251 sometimes as well", + "original": "La rГ©gion de Dnepropetrovsk se trouve Г l’ouest de l’Ukraine", + "fixed-encoding": "La région de Dnepropetrovsk se trouve à l’ouest de l’Ukraine", + "fixed": "La région de Dnepropetrovsk se trouve à l'ouest de l'Ukraine", + "expect": "pass" + }, { "label": "'à quele' is the Portuguese word 'àquele', not 'à quele'", "original": "eliminado o antígeno e mantidos os níveis de anticorpos, surgem as condições necessárias ao estabelecimento do granuloma, semelhante à quele observado nas lesões por imunocomplexo em excesso de anticorpos", @@ -427,6 +434,12 @@ "fixed": "Habitantes de Coatlán confían en proyecto de edil electo independiente", "expect": "fail" }, + { + "label": "An example with 'à' in windows-1251 where we need our heuristic to be bolder", + "original": "faites attention Г bien vous renseigner avant sur le mГ©dicament", + "fixed": "faites attention à bien vous renseigner avant sur le médicament", + "expect": "fail" + }, { "label": "UTF-8 / Windows-1251 mixup in tweet spam", "original": "Blog Traffic Tip 2 – Broadcast Email Your Blog", From 82d6060e286d93fc12f30ff77786e2c3b7389160 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 10 Feb 2021 18:13:35 -0500 Subject: [PATCH 448/502] =?UTF-8?q?clarify=20the=20problem=20with=20=C3=A0?= =?UTF-8?q?=20in=20the=20middle=20of=20a=20word?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ftfy/fixes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 4455dd8b..f1cceb95 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -636,10 +636,10 @@ def decode_match(match): # Other instances in Portuguese, such as "àfrica", seem to be typos (intended # to be "África" with the accent in the other direction). # -# Unfortunately, Catalan has many words beginning with "à" that will end up -# with a space inserted in them in this case. We can't do the right thing with -# all of them. The cost is that the mojibake text "à udio" will be interpreted -# as "à udio", not the Catalan word "àudio". +# Unfortunately, "à" is a common letter in Catalan, and mojibake of words that +# contain it will end up with inserted spaces. We can't do the right thing with +# every word. The cost is that the mojibake text "fà cil" will be interpreted as +# "fà cil", not "fàcil". A_GRAVE_WORD_RE = re.compile(b'\xc3 (?! |quele|quela|quilo|s )') def restore_byte_a0(byts): From 13924b811fbe2175f31bbf911b2ec8ad211c86b0 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 10 Feb 2021 18:14:16 -0500 Subject: [PATCH 449/502] keep python 3.5 support for now --- ftfy/build_data.py | 4 ++-- setup.py | 2 +- tox.ini | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ftfy/build_data.py b/ftfy/build_data.py index 76d84903..f9100ae7 100644 --- a/ftfy/build_data.py +++ b/ftfy/build_data.py @@ -124,13 +124,13 @@ def make_char_data_file(do_it_anyway=False): char_classes_bytes = ''.join(cclasses).encode('ascii') encoded = base64.b64encode(zlib.compress(char_classes_bytes)) - script = f""" + script = """ # This script is automatically generated by build_data.py. import base64 import zlib _compressed = {encoded!r} CHAR_CLASS_STRING = zlib.decompress(base64.b64decode(_compressed)).decode('ascii') -""" +""".format(encoded=encoded) with open('char_classes.py', 'w', encoding='utf-8') as out: print(script, file=out) diff --git a/setup.py b/setup.py index 4bf21f91..fabc4fdf 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ packages=['ftfy', 'ftfy.bad_codecs'], install_requires=['wcwidth'], tests_require=['pytest'], - python_requires='>=3.6', + python_requires='>=3.5', classifiers=[ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", diff --git a/tox.ini b/tox.ini index ed4c649b..8945ab47 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, py39 +envlist = py35, py36, py37, py38, py39 [testenv] deps = pytest From 8da9aeebea01be3d61489a64d23ada8e4a92c1d7 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 11 Feb 2021 14:54:35 -0500 Subject: [PATCH 450/502] update docs configuration --- docs/conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9427e2e2..31828a20 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ # General information about the project. project = u'ftfy' -copyright = u'2018, Robyn Speer' +copyright = u'2021, Robyn Speer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '5.8' +version = '5.9' # The full version, including alpha/beta/rc tags. -release = '5.8' +release = '5.9' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From dd5f594567ae3bc23105ecdce32c5af8493de32a Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 18 Mar 2021 12:08:39 -0400 Subject: [PATCH 451/502] a new, simpler heuristic --- ftfy/__init__.py | 1 - ftfy/badness.py | 544 ++++++++++++++++------------ ftfy/chardata.py | 21 +- ftfy/fixes.py | 118 +++--- pytest.ini | 2 +- tests/test_cases.json | 11 +- tests/test_characters.py | 23 +- tests/test_futuristic_codepoints.py | 61 ---- 8 files changed, 368 insertions(+), 413 deletions(-) delete mode 100644 tests/test_futuristic_codepoints.py diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 1626c9df..9631bf18 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -192,7 +192,6 @@ def fix_text( # Some alternate names for the main functions ftfy = fix_text fix_encoding = fixes.fix_encoding -fix_text_encoding = fixes.fix_text_encoding # deprecated def fix_file( diff --git a/ftfy/badness.py b/ftfy/badness.py index 5c69400c..4e8b04d2 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -1,253 +1,321 @@ """ -Heuristics to determine whether re-encoding text is actually making it -more reasonable. +Heuristics to detect likely mojibake. """ import re -import unicodedata - -from ftfy.chardata import chars_to_classes - -# The following regex uses the mapping of character classes to ASCII -# characters defined in chardata.py and build_data.py: -# -# L = Latin capital letter -# l = Latin lowercase letter -# A = Non-latin capital or title-case letter -# a = Non-latin lowercase letter -# C = Non-cased letter (Lo) -# X = Control character (Cc) -# m = Letter modifier (Lm) -# M = Mark (Mc, Me, Mn) -# N = Miscellaneous numbers (No) -# 1 = Math symbol (Sm) or currency symbol (Sc) -# 2 = Symbol modifier (Sk) -# 3 = Other symbol (So) -# S = UTF-16 surrogate -# _ = Unassigned character -# = Whitespace -# o = Other - - -def _make_weirdness_regex(): + +def _char_range(start, end): """ - Creates a list of regexes that match 'weird' character sequences. - The more matches there are, the weirder the text is. + Get a contiguous range of characters as a string. Useful for including + in a regex without making the representation _only_ usable in regexes. """ - groups = [] - - # Match diacritical marks, except when they modify a non-cased letter or - # another mark. - # - # You wouldn't put a diacritical mark on a digit or a space, for example. - # You might put it on a Latin letter, but in that case there will almost - # always be a pre-composed version, and we normalize to pre-composed - # versions first. The cases that can't be pre-composed tend to be in - # large scripts without case, which are in class C. - groups.append('[^CM]M') - - # Match non-Latin characters adjacent to Latin characters. - # - # This is a simplification from ftfy version 2, which compared all - # adjacent scripts. However, the ambiguities we need to resolve come from - # encodings designed to represent Latin characters. - groups.append('[Ll][AaC]') - groups.append('[AaC][Ll]') - - # Match IPA letters next to capital letters. - # - # IPA uses lowercase letters only. Some accented capital letters next to - # punctuation can accidentally decode as IPA letters, and an IPA letter - # appearing next to a capital letter is a good sign that this happened. - groups.append('[LA]i') - groups.append('i[LA]') - - # Match non-combining diacritics. We've already set aside the common ones - # like ^ (the CIRCUMFLEX ACCENT, repurposed as a caret, exponent sign, - # or happy eye) and assigned them to category 'o'. The remaining ones, - # like the diaeresis (¨), are pretty weird to see on their own instead - # of combined with a letter. - groups.append('2') - - # Match C1 control characters, which are almost always the result of - # decoding Latin-1 that was meant to be Windows-1252. - groups.append('X') - - # Match private use and unassigned characters. - groups.append('P') - groups.append('_') - - # Match adjacent characters from any different pair of these categories: - # - Modifier marks (M) - # - Letter modifiers (m) - # - Miscellaneous numbers (N) - # - Symbols (1 or 3, because 2 is already weird on its own) - - exclusive_categories = 'MmN13' - for cat1 in exclusive_categories: - others_range = ''.join(c for c in exclusive_categories if c != cat1) - groups.append( - '{cat1}[{others_range}]'.format(cat1=cat1, others_range=others_range) - ) - regex = '|'.join(groups) - return re.compile(regex) - - -WEIRDNESS_RE = _make_weirdness_regex() - -# These characters appear in mojibake but also appear commonly on their own. -# We have a slight preference to leave them alone. -COMMON_SYMBOL_RE = re.compile( - '[' - '\N{HORIZONTAL ELLIPSIS}\N{EM DASH}\N{EN DASH}' - '\N{LEFT SINGLE QUOTATION MARK}\N{LEFT DOUBLE QUOTATION MARK}' - '\N{RIGHT SINGLE QUOTATION MARK}\N{RIGHT DOUBLE QUOTATION MARK}' - '\N{INVERTED EXCLAMATION MARK}\N{INVERTED QUESTION MARK}\N{DEGREE SIGN}' - '\N{TRADE MARK SIGN}' - '\N{REGISTERED SIGN}' - '\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}' - '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' - '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}' - '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' - '\N{NO-BREAK SPACE}' - '\N{ACUTE ACCENT}\N{MULTIPLICATION SIGN}\N{LATIN SMALL LETTER SHARP S}' - '\ufeff' # The byte-order mark, whose encoding '' looks common - ']' -) - -# These are sequences that are common mojibake, resulting from common encodings -# that are mixed up with UTF-8 on characters from their own character map. -# -# Sequences that match this regex will increase the `sequence_weirdness` of a -# string. -# -# This helps to strengthen evidence that text should be fixed in a way that's -# separate from the character classes above, or to counter COMMON_SYMBOL_RE's -# fondness for characters such as inverted exclamation marks and multiplication -# signs in contexts where they really do look like mojibake. - -MOJIBAKE_SYMBOL_RE = re.compile( - # The codepoints of decoded characters these correspond to are as follows: - # - # Initial char. Codepoints What's in this range - # ------------- ---------- -------------------- - #  80..BF Latin-1 control characters and symbols - # Ã, Ă C0..FF Latin-1 accented letters - # Î 380..3BF Greek letters - # Ï, Ď 3C0..3FF Greek letters - # Ð, Đ 400..43F Cyrillic letters - # Ñ, Ń 440..47F Cyrillic letters - # Ø, Ř 600..63F Arabic letters - # Ù, Ů 640..67F Arabic letters - # - # Here, we leave out Hebrew letters (which have a separate heuristic), and - # the rarer letters from each alphabet. This regex doesn't have to contain - # every character we want to decode -- just the sequences that we want to - # specifically detect as 'this looks like mojibake'. - # - # We avoid ranges whose mojibake starts with characters like Ó -- which - # maps to Armenian and some more Cyrillic letters -- because those could be - # the 'eyes' of kaomoji faces. - # - # The set of possible second characters covers most of the possible symbols - # that would be the second byte of UTF-8 mojibake. It is limited to the - # ones that are unlikely to have a real meaning when following one of these - # capital letters, and appear as the second character in Latin-1, - # Windows-1250, or Windows-1252 mojibake. - '[ÂÃÎÏÐÑØÙĂĎĐŃŘŮ][\x80-\x9f€ƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¢£¤¥¦§¨ª«¬¯°±²³µ¶·¸¹º¼½¾¿ˇ˘˝]|' - - # The unlucky mojibake of 'à', which we decode even though it comes from the - # slightly ambiguous sequence of à and a non-breaking space. Fortunately, - # no language appears to have words that end with Ã. - # - # We also recognize it with a normal space, given the context of a preceding - # ASCII lowercase letter, possibly with a space in between - 'Ã\xa0|[a-z] ?à |' - - # Character sequences we have to be a little more cautious about if they're - # at the end of a word, but are totally okay to fix in the middle - r'[ÂÃÎÏÐÑØÙĂĎĐŃŘŮ][›»‘”´©™]\w|' - - # Most Hebrew letters get mojibaked to two-character sequences that start with - # the multiplication sign. In the list of following characters, we exclude - # currency symbols and numbers, which might actually be intended to be - # multiplied. We also exclude characters like ¶ which, although they don't - # make sense after a multiplication sign, wouldn't decode to an existing - # Hebrew letter if they were mojibake. - '×[\x80-\x9fƒ‚„†‡ˆ‰‹Œ“•˜œŸ¡¦§¨ª«¬¯°²³ˇ˘›‘”´©™]|' - - # Similar mojibake of low-numbered characters in MacRoman. Leaving out - # most mathy characters because of false positives, but cautiously catching - # "√±" (mojibake for "ñ") and "√∂" (mojibake for "ö") in the middle of a - # word. - # - # I guess you could almost have "a√±b" in math, except that's not where - # you'd want the ±. Complex numbers don't quite work that way. "√±" appears - # unattested in equations in my Common Crawl sample. - # - # Also left out eye-like letters, including accented o's, for when ¬ is - # the nose of a kaomoji. - '[¬√][ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñúùûü†¢£§¶ß®©™≠ÆØ¥ªæø≤≥]|' - r'\w√[±∂]\w|' - - # MacRoman mojibake of Hebrew involves a diamond character that is - # uncommon in intended text. - '◊|' - - # ISO-8859-1, ISO-8859-2, or Windows-1252 mojibake of characters U+10000 - # to U+1FFFF. (The Windows-1250 and Windows-1251 versions might be too - # plausible.) - '[ðđ][Ÿ\x9f]|' - - # Windows-1252 or Windows-1250 mojibake of Windows punctuation characters - 'â€|' + return ''.join(chr(codept) for codept in range(start, end + 1)) + + +# There are only 403 characters that occur in known UTF-8 mojibake, and we can +# characterize them: + +MOJIBAKE_CATEGORIES = { + # Characters that appear in many different contexts. Sequences that contain + # them are not inherently mojibake + 'common': ( + '\N{NO-BREAK SPACE}' + '\N{SOFT HYPHEN}' + '\N{MIDDLE DOT}' + '\N{ACUTE ACCENT}' + '\N{EN DASH}' + '\N{EM DASH}' + '\N{HORIZONTAL BAR}' + '\N{HORIZONTAL ELLIPSIS}' + '\N{RIGHT SINGLE QUOTATION MARK}' + ), + # Characters that are nearly 100% used in mojibake + 'bad': ( + _char_range(0x80, 0x9f) + + '\N{BROKEN BAR}' + '\N{CURRENCY SIGN}' + '\N{DIAERESIS}' + '\N{NOT SIGN}' + '\N{MACRON}' + '\N{PILCROW SIGN}' + '\N{CEDILLA}' + '\N{LATIN SMALL LETTER F WITH HOOK}' + '\N{MODIFIER LETTER CIRCUMFLEX ACCENT}' # it's not a modifier + '\N{CARON}' + '\N{BREVE}' + '\N{OGONEK}' + '\N{SMALL TILDE}' + '\N{DAGGER}' + '\N{DOUBLE DAGGER}' + '\N{PER MILLE SIGN}' + '\N{REVERSED NOT SIGN}' + '\N{LOZENGE}' + '\ufffd' + + # Theoretically these would appear in 'numeric' contexts, but when they + # co-occur with other mojibake characters, it's not really ambiguous + '\N{FEMININE ORDINAL INDICATOR}' + '\N{MASCULINE ORDINAL INDICATOR}' + '\N{NUMERO SIGN}' + ), + 'currency': ( + '\N{CENT SIGN}' + '\N{POUND SIGN}' + '\N{YEN SIGN}' + '\N{PESETA SIGN}' + '\N{EURO SIGN}' + ), + 'start_punctuation': ( + '\N{INVERTED EXCLAMATION MARK}' + '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}' + '\N{INVERTED QUESTION MARK}' + '\N{COPYRIGHT SIGN}' + '\N{SECTION SIGN}' + '\N{GREEK TONOS}' + '\N{GREEK DIALYTIKA TONOS}' + '\N{LEFT SINGLE QUOTATION MARK}' + '\N{SINGLE LOW-9 QUOTATION MARK}' + '\N{LEFT DOUBLE QUOTATION MARK}' + '\N{DOUBLE LOW-9 QUOTATION MARK}' + '\N{BULLET}' + '\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}' + '\uf8ff' # OS-specific symbol, usually the Apple logo + ), + 'end_punctuation': ( + '\N{REGISTERED SIGN}' + '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' + '\N{DOUBLE ACUTE ACCENT}' + '\N{RIGHT DOUBLE QUOTATION MARK}' + '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' + '\N{TRADE MARK SIGN}' + ), + 'numeric': ( + '\N{SUPERSCRIPT TWO}' + '\N{SUPERSCRIPT THREE}' + '\N{SUPERSCRIPT ONE}' + '\N{PLUS-MINUS SIGN}' + '\N{VULGAR FRACTION ONE QUARTER}' + '\N{VULGAR FRACTION ONE HALF}' + '\N{VULGAR FRACTION THREE QUARTERS}' + '\N{MULTIPLICATION SIGN}' + '\N{MICRO SIGN}' + '\N{DIVISION SIGN}' + '\N{FRACTION SLASH}' + '\N{PARTIAL DIFFERENTIAL}' + '\N{INCREMENT}' + '\N{N-ARY PRODUCT}' + '\N{N-ARY SUMMATION}' + '\N{SQUARE ROOT}' + '\N{INFINITY}' + '\N{INTERSECTION}' + '\N{INTEGRAL}' + '\N{ALMOST EQUAL TO}' + '\N{NOT EQUAL TO}' + '\N{IDENTICAL TO}' + '\N{LESS-THAN OR EQUAL TO}' + '\N{GREATER-THAN OR EQUAL TO}' + ), + 'kaomoji': ( + _char_range(0xd2, 0xd6) + + _char_range(0xd9, 0xdc) + + _char_range(0xf2, 0xf6) + + _char_range(0xf8, 0xfc) + + '\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}' + '\N{DEGREE SIGN}' + ), + 'upper_accented': ( + _char_range(0xc0, 0xd1) + + # skip capital O's and U's that could be used in kaomoji, but + # include Ø because it's very common in Arabic mojibake: + '\N{LATIN CAPITAL LETTER O WITH STROKE}' + '\N{LATIN CAPITAL LETTER U WITH DIAERESIS}' + '\N{LATIN CAPITAL LETTER Y WITH ACUTE}' + '\N{LATIN CAPITAL LETTER A WITH BREVE}' + '\N{LATIN CAPITAL LETTER A WITH OGONEK}' + '\N{LATIN CAPITAL LETTER C WITH ACUTE}' + '\N{LATIN CAPITAL LETTER C WITH CARON}' + '\N{LATIN CAPITAL LETTER D WITH CARON}' + '\N{LATIN CAPITAL LETTER D WITH STROKE}' + '\N{LATIN CAPITAL LETTER E WITH OGONEK}' + '\N{LATIN CAPITAL LETTER E WITH CARON}' + '\N{LATIN CAPITAL LETTER G WITH BREVE}' + '\N{LATIN CAPITAL LETTER I WITH DOT ABOVE}' + '\N{LATIN CAPITAL LETTER L WITH ACUTE}' + '\N{LATIN CAPITAL LETTER L WITH CARON}' + '\N{LATIN CAPITAL LETTER L WITH STROKE}' + '\N{LATIN CAPITAL LETTER N WITH ACUTE}' + '\N{LATIN CAPITAL LETTER N WITH CARON}' + '\N{LATIN CAPITAL LIGATURE OE}' + '\N{LATIN CAPITAL LETTER R WITH CARON}' + '\N{LATIN CAPITAL LETTER S WITH ACUTE}' + '\N{LATIN CAPITAL LETTER S WITH CEDILLA}' + '\N{LATIN CAPITAL LETTER S WITH CARON}' + '\N{LATIN CAPITAL LETTER T WITH CEDILLA}' + '\N{LATIN CAPITAL LETTER T WITH CARON}' + '\N{LATIN CAPITAL LETTER U WITH RING ABOVE}' + '\N{LATIN CAPITAL LETTER U WITH DOUBLE ACUTE}' + '\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}' + '\N{LATIN CAPITAL LETTER Z WITH ACUTE}' + '\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}' + '\N{LATIN CAPITAL LETTER Z WITH CARON}' + '\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}' + ), + 'lower_accented': ( + '\N{LATIN SMALL LETTER SHARP S}' + + _char_range(0xe0, 0xf1) + + # skip o's and u's that could be used in kaomoji + '\N{LATIN SMALL LETTER A WITH BREVE}' + '\N{LATIN SMALL LETTER A WITH OGONEK}' + '\N{LATIN SMALL LETTER C WITH ACUTE}' + '\N{LATIN SMALL LETTER C WITH CARON}' + '\N{LATIN SMALL LETTER D WITH CARON}' + '\N{LATIN SMALL LETTER D WITH STROKE}' + '\N{LATIN SMALL LETTER E WITH OGONEK}' + '\N{LATIN SMALL LETTER E WITH CARON}' + '\N{LATIN SMALL LETTER G WITH BREVE}' + '\N{LATIN SMALL LETTER L WITH ACUTE}' + '\N{LATIN SMALL LETTER L WITH CARON}' + '\N{LATIN SMALL LETTER L WITH STROKE}' + '\N{LATIN SMALL LIGATURE OE}' + '\N{LATIN SMALL LETTER R WITH ACUTE}' + '\N{LATIN SMALL LETTER S WITH ACUTE}' + '\N{LATIN SMALL LETTER S WITH CEDILLA}' + '\N{LATIN SMALL LETTER S WITH CARON}' + '\N{LATIN SMALL LETTER T WITH CARON}' + '\N{LATIN SMALL LETTER Z WITH ACUTE}' + '\N{LATIN SMALL LETTER Z WITH DOT ABOVE}' + '\N{LATIN SMALL LETTER Z WITH CARON}' + '\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}' + '\N{LATIN SMALL LIGATURE FI}' + '\N{LATIN SMALL LIGATURE FL}' + ), + 'upper_common': ( + '\N{LATIN CAPITAL LETTER THORN}' + + _char_range(0x391, 0x3a9) + # Greek capital letters + + # not included under 'accented' because these can commonly + # occur at ends of words, in positions where they'd be detected + # as mojibake + '\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}' + '\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}' + '\N{GREEK CAPITAL LETTER ETA WITH TONOS}' + '\N{GREEK CAPITAL LETTER IOTA WITH TONOS}' + '\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}' + '\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}' + '\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}' + '\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}' + '\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}' + + _char_range(0x401, 0x42f) # Cyrillic capital letters + ), + 'lower_common': ( + # lowercase thorn does not appear in mojibake + _char_range(0x3b1, 0x3c6) + # Greek lowercase letters + '\N{GREEK SMALL LETTER ALPHA WITH TONOS}' + '\N{GREEK SMALL LETTER EPSILON WITH TONOS}' + '\N{GREEK SMALL LETTER ETA WITH TONOS}' + '\N{GREEK SMALL LETTER IOTA WITH TONOS}' + '\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}' + + _char_range(0x430, 0x45f) # Cyrillic lowercase letters + ), + 'box': ( + '─│┌┐┘├┤┬┼' + + _char_range(0x2550, 0x256c) + + '▀▄█▌▐░▒▓' + ), +} + +BADNESS_RE = re.compile(""" + [\x80-\x9f] + | + [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] + | + [{bad}] [{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] + | + [{lower_accented}{lower_common}{box}{end_punctuation}{currency}{numeric}] [{upper_accented}] + | + [{box}{end_punctuation}{currency}{numeric}] [{lower_accented}] + | + [{upper_accented}{lower_accented}{box}{end_punctuation}] [{currency}] + | + [{upper_accented}{box}{end_punctuation}] [{numeric}] + | + [{lower_accented}{upper_accented}{currency}{numeric}{box}] [{end_punctuation}] [{start_punctuation}] + | + [{currency}{numeric}{box}] [{start_punctuation}] + | + [a-z] [{upper_accented}] [{start_punctuation}] + | + [{box}] [{kaomoji}] + | + [{lower_accented}{upper_accented}{currency}{numeric}{start_punctuation}{end_punctuation}] [{box}] + | + [{box}] [{end_punctuation}] + | + [{lower_accented}{upper_accented}] [{end_punctuation}] \\w + | + + # The ligature œ outside of Latin-looking words + [^\\sA-Za-z][Œœ] + | + + # Windows-1252 2-character mojibake that isn't covered by the cases above + [ÂÃÎÐ][€Šš¢Ÿž\xa0\xad®°·»{end_punctuation}–—´] + | + × [{end_punctuation}²³] + | + + # Windows-1252 mojibake sequences for some South Asian alphabets + à[²µ¹¼½¾] + | + + # MacRoman mojibake that isn't covered by the cases above + √[±∂†≠®™´≤≥¥µø] + | + ≈[°¢] + | + ‚Ä[ìîïòôúùû†°¢π] + | + ‚[âó][àä°ê] + | + + # Windows-1251 mojibake of characters in the U+2000 range + †+ | + + # Windows-1251 mojibake of Latin-1 characters and/or the Cyrillic alphabet + [ВГРС][\x80-\x9f‚ѓ„†‡€‰‹‘“”•™›¤¦§©«¬®°±µ¶№»][ВГРС] + | + + # Windows-1252 encodings of 'à' + Ã\xa0 + | + [a-z]\\s?Ã[ ] + | + + # Windows-1253 mojibake of characters in the U+2000 range + β€[™\xa0Ά\xad®°] + | - # Windows-1251 mojibake of some Windows punctuation characters - 'вЂ[љћ¦°№™ќ“”]' -) + # Windows-1253 mojibake of Latin-1 characters and/or the Greek alphabet + [ΒΓΞΟ][\x80-\x96€‚ƒ„†‡‰‹‘“”•™›΅£¤¥¦§¨©ª«¬®―°±²³΄µ¶»½][ΒΓΞΟ] + + # TODO: add longer sequences that detect Windows-1251 and Windows-1253 + # two-character mojibake in groups, because the two characters alone + # aren't enough information +""".format(**MOJIBAKE_CATEGORIES), re.VERBOSE) -def sequence_weirdness(text): + +def badness(text): """ - Determine how often a text has unexpected characters or sequences of - characters. This metric is used to disambiguate when text should be - re-decoded or left as is. - - We start by normalizing text in NFC form, so that penalties for - diacritical marks don't apply to characters that know what to do with - them. - - The following things are deemed weird: - - - Lowercase letters followed by non-ASCII uppercase letters - - Non-Latin characters next to Latin characters - - Un-combined diacritical marks, unless they're stacking on non-alphabetic - characters (in languages that do that kind of thing a lot) or other - marks - - C1 control characters - - Adjacent symbols from any different pair of these categories: - - - Modifier marks - - Letter modifiers - - Non-digit numbers - - Symbols (including math and currency) - - The return value is the number of instances of weirdness. + Get the 'badness' of a sequence of text. A badness greater than 0 indicates + that some of it seems to be mojibake. """ - text2 = unicodedata.normalize('NFC', text) - weirdness = len(WEIRDNESS_RE.findall(chars_to_classes(text2))) - adjustment = len(MOJIBAKE_SYMBOL_RE.findall(text2)) * 2 - len( - COMMON_SYMBOL_RE.findall(text2) - ) - return weirdness * 2 + adjustment + return len(BADNESS_RE.findall(text)) -def text_cost(text): +def is_bad(text): """ - An overall cost function for text. Weirder is worse, but all else being - equal, shorter strings are better. - - The overall cost is measured as the "weirdness" (see - :func:`sequence_weirdness`) plus the length. + Returns true iff the given text looks like it contains mojibake. """ - return sequence_weirdness(text) + len(text) + return bool(BADNESS_RE.search(text)) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 0954232e..2cada9ad 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -9,7 +9,6 @@ import unicodedata import zlib -from .char_classes import CHAR_CLASS_STRING # These are the encodings we will try to fix in ftfy, in the # order that they should be tried. @@ -54,6 +53,16 @@ def _build_regexes(): ENCODING_REGEXES = _build_regexes() +def chars_to_classes(string): + """ + Convert each Unicode character to a letter indicating which of many + classes it's in. + + See build_data.py for where this data comes from and what it means. + """ + return string.translate(CHAR_CLASS_STRING) + + def _build_html_entities(): entities = {} # Create a dictionary based on the built-in HTML5 entity dictionary. @@ -170,16 +179,6 @@ def possible_encoding(text, encoding): return bool(ENCODING_REGEXES[encoding].match(text)) -def chars_to_classes(string): - """ - Convert each Unicode character to a letter indicating which of many - classes it's in. - - See build_data.py for where this data comes from and what it means. - """ - return string.translate(CHAR_CLASS_STRING) - - def _build_control_char_mapping(): """ Build a translate mapping that strips likely-unintended control characters. diff --git a/ftfy/fixes.py b/ftfy/fixes.py index f1cceb95..b54718ab 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -8,7 +8,7 @@ import re import warnings -from ftfy.badness import text_cost +from ftfy.badness import badness, is_bad from ftfy.chardata import ( ALTERED_UTF8_RE, C1_CONTROL_RE, @@ -118,27 +118,6 @@ def fix_encoding(text): return text -def fix_text_encoding(text): - """ - A deprecated name for :func:`ftfy.fixes.fix_encoding`. - """ - warnings.warn('fix_text_encoding is now known as fix_encoding', DeprecationWarning) - return fix_encoding(text) - - -# When we support discovering mojibake in more encodings, we run the risk -# of more false positives. We can mitigate false positives by assigning an -# additional cost to using encodings that are rarer than Windows-1252, so -# that these encodings will only be used if they fix multiple problems. -ENCODING_COSTS = { - 'macroman': 2, - 'iso-8859-2': 2, - 'sloppy-windows-1250': 2, - 'sloppy-windows-1251': 3, - 'cp437': 3, -} - - def fix_encoding_and_explain(text): """ Re-decodes text that has been decoded incorrectly, and also return a @@ -147,24 +126,16 @@ def fix_encoding_and_explain(text): The resulting plan could be used with :func:`ftfy.fixes.apply_plan` to fix additional strings that are broken in the same way. """ - best_version = text - best_cost = text_cost(text) - best_plan = [] + if isinstance(text, bytes): + raise UnicodeError(BYTES_ERROR_TEXT) + plan_so_far = [] while True: prevtext = text text, plan = fix_one_step_and_explain(text) plan_so_far.extend(plan) - cost = text_cost(text) - for _, _, step_cost in plan_so_far: - cost += step_cost - - if cost < best_cost: - best_cost = cost - best_version = text - best_plan = list(plan_so_far) if text == prevtext: - return best_version, best_plan + return text, plan_so_far def fix_one_step_and_explain(text): @@ -173,12 +144,11 @@ def fix_one_step_and_explain(text): Returns the decoded text, plus a "plan" for how to reproduce what it did. """ - if isinstance(text, bytes): - raise UnicodeError(BYTES_ERROR_TEXT) if len(text) == 0: return text, [] - # The first plan is to return ASCII text unchanged. + # The first plan is to return ASCII text unchanged, as well as text + # that doesn't look like it contains mojibake if possible_encoding(text, 'ascii'): return text, [] @@ -192,41 +162,42 @@ def fix_one_step_and_explain(text): # are usually the correct thing to do, so try them next. for encoding in CHARMAP_ENCODINGS: if possible_encoding(text, encoding): - encoded_bytes = text.encode(encoding) - encode_step = ('encode', encoding, ENCODING_COSTS.get(encoding, 0)) - transcode_steps = [] - - # Now, find out if it's UTF-8 (or close enough). Otherwise, - # remember the encoding for later. - try: - decoding = 'utf-8' - # Check encoded_bytes for sequences that would be UTF-8, - # except they have b' ' where b'\xa0' would belong. - if ALTERED_UTF8_RE.search(encoded_bytes): - encoded_bytes = restore_byte_a0(encoded_bytes) - cost = encoded_bytes.count(0xa0) - transcode_steps.append(('transcode', 'restore_byte_a0', cost)) - - # Check for the byte 0x1a, which indicates where one of our - # 'sloppy' codecs found a replacement character. - if encoding.startswith('sloppy') and 0x1a in encoded_bytes: - encoded_bytes = replace_lossy_sequences(encoded_bytes) - transcode_steps.append(('transcode', 'replace_lossy_sequences', 0)) - - if 0xed in encoded_bytes or 0xc0 in encoded_bytes: - decoding = 'utf-8-variants' - - decode_step = ('decode', decoding, 0) - steps = [encode_step] + transcode_steps + [decode_step] - fixed = encoded_bytes.decode(decoding) - return fixed, steps - - except UnicodeDecodeError: - possible_1byte_encodings.append(encoding) + possible_1byte_encodings.append(encoding) + if is_bad(text): + encoded_bytes = text.encode(encoding) + encode_step = ('encode', encoding) + transcode_steps = [] + + # Now, find out if it's UTF-8 (or close enough). Otherwise, + # remember the encoding for later. + try: + decoding = 'utf-8' + # Check encoded_bytes for sequences that would be UTF-8, + # except they have b' ' where b'\xa0' would belong. + if ALTERED_UTF8_RE.search(encoded_bytes): + encoded_bytes = restore_byte_a0(encoded_bytes) + transcode_steps.append(('transcode', 'restore_byte_a0')) + + # Check for the byte 0x1a, which indicates where one of our + # 'sloppy' codecs found a replacement character. + if encoding.startswith('sloppy') and 0x1a in encoded_bytes: + encoded_bytes = replace_lossy_sequences(encoded_bytes) + transcode_steps.append(('transcode', 'replace_lossy_sequences')) + + if 0xed in encoded_bytes or 0xc0 in encoded_bytes: + decoding = 'utf-8-variants' + + decode_step = ('decode', decoding) + steps = [encode_step] + transcode_steps + [decode_step] + fixed = encoded_bytes.decode(decoding) + return fixed, steps + + except UnicodeDecodeError: + pass # Look for a-hat-euro sequences that remain, and fix them in isolation. if PARTIAL_UTF8_PUNCT_RE.search(text): - steps = [('transcode', 'fix_partial_utf8_punct_in_1252', 1)] + steps = [('transcode', 'fix_partial_utf8_punct_in_1252')] fixed = fix_partial_utf8_punct_in_1252(text) return fixed, steps @@ -249,7 +220,7 @@ def fix_one_step_and_explain(text): fixed = encoded.decode('windows-1252') steps = [] if fixed != text: - steps = [('encode', 'latin-1', 0), ('decode', 'windows-1252', 1)] + steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] return fixed, steps except UnicodeDecodeError: # This text contained characters that don't even make sense @@ -271,18 +242,15 @@ def apply_plan(text, plan): """ Apply a plan for fixing the encoding of text. - The plan is a list of tuples of the form (operation, encoding, cost): + The plan is a list of tuples of the form (operation, encoding): - `operation` is 'encode' if it turns a string into bytes, 'decode' if it turns bytes into a string, and 'transcode' if it keeps the type the same. - `encoding` is the name of the encoding to use, such as 'utf-8' or 'latin-1', or the function name in the case of 'transcode'. - - The `cost` does not affect how the plan itself works. It's used by other - users of plans, namely `fix_encoding_and_explain`, which has to decide - *which* plan to use. """ obj = text - for operation, encoding, _ in plan: + for operation, encoding in plan: if operation == 'encode': obj = obj.encode(encoding) elif operation == 'decode': diff --git a/pytest.ini b/pytest.ini index 944fc3ad..8aedda9d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --doctest-modules --ignore=setup.py --ignore=specimens +addopts = --doctest-modules --ignore=setup.py --ignore=specimens --ignore=scripts diff --git a/tests/test_cases.json b/tests/test_cases.json index dfaee008..4b4465e5 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -415,10 +415,9 @@ }, { "label": "Punctuation pile-up should actually be musical notes", - "comment": "Unfortunately, the difference doesn't look convincing to ftfy", "original": "Engkau masih yg terindah, indah di dalam hatiku♫~", "fixed": "Engkau masih yg terindah, indah di dalam hatiku♫~", - "expect": "fail" + "expect": "pass" }, { "label": "Latin-1 / MacRoman mixup in Spanish", @@ -546,10 +545,10 @@ }, { "label": "Negative: Leet line-art", - "comment": "Current false positive: it coincidentally all decodes as UTF-8/CP437 mojibake and becomes 'ôaſaſaſaſa'", + "comment": "The heuristic before v6 loved to 'fix' this and decode it as 'ôaſaſaſaſa'", "original": "├┤a┼┐a┼┐a┼┐a┼┐a", "fixed": "├┤a┼┐a┼┐a┼┐a┼┐a", - "expect": "fail" + "expect": "pass" }, { "label": "Synthetic, negative: Brontë's name does not end with a Korean syllable", @@ -688,10 +687,10 @@ "expect": "pass" }, { - "label": "Synthetic, false negative: mojibake with trademark sign at the end of a word", + "label": "Synthetic: mojibake with trademark sign at the end of a word", "comment": "I recall the correct version of this text from a sign in the movie Amélie. Unfortunately, we can't help her twin Amélie, who makes mojibaked signs.", "original": "OÙ ET QUAND?", "fixed": "OÙ ET QUAND?", - "expect": "fail" + "expect": "pass" } ] diff --git a/tests/test_characters.py b/tests/test_characters.py index 512804ac..42e01710 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -2,29 +2,12 @@ fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, remove_control_chars, fix_surrogates ) -from ftfy.badness import sequence_weirdness +from ftfy.badness import badness import unicodedata import sys from ftfy.char_classes import CHAR_CLASS_STRING -# Most single-character strings which have been misencoded should be restored. -def test_bmp_characters(): - for index in range(0xa0, 0xfffd): - char = chr(index) - # Exclude code points that are not assigned - if unicodedata.category(char) not in ('Co', 'Cn', 'Cs', 'Mc', 'Mn', 'Sk'): - garble = char.encode('utf-8').decode('latin-1') - # Exclude characters whose re-encoding is protected by the - # 'sequence_weirdness' metric - if sequence_weirdness(garble) >= 0: - garble2 = char.encode('utf-8').decode('latin-1').encode('utf-8').decode('latin-1') - for garb in (garble, garble2): - fixed, plan = fix_encoding_and_explain(garb) - assert fixed == char - assert apply_plan(garb, plan) == char - - def test_possible_encoding(): for codept in range(256): char = chr(codept) @@ -47,13 +30,13 @@ def test_control_chars(): def test_emoji_variation_selector(): # The hearts here are explicitly marked as emoji using the variation # selector U+FE0F. This is not weird. - assert sequence_weirdness('❤\ufe0f' * 10) == 0 + assert badness('❤\ufe0f' * 10) == 0 def test_emoji_skintone_selector(): # Dear heuristic, you can't call skin-tone selectors weird anymore. # We welcome Santa Clauses of all colors. - assert sequence_weirdness('🎅🏿🎅🏽🎅🏼🎅🏻') == 0 + assert badness('🎅🏿🎅🏽🎅🏼🎅🏻') == 0 def test_surrogates(): diff --git a/tests/test_futuristic_codepoints.py b/tests/test_futuristic_codepoints.py deleted file mode 100644 index 4c76c4c1..00000000 --- a/tests/test_futuristic_codepoints.py +++ /dev/null @@ -1,61 +0,0 @@ -from ftfy.fixes import fix_encoding -from ftfy.badness import sequence_weirdness -import sys - - -def test_unknown_emoji(): - # The range we accept as emoji has gotten larger. Let's make sure we can - # decode the futuristic emoji U+1F960, which will probably be a picture of - # a fortune cookie in Unicode 10.0: - emoji_text = "\U0001f960 I see emoji in your future" - emojibake = emoji_text.encode('utf-8').decode('windows-1252') - assert fix_encoding(emojibake) == emoji_text - - # We believe enough in the future of this codepoint that we'll even - # recognize it with a mangled byte A0 - emojibake = emojibake.replace('\xa0', ' ') - assert fix_encoding(emojibake) == emoji_text - - # Increment the first byte to get a very similar test case, but a - # codepoint that will definitely not exist anytime soon. In this case, - # we consider the existing text, "ñŸ¥\xa0", to be more probable. - not_emoji = "\U0005f960 I see mojibake in your present".encode('utf-8').decode('windows-1252') - assert fix_encoding(not_emoji) == not_emoji - - -def test_unicode_9(): - # This string is 'bɪg'.upper() in Python 3.6 or later, containing the - # new codepoint U+A7AE LATIN CAPITAL LETTER SMALL CAPITAL I. - assert sequence_weirdness("BꞮG") == 0 - - # That should be less weird than having a definitely-unassigned character - # in the string. - assert sequence_weirdness("B\U00090000G") == 2 - - -def test_unicode_10(): - # This string is the word "thalīṃ" in the Zanabazar Square Script, - # a script added in Unicode 10. These characters are recognized as being - # assigned by Python 3.7, and therefore ftfy should recognize them on - # all versions for consistency. - thalim = "\U00011A1A\U00011A2C\U00011A01\U00011A38" - assert sequence_weirdness(thalim) == 0 - - -def test_unicode_11(): - # Unicode 11 has implemented the mtavruli form of the Georgian script. - # They are analogous to capital letters in that they can be used to - # emphasize text or write a headline. - # - # Python will convert to that form when running .upper() on Georgian text, - # starting in version 3.7.0. We want to recognize the result as reasonable - # text on all versions. - # - # This text is the mtavruli form of "ქართული ენა", meaning "Georgian - # language". - - georgian_mtavruli_text = 'ᲥᲐᲠᲗᲣᲚᲘ ᲔᲜᲐ' - assert sequence_weirdness(georgian_mtavruli_text) == 0 - - mojibake = georgian_mtavruli_text.encode('utf-8').decode('sloppy-windows-1252') - assert fix_encoding(mojibake) == georgian_mtavruli_text From 834c602b10abf2f0f7afc314c818d3a80fdcea01 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 18 Mar 2021 12:11:17 -0400 Subject: [PATCH 452/502] remove obsolete char_classes --- ftfy/char_classes.py | 7 ------- ftfy/chardata.py | 10 ---------- tests/test_characters.py | 4 ---- 3 files changed, 21 deletions(-) delete mode 100644 ftfy/char_classes.py diff --git a/ftfy/char_classes.py b/ftfy/char_classes.py deleted file mode 100644 index 1b275084..00000000 --- a/ftfy/char_classes.py +++ /dev/null @@ -1,7 +0,0 @@ - -# This script is automatically generated by build_data.py. -import base64 -import zlib -_compressed = b'eJzt3Uluq8oCANCt3C1gVhB5GqIn/cmboQy/FMT+Z4+eom/s2E7uOc5NjCmqp8DYl/r338afP+XPxJ88z6O80vzpRVHx0vuiOtDXojLCaYKT9ItUoji/xNcifHyJo4+P/DPPLx/X/KN4upB21PxdSjtqn7yffGy/sPFotmqelD9lHO/v/cLXexfsvU2hXlU9+fp678rQ/WzmqY3nbLnf+1TrXLw3mW4e/2/1z54n3w6SPNjb59tndnn7TNPs8/Mzf0sLl8tb/vb2lhaPYvVY2j373FaEqQO+LTzqQFGZzuc0rW078jDK0NYjDmrmxGNUvJOx7H6kmzVUN1qa1cPf7orK8zSN4yg91Jvy4lH/pI3rsnZ1mVInGMmLnzgOI8+LtSvxVbL+6XwOB4pAWxHeQd5mJW+Kk2VJUj0tC9/n5RrH12H+0nwrgyuNsVr5twlSuaYDg/LsjCTL4iJ8lqZJFC0Hzfrw3VKYXl1ha6kVocP16Rmzcc+8Gmwy2Ue2bNbcUgFnesN1EudoH8im0XTFKYu71IvS7m9TK0VB28TKMlfl7qohqesurRIY9pKoOIMqxdE1r7dK+yrck377UyTSpN6l36bcZrHNT59+VTNJPmitdJDIrvSH5a8yUqc/7mXj8udRuyLY8lj9T9MPy9+WaaH+42td+0EFXPv6b9NpNk7bZ/WaYda6rt4kmITlT9KBPv2y4StR3O8sO6p/+npfh0mQfhNpk99p/bcZ+mg6YRlFHqaxI/1p/U/Tbxp5mv5oHBrsq2vpTyXT9K9x0/WTpnzXcfofH13J6yQH/X9QuZsVkfadtW3yNj/dIDhu/zJIePw/fMxKrtcg6ub4kbXJDQ8OezT9/LpS82m4KgkOZG3506wt+KCkTcWsJ1/sCqNjWlycBhVnC/VuMtNyH3FSPrrRbD3zg4Kkk6NTXWtNPMFLu84D07jRZDZt8ltnfGf172/5GaPzp/YspN2rknofuQZHxXbVbLTXoAOVy3G883S7eh9V/j7w1iQv34EdKD9rmr2tWxouH93+1nTOn4ofzfdKTGk4IH7MmNn/4k69f554i77r7Wm9CxTj5R0KCrvE0972Zzl0dQS7Y+rhAao7MRiEqJa6k6aF42O1U6+uXzy+VueXK+t32jotKEpZXn2KrjNn3oPBJx28Myq3nITfkZ3xO9p9wvLveQueXE/X12Li2/ENTsPGp2/lcjysw43htD1fPdf+6+HH7Vk2cLxpIbKmfDuuKi1uXUex2lGL6hz3t4XlrH1tJo6btKel54rXnrb2LTA+H877qu6Ch4f4zdRHCe3taWs5bsu+N5b5pptcCqw65978ZZXg+niTp0MnMcWvYW+pT7lm+1pVkKY89RuTKvzyx5OhOovZKeX23ZNONpvOMMgeC3vWDnXXv+VzwO973Prp7P5H9zlVo+2Mw/Pk8frt8Olb/Vjavt1uX/yj5bRZvnxeLpdwuVqqYq1y0yx32xfLVYp9+PRPbzJ0hZrVs4Gi5aU6fHhUytpzkOozp6w/H6mWh/tjsX9GG2bH89VuPxTHb+VIXPz6fGs+Ev0sl6Kqguph+q1+lMtVvVUXDT6LLYqlKhNVq8RR8Uo8fou3Wqm1t6+qYup3fFV89QXaqL5M2/w0F23XRVXY7VC02jopm2CpRpsAG7V63EY7beZ5M/5DoVctnj9thZm77LHXXXLeeUQaDxPNPDsWwcG+dCePTu9lDIf8+/X7qhGrL3CsCY7OJ8eS3+rEIePe5s8K7hDpdpgbE/kF7tJfDh5qt6/L7Lhwcyd732w3jny+Vr0VeW++2Fo/6m+Kdm/wsvf37/6+XvWlwKaoxdOk/J5hd/HjY+/389J6m8/z1x+b06Fs9Kns0sWfLkC7fPbv1tuhHe9PQtlWgOJIt3iuuHH+OPJ9+8C+9H+yffVdLpdv0OPsOmzBcKl71vSX6qpTFabYqojj7A6xrnwjf7lkRRLfE3+pLMrhDxW+7zuedfT1+5S+jbY2ONobDm6f7j0r7UIdPJc9fNb7O96/vYzv6cYAAAAAAACc5fMfAAAAAAAAAAAAAAAAAAAAAAAAAIAXtjT1XfbgfACcn491cX7LJdU0pln+jdOL/jzl5L7j1/qZYINJiQPHZmK+VlPIBrMPZ2uhsyxZmh/5oDyYxbb+2+b/Mi9rXS7vX+Hj6+v9pkfWTH3dzIfdv14ntJZGPXt2M4d2+0jTblrt/ZMuF0lkTUau17KKq2pequuqpcqdLCk2bWddjeJj+9u4NdqsnGnh6SzWbWzhzMVt/GNdHsq5hYP5r/ekuBhqNjdlhlYL0uTvpuIPZt9Os2H5m/wm/QndqLz722++wsvtkz5I2AL183E0TU7KO48fL3mVWLVZs2nxJ5kdluqsDcf3tgXLYSUMVdXD5O9gdT+f+td+1SBSPfss57q/XMr49s03v9+eKmt7yrh9zu6/AAAAAAAAAAAAAMCy4ZeaF4Ocivd/AMBf5x8AAAAAAIBf5rZvZnDMI+83t3kXzEpzr77wpeq+evWXbYIbFEbDyJvf6bX5eZzxrUpn7nvIg/T3Mb1X5Z+NYV9vH8Ze3sN1fCPlrfun5tvS4WKUR1GUFn/yLv1X6aJpWmU2ajM6UuQ7z98W1YGWb81Zlnxyb9ljt9bPtoKnyzcwrdcWZbjERf0X/6IouE92We5ycRDZSjrzr41fHvfeY/3xwdpqKP98HJEGu82O/WGqvoXySPXScH8sE9q7YzcO3TU92Y7vu2210LlW7SXr7bgZfxtuMbPdvp1vl6fT30N5ozC77k8fxtMMwbstD29Dd7wf8QHT+xXvyWsZbk9e97dX363OlaJzoGUe40x5mNwffGl1On3ltbS5O3ZOltZj8/J/R8jbMXYhQHm4W1s/rrbBmH28fDOZb+KcT7ZLa2X8reK4eX/eKP9q6mXup8e0Q6cyH+WhKG2ORe1cBOl16xS9ClttMm6Y0XC9Hs3Hx/rxbUd7t0HjtXjqvFbZzEcHyNX0F/UFXVq/0v/7zje7PqjIsKAn+//e+rvZyfT3nn+MDu+dveclo8N+2lbonqLNzzByssB/mY3h50D/K4acPE3vfSRdTTIYJtfDBxdIukHmvvlcTX9lbFiqzNapSViukzl4gn1irpkX95rD09/Mp9nIV9qzDTLKyc70R7PfDN/Ol9PQbNT3IK5ii1vqf5iX8pUyE80ML0Gu8vL12eZY7R/LRZg2+O0XELpE07m0g+Vz89VM5iyaaf8kOElfvgAbzATUXMRMkmsXc/0vqc6Jq2j7zDarmhz0OXkNB+tzWqvDXjiQ5sl1a7y+Peky8X6H+mnH53PFD67LT2q9Py/d09Furv88HM2m9T/sEDelP2zzhfTuWP+TDz+G6VVHtaOXZ1/ODW3/kD3s1uuSYXvNnCb1bR0M/Guf6DY9od0Fq/nows537/EnyOChRuo6bzE2rg0DQaMe7AbXcPRJZuI7eGIzN7BsV+SxNAL3aZ4fr6+PQ43UDY3jJprZ3wbnnVsz7s7N2DlIsB+K71n+/V/i6NKvBoFuX5iWP4x/T/TdwFLHuBAfxyxU9I+tzvG4NXtVpf18t/7Mf2mq7JL5r+ElPWY8WXj7+F3S8aHs2bV8D49pKQAAuN2zz52BH2Py/U4AAADg9T37ggK/wrO7MQAAAAAAAADAX+vZXxzhh9rXf0b3L/v+/za2lenxnSuPFru/OWE5f1YcZ4V8OK1WW97VW5wvJf7d9QP3dOv9WRfmBXjY/zA9uv+vKKNLZm9QvUOTnfJpFrq1fMXolIV3yxzfOvOO5QcAAAAAAJ7M9X/gd9r/+eiJTQAAAAAAAAAA4GU88Us6P8GZ+pytY/X9em7bcwAAfqZHn26NXxmfjIUr4+Y+i0+rHAAAAAD44eK/0uHy31ZR1QwP7d+knSCmnfqhfT3ZjuSGPHxb+5fZekhXPWVucpFw/e3ln7Wa/iQTT3Rm5pbNbdIT28ylU+SuyGFa/ymk4TZpl0yY3Lny1LF30bdPZsox3qYJ39fk6jZn8mabooWX+2m0uFG0keJv2/IznH1sr28a//ghTk5XdW7TLtFRHuJ47nPeel6sURTPOEYC/EbtsDoZq8NV6e8deo9+g6mplG52xuH8ttcXfif8Ox1pvrLhhu2VRk/OPsDTHDr+vargMuvRt4mL11s6i9ddOu05QRnfo+e7hx9l8QORbXHxL7r3ZyeHc/CCH+LcoBo+14bW8qcIlg5eql5s/svGtf51rcP04bvXquUgmXQQyTDa0bBeZzBdW39Q5Ix/4LFXm1/PcPcGAACA32nnFXYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB4q3fTsHAIAAAAv6q+5fvDbywcAAAAAwPP4vh4AAAAAAABAZfsWKAAAwF/i2W9PeGXP7p0AfJdnH2EAAAAA4Id49qU8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAeB35xupv9phCLkv+Ms+ubwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA4HX9AwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP+k6bNzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwCtI0/8AIUUfmw==' -CHAR_CLASS_STRING = zlib.decompress(base64.b64decode(_compressed)).decode('ascii') - diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 2cada9ad..59b6aa6c 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -53,16 +53,6 @@ def _build_regexes(): ENCODING_REGEXES = _build_regexes() -def chars_to_classes(string): - """ - Convert each Unicode character to a letter indicating which of many - classes it's in. - - See build_data.py for where this data comes from and what it means. - """ - return string.translate(CHAR_CLASS_STRING) - - def _build_html_entities(): entities = {} # Create a dictionary based on the built-in HTML5 entity dictionary. diff --git a/tests/test_characters.py b/tests/test_characters.py index 42e01710..1c86f639 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -5,7 +5,6 @@ from ftfy.badness import badness import unicodedata import sys -from ftfy.char_classes import CHAR_CLASS_STRING def test_possible_encoding(): @@ -43,6 +42,3 @@ def test_surrogates(): assert fix_surrogates('\udbff\udfff') == '\U0010ffff' assert fix_surrogates('\ud800\udc00') == '\U00010000' - -def test_char_class_type(): - assert isinstance(CHAR_CLASS_STRING, str) From 1d22ad649720c9d8e23c1f7a6279b864e7f01b68 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 18 Mar 2021 12:56:13 -0400 Subject: [PATCH 453/502] move is_bad check earlier --- ftfy/badness.py | 13 +++++----- ftfy/fixes.py | 63 ++++++++++++++++++++++++------------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 4e8b04d2..37596a1d 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -29,9 +29,10 @@ def _char_range(start, end): '\N{HORIZONTAL ELLIPSIS}' '\N{RIGHT SINGLE QUOTATION MARK}' ), + # the C1 control character range, which have no uses outside of mojibake anymore + 'c1': _char_range(0x80, 0x9f), # Characters that are nearly 100% used in mojibake 'bad': ( - _char_range(0x80, 0x9f) + '\N{BROKEN BAR}' '\N{CURRENCY SIGN}' '\N{DIAERESIS}' @@ -225,7 +226,7 @@ def _char_range(start, end): } BADNESS_RE = re.compile(""" - [\x80-\x9f] + [{c1}] | [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] | @@ -254,8 +255,8 @@ def _char_range(start, end): [{lower_accented}{upper_accented}] [{end_punctuation}] \\w | - # The ligature œ outside of Latin-looking words - [^\\sA-Za-z][Œœ] + # The ligature œ outside of English-looking words + [Œœ][^a-z] | # Windows-1252 2-character mojibake that isn't covered by the cases above @@ -283,7 +284,7 @@ def _char_range(start, end): | # Windows-1251 mojibake of Latin-1 characters and/or the Cyrillic alphabet - [ВГРС][\x80-\x9f‚ѓ„†‡€‰‹‘“”•™›¤¦§©«¬®°±µ¶№»][ВГРС] + [ВГРС][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°µ][ВГРС] | # Windows-1252 encodings of 'à' @@ -297,7 +298,7 @@ def _char_range(start, end): | # Windows-1253 mojibake of Latin-1 characters and/or the Greek alphabet - [ΒΓΞΟ][\x80-\x96€‚ƒ„†‡‰‹‘“”•™›΅£¤¥¦§¨©ª«¬®―°±²³΄µ¶»½][ΒΓΞΟ] + [ΒΓΞΟ][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°][ΒΓΞΟ] # TODO: add longer sequences that detect Windows-1251 and Windows-1253 # two-character mojibake in groups, because the two characters alone diff --git a/ftfy/fixes.py b/ftfy/fixes.py index b54718ab..afb15886 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -149,7 +149,7 @@ def fix_one_step_and_explain(text): # The first plan is to return ASCII text unchanged, as well as text # that doesn't look like it contains mojibake - if possible_encoding(text, 'ascii'): + if possible_encoding(text, 'ascii') or not is_bad(text): return text, [] # As we go through the next step, remember the possible encodings @@ -163,37 +163,36 @@ def fix_one_step_and_explain(text): for encoding in CHARMAP_ENCODINGS: if possible_encoding(text, encoding): possible_1byte_encodings.append(encoding) - if is_bad(text): - encoded_bytes = text.encode(encoding) - encode_step = ('encode', encoding) - transcode_steps = [] - - # Now, find out if it's UTF-8 (or close enough). Otherwise, - # remember the encoding for later. - try: - decoding = 'utf-8' - # Check encoded_bytes for sequences that would be UTF-8, - # except they have b' ' where b'\xa0' would belong. - if ALTERED_UTF8_RE.search(encoded_bytes): - encoded_bytes = restore_byte_a0(encoded_bytes) - transcode_steps.append(('transcode', 'restore_byte_a0')) - - # Check for the byte 0x1a, which indicates where one of our - # 'sloppy' codecs found a replacement character. - if encoding.startswith('sloppy') and 0x1a in encoded_bytes: - encoded_bytes = replace_lossy_sequences(encoded_bytes) - transcode_steps.append(('transcode', 'replace_lossy_sequences')) - - if 0xed in encoded_bytes or 0xc0 in encoded_bytes: - decoding = 'utf-8-variants' - - decode_step = ('decode', decoding) - steps = [encode_step] + transcode_steps + [decode_step] - fixed = encoded_bytes.decode(decoding) - return fixed, steps - - except UnicodeDecodeError: - pass + encoded_bytes = text.encode(encoding) + encode_step = ('encode', encoding) + transcode_steps = [] + + # Now, find out if it's UTF-8 (or close enough). Otherwise, + # remember the encoding for later. + try: + decoding = 'utf-8' + # Check encoded_bytes for sequences that would be UTF-8, + # except they have b' ' where b'\xa0' would belong. + if ALTERED_UTF8_RE.search(encoded_bytes): + encoded_bytes = restore_byte_a0(encoded_bytes) + transcode_steps.append(('transcode', 'restore_byte_a0')) + + # Check for the byte 0x1a, which indicates where one of our + # 'sloppy' codecs found a replacement character. + if encoding.startswith('sloppy') and 0x1a in encoded_bytes: + encoded_bytes = replace_lossy_sequences(encoded_bytes) + transcode_steps.append(('transcode', 'replace_lossy_sequences')) + + if 0xed in encoded_bytes or 0xc0 in encoded_bytes: + decoding = 'utf-8-variants' + + decode_step = ('decode', decoding) + steps = [encode_step] + transcode_steps + [decode_step] + fixed = encoded_bytes.decode(decoding) + return fixed, steps + + except UnicodeDecodeError: + pass # Look for a-hat-euro sequences that remain, and fix them in isolation. if PARTIAL_UTF8_PUNCT_RE.search(text): From f6a175b4121f55c3d4877936bfca675be0451a3b Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 19 Mar 2021 15:56:17 -0400 Subject: [PATCH 454/502] more test cases --- tests/test_cases.json | 125 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/tests/test_cases.json b/tests/test_cases.json index dfaee008..657fd6ee 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -1,4 +1,116 @@ [ + { + "label": "Messy language names: Czech", + "comment": "This and several following examples came from the same language selector", + "original": "ÄŒeÅ¡tina", + "fixed": "Čeština", + "expect": "pass" + }, + { + "label": "Messy language names: Gaelic", + "comment": "note that if U+A0 is replaced by a space, it comes out slightly incorrectly as 'Gà idhlig'", + "original": "GÃ\u00a0idhlig", + "fixed": "Gàidhlig", + "expect": "pass" + }, + { + "label": "Messy language names: Lithuanian", + "original": "Lietuvių", + "fixed": "Lietuvių", + "expect": "pass" + }, + { + "label": "Messy language names: Slovak", + "original": "SlovenÄ�ina", + "fixed": "Sloven�ina", + "expect": "pass" + }, + { + "label": "Messy language names: Vietnamese", + "original": "Tiếng Việt", + "fixed": "Tiếng Việt", + "expect": "pass" + }, + { + "label": "Messy language names: Greek", + "original": "Ελληνικά", + "fixed": "Ελληνικά", + "expect": "pass" + }, + { + "label": "Messy language names: Bulgarian", + "original": "българÑ�ки език", + "fixed": "българ�ки език", + "expect": "pass" + }, + { + "label": "Messy language names: Russian", + "original": "РуÑ�Ñ�кий", + "fixed": "Ру��кий", + "expect": "pass" + }, + { + "label": "Messy language names: Serbian [Cyrillic]", + "original": "CрпÑ�ки [ћирилицом]", + "fixed": "Cрп�ки [ћирилицом]", + "expect": "pass" + }, + { + "label": "Messy language names: Hebrew", + "original": "עברית", + "fixed": "עברית", + "expect": "pass" + }, + { + "label": "Messy language names: Russian", + "original": "РуÑ�Ñ�кий", + "fixed": "Ру��кий", + "expect": "pass" + }, + { + "label": "Messy language names: Hindi", + "comment": "My terminal has difficulty rendering the mostly-fixed text", + "original": "हिनà¥�दी", + "fixed": "\u0939\u093f\u0928\ufffd\u0926\u0940", + "expect": "pass" + }, + { + "label": "Messy language names: Tamil", + "comment": "My terminal has difficulty rendering the mostly-fixed text", + "original": "தமிழà¯�", + "fixed": "\u0ba4\u0bae\u0bbf\u0bb4\ufffd", + "expect": "pass" + }, + { + "label": "Messy language names: Thai", + "original": "ภาษาไทย", + "fixed": "ภาษาไทย", + "expect": "pass" + }, + { + "label": "Messy language names: Simplified Chinese", + "original": "简体ä¸\u00adæ–‡", + "fixed": "简体中文", + "expect": "pass" + }, + { + "label": "Messy language names: Traditional Chinese", + "original": "æ\u00ad£é«”ä¸\u00adæ–‡", + "fixed": "正體中文", + "expect": "pass" + }, + { + "label": "Messy language names: Japanese", + "original": "日本語", + "fixed": "日本語", + "expect": "pass" + }, + { + "label": "Messy language names: Korean", + "original": "í•œêµ\u00adì–´", + "fixed": "한국어", + "expect": "pass" + }, { "label": "Low-codepoint emoji", "original": "He's Justinâ\u009d¤", @@ -105,6 +217,12 @@ "fixed": "Liège Avenue de l'Hôpital", "expect": "pass" }, + { + "label": "lossy UTF-8 / Windows-1250 mixup in Spanish", + "original": "Europa, Asia, Ă�frica, Norte, AmĂ©rica Central y del Sur, Australia y OceanĂ­a", + "fixed": "Europa, Asia, �frica, Norte, América Central y del Sur, Australia y Oceanía", + "expect": "pass" + }, { "label": "UTF-8 / sloppy Windows-1250 mixup in English", "original": "It was named „scars´ stones“ after the rock-climbers who got hurt while climbing on it.", @@ -408,7 +526,6 @@ }, { "label": "UTF-8 / Windows-1252 mixup in Gaelic involving non-breaking spaces", - "comment": "Too similar to the Finnish negative example with non-breaking spaces", "original": "CÃ\u00a0nan nan GÃ\u00a0idheal", "fixed": "Cànan nan Gàidheal", "expect": "pass" @@ -427,6 +544,12 @@ "fixed": "Deja dos heridos hundimiento de barco turístico en Acapulco.", "expect": "fail" }, + { + "label": "subtle UTF-8 / codepage 437 mixup in Spanish", + "original": "┬┐que diferencia hay?", + "fixed": "¿que diferencia hay?", + "expect": "fail" + }, { "label": "Latin-1 / MacRoman mixup in Spanish, 2 characters", "comment": "Requires something like encoding detection", From ee19b5df7ef1cae3d5ef9ea8b6a59d7b2d91ce51 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 22 Mar 2021 13:31:39 -0400 Subject: [PATCH 455/502] work on updating the docs --- docs/conf.py | 13 +++- docs/index.rst | 167 ++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 156 insertions(+), 24 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 31828a20..259a4021 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -94,12 +94,19 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +import guzzle_sphinx_theme +html_theme_path = guzzle_sphinx_theme.html_theme_path() +html_theme = 'guzzle_sphinx_theme' + +extensions.append("guzzle_sphinx_theme") # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { + "project_nav_name": "ftfy", + "project_link": "https://github.com/LuminosoInsight/ftfy", +} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] @@ -109,7 +116,7 @@ #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +html_short_title = "ftfy documentation" # The name of an image file (relative to this directory) to place at the top # of the sidebar. diff --git a/docs/index.rst b/docs/index.rst index 2859fb43..20946b12 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,34 +1,160 @@ -.. ftfy documentation master file, created by - sphinx-quickstart on Wed Aug 28 03:18:27 2013. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - ftfy: fixes text for you ======================== **ftfy** fixes Unicode that's broken in various ways. -The goal of ftfy is to **take in bad Unicode and output good Unicode**, for use -in your Unicode-aware code. This is different from taking in non-Unicode and -outputting Unicode, which is not a goal of ftfy. It also isn't designed to -protect you from having to write Unicode-aware code. ftfy helps those who help -themselves. +The goal of ftfy is to **take in bad Unicode and output good Unicode**, for use in your Unicode-aware code. + +This is different from taking in non-Unicode and outputting Unicode, which is not a goal of ftfy. It also isn't designed to protect you from having to write Unicode-aware code. ftfy helps those who help themselves. -Of course you're better off if your input is decoded properly and has no -glitches. But you often don't have any control over your input; it's someone -else's mistake, but it's your problem now. +Of course you're better off if your input is decoded properly and has no glitches. But you often don't have any control over your input; it's someone else's mistake, but it's your problem now. ftfy will do everything it can to fix the problem. -.. note:: +Some quick examples +------------------- + +Here are some examples (found in the real world) of what ftfy can do: + +ftfy can fix mojibake (encoding mix-ups), by detecting patterns of characters that were clearly meant to be UTF-8 but were decoded as something else: + + >>> import ftfy + >>> ftfy.fix_text('✔ No problems') + '✔ No problems' + +It can fix multiple layers of mojibake simultaneously: + + >>> ftfy.fix_text('The Mona Lisa doesn’t have eyebrows.') + "The Mona Lisa doesn't have eyebrows." + +It can fix mojibake that has had "curly quotes" applied on top of it, which cannot be consistently decoded until the quotes are uncurled: + + >>> ftfy.fix_text("l’humanité") + "l'humanité" + +ftfy can fix mojibake that would have included the character U+A0 (non-breaking space), but the U+A0 was turned into an ASCII space and then combined with another following space: + + >>> ftfy.fix_text('Ã\xa0 perturber la réflexion') + 'à perturber la réflexion' + >>> ftfy.fix_text('à perturber la réflexion') + 'à perturber la réflexion' + +ftfy can also decode HTML entities that appear outside of HTML, even in cases where the entity has been incorrectly capitalized: + + >>> # by the HTML 5 standard, only 'PÉREZ' is acceptable + >>> ftfy.fix_text('P&EACUTE;REZ') + 'PÉREZ' + +These fixes are not applied in all cases, because ftfy has a strongly-held goal of avoiding false positives -- it should never change correctly-decoded text to something else. + +The following text could be encoded in Windows-1252 and decoded in UTF-8, and it would decode as 'MARQUɅ'. However, the original text is already sensible, so it is unchanged. + + >>> ftfy.fix_text('IL Y MARQUÉ…') + 'IL Y MARQUÉ…' + +Citing ftfy +----------- +ftfy has been used as a crucial data processing step in major NLP research. + +It's important to give credit appropriately to everyone whose work you build on +in research. This includes software, not just high-status contributions such as +mathematical models. All I ask when you use ftfy for research is that you cite +it. + +ftfy has a citable record on `Zenodo`_. A citation of ftfy may look like this: + + Robyn Speer. (2019). ftfy (Version 5.5). Zenodo. + http://doi.org/10.5281/zenodo.2591652 + +In BibTeX format, the citation is:: + + @misc{speer-2019-ftfy, + author = {Robyn Speer}, + title = {ftfy}, + note = {Version 5.5}, + year = 2019, + howpublished = {Zenodo}, + doi = {10.5281/zenodo.2591652}, + url = {https://doi.org/10.5281/zenodo.2591652} + } - This documentation is for ftfy 5, which runs on Python 3 only, following - the plan to drop Python 2 support that was announced in ftfy 3.3. +.. _Zenodo: https://zenodo.org/record/2591652 - If you're running on Python 2, ftfy 4.x will keep working for you. In that - case, you should add `ftfy < 5` to your requirements. +Basic usage +----------- + + + +Individual ways to fix text +=========================== + +.. automodule:: ftfy.fixes + :members: unescape_html, remove_terminal_escapes, uncurl_quotes, + fix_latin_ligatures, fix_character_width, fix_line_breaks, + fix_surrogates, remove_control_chars, remove_bom, decode_escapes, + fix_one_step_and_explain, apply_plan, restore_byte_a0, + replace_lossy_sequences, fix_partial_utf8_punct_in_1252 + + +Is ftfy an encoding detector? +============================= + +No. Encoding detectors have ended up being a bad idea, and they are largely responsible for _creating_ the problems that ftfy has to fix. + +The text that you put into ftfy should be Unicode that you've attempted to decode correctly. ftfy doesn't accept bytes as input. + +There is a lot of Unicode out there that has already been mangled by mojibake, even when decoded properly. That is, you might correctly interpret the text as UTF-8, and what the UTF-8 text really says is a mojibake string like "réflexion". This is when you need ftfy. + + +I really need to guess the encoding of some bytes +------------------------------------------------- + +I understand. Sometimes we can't have nice things. + +Though it's not part of the usual operation of ftfy, ftfy _does_ contain a byte-encoding-guesser that tries to be less terrible than other byte-encoding-guessers: + +.. autofunction:: ftfy.guess_bytes +How can I avoid producing mojibake? +=================================== + +- **Assume text is in UTF-8** unless you have a specific reason to believe it isn't. + +In the 2020s, [UTF-8 is everywhere](http://utf8everywhere.org/), especially in text meant to be transferred over the Internet. + +In Python 3, you should use the Unicode string type (`str`) for all operations. You should open text files in UTF-8: + + openfile = open(filename, encoding='utf-8', errors='replace') + +When you are specifically working with bytes and you need to turn them into text, you should decode them as UTF-8: + + text = bytebuffer.decode('utf-8', 'replace') + +The exceptions, the cases where you're not using UTF-8, are few but relevant. If you're interacting with C APIs, you'll need to represent your text as bytes in the format the API expects. Windows APIs in particular expect UTF-16. + +We're mostly past the dark days when encodings were "character maps" of 256 possible characters, one byte per character. An unfortunate thing that keeps them alive is Microsoft Excel, whose "Export" feature will pick a 256-character encoding *based on your computer's operating system and default language*. So: + +- **Don't export CSV files from Excel** if you have any other choice. + +Excel CSVs aren't even interoperable between different computers. Use Google Sheets to create CSVs, and keep Excel files in Excel format so the Unicode won't be mangled. + +If you must export a CSV-like file from Excel, you can find an option to tell Excel to export in "Unicode Text", and it will create a tab-separated UTF-16 file. This is not a very widely-used format, but at least it's not mojibake. + +You can follow `these unwieldy directions from a SalesForce help article ` to use Excel and Notepad to create a UTF-8 CSV. You can see why I don't recommend this process. + +- **Don't use chardet**. + +As described in the previous section, encoding detection is not a good idea. It was important in the '90s, during the rough transition to Unicode -- and it hasn't changed since the '90s. A heuristic designed before there was multilingual social media, before there were emoji, is not going to work correctly in the 2020s. + +chardet was built on the assumption that "encoding detection is language detection", which is no longer true. Web sites now contain text in multiple languages, and for the most part they use UTF-8 regardless of the language. + +When chardet sees the _correct_ UTF-8 encoding of an emoji, it will have no idea what it's looking at, and will guess that it's Turkish encoded in Windows-1254. On other reasonable text, it will guess the very unpopular "iso-8859-2" encoding. Because the bug is inherent to the design of chardet, it's not easily fixed. + + +Fixing mojibake and related problems +==================================== + Mojibake -------- @@ -156,8 +282,7 @@ in research. This includes software, not just high-status contributions such as mathematical models. All I ask when you use ftfy for research is that you cite it. -ftfy has a citable record [on Zenodo](https://zenodo.org/record/2591652). -A citation of ftfy may look like this: +ftfy has a citable record on `Zenodo`_. A citation of ftfy may look like this: Robyn Speer. (2019). ftfy (Version 5.5). Zenodo. http://doi.org/10.5281/zenodo.2591652 @@ -174,7 +299,7 @@ In BibTeX format, the citation is:: url = {https://doi.org/10.5281/zenodo.2591652} } - +.. _Zenodo: https://zenodo.org/record/2591652 Using ftfy From 3a8e86b8bf12da850fc54fb7c4766bf0b93067b4 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 22 Mar 2021 13:31:50 -0400 Subject: [PATCH 456/502] WIP: replace redundant heuristics --- ftfy/__init__.py | 43 ++++++++---- ftfy/badness.py | 38 +++++++++++ ftfy/chardata.py | 138 ++++++++++++++++----------------------- ftfy/fixes.py | 45 ++++++------- tests/test_characters.py | 12 ---- 5 files changed, 143 insertions(+), 133 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 9631bf18..4889e703 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -314,28 +314,21 @@ def guess_bytes(bstring): Unlike the rest of ftfy, this may not be accurate, and it may *create* Unicode problems instead of solving them! - It doesn't try East Asian encodings at all, and if you have East Asian text - that you don't know how to decode, you are somewhat out of luck. East - Asian encodings require some serious statistics to distinguish from each - other, so we can't support them without decreasing the accuracy of ftfy. - - If you don't know which encoding you have at all, I recommend - trying the 'chardet' module, and being appropriately skeptical about its - results. - The encodings we try here are: - UTF-16 with a byte order mark, because a UTF-16 byte order mark looks like nothing else - UTF-8, because it's the global standard, which has been used by a majority of the Web since 2008 - - "utf-8-variants", because it's what people actually implement when they - think they're doing UTF-8 + - "utf-8-variants", or buggy implementations of UTF-8 - MacRoman, because Microsoft Office thinks it's still a thing, and it can be distinguished by its line breaks. (If there are no line breaks in the string, though, you're out of luck.) + - Shift-JIS, the most common encoding of Japanese. + - GB18030, a standardized encoding of Simplified Chinese. + - Big5, a standardized encoding of Traditional Chinese. - "sloppy-windows-1252", the Latin-1-like encoding that is the most common - single-byte encoding + single-byte encoding. """ if isinstance(bstring, str): raise UnicodeError( @@ -375,6 +368,32 @@ def guess_bytes(bstring): except UnicodeDecodeError: pass + # decode GB18030 text that contains either '人' or '日', or the sequence + # that introduces many foreign Unicode characters, U+81 U+30. + # (The U+81 U+30 sequence would appear, for example, in the GB18030 encoding + # of UTF-8 mojibake.) + try: + if b'\xc8\xcb' in bstring or b'\xc8\xd5' in bstring or b'\x81\x30' in bstring: + return bstring.decode('gb18030'), 'gb18030' + except UnicodeDecodeError: + pass + + # decode Shift-JIS text that contains at least two of + # [punctuation, hiragana, katakana], using bytes that are very uncommon outside + # of UTF-8 and GB18030 + try: + if (0x81 in byteset) + (0x82 in byteset) + (0x83 in byteset) >= 2: + return bstring.decode('shift-jis'), 'shift-jis' + except UnicodeDecodeError: + pass + + # decode Big5 text that contains either '人' or '日' + try: + if b'\xa4\x48' in bstring or b'\xa4\xe9' in bstring: + return bstring.decode('big5'), 'big5' + except UnicodeDecodeError: + pass + if 0x0d in byteset and 0x0a not in byteset: # Files that contain CR and not LF are likely to be MacRoman. return bstring.decode('macroman'), 'macroman' diff --git a/ftfy/badness.py b/ftfy/badness.py index 37596a1d..a477bf85 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -225,6 +225,44 @@ def _char_range(start, end): ), } + +# Character classes that help us pinpoint embedded mojibake. These can +# include common characters, because we'll also check them for 'badness'. +UTF8_CLUES = { + # Letters that decode to 0xC2 - 0xDF in a Latin-1-like encoding + 'utf8_first_of_2': ( + 'ÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßĂĆČĎĐĘĚĞİĹŃŇŐŘŞŢŮŰ' + 'ΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ' + ), + # Letters that decode to 0xE0 - 0xEF in a Latin-1-like encoding + 'utf8_first_of_3': ( + 'àáâãäåæçèéêëìíîïăćčďęěĺŕΰαβγδεζηθικλμνξοабвгдежзийклмноп' + ), + # Letters that decode to 0xF0 in a Latin-1-like encoding + # (we don't care about codepoints whose UTF-8 starts with higher + # than F0, because they're unused) + 'utf8_first_of_4': ( + 'ðđğπр' + ), + # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding + 'utf8_continuation': ( + '\x80-\xbf' + 'ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅' + 'ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ' + '–—―‘’‚“”„†‡•…‰‹›€№™' + ' ' + ), +} + +UTF8_DETECTOR_RE = re.compile(""" + [{utf8_first_of_2}] [{utf8_continuation}] + | + [{utf8_first_of_3}] [{utf8_continuation}]{{2}} + | + [{utf8_first_of_4}] [{utf8_continuation}]{{3}} +""".format(**UTF8_CLUES), re.VERBOSE) + + BADNESS_RE = re.compile(""" [{c1}] | diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 59b6aa6c..6ca973de 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -14,14 +14,19 @@ # order that they should be tried. CHARMAP_ENCODINGS = [ 'latin-1', - 'sloppy-windows-1252', 'sloppy-windows-1250', - 'iso-8859-2', 'sloppy-windows-1251', + 'sloppy-windows-1252', + 'sloppy-windows-1253', + 'sloppy-windows-1254', + 'iso-8859-2', 'macroman', 'cp437', ] +SINGLE_QUOTE_RE = re.compile('[\u02bc\u2018-\u201b]') +DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') + def _build_regexes(): """ @@ -77,87 +82,6 @@ def _build_html_entities(): HTML_ENTITIES = _build_html_entities() -def _build_utf8_punct_regex(): - """ - Recognize UTF-8 mojibake that's so blatant that we can fix it even when the - rest of the string doesn't decode as UTF-8 -- namely, UTF-8 sequences for - the 'General Punctuation' characters U+2000 to U+2040, re-encoded in - Windows-1252. - - These are recognizable by the distinctive 'â€' ('\xe2\x80') sequence they - all begin with when decoded as Windows-1252. - """ - # We need to recognize the Latin-1 decodings of bytes 0x80 to 0xbf, which - # are a contiguous range, as well as the different Windows-1252 decodings - # of 0x80 to 0x9f, which are not contiguous at all. (Latin-1 and - # Windows-1252 agree on bytes 0xa0 and up.) - obvious_utf8 = ( - 'â[€\x80][\x80-\xbf' - + bytes(range(0x80, 0xa0)).decode('sloppy-windows-1252') - + ']' - ) - return re.compile(obvious_utf8) - - -PARTIAL_UTF8_PUNCT_RE = _build_utf8_punct_regex() - - -# Recognize UTF-8 sequences that would be valid if it weren't for a b'\xa0' -# that some Windows-1252 program converted to a plain space. -# -# The smaller values are included on a case-by-case basis, because we don't want -# to decode likely input sequences to unlikely characters. These are the ones -# that *do* form likely characters before 0xa0: -# -# 0xc2 -> U+A0 NO-BREAK SPACE -# 0xc3 -> U+E0 LATIN SMALL LETTER A WITH GRAVE -# 0xc5 -> U+160 LATIN CAPITAL LETTER S WITH CARON -# 0xce -> U+3A0 GREEK CAPITAL LETTER PI -# 0xd0 -> U+420 CYRILLIC CAPITAL LETTER ER -# -# These still need to come with a cost, so that they only get converted when -# there's evidence that it fixes other things. Any of these could represent -# characters that legitimately appear surrounded by spaces, particularly U+C5 -# (Å), which is a word in multiple languages! -# -# We should consider checking for b'\x85' being converted to ... in the future. -# I've seen it once, but the text still wasn't recoverable. - -ALTERED_UTF8_RE = re.compile( - b'[\xc2\xc3\xc5\xce\xd0][ ]' - b'|[\xe0-\xef][ ][\x80-\xbf]' - b'|[\xe0-\xef][\x80-\xbf][ ]' - b'|[\xf0-\xf4][ ][\x80-\xbf][\x80-\xbf]' - b'|[\xf0-\xf4][\x80-\xbf][ ][\x80-\xbf]' - b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]' -) - -# This expression matches UTF-8 and CESU-8 sequences where some of the -# continuation bytes have been lost. The byte 0x1a (sometimes written as ^Z) is -# used within ftfy to represent a byte that produced the replacement character -# \ufffd. We don't know which byte it was, but we can at least decode the UTF-8 -# sequence as \ufffd instead of failing to re-decode it at all. -LOSSY_UTF8_RE = re.compile( - b'[\xc2-\xdf][\x1a]' - b'|\xed[\xa0-\xaf][\x1a]\xed[\xb0-\xbf][\x1a\x80-\xbf]' - b'|\xed[\xa0-\xaf][\x1a\x80-\xbf]\xed[\xb0-\xbf][\x1a]' - b'|[\xe0-\xef][\x1a][\x1a\x80-\xbf]' - b'|[\xe0-\xef][\x1a\x80-\xbf][\x1a]' - b'|[\xf0-\xf4][\x1a][\x1a\x80-\xbf][\x1a\x80-\xbf]' - b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a][\x1a\x80-\xbf]' - b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a\x80-\xbf][\x1a]' - b'|\x1a' -) - -# These regexes match various Unicode variations on single and double quotes. -SINGLE_QUOTE_RE = re.compile('[\u02bc\u2018-\u201b]') -DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') - -# This regex matches C1 control characters, which occupy some of the positions -# in the Latin-1 character map that Windows assigns to other characters instead. -C1_CONTROL_RE = re.compile(r'[\x80-\x9f]') - - def possible_encoding(text, encoding): """ Given text and a single-byte encoding, check whether that text could have @@ -196,6 +120,54 @@ def _build_control_char_mapping(): CONTROL_CHARS = _build_control_char_mapping() +# Recognize UTF-8 sequences that would be valid if it weren't for a b'\xa0' +# that some Windows-1252 program converted to a plain space. +# +# The smaller values are included on a case-by-case basis, because we don't want +# to decode likely input sequences to unlikely characters. These are the ones +# that *do* form likely characters before 0xa0: +# +# 0xc2 -> U+A0 NO-BREAK SPACE +# 0xc3 -> U+E0 LATIN SMALL LETTER A WITH GRAVE +# 0xc5 -> U+160 LATIN CAPITAL LETTER S WITH CARON +# 0xce -> U+3A0 GREEK CAPITAL LETTER PI +# 0xd0 -> U+420 CYRILLIC CAPITAL LETTER ER +# +# This is meant to be applied to encodings of text that tests true for `is_bad`. +# Any of these could represent characters that legitimately appear surrounded by +# spaces, particularly U+C5 (Å), which is a word in multiple languages! +# +# We should consider checking for b'\x85' being converted to ... in the future. +# I've seen it once, but the text still wasn't recoverable. + +ALTERED_UTF8_RE = re.compile( + b'[\xc2\xc3\xc5\xce\xd0][ ]' + b'|[\xe0-\xef][ ][\x80-\xbf]' + b'|[\xe0-\xef][\x80-\xbf][ ]' + b'|[\xf0-\xf4][ ][\x80-\xbf][\x80-\xbf]' + b'|[\xf0-\xf4][\x80-\xbf][ ][\x80-\xbf]' + b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]' +) + + +# This expression matches UTF-8 and CESU-8 sequences where some of the +# continuation bytes have been lost. The byte 0x1a (sometimes written as ^Z) is +# used within ftfy to represent a byte that produced the replacement character +# \ufffd. We don't know which byte it was, but we can at least decode the UTF-8 +# sequence as \ufffd instead of failing to re-decode it at all. +LOSSY_UTF8_RE = re.compile( + b'[\xc2-\xdf][\x1a]' + b'|\xed[\xa0-\xaf][\x1a]\xed[\xb0-\xbf][\x1a\x80-\xbf]' + b'|\xed[\xa0-\xaf][\x1a\x80-\xbf]\xed[\xb0-\xbf][\x1a]' + b'|[\xe0-\xef][\x1a][\x1a\x80-\xbf]' + b'|[\xe0-\xef][\x1a\x80-\xbf][\x1a]' + b'|[\xf0-\xf4][\x1a][\x1a\x80-\xbf][\x1a\x80-\xbf]' + b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a][\x1a\x80-\xbf]' + b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a\x80-\xbf][\x1a]' + b'|\x1a' +) + + # A translate mapping that breaks ligatures made of Latin letters. While # ligatures may be important to the representation of other languages, in Latin # letters they tend to represent a copy/paste error. It omits ligatures such diff --git a/ftfy/fixes.py b/ftfy/fixes.py index afb15886..f5e3789d 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -8,10 +8,9 @@ import re import warnings -from ftfy.badness import badness, is_bad +from ftfy.badness import badness, is_bad, UTF8_DETECTOR_RE from ftfy.chardata import ( ALTERED_UTF8_RE, - C1_CONTROL_RE, CHARMAP_ENCODINGS, CONTROL_CHARS, DOUBLE_QUOTE_RE, @@ -19,7 +18,6 @@ HTML_ENTITY_RE, LIGATURES, LOSSY_UTF8_RE, - PARTIAL_UTF8_PUNCT_RE, SINGLE_QUOTE_RE, WIDTH_MAP, possible_encoding, @@ -195,10 +193,11 @@ def fix_one_step_and_explain(text): pass # Look for a-hat-euro sequences that remain, and fix them in isolation. - if PARTIAL_UTF8_PUNCT_RE.search(text): - steps = [('transcode', 'fix_partial_utf8_punct_in_1252')] - fixed = fix_partial_utf8_punct_in_1252(text) - return fixed, steps + if UTF8_DETECTOR_RE.search(text): + steps = [('transcode', 'fix_inconsistent_utf8_mojibake')] + fixed = fix_inconsistent_utf8_mojibake(text) + if fixed != text: + return fixed, steps # The next most likely case is that this is Latin-1 that was intended to # be read as Windows-1252, because those two encodings in particular are @@ -671,34 +670,28 @@ def replace_lossy_sequences(byts): return LOSSY_UTF8_RE.sub('\ufffd'.encode('utf-8'), byts) -def fix_partial_utf8_punct_in_1252(text): +def fix_inconsistent_utf8_mojibake(text): """ - Fix particular characters that seem to be found in the wild encoded in - UTF-8 and decoded in Latin-1 or Windows-1252, even when this fix can't be - consistently applied. - - One form of inconsistency we need to deal with is that some character might - be from the Latin-1 C1 control character set, while others are from the - set of characters that take their place in Windows-1252. So we first replace - those characters, then apply a fix that only works on Windows-1252 characters. + Sometimes, text from one encoding ends up embedded within text from a + different one. This is common enough that we need to be able to fix it. This is used as a transcoder within `fix_encoding`. """ - def latin1_to_w1252(match): - "The function to apply when this regex matches." - return match.group(0).encode('latin-1').decode('sloppy-windows-1252') - - def w1252_to_utf8(match): - "The function to apply when this regex matches." - return match.group(0).encode('sloppy-windows-1252').decode('utf-8') + def fix_embedded_mojibake(match): + substr = match.group(0) + + # Require the match to be shorter, so that this doesn't recurse infinitely + if len(substr) < len(text) and is_bad(substr): + return fix_encoding(substr) + else: + return substr - text = C1_CONTROL_RE.sub(latin1_to_w1252, text) - return PARTIAL_UTF8_PUNCT_RE.sub(w1252_to_utf8, text) + return UTF8_DETECTOR_RE.sub(fix_embedded_mojibake, text) TRANSCODERS = { 'restore_byte_a0': restore_byte_a0, 'replace_lossy_sequences': replace_lossy_sequences, - 'fix_partial_utf8_punct_in_1252': fix_partial_utf8_punct_in_1252, + 'fix_inconsistent_utf8_mojibake': fix_inconsistent_utf8_mojibake } diff --git a/tests/test_characters.py b/tests/test_characters.py index 1c86f639..f00ef7e4 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -26,18 +26,6 @@ def test_control_chars(): assert remove_control_chars(text) == fixed -def test_emoji_variation_selector(): - # The hearts here are explicitly marked as emoji using the variation - # selector U+FE0F. This is not weird. - assert badness('❤\ufe0f' * 10) == 0 - - -def test_emoji_skintone_selector(): - # Dear heuristic, you can't call skin-tone selectors weird anymore. - # We welcome Santa Clauses of all colors. - assert badness('🎅🏿🎅🏽🎅🏼🎅🏻') == 0 - - def test_surrogates(): assert fix_surrogates('\udbff\udfff') == '\U0010ffff' assert fix_surrogates('\ud800\udc00') == '\U00010000' From 290f5361752e0ccb984741d9b9d33caf4edb1723 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 22 Mar 2021 17:46:38 -0400 Subject: [PATCH 457/502] tune the heuristic and reorganize its code --- ftfy/badness.py | 357 +------------------------------------- ftfy/chardata.py | 365 ++++++++++++++++++++++++++++++++++++++- ftfy/fixes.py | 48 +++-- tests/test_cases.json | 4 +- tests/test_characters.py | 9 +- 5 files changed, 409 insertions(+), 374 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index a477bf85..2a75a818 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -3,358 +3,9 @@ """ import re +from .chardata import badness -def _char_range(start, end): - """ - Get a contiguous range of characters as a string. Useful for including - in a regex without making the representation _only_ usable in regexes. - """ - return ''.join(chr(codept) for codept in range(start, end + 1)) - -# There are only 403 characters that occur in known UTF-8 mojibake, and we can -# characterize them: - -MOJIBAKE_CATEGORIES = { - # Characters that appear in many different contexts. Sequences that contain - # them are not inherently mojibake - 'common': ( - '\N{NO-BREAK SPACE}' - '\N{SOFT HYPHEN}' - '\N{MIDDLE DOT}' - '\N{ACUTE ACCENT}' - '\N{EN DASH}' - '\N{EM DASH}' - '\N{HORIZONTAL BAR}' - '\N{HORIZONTAL ELLIPSIS}' - '\N{RIGHT SINGLE QUOTATION MARK}' - ), - # the C1 control character range, which have no uses outside of mojibake anymore - 'c1': _char_range(0x80, 0x9f), - # Characters that are nearly 100% used in mojibake - 'bad': ( - '\N{BROKEN BAR}' - '\N{CURRENCY SIGN}' - '\N{DIAERESIS}' - '\N{NOT SIGN}' - '\N{MACRON}' - '\N{PILCROW SIGN}' - '\N{CEDILLA}' - '\N{LATIN SMALL LETTER F WITH HOOK}' - '\N{MODIFIER LETTER CIRCUMFLEX ACCENT}' # it's not a modifier - '\N{CARON}' - '\N{BREVE}' - '\N{OGONEK}' - '\N{SMALL TILDE}' - '\N{DAGGER}' - '\N{DOUBLE DAGGER}' - '\N{PER MILLE SIGN}' - '\N{REVERSED NOT SIGN}' - '\N{LOZENGE}' - '\ufffd' - - # Theoretically these would appear in 'numeric' contexts, but when they - # co-occur with other mojibake characters, it's not really ambiguous - '\N{FEMININE ORDINAL INDICATOR}' - '\N{MASCULINE ORDINAL INDICATOR}' - '\N{NUMERO SIGN}' - ), - 'currency': ( - '\N{CENT SIGN}' - '\N{POUND SIGN}' - '\N{YEN SIGN}' - '\N{PESETA SIGN}' - '\N{EURO SIGN}' - ), - 'start_punctuation': ( - '\N{INVERTED EXCLAMATION MARK}' - '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}' - '\N{INVERTED QUESTION MARK}' - '\N{COPYRIGHT SIGN}' - '\N{SECTION SIGN}' - '\N{GREEK TONOS}' - '\N{GREEK DIALYTIKA TONOS}' - '\N{LEFT SINGLE QUOTATION MARK}' - '\N{SINGLE LOW-9 QUOTATION MARK}' - '\N{LEFT DOUBLE QUOTATION MARK}' - '\N{DOUBLE LOW-9 QUOTATION MARK}' - '\N{BULLET}' - '\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}' - '\uf8ff' # OS-specific symbol, usually the Apple logo - ), - 'end_punctuation': ( - '\N{REGISTERED SIGN}' - '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' - '\N{DOUBLE ACUTE ACCENT}' - '\N{RIGHT DOUBLE QUOTATION MARK}' - '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' - '\N{TRADE MARK SIGN}' - ), - 'numeric': ( - '\N{SUPERSCRIPT TWO}' - '\N{SUPERSCRIPT THREE}' - '\N{SUPERSCRIPT ONE}' - '\N{PLUS-MINUS SIGN}' - '\N{VULGAR FRACTION ONE QUARTER}' - '\N{VULGAR FRACTION ONE HALF}' - '\N{VULGAR FRACTION THREE QUARTERS}' - '\N{MULTIPLICATION SIGN}' - '\N{MICRO SIGN}' - '\N{DIVISION SIGN}' - '\N{FRACTION SLASH}' - '\N{PARTIAL DIFFERENTIAL}' - '\N{INCREMENT}' - '\N{N-ARY PRODUCT}' - '\N{N-ARY SUMMATION}' - '\N{SQUARE ROOT}' - '\N{INFINITY}' - '\N{INTERSECTION}' - '\N{INTEGRAL}' - '\N{ALMOST EQUAL TO}' - '\N{NOT EQUAL TO}' - '\N{IDENTICAL TO}' - '\N{LESS-THAN OR EQUAL TO}' - '\N{GREATER-THAN OR EQUAL TO}' - ), - 'kaomoji': ( - _char_range(0xd2, 0xd6) - + _char_range(0xd9, 0xdc) - + _char_range(0xf2, 0xf6) - + _char_range(0xf8, 0xfc) + - '\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}' - '\N{DEGREE SIGN}' - ), - 'upper_accented': ( - _char_range(0xc0, 0xd1) + - # skip capital O's and U's that could be used in kaomoji, but - # include Ø because it's very common in Arabic mojibake: - '\N{LATIN CAPITAL LETTER O WITH STROKE}' - '\N{LATIN CAPITAL LETTER U WITH DIAERESIS}' - '\N{LATIN CAPITAL LETTER Y WITH ACUTE}' - '\N{LATIN CAPITAL LETTER A WITH BREVE}' - '\N{LATIN CAPITAL LETTER A WITH OGONEK}' - '\N{LATIN CAPITAL LETTER C WITH ACUTE}' - '\N{LATIN CAPITAL LETTER C WITH CARON}' - '\N{LATIN CAPITAL LETTER D WITH CARON}' - '\N{LATIN CAPITAL LETTER D WITH STROKE}' - '\N{LATIN CAPITAL LETTER E WITH OGONEK}' - '\N{LATIN CAPITAL LETTER E WITH CARON}' - '\N{LATIN CAPITAL LETTER G WITH BREVE}' - '\N{LATIN CAPITAL LETTER I WITH DOT ABOVE}' - '\N{LATIN CAPITAL LETTER L WITH ACUTE}' - '\N{LATIN CAPITAL LETTER L WITH CARON}' - '\N{LATIN CAPITAL LETTER L WITH STROKE}' - '\N{LATIN CAPITAL LETTER N WITH ACUTE}' - '\N{LATIN CAPITAL LETTER N WITH CARON}' - '\N{LATIN CAPITAL LIGATURE OE}' - '\N{LATIN CAPITAL LETTER R WITH CARON}' - '\N{LATIN CAPITAL LETTER S WITH ACUTE}' - '\N{LATIN CAPITAL LETTER S WITH CEDILLA}' - '\N{LATIN CAPITAL LETTER S WITH CARON}' - '\N{LATIN CAPITAL LETTER T WITH CEDILLA}' - '\N{LATIN CAPITAL LETTER T WITH CARON}' - '\N{LATIN CAPITAL LETTER U WITH RING ABOVE}' - '\N{LATIN CAPITAL LETTER U WITH DOUBLE ACUTE}' - '\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}' - '\N{LATIN CAPITAL LETTER Z WITH ACUTE}' - '\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}' - '\N{LATIN CAPITAL LETTER Z WITH CARON}' - '\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}' - ), - 'lower_accented': ( - '\N{LATIN SMALL LETTER SHARP S}' - + _char_range(0xe0, 0xf1) + - # skip o's and u's that could be used in kaomoji - '\N{LATIN SMALL LETTER A WITH BREVE}' - '\N{LATIN SMALL LETTER A WITH OGONEK}' - '\N{LATIN SMALL LETTER C WITH ACUTE}' - '\N{LATIN SMALL LETTER C WITH CARON}' - '\N{LATIN SMALL LETTER D WITH CARON}' - '\N{LATIN SMALL LETTER D WITH STROKE}' - '\N{LATIN SMALL LETTER E WITH OGONEK}' - '\N{LATIN SMALL LETTER E WITH CARON}' - '\N{LATIN SMALL LETTER G WITH BREVE}' - '\N{LATIN SMALL LETTER L WITH ACUTE}' - '\N{LATIN SMALL LETTER L WITH CARON}' - '\N{LATIN SMALL LETTER L WITH STROKE}' - '\N{LATIN SMALL LIGATURE OE}' - '\N{LATIN SMALL LETTER R WITH ACUTE}' - '\N{LATIN SMALL LETTER S WITH ACUTE}' - '\N{LATIN SMALL LETTER S WITH CEDILLA}' - '\N{LATIN SMALL LETTER S WITH CARON}' - '\N{LATIN SMALL LETTER T WITH CARON}' - '\N{LATIN SMALL LETTER Z WITH ACUTE}' - '\N{LATIN SMALL LETTER Z WITH DOT ABOVE}' - '\N{LATIN SMALL LETTER Z WITH CARON}' - '\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}' - '\N{LATIN SMALL LIGATURE FI}' - '\N{LATIN SMALL LIGATURE FL}' - ), - 'upper_common': ( - '\N{LATIN CAPITAL LETTER THORN}' - + _char_range(0x391, 0x3a9) + # Greek capital letters - - # not included under 'accented' because these can commonly - # occur at ends of words, in positions where they'd be detected - # as mojibake - '\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}' - '\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}' - '\N{GREEK CAPITAL LETTER ETA WITH TONOS}' - '\N{GREEK CAPITAL LETTER IOTA WITH TONOS}' - '\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}' - '\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}' - '\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}' - '\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}' - '\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}' - + _char_range(0x401, 0x42f) # Cyrillic capital letters - ), - 'lower_common': ( - # lowercase thorn does not appear in mojibake - _char_range(0x3b1, 0x3c6) + # Greek lowercase letters - '\N{GREEK SMALL LETTER ALPHA WITH TONOS}' - '\N{GREEK SMALL LETTER EPSILON WITH TONOS}' - '\N{GREEK SMALL LETTER ETA WITH TONOS}' - '\N{GREEK SMALL LETTER IOTA WITH TONOS}' - '\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}' - + _char_range(0x430, 0x45f) # Cyrillic lowercase letters - ), - 'box': ( - '─│┌┐┘├┤┬┼' - + _char_range(0x2550, 0x256c) + - '▀▄█▌▐░▒▓' - ), -} - - -# Character classes that help us pinpoint embedded mojibake. These can -# include common characters, because we'll also check them for 'badness'. -UTF8_CLUES = { - # Letters that decode to 0xC2 - 0xDF in a Latin-1-like encoding - 'utf8_first_of_2': ( - 'ÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßĂĆČĎĐĘĚĞİĹŃŇŐŘŞŢŮŰ' - 'ΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ' - ), - # Letters that decode to 0xE0 - 0xEF in a Latin-1-like encoding - 'utf8_first_of_3': ( - 'àáâãäåæçèéêëìíîïăćčďęěĺŕΰαβγδεζηθικλμνξοабвгдежзийклмноп' - ), - # Letters that decode to 0xF0 in a Latin-1-like encoding - # (we don't care about codepoints whose UTF-8 starts with higher - # than F0, because they're unused) - 'utf8_first_of_4': ( - 'ðđğπр' - ), - # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding - 'utf8_continuation': ( - '\x80-\xbf' - 'ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅' - 'ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ' - '–—―‘’‚“”„†‡•…‰‹›€№™' - ' ' - ), -} - -UTF8_DETECTOR_RE = re.compile(""" - [{utf8_first_of_2}] [{utf8_continuation}] - | - [{utf8_first_of_3}] [{utf8_continuation}]{{2}} - | - [{utf8_first_of_4}] [{utf8_continuation}]{{3}} -""".format(**UTF8_CLUES), re.VERBOSE) - - -BADNESS_RE = re.compile(""" - [{c1}] - | - [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] - | - [{bad}] [{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] - | - [{lower_accented}{lower_common}{box}{end_punctuation}{currency}{numeric}] [{upper_accented}] - | - [{box}{end_punctuation}{currency}{numeric}] [{lower_accented}] - | - [{upper_accented}{lower_accented}{box}{end_punctuation}] [{currency}] - | - [{upper_accented}{box}{end_punctuation}] [{numeric}] - | - [{lower_accented}{upper_accented}{currency}{numeric}{box}] [{end_punctuation}] [{start_punctuation}] - | - [{currency}{numeric}{box}] [{start_punctuation}] - | - [a-z] [{upper_accented}] [{start_punctuation}] - | - [{box}] [{kaomoji}] - | - [{lower_accented}{upper_accented}{currency}{numeric}{start_punctuation}{end_punctuation}] [{box}] - | - [{box}] [{end_punctuation}] - | - [{lower_accented}{upper_accented}] [{end_punctuation}] \\w - | - - # The ligature œ outside of English-looking words - [Œœ][^a-z] - | - - # Windows-1252 2-character mojibake that isn't covered by the cases above - [ÂÃÎÐ][€Šš¢Ÿž\xa0\xad®°·»{end_punctuation}–—´] - | - × [{end_punctuation}²³] - | - - # Windows-1252 mojibake sequences for some South Asian alphabets - à[²µ¹¼½¾] - | - - # MacRoman mojibake that isn't covered by the cases above - √[±∂†≠®™´≤≥¥µø] - | - ≈[°¢] - | - ‚Ä[ìîïòôúùû†°¢π] - | - ‚[âó][àä°ê] - | - - # Windows-1251 mojibake of characters in the U+2000 range - †- | - - # Windows-1251 mojibake of Latin-1 characters and/or the Cyrillic alphabet - [ВГРС][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°µ][ВГРС] - | - - # Windows-1252 encodings of 'à' - Ã\xa0 - | - [a-z]\\s?Ã[ ] - | - - # Windows-1253 mojibake of characters in the U+2000 range - β€[™\xa0Ά\xad®°] - | - - # Windows-1253 mojibake of Latin-1 characters and/or the Greek alphabet - [ΒΓΞΟ][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°][ΒΓΞΟ] - - # TODO: add longer sequences that detect Windows-1251 and Windows-1253 - # two-character mojibake in groups, because the two characters alone - # aren't enough information - -""".format(**MOJIBAKE_CATEGORIES), re.VERBOSE) - - -def badness(text): - """ - Get the 'badness' of a sequence of text. A badness greater than 0 indicates - that some of it seems to be mojibake. - """ - return len(BADNESS_RE.findall(text)) - - -def is_bad(text): - """ - Returns true iff the given text looks like it contains mojibake. - """ - return bool(BADNESS_RE.search(text)) +def sequence_weirdness(text): + # TODO: do the deprecation thing + return badness(text) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 6ca973de..9576c887 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -109,8 +109,6 @@ def _build_control_char_mapping(): range(0x206a, 0x2070), [0xfeff], range(0xfff9, 0xfffd), - range(0x1d173, 0x1d17b), - range(0xe0000, 0xe0080), ): control_chars[i] = None @@ -168,6 +166,11 @@ def _build_control_char_mapping(): ) +# This regex matches C1 control characters, which occupy some of the positions +# in the Latin-1 character map that Windows assigns to other characters instead. +C1_CONTROL_RE = re.compile(r'[\x80-\x9f]') + + # A translate mapping that breaks ligatures made of Latin letters. While # ligatures may be important to the representation of other languages, in Latin # letters they tend to represent a copy/paste error. It omits ligatures such @@ -223,3 +226,361 @@ def _build_width_map(): WIDTH_MAP = _build_width_map() + + +def _char_range(start, end): + """ + Get a contiguous range of characters as a string. Useful for including + in a regex without making the representation _only_ usable in regexes. + """ + return ''.join(chr(codept) for codept in range(start, end + 1)) + + +# There are only 403 characters that occur in known UTF-8 mojibake, and we can +# characterize them: + +MOJIBAKE_CATEGORIES = { + # Characters that appear in many different contexts. Sequences that contain + # them are not inherently mojibake + 'common': ( + '\N{NO-BREAK SPACE}' + '\N{SOFT HYPHEN}' + '\N{MIDDLE DOT}' + '\N{ACUTE ACCENT}' + '\N{EN DASH}' + '\N{EM DASH}' + '\N{HORIZONTAL BAR}' + '\N{HORIZONTAL ELLIPSIS}' + '\N{RIGHT SINGLE QUOTATION MARK}' + ), + # the C1 control character range, which have no uses outside of mojibake anymore + 'c1': _char_range(0x80, 0x9f), + # Characters that are nearly 100% used in mojibake + 'bad': ( + '\N{BROKEN BAR}' + '\N{CURRENCY SIGN}' + '\N{DIAERESIS}' + '\N{NOT SIGN}' + '\N{MACRON}' + '\N{PILCROW SIGN}' + '\N{CEDILLA}' + '\N{LATIN SMALL LETTER F WITH HOOK}' + '\N{MODIFIER LETTER CIRCUMFLEX ACCENT}' # it's not a modifier + '\N{CARON}' + '\N{BREVE}' + '\N{OGONEK}' + '\N{SMALL TILDE}' + '\N{DAGGER}' + '\N{DOUBLE DAGGER}' + '\N{PER MILLE SIGN}' + '\N{REVERSED NOT SIGN}' + '\N{LOZENGE}' + '\ufffd' + + # Theoretically these would appear in 'numeric' contexts, but when they + # co-occur with other mojibake characters, it's not really ambiguous + '\N{FEMININE ORDINAL INDICATOR}' + '\N{MASCULINE ORDINAL INDICATOR}' + '\N{NUMERO SIGN}' + ), + 'currency': ( + '\N{CENT SIGN}' + '\N{POUND SIGN}' + '\N{YEN SIGN}' + '\N{PESETA SIGN}' + '\N{EURO SIGN}' + ), + 'start_punctuation': ( + '\N{INVERTED EXCLAMATION MARK}' + '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}' + '\N{INVERTED QUESTION MARK}' + '\N{COPYRIGHT SIGN}' + '\N{SECTION SIGN}' + '\N{GREEK TONOS}' + '\N{GREEK DIALYTIKA TONOS}' + '\N{LEFT SINGLE QUOTATION MARK}' + '\N{SINGLE LOW-9 QUOTATION MARK}' + '\N{LEFT DOUBLE QUOTATION MARK}' + '\N{DOUBLE LOW-9 QUOTATION MARK}' + '\N{BULLET}' + '\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}' + '\uf8ff' # OS-specific symbol, usually the Apple logo + ), + 'end_punctuation': ( + '\N{REGISTERED SIGN}' + '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' + '\N{DOUBLE ACUTE ACCENT}' + '\N{RIGHT DOUBLE QUOTATION MARK}' + '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' + '\N{TRADE MARK SIGN}' + ), + 'numeric': ( + '\N{SUPERSCRIPT TWO}' + '\N{SUPERSCRIPT THREE}' + '\N{SUPERSCRIPT ONE}' + '\N{PLUS-MINUS SIGN}' + '\N{VULGAR FRACTION ONE QUARTER}' + '\N{VULGAR FRACTION ONE HALF}' + '\N{VULGAR FRACTION THREE QUARTERS}' + '\N{MULTIPLICATION SIGN}' + '\N{MICRO SIGN}' + '\N{DIVISION SIGN}' + '\N{FRACTION SLASH}' + '\N{PARTIAL DIFFERENTIAL}' + '\N{INCREMENT}' + '\N{N-ARY PRODUCT}' + '\N{N-ARY SUMMATION}' + '\N{SQUARE ROOT}' + '\N{INFINITY}' + '\N{INTERSECTION}' + '\N{INTEGRAL}' + '\N{ALMOST EQUAL TO}' + '\N{NOT EQUAL TO}' + '\N{IDENTICAL TO}' + '\N{LESS-THAN OR EQUAL TO}' + '\N{GREATER-THAN OR EQUAL TO}' + ), + 'kaomoji': ( + _char_range(0xd2, 0xd6) + + _char_range(0xd9, 0xdc) + + _char_range(0xf2, 0xf6) + + _char_range(0xf8, 0xfc) + + '\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}' + '\N{DEGREE SIGN}' + ), + 'upper_accented': ( + _char_range(0xc0, 0xd1) + + # skip capital O's and U's that could be used in kaomoji, but + # include Ø because it's very common in Arabic mojibake: + '\N{LATIN CAPITAL LETTER O WITH STROKE}' + '\N{LATIN CAPITAL LETTER U WITH DIAERESIS}' + '\N{LATIN CAPITAL LETTER Y WITH ACUTE}' + '\N{LATIN CAPITAL LETTER A WITH BREVE}' + '\N{LATIN CAPITAL LETTER A WITH OGONEK}' + '\N{LATIN CAPITAL LETTER C WITH ACUTE}' + '\N{LATIN CAPITAL LETTER C WITH CARON}' + '\N{LATIN CAPITAL LETTER D WITH CARON}' + '\N{LATIN CAPITAL LETTER D WITH STROKE}' + '\N{LATIN CAPITAL LETTER E WITH OGONEK}' + '\N{LATIN CAPITAL LETTER E WITH CARON}' + '\N{LATIN CAPITAL LETTER G WITH BREVE}' + '\N{LATIN CAPITAL LETTER I WITH DOT ABOVE}' + '\N{LATIN CAPITAL LETTER L WITH ACUTE}' + '\N{LATIN CAPITAL LETTER L WITH CARON}' + '\N{LATIN CAPITAL LETTER L WITH STROKE}' + '\N{LATIN CAPITAL LETTER N WITH ACUTE}' + '\N{LATIN CAPITAL LETTER N WITH CARON}' + '\N{LATIN CAPITAL LIGATURE OE}' + '\N{LATIN CAPITAL LETTER R WITH CARON}' + '\N{LATIN CAPITAL LETTER S WITH ACUTE}' + '\N{LATIN CAPITAL LETTER S WITH CEDILLA}' + '\N{LATIN CAPITAL LETTER S WITH CARON}' + '\N{LATIN CAPITAL LETTER T WITH CEDILLA}' + '\N{LATIN CAPITAL LETTER T WITH CARON}' + '\N{LATIN CAPITAL LETTER U WITH RING ABOVE}' + '\N{LATIN CAPITAL LETTER U WITH DOUBLE ACUTE}' + '\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}' + '\N{LATIN CAPITAL LETTER Z WITH ACUTE}' + '\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}' + '\N{LATIN CAPITAL LETTER Z WITH CARON}' + '\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}' + ), + 'lower_accented': ( + '\N{LATIN SMALL LETTER SHARP S}' + + _char_range(0xe0, 0xf1) + + # skip o's and u's that could be used in kaomoji + '\N{LATIN SMALL LETTER A WITH BREVE}' + '\N{LATIN SMALL LETTER A WITH OGONEK}' + '\N{LATIN SMALL LETTER C WITH ACUTE}' + '\N{LATIN SMALL LETTER C WITH CARON}' + '\N{LATIN SMALL LETTER D WITH CARON}' + '\N{LATIN SMALL LETTER D WITH STROKE}' + '\N{LATIN SMALL LETTER E WITH OGONEK}' + '\N{LATIN SMALL LETTER E WITH CARON}' + '\N{LATIN SMALL LETTER G WITH BREVE}' + '\N{LATIN SMALL LETTER L WITH ACUTE}' + '\N{LATIN SMALL LETTER L WITH CARON}' + '\N{LATIN SMALL LETTER L WITH STROKE}' + '\N{LATIN SMALL LIGATURE OE}' + '\N{LATIN SMALL LETTER R WITH ACUTE}' + '\N{LATIN SMALL LETTER S WITH ACUTE}' + '\N{LATIN SMALL LETTER S WITH CEDILLA}' + '\N{LATIN SMALL LETTER S WITH CARON}' + '\N{LATIN SMALL LETTER T WITH CARON}' + '\N{LATIN SMALL LETTER Z WITH ACUTE}' + '\N{LATIN SMALL LETTER Z WITH DOT ABOVE}' + '\N{LATIN SMALL LETTER Z WITH CARON}' + '\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}' + '\N{LATIN SMALL LIGATURE FI}' + '\N{LATIN SMALL LIGATURE FL}' + ), + 'upper_common': ( + '\N{LATIN CAPITAL LETTER THORN}' + + _char_range(0x391, 0x3a9) + # Greek capital letters + + # not included under 'accented' because these can commonly + # occur at ends of words, in positions where they'd be detected + # as mojibake + '\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}' + '\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}' + '\N{GREEK CAPITAL LETTER ETA WITH TONOS}' + '\N{GREEK CAPITAL LETTER IOTA WITH TONOS}' + '\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}' + '\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}' + '\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}' + '\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}' + '\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}' + + _char_range(0x401, 0x42f) # Cyrillic capital letters + ), + 'lower_common': ( + # lowercase thorn does not appear in mojibake + _char_range(0x3b1, 0x3c6) + # Greek lowercase letters + '\N{GREEK SMALL LETTER ALPHA WITH TONOS}' + '\N{GREEK SMALL LETTER EPSILON WITH TONOS}' + '\N{GREEK SMALL LETTER ETA WITH TONOS}' + '\N{GREEK SMALL LETTER IOTA WITH TONOS}' + '\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}' + + _char_range(0x430, 0x45f) # Cyrillic lowercase letters + ), + 'box': ( + '─│┌┐┘├┤┬┼' + + _char_range(0x2550, 0x256c) + + '▀▄█▌▐░▒▓' + ), +} + + +# Character classes that help us pinpoint embedded mojibake. These can +# include common characters, because we'll also check them for 'badness'. +UTF8_CLUES = { + # Letters that decode to 0xC2 - 0xDF in a Latin-1-like encoding + 'utf8_first_of_2': ( + 'ÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßĂĆČĎĐĘĚĞİĹŃŇŐŘŞŢŮŰ' + 'ΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ' + ), + # Letters that decode to 0xE0 - 0xEF in a Latin-1-like encoding + 'utf8_first_of_3': ( + 'àáâãäåæçèéêëìíîïăćčďęěĺŕΰαβγδεζηθικλμνξοабвгдежзийклмноп' + ), + # Letters that decode to 0xF0 or 0xF3 in a Latin-1-like encoding. + # (Other leading bytes correspond only to unassigned codepoints) + 'utf8_first_of_4': ( + 'ðóđğπσру' + ), + # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding + 'utf8_continuation': ( + '\x80-\xbf' + 'ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅' + 'ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ' + '–—―‘’‚“”„†‡•…‰‹›€№™' + ' ' + ), +} + +# This regex uses UTF8_CLUES to find sequences of likely mojibake. +# It matches them with + so that several adjacent UTF-8-looking sequences +# get coalesced into one, allowing them to be fixed more efficiently +# and not requiring every individual subsequence to be detected as 'badness'. +UTF8_DETECTOR_RE = re.compile(""" + ( + [{utf8_first_of_2}] [{utf8_continuation}] + | + [{utf8_first_of_3}] [{utf8_continuation}]{{2}} + | + [{utf8_first_of_4}] [{utf8_continuation}]{{3}} + )+ +""".format(**UTF8_CLUES), re.VERBOSE) + + +BADNESS_RE = re.compile(""" + [{c1}] + | + [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] + | + [{bad}] [{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] + | + [{lower_accented}{lower_common}{box}{end_punctuation}{currency}{numeric}] [{upper_accented}] + | + [{box}{end_punctuation}{currency}{numeric}] [{lower_accented}] + | + # leave out [upper_accented][currency], because it's used in some fancy + # leetspeak-esque writing + [{lower_accented}{box}{end_punctuation}] [{currency}] + | + [{upper_accented}{box}{end_punctuation}] [{numeric}] + | + [{lower_accented}{upper_accented}{currency}{numeric}{box}] [{end_punctuation}] [{start_punctuation}] + | + [{currency}{numeric}{box}] [{start_punctuation}] + | + [a-z] [{upper_accented}] [{start_punctuation}{currency}] + | + [{box}] [{kaomoji}] + | + [{lower_accented}{upper_accented}{currency}{numeric}{start_punctuation}{end_punctuation}] [{box}] + | + [{box}] [{end_punctuation}] + | + [{lower_accented}{upper_accented}] [{end_punctuation}] \\w + | + + # The ligature œ outside of English-looking words + [Œœ][^a-z] + | + + # Windows-1252 2-character mojibake that isn't covered by the cases above + [ÂÃÎÐ][€Šš¢Ÿž\xa0\xad®©°·»{end_punctuation}–—´] + | + × [{end_punctuation}²³] + | + + # Windows-1252 mojibake sequences for some South Asian alphabets + à[²µ¹¼½¾] + | + + # MacRoman mojibake that isn't covered by the cases above + √[±∂†≠®™´≤≥¥µø] + | + ≈[°¢] + | + ‚Ä[ìîïòôúùû†°¢π] + | + ‚[âó][àä°ê] + | + + # Windows-1251 mojibake of characters in the U+2000 range + †+ | + + # Windows-1251 mojibake of Latin-1 characters and/or the Cyrillic alphabet + [ВГРС][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°µ][ВГРС] + | + + # Windows-1252 encodings of 'à' + Ã\xa0 + | + [a-z]\\s?Ã[ ] + | + + # Windows-1253 mojibake of characters in the U+2000 range + β€[™\xa0Ά\xad®°] + | + + # Windows-1253 mojibake of Latin-1 characters and/or the Greek alphabet + [ΒΓΞΟ][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°][ΒΓΞΟ] +""".format(**MOJIBAKE_CATEGORIES), re.VERBOSE) + + +def badness(text): + """ + Get the 'badness' of a sequence of text. A badness greater than 0 indicates + that some of it seems to be mojibake. + """ + return len(BADNESS_RE.findall(text)) + + +def is_bad(text): + """ + Returns true iff the given text looks like it contains mojibake. + """ + return bool(BADNESS_RE.search(text)) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index f5e3789d..94527cb8 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -6,11 +6,10 @@ import codecs import html import re -import warnings -from ftfy.badness import badness, is_bad, UTF8_DETECTOR_RE from ftfy.chardata import ( ALTERED_UTF8_RE, + C1_CONTROL_RE, CHARMAP_ENCODINGS, CONTROL_CHARS, DOUBLE_QUOTE_RE, @@ -19,7 +18,10 @@ LIGATURES, LOSSY_UTF8_RE, SINGLE_QUOTE_RE, + UTF8_DETECTOR_RE, WIDTH_MAP, + badness, + is_bad, possible_encoding, ) @@ -210,22 +212,21 @@ def fix_one_step_and_explain(text): else: # Otherwise, it means we have characters that are in Latin-1 but # not in Windows-1252. Those are C1 control characters. Nobody - # wants those. Assume they were meant to be Windows-1252. Don't - # use the sloppy codec, because bad Windows-1252 characters are - # a bad sign. - encoded = text.encode('latin-1') + # wants those. Assume they were meant to be Windows-1252. try: - fixed = encoded.decode('windows-1252') - steps = [] + fixed = text.encode('latin-1').decode('windows-1252') if fixed != text: steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] - return fixed, steps + return fixed, steps except UnicodeDecodeError: - # This text contained characters that don't even make sense - # if you assume they were supposed to be Windows-1252. In - # that case, let's not assume anything. pass + # Fix individual characters of Latin-1 with a less satisfying explanation + if C1_CONTROL_RE.search(text): + steps = [('transcode', 'fix_c1_controls')] + fixed = fix_c1_controls(text) + return fixed, steps + # The cases that remain are mixups between two different single-byte # encodings, and not the common case of Latin-1 vs. Windows-1252. # @@ -504,8 +505,6 @@ def remove_control_chars(text): - Interlinear annotation characters (U+FFF9 to U+FFFB) - The Object Replacement Character (U+FFFC) - The byte order mark (U+FEFF) - - Musical notation control characters (U+1D173 to U+1D17A) - - Tag characters (U+E0000 to U+E007F) However, these similar characters are left alone: @@ -516,6 +515,10 @@ def remove_control_chars(text): has happened - Control characters that affect glyph rendering, such as joiners and right-to-left marks (U+200C to U+200F, U+202A to U+202E) + - Musical notation control characters (U+1D173 to U+1D17A) because wow if + you're using those you probably have a good reason + - Tag characters, because they are now used in emoji sequences such as + "Flag of Wales" """ return text.translate(CONTROL_CHARS) @@ -686,12 +689,25 @@ def fix_embedded_mojibake(match): return fix_encoding(substr) else: return substr - + return UTF8_DETECTOR_RE.sub(fix_embedded_mojibake, text) +def _c1_fixer(match): + return match.group(0).encode('latin-1').decode('sloppy-windows-1252') + + +def fix_c1_controls(text): + """ + If text still contains C1 control characters, treat them as their + Windows-1252 equivalents. This matches what Web browsers do. + """ + return C1_CONTROL_RE.sub(_c1_fixer, text) + + TRANSCODERS = { 'restore_byte_a0': restore_byte_a0, 'replace_lossy_sequences': replace_lossy_sequences, - 'fix_inconsistent_utf8_mojibake': fix_inconsistent_utf8_mojibake + 'fix_inconsistent_utf8_mojibake': fix_inconsistent_utf8_mojibake, + 'fix_c1_controls': fix_c1_controls } diff --git a/tests/test_cases.json b/tests/test_cases.json index 4b4465e5..bf07e72c 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -121,9 +121,9 @@ }, { "label": "UTF-8 / Windows-1252 mixup in mixed French and Arabic", - "comment": "'fixed-encoding' is still mojibaked. The thing that makes this test case work is that the full ftfy pipeline can uncurl the quotes, then fix the mojibake", + "comment": "A difficult test case that can depend on the order that steps are applied", "original": "À tous mes frères et soeurs dans la syrienneté comme dans l’humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l’égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", - "fixed-encoding": "À tous mes frères et soeurs dans la syrienneté comme dans l’humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l’égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", + "fixed-encoding": "À tous mes frères et soeurs dans la syrienneté comme dans l’humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l’égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", "fixed": "À tous mes frères et soeurs dans la syrienneté comme dans l'humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l'égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", "expect": "pass" }, diff --git a/tests/test_characters.py b/tests/test_characters.py index f00ef7e4..69c73946 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -20,12 +20,19 @@ def test_byte_order_mark(): def test_control_chars(): text = ( "\ufeffSometimes, \ufffcbad ideas \x7f\ufffalike these characters\ufffb " - "\u206aget standardized\U000E0065\U000E006E.\r\n" + "\u206aget standardized.\r\n" ) fixed = "Sometimes, bad ideas like these characters get standardized.\r\n" assert remove_control_chars(text) == fixed +def test_welsh_flag(): + # ftfy used to remove "tag characters", but they have been repurposed in the + # "Flag of England", "Flag of Scotland", and "Flag of Wales" emoji sequences. + text = "This flag has a dragon on it 🏴󠁧󠁢󠁷󠁬󠁳󠁿" + assert remove_control_chars(text) == text + + def test_surrogates(): assert fix_surrogates('\udbff\udfff') == '\U0010ffff' assert fix_surrogates('\ud800\udc00') == '\U00010000' From 3614446cd400667620fb564b9f294e4bb32b4438 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 23 Mar 2021 14:31:35 -0400 Subject: [PATCH 458/502] more tuning and test cases --- ftfy/chardata.py | 124 ++++++++++++++++++++++-------------------- ftfy/fixes.py | 2 +- tests/test_cases.json | 37 +++++++++++++ 3 files changed, 103 insertions(+), 60 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 9576c887..83bde64a 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -14,9 +14,9 @@ # order that they should be tried. CHARMAP_ENCODINGS = [ 'latin-1', - 'sloppy-windows-1250', - 'sloppy-windows-1251', 'sloppy-windows-1252', + 'sloppy-windows-1251', + 'sloppy-windows-1250', 'sloppy-windows-1253', 'sloppy-windows-1254', 'iso-8859-2', @@ -148,20 +148,24 @@ def _build_control_char_mapping(): ) -# This expression matches UTF-8 and CESU-8 sequences where some of the -# continuation bytes have been lost. The byte 0x1a (sometimes written as ^Z) is +# This expression matches UTF-8 and CESU-8 sequences where some of the +# continuation bytes have been lost. The byte 0x1a (sometimes written as ^Z) is # used within ftfy to represent a byte that produced the replacement character -# \ufffd. We don't know which byte it was, but we can at least decode the UTF-8 -# sequence as \ufffd instead of failing to re-decode it at all. +# \ufffd. We don't know which byte it was, but we can at least decode the UTF-8 +# sequence as \ufffd instead of failing to re-decode it at all. +# +# In some cases, we allow the ASCII '?' in place of \ufffd, but at most once per +# sequence. LOSSY_UTF8_RE = re.compile( - b'[\xc2-\xdf][\x1a]' - b'|\xed[\xa0-\xaf][\x1a]\xed[\xb0-\xbf][\x1a\x80-\xbf]' - b'|\xed[\xa0-\xaf][\x1a\x80-\xbf]\xed[\xb0-\xbf][\x1a]' - b'|[\xe0-\xef][\x1a][\x1a\x80-\xbf]' - b'|[\xe0-\xef][\x1a\x80-\xbf][\x1a]' - b'|[\xf0-\xf4][\x1a][\x1a\x80-\xbf][\x1a\x80-\xbf]' - b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a][\x1a\x80-\xbf]' - b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a\x80-\xbf][\x1a]' + b'[\xc2-\xdf][\x1a]' + b'|[\xc2-\xc3][?]' + b'|\xed[\xa0-\xaf][\x1a?]\xed[\xb0-\xbf][\x1a?\x80-\xbf]' + b'|\xed[\xa0-\xaf][\x1a?\x80-\xbf]\xed[\xb0-\xbf][\x1a?]' + b'|[\xe0-\xef][\x1a?][\x1a\x80-\xbf]' + b'|[\xe0-\xef][\x1a\x80-\xbf][\x1a?]' + b'|[\xf0-\xf4][\x1a?][\x1a\x80-\xbf][\x1a\x80-\xbf]' + b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a?][\x1a\x80-\xbf]' + b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a\x80-\xbf][\x1a?]' b'|\x1a' ) @@ -450,48 +454,6 @@ def _char_range(start, end): } -# Character classes that help us pinpoint embedded mojibake. These can -# include common characters, because we'll also check them for 'badness'. -UTF8_CLUES = { - # Letters that decode to 0xC2 - 0xDF in a Latin-1-like encoding - 'utf8_first_of_2': ( - 'ÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßĂĆČĎĐĘĚĞİĹŃŇŐŘŞŢŮŰ' - 'ΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ' - ), - # Letters that decode to 0xE0 - 0xEF in a Latin-1-like encoding - 'utf8_first_of_3': ( - 'àáâãäåæçèéêëìíîïăćčďęěĺŕΰαβγδεζηθικλμνξοабвгдежзийклмноп' - ), - # Letters that decode to 0xF0 or 0xF3 in a Latin-1-like encoding. - # (Other leading bytes correspond only to unassigned codepoints) - 'utf8_first_of_4': ( - 'ðóđğπσру' - ), - # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding - 'utf8_continuation': ( - '\x80-\xbf' - 'ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅' - 'ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ' - '–—―‘’‚“”„†‡•…‰‹›€№™' - ' ' - ), -} - -# This regex uses UTF8_CLUES to find sequences of likely mojibake. -# It matches them with + so that several adjacent UTF-8-looking sequences -# get coalesced into one, allowing them to be fixed more efficiently -# and not requiring every individual subsequence to be detected as 'badness'. -UTF8_DETECTOR_RE = re.compile(""" - ( - [{utf8_first_of_2}] [{utf8_continuation}] - | - [{utf8_first_of_3}] [{utf8_continuation}]{{2}} - | - [{utf8_first_of_4}] [{utf8_continuation}]{{3}} - )+ -""".format(**UTF8_CLUES), re.VERBOSE) - - BADNESS_RE = re.compile(""" [{c1}] | @@ -503,10 +465,12 @@ def _char_range(start, end): | [{box}{end_punctuation}{currency}{numeric}] [{lower_accented}] | - # leave out [upper_accented][currency], because it's used in some fancy - # leetspeak-esque writing + # leave out [upper_accented][currency] without further info, because it's used in some + # fancy leetspeak-esque writing [{lower_accented}{box}{end_punctuation}] [{currency}] | + \s [{upper_accented}] [{currency}] + | [{upper_accented}{box}{end_punctuation}] [{numeric}] | [{lower_accented}{upper_accented}{currency}{numeric}{box}] [{end_punctuation}] [{start_punctuation}] @@ -531,7 +495,7 @@ def _char_range(start, end): # Windows-1252 2-character mojibake that isn't covered by the cases above [ÂÃÎÐ][€Šš¢Ÿž\xa0\xad®©°·»{end_punctuation}–—´] | - × [{end_punctuation}²³] + × [²³] | # Windows-1252 mojibake sequences for some South Asian alphabets @@ -584,3 +548,45 @@ def is_bad(text): Returns true iff the given text looks like it contains mojibake. """ return bool(BADNESS_RE.search(text)) + + +# Character classes that help us pinpoint embedded mojibake. These can +# include common characters, because we'll also check them for 'badness'. +UTF8_CLUES = { + # Letters that decode to 0xC2 - 0xDF in a Latin-1-like encoding + 'utf8_first_of_2': ( + 'ÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßĂĆČĎĐĘĚĞİĹŃŇŐŘŞŢŮŰ' + 'ΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ' + ), + # Letters that decode to 0xE0 - 0xEF in a Latin-1-like encoding + 'utf8_first_of_3': ( + 'àáâãäåæçèéêëìíîïăćčďęěĺŕΰαβγδεζηθικλμνξοабвгдежзийклмноп' + ), + # Letters that decode to 0xF0 or 0xF3 in a Latin-1-like encoding. + # (Other leading bytes correspond only to unassigned codepoints) + 'utf8_first_of_4': ( + 'ðóđğπσру' + ), + # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding + 'utf8_continuation': ( + '\x80-\xbf' + 'ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅' + 'ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ' + '–—―‘’‚“”„†‡•…‰‹›€№™' + ' ' + ), +} + +# This regex uses UTF8_CLUES to find sequences of likely mojibake. +# It matches them with + so that several adjacent UTF-8-looking sequences +# get coalesced into one, allowing them to be fixed more efficiently +# and not requiring every individual subsequence to be detected as 'badness'. +UTF8_DETECTOR_RE = re.compile(""" + ( + [{utf8_first_of_2}] [{utf8_continuation}] + | + [{utf8_first_of_3}] [{utf8_continuation}]{{2}} + | + [{utf8_first_of_4}] [{utf8_continuation}]{{3}} + )+ +""".format(**UTF8_CLUES), re.VERBOSE) \ No newline at end of file diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 94527cb8..484a6539 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -179,7 +179,7 @@ def fix_one_step_and_explain(text): # Check for the byte 0x1a, which indicates where one of our # 'sloppy' codecs found a replacement character. - if encoding.startswith('sloppy') and 0x1a in encoded_bytes: + if encoding.startswith('sloppy'): encoded_bytes = replace_lossy_sequences(encoded_bytes) transcode_steps.append(('transcode', 'replace_lossy_sequences')) diff --git a/tests/test_cases.json b/tests/test_cases.json index bf07e72c..8f17cc05 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -105,6 +105,30 @@ "fixed": "Liège Avenue de l'Hôpital", "expect": "pass" }, + { + "label": "UTF-8 / Windows-1252 mixup in Vietnamese", + "original": "Tại sao giá hạt sầu riêng lại lên giá?", + "fixed": "Tại sao giá hạt sầu riêng lại lên giá?", + "expect": "pass" + }, + { + "label": "Science! Mid-word Greek letter gets fixed correctly", + "original": "Humanized HLA-DR4.RagKO.IL2RγcKO.NOD (DRAG) mice sustain the complex vertebrate life cycle of Plasmodium falciparum malaria.", + "fixed": "Humanized HLA-DR4.RagKO.IL2RγcKO.NOD (DRAG) mice sustain the complex vertebrate life cycle of Plasmodium falciparum malaria.", + "expect": "pass" + }, + { + "label": "Negative: More science! Don't fix a multiplication symbol in quotes", + "original": "higher values (“+” and “×” curves) in the superficial region", + "fixed": "higher values (“+” and “×” curves) in the superficial region", + "expect": "pass" + }, + { + "label": "For goodness' sake. We can come close to fixing this, but fail in the last step", + "original": "ItÃ?¢â?¬â?¢s classic. ItÃ?¢â?¬â?¢s epic. ItÃ?¢â?¬â?¢s ELIZABETH BENNET for goodnessÃ?¢â?¬â?¢ sake!", + "fixed": "It�¢��s classic. It�¢��s epic. It�¢��s ELIZABETH BENNET for goodness�¢�� sake!", + "expect": "pass" + }, { "label": "UTF-8 / sloppy Windows-1250 mixup in English", "original": "It was named „scars´ stones“ after the rock-climbers who got hurt while climbing on it.", @@ -139,6 +163,12 @@ "fixed": "Napíšte nám !", "expect": "pass" }, + { + "label": "UTF-8 / Windows-1252 followed by UTF-8 / Windows-1251", + "original": "a bigger-than-expected £5.8bn rights issue to satisfy the new banking regulator", + "fixed": "a bigger-than-expected £5.8bn rights issue to satisfy the new banking regulator", + "expect": "pass" + }, { "label": "A face with UTF-8 / sloppy Windows-1252 mixed up twice", "original": "ââ€\u009d’(⌣˛⌣)ââ€\u009dŽ", @@ -151,6 +181,13 @@ "fixed": "�(⌣˛⌣)�", "expect": "pass" }, + { + "label": "Lossy decoding can have plain ASCII question marks, as well", + "original": "The ICR has been upgraded to “bb+â€? from “bbâ€?", + "fixed-encoding": "The ICR has been upgraded to “bb+� from “bb�", + "fixed": "The ICR has been upgraded to \"bb+� from \"bb�", + "expect": "pass" + }, { "label": "CESU-8 / Latin-1 mixup over several emoji", "comment": "You tried", From 607448d72af8f068b261bdbe0a046bd7f862a2b3 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 23 Mar 2021 17:24:59 -0400 Subject: [PATCH 459/502] more test cases --- ftfy/chardata.py | 14 +++++++++----- tests/test_cases.json | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 83bde64a..759f71bc 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -131,6 +131,10 @@ def _build_control_char_mapping(): # 0xce -> U+3A0 GREEK CAPITAL LETTER PI # 0xd0 -> U+420 CYRILLIC CAPITAL LETTER ER # +# In four-character sequences, the lead byte must be F0, because that accounts +# for almost all of the usage of high-numbered codepoints (tag characters whose +# UTF-8 starts with the byte F3 are only used in some rare new emoji sequences). +# # This is meant to be applied to encodings of text that tests true for `is_bad`. # Any of these could represent characters that legitimately appear surrounded by # spaces, particularly U+C5 (Å), which is a word in multiple languages! @@ -142,9 +146,9 @@ def _build_control_char_mapping(): b'[\xc2\xc3\xc5\xce\xd0][ ]' b'|[\xe0-\xef][ ][\x80-\xbf]' b'|[\xe0-\xef][\x80-\xbf][ ]' - b'|[\xf0-\xf4][ ][\x80-\xbf][\x80-\xbf]' - b'|[\xf0-\xf4][\x80-\xbf][ ][\x80-\xbf]' - b'|[\xf0-\xf4][\x80-\xbf][\x80-\xbf][ ]' + b'|[\xf0][ ][\x80-\xbf][\x80-\xbf]' + b'|[\xf0][\x80-\xbf][ ][\x80-\xbf]' + b'|[\xf0][\x80-\xbf][\x80-\xbf][ ]' ) @@ -520,8 +524,8 @@ def _char_range(start, end): [ВГРС][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°µ][ВГРС] | - # Windows-1252 encodings of 'à' - Ã\xa0 + # Windows-1252 encodings of 'à' and 'á' + Ã[\xa0¡] | [a-z]\\s?Ã[ ] | diff --git a/tests/test_cases.json b/tests/test_cases.json index 8f17cc05..3540faa3 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -169,6 +169,12 @@ "fixed": "a bigger-than-expected £5.8bn rights issue to satisfy the new banking regulator", "expect": "pass" }, + { + "label": "fancy Unicode crossing-out, but mojibaked", + "original": "hotel $49 $̶6̶3̶ updated 2018", + "fixed": "hotel $49 $̶6̶3̶ updated 2018", + "expect": "pass" + }, { "label": "A face with UTF-8 / sloppy Windows-1252 mixed up twice", "original": "ââ€\u009d’(⌣˛⌣)ââ€\u009dŽ", @@ -450,6 +456,20 @@ "fixed": "Cànan nan Gàidheal", "expect": "pass" }, + { + "label": "Misleading mix-up in Spanish", + "comment": "The original text has mojibake, but the sequence 'á \u0093' can decode as U+1813 MONGOLIAN DIGIT THREE, when the whole string should really just decode as a Latin-1/Windows-1252 mixup", + "original": "tiene demora y está \u0093próximo a resolverse\u0094", + "fixed": "tiene demora y está \"próximo a resolverse\"", + "expect": "fail" + }, + { + "label": "A-with-grave in Vietnamese", + "comment": "Currently adds extra spaces that shouldn't be there", + "original": "Xem clip hĂ i, phim hĂ i má»›i hay nhất", + "fixed": "Xem clip hài, phim hài mới hay nhất", + "expect": "fail" + }, { "label": "Punctuation pile-up should actually be musical notes", "original": "Engkau masih yg terindah, indah di dalam hatiku♫~", From d4917d5377b66ebf84b12fd78f2ea06f9de21da8 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 23 Mar 2021 17:47:23 -0400 Subject: [PATCH 460/502] More tuning --- ftfy/chardata.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 759f71bc..b0e6120f 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -131,6 +131,18 @@ def _build_control_char_mapping(): # 0xce -> U+3A0 GREEK CAPITAL LETTER PI # 0xd0 -> U+420 CYRILLIC CAPITAL LETTER ER # +# In three-character sequences, we exclude some lead bytes in some cases. +# +# When the lead byte is immediately followed by 0xA0, we shouldn't accept +# a space there, because it leads to some less-likely character ranges: +# +# 0xe0 -> Samaritan script +# 0xe1 -> Mongolian script (corresponds to Latin-1 'á' which is too common) +# 0xea -> Syloti Nagri or North Indic +# 0xed -> UTF-16 surrogates +# 0xee -> unassigned +# 0xef -> unassigned +# # In four-character sequences, the lead byte must be F0, because that accounts # for almost all of the usage of high-numbered codepoints (tag characters whose # UTF-8 starts with the byte F3 are only used in some rare new emoji sequences). @@ -144,7 +156,7 @@ def _build_control_char_mapping(): ALTERED_UTF8_RE = re.compile( b'[\xc2\xc3\xc5\xce\xd0][ ]' - b'|[\xe0-\xef][ ][\x80-\xbf]' + b'|[\xe2-\xe9\xeb\xec][ ][\x80-\xbf]' b'|[\xe0-\xef][\x80-\xbf][ ]' b'|[\xf0][ ][\x80-\xbf][\x80-\xbf]' b'|[\xf0][\x80-\xbf][ ][\x80-\xbf]' @@ -473,7 +485,7 @@ def _char_range(start, end): # fancy leetspeak-esque writing [{lower_accented}{box}{end_punctuation}] [{currency}] | - \s [{upper_accented}] [{currency}] + \\s [{upper_accented}] [{currency}] | [{upper_accented}{box}{end_punctuation}] [{numeric}] | From e0f0b507db59e16df8f6a1b5b22bca323e811bba Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 24 Mar 2021 11:56:38 -0400 Subject: [PATCH 461/502] =?UTF-8?q?fix=20heuristic=20for=20=C5=92=20in=20c?= =?UTF-8?q?apitalized=20words?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ftfy/chardata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index b0e6120f..76e4c3ff 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -504,8 +504,8 @@ def _char_range(start, end): [{lower_accented}{upper_accented}] [{end_punctuation}] \\w | - # The ligature œ outside of English-looking words - [Œœ][^a-z] + # The ligature œ when not followed by an unaccented Latin letter + [Œœ][^A-Za-z] | # Windows-1252 2-character mojibake that isn't covered by the cases above From 02c1542d2527a5834e3c2feb3ee8d3bf04897001 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 24 Mar 2021 12:09:27 -0400 Subject: [PATCH 462/502] fix interaction between restore_byte_a0 and decode_inconsistent_utf8 --- ftfy/chardata.py | 4 +++- ftfy/fixes.py | 8 ++++---- tests/test_cases.json | 7 +++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 76e4c3ff..27b6b9d9 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -541,6 +541,8 @@ def _char_range(start, end): | [a-z]\\s?Ã[ ] | + ^Ã[ ] + | # Windows-1253 mojibake of characters in the U+2000 range β€[™\xa0Ά\xad®°] @@ -585,7 +587,7 @@ def is_bad(text): ), # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding 'utf8_continuation': ( - '\x80-\xbf' + '\x80-\xbf ' 'ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅' 'ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ' '–—―‘’‚“”„†‡•…‰‹›€№™' diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 484a6539..06bdfb5a 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -196,8 +196,8 @@ def fix_one_step_and_explain(text): # Look for a-hat-euro sequences that remain, and fix them in isolation. if UTF8_DETECTOR_RE.search(text): - steps = [('transcode', 'fix_inconsistent_utf8_mojibake')] - fixed = fix_inconsistent_utf8_mojibake(text) + steps = [('transcode', 'decode_inconsistent_utf8')] + fixed = decode_inconsistent_utf8(text) if fixed != text: return fixed, steps @@ -673,7 +673,7 @@ def replace_lossy_sequences(byts): return LOSSY_UTF8_RE.sub('\ufffd'.encode('utf-8'), byts) -def fix_inconsistent_utf8_mojibake(text): +def decode_inconsistent_utf8(text): """ Sometimes, text from one encoding ends up embedded within text from a different one. This is common enough that we need to be able to fix it. @@ -708,6 +708,6 @@ def fix_c1_controls(text): TRANSCODERS = { 'restore_byte_a0': restore_byte_a0, 'replace_lossy_sequences': replace_lossy_sequences, - 'fix_inconsistent_utf8_mojibake': fix_inconsistent_utf8_mojibake, + 'decode_inconsistent_utf8': decode_inconsistent_utf8, 'fix_c1_controls': fix_c1_controls } diff --git a/tests/test_cases.json b/tests/test_cases.json index 849f4186..884bdbac 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -537,6 +537,13 @@ "fixed": "à perturber la réflexion des théologiens jusqu'à nos jours", "expect": "pass" }, + { + "label": "Fix 'à' in inconsistent mojibake", + "original": "Le barème forfaitaire permet l’évaluation des frais de déplacement relatifs à l’utilisation", + "fixed-encoding": "Le barème forfaitaire permet l’évaluation des frais de déplacement relatifs à l’utilisation", + "fixed": "Le barème forfaitaire permet l'évaluation des frais de déplacement relatifs à l'utilisation", + "expect": "pass" + }, { "label": "The Portuguese word 'às' does not become 'à s' due to the French fix", "original": "com especial atenção à s crianças", From 74580d633e80083d508a965caa90cf97310878e5 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 24 Mar 2021 13:20:42 -0400 Subject: [PATCH 463/502] tune the heuristic --- ftfy/chardata.py | 36 ++++++++++++++++++++++++++++++------ tests/test_cases.json | 16 +++++++++++++++- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 27b6b9d9..a6e5e19c 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -283,6 +283,7 @@ def _char_range(start, end): '\N{NOT SIGN}' '\N{MACRON}' '\N{PILCROW SIGN}' + '\N{SECTION SIGN}' '\N{CEDILLA}' '\N{LATIN SMALL LETTER F WITH HOOK}' '\N{MODIFIER LETTER CIRCUMFLEX ACCENT}' # it's not a modifier @@ -315,7 +316,6 @@ def _char_range(start, end): '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}' '\N{INVERTED QUESTION MARK}' '\N{COPYRIGHT SIGN}' - '\N{SECTION SIGN}' '\N{GREEK TONOS}' '\N{GREEK DIALYTIKA TONOS}' '\N{LEFT SINGLE QUOTATION MARK}' @@ -508,8 +508,8 @@ def _char_range(start, end): [Œœ][^A-Za-z] | - # Windows-1252 2-character mojibake that isn't covered by the cases above - [ÂÃÎÐ][€Šš¢Ÿž\xa0\xad®©°·»{end_punctuation}–—´] + # Common Windows-1252 2-character mojibake that isn't covered by the cases above + [ÂÃÎÐ][€Šš¢£Ÿž\xa0\xad®©°·»{end_punctuation}–—´] | × [²³] | @@ -585,21 +585,45 @@ def is_bad(text): 'utf8_first_of_4': ( 'ðóđğπσру' ), - # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding + # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding, + # including a space standing in for 0xA0 'utf8_continuation': ( - '\x80-\xbf ' + '\x80-\xbf' 'ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅' 'ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ' '–—―‘’‚“”„†‡•…‰‹›€№™' ' ' ), + # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding, + # and don't usually stand for themselves when adjacent to mojibake. + # This excludes spaces, dashes, quotation marks, and ellipses. + 'utf8_continuation_strict': ( + '\x80-\xbf' + 'ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅' + 'ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ' + '†‡•‰‹›€№™' + ), } # This regex uses UTF8_CLUES to find sequences of likely mojibake. # It matches them with + so that several adjacent UTF-8-looking sequences # get coalesced into one, allowing them to be fixed more efficiently # and not requiring every individual subsequence to be detected as 'badness'. +# +# We accept spaces in place of "utf8_continuation", because spaces might have +# been intended to be U+A0 NO-BREAK SPACE. +# +# We do a lookbehind to make sure the previous character isn't a +# "utf8_continuation_strict" character, so that we don't fix just a few +# characters in a huge garble and make the situation worse. +# +# Unfortunately, the matches to this regular expression won't show their +# surrounding context, and including context would make the expression much +# less efficient. The 'badness' rules that require context, such as a preceding +# lowercase letter, will prevent some cases of inconsistent UTF-8 from being +# fixed when they don't see it. UTF8_DETECTOR_RE = re.compile(""" + (? III Divisão - Série F > Jornadas Classificação", + "fixed": "Campeonatos > III Divisão - Série F > Jornadas Classificação", + "expect": "pass" + }, { "label": "Handle Afrikaans 'n character", "original": "ʼn Chloroplas is ʼn organel wat in fotosinterende plante voorkom.", From 95e700bce52df3c73e50cbeacdf9f5dd85056839 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 25 Mar 2021 17:55:07 -0400 Subject: [PATCH 464/502] big top-level reorganization --- ftfy/__init__.py | 519 ++++++++++++++++++++++++++++++----------- ftfy/badness.py | 23 +- ftfy/chardata.py | 21 +- ftfy/cli.py | 14 +- ftfy/fixes.py | 254 ++------------------ setup.py | 2 +- tests/test_entities.py | 12 +- 7 files changed, 450 insertions(+), 395 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 4889e703..a1d887e7 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -6,38 +6,151 @@ """ import unicodedata +import warnings +from typing import Optional +from collections import namedtuple import ftfy.bad_codecs -from ftfy import fixes +from ftfy import fixes, chardata +from ftfy.badness import is_bad from ftfy.formatting import display_ljust -__version__ = '5.8' +__version__ = '6.0' -# See the docstring for ftfy.bad_codecs to see what we're doing here. +# Though this function does nothing, it lets linters know that we're using +# ftfy.bad_codecs. See the docstring in `bad_codecs/__init__.py` for more. ftfy.bad_codecs.ok() -def fix_text( - text, - *, - fix_entities='auto', - remove_terminal_escapes=True, - fix_encoding=True, - fix_latin_ligatures=True, - fix_character_width=True, - uncurl_quotes=True, - fix_line_breaks=True, - fix_surrogates=True, - remove_control_chars=True, - remove_bom=True, - normalization='NFC', - max_decode_length=10 ** 6 -): +CONFIG_DEFAULTS = { + # Replace HTML entities, but not in strings where a < character + # has appeared literally. Those strings are assumed to be HTML, + # so entities should be left alone. + 'unescape_html': 'auto', + + # Remove "ANSI" terminal escapes, such as for changing the color + # of text in a terminal. + 'remove_terminal_escapes': True, + + # Detect mojibake, and attempt to decode the entire string in a + # different encoding when it's detected. + 'fix_encoding': True, + + # When fixing mojibake, allow a literal space to be interpreted + # as a non-breaking space, which would have been byte A0 in many + # encodings. + 'restore_byte_a0': True, + + # Detect mojibake that has been partially replaced by the characters + # '�' or '?', and replace the detected sequences with '�'. + 'replace_lossy_sequences': True, + + # Detect certain kinds of mojibake even when it's not consistent + # across the entire string. Replace sufficiently clear sequences + # of UTF-8 mojibake with the characters they should have been. + 'decode_inconsistent_utf8': True, + + # Replace C1 control characters with their Windows-1252 equivalents, + # like HTML5 does. + 'fix_c1_controls': True, + + # Replace common Latin-alphabet ligatures, such as 'fi', with the + # letters they're made of. + 'fix_latin_ligatures': True, + + # Replace fullwidth Latin characters and halfwidth Katakana with + # their more standard widths. + 'fix_character_width': True, + + # Replace curly quotes with straight quotes. + 'uncurl_quotes': True, + + # Replace various forms of line breaks with the standard Unix line + # break, '\n'. + 'fix_line_breaks': True, + + # Replace sequences of UTF-16 surrogates with the character they were + # meant to encode. + 'fix_surrogates': True, + + # Remove control characters that have no displayed effect on text. + 'remove_control_chars': True, + + # Change 'normalization' to 'NFKC' to apply Unicode compatibility + # conversions. In some cases, this will change the meaning of text. + # + # You can also set it to None to apply no normalization, including + # leaving all combining characters separate. + 'normalization': 'NFC', + + # The maximum length of line that should be fixed by ftfy without + # breaking it up into smaller strings. + 'max_decode_length': 1000000, + + # Set 'explain' to False to not compute explanations, possibly saving + # time. The functions that return explanations will return None. + 'explain': True +} +TextFixerConfig = namedtuple( + 'TextFixerConfig', + CONFIG_DEFAULTS.keys(), + defaults=CONFIG_DEFAULTS.values() +) + + +FIXERS = { + 'unescape_html': fixes.unescape_html, + 'remove_terminal_escapes': fixes.remove_terminal_escapes, + 'restore_byte_a0': fixes.restore_byte_a0, + 'replace_lossy_sequences': fixes.replace_lossy_sequences, + 'decode_inconsistent_utf8': fixes.decode_inconsistent_utf8, + 'fix_c1_controls': fixes.fix_c1_controls, + 'fix_latin_ligatures': fixes.fix_latin_ligatures, + 'fix_character_width': fixes.fix_character_width, + 'uncurl_quotes': fixes.uncurl_quotes, + 'fix_line_breaks': fixes.fix_line_breaks, + 'fix_surrogates': fixes.fix_surrogates, + 'remove_control_chars': fixes.remove_control_chars, +} + + +BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. + +ftfy is designed to fix problems that were introduced by handling Unicode +incorrectly. It might be able to fix the bytes you just handed it, but the +fact that you just gave a pile of bytes to a function that fixes text means +that your code is *also* handling Unicode incorrectly. + +ftfy takes Unicode text as input. You should take these bytes and decode +them from the encoding you think they are in. If you're not sure what encoding +they're in: + +- First, try to find out. 'utf-8' is a good assumption. +- If the encoding is simply unknowable, try running your bytes through + ftfy.guess_bytes. As the name implies, this may not always be accurate. + +If you're confused by this, please read the Python Unicode HOWTO: + + http://docs.python.org/3/howto/unicode.html +""" + + +def _try_fix(fixer_name: str, text: str, steps: list) -> str: + fixer = FIXERS[fixer_name] + fixed = fixer(text) + if fixed != text: + steps.append(('apply', fixer_name)) + return fixed + + +def fix_text(text: str, config: Optional[TextFixerConfig] = None, **kwargs) -> str: r""" Given Unicode text as input, fix inconsistencies and glitches in it, such as mojibake. + FIXME: clean up this docstring + Let's start with some examples: >>> print(fix_text('ünicode')) @@ -148,7 +261,20 @@ def fix_text( the entire text in the same way, and you don't mind operations that scale with the length of the text, you can use `fix_text_segment` directly to fix the whole string in one batch. + + ftfy can also provide an 'explanation', a list of transformations it applied + to the text that would fix more text like it. + + However, this function doesn't provide explanations, because it can apply + different fixes to different lines of text, and one explanation may not + suffice. + + To get an explanation, use the :func:`fix_and_explain()` function. """ + + if config is None: + config = TextFixerConfig() + config = config._replace(**kwargs) if isinstance(text, bytes): raise UnicodeError(fixes.BYTES_ERROR_TEXT) @@ -156,59 +282,219 @@ def fix_text( pos = 0 while pos < len(text): textbreak = text.find('\n', pos) + 1 - fix_encoding_this_time = fix_encoding if textbreak == 0: textbreak = len(text) - if (textbreak - pos) > max_decode_length: - fix_encoding_this_time = False - - substring = text[pos:textbreak] - - if fix_entities == 'auto' and '<' in substring and '>' in substring: - # we see angle brackets together; this could be HTML - fix_entities = False - - out.append( - fix_text_segment( - substring, - fix_entities=fix_entities, - remove_terminal_escapes=remove_terminal_escapes, - fix_encoding=fix_encoding_this_time, - uncurl_quotes=uncurl_quotes, - fix_latin_ligatures=fix_latin_ligatures, - fix_character_width=fix_character_width, - fix_line_breaks=fix_line_breaks, - fix_surrogates=fix_surrogates, - remove_control_chars=remove_control_chars, - remove_bom=remove_bom, - normalization=normalization, - ) - ) + if (textbreak - pos) > config.max_decode_length: + textbreak = pos + config.max_decode_length + + segment = text[pos:textbreak] + if config.unescape_html == 'auto' and '<' in segment: + config = config._replace(unescape_html=False) + fixed_segment, _ = fix_and_explain(segment, config) + out.append(fixed_segment) pos = textbreak - return ''.join(out) +def fix_and_explain( + text: str, + config: Optional[TextFixerConfig] = None +) -> (str, list): + """ + Apply fixes to text in a single chunk, and also produce an explanation + of what was fixed. + + Returns the fixed text, and a list explaining what was fixed. + """ + if config is None: + config = TextFixerConfig() + if isinstance(text, bytes): + raise UnicodeError(fixes.BYTES_ERROR_TEXT) + + fix_entities = config.unescape_html + if fix_entities == 'auto' and '<' in text: + fix_entities = False + + steps = [] + while True: + origtext = text + + if fix_entities: + text = _try_fix('unescape_html', text, steps) + + if config.fix_encoding: + text, encoding_steps = fix_encoding_and_explain(text, config) + steps.extend(encoding_steps) + + for fixer in [ + 'fix_c1_controls', 'fix_latin_ligatures', 'fix_character_width', + 'uncurl_quotes', 'fix_line_breaks', 'fix_surrogates', 'remove_control_chars' + ]: + if getattr(config, fixer): + text = _try_fix(fixer, text, steps) + # TODO: backward compatibility for remove_bom + + if config.normalization is not None: + fixed = unicodedata.normalize(config.normalization, text) + if fixed != text: + text = fixed + steps.append(('normalize', config.normalization)) + + if text == origtext: + return text, steps + + +def fix_encoding_and_explain(text, config=None): + if config is None: + config = TextFixerConfig() + if isinstance(text, bytes): + raise UnicodeError(fixes.BYTES_ERROR_TEXT) + if not config.fix_encoding: + raise ValueError( + "It doesn't make sense to run fix_encoding_and_explain " + "with fix_encoding=False" + ) + + plan_so_far = [] + while True: + prevtext = text + text, plan = _fix_encoding_one_step_and_explain(text, config) + plan_so_far.extend(plan) + if text == prevtext: + return text, plan_so_far + + +def _fix_encoding_one_step_and_explain(text, config): + if config is None: + config = TextFixerConfig() + + if len(text) == 0: + return text, [] + + # The first plan is to return ASCII text unchanged, as well as text + # that doesn't look like it contains mojibake + if chardata.possible_encoding(text, 'ascii') or not is_bad(text): + return text, [] + + # As we go through the next step, remember the possible encodings + # that we encounter but don't successfully fix yet. We may need them + # later. + possible_1byte_encodings = [] + + # Suppose the text was supposed to be UTF-8, but it was decoded using + # a single-byte encoding instead. When these cases can be fixed, they + # are usually the correct thing to do, so try them next. + for encoding in chardata.CHARMAP_ENCODINGS: + if chardata.possible_encoding(text, encoding): + possible_1byte_encodings.append(encoding) + encoded_bytes = text.encode(encoding) + encode_step = ('encode', encoding) + transcode_steps = [] + + # Now, find out if it's UTF-8 (or close enough). Otherwise, + # remember the encoding for later. + try: + decoding = 'utf-8' + # Check encoded_bytes for sequences that would be UTF-8, + # except they have b' ' where b'\xa0' would belong. + if config.restore_byte_a0 and chardata.ALTERED_UTF8_RE.search(encoded_bytes): + replaced_bytes = fixes.restore_byte_a0(encoded_bytes) + if replaced_bytes != encoded_bytes: + transcode_steps.append(('transcode', 'restore_byte_a0')) + encoded_bytes = replaced_bytes + + # Replace sequences where information has been lost + if config.replace_lossy_sequences and encoding.startswith('sloppy'): + replaced_bytes = fixes.replace_lossy_sequences(encoded_bytes) + if replaced_bytes != encoded_bytes: + transcode_steps.append(('transcode', 'replace_lossy_sequences')) + encoded_bytes = replaced_bytes + + if 0xed in encoded_bytes or 0xc0 in encoded_bytes: + decoding = 'utf-8-variants' + + decode_step = ('decode', decoding) + steps = [encode_step] + transcode_steps + [decode_step] + fixed = encoded_bytes.decode(decoding) + return fixed, steps + + except UnicodeDecodeError: + pass + + # Look for a-hat-euro sequences that remain, and fix them in isolation. + if config.decode_inconsistent_utf8 and chardata.UTF8_DETECTOR_RE.search(text): + steps = [('apply', 'decode_inconsistent_utf8')] + fixed = fixes.decode_inconsistent_utf8(text) + if fixed != text: + return fixed, steps + + # The next most likely case is that this is Latin-1 that was intended to + # be read as Windows-1252, because those two encodings in particular are + # easily confused. + if 'latin-1' in possible_1byte_encodings: + if 'windows-1252' in possible_1byte_encodings: + # This text is in the intersection of Latin-1 and + # Windows-1252, so it's probably legit. + return text, [] + else: + # Otherwise, it means we have characters that are in Latin-1 but + # not in Windows-1252. Those are C1 control characters. Nobody + # wants those. Assume they were meant to be Windows-1252. + try: + fixed = text.encode('latin-1').decode('windows-1252') + if fixed != text: + steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] + return fixed, steps + except UnicodeDecodeError: + pass + + # Fix individual characters of Latin-1 with a less satisfying explanation + if config.fix_c1_controls and chardata.C1_CONTROL_RE.search(text): + steps = [('transcode', 'fix_c1_controls')] + fixed = fixes.fix_c1_controls(text) + return fixed, steps + + # The cases that remain are mixups between two different single-byte + # encodings, and not the common case of Latin-1 vs. Windows-1252. + # + # These cases may be unsolvable without adding false positives, though + # I have vague ideas about how to optionally address them in the future. + + # Return the text unchanged; the plan is empty. + return text, [] + + +def fix_encoding(text, config=None, **kwargs): + if config is None: + config = TextFixerConfig() + config = config._replace(**kwargs) + fixed, explan = fix_encoding_and_explain(text, config) + return fixed + + + # Some alternate names for the main functions ftfy = fix_text -fix_encoding = fixes.fix_encoding + + +def fix_text_segment(text, config=None, **kwargs): + warnings.warn( + "`fix_text_segment()` is deprecated as of ftfy 6.0. " + "Use `fix_and_explain()` instead.", + DeprecationWarning + ) + + if config is None: + config = TextFixerConfig() + config = config._replace(**kwargs) + fixed, explan = fix_and_explain(text, config) + return fixed def fix_file( input_file, encoding=None, - *, - fix_entities='auto', - remove_terminal_escapes=True, - fix_encoding=True, - fix_latin_ligatures=True, - fix_character_width=True, - uncurl_quotes=True, - fix_line_breaks=True, - fix_surrogates=True, - remove_control_chars=True, - remove_bom=True, - normalization='NFC' + config=None ): """ Fix text that is found in a file. @@ -220,86 +506,17 @@ def fix_file( The output is a stream of fixed lines of text. """ - entities = fix_entities for line in input_file: if isinstance(line, bytes): if encoding is None: line, encoding = guess_bytes(line) else: line = line.decode(encoding) - if fix_entities == 'auto' and '<' in line and '>' in line: - entities = False - yield fix_text_segment( - line, - fix_entities=entities, - remove_terminal_escapes=remove_terminal_escapes, - fix_encoding=fix_encoding, - fix_latin_ligatures=fix_latin_ligatures, - fix_character_width=fix_character_width, - uncurl_quotes=uncurl_quotes, - fix_line_breaks=fix_line_breaks, - fix_surrogates=fix_surrogates, - remove_control_chars=remove_control_chars, - remove_bom=remove_bom, - normalization=normalization, - ) - - -def fix_text_segment( - text, - *, - fix_entities='auto', - remove_terminal_escapes=True, - fix_encoding=True, - fix_latin_ligatures=True, - fix_character_width=True, - uncurl_quotes=True, - fix_line_breaks=True, - fix_surrogates=True, - remove_control_chars=True, - remove_bom=True, - normalization='NFC' -): - """ - Apply fixes to text in a single chunk. This could be a line of text - within a larger run of `fix_text`, or it could be a larger amount - of text that you are certain is in a consistent encoding. - - See `fix_text` for a description of the parameters. - """ - if isinstance(text, bytes): - raise UnicodeError(fixes.BYTES_ERROR_TEXT) - - if fix_entities == 'auto' and '<' in text and '>' in text: - fix_entities = False - while True: - origtext = text - if remove_terminal_escapes: - text = fixes.remove_terminal_escapes(text) - if fix_encoding: - text = fixes.fix_encoding(text) - if fix_entities: - text = fixes.unescape_html(text) - if fix_latin_ligatures: - text = fixes.fix_latin_ligatures(text) - if fix_character_width: - text = fixes.fix_character_width(text) - if uncurl_quotes: - text = fixes.uncurl_quotes(text) - if fix_line_breaks: - text = fixes.fix_line_breaks(text) - if fix_surrogates: - text = fixes.fix_surrogates(text) - if remove_control_chars: - text = fixes.remove_control_chars(text) - if remove_bom and not remove_control_chars: - # Skip this step if we've already done `remove_control_chars`, - # because it would be redundant. - text = fixes.remove_bom(text) - if normalization is not None: - text = unicodedata.normalize(normalization, text) - if text == origtext: - return text + if config.unescape_html == 'auto' and '<' in line: + config = config._replace(unescape_html=False) + + fixed_line, _explan = fix_and_explain(line, config) + yield fixed_line def guess_bytes(bstring): @@ -350,7 +567,8 @@ def guess_bytes(bstring): # # This will occasionally trigger on standard UTF-8, as there # are some Korean characters that also use byte 0xed, but that's - # not harmful. + # not harmful because standard UTF-8 characters will decode the + # same way in our 'utf-8-variants' codec. # # Byte 0xc0 is impossible because, numerically, it would only # encode characters lower than U+0040. Those already have @@ -401,6 +619,39 @@ def guess_bytes(bstring): return bstring.decode('sloppy-windows-1252'), 'sloppy-windows-1252' +def apply_plan(text, plan): + """ + Apply a plan for fixing the encoding of text. + + The plan is a list of tuples of the form (operation, arg). + + `operation` is one of: + + - `'encode'`: convert a string to bytes, using `arg` as the encoding + - `'decode'`: convert bytes to a string, using `arg` as the encoding + - `'transcode'`: convert bytes to bytes, using the function named `arg` + - `'apply'`: convert a string to a string, using the function named `arg` + + The functions that can be applied by 'transcode' and 'apply' appear in + the dictionary named `FIXERS`. + """ + obj = text + for operation, encoding in plan: + if operation == 'encode': + obj = obj.encode(encoding) + elif operation == 'decode': + obj = obj.decode(encoding) + elif operation == 'transcode' or operation == 'apply': + if encoding in FIXERS: + obj = FIXERS[encoding](obj) + else: + raise ValueError("Unknown function to apply: %s" % encoding) + else: + raise ValueError("Unknown plan step: %s" % operation) + + return obj + + def explain_unicode(text): """ A utility method that's useful for debugging mysterious Unicode. diff --git a/ftfy/badness.py b/ftfy/badness.py index 2a75a818..18bb4a9a 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -3,9 +3,28 @@ """ import re -from .chardata import badness +import warnings +from ftfy import chardata def sequence_weirdness(text): - # TODO: do the deprecation thing + warnings.warn( + "`sequence_weirdness()` is an old heuristic, and the current " + "closest equivalent is `ftfy.badness.badness()`" + ) return badness(text) + + +def badness(text): + """ + Get the 'badness' of a sequence of text. A badness greater than 0 indicates + that some of it seems to be mojibake. + """ + return len(chardata.BADNESS_RE.findall(text)) + + +def is_bad(text): + """ + Returns true iff the given text looks like it contains mojibake. + """ + return bool(chardata.BADNESS_RE.search(text)) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index a6e5e19c..2bf02622 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -470,7 +470,7 @@ def _char_range(start, end): } -BADNESS_RE = re.compile(""" +BADNESS_RE = re.compile(r""" [{c1}] | [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] @@ -485,7 +485,7 @@ def _char_range(start, end): # fancy leetspeak-esque writing [{lower_accented}{box}{end_punctuation}] [{currency}] | - \\s [{upper_accented}] [{currency}] + \s [{upper_accented}] [{currency}] | [{upper_accented}{box}{end_punctuation}] [{numeric}] | @@ -539,7 +539,7 @@ def _char_range(start, end): # Windows-1252 encodings of 'à' and 'á' Ã[\xa0¡] | - [a-z]\\s?Ã[ ] + [a-z]\s?Ã[ ] | ^Ã[ ] | @@ -553,21 +553,6 @@ def _char_range(start, end): """.format(**MOJIBAKE_CATEGORIES), re.VERBOSE) -def badness(text): - """ - Get the 'badness' of a sequence of text. A badness greater than 0 indicates - that some of it seems to be mojibake. - """ - return len(BADNESS_RE.findall(text)) - - -def is_bad(text): - """ - Returns true iff the given text looks like it contains mojibake. - """ - return bool(BADNESS_RE.search(text)) - - # Character classes that help us pinpoint embedded mojibake. These can # include common characters, because we'll also check them for 'badness'. UTF8_CLUES = { diff --git a/ftfy/cli.py b/ftfy/cli.py index 41c36718..4148d1fc 100644 --- a/ftfy/cli.py +++ b/ftfy/cli.py @@ -4,7 +4,7 @@ import os import sys -from ftfy import __version__, fix_file +from ftfy import __version__, fix_file, TextFixerConfig ENCODE_ERROR_TEXT_UNIX = """ftfy error: Unfortunately, this output stream does not support Unicode. @@ -118,16 +118,20 @@ def main(): normalization = None if args.preserve_entities: - fix_entities = False + unescape_html = False else: - fix_entities = 'auto' + unescape_html = 'auto' + + config = TextFixerConfig( + unescape_html=unescape_html, + normalization=normalization + ) try: for line in fix_file( file, encoding=encoding, - fix_entities=fix_entities, - normalization=normalization, + config=config ): try: outfile.write(line) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 06bdfb5a..d329ecf7 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -6,7 +6,10 @@ import codecs import html import re +import unicodedata +import warnings +import ftfy from ftfy.chardata import ( ALTERED_UTF8_RE, C1_CONTROL_RE, @@ -20,249 +23,42 @@ SINGLE_QUOTE_RE, UTF8_DETECTOR_RE, WIDTH_MAP, - badness, - is_bad, possible_encoding, ) -BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. - -ftfy is designed to fix problems that were introduced by handling Unicode -incorrectly. It might be able to fix the bytes you just handed it, but the -fact that you just gave a pile of bytes to a function that fixes text means -that your code is *also* handling Unicode incorrectly. - -ftfy takes Unicode text as input. You should take these bytes and decode -them from the encoding you think they are in. If you're not sure what encoding -they're in: - -- First, try to find out. 'utf-8' is a good assumption. -- If the encoding is simply unknowable, try running your bytes through - ftfy.guess_bytes. As the name implies, this may not always be accurate. - -If you're confused by this, please read the Python Unicode HOWTO: - - http://docs.python.org/3/howto/unicode.html -""" - - -def fix_encoding(text): - r""" - Fix text with incorrectly-decoded garbage ("mojibake") whenever possible. - - This function looks for the evidence of mojibake, formulates a plan to fix - it, and applies the plan. It determines whether it should replace nonsense - sequences of single-byte characters that were really meant to be UTF-8 - characters, and if so, turns them into the correctly-encoded Unicode - character that they were meant to represent. - - The input to the function must be Unicode. If you don't have Unicode text, - you're not using the right tool to solve your problem. - - `fix_encoding` decodes text that looks like it was decoded incorrectly. It - leaves alone text that doesn't. - - >>> fix_encoding('único') - 'único' - - >>> fix_encoding('This text is fine already :þ') - 'This text is fine already :þ' - - Because these characters often come from Microsoft products, we allow - for the possibility that we get not just Unicode characters 128-255, but - also Windows's conflicting idea of what characters 128-160 are. - - >>> fix_encoding('This — should be an em dash') - 'This — should be an em dash' - - We might have to deal with both Windows characters and raw control - characters at the same time, especially when dealing with characters like - 0x81 that have no mapping in Windows. This is a string that Python's - standard `.encode` and `.decode` methods cannot correct. - - >>> fix_encoding('This text is sad .â\x81”.') - 'This text is sad .⁔.' - - However, it has safeguards against fixing sequences of letters and - punctuation that can occur in valid text. In the following example, - the last three characters are not replaced with a Korean character, - even though they could be. - - >>> fix_encoding('not such a fan of Charlotte Brontë…”') - 'not such a fan of Charlotte Brontë…”' - - This function can now recover some complex manglings of text, such as when - UTF-8 mojibake has been normalized in a way that replaces U+A0 with a - space: - - >>> fix_encoding('The more you know 🌠') - 'The more you know 🌠' - - Cases of genuine ambiguity can sometimes be addressed by finding other - characters that are not double-encoded, and expecting the encoding to - be consistent: - - >>> fix_encoding('AHÅ™, the new sofa from IKEA®') - 'AHÅ™, the new sofa from IKEA®' - - Finally, we handle the case where the text is in a single-byte encoding - that was intended as Windows-1252 all along but read as Latin-1: - - >>> fix_encoding('This text was never UTF-8 at all\x85') - 'This text was never UTF-8 at all…' - - The best version of the text is found using - :func:`ftfy.badness.text_cost`. - """ - text, _ = fix_encoding_and_explain(text) - return text +from ftfy.badness import badness, is_bad def fix_encoding_and_explain(text): - """ - Re-decodes text that has been decoded incorrectly, and also return a - "plan" indicating all the steps required to fix it. + warnings.warn( + "`fix_encoding_and_explain()` has moved to the main module of ftfy.", + DeprecationWarning + ) + return ftfy.fix_encoding_and_explain(text) - The resulting plan could be used with :func:`ftfy.fixes.apply_plan` - to fix additional strings that are broken in the same way. - """ - if isinstance(text, bytes): - raise UnicodeError(BYTES_ERROR_TEXT) - plan_so_far = [] - while True: - prevtext = text - text, plan = fix_one_step_and_explain(text) - plan_so_far.extend(plan) - if text == prevtext: - return text, plan_so_far +def fix_encoding(text): + warnings.warn( + "`fix_encoding()` has moved to the main module of ftfy.", + DeprecationWarning + ) + return ftfy.fix_encoding(text) def fix_one_step_and_explain(text): - """ - Performs a single step of re-decoding text that's been decoded incorrectly. - - Returns the decoded text, plus a "plan" for how to reproduce what it did. - """ - if len(text) == 0: - return text, [] - - # The first plan is to return ASCII text unchanged, as well as text - # that doesn't look like it contains mojibake - if possible_encoding(text, 'ascii') or not is_bad(text): - return text, [] - - # As we go through the next step, remember the possible encodings - # that we encounter but don't successfully fix yet. We may need them - # later. - possible_1byte_encodings = [] - - # Suppose the text was supposed to be UTF-8, but it was decoded using - # a single-byte encoding instead. When these cases can be fixed, they - # are usually the correct thing to do, so try them next. - for encoding in CHARMAP_ENCODINGS: - if possible_encoding(text, encoding): - possible_1byte_encodings.append(encoding) - encoded_bytes = text.encode(encoding) - encode_step = ('encode', encoding) - transcode_steps = [] - - # Now, find out if it's UTF-8 (or close enough). Otherwise, - # remember the encoding for later. - try: - decoding = 'utf-8' - # Check encoded_bytes for sequences that would be UTF-8, - # except they have b' ' where b'\xa0' would belong. - if ALTERED_UTF8_RE.search(encoded_bytes): - encoded_bytes = restore_byte_a0(encoded_bytes) - transcode_steps.append(('transcode', 'restore_byte_a0')) - - # Check for the byte 0x1a, which indicates where one of our - # 'sloppy' codecs found a replacement character. - if encoding.startswith('sloppy'): - encoded_bytes = replace_lossy_sequences(encoded_bytes) - transcode_steps.append(('transcode', 'replace_lossy_sequences')) - - if 0xed in encoded_bytes or 0xc0 in encoded_bytes: - decoding = 'utf-8-variants' - - decode_step = ('decode', decoding) - steps = [encode_step] + transcode_steps + [decode_step] - fixed = encoded_bytes.decode(decoding) - return fixed, steps - - except UnicodeDecodeError: - pass - - # Look for a-hat-euro sequences that remain, and fix them in isolation. - if UTF8_DETECTOR_RE.search(text): - steps = [('transcode', 'decode_inconsistent_utf8')] - fixed = decode_inconsistent_utf8(text) - if fixed != text: - return fixed, steps - - # The next most likely case is that this is Latin-1 that was intended to - # be read as Windows-1252, because those two encodings in particular are - # easily confused. - if 'latin-1' in possible_1byte_encodings: - if 'windows-1252' in possible_1byte_encodings: - # This text is in the intersection of Latin-1 and - # Windows-1252, so it's probably legit. - return text, [] - else: - # Otherwise, it means we have characters that are in Latin-1 but - # not in Windows-1252. Those are C1 control characters. Nobody - # wants those. Assume they were meant to be Windows-1252. - try: - fixed = text.encode('latin-1').decode('windows-1252') - if fixed != text: - steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] - return fixed, steps - except UnicodeDecodeError: - pass - - # Fix individual characters of Latin-1 with a less satisfying explanation - if C1_CONTROL_RE.search(text): - steps = [('transcode', 'fix_c1_controls')] - fixed = fix_c1_controls(text) - return fixed, steps - - # The cases that remain are mixups between two different single-byte - # encodings, and not the common case of Latin-1 vs. Windows-1252. - # - # These cases may be unsolvable without adding false positives, though - # I have vague ideas about how to optionally address them in the future. - - # Return the text unchanged; the plan is empty. - return text, [] + warnings.warn( + "`fix_one_step_and_explain()` has moved to the main module of ftfy.", + DeprecationWarning + ) + return ftfy.fix_one_step_and_explain(text) def apply_plan(text, plan): - """ - Apply a plan for fixing the encoding of text. - - The plan is a list of tuples of the form (operation, encoding): - - - `operation` is 'encode' if it turns a string into bytes, 'decode' if it - turns bytes into a string, and 'transcode' if it keeps the type the same. - - `encoding` is the name of the encoding to use, such as 'utf-8' or - 'latin-1', or the function name in the case of 'transcode'. - """ - obj = text - for operation, encoding in plan: - if operation == 'encode': - obj = obj.encode(encoding) - elif operation == 'decode': - obj = obj.decode(encoding) - elif operation == 'transcode': - if encoding in TRANSCODERS: - obj = TRANSCODERS[encoding](obj) - else: - raise ValueError("Unknown transcode operation: %s" % encoding) - else: - raise ValueError("Unknown plan step: %s" % operation) - - return obj + warnings.warn( + "`apply_plan()` has moved to the main module of ftfy.", + DeprecationWarning + ) + return ftfy.apply_plan(text, plan) def _unescape_fixup(match): diff --git a/setup.py b/setup.py index fabc4fdf..210f815b 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name="ftfy", - version='5.9', + version='6.0', maintainer='Robyn Speer', maintainer_email='rspeer@luminoso.com', license="MIT", diff --git a/tests/test_entities.py b/tests/test_entities.py index 2d05e275..f05880ee 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -7,14 +7,14 @@ def test_entities(): assert fix_text(example) == '&\n\n&' assert fix_text_segment(example) == '&\n\n&' - assert fix_text(example, fix_entities=True) == '&\n\n&' - assert fix_text_segment(example, fix_entities=True) == '&\n\n&' + assert fix_text(example, unescape_html=True) == '&\n\n&' + assert fix_text_segment(example, unescape_html=True) == '&\n\n&' - assert fix_text(example, fix_entities=False) == '&\n\n&' - assert fix_text_segment(example, fix_entities=False) == '&\n\n&' + assert fix_text(example, unescape_html=False) == '&\n\n&' + assert fix_text_segment(example, unescape_html=False) == '&\n\n&' - assert fix_text_segment('<>', fix_entities=False) == '<>' - assert fix_text_segment('<>', fix_entities=True) == '<>' + assert fix_text_segment('<>', unescape_html=False) == '<>' + assert fix_text_segment('<>', unescape_html=True) == '<>' assert fix_text_segment('<>') == '<>' assert fix_text_segment('jednocześnie') == 'jednocześnie' assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE' From df908f7404f78a540f0a3b71347e02e4e88e6823 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 25 Mar 2021 17:56:58 -0400 Subject: [PATCH 465/502] reformat with black+isort --- ftfy/__init__.py | 200 +++++++++++++++++++++-------------------------- 1 file changed, 91 insertions(+), 109 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index a1d887e7..20aca377 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -7,15 +7,15 @@ import unicodedata import warnings -from typing import Optional from collections import namedtuple +from typing import Optional import ftfy.bad_codecs -from ftfy import fixes, chardata +from ftfy import chardata, fixes from ftfy.badness import is_bad from ftfy.formatting import display_ljust -__version__ = '6.0' +__version__ = "6.0" # Though this function does nothing, it lets linters know that we're using @@ -27,91 +27,74 @@ # Replace HTML entities, but not in strings where a < character # has appeared literally. Those strings are assumed to be HTML, # so entities should be left alone. - 'unescape_html': 'auto', - + "unescape_html": "auto", # Remove "ANSI" terminal escapes, such as for changing the color # of text in a terminal. - 'remove_terminal_escapes': True, - + "remove_terminal_escapes": True, # Detect mojibake, and attempt to decode the entire string in a # different encoding when it's detected. - 'fix_encoding': True, - + "fix_encoding": True, # When fixing mojibake, allow a literal space to be interpreted # as a non-breaking space, which would have been byte A0 in many # encodings. - 'restore_byte_a0': True, - + "restore_byte_a0": True, # Detect mojibake that has been partially replaced by the characters # '�' or '?', and replace the detected sequences with '�'. - 'replace_lossy_sequences': True, - + "replace_lossy_sequences": True, # Detect certain kinds of mojibake even when it's not consistent # across the entire string. Replace sufficiently clear sequences # of UTF-8 mojibake with the characters they should have been. - 'decode_inconsistent_utf8': True, - + "decode_inconsistent_utf8": True, # Replace C1 control characters with their Windows-1252 equivalents, # like HTML5 does. - 'fix_c1_controls': True, - + "fix_c1_controls": True, # Replace common Latin-alphabet ligatures, such as 'fi', with the # letters they're made of. - 'fix_latin_ligatures': True, - + "fix_latin_ligatures": True, # Replace fullwidth Latin characters and halfwidth Katakana with # their more standard widths. - 'fix_character_width': True, - + "fix_character_width": True, # Replace curly quotes with straight quotes. - 'uncurl_quotes': True, - + "uncurl_quotes": True, # Replace various forms of line breaks with the standard Unix line # break, '\n'. - 'fix_line_breaks': True, - + "fix_line_breaks": True, # Replace sequences of UTF-16 surrogates with the character they were # meant to encode. - 'fix_surrogates': True, - + "fix_surrogates": True, # Remove control characters that have no displayed effect on text. - 'remove_control_chars': True, - + "remove_control_chars": True, # Change 'normalization' to 'NFKC' to apply Unicode compatibility # conversions. In some cases, this will change the meaning of text. # # You can also set it to None to apply no normalization, including # leaving all combining characters separate. - 'normalization': 'NFC', - + "normalization": "NFC", # The maximum length of line that should be fixed by ftfy without # breaking it up into smaller strings. - 'max_decode_length': 1000000, - + "max_decode_length": 1000000, # Set 'explain' to False to not compute explanations, possibly saving # time. The functions that return explanations will return None. - 'explain': True + "explain": True, } TextFixerConfig = namedtuple( - 'TextFixerConfig', - CONFIG_DEFAULTS.keys(), - defaults=CONFIG_DEFAULTS.values() + "TextFixerConfig", CONFIG_DEFAULTS.keys(), defaults=CONFIG_DEFAULTS.values() ) FIXERS = { - 'unescape_html': fixes.unescape_html, - 'remove_terminal_escapes': fixes.remove_terminal_escapes, - 'restore_byte_a0': fixes.restore_byte_a0, - 'replace_lossy_sequences': fixes.replace_lossy_sequences, - 'decode_inconsistent_utf8': fixes.decode_inconsistent_utf8, - 'fix_c1_controls': fixes.fix_c1_controls, - 'fix_latin_ligatures': fixes.fix_latin_ligatures, - 'fix_character_width': fixes.fix_character_width, - 'uncurl_quotes': fixes.uncurl_quotes, - 'fix_line_breaks': fixes.fix_line_breaks, - 'fix_surrogates': fixes.fix_surrogates, - 'remove_control_chars': fixes.remove_control_chars, + "unescape_html": fixes.unescape_html, + "remove_terminal_escapes": fixes.remove_terminal_escapes, + "restore_byte_a0": fixes.restore_byte_a0, + "replace_lossy_sequences": fixes.replace_lossy_sequences, + "decode_inconsistent_utf8": fixes.decode_inconsistent_utf8, + "fix_c1_controls": fixes.fix_c1_controls, + "fix_latin_ligatures": fixes.fix_latin_ligatures, + "fix_character_width": fixes.fix_character_width, + "uncurl_quotes": fixes.uncurl_quotes, + "fix_line_breaks": fixes.fix_line_breaks, + "fix_surrogates": fixes.fix_surrogates, + "remove_control_chars": fixes.remove_control_chars, } @@ -140,7 +123,7 @@ def _try_fix(fixer_name: str, text: str, steps: list) -> str: fixer = FIXERS[fixer_name] fixed = fixer(text) if fixed != text: - steps.append(('apply', fixer_name)) + steps.append(("apply", fixer_name)) return fixed @@ -281,25 +264,22 @@ def fix_text(text: str, config: Optional[TextFixerConfig] = None, **kwargs) -> s out = [] pos = 0 while pos < len(text): - textbreak = text.find('\n', pos) + 1 + textbreak = text.find("\n", pos) + 1 if textbreak == 0: textbreak = len(text) if (textbreak - pos) > config.max_decode_length: textbreak = pos + config.max_decode_length segment = text[pos:textbreak] - if config.unescape_html == 'auto' and '<' in segment: + if config.unescape_html == "auto" and "<" in segment: config = config._replace(unescape_html=False) fixed_segment, _ = fix_and_explain(segment, config) out.append(fixed_segment) pos = textbreak - return ''.join(out) + return "".join(out) -def fix_and_explain( - text: str, - config: Optional[TextFixerConfig] = None -) -> (str, list): +def fix_and_explain(text: str, config: Optional[TextFixerConfig] = None) -> (str, list): """ Apply fixes to text in a single chunk, and also produce an explanation of what was fixed. @@ -312,7 +292,7 @@ def fix_and_explain( raise UnicodeError(fixes.BYTES_ERROR_TEXT) fix_entities = config.unescape_html - if fix_entities == 'auto' and '<' in text: + if fix_entities == "auto" and "<" in text: fix_entities = False steps = [] @@ -320,15 +300,20 @@ def fix_and_explain( origtext = text if fix_entities: - text = _try_fix('unescape_html', text, steps) - + text = _try_fix("unescape_html", text, steps) + if config.fix_encoding: text, encoding_steps = fix_encoding_and_explain(text, config) steps.extend(encoding_steps) for fixer in [ - 'fix_c1_controls', 'fix_latin_ligatures', 'fix_character_width', - 'uncurl_quotes', 'fix_line_breaks', 'fix_surrogates', 'remove_control_chars' + "fix_c1_controls", + "fix_latin_ligatures", + "fix_character_width", + "uncurl_quotes", + "fix_line_breaks", + "fix_surrogates", + "remove_control_chars", ]: if getattr(config, fixer): text = _try_fix(fixer, text, steps) @@ -338,8 +323,8 @@ def fix_and_explain( fixed = unicodedata.normalize(config.normalization, text) if fixed != text: text = fixed - steps.append(('normalize', config.normalization)) - + steps.append(("normalize", config.normalization)) + if text == origtext: return text, steps @@ -373,7 +358,7 @@ def _fix_encoding_one_step_and_explain(text, config): # The first plan is to return ASCII text unchanged, as well as text # that doesn't look like it contains mojibake - if chardata.possible_encoding(text, 'ascii') or not is_bad(text): + if chardata.possible_encoding(text, "ascii") or not is_bad(text): return text, [] # As we go through the next step, remember the possible encodings @@ -388,32 +373,34 @@ def _fix_encoding_one_step_and_explain(text, config): if chardata.possible_encoding(text, encoding): possible_1byte_encodings.append(encoding) encoded_bytes = text.encode(encoding) - encode_step = ('encode', encoding) + encode_step = ("encode", encoding) transcode_steps = [] # Now, find out if it's UTF-8 (or close enough). Otherwise, # remember the encoding for later. try: - decoding = 'utf-8' + decoding = "utf-8" # Check encoded_bytes for sequences that would be UTF-8, # except they have b' ' where b'\xa0' would belong. - if config.restore_byte_a0 and chardata.ALTERED_UTF8_RE.search(encoded_bytes): + if config.restore_byte_a0 and chardata.ALTERED_UTF8_RE.search( + encoded_bytes + ): replaced_bytes = fixes.restore_byte_a0(encoded_bytes) if replaced_bytes != encoded_bytes: - transcode_steps.append(('transcode', 'restore_byte_a0')) + transcode_steps.append(("transcode", "restore_byte_a0")) encoded_bytes = replaced_bytes # Replace sequences where information has been lost - if config.replace_lossy_sequences and encoding.startswith('sloppy'): + if config.replace_lossy_sequences and encoding.startswith("sloppy"): replaced_bytes = fixes.replace_lossy_sequences(encoded_bytes) if replaced_bytes != encoded_bytes: - transcode_steps.append(('transcode', 'replace_lossy_sequences')) + transcode_steps.append(("transcode", "replace_lossy_sequences")) encoded_bytes = replaced_bytes - if 0xed in encoded_bytes or 0xc0 in encoded_bytes: - decoding = 'utf-8-variants' + if 0xED in encoded_bytes or 0xC0 in encoded_bytes: + decoding = "utf-8-variants" - decode_step = ('decode', decoding) + decode_step = ("decode", decoding) steps = [encode_step] + transcode_steps + [decode_step] fixed = encoded_bytes.decode(decoding) return fixed, steps @@ -423,7 +410,7 @@ def _fix_encoding_one_step_and_explain(text, config): # Look for a-hat-euro sequences that remain, and fix them in isolation. if config.decode_inconsistent_utf8 and chardata.UTF8_DETECTOR_RE.search(text): - steps = [('apply', 'decode_inconsistent_utf8')] + steps = [("apply", "decode_inconsistent_utf8")] fixed = fixes.decode_inconsistent_utf8(text) if fixed != text: return fixed, steps @@ -431,8 +418,8 @@ def _fix_encoding_one_step_and_explain(text, config): # The next most likely case is that this is Latin-1 that was intended to # be read as Windows-1252, because those two encodings in particular are # easily confused. - if 'latin-1' in possible_1byte_encodings: - if 'windows-1252' in possible_1byte_encodings: + if "latin-1" in possible_1byte_encodings: + if "windows-1252" in possible_1byte_encodings: # This text is in the intersection of Latin-1 and # Windows-1252, so it's probably legit. return text, [] @@ -441,19 +428,19 @@ def _fix_encoding_one_step_and_explain(text, config): # not in Windows-1252. Those are C1 control characters. Nobody # wants those. Assume they were meant to be Windows-1252. try: - fixed = text.encode('latin-1').decode('windows-1252') + fixed = text.encode("latin-1").decode("windows-1252") if fixed != text: - steps = [('encode', 'latin-1'), ('decode', 'windows-1252')] + steps = [("encode", "latin-1"), ("decode", "windows-1252")] return fixed, steps except UnicodeDecodeError: pass # Fix individual characters of Latin-1 with a less satisfying explanation if config.fix_c1_controls and chardata.C1_CONTROL_RE.search(text): - steps = [('transcode', 'fix_c1_controls')] + steps = [("transcode", "fix_c1_controls")] fixed = fixes.fix_c1_controls(text) return fixed, steps - + # The cases that remain are mixups between two different single-byte # encodings, and not the common case of Latin-1 vs. Windows-1252. # @@ -472,7 +459,6 @@ def fix_encoding(text, config=None, **kwargs): return fixed - # Some alternate names for the main functions ftfy = fix_text @@ -481,7 +467,7 @@ def fix_text_segment(text, config=None, **kwargs): warnings.warn( "`fix_text_segment()` is deprecated as of ftfy 6.0. " "Use `fix_and_explain()` instead.", - DeprecationWarning + DeprecationWarning, ) if config is None: @@ -491,11 +477,7 @@ def fix_text_segment(text, config=None, **kwargs): return fixed -def fix_file( - input_file, - encoding=None, - config=None -): +def fix_file(input_file, encoding=None, config=None): """ Fix text that is found in a file. @@ -512,9 +494,9 @@ def fix_file( line, encoding = guess_bytes(line) else: line = line.decode(encoding) - if config.unescape_html == 'auto' and '<' in line: + if config.unescape_html == "auto" and "<" in line: config = config._replace(unescape_html=False) - + fixed_line, _explan = fix_and_explain(line, config) yield fixed_line @@ -553,12 +535,12 @@ def guess_bytes(bstring): "bytes to guess_bytes, not Unicode." ) - if bstring.startswith(b'\xfe\xff') or bstring.startswith(b'\xff\xfe'): - return bstring.decode('utf-16'), 'utf-16' + if bstring.startswith(b"\xfe\xff") or bstring.startswith(b"\xff\xfe"): + return bstring.decode("utf-16"), "utf-16" byteset = set(bstring) try: - if 0xed in byteset or 0xc0 in byteset: + if 0xED in byteset or 0xC0 in byteset: # Byte 0xed can be used to encode a range of codepoints that # are UTF-16 surrogates. UTF-8 does not use UTF-16 surrogates, # so when we see 0xed, it's very likely we're being asked to @@ -580,9 +562,9 @@ def guess_bytes(bstring): # # The 'utf-8-variants' decoder can handle both of these cases, as # well as standard UTF-8, at the cost of a bit of speed. - return bstring.decode('utf-8-variants'), 'utf-8-variants' + return bstring.decode("utf-8-variants"), "utf-8-variants" else: - return bstring.decode('utf-8'), 'utf-8' + return bstring.decode("utf-8"), "utf-8" except UnicodeDecodeError: pass @@ -591,8 +573,8 @@ def guess_bytes(bstring): # (The U+81 U+30 sequence would appear, for example, in the GB18030 encoding # of UTF-8 mojibake.) try: - if b'\xc8\xcb' in bstring or b'\xc8\xd5' in bstring or b'\x81\x30' in bstring: - return bstring.decode('gb18030'), 'gb18030' + if b"\xc8\xcb" in bstring or b"\xc8\xd5" in bstring or b"\x81\x30" in bstring: + return bstring.decode("gb18030"), "gb18030" except UnicodeDecodeError: pass @@ -601,22 +583,22 @@ def guess_bytes(bstring): # of UTF-8 and GB18030 try: if (0x81 in byteset) + (0x82 in byteset) + (0x83 in byteset) >= 2: - return bstring.decode('shift-jis'), 'shift-jis' + return bstring.decode("shift-jis"), "shift-jis" except UnicodeDecodeError: pass # decode Big5 text that contains either '人' or '日' try: - if b'\xa4\x48' in bstring or b'\xa4\xe9' in bstring: - return bstring.decode('big5'), 'big5' + if b"\xa4\x48" in bstring or b"\xa4\xe9" in bstring: + return bstring.decode("big5"), "big5" except UnicodeDecodeError: pass - if 0x0d in byteset and 0x0a not in byteset: + if 0x0D in byteset and 0x0A not in byteset: # Files that contain CR and not LF are likely to be MacRoman. - return bstring.decode('macroman'), 'macroman' + return bstring.decode("macroman"), "macroman" else: - return bstring.decode('sloppy-windows-1252'), 'sloppy-windows-1252' + return bstring.decode("sloppy-windows-1252"), "sloppy-windows-1252" def apply_plan(text, plan): @@ -637,11 +619,11 @@ def apply_plan(text, plan): """ obj = text for operation, encoding in plan: - if operation == 'encode': + if operation == "encode": obj = obj.encode(encoding) - elif operation == 'decode': + elif operation == "decode": obj = obj.decode(encoding) - elif operation == 'transcode' or operation == 'apply': + elif operation == "transcode" or operation == "apply": if encoding in FIXERS: obj = FIXERS[encoding](obj) else: @@ -678,12 +660,12 @@ def explain_unicode(text): if char.isprintable(): display = char else: - display = char.encode('unicode-escape').decode('ascii') + display = char.encode("unicode-escape").decode("ascii") print( - 'U+{code:04X} {display} [{category}] {name}'.format( + "U+{code:04X} {display} [{category}] {name}".format( display=display_ljust(display, 7), code=ord(char), category=unicodedata.category(char), - name=unicodedata.name(char, ''), + name=unicodedata.name(char, ""), ) ) From 6dd8fe319f8f3657ce126bcee20e276c0b78336d Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 25 Mar 2021 17:58:21 -0400 Subject: [PATCH 466/502] remove CJK encodings from `guess_bytes` until we're more ready --- ftfy/__init__.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 20aca377..a7b4891d 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -523,9 +523,6 @@ def guess_bytes(bstring): - MacRoman, because Microsoft Office thinks it's still a thing, and it can be distinguished by its line breaks. (If there are no line breaks in the string, though, you're out of luck.) - - Shift-JIS, the most common encoding of Japanese. - - GB18030, a standardized encoding of Simplified Chinese. - - Big5, a standardized encoding of Traditional Chinese. - "sloppy-windows-1252", the Latin-1-like encoding that is the most common single-byte encoding. """ @@ -568,32 +565,6 @@ def guess_bytes(bstring): except UnicodeDecodeError: pass - # decode GB18030 text that contains either '人' or '日', or the sequence - # that introduces many foreign Unicode characters, U+81 U+30. - # (The U+81 U+30 sequence would appear, for example, in the GB18030 encoding - # of UTF-8 mojibake.) - try: - if b"\xc8\xcb" in bstring or b"\xc8\xd5" in bstring or b"\x81\x30" in bstring: - return bstring.decode("gb18030"), "gb18030" - except UnicodeDecodeError: - pass - - # decode Shift-JIS text that contains at least two of - # [punctuation, hiragana, katakana], using bytes that are very uncommon outside - # of UTF-8 and GB18030 - try: - if (0x81 in byteset) + (0x82 in byteset) + (0x83 in byteset) >= 2: - return bstring.decode("shift-jis"), "shift-jis" - except UnicodeDecodeError: - pass - - # decode Big5 text that contains either '人' or '日' - try: - if b"\xa4\x48" in bstring or b"\xa4\xe9" in bstring: - return bstring.decode("big5"), "big5" - except UnicodeDecodeError: - pass - if 0x0D in byteset and 0x0A not in byteset: # Files that contain CR and not LF are likely to be MacRoman. return bstring.decode("macroman"), "macroman" From d99c33a81488ebfbbe754009901e268f2a8ce292 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 26 Mar 2021 12:53:57 -0400 Subject: [PATCH 467/502] more tests, fix warnings --- ftfy/__init__.py | 6 ------ ftfy/fixes.py | 2 +- tests/test_characters.py | 3 ++- tests/test_examples_in_json.py | 29 +++++++++++++++++++++++++---- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index a7b4891d..84a1c13f 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -464,12 +464,6 @@ def fix_encoding(text, config=None, **kwargs): def fix_text_segment(text, config=None, **kwargs): - warnings.warn( - "`fix_text_segment()` is deprecated as of ftfy 6.0. " - "Use `fix_and_explain()` instead.", - DeprecationWarning, - ) - if config is None: config = TextFixerConfig() config = config._replace(**kwargs) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index d329ecf7..a9fccb8e 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -482,7 +482,7 @@ def fix_embedded_mojibake(match): # Require the match to be shorter, so that this doesn't recurse infinitely if len(substr) < len(text) and is_bad(substr): - return fix_encoding(substr) + return ftfy.fix_encoding(substr) else: return substr diff --git a/tests/test_characters.py b/tests/test_characters.py index 69c73946..18af3bfe 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -1,5 +1,6 @@ +from ftfy import fix_encoding, fix_encoding_and_explain, apply_plan from ftfy.fixes import ( - fix_encoding, fix_encoding_and_explain, apply_plan, possible_encoding, + possible_encoding, remove_control_chars, fix_surrogates ) from ftfy.badness import badness diff --git a/tests/test_examples_in_json.py b/tests/test_examples_in_json.py index 0a5a104f..7ccde363 100644 --- a/tests/test_examples_in_json.py +++ b/tests/test_examples_in_json.py @@ -23,8 +23,7 @@ If missing, it will be considered to be the same as "fixed". - "comment": possibly-enlightening commentary on the test case. """ -from ftfy import fix_text -from ftfy.fixes import fix_encoding_and_explain, apply_plan +from ftfy import fix_text, fix_and_explain, fix_encoding_and_explain, apply_plan import json import os import pytest @@ -49,11 +48,33 @@ def test_json_example(test_case): orig = test_case['original'] fixed = test_case['fixed'] - # Make sure that the fix_encoding step outputs a plan that we can - # successfully run to reproduce its result + # Make sure that we can fix the text as intended + assert fix_text(orig) == fixed + + # Make sure that fix_and_explain outputs a plan that we can successfully + # run to reproduce its result + fixed_output, plan = fix_and_explain(orig) + assert apply_plan(orig, plan) == fixed_output + + # Do the same for fix_encoding_and_explain encoding_fix, plan = fix_encoding_and_explain(orig) assert apply_plan(orig, plan) == encoding_fix + # Ask for the encoding fix a different way, by disabling all the other steps + # in the config object + assert fix_text( + orig, + unescape_html=False, + remove_terminal_escapes=False, + fix_character_width=False, + fix_latin_ligatures=False, + uncurl_quotes=False, + fix_line_breaks=False, + fix_surrogates=False, + remove_control_chars=False, + normalization=None + ) == encoding_fix + # Make sure we can decode the text as intended assert fix_text(orig) == fixed assert encoding_fix == test_case.get('fixed-encoding', fixed) From e5ae51b08ab91d26001dc9c57113cf8c3f3a8875 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 26 Mar 2021 17:32:34 -0400 Subject: [PATCH 468/502] Clean up docstrings --- ftfy/__init__.py | 421 +++++++++++++++++++++++++-------------------- ftfy/badness.py | 1 - ftfy/build_data.py | 139 --------------- ftfy/chardata.py | 1 - ftfy/fixes.py | 5 +- setup.cfg | 3 + 6 files changed, 240 insertions(+), 330 deletions(-) delete mode 100644 ftfy/build_data.py diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 84a1c13f..5eb5dbec 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -6,7 +6,6 @@ """ import unicodedata -import warnings from collections import namedtuple from typing import Optional @@ -24,63 +23,145 @@ CONFIG_DEFAULTS = { - # Replace HTML entities, but not in strings where a < character - # has appeared literally. Those strings are assumed to be HTML, - # so entities should be left alone. "unescape_html": "auto", - # Remove "ANSI" terminal escapes, such as for changing the color - # of text in a terminal. "remove_terminal_escapes": True, - # Detect mojibake, and attempt to decode the entire string in a - # different encoding when it's detected. "fix_encoding": True, - # When fixing mojibake, allow a literal space to be interpreted - # as a non-breaking space, which would have been byte A0 in many - # encodings. "restore_byte_a0": True, - # Detect mojibake that has been partially replaced by the characters - # '�' or '?', and replace the detected sequences with '�'. "replace_lossy_sequences": True, - # Detect certain kinds of mojibake even when it's not consistent - # across the entire string. Replace sufficiently clear sequences - # of UTF-8 mojibake with the characters they should have been. "decode_inconsistent_utf8": True, - # Replace C1 control characters with their Windows-1252 equivalents, - # like HTML5 does. "fix_c1_controls": True, - # Replace common Latin-alphabet ligatures, such as 'fi', with the - # letters they're made of. "fix_latin_ligatures": True, - # Replace fullwidth Latin characters and halfwidth Katakana with - # their more standard widths. "fix_character_width": True, - # Replace curly quotes with straight quotes. "uncurl_quotes": True, - # Replace various forms of line breaks with the standard Unix line - # break, '\n'. "fix_line_breaks": True, - # Replace sequences of UTF-16 surrogates with the character they were - # meant to encode. "fix_surrogates": True, - # Remove control characters that have no displayed effect on text. "remove_control_chars": True, - # Change 'normalization' to 'NFKC' to apply Unicode compatibility - # conversions. In some cases, this will change the meaning of text. - # - # You can also set it to None to apply no normalization, including - # leaving all combining characters separate. "normalization": "NFC", - # The maximum length of line that should be fixed by ftfy without - # breaking it up into smaller strings. "max_decode_length": 1000000, - # Set 'explain' to False to not compute explanations, possibly saving - # time. The functions that return explanations will return None. "explain": True, } TextFixerConfig = namedtuple( "TextFixerConfig", CONFIG_DEFAULTS.keys(), defaults=CONFIG_DEFAULTS.values() ) +TextFixerConfig.__doc__ = r""" +A TextFixerConfig object stores configuration options for ftfy. + +It's implemented as a namedtuple with defaults, so you can instantiate +it by providing the values to change from their defaults as keyword arguments. +For example, to disable 'unescape_html' and keep the rest of the defaults: + + TextFixerConfig(unescape_html=False) + +Here are the options and their default values: + +- `unescape_html`: "auto" + + Configures whether to replace HTML entities such as & with the character + they represent. "auto" says to do this by default, but disable it when a + literal < character appears, indicating that the input is actual HTML and + entities should be preserved. The value can be True, to always enable this + fixer, or False, to always disable it. + +- `remove_terminal_escapes`: True + + Removes "ANSI" terminal escapes, such as for changing the color of text in a + terminal window. + +- `fix_encoding`: True + + Detect mojibake and attempt to fix it by decoding the text in a different + encoding standard. + + The following four options affect `fix_encoding` works, and do nothing if + `fix_encoding` is False: + + - `restore_byte_a0`: True + + Allow a literal space (U+20) to be interpreted as a non-breaking space + (U+A0) when that would make it part of a fixable mojibake string. + + Because spaces are very common characters, this could lead to false + positives, but we try to apply it only when there's strong evidence for + mojibake. Disabling `restore_byte_a0` is safer from false positives, + but creates false negatives. + + - `replace_lossy_sequences`: True + + Detect mojibake that has been partially replaced by the characters + '�' or '?'. If the mojibake could be decoded otherwise, replace the + detected sequence with '�'. + + - `decode_inconsistent_utf8`: True + + When we see sequences that distinctly look like UTF-8 mojibake, but + there's no consistent way to reinterpret the string in a new encoding, + replace the mojibake with the appropriate UTF-8 characters anyway. + + This helps to decode strings that are concatenated from different + encodings. + + - `fix_c1_controls`: True + + Replace C1 control characters (the useless characters U+80 - U+9B that + come from Latin-1) with their Windows-1252 equivalents, like HTML5 does, + even if the whole string doesn't decode as Latin-1. + +- `fix_latin_ligatures`: True + + Replace common Latin-alphabet ligatures, such as 'fi', with the + letters they're made of. + +- `fix_character_width`: True + + Replace fullwidth Latin characters and halfwidth Katakana with + their more standard widths. + +- `uncurl_quotes`: True + + Replace curly quotes with straight quotes. + +- `fix_line_breaks`: True + + Replace various forms of line breaks with the standard Unix line + break, '\n'. + +- `fix_surrogates`: True + + Replace sequences of UTF-16 surrogate codepoints with the character + they were meant to encode. This fixes text that was decoded with the + obsolete UCS-2 standard, and allows it to support high-numbered + codepoints such as emoji. + +- `remove_control_chars`: True + + Remove certain control characters that have no displayed effect on text. + +- `normalization`: "NFC" + + Choose what kind of Unicode normalization is applied. Usually, we apply + NFC normalization, so that letters followed by combining characters become + single combined characters. + + Changing this to "NFKC" applies more compatibility conversions, such as + replacing the 'micro sign' with a standard Greek lowercase mu, which looks + identical. However, some NFKC normalizations change the meaning of text, + such as converting "10³" to "103". + + `normalization` can be None, to apply no normalization. + +- `max_decode_length`: 1_000_000 + + The maximum size of "segment" that ftfy will try to fix all at once. + +- `explain`: True + Whether to compute 'explanations', lists describing what ftfy changed. + When this is False, the explanation will be None, and the code that + builds the explanation will be skipped, possibly saving time. + + Functions that accept TextFixerConfig and don't return an explanation + will automatically set `explain` to False. +""" FIXERS = { "unescape_html": fixes.unescape_html, @@ -100,164 +181,105 @@ BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode. -ftfy is designed to fix problems that were introduced by handling Unicode -incorrectly. It might be able to fix the bytes you just handed it, but the -fact that you just gave a pile of bytes to a function that fixes text means -that your code is *also* handling Unicode incorrectly. +ftfy is designed to fix problems with text. Treating bytes like they're +interchangeable with Unicode text is usually something that introduces +problems with text. -ftfy takes Unicode text as input. You should take these bytes and decode -them from the encoding you think they are in. If you're not sure what encoding -they're in: +You should first decode these bytes from the encoding you think they're in. +If you're not sure what encoding they're in: - First, try to find out. 'utf-8' is a good assumption. - If the encoding is simply unknowable, try running your bytes through ftfy.guess_bytes. As the name implies, this may not always be accurate. -If you're confused by this, please read the Python Unicode HOWTO: +For more information on the distinction between bytes and text, read the +Python Unicode HOWTO: http://docs.python.org/3/howto/unicode.html """ -def _try_fix(fixer_name: str, text: str, steps: list) -> str: - fixer = FIXERS[fixer_name] - fixed = fixer(text) - if fixed != text: - steps.append(("apply", fixer_name)) - return fixed +def _try_fix( + fixer_name: str, text: str, config: TextFixerConfig, steps: Optional[list] +) -> str: + """ + A helper function used across several 'fixer' steps, deciding whether to + apply the fix and whether to record the fix in `steps`. + """ + if getattr(config, fixer_name): + fixer = FIXERS[fixer_name] + fixed = fixer(text) + if steps is not None and fixed != text: + steps.append(("apply", fixer_name)) + return fixed + else: + return text def fix_text(text: str, config: Optional[TextFixerConfig] = None, **kwargs) -> str: r""" Given Unicode text as input, fix inconsistencies and glitches in it, - such as mojibake. - - FIXME: clean up this docstring + such as mojibake (text that was decoded in the wrong encoding). Let's start with some examples: - >>> print(fix_text('ünicode')) - ünicode - - >>> print(fix_text('Broken text… it’s flubberific!', - ... normalization='NFKC')) - Broken text... it's flubberific! + >>> fix_text('✔ No problems') + '✔ No problems' - >>> print(fix_text('HTML entities <3')) - HTML entities <3 - - >>> print(fix_text('HTML entities <3')) - HTML entities <3 + >>> fix_text('ünicode') + 'ünicode' >>> print(fix_text("¯\\_(ã\x83\x84)_/¯")) ¯\_(ツ)_/¯ - >>> # This example string starts with a byte-order mark, even if - >>> # you can't see it on the Web. - >>> print(fix_text('\ufeffParty like\nit’s 1999!')) - Party like - it's 1999! - - >>> print(fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ')) - LOUD NOISES - - >>> len(fix_text('fi' * 100000)) - 200000 - - >>> len(fix_text('')) - 0 - - Based on the options you provide, ftfy applies these steps in order: - - - If `remove_terminal_escapes` is True, remove sequences of bytes that are - instructions for Unix terminals, such as the codes that make text appear - in different colors. - - - If `fix_encoding` is True, look for common mistakes that come from - encoding or decoding Unicode text incorrectly, and fix them if they are - reasonably fixable. See `fixes.fix_encoding` for details. - - - If `fix_entities` is True, replace HTML entities with their equivalent - characters. If it's "auto" (the default), then consider replacing HTML - entities, but don't do so in text where you have seen a pair of actual - angle brackets (that's probably actually HTML and you shouldn't mess - with the entities). - - - If `uncurl_quotes` is True, replace various curly quotation marks with - plain-ASCII straight quotes. - - - If `fix_latin_ligatures` is True, then ligatures made of Latin letters, - such as `fi`, will be separated into individual letters. These ligatures - are usually not meaningful outside of font rendering, and often represent - copy-and-paste errors. - - - If `fix_character_width` is True, half-width and full-width characters - will be replaced by their standard-width form. - - - If `fix_line_breaks` is true, convert all line breaks to Unix style - (CRLF and CR line breaks become LF line breaks). + >>> fix_text('Broken text… it’s flubberific!', + ... normalization='NFKC') + "Broken text... it's flubberific!" - - If `fix_surrogates` is true, ensure that there are no UTF-16 surrogates - in the resulting string, by converting them to the correct characters - when they're appropriately paired, or replacing them with \ufffd - otherwise. + >>> fix_text('HTML entities <3') + 'HTML entities <3' - - If `remove_control_chars` is true, remove control characters that - are not suitable for use in text. This includes most of the ASCII control - characters, plus some Unicode controls such as the byte order mark - (U+FEFF). Useful control characters, such as Tab, Line Feed, and - bidirectional marks, are left as they are. + >>> fix_text('HTML entities <3') + 'HTML entities <3' - - If `remove_bom` is True, remove the Byte-Order Mark at the start of the - string if it exists. (This is largely redundant, because it's a special - case of `remove_control_characters`. This option will become deprecated - in a later version.) + >>> fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ') + 'LOUD NOISES' - - If `normalization` is not None, apply the specified form of Unicode - normalization, which can be one of 'NFC', 'NFKC', 'NFD', and 'NFKD'. + ftfy applies a number of different fixes to the text, and can accept + configuration to select which fixes to apply. - - The default normalization, NFC, combines characters and diacritics that - are written using separate code points, such as converting "e" plus an - acute accent modifier into "é", or converting "ka" (か) plus a dakuten - into the single character "ga" (が). Unicode can be converted to NFC - form without any change in its meaning. + The configuration takes the form of a :class:`TextFixerConfig` object, + and you can see a description of the options in that class's docstring + or in the full documentation at ftfy.readthedocs.org. - - If you ask for NFKC normalization, it will apply additional - normalizations that can change the meanings of characters. For example, - ellipsis characters will be replaced with three periods, all ligatures - will be replaced with the individual characters that make them up, - and characters that differ in font style will be converted to the same - character. + For convenience and backward compatibility, the configuration can also + take the form of keyword arguments, which will set the equivalently-named + fields of the TextFixerConfig object. - - If anything was changed, repeat all the steps, so that the function is - idempotent. "&amp;" will become "&", for example, not "&". + For example, here are two ways to fix text but skip the "uncurl_quotes" + step: - `fix_text` will work one line at a time, with the possibility that some - lines are in different encodings, allowing it to fix text that has been - concatenated together from different sources. + fix_text(text, TextFixerConfig(uncurl_quotes=False)) + fix_text(text, uncurl_quotes=False) - When it encounters lines longer than `max_decode_length` (1 million - codepoints by default), it will not run the `fix_encoding` step, to avoid - unbounded slowdowns. - - If you're certain that any decoding errors in the text would have affected - the entire text in the same way, and you don't mind operations that scale - with the length of the text, you can use `fix_text_segment` directly to - fix the whole string in one batch. + This function fixes text in independent segments, which are usually lines + of text, or arbitrarily broken up every 1 million codepoints (configurable + with `config.max_decode_length`) if there aren't enough line breaks. The + bound on segment lengths helps to avoid unbounded slowdowns. ftfy can also provide an 'explanation', a list of transformations it applied - to the text that would fix more text like it. - - However, this function doesn't provide explanations, because it can apply - different fixes to different lines of text, and one explanation may not - suffice. + to the text that would fix more text like it. This function doesn't provide + explanations (because there may be different fixes for different segments + of text). - To get an explanation, use the :func:`fix_and_explain()` function. + To get an explanation, use the :func:`fix_and_explain()` function, which + fixes the string in one segment and explains what it fixed. """ if config is None: config = TextFixerConfig() config = config._replace(**kwargs) + config = config._replace(explain=False) if isinstance(text, bytes): raise UnicodeError(fixes.BYTES_ERROR_TEXT) @@ -279,32 +301,42 @@ def fix_text(text: str, config: Optional[TextFixerConfig] = None, **kwargs) -> s return "".join(out) -def fix_and_explain(text: str, config: Optional[TextFixerConfig] = None) -> (str, list): +def fix_and_explain( + text: str, config: Optional[TextFixerConfig] = None, **kwargs +) -> (str, Optional[list]): """ - Apply fixes to text in a single chunk, and also produce an explanation + Fix text as a single segment, returning the fixed text and an explanation of what was fixed. - Returns the fixed text, and a list explaining what was fixed. + The explanation is a list of steps that can be applied with + :func:`apply_plan`, or if config.explain is False, it will be None. """ if config is None: config = TextFixerConfig() if isinstance(text, bytes): raise UnicodeError(fixes.BYTES_ERROR_TEXT) + config = config._replace(**kwargs) - fix_entities = config.unescape_html - if fix_entities == "auto" and "<" in text: - fix_entities = False + if config.unescape_html == "auto" and "<" in text: + config = config._replace(unescape_html=False) + + if config.explain: + steps = [] + else: + # If explanations aren't desired, `steps` will be None + steps = None - steps = [] while True: origtext = text - if fix_entities: - text = _try_fix("unescape_html", text, steps) + text = _try_fix("unescape_html", text, config, steps) if config.fix_encoding: - text, encoding_steps = fix_encoding_and_explain(text, config) - steps.extend(encoding_steps) + if steps is None: + text = fix_encoding(text) + else: + text, encoding_steps = fix_encoding_and_explain(text, config) + steps.extend(encoding_steps) for fixer in [ "fix_c1_controls", @@ -315,30 +347,37 @@ def fix_and_explain(text: str, config: Optional[TextFixerConfig] = None) -> (str "fix_surrogates", "remove_control_chars", ]: - if getattr(config, fixer): - text = _try_fix(fixer, text, steps) - # TODO: backward compatibility for remove_bom + text = _try_fix(fixer, text, config, steps) if config.normalization is not None: fixed = unicodedata.normalize(config.normalization, text) - if fixed != text: - text = fixed + if steps is not None and fixed != text: steps.append(("normalize", config.normalization)) + text = fixed if text == origtext: return text, steps -def fix_encoding_and_explain(text, config=None): +def fix_encoding_and_explain( + text: str, config: Optional[TextFixerConfig] = None +) -> (str, list): + """ + Apply the steps of ftfy that detect mojibake and fix it. Returns the fixed + text and a list explaining what was fixed. + + This includes fixing text by encoding and decoding it in different encodings, + as well as the subordinate fixes `restore_byte_a0`, `replace_lossy_sequences`, + `decode_inconsistent_utf8`, and `fix_c1_controls`. + """ if config is None: config = TextFixerConfig() if isinstance(text, bytes): raise UnicodeError(fixes.BYTES_ERROR_TEXT) if not config.fix_encoding: - raise ValueError( - "It doesn't make sense to run fix_encoding_and_explain " - "with fix_encoding=False" - ) + # A weird trivial case: we're asked to fix the encoding, but skip + # fixing the encoding + return text, [] plan_so_far = [] while True: @@ -349,7 +388,12 @@ def fix_encoding_and_explain(text, config=None): return text, plan_so_far -def _fix_encoding_one_step_and_explain(text, config): +def _fix_encoding_one_step_and_explain( + text: str, config: TextFixerConfig +) -> (str, list): + """ + Perform one step of fixing the encoding of text. + """ if config is None: config = TextFixerConfig() @@ -444,18 +488,21 @@ def _fix_encoding_one_step_and_explain(text, config): # The cases that remain are mixups between two different single-byte # encodings, and not the common case of Latin-1 vs. Windows-1252. # - # These cases may be unsolvable without adding false positives, though - # I have vague ideas about how to optionally address them in the future. - - # Return the text unchanged; the plan is empty. + # With the new heuristic in 6.0, it's possible that we're closer to solving + # these in some cases. It would require a lot of testing and tuning, though. + # For now, we leave the text unchanged in these cases. return text, [] -def fix_encoding(text, config=None, **kwargs): +def fix_encoding(text: str, config: TextFixerConfig = None, **kwargs): + """ + Apply just the encoding-fixing steps of ftfy to this text. Returns the + fixed text, discarding the explanation. + """ if config is None: config = TextFixerConfig() config = config._replace(**kwargs) - fixed, explan = fix_encoding_and_explain(text, config) + fixed, _explan = fix_encoding_and_explain(text, config) return fixed @@ -463,11 +510,15 @@ def fix_encoding(text, config=None, **kwargs): ftfy = fix_text -def fix_text_segment(text, config=None, **kwargs): +def fix_text_segment(text: str, config: TextFixerConfig = None, **kwargs): + """ + Fix text as a single segment, with a consistent sequence of steps that + are applied to fix the text. Discard the explanation. + """ if config is None: - config = TextFixerConfig() + config = TextFixerConfig(explain=False) config = config._replace(**kwargs) - fixed, explan = fix_and_explain(text, config) + fixed, _explan = fix_and_explain(text, config) return fixed diff --git a/ftfy/badness.py b/ftfy/badness.py index 18bb4a9a..1a09a1d1 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -2,7 +2,6 @@ Heuristics to detect likely mojibake. """ -import re import warnings from ftfy import chardata diff --git a/ftfy/build_data.py b/ftfy/build_data.py deleted file mode 100644 index f9100ae7..00000000 --- a/ftfy/build_data.py +++ /dev/null @@ -1,139 +0,0 @@ -""" -A script to make the char_classes.py file. - -This never needs to run in normal usage. It needs to be run if the character -classes we care about change, or if a new version of Python supports a new -Unicode standard and we want it to affect our string decoding. - -The file that we generate is based on Unicode 12.1, as supported by Python 3.8. -You can certainly use it in earlier versions. This simply makes sure that we -get consistent results from running ftfy on different versions of Python. - -The file will be written to the current directory. -""" -import base64 -import sys -import unicodedata -import zlib - -# L = Latin capital letter -# l = Latin lowercase letter -# A = Non-latin capital or title-case letter -# a = Non-latin lowercase letter -# C = Non-cased letter (Lo) -# X = Control character (Cc) -# m = Letter modifier (Lm) -# M = Mark (Mc, Me, Mn) -# N = Miscellaneous numbers (No) -# P = Private use (Co) -# 1 = Math symbol (Sm) or currency symbol (Sc) -# 2 = Symbol modifier (Sk) -# 3 = Other symbol (So) -# S = UTF-16 surrogate -# _ = Unassigned character -# = Whitespace -# o = Other - - -def make_char_data_file(do_it_anyway=False): - """ - Build the compressed data file 'char_classes.dat' and write it to the - current directory. - - If you run this, run it in Python 3.8 or later. It will run in earlier - versions, but you won't get the Unicode 12 standard, leading to - inconsistent behavior. - - To protect against this, running this in the wrong version of Python will - raise an error unless you pass `do_it_anyway=True`. - """ - if sys.version_info < (3, 9) and not do_it_anyway: - raise RuntimeError("This function should be run in Python 3.9 or later.") - - cclasses = [None] * 0x110000 - for codepoint in range(0x0, 0x110000): - char = chr(codepoint) - category = unicodedata.category(char) - - if (0x250 <= codepoint < 0x300) and char != 'ə': - # IPA symbols and modifiers. - # - # This category excludes the schwa (ə), which is used as a normal - # Latin letter in some languages. - cclasses[codepoint] = 'i' - elif category.startswith('L'): # letters - if unicodedata.name(char, '').startswith('LATIN'): - if category == 'Lu': - cclasses[codepoint] = 'L' - else: - cclasses[codepoint] = 'l' - else: - if category == 'Lu' or category == 'Lt': - cclasses[codepoint] = 'A' - elif category == 'Ll': - cclasses[codepoint] = 'a' - elif category == 'Lo': - cclasses[codepoint] = 'C' - elif category == 'Lm': - cclasses[codepoint] = 'm' - else: - raise ValueError('got some weird kind of letter') - elif 0xfe00 <= codepoint <= 0xfe0f or 0x1f3fb <= codepoint <= 0x1f3ff: - # Variation selectors and skin-tone modifiers have the category - # of non-spacing marks, but they act like symbols - cclasses[codepoint] = '3' - elif category.startswith('M'): # marks - cclasses[codepoint] = 'M' - elif category == 'No': - cclasses[codepoint] = 'N' - elif category == 'Sm' or category == 'Sc': - cclasses[codepoint] = '1' - elif category == 'Sk': - cclasses[codepoint] = '2' - elif category == 'So': - cclasses[codepoint] = '3' - elif category == 'Cc': - cclasses[codepoint] = 'X' - elif category == 'Cs': - cclasses[codepoint] = 'S' - elif category == 'Co': - cclasses[codepoint] = 'P' - elif category.startswith('Z'): - cclasses[codepoint] = ' ' - elif 0x1f000 <= codepoint <= 0x1ffff: - # This range is rapidly having emoji added to it. Assume that - # an unassigned codepoint in this range is just a symbol we - # don't know yet. - cclasses[codepoint] = '3' - elif category == 'Cn': - cclasses[codepoint] = '_' - else: - cclasses[codepoint] = 'o' - - # Mark whitespace control characters as whitespace - cclasses[9] = cclasses[10] = cclasses[12] = cclasses[13] = ' ' - - # Some other exceptions for characters that are more commonly used as - # punctuation or decoration than for their ostensible purpose. - # For example, tilde is not usually a "math symbol", and the accents - # `´ are much more like quotation marks than modifiers. - for char in "^~`´˝＾｀": - cclasses[ord(char)] = 'o' - - # Compress and encode this string within a Python script. - char_classes_bytes = ''.join(cclasses).encode('ascii') - encoded = base64.b64encode(zlib.compress(char_classes_bytes)) - - script = """ -# This script is automatically generated by build_data.py. -import base64 -import zlib -_compressed = {encoded!r} -CHAR_CLASS_STRING = zlib.decompress(base64.b64decode(_compressed)).decode('ascii') -""".format(encoded=encoded) - with open('char_classes.py', 'w', encoding='utf-8') as out: - print(script, file=out) - - -if __name__ == '__main__': - make_char_data_file() diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 2bf02622..6eb03145 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -7,7 +7,6 @@ import itertools import re import unicodedata -import zlib # These are the encodings we will try to fix in ftfy, in the diff --git a/ftfy/fixes.py b/ftfy/fixes.py index a9fccb8e..1a402387 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -6,14 +6,12 @@ import codecs import html import re -import unicodedata import warnings import ftfy from ftfy.chardata import ( ALTERED_UTF8_RE, C1_CONTROL_RE, - CHARMAP_ENCODINGS, CONTROL_CHARS, DOUBLE_QUOTE_RE, HTML_ENTITIES, @@ -23,10 +21,9 @@ SINGLE_QUOTE_RE, UTF8_DETECTOR_RE, WIDTH_MAP, - possible_encoding, ) -from ftfy.badness import badness, is_bad +from ftfy.badness import is_bad def fix_encoding_and_explain(text): diff --git a/setup.cfg b/setup.cfg index b7e47898..1bd30526 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [aliases] test=pytest + +[flake8] +max-line-length=100 From d65bffa762b7d870fc3d1e38f5918d80fa354c65 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 26 Mar 2021 17:39:13 -0400 Subject: [PATCH 469/502] formatting fixes, remove obsolete stuff --- ftfy/chardata.py | 549 +++++++++++++++++++-------------------- ftfy/fixes.py | 69 +++-- tests/test_characters.py | 6 +- 3 files changed, 310 insertions(+), 314 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 6eb03145..8647261a 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -12,19 +12,19 @@ # These are the encodings we will try to fix in ftfy, in the # order that they should be tried. CHARMAP_ENCODINGS = [ - 'latin-1', - 'sloppy-windows-1252', - 'sloppy-windows-1251', - 'sloppy-windows-1250', - 'sloppy-windows-1253', - 'sloppy-windows-1254', - 'iso-8859-2', - 'macroman', - 'cp437', + "latin-1", + "sloppy-windows-1252", + "sloppy-windows-1251", + "sloppy-windows-1250", + "sloppy-windows-1253", + "sloppy-windows-1254", + "iso-8859-2", + "macroman", + "cp437", ] -SINGLE_QUOTE_RE = re.compile('[\u02bc\u2018-\u201b]') -DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]') +SINGLE_QUOTE_RE = re.compile("[\u02bc\u2018-\u201b]") +DOUBLE_QUOTE_RE = re.compile("[\u201c-\u201f]") def _build_regexes(): @@ -35,13 +35,13 @@ def _build_regexes(): are between U+0000 and U+007F. """ # Define a regex that matches ASCII text. - encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')} + encoding_regexes = {"ascii": re.compile("^[\x00-\x7f]*$")} for encoding in CHARMAP_ENCODINGS: # Make a sequence of characters that bytes \x80 to \xFF decode to # in each encoding, as well as byte \x1A, which is used to represent # the replacement character � in the sloppy-* encodings. - byte_range = bytes(list(range(0x80, 0x100)) + [0x1a]) + byte_range = bytes(list(range(0x80, 0x100)) + [0x1A]) charlist = byte_range.decode(encoding) # The rest of the ASCII bytes -- bytes \x00 to \x19 and \x1B @@ -49,7 +49,7 @@ def _build_regexes(): # support, so we can just include them as ranges. This also lets us # not worry about escaping regex special characters, because all of # them are in the \x1B to \x7F range. - regex = '^[\x00-\x19\x1b-\x7f{0}]*$'.format(charlist) + regex = "^[\x00-\x19\x1b-\x7f{0}]*$".format(charlist) encoding_regexes[encoding] = re.compile(regex) return encoding_regexes @@ -63,15 +63,15 @@ def _build_html_entities(): # Add a limited set of HTML entities that we'll also decode if they've # been case-folded to uppercase, such as decoding &NTILDE; as "Ñ". for name, char in html.entities.html5.items(): - if name.endswith(';'): - entities['&' + name] = char + if name.endswith(";"): + entities["&" + name] = char # Restrict the set of characters we can attempt to decode if their # name has been uppercased. If we tried to handle all entity names, # the results would be ambiguous. if name == name.lower(): name_upper = name.upper() - entity_upper = '&' + name_upper + entity_upper = "&" + name_upper if html.unescape(entity_upper) == entity_upper: entities[entity_upper] = char.upper() return entities @@ -102,12 +102,12 @@ def _build_control_char_mapping(): for i in itertools.chain( range(0x00, 0x09), - [0x0b], - range(0x0e, 0x20), - [0x7f], - range(0x206a, 0x2070), - [0xfeff], - range(0xfff9, 0xfffd), + [0x0B], + range(0x0E, 0x20), + [0x7F], + range(0x206A, 0x2070), + [0xFEFF], + range(0xFFF9, 0xFFFD), ): control_chars[i] = None @@ -154,12 +154,12 @@ def _build_control_char_mapping(): # I've seen it once, but the text still wasn't recoverable. ALTERED_UTF8_RE = re.compile( - b'[\xc2\xc3\xc5\xce\xd0][ ]' - b'|[\xe2-\xe9\xeb\xec][ ][\x80-\xbf]' - b'|[\xe0-\xef][\x80-\xbf][ ]' - b'|[\xf0][ ][\x80-\xbf][\x80-\xbf]' - b'|[\xf0][\x80-\xbf][ ][\x80-\xbf]' - b'|[\xf0][\x80-\xbf][\x80-\xbf][ ]' + b"[\xc2\xc3\xc5\xce\xd0][ ]" + b"|[\xe2-\xe9\xeb\xec][ ][\x80-\xbf]" + b"|[\xe0-\xef][\x80-\xbf][ ]" + b"|[\xf0][ ][\x80-\xbf][\x80-\xbf]" + b"|[\xf0][\x80-\xbf][ ][\x80-\xbf]" + b"|[\xf0][\x80-\xbf][\x80-\xbf][ ]" ) @@ -171,23 +171,23 @@ def _build_control_char_mapping(): # # In some cases, we allow the ASCII '?' in place of \ufffd, but at most once per # sequence. -LOSSY_UTF8_RE = re.compile( - b'[\xc2-\xdf][\x1a]' - b'|[\xc2-\xc3][?]' - b'|\xed[\xa0-\xaf][\x1a?]\xed[\xb0-\xbf][\x1a?\x80-\xbf]' - b'|\xed[\xa0-\xaf][\x1a?\x80-\xbf]\xed[\xb0-\xbf][\x1a?]' - b'|[\xe0-\xef][\x1a?][\x1a\x80-\xbf]' - b'|[\xe0-\xef][\x1a\x80-\xbf][\x1a?]' - b'|[\xf0-\xf4][\x1a?][\x1a\x80-\xbf][\x1a\x80-\xbf]' - b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a?][\x1a\x80-\xbf]' - b'|[\xf0-\xf4][\x1a\x80-\xbf][\x1a\x80-\xbf][\x1a?]' - b'|\x1a' +LOSSY_UTF8_RE = re.compile( + b"[\xc2-\xdf][\x1a]" + b"|[\xc2-\xc3][?]" + b"|\xed[\xa0-\xaf][\x1a?]\xed[\xb0-\xbf][\x1a?\x80-\xbf]" + b"|\xed[\xa0-\xaf][\x1a?\x80-\xbf]\xed[\xb0-\xbf][\x1a?]" + b"|[\xe0-\xef][\x1a?][\x1a\x80-\xbf]" + b"|[\xe0-\xef][\x1a\x80-\xbf][\x1a?]" + b"|[\xf0-\xf4][\x1a?][\x1a\x80-\xbf][\x1a\x80-\xbf]" + b"|[\xf0-\xf4][\x1a\x80-\xbf][\x1a?][\x1a\x80-\xbf]" + b"|[\xf0-\xf4][\x1a\x80-\xbf][\x1a\x80-\xbf][\x1a?]" + b"|\x1a" ) # This regex matches C1 control characters, which occupy some of the positions # in the Latin-1 character map that Windows assigns to other characters instead. -C1_CONTROL_RE = re.compile(r'[\x80-\x9f]') +C1_CONTROL_RE = re.compile(r"[\x80-\x9f]") # A translate mapping that breaks ligatures made of Latin letters. While @@ -202,28 +202,28 @@ def _build_control_char_mapping(): # is sometimes more normalization than you want. LIGATURES = { - ord('IJ'): 'IJ', # Dutch ligatures - ord('ij'): 'ij', - ord('ʼn'): "ʼn", # Afrikaans digraph meant to avoid auto-curled quote - ord('DZ'): 'DZ', # Serbian/Croatian digraphs for Cyrillic conversion - ord('Dz'): 'Dz', - ord('dz'): 'dz', - ord('DŽ'): 'DŽ', - ord('Dž'): 'Dž', - ord('dž'): 'dž', - ord('LJ'): 'LJ', - ord('Lj'): 'Lj', - ord('lj'): 'lj', - ord('NJ'): 'NJ', - ord('Nj'): 'Nj', - ord('nj'): "nj", - ord('ff'): 'ff', # Latin typographical ligatures - ord('fi'): 'fi', - ord('fl'): 'fl', - ord('ffi'): 'ffi', - ord('ffl'): 'ffl', - ord('ſt'): 'ſt', - ord('st'): 'st', + ord("IJ"): "IJ", # Dutch ligatures + ord("ij"): "ij", + ord("ʼn"): "ʼn", # Afrikaans digraph meant to avoid auto-curled quote + ord("DZ"): "DZ", # Serbian/Croatian digraphs for Cyrillic conversion + ord("Dz"): "Dz", + ord("dz"): "dz", + ord("DŽ"): "DŽ", + ord("Dž"): "Dž", + ord("dž"): "dž", + ord("LJ"): "LJ", + ord("Lj"): "Lj", + ord("lj"): "lj", + ord("NJ"): "NJ", + ord("Nj"): "Nj", + ord("nj"): "nj", + ord("ff"): "ff", # Latin typographical ligatures + ord("fi"): "fi", + ord("fl"): "fl", + ord("ffi"): "ffi", + ord("ffl"): "ffl", + ord("ſt"): "ſt", + ord("st"): "st", } @@ -235,10 +235,10 @@ def _build_width_map(): # Though it's not listed as a fullwidth character, we'll want to convert # U+3000 IDEOGRAPHIC SPACE to U+20 SPACE on the same principle, so start # with that in the dictionary. - width_map = {0x3000: ' '} - for i in range(0xff01, 0xfff0): + width_map = {0x3000: " "} + for i in range(0xFF01, 0xFFF0): char = chr(i) - alternate = unicodedata.normalize('NFKC', char) + alternate = unicodedata.normalize("NFKC", char) if alternate != char: width_map[i] = alternate return width_map @@ -252,7 +252,7 @@ def _char_range(start, end): Get a contiguous range of characters as a string. Useful for including in a regex without making the representation _only_ usable in regexes. """ - return ''.join(chr(codept) for codept in range(start, end + 1)) + return "".join(chr(codept) for codept in range(start, end + 1)) # There are only 403 characters that occur in known UTF-8 mojibake, and we can @@ -261,215 +261,209 @@ def _char_range(start, end): MOJIBAKE_CATEGORIES = { # Characters that appear in many different contexts. Sequences that contain # them are not inherently mojibake - 'common': ( - '\N{NO-BREAK SPACE}' - '\N{SOFT HYPHEN}' - '\N{MIDDLE DOT}' - '\N{ACUTE ACCENT}' - '\N{EN DASH}' - '\N{EM DASH}' - '\N{HORIZONTAL BAR}' - '\N{HORIZONTAL ELLIPSIS}' - '\N{RIGHT SINGLE QUOTATION MARK}' + "common": ( + "\N{NO-BREAK SPACE}" + "\N{SOFT HYPHEN}" + "\N{MIDDLE DOT}" + "\N{ACUTE ACCENT}" + "\N{EN DASH}" + "\N{EM DASH}" + "\N{HORIZONTAL BAR}" + "\N{HORIZONTAL ELLIPSIS}" + "\N{RIGHT SINGLE QUOTATION MARK}" ), # the C1 control character range, which have no uses outside of mojibake anymore - 'c1': _char_range(0x80, 0x9f), + "c1": _char_range(0x80, 0x9F), # Characters that are nearly 100% used in mojibake - 'bad': ( - '\N{BROKEN BAR}' - '\N{CURRENCY SIGN}' - '\N{DIAERESIS}' - '\N{NOT SIGN}' - '\N{MACRON}' - '\N{PILCROW SIGN}' - '\N{SECTION SIGN}' - '\N{CEDILLA}' - '\N{LATIN SMALL LETTER F WITH HOOK}' - '\N{MODIFIER LETTER CIRCUMFLEX ACCENT}' # it's not a modifier - '\N{CARON}' - '\N{BREVE}' - '\N{OGONEK}' - '\N{SMALL TILDE}' - '\N{DAGGER}' - '\N{DOUBLE DAGGER}' - '\N{PER MILLE SIGN}' - '\N{REVERSED NOT SIGN}' - '\N{LOZENGE}' - '\ufffd' - + "bad": ( + "\N{BROKEN BAR}" + "\N{CURRENCY SIGN}" + "\N{DIAERESIS}" + "\N{NOT SIGN}" + "\N{MACRON}" + "\N{PILCROW SIGN}" + "\N{SECTION SIGN}" + "\N{CEDILLA}" + "\N{LATIN SMALL LETTER F WITH HOOK}" + "\N{MODIFIER LETTER CIRCUMFLEX ACCENT}" # it's not a modifier + "\N{CARON}" + "\N{BREVE}" + "\N{OGONEK}" + "\N{SMALL TILDE}" + "\N{DAGGER}" + "\N{DOUBLE DAGGER}" + "\N{PER MILLE SIGN}" + "\N{REVERSED NOT SIGN}" + "\N{LOZENGE}" + "\ufffd" # Theoretically these would appear in 'numeric' contexts, but when they # co-occur with other mojibake characters, it's not really ambiguous - '\N{FEMININE ORDINAL INDICATOR}' - '\N{MASCULINE ORDINAL INDICATOR}' - '\N{NUMERO SIGN}' + "\N{FEMININE ORDINAL INDICATOR}" + "\N{MASCULINE ORDINAL INDICATOR}" + "\N{NUMERO SIGN}" ), - 'currency': ( - '\N{CENT SIGN}' - '\N{POUND SIGN}' - '\N{YEN SIGN}' - '\N{PESETA SIGN}' - '\N{EURO SIGN}' + "currency": ( + "\N{CENT SIGN}" + "\N{POUND SIGN}" + "\N{YEN SIGN}" + "\N{PESETA SIGN}" + "\N{EURO SIGN}" ), - 'start_punctuation': ( - '\N{INVERTED EXCLAMATION MARK}' - '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}' - '\N{INVERTED QUESTION MARK}' - '\N{COPYRIGHT SIGN}' - '\N{GREEK TONOS}' - '\N{GREEK DIALYTIKA TONOS}' - '\N{LEFT SINGLE QUOTATION MARK}' - '\N{SINGLE LOW-9 QUOTATION MARK}' - '\N{LEFT DOUBLE QUOTATION MARK}' - '\N{DOUBLE LOW-9 QUOTATION MARK}' - '\N{BULLET}' - '\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}' - '\uf8ff' # OS-specific symbol, usually the Apple logo + "start_punctuation": ( + "\N{INVERTED EXCLAMATION MARK}" + "\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}" + "\N{INVERTED QUESTION MARK}" + "\N{COPYRIGHT SIGN}" + "\N{GREEK TONOS}" + "\N{GREEK DIALYTIKA TONOS}" + "\N{LEFT SINGLE QUOTATION MARK}" + "\N{SINGLE LOW-9 QUOTATION MARK}" + "\N{LEFT DOUBLE QUOTATION MARK}" + "\N{DOUBLE LOW-9 QUOTATION MARK}" + "\N{BULLET}" + "\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}" + "\uf8ff" # OS-specific symbol, usually the Apple logo ), - 'end_punctuation': ( - '\N{REGISTERED SIGN}' - '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}' - '\N{DOUBLE ACUTE ACCENT}' - '\N{RIGHT DOUBLE QUOTATION MARK}' - '\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}' - '\N{TRADE MARK SIGN}' + "end_punctuation": ( + "\N{REGISTERED SIGN}" + "\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}" + "\N{DOUBLE ACUTE ACCENT}" + "\N{RIGHT DOUBLE QUOTATION MARK}" + "\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}" + "\N{TRADE MARK SIGN}" ), - 'numeric': ( - '\N{SUPERSCRIPT TWO}' - '\N{SUPERSCRIPT THREE}' - '\N{SUPERSCRIPT ONE}' - '\N{PLUS-MINUS SIGN}' - '\N{VULGAR FRACTION ONE QUARTER}' - '\N{VULGAR FRACTION ONE HALF}' - '\N{VULGAR FRACTION THREE QUARTERS}' - '\N{MULTIPLICATION SIGN}' - '\N{MICRO SIGN}' - '\N{DIVISION SIGN}' - '\N{FRACTION SLASH}' - '\N{PARTIAL DIFFERENTIAL}' - '\N{INCREMENT}' - '\N{N-ARY PRODUCT}' - '\N{N-ARY SUMMATION}' - '\N{SQUARE ROOT}' - '\N{INFINITY}' - '\N{INTERSECTION}' - '\N{INTEGRAL}' - '\N{ALMOST EQUAL TO}' - '\N{NOT EQUAL TO}' - '\N{IDENTICAL TO}' - '\N{LESS-THAN OR EQUAL TO}' - '\N{GREATER-THAN OR EQUAL TO}' + "numeric": ( + "\N{SUPERSCRIPT TWO}" + "\N{SUPERSCRIPT THREE}" + "\N{SUPERSCRIPT ONE}" + "\N{PLUS-MINUS SIGN}" + "\N{VULGAR FRACTION ONE QUARTER}" + "\N{VULGAR FRACTION ONE HALF}" + "\N{VULGAR FRACTION THREE QUARTERS}" + "\N{MULTIPLICATION SIGN}" + "\N{MICRO SIGN}" + "\N{DIVISION SIGN}" + "\N{FRACTION SLASH}" + "\N{PARTIAL DIFFERENTIAL}" + "\N{INCREMENT}" + "\N{N-ARY PRODUCT}" + "\N{N-ARY SUMMATION}" + "\N{SQUARE ROOT}" + "\N{INFINITY}" + "\N{INTERSECTION}" + "\N{INTEGRAL}" + "\N{ALMOST EQUAL TO}" + "\N{NOT EQUAL TO}" + "\N{IDENTICAL TO}" + "\N{LESS-THAN OR EQUAL TO}" + "\N{GREATER-THAN OR EQUAL TO}" ), - 'kaomoji': ( - _char_range(0xd2, 0xd6) - + _char_range(0xd9, 0xdc) - + _char_range(0xf2, 0xf6) - + _char_range(0xf8, 0xfc) + - '\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}' - '\N{DEGREE SIGN}' + "kaomoji": ( + _char_range(0xD2, 0xD6) + + _char_range(0xD9, 0xDC) + + _char_range(0xF2, 0xF6) + + _char_range(0xF8, 0xFC) + + "\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}" + "\N{DEGREE SIGN}" ), - 'upper_accented': ( - _char_range(0xc0, 0xd1) + + "upper_accented": ( + _char_range(0xC0, 0xD1) + # skip capital O's and U's that could be used in kaomoji, but # include Ø because it's very common in Arabic mojibake: - '\N{LATIN CAPITAL LETTER O WITH STROKE}' - '\N{LATIN CAPITAL LETTER U WITH DIAERESIS}' - '\N{LATIN CAPITAL LETTER Y WITH ACUTE}' - '\N{LATIN CAPITAL LETTER A WITH BREVE}' - '\N{LATIN CAPITAL LETTER A WITH OGONEK}' - '\N{LATIN CAPITAL LETTER C WITH ACUTE}' - '\N{LATIN CAPITAL LETTER C WITH CARON}' - '\N{LATIN CAPITAL LETTER D WITH CARON}' - '\N{LATIN CAPITAL LETTER D WITH STROKE}' - '\N{LATIN CAPITAL LETTER E WITH OGONEK}' - '\N{LATIN CAPITAL LETTER E WITH CARON}' - '\N{LATIN CAPITAL LETTER G WITH BREVE}' - '\N{LATIN CAPITAL LETTER I WITH DOT ABOVE}' - '\N{LATIN CAPITAL LETTER L WITH ACUTE}' - '\N{LATIN CAPITAL LETTER L WITH CARON}' - '\N{LATIN CAPITAL LETTER L WITH STROKE}' - '\N{LATIN CAPITAL LETTER N WITH ACUTE}' - '\N{LATIN CAPITAL LETTER N WITH CARON}' - '\N{LATIN CAPITAL LIGATURE OE}' - '\N{LATIN CAPITAL LETTER R WITH CARON}' - '\N{LATIN CAPITAL LETTER S WITH ACUTE}' - '\N{LATIN CAPITAL LETTER S WITH CEDILLA}' - '\N{LATIN CAPITAL LETTER S WITH CARON}' - '\N{LATIN CAPITAL LETTER T WITH CEDILLA}' - '\N{LATIN CAPITAL LETTER T WITH CARON}' - '\N{LATIN CAPITAL LETTER U WITH RING ABOVE}' - '\N{LATIN CAPITAL LETTER U WITH DOUBLE ACUTE}' - '\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}' - '\N{LATIN CAPITAL LETTER Z WITH ACUTE}' - '\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}' - '\N{LATIN CAPITAL LETTER Z WITH CARON}' - '\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}' + "\N{LATIN CAPITAL LETTER O WITH STROKE}" + "\N{LATIN CAPITAL LETTER U WITH DIAERESIS}" + "\N{LATIN CAPITAL LETTER Y WITH ACUTE}" + "\N{LATIN CAPITAL LETTER A WITH BREVE}" + "\N{LATIN CAPITAL LETTER A WITH OGONEK}" + "\N{LATIN CAPITAL LETTER C WITH ACUTE}" + "\N{LATIN CAPITAL LETTER C WITH CARON}" + "\N{LATIN CAPITAL LETTER D WITH CARON}" + "\N{LATIN CAPITAL LETTER D WITH STROKE}" + "\N{LATIN CAPITAL LETTER E WITH OGONEK}" + "\N{LATIN CAPITAL LETTER E WITH CARON}" + "\N{LATIN CAPITAL LETTER G WITH BREVE}" + "\N{LATIN CAPITAL LETTER I WITH DOT ABOVE}" + "\N{LATIN CAPITAL LETTER L WITH ACUTE}" + "\N{LATIN CAPITAL LETTER L WITH CARON}" + "\N{LATIN CAPITAL LETTER L WITH STROKE}" + "\N{LATIN CAPITAL LETTER N WITH ACUTE}" + "\N{LATIN CAPITAL LETTER N WITH CARON}" + "\N{LATIN CAPITAL LIGATURE OE}" + "\N{LATIN CAPITAL LETTER R WITH CARON}" + "\N{LATIN CAPITAL LETTER S WITH ACUTE}" + "\N{LATIN CAPITAL LETTER S WITH CEDILLA}" + "\N{LATIN CAPITAL LETTER S WITH CARON}" + "\N{LATIN CAPITAL LETTER T WITH CEDILLA}" + "\N{LATIN CAPITAL LETTER T WITH CARON}" + "\N{LATIN CAPITAL LETTER U WITH RING ABOVE}" + "\N{LATIN CAPITAL LETTER U WITH DOUBLE ACUTE}" + "\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}" + "\N{LATIN CAPITAL LETTER Z WITH ACUTE}" + "\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}" + "\N{LATIN CAPITAL LETTER Z WITH CARON}" + "\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}" ), - 'lower_accented': ( - '\N{LATIN SMALL LETTER SHARP S}' - + _char_range(0xe0, 0xf1) + + "lower_accented": ( + "\N{LATIN SMALL LETTER SHARP S}" + _char_range(0xE0, 0xF1) + # skip o's and u's that could be used in kaomoji - '\N{LATIN SMALL LETTER A WITH BREVE}' - '\N{LATIN SMALL LETTER A WITH OGONEK}' - '\N{LATIN SMALL LETTER C WITH ACUTE}' - '\N{LATIN SMALL LETTER C WITH CARON}' - '\N{LATIN SMALL LETTER D WITH CARON}' - '\N{LATIN SMALL LETTER D WITH STROKE}' - '\N{LATIN SMALL LETTER E WITH OGONEK}' - '\N{LATIN SMALL LETTER E WITH CARON}' - '\N{LATIN SMALL LETTER G WITH BREVE}' - '\N{LATIN SMALL LETTER L WITH ACUTE}' - '\N{LATIN SMALL LETTER L WITH CARON}' - '\N{LATIN SMALL LETTER L WITH STROKE}' - '\N{LATIN SMALL LIGATURE OE}' - '\N{LATIN SMALL LETTER R WITH ACUTE}' - '\N{LATIN SMALL LETTER S WITH ACUTE}' - '\N{LATIN SMALL LETTER S WITH CEDILLA}' - '\N{LATIN SMALL LETTER S WITH CARON}' - '\N{LATIN SMALL LETTER T WITH CARON}' - '\N{LATIN SMALL LETTER Z WITH ACUTE}' - '\N{LATIN SMALL LETTER Z WITH DOT ABOVE}' - '\N{LATIN SMALL LETTER Z WITH CARON}' - '\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}' - '\N{LATIN SMALL LIGATURE FI}' - '\N{LATIN SMALL LIGATURE FL}' + "\N{LATIN SMALL LETTER A WITH BREVE}" + "\N{LATIN SMALL LETTER A WITH OGONEK}" + "\N{LATIN SMALL LETTER C WITH ACUTE}" + "\N{LATIN SMALL LETTER C WITH CARON}" + "\N{LATIN SMALL LETTER D WITH CARON}" + "\N{LATIN SMALL LETTER D WITH STROKE}" + "\N{LATIN SMALL LETTER E WITH OGONEK}" + "\N{LATIN SMALL LETTER E WITH CARON}" + "\N{LATIN SMALL LETTER G WITH BREVE}" + "\N{LATIN SMALL LETTER L WITH ACUTE}" + "\N{LATIN SMALL LETTER L WITH CARON}" + "\N{LATIN SMALL LETTER L WITH STROKE}" + "\N{LATIN SMALL LIGATURE OE}" + "\N{LATIN SMALL LETTER R WITH ACUTE}" + "\N{LATIN SMALL LETTER S WITH ACUTE}" + "\N{LATIN SMALL LETTER S WITH CEDILLA}" + "\N{LATIN SMALL LETTER S WITH CARON}" + "\N{LATIN SMALL LETTER T WITH CARON}" + "\N{LATIN SMALL LETTER Z WITH ACUTE}" + "\N{LATIN SMALL LETTER Z WITH DOT ABOVE}" + "\N{LATIN SMALL LETTER Z WITH CARON}" + "\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}" + "\N{LATIN SMALL LIGATURE FI}" + "\N{LATIN SMALL LIGATURE FL}" ), - 'upper_common': ( - '\N{LATIN CAPITAL LETTER THORN}' - + _char_range(0x391, 0x3a9) + # Greek capital letters - + "upper_common": ( + "\N{LATIN CAPITAL LETTER THORN}" + + _char_range(0x391, 0x3A9) # Greek capital letters # not included under 'accented' because these can commonly # occur at ends of words, in positions where they'd be detected # as mojibake - '\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}' - '\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}' - '\N{GREEK CAPITAL LETTER ETA WITH TONOS}' - '\N{GREEK CAPITAL LETTER IOTA WITH TONOS}' - '\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}' - '\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}' - '\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}' - '\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}' - '\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}' - + _char_range(0x401, 0x42f) # Cyrillic capital letters + + "\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}" + "\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}" + "\N{GREEK CAPITAL LETTER ETA WITH TONOS}" + "\N{GREEK CAPITAL LETTER IOTA WITH TONOS}" + "\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}" + "\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}" + "\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}" + "\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}" + "\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}" + + _char_range(0x401, 0x42F) # Cyrillic capital letters ), - 'lower_common': ( + "lower_common": ( # lowercase thorn does not appear in mojibake - _char_range(0x3b1, 0x3c6) + # Greek lowercase letters - '\N{GREEK SMALL LETTER ALPHA WITH TONOS}' - '\N{GREEK SMALL LETTER EPSILON WITH TONOS}' - '\N{GREEK SMALL LETTER ETA WITH TONOS}' - '\N{GREEK SMALL LETTER IOTA WITH TONOS}' - '\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}' - + _char_range(0x430, 0x45f) # Cyrillic lowercase letters - ), - 'box': ( - '─│┌┐┘├┤┬┼' - + _char_range(0x2550, 0x256c) + - '▀▄█▌▐░▒▓' + _char_range(0x3B1, 0x3C6) # Greek lowercase letters + + "\N{GREEK SMALL LETTER ALPHA WITH TONOS}" + "\N{GREEK SMALL LETTER EPSILON WITH TONOS}" + "\N{GREEK SMALL LETTER ETA WITH TONOS}" + "\N{GREEK SMALL LETTER IOTA WITH TONOS}" + "\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}" + + _char_range(0x430, 0x45F) # Cyrillic lowercase letters ), + "box": ("─│┌┐┘├┤┬┼" + _char_range(0x2550, 0x256C) + "▀▄█▌▐░▒▓"), } -BADNESS_RE = re.compile(r""" +BADNESS_RE = re.compile( + r""" [{c1}] | [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] @@ -546,46 +540,46 @@ def _char_range(start, end): # Windows-1253 mojibake of characters in the U+2000 range β€[™\xa0Ά\xad®°] | - + # Windows-1253 mojibake of Latin-1 characters and/or the Greek alphabet [ΒΓΞΟ][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°][ΒΓΞΟ] -""".format(**MOJIBAKE_CATEGORIES), re.VERBOSE) +""".format( + **MOJIBAKE_CATEGORIES + ), + re.VERBOSE, +) # Character classes that help us pinpoint embedded mojibake. These can # include common characters, because we'll also check them for 'badness'. UTF8_CLUES = { # Letters that decode to 0xC2 - 0xDF in a Latin-1-like encoding - 'utf8_first_of_2': ( - 'ÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßĂĆČĎĐĘĚĞİĹŃŇŐŘŞŢŮŰ' - 'ΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ' + "utf8_first_of_2": ( + "ÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßĂĆČĎĐĘĚĞİĹŃŇŐŘŞŢŮŰ" + "ΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ" ), # Letters that decode to 0xE0 - 0xEF in a Latin-1-like encoding - 'utf8_first_of_3': ( - 'àáâãäåæçèéêëìíîïăćčďęěĺŕΰαβγδεζηθικλμνξοабвгдежзийклмноп' - ), + "utf8_first_of_3": ("àáâãäåæçèéêëìíîïăćčďęěĺŕΰαβγδεζηθικλμνξοабвгдежзийклмноп"), # Letters that decode to 0xF0 or 0xF3 in a Latin-1-like encoding. # (Other leading bytes correspond only to unassigned codepoints) - 'utf8_first_of_4': ( - 'ðóđğπσру' - ), + "utf8_first_of_4": ("ðóđğπσру"), # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding, # including a space standing in for 0xA0 - 'utf8_continuation': ( - '\x80-\xbf' - 'ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅' - 'ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ' - '–—―‘’‚“”„†‡•…‰‹›€№™' - ' ' + "utf8_continuation": ( + "\x80-\xbf" + "ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅" + "ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ" + "–—―‘’‚“”„†‡•…‰‹›€№™" + " " ), # Letters that decode to 0x80 - 0xBF in a Latin-1-like encoding, # and don't usually stand for themselves when adjacent to mojibake. # This excludes spaces, dashes, quotation marks, and ellipses. - 'utf8_continuation_strict': ( - '\x80-\xbf' - 'ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅' - 'ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ' - '†‡•‰‹›€№™' + "utf8_continuation_strict": ( + "\x80-\xbf" + "ĄąĽľŁłŒœŚśŞşŠšŤťŸŹźŻżŽžƒˆˇ˘˛˜˝΄΅" + "ΆΈΉΊΌΎΏЁЂЃЄЅІЇЈЉЊЋЌЎЏёђѓєѕіїјљњћќўџҐґ" + "†‡•‰‹›€№™" ), } @@ -606,7 +600,8 @@ def _char_range(start, end): # less efficient. The 'badness' rules that require context, such as a preceding # lowercase letter, will prevent some cases of inconsistent UTF-8 from being # fixed when they don't see it. -UTF8_DETECTOR_RE = re.compile(""" +UTF8_DETECTOR_RE = re.compile( + """ (?>> print(remove_bom(chr(0xfeff) + "Where do you want to go today?")) Where do you want to go today? """ - return text.lstrip(chr(0xfeff)) + return text.lstrip(chr(0xFEFF)) # Define a regex to match valid escape sequences in Python string literals. ESCAPE_SEQUENCE_RE = re.compile( - r''' + r""" ( \\U........ # 8-digit hex escapes | \\u.... # 4-digit hex escapes | \\x.. # 2-digit hex escapes | \\[0-7]{1,3} # Octal escapes | \\N\{[^}]+\} # Unicode characters by name | \\[\\'"abfnrtv] # Single-character escapes - )''', + )""", re.UNICODE | re.VERBOSE, ) @@ -374,7 +372,7 @@ def decode_escapes(text): def decode_match(match): "Given a regex match, decode the escape sequence it contains." - return codecs.decode(match.group(0), 'unicode-escape') + return codecs.decode(match.group(0), "unicode-escape") return ESCAPE_SEQUENCE_RE.sub(decode_match, text) @@ -402,7 +400,8 @@ def decode_match(match): # contain it will end up with inserted spaces. We can't do the right thing with # every word. The cost is that the mojibake text "fà cil" will be interpreted as # "fà cil", not "fàcil". -A_GRAVE_WORD_RE = re.compile(b'\xc3 (?! |quele|quela|quilo|s )') +A_GRAVE_WORD_RE = re.compile(b"\xc3 (?! |quele|quela|quilo|s )") + def restore_byte_a0(byts): """ @@ -418,11 +417,11 @@ def restore_byte_a0(byts): This is used as a step within `fix_encoding`. """ - byts = A_GRAVE_WORD_RE.sub(b'\xc3\xa0 ', byts) + byts = A_GRAVE_WORD_RE.sub(b"\xc3\xa0 ", byts) def replacement(match): "The function to apply when this regex matches." - return match.group(0).replace(b'\x20', b'\xa0') + return match.group(0).replace(b"\x20", b"\xa0") return ALTERED_UTF8_RE.sub(replacement, byts) @@ -463,9 +462,9 @@ def replace_lossy_sequences(byts): This is used as a transcoder within `fix_encoding`. """ - return LOSSY_UTF8_RE.sub('\ufffd'.encode('utf-8'), byts) + return LOSSY_UTF8_RE.sub("\ufffd".encode("utf-8"), byts) + - def decode_inconsistent_utf8(text): """ Sometimes, text from one encoding ends up embedded within text from a @@ -476,18 +475,18 @@ def decode_inconsistent_utf8(text): def fix_embedded_mojibake(match): substr = match.group(0) - + # Require the match to be shorter, so that this doesn't recurse infinitely if len(substr) < len(text) and is_bad(substr): return ftfy.fix_encoding(substr) else: return substr - + return UTF8_DETECTOR_RE.sub(fix_embedded_mojibake, text) def _c1_fixer(match): - return match.group(0).encode('latin-1').decode('sloppy-windows-1252') + return match.group(0).encode("latin-1").decode("sloppy-windows-1252") def fix_c1_controls(text): @@ -499,8 +498,8 @@ def fix_c1_controls(text): TRANSCODERS = { - 'restore_byte_a0': restore_byte_a0, - 'replace_lossy_sequences': replace_lossy_sequences, - 'decode_inconsistent_utf8': decode_inconsistent_utf8, - 'fix_c1_controls': fix_c1_controls + "restore_byte_a0": restore_byte_a0, + "replace_lossy_sequences": replace_lossy_sequences, + "decode_inconsistent_utf8": decode_inconsistent_utf8, + "fix_c1_controls": fix_c1_controls, } diff --git a/tests/test_characters.py b/tests/test_characters.py index 18af3bfe..3a004455 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -1,8 +1,6 @@ from ftfy import fix_encoding, fix_encoding_and_explain, apply_plan -from ftfy.fixes import ( - possible_encoding, - remove_control_chars, fix_surrogates -) +from ftfy.fixes import remove_control_chars, fix_surrogates +from ftfy.chardata import possible_encoding from ftfy.badness import badness import unicodedata import sys From c02e4a738e38067dbcc6c39891a0fa9d571a466d Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 26 Mar 2021 18:04:52 -0400 Subject: [PATCH 470/502] Type updates and lints --- ftfy/__init__.py | 259 ++++++++++++++++--------------- ftfy/bad_codecs/__init__.py | 4 +- ftfy/bad_codecs/utf8_variants.py | 5 +- ftfy/fixes.py | 25 ++- 4 files changed, 144 insertions(+), 149 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 5eb5dbec..120dd0e1 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -6,10 +6,9 @@ """ import unicodedata -from collections import namedtuple -from typing import Optional +from typing import Optional, Tuple, NamedTuple, Union, List -import ftfy.bad_codecs +from ftfy import bad_codecs from ftfy import chardata, fixes from ftfy.badness import is_bad from ftfy.formatting import display_ljust @@ -19,149 +18,151 @@ # Though this function does nothing, it lets linters know that we're using # ftfy.bad_codecs. See the docstring in `bad_codecs/__init__.py` for more. -ftfy.bad_codecs.ok() - - -CONFIG_DEFAULTS = { - "unescape_html": "auto", - "remove_terminal_escapes": True, - "fix_encoding": True, - "restore_byte_a0": True, - "replace_lossy_sequences": True, - "decode_inconsistent_utf8": True, - "fix_c1_controls": True, - "fix_latin_ligatures": True, - "fix_character_width": True, - "uncurl_quotes": True, - "fix_line_breaks": True, - "fix_surrogates": True, - "remove_control_chars": True, - "normalization": "NFC", - "max_decode_length": 1000000, - "explain": True, -} -TextFixerConfig = namedtuple( - "TextFixerConfig", CONFIG_DEFAULTS.keys(), defaults=CONFIG_DEFAULTS.values() -) -TextFixerConfig.__doc__ = r""" -A TextFixerConfig object stores configuration options for ftfy. +bad_codecs.ok() -It's implemented as a namedtuple with defaults, so you can instantiate -it by providing the values to change from their defaults as keyword arguments. -For example, to disable 'unescape_html' and keep the rest of the defaults: - TextFixerConfig(unescape_html=False) +class TextFixerConfig(NamedTuple): + """ + A TextFixerConfig object stores configuration options for ftfy. -Here are the options and their default values: + It's implemented as a namedtuple with defaults, so you can instantiate + it by providing the values to change from their defaults as keyword arguments. + For example, to disable 'unescape_html' and keep the rest of the defaults: -- `unescape_html`: "auto" + TextFixerConfig(unescape_html=False) - Configures whether to replace HTML entities such as & with the character - they represent. "auto" says to do this by default, but disable it when a - literal < character appears, indicating that the input is actual HTML and - entities should be preserved. The value can be True, to always enable this - fixer, or False, to always disable it. + Here are the options and their default values: -- `remove_terminal_escapes`: True + - `unescape_html`: "auto" - Removes "ANSI" terminal escapes, such as for changing the color of text in a - terminal window. + Configures whether to replace HTML entities such as & with the character + they represent. "auto" says to do this by default, but disable it when a + literal < character appears, indicating that the input is actual HTML and + entities should be preserved. The value can be True, to always enable this + fixer, or False, to always disable it. -- `fix_encoding`: True + - `remove_terminal_escapes`: True - Detect mojibake and attempt to fix it by decoding the text in a different - encoding standard. + Removes "ANSI" terminal escapes, such as for changing the color of text in a + terminal window. - The following four options affect `fix_encoding` works, and do nothing if - `fix_encoding` is False: + - `fix_encoding`: True - - `restore_byte_a0`: True + Detect mojibake and attempt to fix it by decoding the text in a different + encoding standard. - Allow a literal space (U+20) to be interpreted as a non-breaking space - (U+A0) when that would make it part of a fixable mojibake string. + The following four options affect `fix_encoding` works, and do nothing if + `fix_encoding` is False: - Because spaces are very common characters, this could lead to false - positives, but we try to apply it only when there's strong evidence for - mojibake. Disabling `restore_byte_a0` is safer from false positives, - but creates false negatives. + - `restore_byte_a0`: True - - `replace_lossy_sequences`: True + Allow a literal space (U+20) to be interpreted as a non-breaking space + (U+A0) when that would make it part of a fixable mojibake string. - Detect mojibake that has been partially replaced by the characters - '�' or '?'. If the mojibake could be decoded otherwise, replace the - detected sequence with '�'. + Because spaces are very common characters, this could lead to false + positives, but we try to apply it only when there's strong evidence for + mojibake. Disabling `restore_byte_a0` is safer from false positives, + but creates false negatives. - - `decode_inconsistent_utf8`: True + - `replace_lossy_sequences`: True - When we see sequences that distinctly look like UTF-8 mojibake, but - there's no consistent way to reinterpret the string in a new encoding, - replace the mojibake with the appropriate UTF-8 characters anyway. + Detect mojibake that has been partially replaced by the characters + '�' or '?'. If the mojibake could be decoded otherwise, replace the + detected sequence with '�'. - This helps to decode strings that are concatenated from different - encodings. + - `decode_inconsistent_utf8`: True - - `fix_c1_controls`: True + When we see sequences that distinctly look like UTF-8 mojibake, but + there's no consistent way to reinterpret the string in a new encoding, + replace the mojibake with the appropriate UTF-8 characters anyway. - Replace C1 control characters (the useless characters U+80 - U+9B that - come from Latin-1) with their Windows-1252 equivalents, like HTML5 does, - even if the whole string doesn't decode as Latin-1. + This helps to decode strings that are concatenated from different + encodings. -- `fix_latin_ligatures`: True + - `fix_c1_controls`: True - Replace common Latin-alphabet ligatures, such as 'fi', with the - letters they're made of. + Replace C1 control characters (the useless characters U+80 - U+9B that + come from Latin-1) with their Windows-1252 equivalents, like HTML5 does, + even if the whole string doesn't decode as Latin-1. -- `fix_character_width`: True + - `fix_latin_ligatures`: True - Replace fullwidth Latin characters and halfwidth Katakana with - their more standard widths. + Replace common Latin-alphabet ligatures, such as 'fi', with the + letters they're made of. -- `uncurl_quotes`: True + - `fix_character_width`: True - Replace curly quotes with straight quotes. + Replace fullwidth Latin characters and halfwidth Katakana with + their more standard widths. -- `fix_line_breaks`: True + - `uncurl_quotes`: True - Replace various forms of line breaks with the standard Unix line - break, '\n'. + Replace curly quotes with straight quotes. -- `fix_surrogates`: True + - `fix_line_breaks`: True - Replace sequences of UTF-16 surrogate codepoints with the character - they were meant to encode. This fixes text that was decoded with the - obsolete UCS-2 standard, and allows it to support high-numbered - codepoints such as emoji. + Replace various forms of line breaks with the standard Unix line + break, '\n'. -- `remove_control_chars`: True + - `fix_surrogates`: True - Remove certain control characters that have no displayed effect on text. + Replace sequences of UTF-16 surrogate codepoints with the character + they were meant to encode. This fixes text that was decoded with the + obsolete UCS-2 standard, and allows it to support high-numbered + codepoints such as emoji. -- `normalization`: "NFC" + - `remove_control_chars`: True - Choose what kind of Unicode normalization is applied. Usually, we apply - NFC normalization, so that letters followed by combining characters become - single combined characters. + Remove certain control characters that have no displayed effect on text. - Changing this to "NFKC" applies more compatibility conversions, such as - replacing the 'micro sign' with a standard Greek lowercase mu, which looks - identical. However, some NFKC normalizations change the meaning of text, - such as converting "10³" to "103". + - `normalization`: "NFC" - `normalization` can be None, to apply no normalization. + Choose what kind of Unicode normalization is applied. Usually, we apply + NFC normalization, so that letters followed by combining characters become + single combined characters. -- `max_decode_length`: 1_000_000 + Changing this to "NFKC" applies more compatibility conversions, such as + replacing the 'micro sign' with a standard Greek lowercase mu, which looks + identical. However, some NFKC normalizations change the meaning of text, + such as converting "10³" to "103". - The maximum size of "segment" that ftfy will try to fix all at once. + `normalization` can be None, to apply no normalization. -- `explain`: True + - `max_decode_length`: 1_000_000 - Whether to compute 'explanations', lists describing what ftfy changed. - When this is False, the explanation will be None, and the code that - builds the explanation will be skipped, possibly saving time. + The maximum size of "segment" that ftfy will try to fix all at once. + + - `explain`: True + + Whether to compute 'explanations', lists describing what ftfy changed. + When this is False, the explanation will be None, and the code that + builds the explanation will be skipped, possibly saving time. + + Functions that accept TextFixerConfig and don't return an explanation + will automatically set `explain` to False. + """ + unescape_html: Union[str, bool] = "auto" + remove_terminal_escapes: bool = True + fix_encoding: bool = True + restore_byte_a0: bool = True + replace_lossy_sequences: bool = True + decode_inconsistent_utf8: bool = True + fix_c1_controls: bool = True + fix_latin_ligatures: bool = True + fix_character_width: bool = True + uncurl_quotes: bool = True + fix_line_breaks: bool = True + fix_surrogates: bool = True + remove_control_chars: bool = True + normalization: Optional[str] = "NFC" + max_decode_length: int = 1000000 + explain: bool = True + + +class ExplainedText(NamedTuple): + text: str + explanation: Optional[List[Tuple[str, str]]] - Functions that accept TextFixerConfig and don't return an explanation - will automatically set `explain` to False. -""" FIXERS = { "unescape_html": fixes.unescape_html, @@ -212,8 +213,8 @@ def _try_fix( if steps is not None and fixed != text: steps.append(("apply", fixer_name)) return fixed - else: - return text + + return text def fix_text(text: str, config: Optional[TextFixerConfig] = None, **kwargs) -> str: @@ -281,7 +282,7 @@ def fix_text(text: str, config: Optional[TextFixerConfig] = None, **kwargs) -> s config = config._replace(**kwargs) config = config._replace(explain=False) if isinstance(text, bytes): - raise UnicodeError(fixes.BYTES_ERROR_TEXT) + raise UnicodeError(BYTES_ERROR_TEXT) out = [] pos = 0 @@ -303,7 +304,7 @@ def fix_text(text: str, config: Optional[TextFixerConfig] = None, **kwargs) -> s def fix_and_explain( text: str, config: Optional[TextFixerConfig] = None, **kwargs -) -> (str, Optional[list]): +) -> ExplainedText: """ Fix text as a single segment, returning the fixed text and an explanation of what was fixed. @@ -314,14 +315,14 @@ def fix_and_explain( if config is None: config = TextFixerConfig() if isinstance(text, bytes): - raise UnicodeError(fixes.BYTES_ERROR_TEXT) + raise UnicodeError(BYTES_ERROR_TEXT) config = config._replace(**kwargs) if config.unescape_html == "auto" and "<" in text: config = config._replace(unescape_html=False) if config.explain: - steps = [] + steps: Optional[List[Tuple[str, str]]] = [] else: # If explanations aren't desired, `steps` will be None steps = None @@ -356,12 +357,12 @@ def fix_and_explain( text = fixed if text == origtext: - return text, steps + return ExplainedText(text, steps) def fix_encoding_and_explain( text: str, config: Optional[TextFixerConfig] = None -) -> (str, list): +) -> ExplainedText: """ Apply the steps of ftfy that detect mojibake and fix it. Returns the fixed text and a list explaining what was fixed. @@ -373,24 +374,24 @@ def fix_encoding_and_explain( if config is None: config = TextFixerConfig() if isinstance(text, bytes): - raise UnicodeError(fixes.BYTES_ERROR_TEXT) + raise UnicodeError(BYTES_ERROR_TEXT) if not config.fix_encoding: # A weird trivial case: we're asked to fix the encoding, but skip # fixing the encoding - return text, [] + return ExplainedText(text, []) - plan_so_far = [] + plan_so_far: List[Tuple[str, str]] = [] while True: prevtext = text text, plan = _fix_encoding_one_step_and_explain(text, config) plan_so_far.extend(plan) if text == prevtext: - return text, plan_so_far + return ExplainedText(text, plan_so_far) def _fix_encoding_one_step_and_explain( text: str, config: TextFixerConfig -) -> (str, list): +) -> ExplainedText: """ Perform one step of fixing the encoding of text. """ @@ -398,12 +399,12 @@ def _fix_encoding_one_step_and_explain( config = TextFixerConfig() if len(text) == 0: - return text, [] + return ExplainedText(text, []) # The first plan is to return ASCII text unchanged, as well as text # that doesn't look like it contains mojibake if chardata.possible_encoding(text, "ascii") or not is_bad(text): - return text, [] + return ExplainedText(text, []) # As we go through the next step, remember the possible encodings # that we encounter but don't successfully fix yet. We may need them @@ -447,7 +448,7 @@ def _fix_encoding_one_step_and_explain( decode_step = ("decode", decoding) steps = [encode_step] + transcode_steps + [decode_step] fixed = encoded_bytes.decode(decoding) - return fixed, steps + return ExplainedText(fixed, steps) except UnicodeDecodeError: pass @@ -457,7 +458,7 @@ def _fix_encoding_one_step_and_explain( steps = [("apply", "decode_inconsistent_utf8")] fixed = fixes.decode_inconsistent_utf8(text) if fixed != text: - return fixed, steps + return ExplainedText(fixed, steps) # The next most likely case is that this is Latin-1 that was intended to # be read as Windows-1252, because those two encodings in particular are @@ -466,7 +467,7 @@ def _fix_encoding_one_step_and_explain( if "windows-1252" in possible_1byte_encodings: # This text is in the intersection of Latin-1 and # Windows-1252, so it's probably legit. - return text, [] + return ExplainedText(text, []) else: # Otherwise, it means we have characters that are in Latin-1 but # not in Windows-1252. Those are C1 control characters. Nobody @@ -475,7 +476,7 @@ def _fix_encoding_one_step_and_explain( fixed = text.encode("latin-1").decode("windows-1252") if fixed != text: steps = [("encode", "latin-1"), ("decode", "windows-1252")] - return fixed, steps + return ExplainedText(fixed, steps) except UnicodeDecodeError: pass @@ -483,7 +484,7 @@ def _fix_encoding_one_step_and_explain( if config.fix_c1_controls and chardata.C1_CONTROL_RE.search(text): steps = [("transcode", "fix_c1_controls")] fixed = fixes.fix_c1_controls(text) - return fixed, steps + return ExplainedText(fixed, steps) # The cases that remain are mixups between two different single-byte # encodings, and not the common case of Latin-1 vs. Windows-1252. @@ -491,7 +492,7 @@ def _fix_encoding_one_step_and_explain( # With the new heuristic in 6.0, it's possible that we're closer to solving # these in some cases. It would require a lot of testing and tuning, though. # For now, we leave the text unchanged in these cases. - return text, [] + return ExplainedText(text, []) def fix_encoding(text: str, config: TextFixerConfig = None, **kwargs): @@ -613,8 +614,8 @@ def guess_bytes(bstring): if 0x0D in byteset and 0x0A not in byteset: # Files that contain CR and not LF are likely to be MacRoman. return bstring.decode("macroman"), "macroman" - else: - return bstring.decode("sloppy-windows-1252"), "sloppy-windows-1252" + + return bstring.decode("sloppy-windows-1252"), "sloppy-windows-1252" def apply_plan(text, plan): @@ -639,7 +640,7 @@ def apply_plan(text, plan): obj = obj.encode(encoding) elif operation == "decode": obj = obj.decode(encoding) - elif operation == "transcode" or operation == "apply": + elif operation in ("transcode", "apply"): if encoding in FIXERS: obj = FIXERS[encoding](obj) else: diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py index 473c890a..6483355c 100644 --- a/ftfy/bad_codecs/__init__.py +++ b/ftfy/bad_codecs/__init__.py @@ -30,8 +30,9 @@ """ from encodings import normalize_encoding import codecs +from typing import Dict -_CACHE = {} +_CACHE: Dict[str, codecs.CodecInfo] = {} # Define some aliases for 'utf-8-variants'. All hyphens get turned into # underscores, because of `normalize_encoding`. @@ -86,7 +87,6 @@ def ok(): you use the ``unicode.encode`` and ``bytes.decode`` methods with certain encodings. """ - pass codecs.register(search_function) diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 597f777f..15b1fff8 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -40,6 +40,7 @@ import re import codecs +from typing import Tuple from encodings.utf_8 import (IncrementalDecoder as UTF8IncrementalDecoder, IncrementalEncoder as UTF8IncrementalEncoder) @@ -221,11 +222,11 @@ def _buffer_decode_surrogates(sup, input, errors, final): # Everything below here is boilerplate that matches the modules in the # built-in `encodings` package. -def encode(input, errors='strict'): +def encode(input: str, errors: str='strict') -> Tuple[bytes, int]: return IncrementalEncoder(errors).encode(input, final=True), len(input) -def decode(input, errors='strict'): +def decode(input: bytes, errors: str='strict') -> Tuple[str, int]: return IncrementalDecoder(errors).decode(input, final=True), len(input) diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 0ab13696..fb2a6465 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -27,6 +27,9 @@ def fix_encoding_and_explain(text): + """ + Deprecated copy of `ftfy.fix_encoding_and_explain()`. + """ warnings.warn( "`fix_encoding_and_explain()` has moved to the main module of ftfy.", DeprecationWarning, @@ -35,21 +38,19 @@ def fix_encoding_and_explain(text): def fix_encoding(text): + """ + Deprecated copy of `ftfy.fix_encoding()`. + """ warnings.warn( "`fix_encoding()` has moved to the main module of ftfy.", DeprecationWarning ) return ftfy.fix_encoding(text) -def fix_one_step_and_explain(text): - warnings.warn( - "`fix_one_step_and_explain()` has moved to the main module of ftfy.", - DeprecationWarning, - ) - return ftfy.fix_one_step_and_explain(text) - - def apply_plan(text, plan): + """ + Deprecated copy of `ftfy.apply_plan()`. + """ warnings.warn( "`apply_plan()` has moved to the main module of ftfy.", DeprecationWarning ) @@ -495,11 +496,3 @@ def fix_c1_controls(text): Windows-1252 equivalents. This matches what Web browsers do. """ return C1_CONTROL_RE.sub(_c1_fixer, text) - - -TRANSCODERS = { - "restore_byte_a0": restore_byte_a0, - "replace_lossy_sequences": replace_lossy_sequences, - "decode_inconsistent_utf8": decode_inconsistent_utf8, - "fix_c1_controls": fix_c1_controls, -} From 8620efe421be01ebda155c871816da8ebe0f9e05 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 31 Mar 2021 17:24:39 -0400 Subject: [PATCH 471/502] lots of progress on documentation --- docs/_static/css/custom.css | 1 + docs/accuracy.rst | 3 + docs/avoid.rst | 68 +++++ docs/cite.rst | 26 ++ docs/cli.rst | 32 +++ docs/conf.py | 30 ++- docs/config.rst | 31 +++ docs/detect.rst | 23 ++ docs/encodings.rst | 31 +++ docs/explain.rst | 96 +++++++ docs/fixes.rst | 4 + docs/images/shipping-label.png | Bin 0 -> 203336 bytes docs/index.rst | 434 ++----------------------------- docs/modules.rst | 44 ++++ docs/poem.txt | 17 ++ ftfy/__init__.py | 150 ++++++----- ftfy/bad_codecs/utf8_variants.py | 4 +- ftfy/badness.py | 328 ++++++++++++++++++++++- ftfy/chardata.py | 303 --------------------- 19 files changed, 827 insertions(+), 798 deletions(-) create mode 100644 docs/_static/css/custom.css create mode 100644 docs/avoid.rst create mode 100644 docs/cite.rst create mode 100644 docs/cli.rst create mode 100644 docs/config.rst create mode 100644 docs/detect.rst create mode 100644 docs/encodings.rst create mode 100644 docs/explain.rst create mode 100644 docs/fixes.rst create mode 100644 docs/images/shipping-label.png create mode 100644 docs/modules.rst create mode 100644 docs/poem.txt diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css new file mode 100644 index 00000000..b1a355d9 --- /dev/null +++ b/docs/_static/css/custom.css @@ -0,0 +1 @@ +@import url('https://fonts.googleapis.com/css2?family=Inconsolata:wght@400;700&family=Source+Sans+Pro:ital,wght@0,400;0,700;1,400;1,700&display=swap'); diff --git a/docs/accuracy.rst b/docs/accuracy.rst index f4d1714e..9714393b 100644 --- a/docs/accuracy.rst +++ b/docs/accuracy.rst @@ -1,3 +1,6 @@ +Accuracy +======== + ftfy uses Twitter's streaming API as an endless source of realistic sample data. Twitter is massively multilingual, and despite that it's supposed to be uniformly UTF-8, in practice, any encoding mistake that someone can make will diff --git a/docs/avoid.rst b/docs/avoid.rst new file mode 100644 index 00000000..5e2159f5 --- /dev/null +++ b/docs/avoid.rst @@ -0,0 +1,68 @@ +How can I avoid producing mojibake? +=================================== + +Assume UTF-8 +------------ + +**Assume text is in UTF-8** unless you have a specific reason to believe it isn't. + +In the 2020s, `UTF-8 is everywhere`_, especially in text meant to be transferred over the Internet. + +.. _`UTF-8 is everywhere`: http://utf8everywhere.org/ + +In Python 3, you should use the Unicode string type (`str`) for all operations. You should open text files in UTF-8:: + + openfile = open(filename, encoding='utf-8', errors='replace') + +When you are specifically working with bytes and you need to turn them into text, you should decode them as UTF-8:: + + text = bytebuffer.decode('utf-8', 'replace') + +The exceptions, the cases where you're not using UTF-8, are few but relevant. If you're interacting with C APIs, you'll need to represent your text as bytes in the format the API expects. Windows APIs in particular expect UTF-16. + +We're mostly past the dark days when encodings were "character maps" of 256 possible characters, one byte per character. An unfortunate thing that keeps them alive is Microsoft Excel, whose "Export" feature will pick a 256-character encoding *based on your computer's operating system and default language*. So: + +Don't export CSVs from Excel +---------------------------- + +I know that I'm telling you not to do something that may seem like a requirement of doing your job. But **don't export CSV files from Excel** if you have any other choice. Though Excel CSVs look right on basic ASCII characters, on any other text, it either won't work or won't do what you want. Excel CSVs aren't even interoperable between different computers. + +My recommendation is to use Google Sheets to create CSVs, and keep Excel files in .xlsx format so the Unicode won't be mangled. + +If you must export a CSV-like file from Excel, you can find an option to tell Excel to export in "Unicode Text", and it will create a tab-separated UTF-16 file. This is not a very widely-used format, but at least it's not mojibake. + +You can follow these `unwieldy directions from a SalesForce help article`_ to use Excel and Notepad to create a UTF-8 CSV. You can see why I don't recommend this process. + +.. _`unwieldy directions from a SalesForce help article`: https://help.salesforce.com/articleView?id=000324657&type=1&mode=1 + +Don't use chardet +----------------- + +Encoding detection on raw bytes is not a good idea. It was important in the '90s, during the rough transition to Unicode -- and the most popular way of doing it, ``chardet``, hasn't changed since the '90s. + +A heuristic designed before there was multilingual social media, before there were emoji, is not going to work correctly in the 2020s. + +When chardet sees the *correct* UTF-8 encoding of an emoji, it will have no idea what it's looking at, because it won't match anything in its training data. Often, it will guess that it's Turkish encoded in Windows-1254. On other reasonable text, it will guess the "iso-8859-2" encoding, an encoding that you'd very rarely see used intentionally. Because the problem is inherent to the design of chardet, it's not easily fixed. + +chardet was built on the assumption that "encoding detection is language detection", which is no longer true. Web sites now contain text in multiple languages, and for the most part they use UTF-8 regardless of the language. + +I've strengthened my recommendation from "don't trust chardet's output" to "don't use chardet", because there's no realistic way to use chardet without trusting its output. We've reached a situation where major Python packages such as ``requests`` assume that chardet is correct, and yet the changing nature of text means that chardet is more wrong with each passing year. + +So how should you interpret raw bytes of text if you're not told what encoding they're in? As UTF-8. Text is UTF-8 until proven otherwise. + +ASCII isn't extended +-------------------- + +A sign that something is about to go wrong with encodings is if a developer is talking about "extended ASCII". + +ASCII is a set of 128 character codes (95 of them displayable), and has not had any new characters added to it since the backslash was added in 1967. + +Because ASCII is a 7-bit encoding but our computers use 8-bit bytes, it seems clear that ASCII *could* be extended to assign a meaning to all 256 possible bytes. There are many different, incompatible encodings that have done so. Many developers refer to one of these encodings as "extended ASCII", whose colloquial meaning is "the encoding of 256 characters that I learned first". Its meaning is completely dependent on the country you were in and the operating system you were using when you started programming: + +- My "extended ASCII" when I learned to program was IBM codepage 437, the one that was used in US versions of MS-DOS. +- To many people, "extended ASCII" is Windows codepage 1252, which they'd find in the Character Map of their Windows 9x computer, at least if they were in North America or Western Europe. +- Or it might be Latin-1, the common name for the ISO-8859-1 standard that became the first 256 characters of Unicode. + +"Extended ASCII" doesn't specify which encoding you mean, and often indicates that you don't realize that different people are thinking of different sets of 256 characters. + +Instead of "extended ASCII", say the name of the encoding such as "Latin-1", "Windows-1252", "Windows-1250", "codepage 437", or maybe "I don't know what it is but it looks right on my machine". And then revise things so that you use UTF-8. \ No newline at end of file diff --git a/docs/cite.rst b/docs/cite.rst new file mode 100644 index 00000000..52be8642 --- /dev/null +++ b/docs/cite.rst @@ -0,0 +1,26 @@ +.. _cite: + +Citing ftfy +=========== +ftfy has been used as a data processing step in major NLP research, including OpenAI's original GPT. + +It's important to give credit appropriately to everyone whose work you build on in research. This includes software, not just high-status contributions such as mathematical models. All I ask when you use ftfy for research is that you cite it. + +ftfy has a citable record on `Zenodo`_. A citation of ftfy may look like this: + + Robyn Speer. (2019). ftfy (Version 5.5). Zenodo. + http://doi.org/10.5281/zenodo.2591652 + +In BibTeX format, the citation is:: + + @misc{speer-2019-ftfy, + author = {Robyn Speer}, + title = {ftfy}, + note = {Version 5.5}, + year = 2019, + howpublished = {Zenodo}, + doi = {10.5281/zenodo.2591652}, + url = {https://doi.org/10.5281/zenodo.2591652} + } + +.. _Zenodo: https://zenodo.org/record/2591652 diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 00000000..0ed46d2e --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,32 @@ +Command-line usage +================== +ftfy can be used from the command line. By default, it takes UTF-8 input and +writes it to UTF-8 output, fixing problems in its Unicode as it goes. + +Here's the usage documentation for the `ftfy` command:: + + usage: ftfy [-h] [-o OUTPUT] [-g] [-e ENCODING] [-n NORMALIZATION] + [--preserve-entities] + [filename] + + ftfy (fixes text for you), version 5.0 + + positional arguments: + filename The file whose Unicode is to be fixed. Defaults to -, + meaning standard input. + + optional arguments: + -h, --help show this help message and exit + -o OUTPUT, --output OUTPUT + The file to output to. Defaults to -, meaning standard + output. + -g, --guess Ask ftfy to guess the encoding of your input. This is + risky. Overrides -e. + -e ENCODING, --encoding ENCODING + The encoding of the input. Defaults to UTF-8. + -n NORMALIZATION, --normalization NORMALIZATION + The normalization of Unicode to apply. Defaults to + NFC. Can be "none". + --preserve-entities Leave HTML entities as they are. The default is to + decode them, as long as no HTML tags have appeared in + the file. diff --git a/docs/conf.py b/docs/conf.py index 259a4021..5ff68964 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -81,7 +81,8 @@ #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'default' +pygments_dark_style = 'monokai' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] @@ -94,29 +95,36 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -import guzzle_sphinx_theme -html_theme_path = guzzle_sphinx_theme.html_theme_path() -html_theme = 'guzzle_sphinx_theme' - -extensions.append("guzzle_sphinx_theme") +html_theme = "furo" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - "project_nav_name": "ftfy", - "project_link": "https://github.com/LuminosoInsight/ftfy", + "light_css_variables": { + "color-brand-primary": "#7C4DFF", + "color-brand-content": "#7C4DFF", + "font-stack": "Source Sans Pro, sans-serif", + "font-stack--monospace": "Inconsolata", + "code-font-size": "18px", + + # I don't know why furo wants inline code to be so small, but don't let it + "font-size--small--2": "100%", + }, } +html_css_files = [ + 'css/custom.css', +] # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +html_title = "ftfy v6 documentation" # A shorter title for the navigation bar. Default is the same as html_title. -html_short_title = "ftfy documentation" +html_short_title = "ftfy" # The name of an image file (relative to this directory) to place at the top # of the sidebar. @@ -157,7 +165,7 @@ #html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +html_show_sourcelink = False # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True diff --git a/docs/config.rst b/docs/config.rst new file mode 100644 index 00000000..ce3057ff --- /dev/null +++ b/docs/config.rst @@ -0,0 +1,31 @@ +.. _config: + +Configuring ftfy +================ + +The main functions of ftfy -- :func:`ftfy.fix_text` and :func:`ftfy.fix_and_explain` -- run text through a sequence of fixes. If the text changed, it will run them through again, so that you can be sure the output ends up in a standard form that will be unchanged by ftfy. + +All the fixes are on by default, but you can pass in a configuration object or keyword options to turn them off. Check that the default fixes are appropriate for your use case. For example: + +- You should set `fix_entities` to False if the output is meant to be interpreted as HTML. + +- You should set `fix_character_width` to False if you want to preserve the spacing of CJK text. + +- You should set `uncurl_quotes` to False if you want to preserve quotation marks with nice typography. You could even consider doing the opposite of `uncurl_quotes`, running `smartypants`_ on the result to make all the punctuation typographically nice. + +- To be cautious and only fix mojibake when it can be fixed with a consistent sequence of encoding and decoding steps, you should set `decode_inconsistent_utf8` to False. + +.. _smartypants: http://pythonhosted.org/smartypants/ + +If the only fix you need is to detect and repair decoding errors (mojibake), use the :func:`ftfy.fix_encoding` function directly. However, note that mojibake is often entangled with other issues such as the curliness of quotation marks, so limiting the process to this step might make some mojibake unfixable. + +The TextFixerConfig object +-------------------------- + +The top-level functions of ftfy take a `config` argument that is an instance of :class:`ftfy.TextFixerConfig`. If this argument is None, the configuration will use its default values. + +.. autoclass:: ftfy.TextFixerConfig() + +Keyword arguments +----------------- +The top-level functions also accept keyword arguments in place of a `config` argument. Given these keyword arguments, they will pass them to the :class:`ftfy.TextFixerConfig` constructor, overriding the default values of those configuration options. \ No newline at end of file diff --git a/docs/detect.rst b/docs/detect.rst new file mode 100644 index 00000000..38882dec --- /dev/null +++ b/docs/detect.rst @@ -0,0 +1,23 @@ +Is ftfy an encoding detector? +============================= + +No, it's a mojibake detector (and fixer). That makes its task much easier, because it doesn't have to guess the encoding of everything: it can leave correct-looking text as it is. + +Encoding detectors have ended up being a bad idea, and they are largely responsible for _creating_ the problems that ftfy has to fix. + +The text that you put into ftfy should be Unicode that you've attempted to decode correctly. ftfy doesn't accept bytes as input. + +There is a lot of Unicode out there that has already been mangled by mojibake, even when decoded properly. That is, you might correctly interpret the text as UTF-8, and what the UTF-8 text really says is a mojibake string like "réflexion" that needs to be decoded _again_. This is when you need ftfy. + + +I really need to guess the encoding of some bytes +------------------------------------------------- + +I understand. Sometimes we can't have nice things. + +Though it's not part of the usual operation of ftfy, ftfy _does_ contain a byte-encoding-guesser that tries to be less terrible than other byte-encoding-guessers in common cases. Instead of using probabilistic heuristics, it picks up on very strong signals like "having a UTF-16 byte-order mark" or "decoding successfully as UTF-8". + +This function won't solve everything. It can't solve everything. In particular, it has no capacity to guess non-Unicode CJK encodings such as Shift-JIS or Big5. + +.. autofunction:: ftfy.guess_bytes + diff --git a/docs/encodings.rst b/docs/encodings.rst new file mode 100644 index 00000000..13a892f4 --- /dev/null +++ b/docs/encodings.rst @@ -0,0 +1,31 @@ +Encodings ftfy can handle +========================= + +ftfy can't fix all possible mix-ups. Its goal is to cover the most common encoding mix-ups while keeping false positives to a very low rate. + +ftfy can understand text that was decoded as any of these single-byte encodings: + +- Latin-1 (ISO-8859-1) +- Windows-1252 (cp1252 -- used in Microsoft products) +- Windows-1251 (cp1251 -- the Russian version of cp1252) +- Windows-1250 (cp1250 -- the Eastern European version of cp1252) +- Windows-1253 (cp1253 -- the Greek version of cp1252) +- Windows-1254 (cp1254 -- the Turkish version of cp1252) +- ISO-8859-2 (which is not quite the same as Windows-1250) +- MacRoman (used on Mac OS 9 and earlier) +- cp437 (used in MS-DOS and some versions of the Windows command prompt) + +when it was actually intended to be decoded as one of these variable-length encodings: + +- UTF-8 +- CESU-8 (a common, incorrect implementation of UTF-8) + +It can also understand text that was intended as Windows-1252 but decoded as Latin-1. That's the very common case where things like smart-quotes and bullets turn into single weird control characters. + +However, ftfy cannot understand other mixups between single-byte encodings, because it is extremely difficult to detect which mixup in particular is the one that happened. + +We also can't handle the legacy encodings used for Chinese, Japanese, and Korean, such as ``shift-jis`` and ``gb18030``. See `issue #34`_ for why this is so hard. + +.. _`issue #34`: https://github.com/LuminosoInsight/python-ftfy/issues/34 + +Remember that the input to ftfy is Unicode, so it handles actual CJK *text* just fine. It just can't discover that a CJK *encoding* introduced mojibake into the text. diff --git a/docs/explain.rst b/docs/explain.rst new file mode 100644 index 00000000..c4f01900 --- /dev/null +++ b/docs/explain.rst @@ -0,0 +1,96 @@ +Fixing problems and getting explanations +======================================== + +Ode to a Shipping Label +----------------------- + +There's a silly poem about mojibake by `@shyhoof`_ on Twitter, and it's going to serve as an excellent example for this section: + +.. code-block:: text + + Once there was a little o, + with an accent on top like só. + + It started out as UTF8, + (universal since '98), + but the program only knew latin1, + and changed little ó to "ó" for fun. + + A second program saw the "ó" + and said "I know HTML entity!" + So "ó" was smartened to "&ATILDE;&SUP3;" + and passed on through happily. + + Another program saw the tangle + (more precisely, ampersands to mangle) + and thus the humble "&ATILDE;&SUP3;" + became "&AMP;ATILDE;&AMP;SUP3;" + +.. image:: images/shipping-label.png + :width: 450 + :alt: A picture of the shipping label being described by the poem + +.. _`@shyhoof`: https://twitter.com/shyhoof/status/332621356338405376 + +We can use ftfy not only to fix the text that was on the label, but to show us what happened to it (like the poem does):: + + >>> from ftfy import fix_and_explain, apply_plan + >>> shipping_label = "L&AMP;ATILDE;&AMP;SUP3;PEZ" + >>> fixed, explanation = fix_and_explain(shipping_label) + >>> fixed + 'LóPEZ' + + >>> explanation + [('apply', 'unescape_html'), + ('apply', 'unescape_html'), + ('apply', 'unescape_html'), + ('encode', 'latin-1'), + ('decode', 'utf-8')] + +The capitalization is inconsistent because the encoding of a lowercase "ó" is in there, but everything was printed in capital letters. + +The explanation may even be able to be applied to different text with the same problem:: + + >>> label2 = "CARR&AMP;ATILDE;&AMP;COPY;" + >>> apply_plan(label2, explanation) + 'CARRé' + +Functions that fix text +----------------------- + +The function that you'll probably use most often is :func:`ftfy.fix_text`, which applies all the fixes it can to every line of text, and returns the fixed text. + +.. autofunction:: ftfy.fix_text + +:func:`ftfy.fix_and_explain` takes the same arguments as :func:`ftfy.fix_text`, but provides an explanation, like we saw in the first section. + +.. autofunction:: ftfy.fix_and_explain + +Unlike :func:`ftfy.fix_text`, :func:`ftfy.fix_and_explain` doesn't separate the text into lines that it fixes separately -- because it's looking for a unified explanation of what happened to the text, not a different one for each line. + +A more targeted function is :func:`ftfy.fix_encoding_and_explain`, which only fixes problems that can be solved by encoding and decoding the text, not other problems such as HTML entities: + +.. autofunction:: ftfy.fix_encoding_and_explain + +This function has a counterpart that returns just the fixed string, without the explanation. It still fixes the string as a whole, not line by line. + +.. autofunction:: ftfy.fix_encoding + +The return type of the `..._and_explain` functions is a kind of NamedTuple called `ExplainedText`: + +.. autoclass:: ftfy.ExplainedText + +These explanations can be re-applied to text using :func:`apply_plan`: + +.. autofunction:: ftfy.apply_plan + +Showing the characters in a string +---------------------------------- + +A different kind of explanation you might need is simply a breakdown of what Unicode characters a string contains. For this, ftfy provides a utility function, :func:`ftfy.explain_unicode()`. + +.. autofunction:: ftfy.explain_unicode + +A command-line utility that provides similar information, and even more detail, is lunasorcery's `utf8info`_. + +.. _`utf8info`: https://github.com/lunasorcery/utf8info \ No newline at end of file diff --git a/docs/fixes.rst b/docs/fixes.rst new file mode 100644 index 00000000..d9dd78a6 --- /dev/null +++ b/docs/fixes.rst @@ -0,0 +1,4 @@ +Individual ways to fix text +=========================== + +.. automodule:: ftfy.fixes diff --git a/docs/images/shipping-label.png b/docs/images/shipping-label.png new file mode 100644 index 0000000000000000000000000000000000000000..f3b349542385d0fff19b2579581f7b121d569239 GIT binary patch literal 203336 zcmZU(Q*UL_|wymjcyZvk1wr$(CJGE`w)}4cU&bjO6DLcuR?BpeD?<8Da zRvZon3kC=X2u@N$L=gxGgcJw}m=Fr&pM>5;oE-=V1KUDaSYA?Cm_Xja*3`n<1PDkS zx?01XBYCHF=f-`ekCA)qGVO#@_}G1#M~&n7ZkCbyFCyt20kmkCND@qV1T36Xxj7M& zh$IPgawn@-Z|}#Z=TGO&L)XVwR`ZIk=T-A_(=xE%3k)e070w`x02ph?vAKVHD2K?1qpbO*Y}NACFw15v}(3?%65>x^UPV;g_d+(GvuJ6G}+txs??p zBlnP@&)mrDGq$(EVq!q{*7xn|D`Z-l5Ud%}NH*HpYbrkn*Yp`ZFD2#b>NCAB#jxez z&SV^arcmH%Dqq@wpWQ>Wk0?Pm9o2}B@Jw_Mp5h?jCkmVyNrHJ}cP+0LLgWYQCd8(n zv40vZ4>z!^nIZo5wDS!dE{mopeZ^U70e?^e)u~@%Oit>|je7GwrhdAiNTE^!hTP{Gf0$kFh*L32G@9gANrj4Dd! zk?q#t+VNWNYix~RUG*panwW1>Y#_4l=g|YaWZ~Da@A~Wa!oKghF$7s2c)+&<#S+yN zh4;_K_YTs5%pIBAuP#hotgTpGG0!>pN4oV8Hc%!H6oBBz>@(b_cx`XDJb}p|6*gK7 z#PVzeFv0T*{rS%KcV@ET&)zGszeb1#9LIpFG9tu<(DzyYQ|dC|kFXTY;cYN)h>-4u z@8_L3eHB7E=B|4CN1dUSc)<~IA=pdLIc(6swcY}kh;@nb19Mb*Ux-_ zGLXEMpQFybp5_=dP|cvT=;|B23`h;|F+9-q@yGD|zR$#coQQ=vOhzRu$cwj0Vh~6p_slWQLeG`Yrq8+0E1b=r!kkVoY~^9+d5X7+ z%}CM9){5GS-^k@kTs=Tc+7C zM>BXEkug#;Xfb^--x$!D+8E;L=NRRiA(=56-x*38QtG!)&d)GSNX=07TTV(D$QaO? zc8_okbqt9aDw#GKMe0u&aL>4;0y47u==yE`UeDz8>-PI(dS?YCh)9l*@GEkaCoIZj z9QHX%IH)@WIp7?&_1(o86#K7 zD3>`COP^UAe?!V2q8G9g?~CpW?u-AE;FssuE|AnG>AwNe7@!>h5s)A77QhZ>79=6$ zAw($jCd3}Roya*Ja%ks3*dED4KS^&*h(qlq_L_g25mpoyDUl%&I95lYpsd6q&!X3) zKIA!^N=ibyNzy=)L0UrEPD)FfNMc2*_lHsP?9Wva zRjLP7bn0>vd?Fv|mjncHAsGsd63qgAWV}i$Ts$x7hdMlYt7r_B8XY^y423NPKDiz> ze@S1leqmh6Y!yQ(MagNon@GLdU1>swVw+;|Jhh373H1@nk?PS56%&;Y)i9NuO1kot z@=IAu8J%^?7EayUdRSlJO5g(KO6EG~AnHKfV(uvPFy_VYX6jb(Ht>${ zY5&#&RRHx3qzZHmnGeYiaSm}5lM-VotQ4CV(Ht2cX_b~oaZI#M3@y1Vp(_zBAvPs4 z6*O%&eK@u}?wqvM*rlnWjZufyfKh+fSXQ6bvQQ`0a@Jth)_jL+v^%eRr2zCgL2ABps1^x;!6OIy&5JD5875XEbI(RuyF}OUqHGoZYO5{aU zqp((PuvjC}KNd7Ls?ea2u29I`?1Hz$Dc4w{B1$~RU7%&iu}j)YuS(^jcd4GGt8Ks8 z>bCMa0c{Pv7}O(@J_H(mlQ5YueLZk}zp>P=<>q>RU2i;i$V@2_VJmflH^+Iw{fF=~KTn>=?xBC~ z+V|QEJQXet$1CqB{mC}rk#KP6vLmP?{&D$9HeVFKG%t^v$j?}Np|yBoObq^pM-^pm zR&MssQQOO1CwS&BDLF*Bp&Sg_CQ55gN|*cF@f*>8(V2+o2yCg>Kk!L3x%d36cI&%N z-sJ*k{qsu=aSbPClQS&If0EFZcd7TON~m}DQxzwKuGcH5+w`}>Z98Nt}0bAasqg5v^wFWm++#Rm#*Opt|E^-gL58h87GTxTnYWQWm zZSV4521>nRz7vJCv6}c7JUxtg{IhpMhmtc&6QwV?YkZzOr%bP2<{z`Cjz1?0^aFY{ zT~VIx&z3h~Je#v`Z;%KhB= zuB;1G3jk(Q=GbjpPyRQ;^TBoNWAT<)Qb$EcH9kzQ^Gw&=TTHdW_KiRk4x1o=*%el+^dbQ$Jwc-lqNqVlps3~cE z0ziCfgKcA=K=Z^vVUxW^g?c!*9gzZfVLI`kY1w6DU0>4s*SnR_?oQ#4z(W97x+r+* zn#mZNoGL!+{!3*=GI4hFjS|473ePVuu1^gR`k~^rSnMV-xHkT_15*;fgh*7y*^&SMg3aD4f$+_ zHM<_DkT8{V>BEPz?d++h^V5Jd?|MdipSu|7kZ=ddOc%mVQ&$91*P_KM&U$uMR-!51 zEe>G=d|NU)N5YGzZ*IsM<5b+qaZV=+dUkgD7rX0tVQ~b!joe%po_2y`5x4vs$rYjr z!z<)=rO#4K+1eD@WL_S}_O3cZ*Ww$ENUC7VnDb^wTdV8rUS%!iR2Rao?P9^7f=AN% z;@s$k)Ld_9+pj!^2@dYZdTYAN#1S4FAM^Zc$B}{= z^g+_QS{glHni=0rUH8s+7r^JoVr(pbZ&mHjeJ|w*qKe2#9c~*<7mA>NOyV`JH$r^` zVJKtFc;;GwXrPK;wEq>kWc*+xb8clU_R-uf+%fkcKg}5Rp}_R-YZTIm^vIjUHfm9F z0BIuihqA5&q553~Q-xuvhr>f#N0(TaX(?J@OQLN3by`o8V30xKTts+|MTbV_n`0#A zgzd5$c1v|YOM*+pa|Qf)%(-mE^gZ2vjh7A0b>COwXxK?0pn7BGa`mElN^2@~*eqoJ zNb9(DVuw_KF%E+R(}}u`gM@F9P?|0a@tdt6+k+_q&7@H&0K%NUkbV>=0xp6%>Ll_f z;g+GEixRNXvXz;LnOLf5b4|I4oqe8Dl1i5$n9A6CV_|Bgy7ui2InZ#0P;qp7ScRW4 z)}5&4`+6GRR=5#!c9Iwycwh@-lS4r*%_~XCM1@A%pCX-e&a0#3Q5WSg(xcrR(73O4 zRN2U0@wake`fV8_8@*iAYV^WJudGVW_O{pDL9z;IgTuc4U?TV>Z zu1@cq>VnUu*x#u-Ej--gY^>Je4!6#9PX^u{kCSWZZQLGqx8B~H>%gASudi5qUUwdj z+t2Zd(%Rf;*ckY3Z@xRj^P^6+8@(C$nOK^hH?EF*G)(l|x@_sJ3ZEClkgM*iP~t}V zV`IMJnONTzjjo=e_vXFK^S@85jW@#l!r!lYz;Aw>!CVGsbt`uEZqA<|`N4F-+W}*8 zS!Sn(H2N?mY$iyiZ{s0mCq(0R)_W{R4qK>yyHig%ZoqqNzLES@2;mzN>Z&D*%xhM` z?|i~VDQzsg(+Spa+9U6UVfd4RlBbjdR2tb=%6a8;my9=wicRvbY8%B{MQU|>mv@^C zaug!$)51H;{nLXuyWg5bTaPR%F|3%e;WOu|PXHD!=Ps4cSMZcEnlXQ5(X$7&hjg~v z(AqgWnLA%Riaadc>1--D@S23pN8?ruZ2#a7l}0BVb`HC))sFL%36Sa&))7(=sgt>J z6;UnnN8zyH=%VgPkt%!;BZz5S4qbK0{E1$U{Kj*nWh0cOwIS@`BcxGavgavBokp$M z0v&IsMDv{GZGAcJYbVNSiQCSJ@19{*VtG(ne)`gY#AyU?+`;e4Xncutrc=9C*WKD{ zBNh|ai0R=|O~iNIZ1bzuKNJBN>`OKh#S}2QS(4>8a87x8bTpOI97HL#yS-@+>mLm_ zuBm)4y`Q{Jdsky}?8s*-p*!?<=@tNQFxcNc7e|adPJF9X)mXApvi@m-(mAj>YMWjj zS)R5JyeWC5O@>dDjop~jao>XD1?{GG9e;7z#P_W0^?fKD_irc_4D^^W(f!-I&6U9Z z?#|;bxQrm0^|vp9jB0cB=fRNC4ZJ&m4IuJjkZKb8 zj597mYzHaqlfy8L;xVNrLv{pn1b@JJ0R1gUJ4exkXo}3&*Ek0CXE;Uw#z4(Zk%4Yv zqX;_6J_1Y)0EIMR%S zfwkGb+4|ID$XZ%^Nbpd?zg?6d=BM5~*dLP_i%wiiOdeJU2RM8u9 zJM*0b$q>EUTu!^?2bD^;cU~!SJa6Ttw4TvBBsV*b=quYTf~)+1+f$U9emgJeY1+wmm=m zN_URz`<1r)&(SMtYhul#b@9;CCp})CyU(rXwI@C|iQLC6IX#XJ*PrkUd!G@&j^pnb z$i+cG7K$00k8NkqxR7GGIg<6QACHUr{ynir-2g(dK89NQUd?A@* z9SX(F!90+!1kvjY)XA+OqGrtWm9K$}?p!v}yLkAZ^!$B=5eclxlMqAB#+<&LSe+go zGakzyy+v#cf$o?<|L#u84~H4R8hVWtjVnVJZaW>Dq(G&(R{2yNSPNP^UWSu^oRNeQ zjpEAsV#KNf9QMc@c8=EuD;5R8ESpQXQ>6e;E=AmZtl?Q6H&0Z z@v*RG5f`w{lUPQjU=gEjpyOv!WIQ^(e4~ExQkT=7g?~IY-hOyCFgrv!&UI}QuZ>#A zCc@~Vrl5hRjOA)l0r(SD_S{sL99+*wMVPxTDa0OF6OFeT@g% zx1)FTP(nchLZAk}O%?9X$WE8e5N3M`E07$+nPE#{e@DSajYs9C3#FIn`xs@ACYVIC zhRP&ek#*9C`!D`Q8mF9i1FpGLwo$_))ivFA?;-q_^%v`>%2#x#zom?ane(V;yX!<2 zPGFu7gYB(q$!h)d39gJ#`uj;`Q$fzwtBAf3Hs6&$T!LG$P)%Oq+owM_w&FIsI9)X} zFzqnAK4UtUV#$8Cd&RMTdBt>ucJ{sSaO*OTi@VK-T4h_^t3|+f>LH<=X8elL47)u= zT8M6FJ9I+⪙(7ETysj!E+@hCS%(FU1Xh#ldP?{-AYV6>JiJHL&5Fd#&M;rZSk7! z2b0^ly`;;Uq*?mwFB}azKijho=T6en{zk>p$UL7%V-N9Oiw*l}ySv#6Z}SH}e$tC4 zk0~C@Fl2q?9h2al+@3lG^o!(oE5qPhI!IJ&c#(Me9`_8CO z(GTj^TKje5-p{FkN?KzO!hkv0LM-vcMI572Iee=~z@3uSdDbs1?cBU=ESfw8Tj37s3j z?%x;z2#Cjx>mLg+aWWuq16bQQa=Gym{ig-jKmI?&^h5;zY2swXOQbF%PatgTU_!u3 z$4JLS#0Nt_K)~Z*Y|5o5BKF_#eP}v}V&Z7zU}5KEVQWM1AASuCZJnKXiHQCq(EreX-P6R) z;(sIAIR1B9{}QDC4-Gvd9RvOU`2Gjw`41_VyoH;IwYrD}z{JM!UkpB07EYf3wEzEU z{x{oA|3?Dm5UFChyH&`&j-VD6m$s$Bmg8S zBB<;JEWM@SLL)PuZN?IQtz0j$C8@a&hhExWjASm4#r*RQ?{Vthdh#{|6gL)S$Iia7 z+##OW`g8O1uyfPu?c>wc>;1L4xw*5osds&SP0!EI&(6Qo>iQxlH-mqZi;s{0^1Qv} zz4dAS`Sj*()%o-0^|O5a{4DlxbHlZ@y}7xywOL^A>g#wxz{LN8eshEK{=Iy6?DN>` z)6>(-cXLCR8VWfy`+1L!6!p2ewZrA)==kyUb@%k-UD<2ngO6upV^jBWcNczm^j(nJ z-`)N6)-ui3g?;hyF(K=;Uvdq#x$*FQclUkwZP#w@UyeZRW%1_d>ucu3<0eO+4(K+uq5;!-sYIaRK4R06dBT6=4qyu1y{(%t6RwPS@h;u;s&t#UI7p1=+tW)_@PZ`Ss&&;HW)}L)X6>_=$ko&16|@P|?*5NFyx2BJPavnq z|1=3!*h+Emyp{eUD)O720l<8{zq);RtsLInjf~I2EPA;m)(xob7+#hq0Y$EX_=z^ITu!34L88I-zdP;eYiqm%w`w z-{0Ry7VPZs&z4H>exJIyvjo}mCz5hoV^^!cjZm#JheuuB#RmhpQ-rRTXUqd)D`rfb8{JX1yV;ti7WF9#fBH{JttcH2KjsW6g`66 zKsH!6FAo_c`(_Q;`1x7>BI4uuY;Exub$4hmAnO zH-|^pwRL-beR=x0hFbAb^w}I(9+(%%7=Sh)o%58M&t_`znJ`mCy0rn(THH!Yp`G~p zb@_fi-Sl=Dsp|KufHG5%@t$dfDKIVD67GL*%(C-K&mw}CV5E^)6jHQr$j}3-n1~|baD__61T8pdU+|frNV&2LM zwi|zASFWdbWWnjN|I72m=hxm?D#>h_#<`re?~WZcQpuho3F<*?aBb}h*4};R?tu~A zA<9{ShVs{U`>E9scW4?%G4c`_y>VV+a&Y*}#|PwEtY!nQcY7IQWP+`S*OpiDk3tNo zcNuM0Lk)-33VN|n5S&*}FX_=8I};Y)z%bOJ`<1A)F#F`MsRwD=1fb8CSZUZM_k-|$ z@IKD|G(wFK#oIm%mG{ugBc`;1a(}ftOEEMlN?JdC|M5Wdli9V)^*iHjIPmf9V`U#0&QJ@SowW{<>q*n>!-B`0_}V zyPBCwsTp8$oqf80d<7~!Z^>-Utu4LuS6;hy1sP)X;`j|p7xSJqpZM`7`~Wcdm6RVz zPstX9*~$N7KV^&|jj4*^-n$Gq6*!1kRl{X%08M1h%R_!{xXNS~kc4IA1vN@od(JXp zun!G`L=0gj{A$2q3+{{?WBk-hf}*vVDU0&QFCcRKxl4hnGJ&hKLg4(7+*!4D8h>&W z6*2ePhA|Z;)i6%K5O8$`zQ4OBp65hDpmNGn4w`$pPFK8U1U$H(I>4c_gdgSUTKj1m z9R)=rp#^)QJ^BxATu{S6NangZ&wkX!rb^yHI&ueqFlJDc1X3M+tzivgkSo9En%{43 zt#}i%e<*0g{0LCL9CoG;jvOrfZmsVN&VV_U`<`iCiCuYFZCBw9d)4gwxscV+E)qUi zW>`bcR@!tWWOvv2F2dND9OTG1>wT$;O;@PY{Dm@dIg3G+^$9=#`%?XtqRD@=P3W01 zSs6td?WJQgH#VxT2HwBc$OcN9B_+a~zjxvA6Ne~2LV@SPvm^+N&sokl;YmoO_;7H4 zyIy(OKoJJSxw-0EqIXIVt1kS+o_e`VC?HhZs(G7#FMc3_eW!268aALhm$m}KYGLk; zt)*{v&@Lvdqxv(T0BV>ZKBsRgBnFBiKf&fCMF)<%5fa$$Lz9eHzB~ppA^Jm9o=->u zQ9|odjP<6tZeA~ND}8MW^3`UR*_N+d2KvhQAea=p*sNo$@08si+UHTlb#8B5F> z1uq4o8!mW6nveiM91id6z7H+P-$O((Ku4NPLw0Vyg}l?t@ulph=cgfhe-WzT`@51$ zmc}+GpNsq~#1n}QUN;dnYEw@}p@b22Bhy&qLA1zD90OkK!&K*#n)kIMZr{HllnU2o zpH9T?cB~m@<={x3kEY2amZWcE95e*K?Rs8C5w4Z7aRZvkf@R49SY$a=i%)R&QQQ)H zLZ&CDRx4*|jRXdb^-&;eF)46}>NxC~$3Dg!X#Jbn==FV=X!Zx@U3M?pnB1|u8xVnl z-(BHz`*mj5Y<@K11xP#WvD^U^E}otcD@r2#`F&t7piZi}_ph@v-u8Ntl*bBeTyF~v zqk_|f8RtJGwNbTWI6VwhQ$#sJe5pVy6j8phUMzuDrdA9d(QCE~l0!%mdiGN$fG2yNR z<5{()aCC-o3U1{{eLzmYdkKFMn2^aR5iMth zAc>bc-+Vap(*R3C9O0t@qDx-mQEIyn2deE4twIjhiq6(>>p`QMXX=|e_xeB$uw0af zd-rd558H&p%49;p-#AalK%yR?=MJ{ICSq%2u})GX44C6~N|+b6Dg);G;jHLYT4{c` zFzIOVisKF91d&cQcbLw;kw)Us@*IOso z7Z9*UpjAzAQ8Wql9$G&(<8i63TV_|nVFmMNW^bA5Nzgk}o)|Rfja>oa5N`=DIO)r2 zBAfViT99JH3FppPY}x(`CV%Onmq{J5m5t8cXk>P&&K4N%DH8?p!ebnKF$ zCJC(25x4t6<YVdQnnAk`YaQ(sS@j~|DWVjZ- zSHYsQnA7=dg}}+Wv|o4)U=3!>y#)_dM>rFiF$DtI&?s-ux>MrJF0H^qohkUJK$R`CCSkn5QA2rNo)BFU~SfPHd6X z_c;|UX{h+Y>bwP5pMGj)Yd;lmjq*Z5`03HGR#J~1m!Mm!vZcSZJ!GPS?Tx?F+Rr{k z^^J&s!?vDo?sz?*ri4H4v+1a%W<@&*kthXTue=lw4@PlZ%dlxe(y@WrKRaD^-ap^a zYP9xjW*YM=y3YvxZQ-Fj+V%c3p2raqM226J?OeDlr0Bp#MDX4fY@7Ed z^Gu1%l$5y>J{k)o2Wo#f+M5~s>Q|L4{8X>?^^VcBE$py&o$;;f$A54iEq?@nnK~g5 zDyE|FGvAMSRZfcwu~r?zVjRyw*wrNL9l44B1~CaYfNAnzOW)im#^5~bi)>&}`<}w9 za~*)M3nqwS00)7cpeE}yyHF@+5;?E;Mg82H)F_&p!&j@h^dURASCK)nQB2r@cixLr z6Iv+rGeg3JrjXE}?d`|UGxGrDDXNbN+9f~9qQMfN^?el~v@qo`SHMowpIH9l9L#>f zRNI~_&~ToET?H#b>;$pbqzJRReF-t!n7>lC2p7c=Y_SA1HY;*eCD@fx^7r8T(D9=+ z%zk8*LxK((LlXPYhRZs<_Y_4zCyR9fKOWsn+kF(L4N9K=(8!ZoE3EoNfxCV()k#Th zwdTIEHa^)i?8}9M=a?#Qb2Q~Qexf;?8nv0skOC|mQPQJ}8BZI^XxNHRxvd@*kGC)K zOXtU@4(NR-qK=A)`IUg(#ci3>W^=>w#+PY2SBSpFL#p!-a^d2%DGF<*bdeYGy}XHP zLEvXF*lY_AVmA$-%mn(($dDe#!;xPjYD<6WC*VVb7}=){M`{4Kw%`r-!#w^vh?f~y zj=-BF?wJL!2AdETf-Lk(?8>1>(iRR=V%jD1S7A~uMGgJ4>9EC_z*iMV0rlBrc^Qrh z#JExu@ZmD&pm7<=M$pFR_)?OL0^R#6%1@q}vJ&%`ryAW$8FPp4u7gb>edtHTfzkubEseZ0EQc3vZ+u!2FSWTI~|tP;7H0Bh1T@62P#EA~U1 zf^WdUXK#&T8YyU~%?FWu^ESakg<(((y~IG8&vm9-DLdIVq4T z2a`Wf&nlr!q)D1TKW>S%aDGYh=7*XF_-f)k2TI;%&9GLXf}VtAcf*? z|LE2+AoDv0qmEIZq`%QB-X#r3strFWG3qBnOd|onh+_^=F)JHaj}OuWccFBN-ZGW_ zCI?osZvl3H*7>pD6-i6d&uUB$%1Rwa68sFt{19_5 zm8SEz8$b;Op6)qo4~6hsn@w!&Pt{y-)?v(%p^vYBuAlCAVKyE;u?bY}vf)?vLtR)j zS8OV2)W2~`{BDh2b(!DZfm(@vBW+OOr1sC20;0^n)z7VVZR)uWF~we~YL9}NIkSN! z`#hY4L|9M{e=D3_<=e0i?bZt=;21*aH>PQ7jc!5^8lUEGzL-G^j6B1NwkXRf<5spn zolVq&utmncG>b?Heqhe3b0s4D`+(&-DRr?;U!@DV@XqBBKX^w3)W-3r-tMV8HU&*B z8aiX8IWN>Rwd>v}%PddTf(j?g$lwR^3W1tmmLcWuVaA(TIziPB6#dXz zS}Sh5I*PX~a*Ky+*}mLMI2js6V>k(0n{-|?ox6VwWp+6`LR+ur`%`39)!sq%Lgx0v z6&wYZGUxqI1SC~Ido69}ArN1-vr8f;cop3=GI~#q3r@iy1z{!Y{ihthVKy>On*u2J z)kjcLMtrkIsULYGatSb~z`oEM<;CauN=t(3sZK_NX?R3AEOHXtDp6uzvHi$Hp37#QsXG z+NHa0;)nt`M3}z1U?Qpx$b+?FwJ42NMONl0YB`4k;}weD2iP;Sgps!7)LSIaSmC4O z3)pb)m3JxDh5zMgYd|`gys%8zPhExr$icU@gY48-I3FZbJDto=?WrKQek)~fPs2Is z7Fbv{)X&augJs7%vFcFds-I<0G#F!KGx;m9x=%lTr_9V=Mge#&h08^5k74%!cpkp? zl1m#K+(I$h*fT55kM=zaziIH1Wwnxu-Ea&K-BjzE)Ezt`ZKj^1g1neBKi^S-CHt*! z#}yX!*BOG>+3YXlLe`7M*vii%9Fdas5*kx*|2iihrb97Qk=s!Aflg7`n^4*9N2uU_c zwhJ=|#_Z!?w^TV5b>q%fJq`?q z(h1o`xcq*H>AwDaeckx}90?(vi{bj-ZS2RUL&ZA7vA>9AQ96y*4MvG1AvvVcHZOd2!Xewb_9lxijWr#>h?@k4U!Q>i=Y0Br=6|^R-Uch?n zTy47-d9o8GsICwMK5B|bwJ=QaMm;-&N7|Den|RK8+zFX_Q&BV9gEk=+n5iUWDdgxl zk`)YP_VpSwP{hTK^0dYx9d*Q^NQ2yv4nRC9Vk&egZ+dAruVV;=CB@CoA1|=a6KZy( zf~5<=UsW+na@0_ihzdZkiTL0%g6(c&9ynTCTCJd1pWV?&$mTV=;FwWoW&rD;@VgX^ zy|w0jz!6O^k7HNL`s2V;O3Sj`K+tDyHCyZ*hR7`ac?D(tiLb2C!oC?0F<@11&kU~v z3ls<&)ykF}A6auZAy`-^*<!YM^uqFE7lU*A-?VvQ2{Q6nNG_&l=KY9}m0r z^6+UmXybf_Dk!WED%q*!9#*MZtgTrsv@In6(fIg!_wn#i-jSx@G6`0wVMIXQ|6^e} z%O17skH5YzVBDu@UF4vshDUi<&y6s3d~}xC86XCtno^u(D1<}jNvh(BP>3apkYsPwGrY4%{aJ9yC!G_!4!(etc^aP4eMo$avE`0;{OySc_8- zv~*a&+#a5!pcqDQT*wAK2Ec6!*Wju#KEZ}C+KK!=CQV`Ih`!ENA2TQ^)9n&p@zb3` zvja-M+n@*=lJ3YlSwl(`UyGp2uBsi@71^w7&^K|vB986(8&fnniFG-f+C4)Z-(aja z>LhmmrnPR7Gc;2%K}Nn9H*bg$U%jx<4&pngyZenvt)9`hX1D7tHv1@|vZRqF-RsXd z7RhJJ0bGEU8sFh#?RD|x{qcOwR|;hS0n**b%~nZc-a1e}aUOBja;qsKEY%CCcB&<- z&}^S5(-V5ggfj%2q{;0M{`(|en_@fRoDR~wD9nY<2~CUFjWzInB=(D3^UcjzHZ=YP zo8BmEUpBAG@93NjcO?%|<#2VV_4xv-9#tOhLWrJCW^zbKCR8)JEmb52eLalMI>Wh+ zAv%)30EC)`q#%i5=~Q(3?G?I2t8!lBg3XqT%OLeFP8_|N2VWqXWI3Po#ck%OW5^h) zYDLy0Um4u=XIqq$BL?Mp2dcb*#GY5IE!+$NV?HofNv>kN2P zoCH%OlhWtLca?S+j;j8~a!yu@GD7I0!`6N^!%3)7wY|7vMsKCwD$>8DBbL^w?=U<` zS_W7aCv>wLV>T{D+03#&CHrlA^&-2`svtzC)*Q1R5l5^4}aVSN2|( z$-jx_O9_w679k`nhSY0NN<-8Wf9`g^e~B_-*X!^mfGFN~JZUlz$Ue^V9XHfbp*Ocw zwK?>3ol5OFgh^Hcn2cPQW8iiGIHJWV@VD^^BV|8}3ZD4s`!z2TxNzC_)HC)=bCahb zrR8|q=maoMt|Yq3Qy|E=X9;}o4+&9RE_;_`bQEu0(AqD|`+Y({FzaEeX>7WiubIg- zWQ}0Q^8KtjkflPv%)TW0WX-m_U9mCftbIu(g%c7l^eDL|1WPcAKqD$4^Tu^S_+vv6+ z76ETR4S?~EctrRKwQz0U4}(cVN@grOH3paOfv#HpJj|5*ikXe*9wSW1Tp6Y>IJ5FC zmBftqj8FI!WG@!0yWCccVM0V9VNikC6B0o_i@oOvC%rF8#QfJZni31Tz{>|Ey>P$H zY#i&<|M)3qocB8TXl6(sLMcZAYHNgs`k%|kiZrkv{3J7A*JNcf0MDTNg9{}055a3(vf_(Nk zjAgqjOhqw3D{@&yKHu%qVBesqUKa;nXARdNg=gv5So6!~;dMaG`Noj;6Q_+$NDD<2 zRtiCh9;oQw`r3@>0_M-=73=}YObOd>ggHioVFZ^&Qw&7G;UAD# zT@jJwc9Eik5iP!nDR1Gyx0TXM%p>7%q-uZZZd`%t{a?=S0%h$Y6({XF4TPbe-yGDBvnbD5;-uP; z0R4H)=d*Cp4)%gK@#8FLVN@2Y80*O zmsx2IP;S@PWSpr40$xX-wq>tJtT(Y(J|FHiLxv{_*dp51Isf%sQOFQ9X0_= zfBI!ctL%*=D80u_%hUEMEc#T-6QdYtaZQ^{=M3qL&)UUT$eSc$x&3~=>gny;-ubx~%OsxY zn_d^csiG4*@tVw7RbHNmBhc9OFKPIL-Q9KnI725~Ig=;aU@m60j7+&WKAmmU{ISSy z-a71qY>Z=G?TE#nNhoUeQM+jwVw|evigYY{1B)a*yFnrqLOh;d&U8`JLs#?%FAoHi zGid?D6kR*|tQ(@GGLy{`Vx3n}l64ECU9%0#2o(?5TBT?_M6)sZy(8*_%E-2EFgZh3 z0|b6hjJ>HCqP7M=8zY)baZL~bDJ*o$ymY++n z(q^U;7&~8P=&IcWkGy0Skn$A2qM4#|@eu0*LvDIn&Fgr6kh!fd=U0Rr%BgOvuTUd!A8&^A z00w%4>my2o&Kl!%(ik>TYWVeSmRm>2CJ8BshUbaRPT`W3mRhr+7%+*$h_BsmPFe9P zTptqySWL2nPPlXR%pKXvgTWX~QAi8F#Emm62+z>(eWODs;)382#ETgxKw`@Pq`?4ENi@-Gj(~|WONQ!%b zFf)e)NLrl!BRfo%T~{52@QUzL0K+JK4rMEYw7Rz$y{u#ZuHL<8mZ85=4d034uz(c2(hr^;$^hW6{!JWc%D zec>4ndxFiCf7CTF9; z=IH$ImuJB#Vv>vus*S_QHQuGCNyMHA#ojXCfb8&=&QHj(f@&3X6Z*kYo3qG}ohM60 zw%ujKdfmbBN~cF~oYjpEr(ePBx@lOfaiuTVXr+Rat)o2fb}f{x6WuvW83Qf0RN%P> zv}5RoX*54Q{fs#d zdSmFDU_%I5ZFc(;8H^b-q#nckE=+S>VZQ{N{4CJ#0tm~oMXKE>U-{&nlTJ4bK3Z8d zE>wN?D0G}5QkL?Nl?X%mwq+G~qO^vD#HSC$vS_}N{Z#eA}gQH)i%>E#4jF~MwfxYu( zeR*(y>}m)hH)`}7s%c|0hrMvOG|Tmxp_sAsVQ^|%*e-F=-PL!U3p0|wiCYrktj$@R za)gD3=o38S=FdMxcUDa%{~0!QhygLHo2rxbZI)G=yfemY?&QVlegF^2+GdK0xC}NA z4f(-rE(4By#V6lmD(RZjDSWhV$wv2mCGGm{>6#wQGo-4dN*?ziISooL^!kkIM~i2- zi}Rkh&B?WKEx=zA>B}~27p)5O6#MB|oWh4Qg8;_(@f7v7N+h7w31{wT=j9l;8O%eVE&VwN#|N^6_EDRf5n1#Q1Z84pJWjWsRl@7LO*! zK)cFqbzWu=1$|v2@>(F_Djv~3fbc#{^!eOSrBG3ANHIeg1HQfdX^o*a=4Lbq^i-)) ze+iNtqyq^VAcSqICj6Mr6yjZBr= zdufG0T4>)&aZJ7cv`{OOx4O`b5OS^irJ8Mcwn*IY3Ac(M?vD~A8_lRJMmsz`c=aC6 zD|Di`-ML|*tHA5HPBRg``UL>}S(C7JF|;31Imv2_vx3esuUtKUbHE<%;ZVVOqq`C* z&vo>$LAqm6%)!3Qq?55JtK`rqr>{BmA z`2goq(a6rgNR-I&yPaBwz@LwqSDiN^FnNKa#;H*o=z`R{6I^K3&OYL-gk64N32S@U zh)*Af7m=?ju0hMC_%v57$`CHItsh z;Br97u4cu_k(3Eg+K2_qo`jkRg$M8!rN~KRff)Y@)cEtEGSHX`!ULm^wm2a~7v*-E z%^_VFr4pzME!Mid2^5T8V@w2-=C;?)gfgO4_5wl|wk;}Bv}t|Ev%+c6hkUY;QOc^v zLwwqSy4Se3rO?c)3VCtZ5T89);BR<2dw63vT7OJP12wVqS!}I@%TxU3k)ofcXhvb} zClRDOO7N~7PcPMiv!9_C>3zJp5G$B_5~`s)Lsi#pBZ`lpB%W*qrc4{LyuC}v`DLYg zHwia>hf+W&a5QZ~cM2E?fdtU{R*P|ioKOQ}4KN|qdf8iTQsRmfJx76oj#x{HFh{>pI1lppq3_bSs`09Q}U=J>N>G?oYO~0 z;v%t-xUTO-J)Vis#Hg(4RnKM{MMYgQ{DBrP=yWO17LhU{2ZppgDJO%s$LJsfMTtN%!8KdT;YKFo{4t1ZR*MZ^xp#7mcQ>excUEM#{7 zCj+Ja#Txnyr73`;6f#P>?bryiOOI0fpFgfXH+S8&l?_C$lafy%T>C59*%&}O%X;Y0 zx$s3<#?mpxQ4M$(WeO~oR%JkZS4+Ob6Z9E=mJ)^{wc*l1UM8PzHLU>N{$bKkM+r=G ztCPlBqBtZRpAjqgO-m>45zshEOGBMrQz4056-7;Js6aP^PBblOp{_%wngO~x!q-}Z z-SS1;lpT)i<`nXs&%&@?lD1o-P+%kBl?Nx}C?%dYG-#P%qlg08j`vOedTkT9R?MuX zocs{iGgA28hG0Ey#4-#1=<p-aDETWHxZ3!%-4%E70j66TPj%6b|Z zW%`nYjxNw_PMdCK?u^e^o!u|}v$Por+b|C0XL_yst$6Afi;ug4yY@_QTLO@Yx59z- zvKgo(7Yw_nD|Y6u60s!rrn4Cl-zZ*xXvu0Z?vAh+QXb&{#D^M;`9gi zEA(eiI>Z;!LC&@^t&I@7Hvh&W#pMxp&Xo8|1Z|@hu+^423I9yjS=-H_k^O#) z89R*yrq1dSqh}>Q%)}!y=l#Q9Y(y6A3llStF?}!wX8*`>IA~l=i9(8VEE6i$*A!{5 z(pBbNdN9$mC3)=-Jm&gn5&WaxgHn}Lsn3q3qw+Y()K5aGF-;H^=7a-H3m zBpxqE3Fq;gCkv3NX}>pC6XafG%aTk%DmcdaB?@}tgDdPY<^pOrCYm9=Cj0fQ0x2YQ zy)m4gkN=<|)%MAcm{30PhIr|gIJswCQj2j%f~e5IfqK(OB`k z%g!ibK-|e(6c@{0!5u_Qq=f_a!kJ^DaXS}iovy{O;76$tSfZ*5Z;yRZt|r-HM}2DT zAfI}izjXufN!Hnb?nL2iopw?~MAN?ciari7oX_H{L;h?nkbHi=z5mDOr>kgQ)Or~} z^FO}@W{*lEX^Js^zGmv#i8_OLLVvQS!Vbo=Elnhwo#NVJjRH0`=$>TQ)Fw1r8Z91W zj3LvHPFdpq&4|%41o9Pi{wqjVYZDH&R>!oB(O5|B8#reoMik3+_DXsso7wu<6jhmH ztGIKE^4#nvJ{u{9Gn};Sg8eGlYaKRW$s4qo>ZHV$0-;x=`*Rympaj2gL*%n;v->4u ztzei_-MVNK0@yruQLV@x@T8+A^PXzx9@1^?lcYTUrWiO z^}4^=NR1Y0o3NJ4JC7CBKote3I7$D6I`5Lu(O@^^qR7Ui_>Of&N*FSja)NkWDsBm{ z_t2)dm@FQmHZ}mPXB#$VFei^LfTfPc8c8~>S>yyram*1p zBZRE9RHCJ=G)1&d6XH|XuKFp0kiK+6tp!&)Fa+J@S}xphV#(en2n&FNOCQ+81i@N0 z%`&{d{~!PDUqx^qXsQ?|$_SIWe3ZEDBp)wq;es(>N$1RstO?c1jP)05<4b6)S($%PZ}A zYsbN@E})&;Yhk|ZP-k;`Mn%^|lxmLA0KaIwS1r{LEw|z(b_~9Eu#dP81Sbqf-3U-W zLR1d{&2PZv<6xpIdKWXWWi*m@uuA(Ivub;eg_$pdq=D3qRRA!hQ{F1hp3 z0WxKPD|0=+WI0po4Dx)++eABQFOS*0LDEdS(NB@b>M%MP3R0m^37H!O2_?pb!PAkx zrJFs<1^7wx_4WJrZvk$a&)`dMaUS$G|Ge)u&LSvLLv*Xa5Qu zFw1P?;nA0;yjVT%-mgX4w6!Bfo`bZICY$DN%+O*zkRrXgK{2I?PM2inw+Y1UMfu_m z4oS9{nXD)`ZXp*&O_HeT?epWW-~TqN=hP2RE@+K7-g3$=6@ei_I8P(#+?HKc<5v$A z&3g_)7nLn!NHDse)_|*@oB-FxuQP_+`N%Fj!fM=jY%{ChyO)&+lJ2}_>P=NzQ3s(DB!Vnv|M>kl2 zl;voZ3zXV2t~^H|FVE>a0eu`bhZJ6OpBmA@hA8xfzV(L6>1@kSD7GJY;S9U^h|deN zU0C=1_25?Y&Aq7O*RSX2UoQ(u7SFoSy+GO|sf(eSJ@mi}_sPGY=&>Viy6iium}QZz zemDZl+zM44`*_~q^WovyMPaYLr$B^x=qq!mgW(uRq;l?nnP48XY+|JsEI##iHjrZ`Ufl~W(@Y4`1Qk6J3)rM&Vor^zZ~ z8}_B@;-nFZ{5rd1anUc^b~d6i-X+D3;9`B)U%^s5?xFVb}nbQ_=u*yt3eK37fOnOY`b9T%IO_@Yrh$%W2FYF+N z>A{EVkn>xITc$K{QO^rlQL3m|hbHRr(ZkRfQvy6H!NjJvgRGVUZqK`XY^5u=)N8Su z6yWc=~W7}*p04!wQrPsxz|lFC-qfp z)NoX!ou7%}TR)ui=XrXDbrlDymJnqsjK@^&qrB)`Uv+1*n={%jb1;1tZrlwXun|e) zu+Dgbwq>r~1Vf)wSBZFd#46+o`Gn~?Xcb5Z1EMGua5Ag!%DV{k>CJeZE zUxC`l8aDqlcnBSZ9L;cP0VB4E=y3_|OMsC;{9x$>B3yJhu%{7nssF?b$GFWob8PQz z%n3HE7~m?Cgrd5PfNE-ee)`Ayql`K7$iPLvRNC-<4*2V6BuU)&J9P8JA|^_jY^$_F z=(?KNH21H+Ts{0%NLgfBD~;itL6Vgdm4}R+ewM{KBp7BSsiBithDt`=QR}b<>Bxyt zqSPwg-dpF4ZK!5{#wh(k=D71&+bOIMp&Wz;ZHR?QkhO$pHw(@g(ekl6&GzlP^UP3F zsKU$VX#siw;G!{6`6bXfsaSRI^NY^@{8Mgat zZNJLl_Wf1Hn^HPps8MaGd_awXz4Y4Y5PPewwr8Y9bwk!+){wn#oGP{gWOWj*Wg0H@ z{_5H(PZRk%$y`^x-rri!kUS(FS}6>ImO^gc1p=2mjH#ZtQiB{+{W07c>pFaLEh#8} zaVUM3Vf+%C5?2og##9AY4BV;>Pov8tO_7_d~1(h)-;6AQ+NDGl$r>LM;Z&x^(%reCJEJdTzHxu5%iVmcrmW)u6=EC@_ zubtjU*>?LyK|t<;OV6kbw-nHcuP|DHQwq^QQt1Ig1SASym@nFW;)tspd+#AQ{;(PC z0Ms!>7`MIgV7Q`{51}`M7SQPCIjX8gGW2H%L1LzWCz{y@(}m4zrdB%{qM>|lY`r)eV=izm#IeM-47WzPsb5aW&R(_Bdm`SS+ zAYj2#EU-%L5zZd%Zsg7yG{Ld}CC7WLyDvYS8I5;Mh*}yQIQ~JCvc=kLrIA?xsB0K} zMO9sz6ehLrG$D3Nkv%wv*EysZX5|{EfEx9j36ZBzz=~`$pkd!NNgNS%r{4kqh8?G(&ln62khl-*T%-sg+T_`ceWvD#N8wzd`0P##w zFK<>|7Pug3C~%_KnVf0Lp0$j;0lgXpLl{KUsI3g#T|Cr8!YXM=#Y0%vXqH z>3LI+WyBk`Xx6G)U&>Il#wm%uoh>tuVm)Z#u~@;Mj1_{pL?xZgn$CdE^^fMp1ao0nM&rXa}w2xqJ;L&8o84$daJK=lMx~ zfAfTGw=Vf=3yT`fx zx6OX!wAXn#DJ3kdlo)9(wV;X~ zw1O+rjJ#~A!{Vu0zO|AW3`wJ(DE*D~px;p+(^kKKHhWNjU2&Xig30JKe zx`SahsXKrQcvAQ(x0T*UqOieb7!&}j;_VU#(UDv!v~XA82DqHCH?d_yX8LGiH$nQ8 zdi1hstq8q zHDMt$%d=Pg8!jV+;(h~{ZzS@9M0nNUU^|0HbaK=LTeHF4tu<{lLAlbz^qy8_=&#p8 zZHf2{lo5EpEzGPrd$8Tl&9hZL0(Q~DAB)l^&CDt$J7!{MwRL*7^K9+RuYH0<-%WuL z2;K-QmdUCV{#;bb*b7=2^I{Y_3z3<2$i-7I=UOcru-HcZ>GmJlTetV`*KlH3J~^#) z`FOD8B-+jbD50V)qbPhIPw6s16Q<7YaB+&s3_0oMKn;tb1|cn67e=#6*SnI!zQlI! zoV!({w{lt(6}%q!_8BZD!<<+amu5}LSKu;SD~0H1on^#X>1wKE00c2b1=v84WXwWa zlN31dVX-?S=}50QJ1LBLG!H{RS5Xv#2zzB0;2>gXS}8%V>-aN_@pZ`F<>jSw_@um8 zBHOZf92}QQ_`HDNMRmU^8?o&xTH( z%ARyA<#3%UEyL#-vty_&`(FbSQh}NBV)ewJ&~`BkflZ-fxY5OluW)%6eR=T3@aj6&;}v%x9`0yaKlIbJZlU;JvDk^%_9u+{9(FHEjsU4u$m zk=(TPd9T#2W=rN_)I9wCBX4njhVnnMeCF3St zGkt4D;LTu%8SM=o$1S}CqCVSv)UvJqwtk@hNcAYLhf!GPt38SjXN3b(Rw zEF5G~d=X5&L3l=}1HUUOWR!7RAS(b3SJSUnPYSFVSuMhlf%sMep(0(<{amGTjk6KU z_lL8Svjs&*a#UvA)rx&o_()5r6lki=F_Hl<{N>#R228SrZduK4Is^)@uvoxy6ql44 zDt&Qxe2}5=X1}Q?*3;=>Jd+8!&U4UjcjiKcsnLaX$maIBN+^IR_p2YXajVml*Rg;; z;e$ah%vh6SJq$?~ozA@poip2mv9X6}jl;JVS!gAF+E4NM*W#CPNDRIlATRY>XUE~t ztwuB7Xrw8PM>u(}Bx~C>CP7=W*FLw1>Y^=bgSY&cdt_4v=+mFy8e}5=xt~1#qGE(- zK{^m(O$W!Q63|^!v*i+4I)g}zIIMmzPsYksWi-fl^5H3h0|6!dB(cBD0Wu#XGC#zJ zf12ox-Dl1Zs&P z;7w_GXe13-PEX<;2Cys+wJqf_002M$NklUopA;+J-z>Z|1}PKhu4>4Xi*9l!ON+`_IKEqWveebHye5?-8~ z)Qw*lX*Q4J@;{>!&H6^;mMT&zDwPk7lX?;UYLUVHZFdt;4P!(GE8fP!LDc zpXvs|DY{_P4256f*%t*MV4QtC9+JL?iI&+LeIaevpQWBmy@GkR5ZrtaSD^_{-C+l~ z3Yq+qZ!D1Fr+EbdqZwAj(#w9kx=|N4GvrfNkslk%Bi^hSUM$S^hD%m8QaOW*7CTmM2j6JgmQMpPerXlQ{x+mtqa8!4&0{Gc5x39 zyG5Y2DJvv7$XU-qdY)f^O0-@AO{l|H>O@X31zQaJ@*><-js$j^CVFt>^Og(^dxqhH z$(vHs3@g$a-V3WNtO)D?9gnxU2TiA#O0K-l(b)Ev+zXe9mbH||y(LI2wy%E~QJN`U zJJ!<@aXu{yI=oD~9-|OPNUw7(*Egybm5MaV1s;X}0%rch+91Y4dPsUI7bFB276S&$ z>5^h<(>h*L3fOMNEw-lx_r5~kg>GuB)Si6lKt9X0VTgW~1+#8Y4u7s!fUA!2(bh># zKYFU#R(!zGag-)}x^a?2>`7P_;Ox&#I3n7h>sOCaBgl5DT>>$#{Wlbb;Dwi<4b%)} zU5%QI4OcuW1BGLw&qg8K1w>TevO__WRpi>AEeBzW+7h!OVwx8=GR0{Xr|E^pv-MP6 z;mfjkL_sWuraD~Uyx{)m+_1;3cvca<(%kG&pNW!TU{iqZTr{4@g^3^~mP4H{WI`w> zBiYgh4O9zyckw+meb1H!yUpYAuq*B+6m)0A32WHw=2cx;!4jTtnTpKWTB4y{d`PGk zH&AC<_TF2dRlww`8C!tIICk*TN1x_nm;e3$`mc@!WC?jmt>Kh6DG;-9hqvg0g%9RM?Ndo)T-IR=@sH2{&@?#&n~I1o zjP^X6l`lq5?19`#{1EjGp(a))2of~CW*vHgn>``kTDETsVOP}b%pb6w=+L%%*Rze| z44M{VgOP5mpJ=W#mQYp^_wdODvSEn71?p7+ixgYx5Q*fejhcTU%Vl_5NAewYE2FP`t=+Lf{ zv3+|@P_V5^zE<+AKm7C+S|1nAU5{A5gB3~;d z%rti#t`(w5sO(^~@1MW-7#h_~NZ{)ja+t z86?U@I#PEcFQs;|meYYnc_S#P?|kd7`}q+~!9%97Uh%Z~MY${IkRQr7?7+=Wlo_|7 z>(yV-QV|F-v}tHn&4MkuhRO`ei1_BcW!)4|?6sApVJ>E!UaTpNI9g0lhC9#1R9s=@ zTz#b{*?9r!yhoil_o-fGT20mACO{FL~Kkk zSVba*!h8$@5!%+-au9obaD-D2^_C%F>v7FzWu_)p<;bP}YNSF9Loy1jX`J5{a!U)^ zzI}Z>J-+memaYJKbM8zzDxWZHx+6ZWD?4(ozK~5R=Hw1q&fJj3b3!mL_*N16>?_(P zCeL>SeQzE$NUVEdXg<%-tc>x#!chqsghJ5&@qhkT9c&q=ciCSR7XVo(mf$h56pQrL z^to<>TbFt?UeI7=wVM3<>Fc1x^eXOPw_28S9fQ+L2os%Hn|V`@0yj))EEDl%Zp`iZ zAOEDz3?Q;JCgAW8<5qDrTF-j`SEU||n4by+70cqyDz;VK?8}9^jbLBB*Nr18xUVnY z4Z;va@vel0heg9K<(q-F`UjgVgwNv^+Z9WcQCzV6-IeJmGdy2#3LMY$#=`4gzkx;E z`~fCTjy=SHVH5%p6kPnnUUadpu51j3UHD-gIwp(WG@H6~7hX!Tn4934{PCPt^E8Hs zz~C`k$DO#P@l3eHAgqfPu)lPRY!|L1Nx>~<^*_TlTJv~~^8Nl&`1`k0g+92I zL}sl6eyZu@>;e*pXBaN3rpJOyAwIG5=W=*q5OW%EeWR+Dvfmby?o>(wJk^%AigRrG z#897f?>VOq-G0B5p|(NBN!VCvjJq1tJ|Piih$ipMQtRB`K!(i^bN-Oey*3L%mHhB) zZaPC>I>BtJDkT8K4fM5%YL8PYvc#Xhq&Qe;(1{6O{u+S&riu)!0l*a>W5qGxCvvqs|n;7u1>~m z`9L5@HAccS3M5~r6>ooG)K9H2${KyjKvx|_V~*t!U^ljj_URd ztPzfpZ~VE%@|8tE!fC(`wIUg00fNKzGMxC9y{t_j!<3Q9iAJNgwaDEHmt&6y*W3)J zIzO7wtiP(nM4xNrz*%5B2jxjC-Ww?*_B$-9hZZ&G9JE$*d=)*@k%EQ}K{R!9am{uk z2ryg;Of{J?&ahgL7DHK?%1@UQD$-OA>tu-xVr4==d;4&@U*HqiGNd1OWjV7Wrb(HF zR2pR!bO;m>6#JBwTu;DozMVP>ocND(bEQ&i5afM5SH#zE^X(_~usSt+r7NO23V)pI zlqxNp{QAege6r~zP`Ubz{8-HHZGl)(BT!<^C%wD*q-?$bZvE7N-YBb5Y*!lX}_n0DHc$3We3@RcPo-(1`AFF5; z)m2+A$9jKj#`5Gl5DT`eNz$r4x_SJ85xZ|rCeM4ZZ#Tql-4@mMUIE0uaMOBbqvpe% zbF48$Cvsh*e`Rcmp#Z7`0ujH>{L&CK{08AvN100lE3KiEpolb}?o{Bbw{_1PQ)vo+ zW+dM?E^72RVF?^`@Xr!QYqV>6o>>C(F^uDGoN6Zf^m_mLm-qkr*ME{|6%yhq3j6Nn z1~a;$=^HCK$80NeCp-$+?x7bSo+9`1k6*!oyG8j7nZ-L=_A26GoEgQQJ0)FL{%H1d z`}FJb{8E5(X!gkRON7i055inghF@WMhO4j14(v7F&Y+8UWkYHLY=BbXgXb%!+;%!c z;Oy7Dk#*cZJ)2V_$QGhDjcob*O|&OK_br@dZ^Yvu+~NQhOpx2A?QQ{ZFnL*SGp7QY zW|v0~tkIg-0!V0ShEMj30B&1@*Med*)vOxKObA;N$VClE)Ork8ju&P`wm^xAd4wobxDa3V44NaT!%6U zi7iOGbeY4k-z(9Ozy^l@-T+x?pz_z6{uY?o}Pbo=A6bk zZ#QgZZ1j?A6d6WX5S?=z=SuBijaijzL7`JylF_~8nC`L^oYmx?T{I^&wWvUaRvvM5 zRA&JCcNeJO&p1p5VCXACnZa+j+x%Etdhmszdg1E({zX`F3)Ro%;UE7+Z1&}zH_!ic z^=TIjxRaE4Zq<1E>$kSPW9!9NBfq?4W$&LH|NiM>()9ZE=8KBi-Oc;A=U%4R*-&p!FrR9%3+z2m4dpSyjAsf|4eZuXc^2Gp?7NDz?U!3YKHQ=FXU+I zrf?|gSrww2oJoqgJW1trhB2mvGf%ow>%?W8<d)DlVXu$gLt%&H&Z5R+VUfO+edYEO0a-qUNk!bCTQe7JhN8Zu`%uxe z*6m??92O5gIT|p9!pfg)gF%@Er;qu?j;~whZx=pFAkz^F{!7V)x2?4O9r$MES-UpA zs9g9qv~2qN+;#M;7J_;dSgh%2HTQTC%`4f=K8gSG`SC9wA0A$wx-yb{7C5-LuinK^ z&b%~V?UuA>a=|Z4lVp|bL=u((aK!EFHM8lz=tWT=Bkn#{@lg>s$Q^f7Tb;b(Dz#>0 zb}=fg63~{GHs0Q|d$usW#e!pP`b$qOd-CiKlS1#_2XD%$w_Um%vK?4b#4vEW8@q{h zD7D*icCoHogVApoJw@Tl{B$+qxs{Jg-j)vQeh{NhBV7^kuXQ%78O$?+Vz*JeLC74X zD^hU?kc^48o2S$#IqYH-`E>v%OcSE}M_-r1$qrK$h=vw4HCv`rn?Ij~T3%V*VR)l1W58(xkszV0XqOPg zg#`w5NvYbxNeRsY)Q@YIT0YgCKx{}`7D==$)#D+Y%NE=A1;EzsZ!fB-LKrMC>PZD9 z1rzezR#^Ldd(fq#Gev~i%;B2-)3}18xb<-Fu?UJYT;#E<*lMk;fBSv$1oz5psxf=} zP=1m$*LZ=0FL5ul;9Mvh7@5yB*oy(P41UqJ36s{uI2Rp^&%=DdVT@JCZ`s+c2*3XE z&wu@I1gYFF!IbJ{a`gvi{+46KEDs_<=1I5Dk0O8TE{wdwwV_)6B)ZYPjxl?h1r?>U zjemJ+ev@WgcG_=@`rs(Z9(#PHW7}K2V%0ns-K4M)?$P7n=S5349&7~)Idv6N)|=SN zmqn#HVQJQoA-vzQhx!0xf0cGwJlm@rDx5;h7{z-^Z2mF#{xY{W&&ayj<73IBYgBGkzr4_{p^9smWD=nn(`9t(Wi*yzF@PKz2-GMzPpz1# z)Oq{SB5Cut7+A_`RoiIA&xOmuaBWZoQ3%5dtP;k!OG5TK2fMmZ^dZU8qJKW+=LNdX zE3^zRl$n1C3{CIL^vP>zuL6+1h#&geBVnh5udmKU?G$AIF{<^rzM?_;Mxo4Mk_p9L z0H9wxyfQ%$EIMyIAmv>BMl<1C?|gb~of4*~=`H}39J}qj;6AHEQ3Oz>T5ho-Vltss zr-x2%JNy&S8BfGH3}v~H_20k$r?l(4As_#tto(X3H>vQC#R7Eik5)^WHvBDVB{L@$ z*`D4X;bx9sHi1ze%ZeL_OJ%Fpht{#1-Qo$&tt`*%eHNb#V{@88yk$TqLf zG)xX3o}QK98CL!wT)xwL)2Rw}{{q27X1nEEd!6)Gjq$zz^soQ)^73LQPP73%O!4s9 zDfAx7X`@q5U3&ekbo<8SKmYT;e2+kN-`p^ju^s55oLo5t;e5)Zn&2hKw2S@B0L_%V z?GVxP?c@FR^)Dp~Tk~99^3pISnFuaSZuEV7|6O>igfkOznMmggjCE8gFluAk)cDZ% zYb!F^9h)l)i62VD&9&ABZdfX{cJnM@Jw&I6+|W4$rsNb|lq>ePIiX}?ioh|m@}gV* z6RlW^uJ>w(ta&5*9~VQv(P!WW*GxdkAp9%bmLBR|`=;|u4uwK;x{l(15%0#3N!m# zq!1G=9J!O(b)4mRmfG^8*8pmOOpnCtzftT#zD>@7LW#1x4CJiX#sPUCU1yeem)}0} zf~VFf%KHUw*KTpGE>Eb0v>_*E5$<#sm86=7kN2nR{HARE2r*ZT2s7M+;F(BijBdwj zHoz=BkQ_md3aP(de|Ha^AiyN(yQL<-nQ*PGqTt8Ev8<514)_ZQ{}+^yFWHa{=XTnH zg4sc3Qyg@>Mc`3TUZI_T2JgP{rI_3LA;a@VmPK0wi7*D-LnHf8LEryve%t9{1N`;s zM!J(S@4jBXtaqE}^$5GirxMliX@lvkbX$S4?$#jzgC9BygriBT0C#T+;e^W-;j&4O zCZGg$&pcSt!hoakP;myVCfv_oU(cLPcx?;}|2+Mc3UUj2W(6eDkvZ`-)oAh1DVD`Q z40UGga(RAyLhASHD`AgSPQ`C6Bv<8!=Ctbd&6Mr?8o$Bd5qH?HPBNaM>1Nyn-X@32ca)NAt*kgIXPM-&6S9ZNO@s z`hI&SA3n+Z@Y|_`P6zz;*YBlL`MDf3e-rxHw#z|yY#Wi!t2f%5kFl4xUN2H;oh`@lx=nXGSdFI5mR_Ll-NBr}Wdz zt7TttvE0Ot9VnX2c!a)qZM{n>hyq7$5lP47$XOTd`F^Q{CKb+Slxt9B&ziJz%BeY$ z49ZRwJRx}kT2(!w1EP$l4oq@5<=~v4?(~2I0)jZ=aEe1o4+$e@Wg+C3a~QW}#Yp8t4unI+ ztloxiX`_DFcha;Bl*m-6RTF)LqAOvHc<$H^skHO3=OeA?WEC;L%C*zSNDdTqI|b1w zb>w-*!hO$b0@IADaNIX4l$hY!L_2$kVWeZejxO;T{_s`t-90+*;<|vHDZjsnhF0FG ziI|n2;w04=jVaJvWHAp!BL*{8fQFB)-}N>){_IBsoj@`J(^nGhWDG7`hVT4D=kQQD z6pG~~+G_S9F7(8#ELl{7h{DTtIkI?az(S7>BEwo5!kFD_x3cTb+V_^2zO70-eWTQt zL3~-c(xmHx!r7$2WdLrIu&0@^xadm~v*aRWmu*>2fp=0aDP$@Nf^=G2;xu&E`xItW zsd@1WHV-CAod&Z|DozicFX$U7y2!Zr<(QI`QE2Hu=_Vp+0Xy|m;c zSujdQwO|kVEYt~y&I=zS&rTly@p$hHW*v(~oP>#s{`1H>PY^F%D+IL5n>pmc*BmF}#Tx{MO zH=nZ+sAGZCeq60=9$fZ9tIXO>jSiTY++Ci#&`Y_HXH(XrllMj^!Lt#$F=nJ6ZG#?# zP-+zN>0JPAG|kG~u@%@@!&_#UW|v^JNmu)rPdN+dBZA&?mI|q^58(xTs0-+HZe4&u zbV60?btz0}UhS}F&RdDYq_GvW-jtJLZlmL`;M0dZps|F;-Z8`l5u!Bz(4d?qmlPvc zWz*7(scyvW<`|N>DkN$aq&*}8Q$P?9tI1gxpT?SI?4iI>gUDJVbF!EqrRJQ;7@NE7 z)=>ckTr5zoENC(zfJjwcm(DjU5<|5itih4DIZ{*Dt-6k&q&t-?$2QSOXhT$THnGWI zh2+J{5@v${gAR+$Q8xV24_OVET(?g*&q8v_U5pPgtWUe%+u3`4$yW<6$1IMQ$Q-9l zUNc)v?*^^#Nw*cvv~zdwyJf=bx_D5MFix-ehsGIAe@+XCsuY*P5*EZCBZ0pF5kCLl0{d2m78hzsd$D;-bI#V z3_+?D`)9=nH;lh5z9TA3BwEMLv76pG12f}(G(O9SG^k-;FfHZiE`+<%5`|W>cf9}s zwV;kB&dEO>yW%ZXIOM$H?W5}e(w-Wa4=7+3a-4o}`KMKQtQ0l?T#zmm8>N^eMg|kv z&KPu=9t(z!FUa(wuRfUP5oJEE#23Z<6l4(L`mxi56mCxmYoWE!&{&w(^SU})2^_eS z#CQbo$*&i+T9I*FG~e|Zp&pviO=dkegc%uu5fL5KaZ@JFX}AfVU?-(mxWq98f4~2> zWGE`pcn8H!zHNor?bsK0Z_K@%t%bFU2jEN)2JEpwMs2xDkv=cx%|xF`7uHfNSpBZx zWn2aE)M(~ikOD9pI7L}@?^8N(I*YG6=t7B|`bM`@R*WkLbj#OVvx=;_1&vU$1elLF zsY;4cW@TlwUu|<0ziEy9PJCd~?k2w$?|bZUx+a42-ym&pcB+@;bev!3uA|79r@I-+ z?6Z=KcIHaJPASADfV0T@YVBy=Mz8D6*+VhT1o0U1M^OXNUQz2AMc0ILLikk+0Yq)+ zQDnIZGk(Mj%Tmnfm}u;THgd1(SVcPJNL}jr90z@W{XgM|jM-Ecj=y8-&3(DH&s?~8 zD8qHLh$xA?`{F&G{uD@s#2gDbe1W+02N)(ea@CP}du0-X`EctJSQ{}Ntco_i>{aWIupvWX-ufqg#YJYN<&WE)GmpGAm~fz>qlO_$z>|raQG*Fo7L(Qr zpz7g8>`t77RUYiEUfz+T;u0LxOk+&i0S2$0%(ZgqBN>CW&|H$hGd4;OX+3=)Oo$L{ zq;!j5V>BGMkxkMVO4&%dXpk|mYf>_xtVB8uEr5HW*#$cM(F~(9QZLZ#xvPGnf}76$ zJw8c=)~jdCk!Xv5u8T24BEg!oX1xq~FM7(QZy{J$aFBqG4(12Q$40$7dNQ>`; zkPQ6G1QHJeu!p{oG_P2jie-B-?liCw)DD=NhDAf51(%>=Bz$GHS)6@6M&59)0m_rn zZ~nqgw5&yj?=xFCTpp3I7jgBs;`W$8nluHoYu2i%;;^kzrHA8X^{Y;tQh$QoOejxg?4yMlxTA(wWrB*w)j!|<04@+{;#!&G3xYIYwqni0Gc?o6$w z74%vydN4CPn|=o-6&MrP9rLaLSm(^B{Fj88Z!APWI~_EQpZvD}Tm&cM3ZkPF^-31~ zBM}odjD&{PHySMUWO*S6gLGgii*?s(wm@|T&z>q>Fr4XofRI*R9lgfqpSQuvwfQ;S)KAFCZt7 zU!Mc7{Y-6%jblr(G#jLZ%~t`T;INk<+s`>G0jDxU$lAo6|27L}wa| z6&q!~8Y`zz^1mM}ggaYeg5YvFOx)gi&`=%qNfS9HUcKjtyv`nNFj-e#gV&u78Qo%S za<3A!GS9#Bc^njvc#g5_lSab)&oBX@wv{69FR{O1re9!q-OQrdlkv z`7|KP4Z}9$AxD?Wb>hz;Qgz1JMmYpGYU~X8$|uotODKR5GOF_H7*pvaUxoodb%Il1 zi`v_AiqM2gpxDO%)FU;ESibEIArde*Wm=})xo)iOdTb+Rb@G~Ip$k?Uw-)}yk7Y+5 z6(Dh+Xf1N)1Wn+`w33L0VyxsmD%v5#1oc$In9R~W#o@Vdb|Q=h8*AoP{9Ztq_vH>m z7ZME_r5SWQA;QkUgceGKgH>jh`zELyUcr;|B86y`pM#Tx-5Ilhr*nw2;+P3%jUo_L zq%&axBUnVxUo$S+o0!H@AZQ-(9oC>9WK$m1@^{0fJ$gk6%!(6_KsRA>OFZv zRKk{62&<5bT<0Hx!4s`xF?WZ|gHN5KY66=xV_f)ig4~pf)Y*)&x7wI-=!nEBiprW| z2NS?sF%Kk=YT*epM^@yTlp@izC3IB5^x~7swlvTpB3~Q`MF?D|0rTg$?v+K>0Rn`Y zjCQ6P`lC{${a2h#oBeC|T4q`!K>FPlxKQF>|9Qr-y~o>G2v$2$>81Lct(m=?DwAT_Ok!@ z$?h44ZErJ}w4W_zDM<+>C1zU3Vo=mHnR>}Ykj$#Rt8vI?YBOgwiCsBqM=MUVYa`P6 zP9xDQWh$LR$f*uXgua`mdTz$+f5bKRu5zkYodm-Z$Lv#^(sCy5)2fY)DKl`R6q{i- za^|T!FB^{s{CIXy&_Hmk7RaX3t5DI#z_2Os)Ud#}Ef8X-qqYW2z?ngwer$?BolM@v z{fqAqiii%GR{mJv)*T$;X19z8lgPA+VgK}cGHjMg&?Z~>D#H4kAFzD`h5_5Dmv)jo z1I~dGcW9V-qVGFhv}lrsPzYxk+6TR62={r|gi4_tul7hPHSC4JHSy0n$OOso);-th z9&&)CA%R`X5zK<(vKHR7J9+sX{DZKye*4pQ<5GR68+3qcJ~Hs@-(MVZcmI}w^iGI* zdn)mv9wIm^+GYUVl;Dj$S4nrq7O-?M(=XH#!nt{_*1QezY%tau-JlL73({;i{3pna z0MP&q$+m>Hz~$8-I^?fevNlOO%kmQ8fiI^O{LU4po%8^23`f8_|Lc=TNn>0aHuB@; z5RjP#`kqWBU`*?mf(n(*97t8s!FafXDeNh8E&_i>us5dw2g zU;8jdur?alkF(J5exL*=nvv8dUDi3|Xs!!mEN2$nd=F;zP(W#G65u=>2AmXYLCd_n zz-osJo5Wm^)>6op{CdVnnMuhp3EoH#^jhCejk-u@l_Mk)XtHucO*7b{M{(mM%$eK15xy1RUhs zKkD(&G7ygqI*Mxd|Hs#TXiIV&37Q6oLTLa6pa3e6pqf6t(<|QpF?Qyh9spfHqQ8nJ z*;G&kSy}@7{brF_mzbILx)tf+ZraSvJv<^KN5+{O918HJ?*=z4vV_C8PABkG`zp-H zrnj)@eIYz#W#&DP*#q9=UC(4~ON&2d)n7^n1jx z+jK2|XbDxk*@>ff^p5cunS!1SGHV$)Fs%)=BqVAe6?5oFTiu`>N6Svl(OtpEl!`kt=O?$a?%qaP&q5oT}&Ue67-K5QB$mP;m;c9O`3 zu7{HG7N59;o?MsRle9?~VR=rdqjx!7;FLN;R%ZC3*zjp}Q)Dw_My3`*$R!iL>-Py! z=F+m^s-ld{^lKbCEl*B%NEY5Mf5#=b5x|=5Qs0&+v|>ebQBlZ`QGq2;QAE|r4$!)j zIbsAaR@K)sB$3TV4(`a&(Tm`zkV@NdvBj2%8QC!;@y34a)Rho>;#*Sd;;Qqs2+jXO?qx~5a>6)YZ4Frg22F3Y|G zZib&@FGY(^hLS!Y*zjm3GXFHAGfUw)rlGn$6 zHj8CehDds2rV8vFewu`UgUWAlB==UcsAx1hw02M;kg;GU!ANELTrsm}@!GJMbR0vK z{5cY;gx;v#zYqgDmCg&Q;MH-{Q#vqRwr@76Eg3Wuu|n8Y3ViL;sJ4AtP>V(_2g-mP zhr!7pgK@y9Q;$2t@;`0;%boTC$nOz6_#u^uvW5*(8Qiov>!X=My7-NMfL**Mg*`>Z zvt-MHg|^_v z+YDkwG<(rhA(ueQoKsYRP=8s9fnorCbyXeO=!T@wmcwf~Ixg zSW@L&AGR>`GjVz$heUoFC&X;AG%)nUAsxGN8vRHxJsr=bF_8rijw9e0XS@x}=cb*D zf{hS$fd@m{MJyKm6E@>~TaswATtvtgwA*ZrV(&6=y31AGJgqMd3oPFu9;;>NRHX$( zrc=K2w4+W6D%KKBU_x7`6oQopdOK$E=2XpgrGTQM4E0FYlEnZPf7;cD!3^$^dU~sY z*EJa49fEKJPv&IjF~!;fWPycX5VSm`urXz0qeA9n%Ty6Wq`;lBXJ86qW7Tv#k&~lV zfDwMkxWTqRMwu1~KgbdyH$Xy<;z7=02+V@5*sIyL#BOw7863~;qe6#m0kshfUW!Zn zQ$(#u+DTFL^V3MRqlWzSNd=?U)Y3x60tDG8GvLlQkkKJP$DxQJD`W~J3h24aI14%f z>TD5V#2)$t0nf8q2s2TcY1pMFoU%4jVRd;k#^)3D+J%wT@)$uR55czZ6SEm$MAW+0 zuC;QOL7f=qmcs?36cm;mOxodYtyQjhSx&UaRK=v&o9GH;LLdc&tl^B*BG%Hg#m%GD zM=*7Tw56v_FaynL1ymW4N%e1ExnxlsM+(G8(n(2&*eKIf+#W3AhU#d)Sn(LQ7aK#N zk44wzB&`l}1-xaLz7zmJybo&xGMqK%xE5_QC<;JMoM@p(-Gk@|B`Kre@2E=zfU{3p zRO~2pbsG(5S_$SR9Z}Yw6%mGb@_AqBJBN_o)q8&!<`NUbaa>$UTT<|AeDnomSjDx17 z@Xqj4;DK4e_ISq9j^F@TaQBiPaHH0u9^!$SixK0jbwdlg0b{ASA0kv$&g{Y(H=(7z zy*sjwQ;li|t3pfFe-iXUj_n$>`nDT{mO%+}Dp-cNUYKKfaCjexP@E28xIl;O*0rB) zdc?|_VB6K(5Rv|NjYI#uBP!TnvZ*r!+f5os!nCW_IB8dMMrlHip_jK2XCX$&Qn2}; z8)D4ZN_>{!#69F>2Li(;bCS&DvBJ-|s%w`O08|w43kg>vE?~BCb>AbAttV*ScsQ94 zBzRaoj;uDi9@i97tgknCPNLb!MHgFTy2ux{V`}88hmWC=c-<0Py34Pi+w5Egg$CB>?yReE)|gF?3^HD2lcb!WN#rrUBjk=# zU@0KC;ut)>VF#lsI-dH+lzC$dyJ0H-8a-xFun+GP8pA^^POr2rj#`09zTp zkva>r@Y=a!*3+5MtQX}bz(Wh?rybOdnST;~MS>V^>RO+|M1e~)L+KzPB#dhnnstnI!y}!Z!qnCQ=@}jBpm=s^m5~Yo8eBSM)$gEX*&ZEa5_<$ zH7glst6&syl5~?8EoK>gLAQ?egb0Re2|Jg7i6Q_dj@#{Rtkqd=XP;!96oGz~ zqAsdg$szHl?xe;oT2pZ0`lw4w?Z?_8@3l`qix15c9uZ&T?-Ex1MDkEGnG!7G(HzQW zrYaM<%RF}5wE^i=S?i5Z#W}aCw~ufx$gU9Ea;AtH(Dlk0_*^;bzb(r^odoKR=lU6C z^f;`zjbn`<$A06Kt&Np+oHZ0=vr8u~L>W(KlClpA#xh?urzdpeOkp#@%=Wb4DB!$3 zjAV8+1K6_kky6gQE}CyGqxXdIPoHs~qH-okgX+x%rOzWWPL`?b1Pt|}duE=sz=G7S zg~5v~Dz3pIGn6K(c37Ei=WK#wNv$xBD^KVrM{FekMrY00z)P4Vo2;yJ7VYSPeQHb~ zLTus^caqVl*X6UtF#2eRHaP05S`v&5u|65uPILocHpP~cqu!3YRHU(V+>Bdeq?-84 z&;!5Lb{vk=q`r)tojD#o7ma^z5+p6ytxJQ4To17e!~Pu!OYNkZ%1OE9n*b%6BcwD& zB%qGbJ5Z`_3FTDpbB0@p6kX!eyVzCNXj7S<6aDSlro(hiij#zy4Y5%Gp^;(r3s86y z0^eB2rxAn>cN`3c-0VY>6{SQjoJSZGL<-RHMCuC#bLMI+UWUk_<3&}4b;Nv~DRRTt>hZqOSoHmH+C2^!@N|=T-P^hHD`Xr35 zB$Y1so4S!tnMDYGYsezYT)ikBemtsaJsQdIaRQsGz$?xC0ZN#_tquU`rfFQNbqf zLhC+b`@C)l1DPuT_XNG&WC|PG6*OTiQ-0|76w$kRIBQ;{qcd(LUW85bc4gzhIgAR# zn%>gYKP^j{sB29^CF_L#%71QmdImn%S4dbYiVAq*RUexaR%3@?O7a!GNX3cD^K`?) zgPi7eer7bJ$oPLu(Iaz|jWRk>z|bfka*dFwToildbi7z009>5xxp!+YtM+`F#=@YB z%3hV7;*B~s@(3zV128^Z;EcB@*pj@9t0t&XYA7Xe>P(3drMMEgiB~NO#acRevGlxr zMAu5Vbb{z=w)rq6$0NEz99|{TvF8{v3-R{!1CH@V2rbPxz>Y`#E@agA>mP|T~cP7+F8lqW{IQKu%yGCr;yW>TL`U-EX`ygo? z?P_wd?^FTP|D|~n*x9y$SBjH9UYgn&TAI7gqVIH*Vq=MJM54uUe`UD{i6b^xE7p;Y zs**KQz7oxh=DRRGF^h;B@VL=y(DE3c~+|U&2O8BK;OGY{#xb5uYR%oAS*#;6v0 z>Vqzofvv28ioDs=Sjbg&1p}kvcqW3*kUI2@%9I6D)oV<#W?Kxd_;cT}Dp!G)36!DP zr{Ocp>^QoK#|};}NJs*MVW(j*C=5eGcfuA;7A;4|NI8qQmBCOAYg=+Eu5F`b4QO?j z+sH&h0|RB94Ii@~L_{2VC};%MoZ<(}p$ye8> zj5h<;i3a9^f^=Oo5o7$mGy$kg!KzOU2hH(@^>}aGIo^K%Pm(O!0^@Z)WdvKTwG8zD z5~UbPoq1j-Nlr!QNSY*cHKv@eKY?hZ&e9N6L#v}EFe~jcKLcy02gQ{WVwmHS6*HD# zhH+x5Gy;Z@Q(3YTC8521rS$?UPSAvRD2q#uPHeEq#MtEIrD~<%xs`m1kR~$leN&{X zKNLnea^}ssE$qs>bxM8XXs>-PdGdjgiR>g34P%Go4K}f~rG+wSq57&{$2rTVSM<_$ z=mQ3p2XUVhl>w{TvM}`ax z(x5&z(h#3*{p1{%ux&~--aQe@S|@c!wq{#AHp*jx-134UUB&c}EXEN!;jgVUfHwh; zR}OHRk~82n#Z*Y^ktr*5sd7y>&1)jW%@I~=t}o6!PCBqnTROR|6%+VFjOVbNxEk6#sVdcg!T z1gqtU*&+b+@bBV4*kfJ|!lO#!Z-tB98VoXwfi z6cpe1>PFzCLBb8QXr@fz<`JKL!Pu?%+38eME+ zCw}gr9_HQN6%zXx^__Vvx<{RACv{Y!pbwS)2uRat8zx27l!fOq6%CHC68KcoF0zh# zT5@IU>Fv{kr_hl*o6;?=QP93*ndDSi^wOdNd%27~;)trK;mqO^PHqL&f4yfc6{kvl z3o%Cta${ZlmNDm0iW)l6fH-HP>4fo~>YA8Hvg*N-BrT$YVy)Zk#4wsz?d#E{Yr4Or zpMmnWNH*EDs`!DoICk?wy`_WzZl6IA5D5Kz6VZf>TJxvRQB=D!B!~Mzjx0ym=uO7M z2!`}Xn*Gtr_zuO%X?+uJ7k7C&Rs`t9X{gwEVHd1=#Dg4@#sZvE7Q7q25-2fhL)0u39MS3D@hhg(z3PNPV7F)vx z0`B;51L(X_`}9Gv-Lb008pF^TI8}3TTsW)erzb_D#6xJtJ(SfZzFRcLF3rvBqD4c! zdpyIaFx0j$k?l&-mjh%-MQ&}9X}US=rQDH@J&1nbVOSh3+UFW5;eaihp(*w%-UH&8 z!AGTxHTd91ojnbJ5(@)B-%7%*PItB(7+=-$4fa;uE|J6yu9gzxJGV4;b zdlCV#0#ec9xTkSk%JsMQA;5MSnL)!E3y>#~W?Ie5V~PnZ?TJ*)98K()(D)tYlzKw;&cXEU>}qMiS`VL zI zz^vCyd=17N2l=p_q{d})7YE-#3_xl&?IO_D*PwL&dn%yD5zSu~eeT71f!jC>DyldPjRyc&uw?K@Au zf-XmG6vM+-rzOxX_K7F^q)2)(5jzUW4n}Qx%FNOfi;JwRCxk|52iIDhuGqUl27PMo;o1 z81!C5Uz8LPeQZxv9@FV6c(u#3Wyb*j36KA>xX1oULG_Ywvwh@ox(B7DH zOsGyBEC(`@DhuSSM&T+&AgehsFE>I_tmdR`d3x)yx}X^F^M znJ!USL$6_FQi^O3dG|dsaZ{U{nHH}r95T%W!w|3-m`TV`6@0x@WbSJcD4~$3sltVNoFG*eV0*WHffsU*JIanyd$_jF1H?pRFUm?qi&jXO@KLmZolP}Z2av1=z{dslYy2!)ku zMEZW7TKU%#j)HP!+oN8Ut*@=Y5GhV&=2=h*N6C6(sGgI97c5Cm%36|BacLgyy5N*( zO4ABnF8h?m^WbJsdytlCGg)dxd5LJ>%K5n5wS&n$E?ryr^g;l)5&!@|07*naRFVvw zJrpy8sEK)*>9kR`NbD_zk2a|F95g3}zWyNs1VmMXVy2f+w0-R5`5?HE7O#K)f zi!~(1&W-&Uf$LB{U9=*5S&SmtM?rasM|4PS<8*?_aSF=71r{b?*_5^jGPU7k5nN7vr8)4p-Srb>2z?*C8_GCE5y@LaM zS68t~DNI+?rm*o9e2tLw8sNbdI*i}EMFI}*WDTtrgEf1rVis$S-4-PQ4Os?}3^K3Z zM;LbVyYH`Fdi>yiwqh1QsCI{eTqIjxqx}6s)N|9hp8S+ZX?(Je0_+Un6r;eVY?rEb z$L(??+ZOSLO0t=6?tK2$ty{OAJn>_z-}+TMZyOu@486H;t)|!7`Yha52(@&b0&u`@ zyUtvlo*Sp0=peBs9xx~5P-ux91=(|aHDS9$B9l!bK>pAj)yhz7@oy29%NnozIJxR% zf{?l{nlj@sC(QGr{JnJI-Y#@r)v}h_E_IcQnz|$>uy#j8^`4Jf4yClW#F_gL!=z2lLWvO%N4gIlp!suIvj%-@CKw{R!9q!w>@?R8YTZT=CfV(Xh zD`yR%m7$g;R%I!iu(UL=&n&k6d^)Hn*!kR+0U%=`Zrh02k;vSm&Y4BP6Z25xU0%(e z3*b0(>iA!_Xq;=^g&MG=-Q+SMl!_CHX%LeZvE?NDMS-Efi-$qVmic{Nz#8o0lDUz) zQ(AH|-AkJcflbzgheG~XV&4j>uxSB?$c)foTl|kSD6L@Y`%!ZP10A7g2sO@{r7G-L zf=my8Viy(cHz+A8pkR+asZ9e}cM?YS}@M=c2?0qt4*cS3Jj0v` zYB>{3&{-4cg?Q=rzx&;Lw{Jgw;!X9h{P-t7x%R^!q6nO{&=mUcq9*X#piMK$D3i|y zyf!`{@Sd{9q3ua407D`8s1ExTYXvn-K$M|3N*HU5S7SLKA19bK!srvP+;LlMO_5Rv7Wd97W+oB<+R~f)WEi2fS01u$wWHSYtq}SBMsSQ_jL{MO!eVPDcOd z!Xj>njUtg5b zBpqzGS`M}YfD9YkoIV)U&DS%|G}P?>7G2=ab(J&XRKi6s7!nGSA*+$tfS3MaRWHc# z?sLE!ipvuG0`KXf1YGgS5RzXZbZX3Ybrz8^+D)A03tgtM6+adzrL9hyhI2ud2g>l> zYC>}^JyDlkj_EnBV~w*E(d>j%(ifM+5(%@)N@*K!Yk?V4*v@;m3uK@iKHeX}o#8Vb zA{pJr%8@+ImXxdgY>gLU09B55^!wph=N{4WlxV|gftDk z6M)a{MFG5`g7sAg=Nx6w zG_bX1ny5qNRt+Wc|YsM;%)`kt%(2qn$7Q zozz?IL32jU=w4Dd47V~Q_7q^m4t8whumH6eywOp%7vvVwozW9_^_-lw##vKp2X;m` zPS8>Gs{2p$9toW-k>YF3M~H=+3XW@y6Z4W$CBzK74D$Yn(TF$?n*u!$L zeDp{NYjPA$oclCZ8NrZUl$9jbCpz2i6p?Mr8>)FM(%VtOSre6jCk+SwPXz_^m!P_W zz?nW!EsoogY`KPB8K<$F23bOhv;P(85|y5%+xD}8ovx401VP93q{vy4sW^zQmohk4 zVsd@Z#cHwox8)+v}V=(sVhSVWT;4wPum8k2+1m8u&3uoePQ} zA=V99U7GD_nd!&B&4Mgk0XFM>Hi^?vgQ$wu99j4{i@?zyqs%C<2Yr;)h%gm;(Gbc? zGX(bJNt*t~lg(%Ta3DhJqg!t;LXvNtHcTdAMR4fI&f5~*(=Z$zJh18!2+yqcc9wo% z4lQXEaZ&QRcfh;sDf0+3`xK-JCi|egW2)Re+C?wDu$~4mG;xvS$tk|@W5N4%%T;7X zTZv~!=!)(6f{D-6(S?vBR?$D1Dbvz>hHD<^=Bt~> z(a3X&GY%SS9hoN!59DARhs8R#k4_!x9X_Q~KXN>-Xu{jk~kjY?IZ z_ekH|TL!aY&lqXwCxEGX;-3(Wo+-;3I~5?x$g3CdT-M{&vU({Ui%?g=6flo*7PKYe z$R>^}p$e(a$1^*Xm3CXEo_ljwH;m9sU?n;O6rG8x72?!_O$ zJEbzQSzvUJD-zmI;W2^F1(H75TDgwQC!RioZPM>Mqz zo|UWRuVpn=WP(^QchgKyYD-efBl9y`nRM(tRt-7^hdJ|e;}NMsy>T!A#5?kMy_Izw z3!oXt<5(9(n&yju+GnS1L1^AQo&cC0gSiMnDr!@EARQ{FpS@OuX}cCK%Bhk5H5%B?Hd&n$0(XM5=K&7TzNTQOMNun+1JaWOQ$YZdwqH-+hbu^B1PYrz}V7{XHv2k(AsMxpKvq z4!kn895sVCYr+ea5r)MmQf!Zq8CJnUiwYU$e4fV3kvT|L&;zj2f?vO<^^~yI2m=pn zi^lfyl$qW04Hr1QcKycHYu6SBe}e8GKQ%Q^#yOF$N6`pT<$OeS>u27269ZJbOqnRJ z#wm%dWcsPBwrQg?x~oI;kH2IYP3sxPAz9W8^*tV(VGco*i4<9 zA+A%`9?t}7nG@|;AM$8wG|k^|Gw$2Q@s!-iuq@Q6cP3TWoGh0;Q$LsB1D{-R05jJj zg^9ZUkx1)5>4^vcM{)-G&vmD(*Md#|;+IcLhR457YdE3j)`31DPVkk%jhugCw|vh6 zY-NwSqLcd$G2+eIW76qO=4Djnqy?-nk8%*vJJ_e@;DI@U9N?UA6f&-6&LwH%G35RY z(1H_#Y#W5@*C=gN9&{v=@TO=Y3Q{|;hDMD2tqLeaCO+yQtLhfAB6kRC9cGG zYP93SGKp9nfmEwLWyE)yWtsu)lQ?dA2`mW(&`h$&V0_7{&Y`u(ly?giW z-@hkK8}&%3Jb(WFmFs?V?)7Wet}mAnWG3d){iDx5{p{|&yN32vzx6prL;uFjYgeva zb4%a+_3VYW%|qf)7HXb*;)@gCy>jEmjkn(FSq4}+Y@AT$WB&2ZDsH$JeZ-xPekc{C zon)Rr{pzbPA3c8X>}i_v%jGLqE?>SZI~O#s{@|^*-g@KBH%w_#9=EiX0;8JQAI^Q=jDCT2uhEI3}muxlTY7Fg z9M=d*k?{zKZ&wDWaETQ`YezjJ?}X`?uHYgNRTDFG(QsiAaM4OQZfvLokBY}lmTJgk z?l#N9EaCuWz=5WggaV)~e>!Dlbi7um^VqU`P$^Ps zwkEPb7?ajAtzIp4d%U7;GfBNjv<3}zp z2#y6Pt9Y-dr}C}0-}%{3zkU1dwmw}{m*D0p1yZ7#WefRF8hY!t) z55Bqo{FP^)fARUFhYxl~P5^qDe|u_dbmht$H*UQ9?(MhVxpnp0^{WY=@h|(0Ro^bM zt9B~!;Mz%Xj$2QFv8q#Ul?=Cw4;XMlsW9Oz2plNta@bsBb0{PV^n;BWp^ndMv`QF$ z8l+xyF5PpFNvgFV1?1$SYQK&;gC_*t9WqIwOGUGe#u-A?KqVueD3iv8wY8NW7%k5L zohf;;ZCb~1%wx`DBC2HIWiunocj((#Esad>E6&(!IF2t8QrFn>m+ATY2dSz_{~sWW;ALW=$otSbl&ihYWZ}7iP4n zF*j%2e!^7p^CO|z8SuP;eNu|_kqDWyBpYQ4_TvS|rQ2D%6gx$HyiTVORkeUruWK#I z1KJ9c(&21}i(BmaRrdPKcrzdpddhe>Rnze_nVlH{(Y{6RQDFd-V3sj64lvPAmVw`4 zRMp>7UjJxXdh7IbA8!s!iy>uGF0w}*N$ZEr*N_d}M($+7GapaL7*O#~95lx>B};QK zmW5njoA1f&877&I#sEfRkNOXaSYWdr-<=Jmi4IXCa+Hy7Rx{S=a-wx)9zV)j1}lW| zaGDBc<-9-51IWg;bDvqWg}9&pC>n*Bb$vKFG#@7SmcFA>{)n*RF2=md`swrMPu&-3 z{P&~aqLV;^eb;)g5812zT-$3;o9Djsyug#kPyYV*kM7?6Mq03+AotMAS4}a(v z*HJJ;R7A4-JAeP^qd))cuXe87clKh1CyyV|P7FQ&$BS!kym9l*H%Uzx*B_TJzn<&d zTH0@De)q~dZ{K?S=+Q5K`HQ#TdAD0D+04%?%RYpFv95PZ`#3|cO)&^wBN$!W8~EWMQsVC}Fs zfqK75yJU^mXwN?IF9e@9l0s;#4>7P zLi7LwuwW`q(HzA{DCyL5;)Z*vf?PF*mq8Dc<_{`H0X(!Y2f!f<&N^5Z0}w`u0BKR! z2Xw(7u@}`X5qN19XGej4s`iQ38SKCutZn4&L)xh2D{BJ>3~9f6(IAH_fC!G^&{Xxm zk7d#Ml&v@_=Ps3KmX|3v0Y65lb%+df1U2kmDi0S_DK!+vZsQA$gBw%00JddMwawq~ z7}8mKB(yBU0T=+?E-j1ifoXOcj8WJqAEp^2SACDjUs z8SiWF^-<7fK9`yV1k#aT`o*xb8O2zn^U%!u4U?tgm>5a-?b8=B__GGYglo&D>_a-k z2iGVIcQrZ_v+2dZBS^YZSqVvGL`Jh^&KPTyOy^m{$#^uksB1oolsoOu-Qjrj{yJ@Q3#wKYh&1`>302_2l`}2ag^;dG=(~INr)JMt$}1vuCa| zY^K>VgJ22RbxdA+<;g40AAR%u(UT{xMPEF9_W1D=mt3~GBtQ3u=F0`IJbwJp?D+WU zlVAMe7xL}3D_3t^yGdl1PE;5(HyxTrn9<(gS6_ei*T4Psx4-?ZO+2bSbtU!uiAxuI zNGA15o`ELB>#s2nIbM14e2X9WA#f4$^#hrSOV`d-dHdno;l>n1EX zDSu6UE9)sO+H&j$kG{zpptG-`Z$&;YClm)5ZJM85rIU@IlU9~Xf$6!%mkANH?$QHQCMxHw31&o~YIaG0I8FjihQ{Np?4vl=H0J`% zoCXaPqtf7uenUvJYJ;IQVQn;6U|$v_&L^2;O8BR+2C(%#F3Y;dT)sZ3Wo5qP|vzcP4V8GdzkLJ+mV=V)NsHzFT9bnAy3~U`SG{7TX6B0U25-Rb*_u}*-RaIdaR*mYCu{M`D_30ISS2?V538V}=#>F1NXbS( zA`I{IfTAYJjc6pkd)m`6YpZzxtTF4MHQwAv&Riv!nI2TyNnmZk=r^NEAdjCsy!`DI zoRFizT%mWDivn0Yd-~+FPd}BOUBuje@4a{4e&@CKFI~TOoxgCww@U>w&-L-wUw!kZ zKmF-%fBoy1UwzFRH*UOf_3G7Ep5_5cOJWFu&uy*Pfr(VP;k+~#B|8|Vo?nL_i-_`$UDn?=yc%^F)1)UL_j?ABJV zpg=F-p060R2LaEqd+LtuFy|=&8{W%wP>9qrWd#~fA1IpR<*U0qx|)t4;+34aJ?Bp- zSDl4xa=IWWekZx4H`YF)mLta8b1LgMC$%)lM?x)v!xW!EI&|IkNg^_GG)Bba*0sG; zPu#H~j#1K;L^-Mbt}+{Z(rF!Wgj-iS9&7CBDNt!R@jUz!+<$0VqZ6}p3A8PaMpu$0ud;LuE2wi!sIu{Iuc=t`1(hiSx>0i9lAXO@F+waHD62jno= z`Zk_;7Dijwz9q^h!=Bpu)Q<83FV^;+Nqz5h$rQrm-M*}zN<;sjm@E-cYq@dN;uNK4 zA%U%Ui+rwUig5e8VB?IkxGT(ZIq^Eoh=?-i|KvN3C&*)@Y|B3%W1;25<)CxM9g&vN zA^AdLH}g;|#D5fK%vg6C44W%Z)|Jywvwm0?L|vJIA+lZZ^&iY;hv`Trm(8ZP6*%{pC#i3VuN zy8|^E&0Bl3$i@y5_I(H5BKdMzzI!KO>{p1)1%CJ%9wba8=-ym{W^1x;UHf!48U%(^ zpcCh$GM8+9=9e+=PQyv)?Rvq#c;?>cYnN}_fJz#7P^39lHYp~ba3;|+)COF46|&@n3*Ap zfL&4p(7EUub!jo%wS^`H#mzr;1bRAe+-_rkVvDNHk#_p#Vc*fb!3MVV#mVl(TsdNP zilJrcG;fh|1Z6i6Ht)w6?4Cn2k=SicDFr65qaCio$LgthdwC6!!`W+^&27v8Ah8gi ztyL9XLRjh2Zib)69gr*AP!PmGvU-;`iJ3P1HT$51SsDqoD!>}?Q^61tQ8R4XXV=1k zQ7{UKY{n6~4J#)>Y0MbBF#ZZ5&2ow3oE?X=_4b_0N3bD&f`ihs%jlUur(4b_TOrMG zBiY7DP}#ldW-3OOWRW%lOGKEvY#mik3}1c{X2W1P?h%3TMiIvqSc}S*Gw>2x$crfI zOP0Q87+`z47GEu}$)uY2-FIhmUjb9zsxL7xMa1#UcR4y1y%!cz=&!+57G}AxSBBzC zb++q&cf<~ni%iuo7Kl!l%bA^1Wx*C;>)Y=5J$3`U>93m|XT%IZck9V)mM&u?^No#8 zUoI?%X0;5HiIneAc8c2h7})WzWV1+%$zpC@m<|xBV+alEK~w^Y)XYP&UILsJA_JSN za7%i9i1ek$=aRB^kdtPXY9Q$|C7d}e!TBmlp;(R?pprlUn@263v9BH{>7@mzFaJtz z1En?QgEB<=mV#VoC(35ZYrTfmqtFd68*A>cpG^&=q38WKvvEWm9O&p9Nhox2zK6Ydb-bb-nn)j)-*cy2Cy^)?OU;ocx zFG>0Qi#vb)>t85skef|2) zSMGiN@RLtI{ph2Q%s1C>-h6QHKEk*&g2KA%pd5NKOSDfVJ8fuZ93R>%LEu&Y#G{)^07tZpSDk!ZM-Szis*)$&SntVMOMDGOZJfJhf$tG*+_45qaCNoYjMy zvMr7)?AY1nzNFQw9U z4n%;2al~R{>EYea2 ztm}IyXPChn*R{Am2{Qo!h86O1TWF)vm?^RwEwZHNgGa>Wj*FtcYqAwSG80seCa>=8 zraL^xu|hSZg&$uu^oDW5jCBJG7qHN6L$3H+SM5UzI7(1$KDD-N$e<{FZ@5C5(56g( zSQ3(ij=NS!Bi>mZpw2Qb#b=T$A%X7gGA0QZ80O>&?N>|k?)^lh``l$2yygN(wV~;1 zs7=SY$zThx-HPJN?ir*O*;tQinQ8?yJtZ3{npQ(Vd8GCG4epn}dd(wlZpl1#bEloS z9-vaZl=_9Y)X9tTC=zIDq31rHxSeByS~lgF&uFR#3K{`5QVRC{W_{qohz4<0@E_@j@$ z`Sq`^|L}bYpQH&gN^Ka-)#njZ33=)rdyj#EAI*Uk2QyM~ukwgXb+yR=7M8<(8b$t)vbLu`_r$)&()cxo7-;)?)` zmuV$+Fqpv~+(7PeyXy6y(?%D}7+HbEvhTh%dX{m_&uX(P!<@xin5Itk9>{1mZf>_ zI-g#wsIN#W-z(a+V6p8h*ix%~FBe9hK7 zeCFuXXlf>~E0H?cZ3J*9D3UX*@hPk{1ToS&mZZm44XnhQ@G#EwIjd_{4uj;3OA|9N z`mT{f;jS5^hs(&nCDiP4Owa`rHlxF4)-H)8*&kea{pz(FSKfHzt+(I4 zCEY)I@W>Z?JSNAUPo6$07A1WXF>VUw-k$mtTG*@wzv|r>$jQ zd+p)|jR^=Y%(`{HgDG`}U}R&dT{*F;OFu3sBud9V6-F0WGlHNc4@S9UFJNn! zmZ7VT__Q63>WIX0iaK=XGkOzOH@3hW>l)w$SqBZR9anlw3W?H3-^*7q(4KeCvlmXy zv>KnLC1VA}VBsOSrSlWGvw+xky|cJaosg`FY+- zn-)}~eti*&W}_f2>TqCzGtFR5e1kj{jcYv>C;?SQ?(qpg=jX1i=+=0m(Pv2Q+4PDm zrvMxmc#jL?h~4%s1;_6JEczz6sGf>;eBc`S>Dh=s6|!WH@_EM@1n~H?YAHd;RAnK< zb4RF^W)x2XsO+g;L@;W|Qf1VU$0=ki?!5tv@}hl&C~FIvkp(f5&(2u#2;_zpu!%q& zYbml1J4j5K+2wA@)hXxY&gN9^J63oA*lleR)s#Av?R3tmn0X@jt<^n)xKinneJc}p zWuUjI*jl-Hd2|I#N;ztm_AYIx_}XO+ugx70+PvZp#FKs!I|}>W7>lpf(pTTWW{LS5apM3Pu7oXpG`uH&u z8h_@%Z=XK1;ptTXW|IdGA9kt7${zl1iTmB>o)Gx@t7mr7E?;BZD}=rL`qM{`*!0F5 zH-GbAe&riBz8>_}n{T5sbR=OQ%@HFd_aT0j+d@q zzaHT!fX_img2OAt_WVZdh;k5^I`|cC#YID}7G6S8sM6y&<@h5@6%Qq4(E5Ymw=%fN zNQ^sp+GWF;=XS*=5(Nb=)+dGs!L+EmndYKWyVECKxYe7}=11f#M^qx5{c&PYB(s4= zhFNo%F#X1a7XnNu)`%65#?dfcy;EogM@v3hC4b+=0mcpZD%5Gc=OQz2*j~+%6V{v% z(qW%9%lMcUx`1u3ieivtArUZ)s1fTrdTmn=#*h6Rwe9ing@<9r3;BY|2$iZ)8I(+2 z53E{Z5H_=EsAGg&;Bck@PPIv$8Za7XE!7}&DO)+8Y8eY106xyAxF~~kXnn@j9zvz4 z4T_u-Cn-4jH`naVUdm+dmj%r;g|kS@=zHxLZ2KG!R`D_`L#ZB2t@}xc&@ujRiJJ#J5*3ZvN^`t7qPZoG>;{)=WhH9}YOJvS1_w|3CG^cN0mWjHi} z*wo5eGc)+qHCP7WEU6-57*P+s_NKNFNm(HW8`3QACMK{0?|WdlG`O1c{pJjRpwZY= z@zZ~oue>e=B5hD^G8`K@D;;*P*B5#A$A5b*TaS=yZ-}5^ru@Nt=alK^3nrKRw3B69 zuQ0H9bIYeW)3A3D?c3@$(wJ!WrFeoOgw$H(KJauGTLx^`pAO=pPITP`LS`MUBPP$3)6#HG^Y9l z|2A%mov@5HX~ZW9fnmLo27`F@PT>w%TUbxq&=zu(=FPbO;0@ornTRC8KsZ~_D(mtk z=k!D^;Vu^K8y_m$jy$ofJ9O`boA|$YKFw&)TzN}{ogW_~xYxM`_dz8;D;h!RMc|l%a?H94m^^r6!;&KA! znmvu&3GrMx<8EaXC{56s=H&o`j#Ye?dxFXhtH`K)W^}F$pD@lX=Suif7G~6ln$wfK z2i8f_e}X@uHX4tP`JnjsTE2G*>i?%j*OrT5F*pa%Z~-&%H&L>#VXCnWyAAHd32i60 z{u8xfFl0JJYKo6Bj>)6M2Y*iMol1}j8+BS{@z-boV0`(51xSA;7^+2N-aF1AR!ULR zHUdM}J;^q3^2-~FXn7g5i2BzMYbhDCk#GU~=sX@V%$37c?gkv2fI= zXc_oMS;Ck`$RnM*C^2(bvzpVbZGCUX^GD?cJ6gwnUH5MKss@l&X99&enhB8j?v*Ru zxcPzGHty|@0b2x{5`OmKFMjOJI$qu4VF43W-{AA0T5>#j^z9d4e0KNl*ONa=6w-jN z%dp4m)~emMX`aDVR&zac>+M@+4&Ufx9O$kM}(LLP2LUPU8P+MRL%Ng#r0#UoO{&pQy9 z;YIrfX%M0p;rW5(OTCVQXxumgwh^od#Ek8~{5RU7JdpbW@OCW4rJjs7Vw@{0{2muY z(68H9$H)?5h{s^2-o#o{Xqcc`;`&weE~x=`*7%=LNgroL%U;xMPuE^^D=!O7`T#;_ zO3ny8%RK1eY~?r|37N0)cACa?CHct18cx+$>#d(5wJS?%u@+H%wSct%17+Qs3*23o2-<=pFb2%qgblvWyl0u*$hC*+z2K%}LfWTLH7Zhn#tGW`U!*a5c7Z*_Z)* zE@#|OMz*>-;kBw4t{OXEjbN#aAV5jebv~0B(stwghHIOGTh6oN_o|q0W zYYm!Kh8Wrj1NI2Awr42LgZ^rV6(O5jWVMLltcJNI=osKe=hz)iO#CSvg^&#!U!*bhmAq`S z<;PdPea4}ey`RTSjLRg_ewx{}k)zu`SFT*cnJ`uhSX(EX)-KDX&!0Yi@`NcG<)amX z0FrZt>S~&vQ}&ecwQSL>(Sq7>65v>{t%V%qx}}}7W}3Ok?maIPN_m?J`dC+b5S_;^ zk+^RXnQZ6Fp7iPUJ|}+Kl`c%CvNBEe7(7~twK8cxgd0ygMQYM=rXQ7AaOExvL}N}f zi@1PfU-#7VoJ-Djtj}CDZdvYoWN~7X6S`R*eekFg#oS^_<1N>;SePRQC94d$NvlXQ zbud#GBjTkSfla-L;Khg&Z(TVmh>q{uUe$}*stj0N%Zj?hc~K+rv?sybrLz`1rqL6k(09er_0DHBaG9Db@ z%@@VJ(*?_F#uK(^yZy}UIXHl73Ym^MAIHG@92^?Tg%SyA6nxL-5xo6@4ed} zz3HZXyq>*y?9FWD;Wp%CnAxQteY6PF)b*@`-+Z&HiP0>@t;>yxdS89j`~=e5KOOR4hc)~t zs5)+5V`eG7V|XSym13~H`z;ZR!rY=jk+UY>cJnOu4<0id%2uMA>(7h8mvs;EEk~pyjiP6lryfVxD zxw%#N5WAo1L(q{Ve4uXfHN`YDHgeFmBb-flNF_`!v}m?i+2DdyUphnuvuRGnnjzdN zM(Re7c61j$0kH8gWvDhD|7preyzH>Yt*D=niq`S;0;CbxnSQ_k4}A$ZDUmni`&Zvd zeif(6S+H|UFy_$#z6dbjhq7R{KwoBhRWvTx9uwr$xRnK%I$>gxJ`?VaZn*M{L9T!) z#8;&n4zs8rJ`Ej~FJM+DjN?TC4oE}VW8{{Iw3>CHOt28nlZZ;;EFvFE_YLKuw^W?TxJRsdv<%&fs$Yi|+X8A!)O6$um44ffoqF|#FNE3xdX z03b5?JITiV)0;>GGBjFt5L0r8rx&4?s5!JLLZ0rfuM_L4cuBML^R}2{Z?IQ44!LA^ z%hg;XgUA{)eoLQbv$olc1hQb>%PhK$Ga_oQuM^hBp|3Qif;F2dY-)9;L959fvm}@Q zCB2x-f$~qAAFf(_Z>F~c_%bHb!nXX3wR=35TmKTmh17uC;-jsvk-qST8&3zIW3sW@ zmRQuS>2KZg{>!&N`{a{M^xR_ik~jT(a{<}>k`F6^WD4X{taV6pj53x__ATerG5B1| zX3|6lOg5EVlW=sNH!N$I@yIWQ;TQ%78c(x=u%)?)wd$gdhWsm_Sqd@I{L@g-`wZDE7z{j_2%{K zzF7OCAN}O!%^M{xBF^g~#&(+=5B-rk6I4mz;icj!LM(Xo=9_Q)?1K-?j!yM@zAKDn~l_ci9*wUazZC>^G(l&_a!M7oIGEUZb4({q%pFJQ$-{Rghsw0G?_H} zEKX~s`9i3!Ad}wVXcVYa2(2Qr{4H5J(=FguWaW&orT|FVu|sU9H14M?x~j^;5AH=D zrO~iG9JGk1V~`OMJ{Jm(+Rtj8dxrS%TCAb6z!w8x!A>j=I*6=<3xDt(I=Y;`B>qx6 zlIz%DFHs3+)|~Jr2K&Lv2aCS+m)gqwJxRXDbYYOFeFUHd@s9Q^&{|UeiH@H0C^{+^ z;RpETVXvspl5EGy0PB3J;#Aqi2N|H8MXl1z!iTDj{1RPOk-?98DW=R<#!?2?AZdS&Mo`z((iJ?0A6EuM++ zZi4^*-~V3|%+G)Ri{3MR>Gh{}=eoSR^zJ+F{PNeoal!E5{yq12-hAUNQ;@f{J%0Gm z^fPOM$Y$sX0BXzp8*dvA+E;RwRNT`-$|N!-W=nGhGl~rT6N5W}W$a3Nt!7+w?H=Cw?9QM6{1@Uqeg5>VTb?5CrP&}908Ni0%Y;G6)WoV?SrN%?z^s%$ z9^AXvw_88<=GK3F^6AI-?mlWM;nv)X@2+3F`u_XgNc~^`@Bj1vInTOeLJASmr$qxG zf8xIMwWy=dM(V!D@haq% zxy#it6n$_&<=}zSp3XT9$&!nvRG*e}1u*nbrae+S9}fH(82kmOxD7Z}jZ#8}l%7j` z`iqVR<^CNkmJz*n!GY`+?JCeCJl@~atxp>|+nGj@n{r^CPA@`k7hW3AV5fR&<2N^M z3J>i?M~!7d19qf1mOB8}BPvU9Tr&cnSZmoTQb%P!3Oj#CIVl0^rAAQCSOC^tC&F1q z-SGm<(YamQz5wSsh%@}fiu#^9t|3dN{*fp5r@Viq%~fc$ap6}wSk-1{X0k$Jtl}$) z(-9-)D^KPs&$W|Qb{5BF2=+Z-v@MY{g&jiL;VMX-OzSRA8wmuDMi2T{jnKFl1sxC# z6ODt|d~u@VQoiA%W4#Hu+jVFq_H1A^-E>B+a?a$wx!!J~nAm96>Bj`kNkvzH_8K>b?tb%)hZOw8j08x-M~@!9d;7M#6^M*Zd0>p0 z3&_6plmKxu6urizZC4a`T@Cgxua-~D)H4T-Nuw}6(YG|+$Y}Itv-y7pbG65U=G$ja zo-)?`2M@pc@@rc00QT$hb|rVKYS-GQWViAmIl^&51#k4exp(hxfBV}X|KksLAKvv& za+i!4d+7p8yXNphR=@4`^ACRJnGIKkgt76)7IU+wFZ~)ztlR;QDQgaE3u&ELe)Qt^ zzx^K`a_3r&q+4zXUj-(_+?QLCLG-pzXF(@*bw{+VYTj3=|Eok2#oHi2{1m%Xmva0RQ}y!nRj z7#n_uo5Chs*@zimYQegWoYZqDNA z|29^?z+%k9J0;U7XQ*h+$OAXAJqyr!hX{0{f?7HUuW7tUSzW9`QQ2dVu}#hrdlmzMaEnjrb;D^S4?n3J4LjBJAl7=29a8RW-i)Lv*je}^ z7ypWAFUkgVq^+Zo++IPfPtk!i5itckFa5B)qL!kg(TO^)e^BG z;yWd}T~+1eaFI<@B~wk{&qp)mOg8cLzUXXah1G-6eyu<Os^PB=r>6?O$~V1%=ly`DW8^fA-N+}c?waqd`SWi*5qq{Fh!m4ZAu7O+-|L0%+^FRLZ zCkqxcjbaWqLyHNLm}Z)5u3z_UFOT8>{O2E7wqOv|hC@rI+~@x!Di!D4ON6y-lfyIu zuCb^H+`?zjNjQUhT?(?OHb!<71*~M0ygnrasKa33kt5GUMp$tuh0p!aWYw5>$ryZY zJpy3SmuhA&?!cgiUtCwa=#4rN%8||31en%CrXiJrBRmzC3~&99bhLiI(Q6a`BW;Vn@Rh@XGV@N^haZxMXQFp0aHbkT=WAFQKI3 z_;8XjB)y|hU}uh7ce*8rVJ(-PTUI1pL+}7NLjoTgh8Zh@j{u>k2N@f9S)aZQj^?21 z8dQLzu*?>rwz$+eb;pD*iBfWK#yqezr+?iUAv$60$&A*ntMqHiLM{NLuYqc{8@0On zUYN$l?o2(Jw6ihEx_FE5#_I~y?X;#>UsFk*%r5t2B&QzJi&T=g1^h+XzmHFj^@ z5SB6Qs3e!p!EqRsJC9eU-T)@{7_Aoult_Ff$5u>1OXn_I2n-9%wrCs^PZafi>WuR6 z;R8E5a~YA=o<9~fUt7F?`L(U1J9j>_poid^>$0~6K=9?8=WbL#f8*xOcW>Rgam~|Z zR?hvhwVko|-h0=Zq^+m>nF7CgXgc*&_IzKGL^N|F;Le?XdIw)-mAc<~=iOalTR_WB zqi$O@k$w79NKmL5`><}d*ci=1M^X(=R`} z^Lf~pA3qlf@osb;Y+5GAri=eoGX@-{G$ybFn3AO zDscrS#(02`WjSVXA((P4M2Nncm&PubBI(CC8bO|q=k~3vq4XZ@3+e`MQyTpK^rl`G z3U1oh7So%q*o zFnU#O>-@P}LF)1CC8pOt(BfPTJC@r$Z?aHR*7U{s$(?2NrVLNrZoF*3w2V#tf?=cm z2OeT368TeM#YA_bgehp?GQTFgx4~X`~%bpk*#Yh%~87JiCj&|}%s~gGW zVolGbkl?AOk~O(r61@|Fl&O%*T-Tai0_g!3&5s{F{LerA`OasbqG}9!&WLRvSYk^+ z5AJ)%3|kYzvtlOD7o?5M^DEbG+`jd;4YW*&9riC?ym8~273-r159T7~Ugq=5b^5$e z^wK52p+lFe*Dgyy{@IaoXF)pk$o0Sd+rNz?x_Sm9QQ19u^yn2|HNAG_^)dT6ltuGk zF_{xzeCv63!1%C~qdGxm@1>SZ)@VS08LSDWGTYpwE*#mfTi5Ldxi7V|6uXN_i+ART zLg_8=OW%C$uKwqD?>!J9*RNj1-esU?w#|#4&B}ajpf1yf&W@OZEEBi@p;>p3sEWF& z^{XBexbw~3Z(jGS;C_e5l!wjrD{p@51q{#c-oGnT`z@qzzWLe@B2kYJt~&qp=Rg1P zkAEPS-ypj6;fF%bTO#g!_SwJxuAdwF&2N6w^BAwcZs*Ou847;-`qejIy?y&V*COok zx4-}W<4-?%<+bP6-?~Dg&p!M7^E;oLW7c~ z2vSdR^zeNtVUxDdYcMoc?b^)`$67=A+SNSLa{1LBdT#sm+iyLBXWP&ea_{~@~xq3X@bUZWLj2;86SSY(Rzo?Hp=V&Y}j3eXcWkGN% z8f({6XRD9~Y#?!*kz``DTa$;1grpvTgF^09i}X336|LDjYOsz67kcDhu*j?eH zM0a?QahpmDhe)*~CvX%pWbSy!$8pL@rhjBQIw){FZbka{mq$LH$Ll+2U`HFp* zWJ8-XRB#{HlH?E%zw!S|0B9Nqy~9O;ClZiBvmXE1jW4qq4f2e?mjKn41S)dR^JWoc zu#ME7z?D~r$U4)d5%9$3* zN#FRBmlQ4Y9CM91u}`KH`c~-%_~p33j<~!(_y6~c4Zh>w8$2roMu#>%EHhs`BUhOn zyj8Cs2EF`ckNZxbj-GBJ%?)FlV%=MEHDG>f287`@^HslH{r0VQP?KIX*)y4ld-bP3 z`iV|3K|Zm(y>GU*Fm6BjEq zHjU!ql$97rZP(R0xV~Y2$yL}JZ{F}4hUUKSydmbQhXrOAv?sy4>z1s@HRtEo4c`2Q zC_W`Qyy@nJKd)VVT@*i(&YwI6ohD|8yZ8LEx0@Vq+U2(;xoRBkcdzzx~%={`^Bqn_66m{>LAFZ>o^xy}99?_il?bzvlh%#~*+G`4_j} zeW!eX`E|dh_uJq9?&D8B@f|V~suz8K@`{VJkKcd)y=>=K`MBiHoqv4vcMrB<>iUDN z{KWS!e*Uw!Z{PMp2|l7d_MnnSlUjell7G)Vn@Qd8O z##qrmyST{0ODTqo*YvfVD6NlAMvLm>4VgatX>3@)%UDUba<%G{{m3?kD)Pb8SPP

    Zj)&vB zE8c3uc|lb;w)kYNHYCW1I-z3)oRg1?lboe3h7oWU%TT5hFG>J2^1t6lK0(;xJDb-F z43QF0L)}qo87#fS8uys=5z#R(eWy0(1AQFO9BD7M{qO3dSqD_Mt^!#7jIu4|y)=GF z9D=S4bS?u5Ke;sysdMLYIiFsvEQ^aR>MO&UJH`h7FY=QtHDKtivSnf|K~b7>&odmY`d1@wZm~u)`nCI8S$^E&+_Ih^Ef@UV^D)6 z!_u7Uxvnj&6@QNhabDZsQ<=;QYra&N6H{ksoexGX+~wdX466qlaPUJ~g!J0g zAKrfdgP(n{w=D=bK0=|a%xj)_{pE*#cKXf#y!DsA{q--tc~d4vPbNa)GWbgbdAB$^FGFQK5fv(?uHCff}TrsI=xwJPMsBL47d;HL8Z~M9V z28EqcE_pzf24cW!46O9Y0!?>4aLl;(A3gFAg$MC|{()e{Z8&iD%xMD_8IK(~ z3N0s&wt<|{v%v(_p}Cn?J&%w_gG2_^%ovbn?eUNywd?Q^??3gscxft(Y<~Rdr+H6G zgJjsGN#J*%y?(X#e0THMu`j>)^3w`A+Eve6)J$w1` zjw~$0dsWY$J#)U^hYwHRy?*y{-%;exBf`<6DUaGAJxuY8(y30w)AjA>6T)cKAh#7g zDql6RZ3NVd`Car6I3#)%Ns+XBkpj1*M~WP#lpczFsj6C|bUe}pNiC5h%O?vJg_v^e zP#|nH4F2S7klv6XuP`?w#oP|0ly%~yfW%LjT;-Co;8cQoK&ed`V+ZHJMWS}Xn=*~wjI0-jl=dd%SQ`d!br+WU@pPm-d@uqkw+0P;>Jt{&`pLjCVSF{OE%_`15~q`b0DYw^9!3t+G5BR z=~rc;^x}2Qc7+}XKjg#AyS$sn62vnYhPb`MR2oL&7!+G zWY5pP{A?Y84&EPr_v*v`s7_CtOU%S_zAgR1sh~#=X8Py*@4x4N4ALAupj*IsK`e`a z(3BBv(FSLTM`+#1vj1}5P6sIINYygJ5%T7%XGfI?+J8sg%3rL))+S_ zxL8TXgWykW+a+*n&&dMaIAi!Wp4#SRZy{0rS2sS(h)2Qj+U5(%278KaF+}5r*v+$ZqrL}j z0H+b65{fcfVuE4bO5=z3bS;Z7yGBiMq9psxchM(ESEc1Tg)J`f+wC~4)M@{sTF}!5 zf|TGo5%J${A!K{XIq#&)6WwMaEVVJV9{1SniDs(;+aYNBZ6KAJfS{4PpkYEnI!vV& zKZ2IKNf8hJr2G~@Lnl#~yduM@TZDj4iuj;)i#(uYtJ3`<>2cA!3ytNk{5R?M7uX;l zoI?@B+z!#A34jRd^He1Ii6ogb=Ts2$KLKQ%wJZ}23#kO1PD^rK)xiFaL^TuDPbe}u zm$F*}4>NN+@wpC8*EOVwFiM-a>mXl8Cf6ilV1gb;v_RhKM*-zFB~OAfv?5mj!VFSO z`2uH{O<>GiH&zC7a;G$wqE|1m88ud=No~ghb&#&IrOlHelYrU=33=o`*OFj;e%+QR z9)NQRH6+mqwc=#yAc>1p0rP>*kLIkoTq8Ti-=UU;kd$cpF-|KB!B|&@9wCtKMGPSR z?Vu%Y065_!DGGq`fEgpzJ>DEwh-*`_X^nMoKvvhww9B$4q>umltwwI+XbuL8J%s{Z zxW{-`F6!{+F?97o?bkhb_WT#0e|~vKZ#!CLBS^I3q$a|kZynLzyy08UoRKe(A-G`(6U&XmuNM_VHnzVEYyJjWCWM-#qa4z}~YxD-I`5#z&dH0`vOCi;!5V zQxxbHhfzXbY9#F5-kvpY9>BEIk>z7W zgjJkmgv9vUoFT3ZFbBJ-{|bhpL)9)RVoXPK(qhE+R-zL8KyC%Ttv8 zYo{`l9ha2?Zo$vB*iz!cWRLm8Kc~(2AQJ804)&+yB?dTK08dCZMi6X9(Yp_kC6xKC)2gY@cP5LClDTuqq8p}rJI?7cT54!%f zKn7riCn(W}t>r+fRU=mg09zp!$*xo_k#+%Gv!r7mJ}7~cayGttgcI2s;_T^=k=A$Q z25{vax@n!_{fR$%2t=AG>GI&J@Jv_*%<4t^h)f+<_KjJy8i2@ZVu}+}57T#CD8tic zDBD7yL!{*$;rW$$^~_OqMFxY_?=)EnI&6Y1rJN>f`_sE5v6` zo-#?NpSrcZb!z+6#S0g_tqGgzjSuRpDXDPTUB*Xi#En-td)W;Dyw!g8+<7lDK7F*S z^Lziz1MZ7c&d~v0y>KdL-5@-U!#R5B7}x*v&o>QhhQ|7Oo{z5_(T`q^)ArK}A?v@RXcWjS#46Yp+7aKuNoKgytu)`*h&`Q9FY zlc%&cLWn=;B-~r!2 z&7{>3E!IwR&BumYZ0hDcUI6 zV>*DJ;_>a8=dqP_QKXV*F37(&lOC6Zd}T5wqY$ZJxBPcB`OW%!XB&>g}Wv? z4NhpLJ`d4E(rzULF21E!BEtI+2f67D1^3cc$qP)FT%q)dEHVmVcQ7t|6FgNp)uQsk zwtD`X><~g*1YvX{0}pt8J1KJuY#;$Jzv50SoBXl#=oY8R9t^Sj{#lys*d{EcjU18u z7T;1XZY%fFdHnAbCuPmsL|i8fwlxgQpGpU2nh@y&KTj2Cc&~mY0>j(_P>)!v49RQ$ zK%w9{wlb2YPzVFQIK*S(gyv<&e&i}e4c?;Cbap+nV>!oYcxwJC<{5=$ppmChrb7_} zFu7!)fs90sd zTS_fC9kSGvfC@ZM%?(5qnW6)z7Ymn#ouIQfdb8>Y2M&(E9>X2*GM?l z%a?X8TsY^ru;rhqbK5@D=?M7A{B`MjH4 zfYTas7Y7fX^ifaem>k(SwRL)L?}fHv->UfDa}G^=@3=HRR58yiOXd_meti7!F}LOu zZKlXhZ5@w*t*5Rm!v82U6Jq&Zj!wL;`J)Gqbn!QiZDvEJ(_msbX*5%wAL)))~#D|IT@~ADXXj4_ijm zTDRA4@L^nb?xIC?!;Wm$#%0PY*B(SRmd!bKn_NdNanOXvww+~Pe)+}Qch?P_e*Woa z)pWzT&h`an!ykjL4I!;7^8TYSpw^X<=`H9S#d;M%K6rB^meN|0L(j7N>!1}TAA`uC ztc6GXZ2V}?!p3d@@89St9Q8a{;*!03{K1#^ph{cqi$)s3_5@!PW6;j;46?{q^ z-FYOQ+bsF3L<&8zJX*ozX)0tA#a-Pd3{a|CT9VXfx=lLS;}zS$z$YmKmUx{uoC?i2 z*Rg~Uq!L&Gi)-b` z-k3Zs30ORiZB%lwE7A+JAMi_-m2yU_foSklis%rX$nWBAkpa=a$Pr48iZF3kVTjJ7#-%J82(~D5Da46L{9TIM32S(_6Q`4DW(Lov zVDo6ehB{^dj~e+7{TD@_#ylSjq>E$suVsqxh_hy1UU5Z_nbg^Ka)VhXX(<-icBSM$?rTo#iKOGu80}8Cj0z548=9 zTyUXg0{+^KYk&IVcis`yRM;riiqX`n_MShx_wb>?<^^xxT)dP?Jj_xH z9%4rsF_}n}%R5Aht+_HjaMIBBneC%TF3_|wua4Q)v8_ux7f)@QmkV+XkGUI4b}EZb zb6Sy5Hk7wL(?+Z*cp;-s6yLoif}%O*wbLBTgHyA7u~OLBzG4kiu(x(iaU%C=@KA_K z({#lI(+x|7NuYuOmMr3m3P@qnRgxCBcNEgzFgTYo9#rErMKt+Am8zJjkWQ&(w*^Z< zlo6|qV_-9)jV$dEk5NVseVG{n*gQoY&w@y9TXR}@hVcxf=IQV`{7b=J({WP2)W*?M ziMdt*aq0DllRl3|Wbrlii7a|p3Nd$JCamKkfMFZztLG~yyyUe5MM~^Q2bSIBMZ77L zUm0g&Q$;;!I7MKu_t z;AE|NVCFnf9msSu;^y*ijYmeIiicC=5F?M&<;u(eyBRMOAH*Yvv?69^5!Nk1;nzbj zH&bx&LC{23-};Yl7Ur-fuYmc;M>1RrBG!FG(NWJVt@eOuPBZ%^!dIF)!a3;^ol& z9PZW5`+*}zeSY-g?$f6>g1x(Vi2xi4J@c`nZ43Ywu)!Z%0W!(XsX5P@chCx;Ye^(J z%&Sk4HW)tW#^)n%$!(lRNu($qrP3Y3Ny;KhUTfX>c(@(z=<>ay%6Xj%i#lYnN@1bt z+MT;xxz|%(zS_Ta>*n`A{9uKC!;Z1z$28OyQ9$A~rk~MPYJSR_**OF{hsd4ih`})E zgQ1*Z-Cy4R+t%^zG>yvVuoGzBQswpQ*EBXDncnMHdH2O?&!Yzqy)K*0Y46`@nN7_R zOVgbxMTMt8jGo@Uv*%Ce1X=r-aY8l56LifK6r@&*Lbp$BS(?#;8oFt@G$;-@yK((G zv(RPu>BpaZR9qQH@V1#seXO8n4r}bb@hY@UtRkd0M^pGZdF=RL&atrd?#`XtUME!q zRB-fImIRFjn>|fo>f6+l{QMRGtixKRC~~v7RhmpBw-eOs8J>(IAQ-lStY4T$g4%KL$bveUaTb}-ieP>`N$|+^Q~XU{(mf@FlyJCG7vQ<| zR2cX*%-jwGo^pof9McPLE2*Z>XfeFSw&oZ9x=v224seAnN+9M?6c-UZ{UnYVw+p04 z0I)~xGEUyfvcAq!x5D6Yl?Zq(*Tew~EdZYq^^{UP6%mRC;}LU>Q=~q~Z>&1$krzOa zrO<*i?bHCkpAZ1>!3-canARvkO?M?3AlRDHp-i+9q_M-$5AtMap>b#ENX>(wTFTKa zzBNucMui6Rc+sIQRNh5Ae8j#m+@Z=gHLtdT;fjMY;E)+6RZ3Rt{V7QNwgF53E4@JT zCFzUJm?fncCgqBavVm~&A*%qe%fSn@f16&3c`-*iVgggyWH3k6r@h5#_*AXjP;W3c zz%|RF)Bq?=Vr(GCfIYOe>HdQ!yH5d{r}9XZ#+#KPN2T#9G{tS|WJ|no!O4wg?LOSR zd2@QkA77ih%ybUVuQFIC$eB7%cWG>D7DAOb8gC(F7hgsyTn7%qFRyOj?*U$vMmCPd zG3516B}p6kAtk#wZiYyOF(inB%t9V<{D0mqqMy`^zU&SsjYJj|lX)C)IJRX8>lUUK zot6Fmtx3U0j~wyZIQVMs)qniu-yht)Z$i-2 z=kDW&X1=ygY-_qP&JV>B%m#SZ$4iyknb&W8b_hfAQv-6=uAGp1>*mcPUw%nb^ym~= zSK^s`b6SyzZGX<0c=+(ii_QI0+b3|?H$W+ECd61adrUQefl07~Z2x%srdE2udqsEd z-7z>$_hEPU(S!SUZ{PNcP`>MT-2f*D%Uy=IJg?b1y5b7!jw3_1|k&OzY70CJs3T zA-TNd`C5ceDX!xpV4{E(+Y^y%$`Sv;Qtjc<)47TWXaNwB2CVegJpmO^unD~eq zUmr4|wGv^l4hhIpg6;DpC9t^#%dy6B$41Gnvy#25DSX%#6d@YS+@(%aZ1*>|p%IeX zzi_as8VkY_0+9S+Z9!?nH!+m|+H)xS93)}S-NGPw2xs+_q&`fs< zGaQ%a>PV^ycA!^XQt8-z4S;~A&-DYcV$puP&YD>O9W9Nbh9)hMl3=lGMWSpj8W5$Z z4k|s*%R*va(Ll?6RODJcq#1}gkQQN1WwweRc~u zJyJQ1&CwhQxdk|W*0-gMY%Z3PA)F`Wttw`)@iHd zs2|Dxye~et2*3Ay-|XJ8Ef^QOcY+zyQ7lrRlnwdOk^7W%7OJtGEm#(M~{6G zK?jU5>|zGR@@mYq=V;ot@#d8eeZSB$qiF1Jbxx^JB6%J^eexgw@gI5z7cZQ*A;h~x zd(Zc1%&7ChqsMkKJl%cv6q;5jW!tH&s?dYF_|;d^%RnpPgy;9oVP+ z{a^p;1tSKk>3V54?Zpeb4wT@D6Q>-vcgi-EOj?+1Jmf0{s+?JvRI!GdR&6xZZcH)t zm_D(Ti2tdIwTK#)eDlZyXxH%q<`Q6`^;l6=1ZXl`P@n`oe&~426n;1Dg|cE`L=e&e zY@m;kTA8N8tQbz=k43efB+n-Sl)JTn2CB_w(~=$J$h2Z6FmR46 z;=`E8#EUI#H{I{jkW9L3lA3FcVIu2{Za+A1T%aW4j_A0wvrG1LDQtmBZw-_@CFE3s4Gl{M!C83EB+oR(YVDBA#)PSATzo2y z#>PET(#v%XK)JA)2+0d~-Y6eEdA?^%W38dE{HV=iUM9&2`V>fcLqgTYl5x+qYu6q= zdU*HlUEN}OJa%+*|DD6PURO0fWWdo{aZ_&fWoW8-Z8BmKCOEn?aDM?VV!`BbN*!Js9xJRmr;vdUq?*u84|pH`TC`u zb6sZ37C7a(fgO}gl-XRsb?)08u(hT6Cl4mho;~HgeT^O^;S))W$~lFejzb#E^s(*B zGpVV7G>SJqyJ7f3M|W=DF+jA01D>1&J$drfwQJX%o{b@Ys9^t}{_&qodVc@I_rCAI z`B@0D|4$3~=EFM<9pCzm_#CZm9Y29vl#JlWi5Xg;0)q0cOy%8+clRFNw@bwCZzIq@ zfA^g!xO=y4TX(A4bDXG4kYBrc`TUvFm#P+0C8T4=sKv}W z-hTG{&>ii+jer|jG|OD)0UtQJe)an0E0;Hq9XG|v7-!yAM~RhC9KylVP&U#Va5r(+ z+}0>saT67hE+k&Z>Kc}sQM|n?CXuiPq$mQ)L_Y$nRu%wclCfaHs5pxz^B~Rx$tX2% z?S&=4A=E+MZ?u}T?0dMlxqubIy*w;o@Q++>i2x&nH~|QL9sv^>JQJU%>vZsuA&tRr z&BLcK;z9*noa4k{kY!E9t$@(b@@T@O!!SCIulZzvqMzWOl(F`LEiJGj3G$A0CR!1t zEv@VT4@K8JXo$)~LbY+o=dthqv z%AcH7g2LiB`16&x@&!+sPlOyL8vPW#=L-)-koo-1YjoMabPaQS!5#*l#6-HI_(KKt zW`HxD{)1WT&PyR)1gt_L&Id;ezB3(kU^7iimhosSrL5xS{dsug(%?WM9Sl0juDz&R zNS{2Tq)Z6Q2`8)NM$kjtu8Zhd!&I4IiMvi?Q$#bBl6Upve++Fpvlj~yipOTyclMKh z$XY=r_@EZy-+0ei<)noos#{X@_KOev4O1x1@%t%r00%&C+8I!umJ z%d03pWhpV#dDI4RUd8O?(G5cd^s`o^F%w`zU7KX5T3Q>YqaCp6A)o7^S}Am<<4Z$X zzX^nO6b;GN)`5!`yVT=28Q-nR!P>}pzvQ^JmTp-YMkU5NqHs=)U9>q@1G5B|8AdoM z{qyUeIhWI!G<47AT7n0S(LB|VYC-uzc#N*jo;l;ApAN5qkN_s!2&2J!@_g4+-PZQD zbcz6c_RQIDzx_rdeEQTGY;y-VSJ;}UM^B!-@GhuBIggmVJ**jE8Q*>T9c%sZLQkHZTb)L6+Pfm|;RaP?n zI$^M_F<~t6-;)S(@Kb6i*C4@3PF?aSk%lxki-V}&oUUPO>SS0bhT$ZX+QPheLh(#= zu9!x^-%@q{oA?7JUl=N@>y#psJ~~M9!oWbRt}C#GpHb8R&!YjHysoH7ah;Sxu0fOPNMWqroJW=^Fpwctp3hLAyj49U#YE1+Ndps( zT|W^S!Bi*sTX_?SW`JY_r3vjII=S#OieLj-Y$k@isb;?p<=qf1AUXG6yn68H!MO|P zm5%+%GVm?(3TY}FuX1u5>YUV)?OLAtoY)a%Ig8o+BpUux{g{IxUn>svMYA zXIo^vCYyiZIqoeTFM3YXOU!xSldj_8v>j=dIBl*tlZKhh@6yh@V<+C8pcu_dK7n@d zgRxQG_Y~x*bBvW6fstI`ujTbYAj=UKFJ8bjyh;Xq?6N%t9sD%ZniDHCS}Y@sb||L3 z**xU)L1(E$M&(mdImKXj*YLpeX#b%sb_9I=%^&jNqZ*X#5Jm%=8`k`_y?ucQngeIg zojrE^PvmFi+uJ9!v?L_z*MIz@X2~AGr%#{E8##tGipS9B+duyC+O?~epdLJYKtL^) zW=7t#4$EVKJ*y+-!Q|fPc|^_c(*2913S1vb3wrL1Zy&t$L2!HRiT?bV4e&46q2|Y8 zl-(oOZ``1;saP#1|HQR8o7dW6L_dviM}>s-m$fam~-jIisg$W#N5*w32iKz<75 z(cDgC;?PFhVOtehgf43-^xT;SBq!NBY>OJR@Dl%-y_sBXPRH@_=>yJWdF@25&p!Ly%j_Wd zPp9vPx6U@9XpkU9SKC+KsA6N&rmIamo9qOD-!@%MZK}MMAk6;uP7BZJsYZ)?_XLv! z2f5-s4ck0|hgMYb(1I{}ZXX>#mBkz&B?xWQGT1ZMmc}pAvb`>wp%E{Z>Jj^fVBa?r zd}3i!zyE|cmDmATNUVPxed$z~ZWZeZIq@zZW|~X(9Ax6Xea_Z#pWeWf(KI3(Nsa1G zwicP;YU(F-8zLOfuD4?fi;#}MiP3N$rgw6zz{a*2u`{PKPV81cLrJP4yje?HFv$oW za>}xpZz9V=&bc#ZvdsHVZ_7HfKO|z`rGm(TpWtKr-P9PX9 zI|YvYTi|d@LRMi_#%BxE4y6XCCbRE|W~-WaOwXJ>efi22-{CeeYV%}XlSgi=pueUI zl~u5W6!t1mvXtn&fgnjyaycFlXF1p}?GJCL*(W(yAJyrn0K|Jx28HK&YW!?GW^V%U zacu!hQCmN9{Cvh3p^ZrEaQR3|`XdmOhmmz>6 z%0$FbZmRw!97{tReuwgKvx-Zf5QZ9XB&Gm$*YaJ12+^{jm}H`(R8PqV-gXQ{r^Kd!q*HIE6yw$eG+nft z+zM6=OAZ~*8PEE*U7>k-%=X&XZ+vdpl!fe_WqD_=h}+S#%ti(6U*A>dn|M*4&SqKa z#$^OZ$0(&CaG}vF@d?l%(Jv`CvS`)w%<}e3B#?#0r4VIH6Pr$FEvMDIjfez~GAXG| z^4>_B9hH$c<(RZ|^dvEZAIPB4`Xi}rhsjJR2l`_Fg5`U2Vs?2ydH&?}zU69bXH8XlUYT%{&v9)at{P2_e4>TuM zw@l^AZ7fBEnO@XDYFCJF^35&==B|8!zty0kXwuaaAe-u z^UfE|S)zGW>yV|Rj!)BQetfd=u5!rF&s11(+0hDfMrS}K~-1ODnnqGSVI8#Wt= zHqI*ywSF!Q3?9{;?3%36$HB)&jHNUN3{}w@kf$Z0apxcrhhiB9C!PJFDeIba3|#$X z*bFHumX!>e!!i0+4lQGr_+g84$w#v7S8S)GEpqHLJ;!jEI;6gY0YvL+vhe_AkF(@8 zG)Nau)KQw8saC05p|KLYNf~1)^6*s7L?*qrwecFY#5~AP)r(7pqFtGxUfnk{^IUpl zO20)U4Z{$16^2mLXe3phb}AXh`~uc1E5GPW)ze$BPLSDia`L&5L>1Afkjfb8{j~Ea zVq&WEAZ6E6!BcmBUF1#n1`#GUOiBPLK2O(I(uBu6D4M|OBPOE|_t*4@EKf_pHR6}o zwvEkSJn=d8aj~EjAh3ARiV(@6i(RGF%)AA%xMzN8;?i2Li-tC?Z#CS(R{d=7@(igUF|8nQ(;mz&i$91N3^q=oO+1pLsxPSltqx%ouzI>HR znCmdpx34TWSh?W?vINCLa?u3ZFEq|iK z%6^O3P(4tFRdJ^#`W$kSW=7y)>C}@noA2LpZfd00HLbKifARd(({tx9?Ce~*%qDEf z@=>+_@$Y}}iD(U2P6e>H!Aq1Ihs-;fG}Q6`>dUXb`O~+@HjjBMrfICM^@?|b@of+pP|o4$x+ zXyQF0*$p|Pj1BW|4hgm9bkvET81YW2Nn9?T>2gWRI9k!Z=B-boNoz1>%QmwYB@X7` zhd0$@w90VuVbfz<$2wsDXzPV9QfR=pj%CY-yzEVf1Y@#h3={RmEb*_TY!F{nWr$N? zaJDhX1XnD$DW7_oiPOnceKQbhq5-v&ci2SQk*rDT;if>rs3vfxWx{Q|m2ao3IMB5A zr^Sn{3`a7SV%Zdrtuve~TLIxmnUkTVA-pD)9Ly@V6 zz=w*^yP(PF++F@;Tmnd8oq}V*vq?uc#gq<6Gztukc{Bt-GCYPWu@yjX#LLdX7<%cE zC&(z^Fy+zKXwiB=Q^XR{0T4B$*99l#^p+$w#><%czqSgPN+P$EtEdpr$38Wp!zzj~G(^XldA!OJ17h zEhRDaOTXM6Z`p6@K^+O4WXtfiwD z9+mUmJe<{QXJ@L{=T3Fm9Ko;*DCLZ^nk6?;9+(av;7{W+dqB*)qndYRo4k8tOVsg8J3HV1_`}}bvv`%5d=}S3H0{UxJ9qCH zcig>m$4eeGw7GTMNA7?5`4=A&wcJ1!{u{4)vI+YPX26EXE zMy@cBD!~9!1XXp|l|%u^DIx$KWo=BT)st2PumCOdXzz_B805tOGCvfR#AR?tixE2i zCd2I%XFA%ff%ggyxA8`{Z|8fOPoF;Cd--}}^91=xVj}C|L+>Ts-F^JXdndqc727)R z41r^J@7?*gTemc6XxPVPfy=>~Ds#rOURyCHym$Bbrq?;&I*Z%3ino{FXuzj1*eLE8 z+#8NHos9$(=|z_t6+I9GVJ}x|4tKSJ5AOfVzx*$!&Ye*oG>1&XBH!~zkB!C*oe+Ka z=+V~kV|g97D^h8aYq+MIGXBhBTYiHsSfbkZ+x>c+45=j{q1MWm7-VT*=VC3~XZzspF~sLn7o3HS9D0tljS-9DUu2}*D%(APV$6S zNEWK%mxVA>8j-V*?vgL{CpCkPSaA4S1sI0#fJavq4Ty9)8^<=_>99j#q#b1@Bb{U@ z?-2inq=!c5n<`zCk@Fj(^)<;V<79CoE~8t;u}+i@9zV&sVp4okl86Nih*el5QdGq% zn>?a(4=@E3D@Z@5oW!97ZRXx*iB9Qdlt%NBjg~wQ<&p^xuq6{a6!7Mm;7J^6gLDu9 zs;SlFgD=22WjnOR!8T(C)Zsc}MpDLxhDWJL2qi;|ekkT93`LKj+K|%1_HbNgtQ320 zOOcO{_`0GuIpMYn-8aq@s6DF=V|;V_*W1r`vp;t4<^DUbG?;ExkoH#f^%0FlUeq~w z(3F|$`7@4tyl~X#PH0OPL@;gD{t8RxQn*KFBc}(5efEJj^Bs$uvj58SV1qoCm@?(Z zG6!y=CCc%y4kxmwvk2D6qgf<5DhwWHtv@V&0KmpQPM z#r4C7@7=rSbwkr@&eFHwZ=<$RIQoi(FR~fdug8q8G@16V?& z0^JJk_qQUDDZ&*Dg8`#h)|bWxVDFT4=&Yy!rU&NfZlJ@kf zOr}|CLV~9C9W*U?HmHI`U&-8^TRXNf@5`iykH>W<-@YfOQ9!oUH^>V>5K$f?Qz~v- z>r!{6xTJL1hfAWILAA4330Q_7Br-rU6}8pV$`JqtNoAPnr9Zjvg+I&p5LM_4y~ z%2q#xlsFiO9btMqkie?O2&Qd}+go(f6xkM$VF>dR*q!WgeyiZNfdcXfFYtKC_e_^C zWe_JF9mPvdJxj3#N|uFA5z^=AsWN3{V$iydT!mi*3C@Oj6pX=Em#p)TQY-a^HQKZ{ z7k4h2QgUibPVO>c_VMuR53lJ8fB~8r4i%FIjZ`uM@M;bPbs2-tBX?C`mizFT(`R>f zcI=Ez*ARh>9o2;n0`M6)8cwSa`)Oh0&W(9&-Lh3el%o*V%Gcq;xllcOrWuwy!#L`p zU2TzGWu_?KA29cG`O-y)>+?5=VEOI-8+@8mbNlqsqnCU8ri%D=b5{ zD_5?5`PG+pXPr8|O$9&x`16lH{h;2x`SA9qpML)O>#wOpmfOGHa%4b+{AiWs%vXQ- z+V|FsJQglPB}@15jtZ!bn3`~`3K>`I?>~2v+mAo|bpPI?3p?4(P3`8x0L+&QyyJQE z*I&J?qa>VSq{=Arr_Xj3?NRS29Y6k$|L_lAef5=Yk*(}0FA*FxaO;;_uim^e zwq;4D&z>f!w#Q2SwpXyUDINVdO@|3sw;U@ZtGsh12=yvxS1Fno4nC2^3j8uAA-)k{ z|HWj@oW%_#P+qJlW*k&;=zu+N71pn3F+69e$f^I={ABx0LFtnt9NrdR3KoGAu&lwAI3*gP|4&!0-s7$@Fhh$farIn z)pC=ZA`@SWIdscqbMzl0y(Oqif*nEK8S$zitY5V8zewK4<==u z9+oHgH8#jPeP?}Jc<7oP@nQofb?erxJHOucRt<2y*aK?0X4u=7>aoVN+pNvgMLb1c zhZ|M)OJ!aeG!D3SJ6zgd&b3jK8I|ulBjo;=Z>{U+XM<@CK6n`uI@zVy45#xNLJ`XU?4a{Ik!U{|^mUZuj%m zYgc>$Rl{q|*_7A!Km1^J(LCC{`#yK-Tc;bRx6fR^e%(n7b|k~x_j5VT1RStjtqt9T zP0ddlZv%z>+rR$$m;d8ZnKP%!nj@^=y)$+8U;p)AP8P7off~(#Ido0C zVuPUK^xDmvhmVbQ`dUvm+9_#0ACtrOMBce~&-$A=`Zw=iTPrh#NYYr(;p}wsVFm5n zy*s`l{))Tdo=sz#KQvy!1Em?t7$iAp@B8onw);Hq4|(ay%wcun3S;&%= zO^eP!>J(xc52ZqCQV1zD%I(!{iY>*JL|Pf^9d`ysX!kvl0KwAsC9a|vl0b}v3|7`b zI(-&@tcIgN*ew7zVPb)v!O-7X+v-su*Fi>7z#vG3-*w2%Ee<@ALZsN;CorGJhXL7x zq0D$HZ!$ECbTECM_F!2XQxB!dD;*huwek`rxw6r7wm;go)pUCC*gRFU{n=!0XqqX+oZx&w2I2cB#zOD`oO~Yt(d^(M&e|9<)@;LAEFy8D|HR zX_TyZZ5}?x4PL!|?b7ATy2hyrj3JwfSmUB+{4C0^5o3dxd__vAWgjK4yRkKR@GXaA z1rl~zZuK9KXS+lSUGXnw7#$2z@UjWrKr-!K7NAo-Qdv9QWSGFwdw)_JZ`Ahc&AWT| z9vnSts?1Lni_XzrS9Fi^Vk3SPe*wg zW7*AYp+?Dgg@$3Ht8~)@KHlXK*u=48#-!&iUfOYnrlFRRX^Q>88$%E)h34^|?LD{V z=3KZVhg83h)}kB-V5*Rwx+#U?(jT;U-NAWIkki;`G7Us#7Xd ziYg6GZDO58b(K8UmKvVtAex~r52QrdlfDWP1sPER)xJop#XKmw$r7UV0?}UioQS8g zW&FF?XxN+F17F^gcQ}`8JauxcsQ6uRw4_bmAswnJJD5 zC7#ljBFpeH3@1miuOlOi+W;_<)Q;Z8RY_{8{XCI0yd}iK1$GhU21&!o&x&cnGf~9W+wLfYsvq)36bGIE)U`k(NF?+&oEAgD!nM zU;rymuS1T`*K}n;$ZZ4xUd)<^uBnAf`N?eDA&Q49SFV^kicb&)ZRSmm@8Fz|K)3^UVZV!7mh?pV{^!kgS0E?sibR3MZ!aM!x6)-qsb+D zLn8MDCaL-)DnBM-?vsg zN1d%2LcBWe%4?gWm(!%BRD)nH>%TAW*V7ZX7YZ(J9bv zA0CyHp8nNqSHJuAn`h5=4UKFme9D`@e&vOKcs4f+pKn>Gxo~NRN*+9Zpe?j{CPQO#zk9d)Y&Ye1BuBKK zJiqnjS6}&5fxT=V|LgztfA-uppL&f2!m1ne%dMNwpFZ2(+&Xq5hlle8FJ3t6=grnB zA7RDV2Xnr<7r)-QegEzq4awNS>9c37Bfi*wZobacF#cz6=#<<%%(H?pDf?73%7 zr`~^I9DmyTW*bLdKHs;#r8NSUaS|F!!HWJ}NKH$DB{4NK#*jXZpX>usM#{HER2KvQ zc#7U*qGX8P3&{3MVv9pyD8gT)?qXRAcWi6T4}ew`6jK4f#F`s-n-F8_?g{{CU0j}H@Zx0)jBB%j!Bjr&Yb=-$a=Gp|}J%v7O=&q%TwgK0{mQV74IQgmMgU%B=f8jhzdX3uJ>6 zLy+Ne(0jC`!&@_hZT(1O@}{_$or#MwfdAz3CdS7@nSaR?{pmk^`4wMeLPA&1CN%qo zOwUkMTGkvtpRXYMHX0ZoY4*}LLblPG^Wd_qc2E?YO4#U0Q8RJy(W*Z>{P=O}_)&Wk zbUX9X{>S&*TibZ!wziLZ0X*M(#I3p)g(onJRK=<*m=wPepGew&10$cpP;6R#U`ciAtOuTZ~v<(M?RD#a1wFx>KTC5L9A+DzK@ zYd6lEJe`p}%}1{ag19LKapc2!^*jbYW+*YBi|s+G<}nkQ@_(^C#pqTM&F;oLNXe$8 zTcj`~YX6O`@i`AHz9KDnNc&QYw@)3f#+}7WrJEBgjE>WH^R%Z=%V^Wyd*9F&%l2ib z=$NrOp7$t?3`kBdu0|V{r@sEx=^G`qBzIDqH6c;a^zwXSNy=M=fJTa1R+5W2;-zk22B_dot%Um0<#m&i;`dq2F@WitnL7&B>;!O)Pi=Q8=* zvvZWNz3?^~%1Xx|y@=yYQ~u@M`{zdN7II9cNu(Kq%avjF`QFu8gi=HqZb4W+!0Uhx27 z2nk~05ouQ0;u{|3>3_e%%mQWNFt|KHaYzhgnV;-US$GE%jpbxjWf9g~$qiDX5+5p) zyNm_fnhlXb@7ljs%_iYH?f61h7InV_$_^w8VY(%oYnGmA+T}nqnX@Fh@ugxjhv!5cDNOEv+Ga$xhc?II#+iinb)UH+<1O=i;os@(QuD>vi~ zzz%+RcVcsE^Xe7Iyk`T|{rmU+`q%Fb-K-q<^vI7-A3wD>*g(yRoQP)JHea5;a>ZFH zr%#`uE4?o5TDlukP)mWk#Xxyznb$h7qOe?3#6%diH&p|b79p)&|HT(EndnKLSZIu( ztxct$zGW&~|B)Y&n#ZDlLxqx|n07d2Flh=BuoQm_%|w znw9{J)`hh6nntS!;kmV(^X+%v-Ft9%_t9euHY$gqpidJ0>AOFF{f9ru!YSeg*PYSN z{(bGXJN}=`V5=EvemvlyXd9q^ur9Lu^!U-uZ@&5F{Dn(@`L}=j;6)z|==-;Zw8uA( z-QCNUFkcLyEbW2!zYMc@-0<_7Ssd5k5mnK&P!3;7XtI zYiY@}=1l|V zCP>Ws1X~Vf);|i~3DF#HH~Eqg%;XKJ`p5Y2NJ{X|b_FZSObB2)<9v%fUf$SDC)Yu` zZ(i@e_9c-V&;lC`fXf%EtfX${RfQy`2!`@F zXZj@t)Xq23+&WIc%V-J@HgO5F=%rX0HgBfia6?grCpZhf+8R(KOhJm$G8- zd_aielOCe^mRDdv`Ef7i!cuAou^+Oeu6PzDrH(|qdD}dg3*h4q)=HvSbQPAXy6@Hm zknh3dJrWwTjuuVkjqg+=^GFJ<)*f?>H{RITf4}dQ%glxzdFyo=ADFWM-Xc#f^l+>Z zc5=*(YuEO6pD720;(gD{mv=t@>@zb>O2Z;QKSZOr6XZ#UH2Z(TA7%;o?CGxY!B0Q@ zn6E)>96o>k!qscnH6F+SJL0Sd=bow-rAQPgHjih|G4IdRH2ZH3yz#X>EkDlXGWe!e zIlOSIkE3gVq-B2%gU=j6uyuUHydURd0EZn#B$w((C`PmY!V-*eQn*&k%<#qD``u^H zyhCb)#|UCj3gW;=@A25bU>wFfpz=!7^Jw?_wQJ@ck9JTf`)HVwi7!62%XQ)@W@=eg zMKB)Kf)#3GV-kw3A(bUa;)7&ziGfX~=0vCsfOW-m3Fp}WD*{9pJZLC@(h!-rrxOVW zOGtEBE&6r(f~sbeCzBCE28DrD`xc4lQLH|%b$OO zl)C0r_qbuq*9EMZAz!RH2HvXDYBJ1JG4Mw}Lnz4V!KcpjHV$r{M&l{ECG=?|rFm>j zfXd@_iX84__jE1DL|j*yeiK=u44DB?JyesH#abtW1pZ`f{y{+oV|ED(tEgqS+60bb zlpjvQovgCQZio}Ds3VhxBYDW~Ip8WX#vn9@%rgXx4~qLXr;z3VuCyuv8%WvxRavDP zhO3I~#~p3xDJvJ5Jt~DEu_!9MQ2=~Benb|Q5oKiP5J=eKA{)&MhM9~&(03lUkp(lt z$1|q%^3z9ZVaU`B1m=#(#G?yBkZ&I&@71T7DcX3@v26 z?QF7WK?H3KGLt3wLnKB!lNL(Fe2pccjxabheKQ&PXE-;`9Nya8+IjU7;jTXP4P=vV z1xS0-&%2<~hB(bKe)wwy!*WgunoT|E3R7yBXRB}sej*JWE+*!#jlZL@O)#WbSjtlS z$PrT^i--rB%bW)u+t02C_!XO$Q<>T2y;6AO&D%jn3B>2B<%d9Z)Vea&;5l^Y#MX93 zWCsrUY~|%kI}Y)-9pL%?D_e~3-M#bIzn7#dTd7YR zJ7Gc3OqW5Dv63C)K23H&!xrbb%r&s|>>NL|p}ltWj~Th|zyIDwIJ|9bpV+y4N!y?) z(CcD(j>>RUPTqMj-a2yFPGwaiuf_EEEQaY@(FkBA1zfHob~L;6PZ^NC3MWpUwBiyY zvRlPEVg@UAN$jQM5Bh|UqvV{qof7iw%-LhsA)oC%)$-^cA%6STulD0zzjmG8^VZdi zmxqoxi2IcnNF6}u4OBz#1HJ-vAS-@0KEP>It|(2Zg_jm`3vWuU7O7ZLybMKI#=U|9 zBQmu@TxeG5j3yW5pDJ1Nk^++TV7^%C4Y0qY}npD}13P5qBZ6&>snh7(}!S<7H>W#vTA zuUW)003f=^leFlXSsowrpq$YEiZIKA!vfAMoGy`W|_&14r8ma)+%c_#b*j6+U<^4(Etdssi-hY z2DJgFcM?Nd#KK7q$;b`@+v{Q?J}zLdq4YQAlumv7-M2V3#kBjFe?R!-=~Hj^-nenY z9=@qLqbc11BUxPlEySyRZ|@)4I<^VM>y{;zFKlOnHD?h1)86EVG7NES3(FbP@9*8e zchkP=m#;ij%$k8aw;k8^vHLMnBwoAu_4cn3JjX6+2qL0_(bOBSl`>t|NT8ZWMX+EV`~xG7k7TfuXJsA`3VF1; zu7LwM!PD_qt*g^S!8;s$TlU8JN`NIIjz^}+QA%^ z%2c!09AWZoeW^~5qB&1)EGiV{|SSkBnEi7 z&Lim!z-*{XSW6Yf(r5HG-Qp09w1Q}~&A*Ma4a~3?^;U!o`ZAl85FIWOrUat;`SwUG z#PgahkN75Q6FFp(JXwy6sa5v}bb&M7rZN|2`c`=-Y3721AhO%Do#r893rJgdv_uH9 z#y%rm%C1TBBM`R{Izf>FoU(?leo}gkhqN;V7eQ7Hn4Z|wvlMCdU)Wf+P{sL;p4cW# z`9O=Q4&_Xpkw$+|gO(lMW-a3tf4^eL%dGY+kom+RF zJbI#|V09$lDL8Pz7cBnzU;kx=#&VHwC`d8b+4d%8<6irG{(P@*{oAF+XXS)Vh0iFK z1}EB#-dUy2sOHK(I!Sq*^SE(R_V8ee`pg1puCCd|a_y>}@R+tG{PELAlom;?BXwYV z{@i)PBQ4W6jy?GHTY4a}W!oor?|yg{AmcwfXIwLVytH#uy;BHijVf}Zh_W|d;#xP$ zQ{DqhbOuWB*zM$1tm#H3Ep|USU7^LNE3ml*G|gUe1I;BFTqiPl*O+CTyiyQ@e~n?_ z=cy|~>-anZNk7OXZ|x2MCK1FD7`M_V*W6Buxf*i{3nwcJ0WblfFg(?T+>q5Nq5;zIAcn4kl~4napM*TUR-42^GqFgI8?*~m=%7mC-Y7z28Q*^h(d_Q zylkvs{7V&pInHnfB`f0riP9h;aQ_w!1j(X`CB48^L9oVrxekN_<`0wUcjGK}jh_*A zg*?CDT(OZ7Y#^|IEscpm3`W)^l zb$L!&?&g2BB%^y7M^`YfGU6bdt5TS%mUu-tVX8>A(m_jDCpVZP;du2-Rbg229;)_1 zL^-PPd6i^uPoMqD)ypSOo$^}$)vH(T+`Z!r3024&D)ttC{rKxNCl=7M%yZa$qPJw= z%E`o(=$vZPPmXHV$1{rZDrE*9cuJj)f>%5x)n|q*9huU4zWZ|T&B0R}hd&;9{c7K^ z1`hifwI`>xPk6H@YbUSYW<|upi=oufa4WBX4WV09_E4#TWX&q68D7vhg{A4>3N0(z zYvw&4%@r*X1j4->OY`2iMk|qZxs44cXK0G_3D2EBYktogKX#O8Vyz0%Rdi=RwzsZ) zChkPOKq^u5y!YiZC(!Ng`}C7xb>>DhA7vw4+Z%%8D5a$s|C@cVz3AwZkV1)lW#zL? zMutj4s80p19ZCTyp9}jO60Q@W9{|NVPz!-oVwe#;o5$jK79SEY(#Gg`P!%w4YhXkM zqlhp_CS~IA#O4CE07x%vE>hQ7=6?NXo(?LqtbxJDNzJSvq#7QdDb9N#vFX6Ca)JzO zDj3G%m2%6;1O!*Hj?|OXw3Le=gKH5aR}*4ARLg!6NdT;_ChrQ^FZv$W&QoBsS0bsy z{4f=uVp;JRisG)Not1+$G{wwVTBN-R-Efb#Kip$37a5sEEd>+GC=!PorU0`$?j+KM zM7p#ti4U0Hq^xu{I$6}1Fo2m%E5lMM-BH zz=Q~rCaG##@fNSfJ-3vKH*8PiQ^qtId6sNT+mMMkWF=P7DW*|#?$f<*iQqgBY-q`W zLS-fjO2%}G!dYD7JW!HlrGXSMF|*PrALY?G^m71TyDa8G+DVK zgwzyl^fN3?T&8gDQF=2;V+BDUU{8`FB*jl&JVdX03B4#ba@hS<-)-BLC5RWV4xZ26 z4~z6jaih-7zJLD#Sf1!f4#a-swMWb5F!0|9^1LihL`#4nEBjN+VNNHiv~d6%5r;u} zC8tqwN`xVWbI;EE*0bYZ?Q%S!Y4W<0LqAp9v{)uaeLtZqPeu;fF|HmaXW7^R=?$AK z_Go{yju)+^wwWu-k~Q|6S;T2?I7%EqPrl1MFM%~YIbhVMso&6vbd-sbO>~x#>`Hk2__3v(liQfLHG12G6bzOak7inK`GOwgOChxQ z-NIH4q+mQtDJ~u~DMH{0A}l6Q6uAp@YF7N=pbc!bn^LDX69DGT1ecTK^RB!i`5D-KUgBb)n*KDXB^wQ|-Vd z^(G;=c`+yhf`np#gc5_ABMXfC2BaB}YA6&zkk8n_Bl^tSW(Rfj%W6tbW-!4P<+SXI z48o``6c;{PkCGe~rS;J~&lRB-XqG;LD9@AE6@25lM4sr^1$j>cqDz|2M^q8`DJ*l3 z=@!QL_~NeL6D4hp`Vc8?+!bL6dEgH9r_ zE%5oXXRlwq1ZoX8lZ z#{m{5f;3KI9{H_HZFD$sosew*Ehqzc7QYf3iwM*Co#v-`gL59u6%0~^#lymgN6@>< z?rE&E$EAd+2BUxlbnbgyF;M$1d$PweYS7(><3`1d6+EP5T)HdYHs{fWu^ie zLa=`zve$4hxW@bRQAJDHNt|ndRsj+G5QsQ^`Wpx?CKDQDl!L~Af^a*L5p<9(IXm@% zCxt$cuIUx#1GEj{ZK=$|QDlJJ5rSbbehRO7Q?g?Ty{*0rt0Ibq6kEW6Fu*B=x-%(I zQ5JN{F1U)Y?B&kwDj_%nG9v7`NLCWR55s%WoplUg;1G0#XlSRO=!nJ3`U{2Sr+NDixtd6=x zJ#;1)jRt$@$52@ZYvVJq9LtsxO_tF>1u%NUddsQekBcC60&G)E^4Kqs;>-MV`u(>2 z>9TXOj`cWj=ETWvNXzGBF6`t)0JTUja{I*A=U;s37-nZv`ksy9&EB4k$NL8^&0x~?jY4QYZGA)bFnxS!(D2n4U)pzU z>mg;QEy~dVo4zk}^VV&1Yv<0N`{JvwzW(|TS|)4umo8k`wShXH8E}@`!)LoswLz3) zXR}gr!i@;`iBlKOp1*kZ!q(>TOtxxTvOVd|-CH;R$Hw9P-95$gNOoWE z`S8|ZH%=GGdxfi(X39W8rL2@-N-gE4;OZ<>Nfc#AyaWeFDcD?Qz zQ)m`u5&QO3MubS}8YJ1XG+-EsvtvxTZy0#diMYx&w{w*uqJ`$gsKqTA1rY%*Pvw}n z+c=vv5!(w`I zdi1bhVIbsX1YW~fOXKrtJawAweDF6GDvZFj9%dDl#Y&=@Fd7MxCFJrGTcnGOG%8XW zp_F?}AAsaxAdWCioejuxRoH2c^2vLcGjF)JpCk5(VqHBr%;?$vYt&^;yh{_q9cU;> z8}b4sBVp*s?h34|MOP`YB_#l|xFZsd>4<1L@A+H8utSVLV#9y__Pfg$cAPY0F~%z|J!C*hhb0Om4ok^ISoSXI z?_=1DE#9UClBqMzQI|$i8v@*8_AXtu%lgXWCy%^#d%@1)GiPvWj_i+L|7m;sv~OKJ zxOex(^ZmEJE_8a+AbEJ|zc{I#9 z+1+hb|Jvo9-6viMz4O9HCVZ7(ch`$oP8@KAp61KJ0oQI^Kke((n@4V3y?XEFtp^Y9 ztCtoxZvXt#!`rugca(MSzu2=|@95U?3(hp!+3`A-sUMrCHx?-~oMMseF2|#fx$9PW zvw_}YTPi?SP|Xa;7<-C=FBYDhfSGJQIgMK8tWuRSVpujZhzrRW-HWjAU=S2}Fm>a@ z05GiJX%LBdDfBcOv6EKPOj@yJZFBv}suz~Zm0Dz$6Ie`x9Vl7{B^oO`JmQEi)aArO z9Hg!5+^nflh)9W&eUdS7r|Ja_p|w3rO&I<$wdPH;Q+nDBc@0LA(Z+FO9WO`Mw3MgS(KZw9%dy% z$t51jZOl}1at|ZHfMM*30LGyZ=ELUV4knT-OkGb6JL+slPGKq2JMn<+d}?C}91&yM z6qVQMa)u%rjK_H*@lt{JkHe3dN7FTctYmhSqZbvJFnZ7{{82}`&K0;@sxnzPd%j~) zwI|L7H~fAIC)G9v1`Z(vqJHTq$8-ZgMNH)7#|(|KC67ar+M)rqAXW-1?=R9+{EM0#|{BDyg8Q zNj+e&n@7)|J%S#iksiULK{*d`vED|B=^s3B?(`YBQhh!;&R@0-O&1={`0BvPGd54# zk!)4QVmmfkwL@CTVFTR^@cHP}oj4fw4+fnA_)uA-lYu6z&tANA;h+EcpADx@o;hX2 zf-9107@PsBCAe_z!u?A-`8wW_!=9eMa8YaY^vSMO69-ot35`WY$NG$MRpi<@fA;j- zH#^<{J$Y*T!uj(i{xmke`D3x_+SMyG=By=jJPtylEbs-WCL5csBbLddRAg4fe%%PEE(zFnkqQIZ^Z}L_U7*}7q6y_UVj&a_;VVgDuqmMvUPCBM zr!Ud08Tzjwl)Tm;LSgtD~RKpM7)a~AMW22W1KuS zrCBP;VIUOSudk?KMZiHQ)l#7INH$(a0zX;3$(2<9{T5Zrq@YE$zF9;_*7IAQ3A-j$Gczd)@_+^5pO;|v@}DZV^!2jG7RXFd6Ql|1O0tf_Cf|47L(((Wrh^xt!9s>}zn=Ua0oGsm04|YbD zcDT0I-rjTov>$m8Fcl-XU9>+8l#!%Zd_Y{a8NL&V6N-caZ*G@Cc9WY6>vxx@f_bzU zUyQItt=gfTHY?jq0uUJ>OL^m4b0)5l`SZzB8fTB>&@(l1%PPxJ2RS(2^^nsY3?_(y zuH||u-s04k3>zC8PWcRnH6_Jr9>t@bt5;8-Gu3t~hXkbY$fp;GD6f?&X_t2{@0vY) z{_NF@m&TUxXq*hNjIQzqFGL>>o<4E%AHMyQC7;tLf3~SXi%lN$bY>T^v~zLitRdd{ zbDG9<`R$*^qClk#t7T9tjDjgPo|sQpO;%Roo*slKCH%G6TpQ9>XXM*;-e@LeMqxSu z15!_svXisxRP2a0GD`LO5H`ujPlS;4?Xxj3vbG(Vgz_9tYl@9XJL!Y zVyD@|MiHb82%I_~8#xN@lE;<7RXohViN|IOY_X#6X^^=_os#oFD;b&?mDCbUIgcAm z*+ZTzVI~m|gDCpSy2#i{KapKNSXdamWUti-m_q_Y1@w#r%a-uK$Ozf7_PbIMOyvNfb#b z&w7kHufG5L`!l(f`guP?U`RgU z-sJt~bLT|iu8Mg|YB#bi1FE@I?WQ7DmO08;Fz{f=z2^YZ=_LBO2ydf$<^XeO>Vggg za@@eMRFG()(tO{uFIG@u%sCIH?TZM0;0g~Ta6*Lp>8Jni|NVbkg#65p=MFg6_d|Qe z+NGecW^g~UpFCKcK#UUPa2G4tUbP9Ok&x&hujmP%b2L!Ch zG2(^_TbWbNJ|VkpL~=@5E$rj^$U3(0HRI3NoHq1?TJR5nG#ySLQN*)idGYFYOhi+; zy~v)Qj|y{cgxqZShyc`_Uc)?uPv8KlA?_gH*Fdh_X!N_=cECF|q;ovM5A!{3hrlHz zc(5VV!+%7ty?5r=JUmRD6GYXM2qG3W`}v^P$enblzcqg88L9v{<79WE|&s~dYX^f>dqqm87n&cxNu#( z$T-}N;1#gP*ojC?=ocg)+p!Pj`#MF3Hl3i)S(g&Ds(L+>;;SY}{Iv7aRx9Fy&iJu< z##(dO*@8>JCS$!;o|>(6x5C8BRgC+ZK6kjsJPeuY^uMqTcLEr?P!^_Le3)*8R$sB4 z>mT2Dh6HIP+EH)bE{E3_N*!5LmR?xj+6cYCSkgZ2F`DXa|8qMCoTp_|7UJ!{y$q(? zY%cPd*q^*)kpt3p27uhh)3`*s&K)7k$ktLfXT9xUSKFyXuZ`!fl|8e_>J9S8p2$8bCeW?#@=(klzBK>FFbR)3!0 zYUNt@`yc+XuQ_*`U6vU@>}W3`Ihr6;9%y#&?^i#8ULL30Ty@dvH<75;g7<&@U%tT2 z^&Go?`|bZj2blCBoSvGVGiHC?c@Px)j67*yGtWQA|5#pyHENGUhqU(<(eZd?b` z*|mr>FRr!HW4Ty=vS^|c)!OxM^zFxO4V+^k!UsGnZphK7W=Q{Voa#W+hOpe8S&!4H z(5|cvPDPgdTZ1AI^z$yat(KCp|8o_8EV4d$oG?gm`^Jv3^33FPD~&VrK`nqW=o8;;25cbFjusd_923O&m-(5!xvSU)F`nY;Me$n$ny39s|fgCVd&Cxdk zuE1VLQ9k}5R~jj;pa;Z?zn)SYc7z?rp+56S&g{$pUf)UCNb`iti+Sqg94eal_bx-> zRQInKvax32%WOK}7>g~FI#jzH`#5FtvC%22l2|Kf1#SZdYal)>K`k{t6bj)w#< zFO&~ydBsn4Rs?_bxN+PiB@r!qkrX>z?fc|*meUfXw|um_xXGX{HXU(Jm31bkdaYgG zkam4pNW@|iuf1v3XslK}p_v|M(J3DvZ_hU?+#Sj%E|wJ8JsZU*R)wBj21#QRxA;?K z@1Sn+I_YS!vWl#T_mLD9Xkf*spQ`xY2C)>KEvG+(RDv-}6A-)n5s~i4a9~W%Ovfq1 zf&z3Fy+cKvs($vci~U+0Q@+%ImI$CR*=wU^`=p;+4k!3zF0f&s208}tIrdn6!B5cH zb!+M=1``lkD*+F(&mM=81(s%)73l^((DH9Rz!Q3`PL`v=6LkHT3fk#?m3iw=|qsq=K8;I`Aoef=(I06LdQ0>zvSoAJvf@-0?*_WJi!| zh)I@li$dN*Yv?#gCk&4RNQ7g0eDu>V63T;VH?2DPk`*mE5|k&@L`V|dk(DOk4(LZv>>o+`vZ%23}rviLvQQRnTCmiSv?ng-4 z9_oYiQ4VlZ;d%Xld*#n6!@^I5*O1WbbzAOb`q9AY8}UP2hblX7GD=$SsoTW`)yh!?^1&}Pdb z3jFa+;hjBW{>+mYrC4NMs=u2CwCJj5t%=d)LP^{W!1XiWvFO>kQ1IcLoJVzN*B_IA z|Ja8;TWL?wIkDZ=G^V#dr%eWq__up{y=P-3af+}bWMmoR(GeG@&$B0Cv zb{$Rs!bFWbo#xLL;rt1Qb&lyeg`AeBBQb5p$=o6=I}az$E8_3zp@Cm>dK}&(_#q#} z=`@e-9lFjT9K#z0C)?CBM3F8Ynjp32n)TrhZO(r-Uf_p^eJSV$eQ}&#HMitZ=93KHm>dUkOUo~UVlh5mFMeD?1&WLWSm&I^MV*d= z{6Xq@zmjvB7?aiC17hzjwrpw6h<4+hCbwWJkqs71soCvv7!vC2ePcQW99B<#62k4| zw0V394ZarbXu2}%={0gTZ4IPvEiPL{zQS(kPsf(Wr$%?TH=D->D|TeXxZN>sgk{%U zQ=GvvhIytNF!LQ-oj+((FYg`I2tHcm#fXgF-{bm$?Po!m%oaX;beK0q-}kaL;k7A~ z-@EQ;m#jIItuw7-uE(tAbZ@|S8lOa;8P_QvK=30RiyaSPV6aWe*mg;lJOdl6GXu}q zMLmxuJUc^9W2gH|BBz&eTkv`(^HS|-K2eYRMYJOJy;(7lsQ#lJXQ_d!WJP5ufYoYD z2~VCGqDGEKp0B*0py;zlLqb9L26zWOp#HUY2(SXZc2ff6s8H%SWr?9_(B)1OX;az= zZift5?Mp6k@jNpv5)hU~G1@y##Qko$LhAh-pC7W2wzkfi;Vc_WuyoIMKnsb61?`9; zj6p^__er21zxNvm_KAE7c$cdeE|b&*o02>l{=+xkR9*6QgH`j<}vW(JjX=LtS2bu2t#NxR^Ot3ejD~fJPuLm&;j>#Dp+%0 z?nkvIIrR@Wk~f7fCxlk=2V3ujg-N}yHzq?FPW z*y!UxLj1LDS+HY;!_93SqaTf*p;?!Z)aW@&xqY=zgZ$pm=MIZbY z9(?>ZIq}QVGEb${EH)3V6y5@iGst<#D}xf61$FRrhUX?Nw4P@2;ug8b65LQ^eAp(o zvT_M)eUdEm=iK_JmnXH|+#tNgrs-K#Jtv&sA}K2`n$&na35c3mIw|*QgrmQYQY@Q8 zD=Y|4i*;rDCDjA#>^wPQh4ZdV5Z-`7s4Xhsnlplqr{e)nN(wEj4}OYr6BU|$AF9)62FGWeZ~n(4 zABah;w}dadcqhBiT-xMw~vmHqz z+;#WdW}w^kAP`bf`|O#@P_@X<1$7a1?Z{sZQO#)!mN3q-h=6Bs<)on9+#x}VVx)7P zJf_di3G_2V!68f|FfM8hq&Yp5yL_Iq;e*=ym#iOE_@#>mG`L(imnYHZ&#vUEhJBUU zypxy(^>{jsl~w%N@p&yCzwMKWcx1!8e^*7-U%$(#UVF4`efSi>n1Al7Zp8?>KWsg*2B#CJ;_TlnBd|v=E`jFE;AP9yY*Bd{ z0EyRt7R)S3zlGn>$R;B>1v+s}!pG_o?8%`AH#s%8Yq-?e&F6s9p+s4`7e411j^z$I z7&-Dnu;8Nk#cpmX>*o5+lS6hPLe&H_Wq2uv%ILh6^4&qws}fhiI3q7ab)MW%+%Bnd7|$X77?n7UM*UmL;S5iTAFEcHWThvF)&y>T&WD>I z#W-F1pe%&LE)6!nIyL*2DuoCK4EU{wjpyGg{F!({h|X!bw1$R#UEJohIcLw2OvRP^ za{|t^r3V^?!D;?k2_BympTKVOV1)|bGd;GhZ_u2X2=vzkO$?4hM`S8C=y_M}FA6;} z4_}oucI3HaRTB4&WuE_cKt-WU%ubi*Emn`ncTcQ|C;`-nZ@m``| z_T&bRxFu={gCMp`69ywMe$88v22So|UD6~%oW>@7b8<~*k3Y5eJ9~l!irR5Fanp!O z%86Nqn`Ht3j@ot1y~C!^+2LeJ%*JDi(f8aq=0Ir-!D~LB3r~vUhQlBkf5#Z=(~(1F z?7&}?%$S%-Y8*0fms`Yv53=rbGBtuWXLEQl!+fA;R!xH|YA-F+aAJ<*X4EH$={ncw z7`=XzXZ_*_Kxa|Nt;c7hbB>SbsmCAJ@9-< z&m+H30lm7t9SqN$Q%`^VPs&%ZIr(s|qkmKVMY_?Io$~EW#Z5aAr~E-qF-K-lmF`#8 zjUJkzul>UZiKO=I2Zsie+M(QpJsij@3tl(21XF5)_pa?eSGakEouU(7(w`V{>b727 z@wf<%oYA1QzevNs{=Vtjy~mrBY-vgQ zgTkc)M7m1eQWh-%b5s4hle9UhrkI9%5@WJj?a~Kh9C8^gs>}Jy5Ucp13bti~A0E4x z-e;en*l63mMz|7j&UCB*%?}vG(7&la^KB@44}xCWLFi3dm%Zt+`)TJ~32ICdOhw2o zbZ|0u|HZg?Q}eTyQY)s0SK|sN`HPezAO@+Wo7vNJ^)fBNKSi$DJfj^on6X-8VUHSb z(af{PV)pp-kW+7eR{~UpgqF$3gmW$*TO}VkFe89vYVnRbac>fz=!f}a4Goc8nV-4u zslwTDy5ID`YC5ur*O}?RyDrePQQ@*z^!j)3;q`H(%~&j<91ti+pTmX*2d`7(R(D`P z?K&s-Jo$R>5ORyRzJ{-VnO9XUG>)g3ukwLTehiQ2k07r#c6LeCiF5XdH^j)ceSK$ekYK9n}GOX_0Gq9$MH-c0ipw4%XqA0KVkMZJu># z7vQBdA2|pf^g0fZw=t-E^ec?Kx=OjftL>U_Lsps@-F*DNtJPeCi|9Av|grc&r$! z^}*VCt1(>jP{$zu&jT^M^t$0Yw!mo3$y}Wtv&Cu89L)+i>G-i|ijEsKKRJjLGGW*p zKD}eCb#eJjxA`qAPD(0mEuM$NMKcvT#B*uZ#;C7r9Fvdt@?oY`eZTq=mHQ!z;Ei>C z^U*v3*GG(I$I^hP#?trs>onw?LWz)yiL+rgme2m9Z8VO!Z6~&ikyeGx1A=fnCMO*S zr;h@Y_#Zq2T;9J!$L$uU7pNl!?R*MD^W@XMDcT54Hj7?UpwXcn-0#+j-3@IlThi^h zdzLF+oj;tPOe^fWXnWyr2bF$FxrRf{Sx)@}&sZ!xy{z*xInpP1PKeV<-H7OJDlRj+gpR`uvIWpOiV#H+zr68Ggr`$$#PyT8cdy zIfHgHU;j`146S6LznB+5$5* zA6-#h<$RUQH2>EOS!{d)LE&T3eM?@v<6LR-N{ZT$uW!!kd>Tnl0zb+5#&mkW8h`Nr z^L6gYLh3vke~?-i*Sbli(}xOe7H_u*X@2=2|M8Xb|I6rqg5b?ghySfTF51$O`z|J_ z`dLlBQ8hk$cQ<427IisZbhL2_S$JNW*I6JCE(GuB+hO{)4Ig(boJO&|TM=kOE-mt* zCMr|racmrBDHdHL%{x6W5b78Wf=dv2a2$^pD0;K4Eyk|*wqAspsl~tTzXlMu?F1|Y z`V+cWU^=iEN^3|igpSaQeJ^rS67*SkbWmbXp%|T!-xS^wvUlB$!a^%cBEL(SVH!%B z%d(cnZq`|+-9Z%lwIf!JG!l}X`pT90?Y0WjFlVnp$mpG|ALmskl8MO& z~QPOe#lPyfZ>7^~ z&T9%JN7Ikb48PNX-{_QYH~MzYGY356JN_r+;PcSO8+mEqJiV}`!nauDnnFtM;u_j8 z=JruOHeUaZXN5iknXLUuXG0+1F!k>kjSg4fdIGk*aXa9BLEk#AVnbEM3fe@Vrq+5BngJg>5X z#?Mfk^YdK7{|o=FAHo_>VOa+v5mR)lX66?t)S1x^c(o4yG+%wJ`mH@$_&(-;^?jau zinq5gxGbCv$9*zz!WEt4GZkr`L;IJ=MXOtj`vk|z$>Z%QlTOfCw)@;A9?t`6eK=8+ zjE(Ff1x}n}ic}B2FPLGThSkP!}A2Hi| zG#*nnHeS1FQRGlfMO;va*}pt@PH!;AxMsZG9TRjT*MYaU`XPv4sGoKu&Uk9Ro09nK z)|geIn^$9!1{?~NVTl9ahv&MIIH!Y$F;-~-vDJTD$np`tur4SB#WfxM=qjZ>LG{*@M}=V4e%_T{?EOs_(HyVF?)0O8Q$wG zxgtgS+L;##k)p!KgU=87X(lXcn@bWq38ar(qgmA0kiAbcPgXhuOC-&6g!9q^C=U;IbRvgmMb=q3Dr>5xV@`fW)qQJTfghLVX>`cb_(@#Fd=K)A1{-m+Ik? zE>m(x9)s_QOw;4){{rP!fCd^31BMlco@Owg^&B-Y@`c<9n{#-4^wo86Pqvb~^>A?Y{u z>L8wvP)-NYE<#S+eLU}*Z}57`d*p=XrNuG3fXC(fb5PmBoGSE`GqetUa*qDm?)$WG z`OBOZDn_hMI;Dtq^K|?kvY_cyhS1&+p3n^Ir&vY1n{O@@adUlB;Ndb!fBs1-$PuU0 zlmMtMmWPo9zXfSJ&AU)s51Xk`829D`u}U;R8X%22m$byul*FvcOSeaEaJpv6YLLDD zt}D`CBiGT;&N}3w0qIc3*k%YDlC)9z$P4;MMz>LXo3V%@Uc+aPESH)Kcb7X^e$$mT z=<`5EV40tZ9Pc>iSsz>4dG8X@QE(wu$KSwHO+)D47;U28^g@p&9oOsZX-_WBsOV?L zc?=_}71ht^e0f&TtBy%YT^&l5an8;fv0pS_$HuzYB*0?d=CoRNiI*DmRiAjmSymhqr*B99+L)DZ*B+`ST@8-^ zPj3WdiVSrPu&|AQ@oZ(|v8;cjH;?q|oPUNg-Z9tS{+V;9FVNa*XMak5ic%!cl2aP(E~P;Re4|_7~y2I!7d^2(fn_+K`;JZ!mHhgoEEH zh}~I~4oiahVH$K7Oj%?^VKqL_bqKginn?%H3Ot_aNP5q7^(18SmP#L2=g&(9$`HNk z5fG*$?mEGjW#fM#Xk0>9+7AHx8}b99>v+zSAAjOF+@)eUUIS}pE$G+`0XUYQ(_p8- zC-I#XnH7q(ISdgDI4P{}8q~>nk=AlOClFuF9$IWfr^w#~USUaor4^3q(TE=RaUe#O z)?%b97?eL&&}L3*`_F=%MyfY8>iUn5VO%C5Cm(IcjDeCxy-fXsBzSH7(xNa73Jrd5 zBQMf?DlIst!DLS?-lK81=Xq**d!W9>?-eG`l9}l)K6;MVu`^E<-aCF(qspMaz@KMV z3Rb(w#8F)EGll{?+-)yrpVylELu-Z78JbR#J;{Y+tXf#0kL@g!iS*yd`8}p|0uRGKJ~?O{j)y_m+Ck z^={W!V&1^FyB{CabS}jfYadWR8l+sxCb=wJGbawoLAhLLOlZfrY!h92x^$TecktzR zRfh;;Si-7zNozkFNq0nN_`muzTTfAES`>Hl3?FFQM)WY7AO6ty+cJHk^k2VyXG(pq z#)@zcaLNH3=~TvS&RIdh(t7Rf+^K7?qk)C@|=BG~ec9GDxBFCljU#>P&Pisr`UAEyS=p{|$ zPO2f@Dsa9yqVhdZejoX_shc7Nca0zkoa@&FlciH_*_^74;!Fn=>)_Cv4zEuht4abP z5PAtIa!EPD!xXZ=|KN*J9`vkw+>?AtZKH$2*`-9TfbWHT-vb{l*O~G3q&PWq7Zm}F zRvg%UYjL#GJ1y3UejRvvQkrI!W~;Y|FU^HupApFrBt~I7ztoHf3!8IGA}f^HM$m?(Jkmn0|rRu5bA3~!%sI% zXcUULPG)bY&`@Pl$!O&uok+M~oUFY`F4a1_^zcS??vj5Taw4`}Jze1Bt=qt+l=(ik zfBm85(>*BS6W1n?LaR!+WI$6oh0|3kcnU6P@!6_DH)Ob(BH7k0vl-jFo4Mc|c@ft? z|A>{4Wh!Ogrxxe?!@LTp-~BQ2+!<1ie^r}iv~?8+qiR9o${&ZnSWKFWb@|QEWePDl zmGqg^DD>Gyga-}Xv?z?wh+ zY~M%s(=3mfGeX?N?U38=zn4jCptum4>7LA{pRLnfk?fj$naJ&}9jyxSV;}E-58v`O z?frL<@GvGQK$W^*|{^gc!1)SSl6SL?3459Yph(XxGmx~t)g zNpFhVWo)YCtHt1eb8y%p#e?y!Siz^Eera?Qfu^ylf(L~=U%^4_V~H_3cLk~u5A94% z+l^*0=7>5q^V&H=cFMtcw~|kBSdYvpGwRi$an3_hV?&`$rO#ff8W#8coh`X=q7km! zVe0t`?O(WZO&w z(4rklQl_m9_9;;HLKIC|Jt3nxhNbwWm2s{<_I0jdHgj{+>6!PMAVjCiR*sUZn+d3a z4GJr%`|=+h`}l3HQbR^9Wr`OOh!SdP@ZB>RoBjafR0>V|iZAUd3#TL6+^ncHF?>Hw zgc5mlP)sE;rcl#=Z{GA_R+gyPfbulonm)g}N6`0Eb^9$%ECyzc(u4^H@F>l8^Xr#t zVPCYd@7j1nx(hau{nuZ9`Sm9s5AK&ceEmnCOt8T$MfWIl>l`Km;eu(DYMH^~%;^fFe;S5#dgg1K!im$}t>F~-=S?hbGqzg7X*l>ivk$2w z*)%s3`+%g=0m*pb8zjPOnI34LXeB_jHn#T z?^*Zga#V1xgVpt0+j4KufV5@WcPA3!_k4?n^>1I3PfA;r?QY|s)0(JriCnVvOvN4s z%u%^Q%9d2&rAKNNgSh59L-indq0k|Gj>}B*i`~}2ViJc$o`cUsWX~czJ3GW&(PTq{ zns~np(O$31S-Wx3PbDZR%@G2(Z*O*Vnv%2PNF1*vccz8&X@iwo{x>0J)Ez@f?Wbk-T9C7M zd_Us@f_QDd6|)@njX^$n+p(J1C;Gd_je262&M$Tya#ByDZRM9fiHJm=mRyNzjmv&6 zF$0Hx+NLLDI*k`HiF2WDNIRwmHVYESp=oEMUQ@~ zJ-!9~1mw8u>rer5DMQgljBVMK9NqIW*@oX$#=@BqVP&=^FH|vuTZ5s3RhsrLRz|fD zb}t^)u0^6v1{KCRV&%18KAwts+5F0hM>2K0BZX|4wI^!N{&axgq;QQY-{i~$X^nTD z9M_l6Q%k#Q;nI2aZ|-mo|I}zn)-cRt`RsAigx#SJRI%mE31@3tWc#d@BjDfpl`UO0 zlAf9=I)yH0pOXc%eAA$GTmT@GjaM=`BXP-u5CGcR*M7E->*5is8+|&g{E)>;fhvvb zx__AmIaxyO-)(qAaWd0fgM9dTbpHSeDOziJTPpD2Fkko-NS_V3+g4`gpR9-ypMVoF zU+6Y4Kz<16kFD_sXDr5q*t7k}kK)5>3|ChU4!NN*x&G0$JX)MLG+Pps1B}w zLpsLz^AZ3sycyFZ978TT;FA+)>IdzH&W)GLHnXIl)y^TQUO`%R;U*|e`Th}D4O%&P z!Zyz|@WpBRXT`xEp>5L&mBUH=YTc2C@NxKc2I%lIte=Vt^0EGMe0hSMl;iO7A6&6` zG(5OMn$4dExJF9Z+pM2+Ff?%vi})TNp4CBydD4;OtvRTXYLlSoMZWNaFAA9OeMN2z zbmW;$r>p7aPoFp_2XIjS+o`R;SFDN+Iy@n42t=Ky%C)rHi6Xdf8-mtd<+h_M3I%yc zsdzS$ctSJk8lFm%>z@g_QC?`nxSbzi*>zd;EH|qJP^tqt_M_0^*yg%L74S5yHgI#U z^}hv?W{I%A7J)p_-5QTfwc@laoJ5AjBwxl&cgJKR*=g@GH;`-d^On+^&Y5>ZvtD`a zF353)f1`-R3I8?&d*ADm$n(KMP`pa*D){fe_Et(?^5Y#)?<{n2uxiJ`c=v!HDZU@9 z+ISyq+IKw&RjHuSo*`vRYEO+ws7k~3{eAFOH4M0L?GRBG#UZjhu_Zi0%~0ac%e}TR z9`Z5KKb$dd`FA>xd_w3PBdXoHG9|~suA=KP^d!rdXI~cb8p2w?1buNTV*muxc-jpU zBvo*k8Gz0l!M~y*xKpDtIPdb*-0b%~4b7E^O7d8fx+k0KYg0kaEiA3?k!;aI(ztX$ z&yd4p=A9m+7XgqCcls6hwV#H6WcZSN zB?o*@;cG_-{AuSKH(KioVmYz%%EL2gmKje;K}ne{w|#iOW7X z>oDg@$U)NOMjgdk(*ZO|Sv1Zi6^J37-B@t1sBK`F2(9ok%mBf2aRrpj3l$Aah1&Y7clj7yW+n5eT^6(rsGVO!?=od2vttRB+*4?8_YDM_VOW3>bdkx1X#-vas1Zae)(`V5ybZO zD>2wj-%@Vbac(#{iG{&!r#U7~goK6d&4O`jk#ow*%Kp7?vAcg30_^8uS3bVV@jGvb z_;kz@{{3{U6LhJvz88=ft))cWorblGi@aPr z%)10(uhC7_D375dH2lfVnlb7)iO1oz*T2)N2*V{co=Lmi+@og;o@B((s=*7r+xcm& zG<@jDtiS%nwX((+DZa+JT_4&P{Q9r=9T6iZ(UHkcPamh$J3@W!`iB7X(45}TP=>SM z;nLJP$KKedk1*eU+Hn8`NCBNMJWt9UPrXyX@ofn8@Av{=$T|7ouV@*v;Ple;Uc1CO zn_(V?u}cVOS7^f>@m`-yAO(p#8JpT|s98fPFFTwCt#jRBu&xzcIxoyq{=q0@e_l3@ z`E<<8Yial)1i8~YzL?ZFS_p^i_&}Nos4AoEF;48+di>AZ#~I0y1D>8@2sn6ZT+7x~ zl^a5vAKK9FSAnJObcXz4Qqc8F+fU1n4_XB?%^rp z*WCTsBQ})hJNo+r$)|c5-5rYF4>-4mIb1rnAbRy{sPkFX!3tK|u8F~5mlRG&ovb2}Ob?$h@1@h+A8$>FZxA_d*QMxfJ8U8Zvk%oy@mX3dJ{A44f z+D_$f{yfM7u3)|FK5QFX$;=PC!}mwJZ_KN<{0hfA>OxnXkL>`h6@Vht9 ztJYo0rxZOx#jk6+od59z!1@=lt^M5q*=@Qg+P3#F9X+o)>oe-wgid7YcXkTZ&l#QG z5oCRJTpiw;&EqYD2hUL&HYvET-=bj}xBUJ7_wV|du0ORSX(H@1QaVIDY*acMlFwH1 zB|HSjYiE>E4y2X4;-x{KkMAzT3Ufzq0jkfp;2{j6SyiXJ<%WZ~lz%irDQqFboU2aI zb3Y9g=Q)=B@u;1a@H%m!j#afhzg#OtypQ+yt~N3+oZ?vf_#jLi_ubkTivKh|($CIL zA2p(JiSSOXrFSh7<2}Z1F)a`o(%R0X^h2R}U$2{G0DogIpvSzI_evE^3V&}iU_I(Qf41A*JVoJ098%2J7@XMo+^Vb<6`F4?i7DJ|RFB3A}3szM9zCvObApjtkBUZ}}s$_{u0C4&%|5 zTJECQ`XxeL+FW#TeE8h%;gR>0>C@RH$LKRsVx0F#0XzfWQA4Oc)A$A-kEPDqkzfBr zEZdk8n}Z+2>*I1A$7B8~aESqa%ADLOSPC6nD$Rdg6P8w$K10jW@jWdQJ*Vls-i99M zy@&DYhG>1NMrzLUIO>?jqZZDoz}Hoar*XF#OV8Eb$oz-SWnGh+M1Oa)?VJ!ZK=dMtGJp%&uvRY&+jzlU+c9v|da{0kffd1RRBNUPJFw)*+@xXdAx8jpfb z^cc#}JfMBp=fcmAl@IY>|N0&dj*!}SRuH*^psR9|P+agZwpo$;`kO|tRxt5!R7p?I zOZ7>&6cyG(=)*)6*)9#Q&SvOSn$+DYXjDXOy|ET>{~2}bLS$r#Rrj(gxzreQ4|*^I zaxEqRYe1C0A?A&8^3!qs(PIA;;?S&1}fak$hOW6}w-t8wAllXCB(D^orH;-2+NzrTys%1Uu)5WIo zMKCY?>Z<@Dxm-5dmWSSq@GH)KS5Qg9rT6Q68brX#z9rX*P~b8-?@Zxk&D$StK1;VD z=zl#u7v1RpdOQRqI&V8nrw`Bd=9NY!A&I14LL7Nx8-;#2*Ky=zNA1Odpnnj2u47z> zJ?I?|86}YhbOe~Qo$8buus`q@`3Nt|?mF6>bkkDW(d&*jN@^`mo735j`I8z=mRf4e zkos#Ei$AFi$KZLu5i*1`|BlDx)Vm?aV|Y|w@gdht?#zGKytF6iD?V{Y5z*61x!GF< zG(Petl;!pKnk=SfwLtRin)~RD9xF#ZED1=;!ZC5aUn5B&Jgvc^w43|jyy@Qpb%9*r zXeyFL-fsw&<1MRnwX}!_GW_n|y=QH^xBks28Isf~bvFr8;7GjSjMfb*TiDrHP|_m~ zPjpPLrBq~C!FME`5`E4A*`{+++7U+UKYwARbCLDbaf#v+b-#Ipr@V!~R+r4)*663- z^{Zp#_?t5E``hpPA0Z@>4nca9pI^Jg-SLAudtMzqC3 z>0k1M(#^#0i*coY1r9R3{R?-k%q%^m)Bep(kWa4TEk$tqQ0w+;@U$DLH%99+O8X8j^5AI0d6~QE9@da{JhnZp*@ufxm zVW!pkc!u$%LBJ3o%po7EBYb5+(8mY-+EIO@9f!leBa#>Y$nO~W)f9%n8M^`!JSoyj zm&WYxNbb$aIRSzqB~5kCE=Ju-V@k5-JVYJkfO8}P*TuFsC;LFb1ZH+#_6wi5xd{!?9A_38Kj=yaQn_oE0;{!qmn37_Ct$Y~8 z{E!`2f6{g?^qLDHIE9I{Ql>XoSL`Mz8)qiN&MkA@_Vu6WpCEi7PhSV36LpDcA`zyK zdC$yV*r>!Z)j6S;N{U78Dh2vxY)=^f+tW`B6cLz#U07U-wga)6$Af9eMnvx+#|8T$QVX+*1R zao+de8~X2Gf6eI@Ca44#SX`OX9C6~9;mao&#HCw9p&H*$CC0DZ{q+|qg5K)Hg>%Ke z&dVCW3mEyLOsxLp{#(jh-rk6?F8OU#3)bY8Av#Wtl7&=fO9(~yf$OM~;u+yC(C#Vy z=2K|R{T?>kO{4PXoCD~$AHUxZ+oQtM+v|i-mNMUe@8=nR{Oy-tyFqp>6F-@}9n5`N zQ}rvO`uJmO7!Cf2ZLu>bzmXfFI0J^S@|e3n-|<-57_gv>V+JjH@<{56wu*VB^7s5PML zyhp6`X~Toi!X$Nb%amxJx1}YV0b{z&%5ez)({9N@bHI3g8AabRmM&G~Yx>D;!CBr- zJwq&LowtO^n^wDY>yrPObh2bqHliavet#%8KqrfW*NMre%n@<$B~#?*w!?9HW=*fh z^n*Cws3YYUe%;zba-ADpD8MoQ${F&h^zuBmW7FYAb(Yzk9^~-#uaoom1m2l5=owb` zG&!sFVL9f>u620zPR|3-oo)#omraAy#5|rzAf-otlhPs ze@Lj6iSKO&^U@uYIR6~x3j}B24V@}S@*P%+s+qt55Lf&7^SzmZr$pd=;476{oimO^vt=; z*WYL><0;2cF2?YKj4@d;+m#J&Q$(eP-4jbl4l=d#&Y9#A46e~a_t{>MxaB?Fik0vtxq`8f2y|k0%^_r{`-iJvvB#>LSaqk&^<4} zX=#_YagaioYg}r%v0RA@cuFo4@V|ba_td)vDm5J6240^D$~XC+cJPz8i-^DVBTH;^ z*wkfEbS2b7v-D_j%7N1r&LjBq0u@3_%fis9g zne66D`x=c%mK6i5Jc3 z%M*~eJAL~HaRh<ZA2R>#``^C*;lF+mmebIbY%|bC)89(MXX0J@-q!LpXG-O?>kw+_ zP90ANL;Biv?#Q|$0f%ei&xSA4iiu4hjuh zEv+ieBL!chz}(o*bGK`p8tz`?!f~8UET{TwPtgv7wJ2?GREAGVVQnJMN@C3 zsT%PoJ&y;3iE=@fyLR`!?6KGXdlv?(s;bn+->BJPH?BBW`TB2;wn#*{mF!e5B`@~9lF&2egdpRtpRRawcns?*fYvHlM&iq4%Hi2H4PGcbtPInSL$ z23IxNn^vERkn5m1f`a38`Fg74{J2l&M>od8baF>Rc_Yh8_guYcbnCav_g$SdZ5l-q zyVB;-995?&kx^rk77#ewTjCU$%B~L-aok%B771%?nSUQfq#`rODA3iq-c31faH_N= zT&ugt%rmJ9VGfDpEV`j1yxj1KpC-IN;d)EZ9E_xh6qNaELBR|1uAp)Scb&(mJEHC9feioZ(N^gOg^b(~M8Cu6kty-X6|_g*ahsU7UvfrhNf08pJh7;h!H73~fn#0&%0&d+v;4AcVoe2N^*9g zTl6Sk0|gXx&6EiE>>TM*ARM*0ZO+4a@gYb2Y0wU$19p8JbhHNXRqg|qtsa@tmHIGJF(d%t1@O4xd{ZW zeC)nWk(D3)5T|IyZ(fY{I)v&NQ9UVk65Ar1cT&8CAYelk5W7hl-(oXsH^cx)luO+* zWgTiHy=KyP>xa!suM-HIGe6|J=~l3^sI<&M>vZ?WWR7kB*Ve7#FWux=sqRALx4w=n zVT=1jJcS!v#2w~dGT0Z%ClK9f#PHIOoNMV4M#f~@VPi2prT>(C!LS5JnlQ&^v^tny z`Hgo_^^`}FQB@)@1uqTU!|O=IG%PsUJOk^pt_@=1=2#uE?$%Siu_oyn88E9aMg9rRL|;t_ z0$jvP$%kLCwZn9@URFnE^C|HN_~Xlq5BzE}rE(?USA@5ZVQgr0>p=>UMB!F=y6lc3 zHq^tRyp-BD*vEl)jGe?s)?N-0=as+fAUQLgzR3FEFXTFN$%AqThvP^h$f!O+(m7`s><3|9rUq+S9GSl34kO-2Sd2E%HOq! zs|e942c~rK~G9566D)=r(u?m))oJiqE25e47Oz zv4*(%o==?K1j#fByn&XGB`1VBiy%EAcTv-Rqv^))g}aH|^|1!1SrT&EyeFq!KcCRj zA}7!$*9@mC&}Wptq9OvhjD2$WZi{tc@@&l$q(Me_8Xm+EBqqC`wkRjl6)~;waoio+ zi)OTtwdeXqFZz$=#~sb%h2pl*6}cSr%GME#bn4#7Go@ehQ|o3GZZuMJa>QwmX`Gf9 z9U_NzemOxP#?NPMjk=Q_xpq1`y>`GXn{9;jrJ=S>yigK63oK+7G&#!Dbk>kKrs16X zG92<_FXTIgd}v2_$X|yJ0VKR3Uibk|rI#(>1OhzG zDWkq>?({Rb*X86NAatYZya8G68C3-@hsLUX(D_RC6Dr$oTPN*3SLqajAxE30Vz+zc zo1T?pDZ?DqVP7mzKJ-Pa^U3)LPeLhwX1+$<>3ck!Qy$KX267zUA)WIBm2T9jc4*t3 ze^T*i{&+lzM|dP?Pn?cQ%{iNL((EsrYw<2%PxK;c{ZYlX@9Pud*8;Za!f?duyhl%- zCqgXzy)ny3Zc!?>>z{}Y*xcO16)o`;vP+(BMMSPEE#$Z>0|)_{h=6!2WAESp*1Mjb z>s^{%B~>zd1N-|Q{X`@tCZGcaPB#$x{`KF#h?+;JM4S1WB~Pknb;)oa#b9XL2(4uH z1(JJDo?@94XwJ%*Tdp3+2D8~XwL2|=iir-5yfgBB-=ipF&^DSoWS#W#9hdvn>XsaT zMu_vjVilrF>ukEo3NX+&9&rY3wTg)+v?{$;QTi4 zDT7;YXeyu=)3_zbJbg^l3H$st3n9Ft@UMZO*bb7uIS`t5gvRt8pJy#X{pVXkU-#4Kg6;-;n;bgX`-&qZJow@ZO8^&82Y8 zMewnDspgLCy@^XdL)v^W?@ZyBqs~{w!u%#KtYfY-HXZbdJ|#ouwGf|HbY_C1yg4&h z2iw;AYT)YF+GFPK)T>}vfkeFK*2rZrSE)Dj=(&~NW_ORGvfy7N+mWI;J*UpYfe?D$ z5YF0Do0hwWbO?uYs-AA=Tm7i!UVUgcdYCVMX(L@IpR@<(@ZsP9cnozwZaN(+{hxpS z$%>;U5EcD{{Dc9523BBjQvqExPAeR4VpCB+TRyvi1UYksCeTZf1z|YYDXxeqs>{@rs;S8B(&(#ih6eqZ9x z%@Jx=AI9;R*D(pM*Gbds_~%`=@b3s(dXVkrmM4^VFDWKg{bkSgF`OqA~4?DGpOP_dkXDfxINPurx#@$Y}}wD`FWzgEQQTn^9t4lSqK z27UDGtyRLTh=%jP#Up;L2#~>VNZ%;nngCaJwKMYwkLu8#>zkv0k2mHN7sn~TuG3Ab z2eYvHL7JSkw>U|kDgd12%eYs{;!_&nDhqn~XqPeN;RM|~{M>Z`=c~bQ=-bU$zT_20$?cpXsEVgldhkeW`aHEr_L27;`aDumpDP0 z)j?}QPKWLk9N?ZjV(_^YTL_NT!g)2cYtH9HhR`4L&}P(;zzQW%XLw`>fq+E+Fde@% z_}dZ=7sEqC3+F?kp9PZ^6MbdcmCesTtLi;7&Gsw1Jl1?&;vG&rsq5;Y4ySMU-loE+ zOPskhJqk72T<8DV=`e-UZ9brkh>BZEBkS%&wWy=eEoy)5apgUJ_V3?+R6P7vmy1b! zltEY#E;{-osxQtdETbr#Eq%z^<3`VC+OJao<=0<+?w4Dd_+)w}_AUr^>C^O|%B@na ziOx<2EdkSA1Xh;*-FHR$4&Kjv@$i7Lyw;SQ84IF*{xG?GC1t#`_vY~-!O!Ll$wSh= z$1-?F(xLh!K$2l}2suVyh7Mw^AevjdDgwmN(bEl_uy;7RBAGc&b~!~Lt;gxvLF&(+ ze8|I1G-!BCgbP!8<~_9$E&g5T6z!Wtear~31b1`l-jaEgMJP{vFMb}t?P&}yc?uYK z3YW`ND3>@CXLH-D(ILReE1~2BQ@Z;NpVNSv3#ohm2E&=q64JHlU;gq_PgmCrX^$G0 zj@_pyh(^JU&P$QVR4$ykWbH+Mxz!05+0%8#hYt&7WrZn7 z{KDz5MQ4;ju8}#_V%Kz?dgchivlr!Oo%+yxUPG{A_`K3D6;nFz0bmz5ZqEA6fnSq8 zMUNZ&C2{Xe9*-HySI=L!ZCV0)+xbeN0hpzY)zRIRxfD0836V{06~9D>&)LA7Y!(p> zvLo5CS`Zr@-j1pDaM+A|SE(a89B!D0Bm94EC&IYSOPv~cj_vInLAvz>kKC!}>cLL_ zchh(GalLirBP|kdiJrMRqBeO+^Z)=r07*naR5e~)=7el^?H6}P`2o9HvX9r0M0l$&jxl{yc_b|#@?k0R?1K#*)~LHlh|2ghJWWv8 z37Eyt%z_}+$NbU&c1UhiTG&RrlEGG`Kmef5VO-2x#WP z$HNS*eB{NU-D2qhx_JQ!gzlyb zv)EAC$p!PP4)pl-y?>jYJOA6idfQnywax3c9X*}VcdvV(&@<#b)J{J4c{q0&*_*Vl zy>BOVX3nXXta-N7z!i@j$##09ywp5ex}8=+!9f^xqhjsX{cH~+gx^V|4=vvQVVPVB z0u7}a)Md;*E@AdFzt3LvbPwoqHHrdmhN(!!rR$a6(m#2d(!41T?g~b5oZ@mW;=tZJ zareVQ@%O*mW&if;&!!qDQc11(^ykSgFJ8g5o6^lh5Dk+$|@r7%*_1N6~{eS)P)6Y}2DMp1Zc7FNo zr@#ID{f|GmsQB;Sf3m@~tNyXC+xYJHAO7|0&%gZkT|fTCVc-4ZZ_Meci2r?whZO9w zs8UXL@QRNd{_;OQulcJ_ZnByhJW4VQuC@Rv$LE26{_`K6^rj#w+lc%X1tTzWxAiv$ z?kxoVbyGYF!qd|o0w}1We>u*06>GG#Zw%v=I=`kt(T9_g!yBuU?)q_d` z{P!P!kkUWH$@89FmCnAhp#;m|{usEYSrPxGZyfkRC1FDwTz3FL{j5Vtufq-AqohXJ zhXY;qU155(Eh~uW8U^E&KLdXG+2=IB`epX>{hyzICikEJ_~(y5{xe6lnUORt_B3FvDNNbwCk~qr&To0{QJp4Q=9xLQ+*y&N z0KX!%6JWM`<>wDqzO)F_wGrem9UOH+^Utfp8pz<>mjN--Hls-se5l{x8^rCb1wRu9KGqAD1b8eR zGkC>K6|)K@?tlhEb$N~wU9q${K`VM6O1b0 z`GwWX;!&NMk*{U^7HYUtZJOn!9G)%@8oE%Ixoz~0Jb!)=yoR)PtZtM(I=jKrM;X*y zD9SXuHuutVsUg)NwnzA|vXk&I>Ajit+)aYPHr26C(p4+5sIJ^DOLZ;~R;Bv_t$qtD zH*g95v*}zjMdvW4O zF4{2t{L?S)sN97wQUCnsf6cU3QX)KEIFtD^pG>#&A?qOKe*^e`M$^31E;ko~IZu!x zo9lV4k0|*T}T(Gj9!$^=foncDY3vugk4TkL;t~!I|Hr zI#sfuc4w=Uoi~k?#&yj{)^t~BpOraVEs>8K zvx`2iRvzQy*rLh2wkhu=wM61!=evT0oX4|whE5L>^IyqRQS4%CZ#!kl5Ky1yjA7Vt_Ag+MrQA*PQKnq83l>LR@G6lQNb`ZL$dwKmP6# zinIFU^)8jpw{CIJHZIZRNxQk0sek|avt4_)sD;e|*Fp(=SdLUA||7*NDSp6m}mf8$D=Xx4bNFnTAV=Vqf1pC|Eqaa+IH{E zgW1rU&PP)~T}5%$jE((-^~SbcVshGl|F^GS={6&AsQI3EPvG%9{z;8bZ=1V5z*nsx zFQTkJ(KZp;InDPLBdtIr#YDU45E4m~^41WirBbfSTQj$)356A-YvtB~l7~(baaNfI zU3YAM>L~D@Tq8!GT&6MKs)#{luz&XWKGmHgUKgPGt-8eF8i6?EpHDj_An-@SpH{-Wcw1(=k|y!){7@`6 z9n!}T8ThGe4HeDa^m0T<1nr=@K;s#Z6Ln5-yf)4E9yYy6__Ue%)QVXVb3hBXOt#>k z;g16CEu#etNkn$taLh=y8O&S;cXP5i(T%r#BFZVnfM|G6Mc*pY(PV9AGzjNRxMSCg ze!P6UB~pYEoKP24e)IHn0|z_}AKkVZ-91=C*Kl5>PT7v7^t|PT)yyvqj}g!z9?h`; z$%a`H&7xAZo<|PnMbP=yX4e9r2OkqZuV;oM2Qbyql775 z?%0RUP%YI??IUqyDrAJHiWU9ir5PL3rAIg`7Nl!XaCX!W_gpd81OBO{%_qs;p~sLX z2_7HdHEA(~xi{cZ%9YLGz^f#ev}8Yw)8QdpTxo8N05rf^n`Ehor-|aFDZZ958q-SE z7O6ajYIaR_mIgr|xG9d0um6+pvl~BZd2hGEhAZ}WquGgylo~W$&pLVs?7@`3NcsIfX^>393*kYP%XE2tO6B z6{Tco_zaAqzCVP$EsQ@19igG?I>#G2mF9n2ivIb}e_~?PyBhdaq56K|UwrM^@4x>b zL}QzJZ$sEaTYC#O$&ahUNp5E5U~b?>^GrG%iW~o zDCy#d%&{WlrMl3R{GUb-A}|L$ckMJ-Z zzv#JuBqruZ@*c71B93B%xx)rsL6onsu+ z^^XTV%c;w}>Cd(o^I>#3x4D7#yl|Wy@d>|@Rp)W6ZR2JFM**flD)_`;Ww#FzCbxfi za?&rVu?kvLv5fs9YU9NyDR=%di^5!ryqd&+uGg^}jrV`F5p_7^OPaHY0&NkAHUoNV zgNTXyrX&?oIC@GFS3ixMGa}mAgLx}we=XE%*ciue_p`EWwneb5VKYqU^yMm}cf=xB zZSi$UqpuQm$7PNF?eSd7sB}Fi^P91@yKX;wo1sSry6=`EKBwSOmQ+v8xVN9$G%(p_ z!=$&Coph^^pyy?%m(WX*>qA-c9z5!Yu&kY&_5h?iH#3C+19>8dGIn zYE9i8lw;!fw&UzHeifJ;ryV}n33UX7ro)3=DjyE*39f;)m-8WifLu`b1nHo4!~{86 zA1j)shhr_S4$c=Gqz5eym4V$E`#1xB+FsE4&m}O298lQpS$g4{+9}SypHC>xmT3w^ocP94WrGI?m@nU4;2lY zGG}1UsF&Kc)Dg?Mrru|*9@j_R;QUFzmXN5*QKipny(GJ2ScHtG%N-e<|Hu8gB=1%m z@&{<7>DWixktJkNe-+cwLil*?PC-C-MWp+!4Ly7=(tllla7@Y_`&!d2dwOSqH_0p* z3`$TLfxBa7qWrvn-rE}GyF~QTXCt;+rAj773B&$X-rL2I<9{PFEq<7_%xR~8VcTi3 zU*_Ru_fY|A$=+7g`^P_9PYP8f*!t3)B5ThNf9ow`3V9oN)iwpv0e^Va#BB(gt6io? zzgU#|*-%6Ht~WQFdP|Huk_AnlzZtsi>4&(Tc$?jEDk4MMS#C=Mjf?Yeb-7ae%-v6Q zfF05*-NF2krQKbEskjbFHqi z&uOpSy35I&MvKfy*xJsYNFBuSxEXpOAM`jQE?u_QacrWLvrp5Xgdr|Hnx1pzp$XH= z83c!~$E`HENCQ9ha5yv&eNAVrah<9BAcGu^E}Vm115BRG`-2TbR#ptGvc~29TUh6O{?pZ`zZX0O&r&tc^&#ga)hs{ zMzNJRyq-n0!&!c`>a6fMS&`xXh57Imt>%Vd0#soB7jKi_Q7}{P+JL>HUfBqogK@rTENQ!THW@ zB+})NAY#?cj-K`HNy0T0RjgZXPDYqMrLi2iE$ zM>j*qBtZ6JePzllPm`$6tIzRH;mCxIJYqk%;RD^|TaIphB zRSE^&qOJzWO|V*T4Pv9y*`;OLrVmb}Cu?pW@6MPLq zB75mCA3oa!LP@`3B0i6}CTqJyL~owGLQh0IKkbH`&13lr%a@s{D2-ribkX8#0VU%CPA)K9pBkCRW?lKgGQ^4^$L}>a(9rfwycq_t@ zA0|HF2XVCHJh-FNJ3>guFE+k7hma2Uyub}OM{{Hbt{Kt{ohteeI@->=>K+RaYEl`= ziL9!&2TcIuuhttnp3AMmFS?osGU9O8;3($j6O^5H^m#a!;S>LgYyW~iZ-79LNzy(p z2(6_~#9Boy27S|TGcd@TEy?a~(;&Zu92LR7U|KlJ3kbK=6@;K_}CsJ@rzKyWD=zVWB%RM&*tw$TW z7`ren(>>H{(pH1=;qSj3e~zUbNQdVfjTJeR)%p3w$_%%F{aGYz|Z0h_09 zBHj76>NBx5nXEiC=_mIQu%gOUA#4=#;g1bL1(ykwxs(70@pgg3?kUX9-<+i&Z_HGLFxjZ{)n5n*||8;tfNfAV| zK<4mLa<>8R9@>pO<4|*a63-tukLXvpS}7yncnFYTqP^pF);C%?dl3xX=_w4p&=-->-2nrTzduSXomw1?LEZ=&l>`~Q4sC?f57gF=xGfsVWBn~#%`~l_zeQ- z1s>s@g3C&M+2h!UNIF}2n8x^bzA!jCy%{uE2cGh;$@UcGXXFT3LOo6F5A!k`QgSW{ zxOKS|x6|WO8u08++}4W>NcBX1qy{(-7$i&IF+P86kq&tJph~5We-HOgU)l%+F%|cB zJN^mJ31Ds*`LCy6!^XL%#M&D>1qUkqyLxDRYH|6kEF%I^@|JndEA-G1SfAD*>uqtH zbs$#0_@2#hI2#0FR`+J+v*g0oEDc&B|cms>j~TZKv~Ra8_yLiBsw4? zSv?e_Nddpn_^!d(Yoe~oF@fRvVBuR(34JnQb;m_OLJuF|hsntWH>idl=WhnYC3efe z<`X_6;n%fC;g9}bor$4Q47fot>C;qQ3;~;H?{+vV7d{os`}{6pyQET9&x$QbxGF|z z4|#t+@#7AhHz$7ko0#vbTr9lj(Q^xjcvFZl^>D7m4Q|@zBsqEr{a?7Y&a=FSI`|V zs5VSKf(73SOalQR7eiFNwO%!-f$)opdUa5KLx35wYpKs&3w_~(3~s}zbC?VkiDh{{ z+VbPmygs?uXC{=_v+m>)haB?pNyZvg?wwL_IQnPQZDDY5s$uAOI%Co|VXQo*|6-Qk zw@*2?YI9E%j0(gm!-Q>XN)EVy-c~`|UGx0ljo$9fd$)IPA9P__RGO-g&wOJv!G|XH zAoDJf$K)gL+WzyHtwA>PLxfBh*SS=69Xn7fyI8aiGxgj-xidMg>WMa&JGzkm1J&;631 zdjy^hHU6U{*^~4%*XH2tJq;Nt*x1tNJsLV>d`?y3mXcfC=PNkyX*o1NF6=sf?V* z=!O=~9gi&qwDCWkyB6!4A%FE*O>kIUJ^lLo&lvpp&ma6&y~|coT-=C*J+R$O7g5_x zV~!gMJ32e_yOrm;=YRk3Zz+vv0jK1TC1k(d`6EG)X-h{8=*(e&V(szoa@e=Gxv$XY z^C^pn=XLH|+PtE=0<*f{x+Xf#1g5(F4%0II}wrEN~x}0Tv#DP78SDpKMPqO5nl;U`q!4f)sH|B zx#QFIpfACn#)rVmmWtMx&2ep(RTqrxF(`c7#p3-)zWnRH@UAe2Q&%ct-LQN^fl=2} zh927m9&Gg|D$7tJk6^MR%p-$jevm|x zgiJ}G=y7Aq1gqYSX)Wi&nEj<3?dB=>$KU_bGuglHJ3mB>RO=^8=FT)6 zDm9g>AJnUVS`eDPoPw2vHykP~YcDe2@NmuzA#1NP18>d1#Rv-nb&~Pm9oGpw6*qiu%E{~*-eG3^P0NRU5;{A2mBsn` zpW`bkThQCQHe9A@i$>F>)4O$Sw*Be5pZ@rv9WYs+NuLw2qr6g*bwT+=_I4WyU4Z^p zxiB3XE}cKW?H(?N?|3SX@{T8}uuk;NAE!lFyPq(NCCdW6V6{{S^T;~BPSM!gA$IRG z;G;;pI!cQ!u|#!hQn-to8;7GlKb&L@sTuD&za36#S6$NPkA4a%iAXv7?yWjG^5Z{( z&+#e<(0}S5P1n*TB9!)YWapUXzK^fbDBym*cENG&j))s#M#sCg;bkUcxnQ6#C`nMCdhhpiE3(tfXCHaG*2=1nwO)RVYUU!vhz5Qr>{_t~yv^$iBt)1(yI=e^8ao+Fwvu`Et`{!vikJ_Mo1E;fGECBeupZc)Yka z#L<)6MPU+IdM*=u-*b(a1b`{eyHIfI%5S~od|Z_4Xxzv+ zCu9djOaKODC;h8vyyK_hbcWZf9G5Yb;>KA;{gp;nV#|by?0qZYKs7ani+__Cx%XT`B0_*|v2SMQ&F`ZjlwfTZC(r5b|w!}q4hxco^H zPdDttc1%P-f@6Z`{4)oDFuX#p*C|oHNJ&@!m6|q6lH9nheAv`S^~4E{7J85!vHCXx z>W$2Kp?K~-d*VjiONsDiOjC&sQpQks1yf0qBO?p6*R8aOwZt+zuh(f}xAvX{LURLG zn#C^I4d+b*1K^#UKH=N0J~Ey1b-UgpsKIR-u06=SKNFsYg4sdRG$sB3c1$jkP-s6U0XE8Pd{l}Jfu-> zfzUW7&0_PffBioyp4{Et9(v6&l9Z%5c`hq=|M3rBdE6T_#zvjpH7+RaqQaK>yVgd4 zxZv#@BB;*rj5~jcS=AnraT{#kmt)=W{6XWQar?{#A=!PH0fu?k`(=Re^_?CK%7q>o z8gXsZO}m!UivZQy6RWlLZ7GzuSJ6JB47>bu(ui&<&Rbd%=Kn!be;1EO0+tlLV9>38 z8Ujf^8dR@ll| zH_J8G$c-cin`sHg<+MQWqh(FJd*4tmd4==6rPHm;=mR(d2Yfez!ZO=&4O;{UFClp& z$D~iKT;L_uvE|GOt^wPPQ>m@;n#nqt-#8}g<%h@>qVllE7(?}MlGMj!L}Zg?FC7hSy{nxoYIX;E={g^->0HaEQ*g+`;w*P~O4 zPOSEcC*(L9jg`hm5KuZ(3HjI|*DLxK(Xn`7DB!Naq*51%HiE&M8`qgrm`Ol{*R*8O=4G zV9_*3cuhbXoEFC3M4FKgBTN%Qe(e600!1kA>g=$}&^}yy!C~QzWq9t$5yqDOl!?0S ziQw*5KqI)vWgC60KlikwtDFTFNh5EPd#qY!3MsWhcLc{`RCjm=_y`0Ou z`~=X&PuibuTF{Lk!r$pV= z3|4A?{+Y~xghIT}CreN=`Qul2fW779E->HC=^^)afD2Apur*?qeajqQ5o*^89(6tu z`rSVpX4W}7#6D!WIqtr%_ci(%nT9YRj$-lpf!1VnoEJftCV%^ z;b-16t1c@pDmz}DsixwNr!4vup8F2i5$HD#miSb>phijcq&~q_*TS)cs2}MOaO7F4 z_qe7l(wlo4{rNkGAl*r>q2TgG%QxMtF*&l$RvJRNw9XfmZmGF_MV*2GDtFSQ?!i0W z>k1o+=T=*uwD2@s_wd3fT?O1eAT1%ZW8F2koe8o$zm#WbgEw!cI-8z=JYF)jpXynv zuTug$E#b!7#yVH%Iw2Gk-8o?`dE+W^zC}IN{8sw4qU<)yAvrCjR&QFk7T=ptSm*Kv zpb`)QF>XJgF1z8n^giS-;Pj6CFNY7j7gRXa0aHOb=U~iQ|C}l(Q*;m@K1DIUNK~-8 zm%PNymgdMlxr-{_ABc}A73LV1x8^vtQFTQ5$InGjiSI3xI@PG5n~f%)iFSHjM|jxC zLFuSyN!E5A>RQwt(XV-jCDQZZn-g^1^ndt|T6zOZOW6y8|iZTmi7CpyWYbrb6MJlmH{KIQCA~B00Llv*%Kh z5y(%NP~5bV5IKooin`q5dsAT9NcG9= z6mvH{x^Qd|VEH%$5vCb(9I4Q&Wr3LYHA2hu!H3 zVRrSBv%kJ=c;GHub$u>zsyFdkU+5`1_L%Mv*bq>0DjhS*wQNrkbn}CjX~qG^p91MaQOko@nRf$3 zmu=Z}**#$61L~@xBv4jVRo{bTxKl&&QvyatBtww+TBc@=cbeRTcx0e7)|KUTjTvxC z$5+mk@>7&qyiQ6}%u`MPUO=J0i~cBaPWLpoz{YWRUryXiI=hzY4;OaO4&+o7H4S7= zCvW?fyk`%6%Eyv9dAp&Vr|%!%_eRB z;>(>!>q!SWo}qeLMvwC8QaRaAx|ENH*EB@;p$}3{9_=eFA#2G~+izlAmR;sOSo^N_ zIzVovNE?(;lRZqE$F!B`0%eZFztj#%*_11gW(W{Dt90+)b{$(^^2*R$#@$vBU_YIM zHP>tpj@gz#Oc0+r9_)=VipsGQcD_BS^i%g~A#d%cP(8fI8-(1~7R1{%8#*KV-dC0V zEN&JUZeT1|{QDoj_&pM0ySK)Sd;@F{Od#I+@d29I`+u9zA-6azQ|SuNK%6soeaJt$ zzx&RUX)+y>yK1~Cg73=U-uCKwO)B!IuOZ-W-=OAcsm)iUGkbjJWL?oY zQr$`a4o_mZf>Le{YV{fN0`f|uBgxz~Im0HbU8U!QY!Qi{_@wDCcv^Y_xKV%Qr}i0VXT*Y+poJvUiWJA>iGb{iBRD~e3yK}f&x5ZgK%q;3j%}tVDU1454-2ax zK}9H13hW3St8*qs&-i%6`GRtyvJ}DG)t<5zGMRul%Cs{_d1J*k;`7SX~=I_An)Bq`ObH!O?KhzCbF4juw z0jgJR;2fjgEk&LsN6RBEl9Ls=MlX)h&$*JdNEJ79#C4wIoauA_)pYPtt7 z@ElSO6|+ZYJA+UTUkTC1riqObWk!~`M*es)mnGYrlnUCjgB0*{az4rQ;&?4y3DA|m zBzdFWmtyin@@sW#$};NaM)1uhzgbP(fp%5kI;DAP9}&nA{ajUVwU~xnzAi<|PScEa zRhE?!d$u{%gW@8@l><4FzITMNdMqdBg%O&}&7v-L_FXHzx0I?Dv~cT^Qgi|L$z9wf zVD{9y-?-wA{JfUc7r-8T~uxNp znkhk^OP!S#F)^X(J0N^Nf`%CR{VyC2oikA#s?yW|TegN^zMQ&wh|JDU5Mbi~KQN8HCOig^+nyr%sQJ68Fb&IQcS!&! zt%;NH=+O3s;hC4~kS&~zqn-wqjbdxIUW86;-k^x)!krs!kN3z)U{V~;T$y2wJ7)>c zOlHSgbOP2w7jCZK((pPepAynjb2-b38j@g=okJ35_RJ+~XeeKhsG$`Em%yD(PfS~s}&tQu!Lk&o5ecLyHcyeWJjtAm@)cJiWQ1UT@zr#0oLm`1@DtHSZR4 zB?qcC6rCBBzgr7C8HAk7L9!Q9a~KKF^8%1579K7$#+&@InXViYDTP@kOG{H-TrVNC zD^8VGtYu1Un^$&Sl3^#I#3Nx4eU1wYrPwp0cg39LR8i~=NTX}g97Pc1+C(kqGs*;+ z?e!(r=27tS-(t{8logf@o)%SGa+19h-Yp~yL`dho5!IZR$#F1oqpyPwOio{erl?m>Y)Dcbv`<~=F<{r8@Z`@{2a=gT6e z@0T*Y-*mAa^y!KvX!i7MhG_VqOUYxcZZ>N$v{qU`qr1xG;r zrp8$J^(?cdP9K@%g*+(zR#d3wnQ}JYJ6*a`yQgwZC?oFx*OLl2I88sx$#p$sR_=oS)NE4yuT{gNif4SAwuQyNt{xe5Tk#lR2>sA19lCoksnO1WUIR_W^G%-v?W5B5pvHIs#IbJ@noX4$*_oB1ezat~)$)Xw^Nt? za*kiKecybIfYEZ0f9yAwCtTs@rTwF%H#84MXRPV8`Ly(u&YORlAX+Jz>*4=f6>?S! zaI?9f;5q`CZjAYo?$U-kAma!@W`#4M?6`pddA|9_5C8m+Y$*s`Bevbh%Fg|qt|_Oh znH=M4q(|6cYd#RTZuqyw)>YI?p?R1f3fsGg@YLKaPIMt_=)32zK{W44iyn?A>PLCd zR2gnQWlbB1uAeQ|l3lo$@FV|3Y1EN?jt_QmIOf1q$=n+5m zJn2{z@-X{YRY7(~z?OJ1YQgCE$mzATelpAQFzM~?hG&!`YK6#Y*{SQW(1s&lFLGXW zw~=JQRt!>1zInB&z0$Zq%`u5UCMq042Qxcud(>KDP%g^gf?UbQ;5IaaEFd;VH#%oD z8OA|0;ZNkrpCWu-n`MRjga&u|y^D`h9g8l!6hw!jkIR`%_6vaduWXD!TKc{NN#4A& z2_pQthYjV|WF^jL$z2!8XO{XbPRoJRL8UWM&lmuaYa@AeKCyxoM{6!ea=-DGgBiOFS=4pL8zr?9qm+@n;=5UIGx18N9S) zC|#x6w9c%#O>7fUE(x@MK?$ms9%`?5GY{lR7`Agp5*< z#_PJm>6woh-#f5kY@aj<2t4Wf62|`FQg2=uuuX&}QehI6Z3efO+kN(lC&;T~zAN+< zXK-BQz*V2o-{@b)2&DKRB!#=E%Z-)9r8*-usc~x6wVcRHGE=CX1iFXiV_cjS((YL0 z{_}}8T;?d1CgQzNyY?=M1*)EjNVo{z$JSblb4szfRA{_1-nF(3!!h*BB@sP_9JbF9 z4HF*GFY??MMF(BZ-V)pm12>cEdOiBM)+LZO?VI0IivA>MvEz4J@D)XlUta0qV_#wA zvMZ;~#+0TU2BW?EX(Y4_?#5(WA%A*NfRG19JNX@Cp@L`2BGIRIEh-vk?jEIBr!I^hvsp~B3dC{XraJn)pxtt=e zEbsD|N(O!@M{vgjaGl7`yiotO9D#OGB@-o3qiftW*w8+Pp3tW@~oZpj{JYi?~b!o3-1y01wI~FnpwFr(l z`a?6~jo|Lo%(W(gU@>>ZG$5*>mj<_rwh|!4w@(}iBBspds*)>r#W%ExXvSwiYv6;Ldy_0861!$Jje*%<6K~`C*c>PGF@L>b22WNXZ?Wg@By19BHwW6yg0JT zETCxdWvah*f1#fU8GuY!X9rqB`;kakQ{m$c1o?ACCim3n?tg4G(wcnUu|&Hxao$;A ziB0%gb{ow^gQxts3qB1b}`T;P8haY9&~h-C?m5IxVQX(A}9mFM(Mz8uL_zBE+5 z7Du9;4rI!+W5elh^s%Hm;tW*{$|I-R-wKUKOyH~ibbU?}Dar;4$_4kEjBRt=HA2im z+6OLM%^Dgn7*mqe3nUpRn{mw+UF>sIgYZk%I}}pE8o@_jVM7 zH0FxHhj_S*t6(UpXri*gYn>b6w-6w>#xsOzSMF4=%9XdL5qs?_XROn&<>l!0+CP;K znHEErgoqxhZ=Ii-ir({<;s1s;$X{u${`9_k;(7c_BKM;8w`N=AO%QnyKAZbBgNr zh5s8VYl?St9CNn*jZ)*Oi1O0wrEtCw8)7RqulvZUWj;xz&AanrF+PW|LzGzR7< zTC@>$o14db(Or2!rsyLl(JMz1z?GakFLzuEfb{Xn&UUFa@TT|&)_@XtWqggu#VTal zA6^J%uw*!5r*2@B8?)ia%o)sk>VNgyNox`b{3CU>lYDVZ8a=Nq57{fQSFYmk)*G}e zgaaZ*f|bFWsIHe&H6BEAa|F#=_;Rap+LlSteJOUeBz%gVX1^lP9A-b3U+q^eBRAVp zyt{JSF@5pcM=JL7ph}I2vuP)QDL1uIv>-!Q^{dn zjNnYTx|PQ12)xQ6H<@ezIS*fNr-OOW$$sKOq^#r(Ev6v!qUqg<@48p@eg(a27eDJM zQUWyO)~29#OT>227S7#;Z8nQwl~H*grZ;dFgf7KJWt(U?Z&)r;T{2RtU;+m0jIy=F z=)~PZ00HCtvdaHJ=NATfn}S>1SJ9tQ)hq*3DpK@K=73|s_ukk9^ z@*&W8bw@tD%`OZee7bHwLkycnl|b|kvpB)*8ixEZInN$NU3U~$_`k5_dWyB<(&MQ0 zA=`_?LDH+D56$hQa*x0@SD>0pjlf;DQg@DkxmH{gp|H&li>fZ2nWBK3RkgT7O+A~4^sD9_GUW-<8 z_jW@UvisgAjb?-|R-~wM`BoC1Js*(U>E@`qwkZ~?fiG8?xGR}NOj4AGr=}WnF{SF- z>*_9gY3aG_J~(paNDmDnJnD)%QoU#YIjp|kYwm5yQN0t)V@&-RuM6+7m(Ussgb8Ji zHJ2|JFQafgApgP}-CxV}6Ss^{<{Y7+`7cn1{RuZ3KS_VBuwE|Pjk4IgzeY#fvC zt@%HGY9&yZsyh&PlZ|s97xG|?w{Yk>U!n|@e<8N0{J1r1hWLVHD~txWA^akE?@L)T zpV=VEB@3pYcvP_443^RIQ1R(kI)N#5W*uLt5>oqHJ`Pt=M%(dI$PD}nvDzJ-X;(ik zElwS;T$gN+WFJ+>f?$*Z7zZIqTQuX&hp6IHmGE`QP`#H-nDED%R zj^zY%2YRwT$vKhV);)~u|JYN22K`)|WbhgthJ;J3w1b5uH@&#KY>6&+3AstRI}l8d zdVm|0Uo)qU`SN*b9$Ax7DOU495?SPh*nnk46Gehz)|{hGMYq^CaENMuI!6(y`#jlQ zNsLN-GwejTl8x`l4k+s<5p>kdccu2$Q(LQH<9Hn)qL z>MjT;Ka@NfXn#|&U})ZuX&&06rjeDdbOe7AaEDaY03QT{R@8_;yPTd~Q|` z;aW7NyON?i9e+=EDMB+ohptqTq%1he1!Mc2u+K^PXqhlDt#jI4XUOhn?m)OcS7g`) z$-h{xuB&?Axtq!?l9722x-hz>)5_6j-2qU!qn;**^P$$HG3``g#y-5HkZ&;xr1!p> zsq{G4t;D!;bz~~nf^x=pNZtH@^(pM|A;zfdTCP!DqjHk33(d-rIDWH8`@zYSH%vDm z-$6=LGy}*SvOjC&LDt2)bA!vy9*as=4R7{3W0#|HAg-Q)$aFoH=DxQp0m%Yu?oxpk z3!0X+yj{K1{Cw5V@vsR*EqKqISw6-M;i8`1?Lr6_vuF13A)#NNWG%~iY4WIfBqr7H z^Ok9!TfD5CmQ0@mqL9LAp4Eh`EA7koBJ~huJ`E+lEbe4(8EaoU+WOV0fHEmYSPx@aOR*ob^?$keJ8ntm7ZElpJum-AL| zT#Jsf8MBEJs+E^-i9X@xMWRhJsC)J(xRs+!tSIkvqpD_n4(*|H)W+JBUKc$hII>Un z#3&t-6niSheM87?gc9u}-z!T2%YdgW9akSOk9*SBjA?3=A7^UcHQ-TGI1ecN2_$di8=~re+GUG4U=^TqjJ|A_m|M zHj|k>Z8)x@r#x3$Jo55{x3g@+2Jzn4iAXO685gdN!0sdx5L%VyN#-^2H!!XwZUwu? zF~9${pNnxj)t~ zuNQ&aLORt5j}r2?-07M-9QzTUkZbvLePn)UbfHv2k!^ZVrB00=1jqFek9J%!>YO=_ z&jc593G}Tb)zoXBS6dS|f2XksBgq+*cii^AA^zSeHN2DSai=9!x@x zLaZ{ud-hA2lx8NQ5?9JZP*&d=Ff$4m{d*KAiA!nuX-qcAMnyz4JsdnyaafXeo!a~V zjnRQL1!W>Mr&;@kQRJBLQ1FDbn4T4h4(x?nEws)dIQ;Wc=NTTt=KYhYk;nr&#t|0Zlg-UZFGwNb&Q<5Qt;}ZU%hW+2r$I{ACaw?sI8(2kpSMNo z*WC9`OP&#e?4J$AsOoUWG=Qm}Cz^MG(N3sVqZtPKJi>0CnNP@r0?vYq{&r;)c`q2< zCE>Y9oJQFd6bbC7ld@Iv_6&dYk5YUW)g?^SvUw8sN#p9V-4ddH1~^qG#vIK%IbwZh zCwfIg&Z%Qz-nEW))+|40jmAmDA}z$sYl`c(l$1Up>n*at*{dAD~_+Vb}UP2c_a%O9S>&|YFhIzF>}W2@DN zySrdTa`+7(zvcXIHz{d{ux~OXV2jkSN2$C)7Hr?WIogH+t~VkoL@-tJp@lqOJl+|_L0ob z?<3SbdEYAa-+YwjpFeMv>UTjXl!hd$L#V==ZV~Nmm~7tC2ayEJyLzpBx89sj0Ps0d zh%<20Rv+*YK>l9@*}|X#lKao;*`6!k^f>1bsG@*z<F}=DzWHdM> z>ge_6J*mYK|M+-vj|`1798TBVVKDV<_>@iljsW&b=kNF;`5LKFLFbD52bU{lK?C72 zZ$k_YElC*2>3U>>45xO<5MZCz=}Ovuxz*hR&bW@3rJ>cDzpgD?zgS#4Df$Wtnocqy z?0pwk)>=>yC=HsylmL2gz%b&jTH@VXb^Clv^jxV|(b*)rtj4f7*Oknftxh8hD#S2h zVH%nzMy2|hhG})yrA1v&+LlD}SR;fTcYT@u3GqT~f)0td7}q{SY|wZAx*(GIvSja# zy7zmYb{15Aeg-d^$DAS&|6}dx&5Y%32E10N?(SAT7Vbf90)_lsRGfC3%q)yO@0)gP zVSD&9!6g0svrp}SnA7Bf#?W^0R6q^e0(MK?=86)A+&)DIaGNRG$7_bf(&}-}N#RNZ zLZJ8XVACkMD>+HYU7CLH{TqY^r6rr_+$mzb_tH;+eMZLvxt-ld~Bp$(V{)J*~_ccO0R3oqD=-y!nN?jocpr0t@8K1X@#L?bLxCl zt>n6mutlpc%-9baSYKO8TtNS3E8^81#2lD!AKwFDF|sni@KdwBU#8R>RPjd5$gFma zoM$rF3Zc>eZi&Dy%rHY)Q1hM_@^g-Lhb;e$dujS!na%nRZSQG9-H@)KlZMA!n!@tu zYXJrw0!QsCSFT;-mGho1?bmg{usGF@OpCJqN9NJ%(%5KxVMP6ZC==d6LhpEE#4(w3 zMo&mkH1}u&q5o6f5O6vnX*26IeIwpj3*HGjwylC16R~m?N837I7DUb@dlRU2h(yG< z(h#dxbMw|>T4XLhIto0UfQ4-p;@BO|==&E4q#;~r2~+PqVi8GPQ$!Z-A7ZniMK3Ai z9nN5w{WS2%7b~J5B~=5kuB0i$@I(!}qD8TyLHWTv3p(V|J(a#lwh*B8H(^iC1ZHcq zM)dr|U4}};eFh+-D5^KMv|}kUHR!05DH@~c_{0U(1?{?Q37Yi&S~N7m^X)e+d9~t` zKtJzEuOuPhGCu5*Ewel;B)GGB(4+{|C|xSC0RM$v<>UuS3;&~yKMLmw%Sjag! zjiROw-XLV3OVAvxg1aC`=5Ea9gj%2OUj0A8kMFLJ(4%9p%AKmDarxBI`O4Z-I8Q?x zW;{nsL#~~i8ef5EB4eaQwNS(OC_*!yo{l`>9~D^4iJ!)kkf2X_Y$G{)CsKXswUazp zV&q6b?H75b?{TR;K=Ag-GZ5~h(A!o(6vH3B@a_x^f|)Kg?IyAs{#wU61FG}?lnGc7 z-H?3pua>vqQ;mJ)dXP)tE#k06tW&VRFxIZ*G0R2Zi-I@$9j$%dvEK$|Z*H2G=l;H) z!0%97Vf^2WgRKj8b40t66lvE-;J~Nj+GlKQwHB8(H)2~|7Hq80=E*r^(`RmhM7qp# zc>%$*h#<6Bcy@-DP%Rx#DKQLS7h~LL%sZcPoBmEEwFSyNJ-!?_ob@Sp8ko0yt)G^! z^hd?;1WnXcKF4Fna=lkkr=fpEm)DysKLn~yPmAMn^5Te=*Vtb05f^Wr<@ z>Uv56%2o9*QHaP<%67gdz!6~aAE>&N8&_U0Q%Gp7ar1f)hCD2J?7(Ey7pnyzMFUM3 z@?=iCV={|4q1C)bg5D_=(y4|yiv@=n5f#aNqPwRPUHEp3Mpt=TKJU%t+ds5R-E~Eh z%ER`n6Ogm0AWaap=h(_YKxu$9Z#FR>rG@aGDzqT6c=$)sh;qx|Wz@rPjasu~IYYbkW~*nAVlYu_Xv32nht1?UZmw%lmoeH+zNksi za0APZG@|$-^NQePq@;F~>B*frlqa4sQH~=n*(zZ<($ptywl`U@S(l>eb5wzZ6J~BB z`5_wFyp-##*hclZau^X>3mlngSiv%`nWptE*LXgsCbYCZ!%Y^=;k)vAIc9W_$fe~9 z?V9N8g_*OoF6X1T_F>MBdO%B66HF`<8G}%D>aUdQ+$wJTdRbnr%R%2LC!$I9)Uogp zcS1K5uj|?mN{WhE9@L_!!*%rx&@qVlbb-7sTo21q^VEGLqtVMw12G&gae>^uf~WVd z?YiE;WOh^5GaMK`XQ7~0A{U@$hx`@}boAL0+3#<+ufGoC`I*k-w1MW$vu+N}WzKTa z9GStzjZCtPQGX%t&uqegF56Z5-j@`veC)aw$G7F70R@SnKk+3!=&-8a@>e|ou{JZvD`xgVZ4Nj<5H zb}czsHyp7St+nU>v6Lfy&FuWxjf931{LIR_6>*l!#_aBSG`y7nv7^;0ZO+oZQoW4r zWW5;16JH5ApWRv&lfTC8Msg9EojSf0NOE<89~E^DXCM#HR>4lwFohptl^nR29XKK| z2Qo`ws+*;p)&N%5t&yVTrW*u%n27C6*V6SKtf{1+h zH+ruf0j^SbMWabk^(fV<46H^-O}4vjuyhy;0=SBt`D9auicz86Y@%Jo4Zid4IC8hj zp;9@J$9n|2cxq(ba8yi=4y;m-#XOZ8`GU>A`^7C6bZd;a_Ksot=;Ek>a_Hvq#>|C-RVXxBE(^i|Ijkh_9k0&I>d_tMxnmuJ}x zk3Xy)bm?1W3s%Nm@ofdp#AWy6{5H?cmiYmJ|2$jrm;?K=_ulI|+xrw-7nQj^OAASF z9<~kd{miT;L6`FobcmtG)1899l&=j(g5@YqDQU{5@n{feMDSrBRV_#Sl}yXr5{!AK z%1Lk}`GiMSypn{lLs~Mntq7B4NZ*%ZmMM*Ia3GR$EY&RzDVXc{@wVHK6?4mkGOwgg z>-OVbrx#&Mz6H8TdtJ{Z54Z1jRmf=F6!HFYn+hutqkyG^QWJ@-Kc`SBpm%A(!ij zzPOQBtOO_D@w%wbpm`mE_AFcp5~3%CcGf2;$|X-}HWBF>eof?Rzte>|9r9mD(AaxHOu3DY-{VwpTGXq`T1yC@`UrydPyhA&mTNN$u)rH7Vf`T!>(O`Z5&OH!Kph=_A6MLdFcXlzVn&h$p| zsGnjdf_}$CIBv3<=C6K3+&Yl$33}Qe%K3&*ko#vuciqKVO*LO+1Qp{+7%|GRfn#Gzt}#SKajLMDKU7tQE549J9am~TWpdy`;Xv%J?cEy6^rhUG&v}E z+alO&6o9Z*mL+w$TR6pQE0aDnY`G_8P7VU}H|pn1GG1xPyLFDbyU`I7!85(YGw-MO zEw6;n!kUB{zllV;mi!~B^1#cjqYbjt;`A_5k;m#ArB8Q1UNcXlDM^=GK^w&wp95i+ zPp2c!mB%}yCooVbX#&R5y>!=D;&gY2>Mm=x;hOotZy2Td8g@LbF|P1zk|TM2ws0?e zu&{*Z^FGQeh<(vr`R7qB-=bRJ;@2dN+b#MA%YNOPE=4%e(eSh9O6EC zm}Ci^Sx<7NrLcXLa*BK=f5a2N{aRvPsL1v?*WLENyG*m;lzQt0hD81LyS;HBae5Z6 zZwz?Ksi%Uw?9gtQV=nzVt;|uiTvzhm-s;PC74(i*@NwUVUE`-t&`r5oLY3bWt&s^=UX6j7 zUKGfqxk{`<)XTi4n_2rDo=`F@FvxAzqxCG1Z&q&dnNIEVtFyjFc8$fqRr~eEv*&10 zvA6B2lOKym!?0b2vwJ7rRbo=7vE9~>t~@=Dn3+TDcoqTqJV!TYlBjTVNQ0V~|Fy)< zI*#;Kf1F)ZCa$R|^rBpb4M6OgXXIkoO)cf+ACTcRCjSMV3CJhP0$?CO{=vuhYc zql}qX@7iIZc~(tBLr*@RD3xh$G|H+uGWD~%JPKUf=DjlncuyJhh;Up{J&OkwqkJ;$ zTruHQ*9%eXAHC?e-q?$pHLUTQnYcHyAo751LJnS@VLf#<31%>#>EfeE<&eyK=;375 zOv9DYPl1ZoU3VqP>G^#|1yxgu=bwbsP`DrY`s&?0UL`Y?W}SO?W~#P7h^s$DoBDd9 zi3ZftadJ|xT%Fgie#aqrdJp@jS4GRwN3nABT4Cu%*~I298YA8%YR=BY8!`CG@U$<%(j!u<%x}}WiSt$bq1<-KLp#*lqh-CrN_|jYtYz}e@mIy z2wUDBU*zXG_P3U+dBIKL-uqdfkB8*(^LZeeUG(i%9rtZIXM^7w`Q;aspmKDqi&__M zzy0BZ)On*6$WzP2!T~2I+6XwPW?(-9Fq?j{Nh|8Pwioc=09geG9>7qK{FJ$CB5CzA0O0^<%8)-Q~? z6n{%~&$a^eChg`Wl1fYxr(b1qrg!plH-I@uAj{IG-yBG7r()ZKFG+DKR5dmkNZucR z>+9*d0LtuZ4oxZbma1Gb4OQ7#G#r^AVgJ>z)19AA&`DULxtekoW+-fXaqzG0 zayzEl_-H%h;Plwqdh=Mmk_gi1FzSME+w{575d`dY=CwMW z>=3N1!PZ)l-a8|d6%8P5aO46`?%ARy6$t0>JgK@Ynox2BCq0{$2w{_a_De)?W6U+# zXlkPE01w|x=^k3SpFA zoen}#bb5r2RN002NXak1{K~kWeVW8HExMjq_v>K&T(Re?Tu6JfAkUJ5-v_zgHxKF; zA?e)v2|XW7lb;rE&TY}Y7~VyMO_yIP`>9Vs7PFRWA67(Rdgy}%#k1t^maICj`_-1v zHt2%i{JGzU+gD`TtovXKRGtL(iG=wHew)#L8m!Dsc8T7FPV-YEwnfX37SpWX`yD>H zX`w0{y~CeE?RH761~z7%jhTU^wdkYsVTZE+&)$*gQ#<`?CwM>hXl1|V)%0YtGLt;d z3>6>0tM*^txzK505<5TAMhM|WQen4He$65ypD+CbjTv1-aKrci@za0w`5-Xcc2H#x z?K7uTcrL+W+E2&*`OlfzV?i5iODRgAPZeSvIGp4LhV_=(YIQe#8Q=kHV50A%+Paa0 zfA5yxm=Lmj%|7gimJK?Uc#A$Kx%&$xQ=+U1ZE0&6@%*+nLc+p8Bw$*?8%3 zEO}eH$J8XCVm{(Jm>AgH&3g&&==*DZ)~i(SPqh7QKO;1rGbTztj2J^a1o2&oPGf$| z;fbPG!RFUZVI`B6v0+C^1~o}X*`Fee-?FP z0KGXRClpqc8Bde|z;c_Y-oGi3e=R4@aSZ9uuZ~jws%bGOj<;|oxyFbF9VdVGD=uSf z17ObF)sIGcc^zbd>&r^8X?I;M(8JGvmTJ6es!s@VD+wmEQH zq(me0Kfe_gE&^_IXZ8{(in={Xl%-jjP{27v%E&%3BSm`wMnA_kVKxSWd2&SQDvu%b zwfWFSE9!h|z!p~tVm|PLp6}BBu;@wr`+k5_YC%Xplm*@xf{IlMn_0A!hLdSbb-5!# z?fEEGcadW%OYkh3u}f?jHEUk~Td{AoBY}lov6aXr7%`7VlC!B@uK+ZUl~iJqCm*t_ z6p#MG(`vJ~Ob0uHny9E`oeV3WHQ_1AW5!yBrLeLVayC1(kcoY9$A@K4 zs5DGn3bT-mb|Ded%gCf8eRfVoS^Alwbryodh-jgzh2-5cCqOH1t4oT<3v~PHm0^4` zwh^%qlpO8PIEhZ6?2X$49h|(9%5|Li3@3=dz%EI$Z%&X5LQqdq4OH!OrzPaj6k^qd zulA|2q2Diu_cGa)zyC^YT(>#49SF@`y$B_6}xC)K=H- zGRJfZ#kODMYbr;tOa8arb^Y*tm$XU%1D{`}J`8HP5yMiSl6_ z(+@wj1In!Yp_5*FRV#F3MuUGOFf(4dm-0AxfZ!YhYT5SuL)PgPaa;OH8cp#oz*u@? zUgI#};LGHPz)h8^{uk`>zQ?`YbL0eh=nKX{M(dm<(`^v@T zpk!^4=m8iPJwN`oyWB~DF9H7Sjybp|+)~%0cZ|JVx%Ec4dCDV_w(k4A;Q;`X;MCf{ z6%TOGWpu+~10oB#5j|j=p?jsOhklQ;A(isjH`bHbQi`iJcxSj8z_~L&w{!~oi5;cf zId?W?gd)uk%cI@P!FXkUPQU5R^Ct>>1=^Snasn#pUI*f{jvH$Ae}<1S%*qMZoJ(K% zCwcwkO-EYRz%7m0hMlos+e>hmlxr2JyB(cFDi9&YwkW-{4Yjs}buJoIo5iGQI2LRXw=dkGU6=kSj== zpozS5l0F^Z#Dt)dFF5_^n}GINiIYa>T56m(>4rUU)FvP(=lV`u>TfpZ`rt=cjGa1` zQ#Fpo`CG10JA$K3OGVq_rvrd}s7jaYY|5Dm?IZMtsC|{0kTF}FO3=!LJItencFQ3C zX~>8S&)Q9-0=l_;j2~dbgQ59i-n!$q|0KV82}$+|3GwrJ7XhV0PNwQ}R5?pCvfEK| z^KOFbd5|A?Xw2zri?+sgN|D|9jngt6Q8x`_DG6$Ojibg&6s$Bii)B|lZ0W+FB`1ul zJE;lzI{FIEAkHDJL$2ZVw6iRxoT5v6$4Z{j&Lt?7y+;`@nW6l9Io63J_?bI2E|DLL*!W=^{W)z+JEVOeujz=-Ki4v@ zQuiv3Xs-T99^EJ0&h&DzG4WXUn*5P)G=Gz$F(LlHK@Kb*&)$a)02c(ndFobz0_7O%#)vwSG^204d;!dFAex4lTw+&Y4~=iabo{s z#p8fh7bW%urhVZ|z9U{0LXmHQ%D68R63o+gv=jS$57lj5-VFK*dOc zY(PKJ9qO81n0S@%c2?Iyl^&IRI(Da5xnA7nFMoE*8w^pSsSq(@^fbDw+0~{MyUCE> zv6f0jYB<_+XlP1>f!wZ%mSad)#zU0id`x#d3pAieh+a4hlgs<-a0Ci(skM0BL-^E> zW1>J49sx=LDVshPU&!wkQ{caqOPpF}a)(S>S}|Z_JR@sbpr$*Yq4TEE7-_R7D(x58 zF7G_J=aQrcsY@LWZ2vEGzG`+~H`{`JGnCNWbmOfuig0%Ce<^4ZiTy_pQjhytZwlpw zZ@#<@#HPYa41Fo>`&&P*9?n9~{A`?rj1Td-{Q=+BfF%gub>agxxzE+Tw%k76d^7T1 zl*8tz9^GHd0z+}~5!^8x%eeSou50|9uICX>%) zy6-JA&Cy5sG33Lp_v+`OC|hgpW<(M?$@)B__f#z0nO(;W|IL1&*=a*)jeMW;xwfmx zJkp_@Y^!mQ8dS^kTqgv=6^=PyL#-t&UZ0C|zf{H2doUc+wgezB%x{v!6KA{X0gCo7 zWp>?60_=5NB@^&yV^B$w@tdnq!G9N>)kBk$yFry*EAtZiu{Og!2%Rf4D0Oouq8WGm zTdj4WnVtkjiR2K@E%{{r{79bKQEoN}qNf?>J@*Kxrx&rJu}Q44&7~d7`K)Ug%n-lG3+maY{^ONmXuWT=dG6o-;Nq3B}if z#Jk%8T?>&Kj~?5l(bS|m8a7JH1w%Esl?NooPtla4S5SO1nHc^$o+CcWYy7mROt=}a z?KbV+DdU%+D0%Y7z}HkV<_^V6r#UNF)7xF3(tpP+=<6)|))(bWam8d^b4miPNvetk zFe;L4xv1m`t9kOg|9^l+fQ*)gsneocni*`w+UaKwF)pjS`MR5{&Uj+kiLuer79;kz z^yW(ihW>ghBZ%fI&N%0r{j<_4u&pBVy2>-jSQT-UTdg&lzWc6YqJs{=I=2H@oY%Aw}6N*W9hU6&ZUn^n3vSaO?O}pu3XE#tBY1yOgg|jzaXnT2Ca54&3KN^%7 z37UUq$KEe6#qBBO01T5ZI>(%}@u8 zLQ*u~jkx0_>O7}|kS6F>#jz82S3?cNT^Z%*KmYVA1#W`xahQfr|64S7jpuo@y~%kh zj*rMGEsG-WaQ_Q`itskrjB{j>*S

    D;&}%+=mCH{Ry(52+f9*m{2+Ak8#VaR(w>7}Dk!}8= zxvqLwPW!Pt>b>?^|G0?!Z|xg;Xb#w^ZZQVnMR;;3P+1s`_ClepOsuB~R9Y-;rVqr3 z!g@K`mBK*}b~<$4GduGPi+=aLtuN?RQrUu2=;7R)gW$0%IbY4B>k@t%+m%x=)Z98>|6_YcaIwB~ zHxld|;}5?&zGFSHb~cxw*LXCP5^?$%S|ezLa_II2dFg@13pySnP#!403ts*W=SA*&7QsYkbw)>uTj&+Ur(@+2O+%HD#e86c{Cd#l;YFG|vJB>-W zOA2#G7aRpiA!J>+Tl2J_@9x>|N(@Z*Rbgjz&!lo` zOpR--KYsb{*PnWxM*sz^;d$Q&gH@iP;sIc0t_8JIja$63ib6|k7Z|hDM`1RmF5BeN zcRmq|?0N@`6=W1d(YUTpFGWu_a8xr+!OWRhq+yF#1#kHyj{8?Sxf|a*GEZbjLq%&U z?9&nkr^<4mG@IO7bX}RukST?7>7z0z&Nlx{Y4pr@)H&VqUI>q|NCf##1B_71*?YlF z?8>7niP7v>2X}o4uInURqh6!+6aqN5xTJem3G&))=bu zGfu5xr*gOIGo&dvzq~qUxbygO4k4Y*Y5BxQAolraT4>A8S*?!v=pBz&Tp6ieY3k1j zj$g^N5ZH1M}_=Z>fDR6j&An2#?5z8UwxLwQ{h zuW)gmNO&17P3}(CCq+_t zuWunp$gYqMwt88tvxL=F)Vif~_L@pwr@@O?IphcAA{EYhU+Y`u}3* z$LeN@lHg14fC2ec_@|Ck7l&NS;Z#@ol;d^!wf!0$kL)Ax|4-$`0#e*33gf48LYyko zJ4&-K-(P;ulW%W~qIO}$tHK9n3>BF|OVMnf!7cMc3D2BVD^qd9d2tx5!@!x8s$124 zcI>#dEJ>P^5S;h618=1nw374*Z3+^$>~ZquC_pVYGT))*3tAKlhYW{UyV4#Hjh)4C z&xWqIw3ooI>qtQS!5qQL30(Dzz2?Ns|7Z2Zx0dVHnYu<|Ao5v~Z1ya33aII-*Dus%a1QuEox=1}I#VpGx5M79p>cFu5Z9ZP>3 zr4TLM8^|f$6rBSIUpT(|^;g#o^`88!TbbsZc*NQeP)a_8PQSK0y)OMncD)-;bz$z= zU)H(3)bq-2UMG!~vh!k+e+>yYfx2PlJ-6QI?m>BEJ*peAyfK~r@N+Bq?|=OJ-~R^2 z&96f{;K`#<%R)^{-hu=qERGJNY;J~33(SoJKIwuJYj1WCikGXN0d5faoa*l2o%#Bl zB96VhJs*mTJ961LEpv17v$W(Fi-d|%Pb#B9SfolO!#^sG{D6skwC?jeZnE&1S*g*l zw7CdVYFxf~3JF`nmlc`0qVuPAdp^wz-g2OIFBK?r;=Hkzp;QBNaRy>PH3$}lK7!hHcM~d*1OfMmeLc{|Vm|BNQ zMK6}MUCwQ4ZlDQD;ZK z-FExHQp*U(gYkiVO0n!cLzgOcB7Ja`NMg&z6`$qh87tIo4?NZ-c`n)ef894TWCW;~ z!>}c%iAw1Eq^IZTd7(Tfxd{5CS22Aeo*8nqX(N_e@0w_tTV~$`Wk^JstuknzR$pqd z_h!V#`zD`M7s|_Qb)xSZmGGuG_)}U{xI$*2U7eX?*}i-tpy3S zZ~N%Vuk1n`QbVZ8H+G-Ymtp%dCGKBA!7CW1`-Dg8em@ZMX=hytHFRTs{%XmhbtfdVv$gLq*y$XEewYK-dx9pjZZPGX z1T4xyRW?J&7LD=mwn?g2p-4D-2drRbAF%^djPi7U&HecTD%W;}Z<0Hpilvt{Ov~T8 zfs=l5r|Wpo=c9OQ=--A31X6%Ub{eAfl{RUAne)%Es#MQ3=l*d)g{mVb{Ae7n%Q5XM zSBIPY(Eqg{Io^YP^&|OO|Fj%EVik>OYRTjPhQ~kKmocAGRg^cp5r2xtx%?Kwfa!AB z5)dja*EnIFA`pgRlaxYtMs-p?)Qq^!ksJ%Y{e_nMVC&n)cQ5r33bJtUtpdDR+p0u- zjNP#@W@4S(gzVDCQExX`VzYQ4ti_z*Xz|b$jfB}+6(;2(0Oz~dj4jNZc%8V8cnW7q z!Jl>a@(9tA&iy?RITloFw4&6oJW1wa)I6a7Y|5OstWkQ?bv6-`Qb;{pE?u9%HZUuX z=K0lW)QQ+|v)^I`Ic>4*c%IxtcF7a}{DVx1vBf@TAaEMNin=AhQHV9v^@b*0Gvk!L zTo3%$EMJ9$Aqi`VqoKEOpK23DN*{(^4U~QE;o%8iU?4aU!(r}4dYF#jP^+t$=1w40 zi^39ijNxCJypcHD&}ZHP5QyYV@73Rih_B1>?Rto>{RZCf-bM_ZpdI~bfx9h^iv|_9 zr@D}f+u4mH7k9;vt29O1wOtt?t_uJy714FmSfI^mEiU0tWY~h|WXIdyJ%CBU7NkTy z<*O~awgxqTl5_rFyo1zqlY4OvV8gbi&G>X+BcRm)yk5b|2^hLS%dBqorX~IL%YcPL zf}*@6oI8?4mgTMNR2Y|pypWj!-AV)K9?CqL(wKPNiz$IeoUl4tS-ThsVX2QW+DZdr zoUQG*hFzD_mnO@djy@~EZF9Ir{AsPEoVIrMy2LMaTA!J6l&3TRn*9^&IG<)3s;wpM zlf2i{^^Vo()~i{yytwnVU+*0!YNQX#=iR=OpIy2aj+6Pc91KUTNLVy`BChe3Hnp`} z`{z^($wi^#)4WPhFFH`;(|F-es1Bi@Pcn-0icihlZ&e#5nhm3N5f2`F2;L{Uo4m8* zbjlWKWPyo*-lee_)|wP0xmzu0fhCk>*$@Xag48PoFM{?&guFD7mSQtM@~4iw3ymua zbhexXX;IEkkykzlT0@4NRfv<{ZCo~chCwR^=*sWj#TezTW+GFxFm&w>@($?YvdhGr zcvIjfRY=r$G_UDqrZV=VIR<8Blmju~+Rm1%@YtOq8-RFtOna4c&U-wX6Ns1=b*K!z z$3Ikz5fW~^YqzT7NprwN6 zxc$IVqYh3%VZ&c?m{m?<8*~O|7@OXgIaJ-9OFn9mX}3Ro#CR??s`c zdS*hr1#1{%(M2YPQgP57uGB=VLv*cgpfnj3O>%UQTDpsej!<63v$qyt6l>k+Q>nN+=UPNs~2R#2Q1O zh{oFw)vogLLN1Ci>yG`%Pm%hyf2k}mU)n$5ui}XKQ&+kAN50b|I46{oZ#lrl(Dc!b z-Y!y$&aizItA3{^oM~uk;`~h?Yj-#gr1MhrMv3#MiHb>D?%4wLDE-uhDL=nusLPUq zo0QwS$(&Rx2+72<&%x)+0tQTjjQad9#kGHylg!j^sExAC{|ml$lu~~Xa@lzmM$WIV z(vCykq1lpV6r1_xj1m&nb=-l&vf7Z*KCz&<>0 z_sQb2y+Q-uFaOp4YEV)rX1JZerPgBaJ7L)7R$!cIY%dT zm}+O*7w}q?P&A&KSQIqed3TSG;B(|FL}^2pRWoaR?90-DCDu?XEO`?RqxT9s5)1 zf*|_P?%O@M~wTVT@B=eODRfb9T?>k_%nZN-et*xi0E8N5CS3*&b}@= zH#DA4Ys>M?rxt|dop%Y{2!b^6J0pn3(!q!U(6zGmo3Y^&b%aFl!GO_>oF}}^kHwv+{8B(Gq+P9njZglAiWPKqoC8$MWN29s_Ij~qvquz|{_H_DZI;LPBqBpxR zGHA$Dm1!sKExL>=@m+{(j|j}?y(GAH5lYS+q<0>Q=CUEoS*~M;y6ej1S<$?>Nvs+0 zEhi*~M_@zpbE@|U-`BIDZGus{~QLKm?kj$Css@C7mU=YHJ-CW89= zi!iXmAVcT9dEd4wLN)0e%5XmVfv8$mHu zUwZpN<$PfxbbmmYT@U}`7jrGgZ>nuJJXEfJe9>qj?@heI(+x=w?%MJl0;w(C@k0_P5w^wRks~?Mw{%DR%%jJp6CGb~iM}J)(;m3}+-u5;* z-FE+AYpngX9r1@=<9R#h%dULzjE=oluW3fC@*F0zE{}u!^4ss8$yX$}i9(}ZCz2*O zxn;?sTE@_vIwIt-g_ zybKeL)~DcY@rch@^2TJ#0BK+VVJM@?GmN5$G>xf15Xvaeq8S>>rd@=YkUzKnol_~L zc38J?OaRS+s6ao4jM*3d(3Bw^E|x^fm5h>%I2FgYQoDk9Uv9hIX%kZwnCRDOeepn* zw{DtD8x)|~*>`(!nNb5nyo4q!!Cz6?+clRf`EyIuJP_&1D%J59q>?PgS`;4UjC5L( zn>rM|v|XrhE~QhQ&OXsZrGO6R(cpAzPcrPH+qw?I&koFJt2j+44orYTC~gIJGP#l0Pty71yQrQ@+YP z)w(p^ajs2cFnaTbVNs`IOnQ1Ms+A^==7(m%~tv&aBM2eSBd?wpmD0#@w)2f zAkkzT^U6mD-<)4kU)F_ z2xykRqyPWq?#-6uNU}7&SUolb_AF#(h7$YQ3TA@RL_2n9#vB_H>U?lqM-*Q4(_nWM1!_c zBs6&HnY}@&@WX2UE{$ptaQ;pNj-;?CmKIU!vW7`qr+`l)g)t`-m?G-L8VM0z*M?IL z1~pn#9yLJheqaF%LKhTdBS~dfsBF|K5@ztp4hwjm1e$n2cNl2U*iSRX_u7b&!p#Xd z2RN2WNGYCYK5o4L06+jqL_t)f>C(*XjhX$TJ7k0#B=qo@t%l>Fq$d%mlWyEpH6wJC ziYOpJwaTC=FjycOpaQZIDToS?*qF{-6}|?_7+tF5wYC;jB-Q70Wtm@O0hu2ufr7MJ zkRZi8qKEuoykyre^FMN0wcE+YH9&h z)w2|n3N4|evd*twRH+fCZ4cIn85)BvFcfOWst*{V8zByw!DUeap1NMkV#xewa*OnS zS_H_kEUr#98C~X=7Uc(6gQyqEcm<21bQBAFQgD_lSi(r)0e#S=&~%1L7>0TEfL4>54I(R* zc9xQoVI~AHVMgKTU0b3j*EESj;L(Tt4ykalRRm~|2y$bI4iupkAW28V7$J5~Lc&aR zk{;t#nwS8*MIJwpUWia6hm0a)W2uL1qBZn$TBp>A0AM(^oBWGSY>YJuNlW7g`o^t> z)Ye%FMApPl+tVj6g;1jc;9(KtC&g6Bz~I2l5dkIv(aJ86*u7&YscQvlgE9a}n(b|* zqd*^Tnr1msn5e*UI^$=SYgXNXZV8|+AU-KCBoP8vRVp`d+|f;tD+5J1m4Uc{&d^z# z`AVDR^&nwDu0jk=@XW#dw2k{Yd;wXsr8%=b4++S|cZex@h`FF4ej+AeY_!A%L=<09 z;&-UBo`qev4#P{cgbjFQi85zFULIbj&09nOi2plf$}q?!r>I#ALs zEOaw9;#jrLt!3Q_kk5jf={dnZ-2dZuN~B^)%L|ZfhPiT95sZcAp$S<=jPW%>4u-iJ z7Lp=gU@IRe2(MJ>kV8cELw?HSlZ8e-r}-Z$kg2UXlEUbdf-Xn?OB4=#fpxQEje6KAWjSP0a8)59)D5+iuB&|CwLr*a?85L zEae$CLV6WSRB3aEfvK*(#r6oonS)y;5Ctu%DUuC*O!->#Qqtyl@{K#3K#=0aQ0zbt z$U4X=Ms*2&MbA|j^e7o)GzI%g|%Y_5p{Olnk;1cdbbxdVfPnU!{0>=lyZ*pLWW zL(yEyAW;Sg>wN5k!9c-NG>V%sNt)yL)-ZtofQti3bJ@VtNnzqeE#ZP+eVwpu;L5Mh zWrV|D;e9($#_Mp7K_M4}Fni(`zekB<{ApV-!>;p}xfX)xq&`r?VC38m`$$X7a1Ab$ z{dJs4O3Tq`z-`QCJSi(_sU@QsZ4$*+7jvHrSYKvq)Mdh@Kl5nF2%g;}t&Vh>!8-;D zlCWUsmCK`ea+$9_`oohkJI>(8&=?ZFDa>r<0PtR`*C8XMz`{!gfj(pR&MO%qDKx^{ zxCt};V#-z<-@k(xT2I0#qaFQo==cMO3}2^tsBlU_aE_%)NHw70Yop!9!staQAI4L? zFu}3DpStK7Cb6-`ur!VxKxu(Us`{g;22m4Wej*R>Z|4_YJI?s2y?98a)C_t0SgpKF zm&1>C#;pKk8uuKUm_QF;O+5;J1WFAtIAMI!MTO(2YY6_CJ$-u6*+=%0GgO!ZtYip5 z!wxN{QY;A^HMXQcvjv5Kq&L0;)-ym_$!karOJgl}lnh+~+_Z-R^6FWb!;VzRppc{g zvf7a}Q1tVYe1Kpg(?zd&lTC6bK|m5u}2&%MKs5 zWY&fm;l!6csc;$s+CCFHy}Wz@XSm!8Ud2&s3c!|XtpODqU(iwj)3Nf%0>C!X0%Oq= zV^qMR5uw2+HA&L15sIM$GcF3FRLlH;6(Tu9VqWtt^2$ZeMh3mjDYk$SJ^2INUr^=8 z;x!A(k^y0)#@Z_BLOB7?dV`Thv!TMUgRp-&H|tduDr1>sI`khvQWpXnBpxG^;-OHE zh|?(nB8fccD}x@2(p*@ehV;aWanXcS}kjbFNr#0H)% zP(WZ3a0pStNAaqaM^dkjuhQgNE0n2TUR7RC6Jw6b9UJ`D;>NVV#bevQ&H z!chS%Qm4bu!HYB6=n^%hh|`&$K@F}@V)U;#oEu^Kg+U=T74+!*=GK#KqLhnbq^ZXN zA0CA)LwXkg;zO_>*N!4h>W7J=C;f^Rk{)XCg#d|pF)7!%(1ekT_!HJ?RE45UJUyi> zWa#Pj)y55hi0x{bWZpHYE_l1gz zu!w9hLL}FTG!Nm%CujCL&rHN>wAfB#bfXCvT^%M$&{zW{N=UW%Ii+eqEaFY0+woq( zB$g2|5!hMLoFqIe3-A%&CiJmZ*s#h$I7wi4ryq?BqHv1m&=eaHQvx&Ka$SfyxK%8o5@T(T1*-RzUtixl;`kLP>BN5o6G1MP=~0NdpK!C(nyQ zkZbsd4AWN1?k6s59WPsE#Ezl5kOx=@uAW;l05?iQ$xs zVX=bKAkRvY8f;eo#3&T7kOYaCl%OE=SX?B45K}{H4woEB-~1FZt@)m(lbQ2LhQul& zIWa-PVo=y4uxuTa-b3Tlg&Zh}TIGN;h^VfaY5{`@fzb`|T<5q#=TAgFB_^P}Xi_O` zPFJ2A>4lF=1561KR2X+bG^}J<%hVK=Ez&i9>BVjwRa`wRp`Sw}bj6h-wajqJP)Q<8 z+a*GW>pr8SrKI8-80UA=SzUzHQV^CbL*qi@|gcp$~>#{Tk-YtWWV4ET_ zWgMb>^jGU#+o%pp_gvL@QAI%mQWYwzo+og{FsQ1!6kSv6zyX5lOLLkb=tY~QB8v=a)DSPjDF$r99l;% zUP3m|+)T<_sl3Rg0h>B&4C1-=>{X{Xc%QMfVJ|DpE!p5a6Xb@E}es1yn5zR(-<~5Cquwg?!B26 zu3Rz^DKr61L{TA_0q`f*6!|Sz;if-z)1t;a;pnay$ekV>Shy7fMkTVOP^3LCt1%&0 zz>Z1@NCPYi!47e{`f;J7ey?@$RFBRNr4@oqZcRT)WNSK28;tl@r5g> z(o?W)N_t8|!5Rv3Nuh;48V7_q9rA-9)}bTBXtNY+(b5VUqPC{Q6#l1RD9^d)Tr60? zlp=j2lPm@~(F9KchEK4VewYj-mjV{;(5p&x6%2q@dMq%FlnT~%avs-Gv_OYrl!do2 zsV9gXS~MtokE&1|j8&OWV{gC+ExdZB!qC+Fj+fX1Qn4iv`KQtvL0|{7#c+x%nP%Ez z{3I3|PBiLQWkiI)mt6v*CggB5Q$D%x z+EW4AJEBW!SU#poKyoS;f5<>G)0*ius$!7PK|`n`4^{`;#dyllVrW&?#w|8xum&XZ zhl9#kh!*rPXK^HVp)pqubYNMV1lrto=8)C|I)6r z6u`*Pg!}=_#KXM=@BX>ClLYz;L4M1v3-!%q8Wf!z{Q4@k(L#2qvEwO%hNe>3Luq4r zho=<|krJIEV}9+Zvja@8h7DLKL;)wkq}O_d2{k*uO2P;*Ur)&5OhwTKslo`v;OH!n zE)G!ONDmcUPLb$T3XBkiDQP-eX(Pv0Lf%LP0L9$m)nyo+f3&JN{KHCK689ttD9N>K zOOWG2m{vGePL8chaT0j6g!%%kf^k%&lvI=y_@dM`%9j!V4$@wLO?(g)j&uQR0I|L( zi{kbeZe=H=ZpeTLmbNg=kaI+QLWNV>aV2ulPMZ-iM6#YjJD;FS&>T@QXcI?cUY8?v z#H)e;$uvd9km}7V*e0C9jj<>$RiQU&%sW){EGWJfS^i-Q!C-B)3U+fDA0d=sB!rX zvYdx~<%}$e1mULAx?-h~BZX58$Z>F-dV5-!iNLi_5H*og5f%n4>L8fpVGpPwg21|G zaJ11oph}M&=qW|5X5nZQNa9+_4p&SKaz+#k8Uzk1O4RXvo!TdUQR&!IIb+t6k6RH5 z0kxmtfDv;GiE(Y*U3lqut(HUgraWN)h55pi0psw1dUK!zvOQ=#vgWP>4P!R>P2^=I^pSjcuoe z_5y%Gqi{*69z(?N;zY$vRzAE8StLabIqr;)4~zYc7z7R_@&g>-EWYD zZXZX9Kq=TjHaI!9S=pj`6_nP5sWV6plET{H^EY7g;i)|Qpd*S#eHyYl!_h-LNjfYi z>c!GBF_44YE)Y#YIYm6vK#J{Ff#^B(4fw(*VIkib0<_2E#72nXA!S^U63L4Wq(Q4X z1qk>RF3c(ovsKy+;8>6}4bkX92VPCqrb28*GLDIEMRh6MAb}b%Sq(ds6;)v^?Xm{t zjtZKtl_6D3L9<-!<)BHC3e1onq|)!y&1j4v^ddMJYRv>dj&TpY5sA7lM2I^biJK%un zn1{YsP$267WTKksRO6PZt_2$sKuQ#GyfUfB8a}=On|@_bGo<2bt$c9-5;V z$u2yc4qc`|9W=eD$J}65Hfia?Y}KiNVhhZ*hK6z0%RB}kY5^vsqJhengsr3i%Rm1{ zU^%O@6pUGbN{0axsGz}?x4qj}tEQrgDdQuE?Xg+%5^2l=+~Jnyt3%<3U~tJaAYvqM zI!2IC^B}b-LVaR##iao#ca|-Phm$q9iZd5R;J7ASv<}7i*4iCVKAJeuw{wJ=Dwbo1 zBBzuBxXBh%EVQ7VUf^36b#X!*a?BQs-%AEQxS$Mxn!i*-#B7zKoX`%PVJcPDWmWp};5CR4 z=?09O|l#8xVN>GIwSdmBoQ`w>@B{GcJixV`iojEcJ zxd~|?`z((oWH8A>=1efU6E6`;YKRT7B0z~O)EmGM_v;XpXpIQihfAU6htl_bozRd^ zM8qj{XcW#J2{wD0{s`3lN1^*%qK+K)Al-wje?WsP4=AG`b-he0WaKSih7|3{mNzKZgi^IUVaLUjD`Dc^2_Qj|hbc8^?2E1VLNKDy0&6F=X}zs4 zFbq3u5Mcs>1K&nldb^%LWzMOE6-w0vsJzOh7ra_>RI9SK&d5?9eFIOG2hJ8s0!FJ+ z5l8A4yHad{S|Nj00ARJCB8(yh>`9B*HYU|rgWv!Pg>djd2B%#)regAtj751;=#*@V z6;6ltgL2}}CdF5GU?kh25j|w&g@RGUl*{Gr)ocEMEOD4p;WYd`aH?CUBEM_8EC8sK z$fCEE;7PFbL!5MztRbWFvO*~ljucK4Yq~Jb5uoM*Pz4A4m2E}S0mPI4C=0$eP$q-I zR>=5C5oBSs(1Nl=kdTp$c2zLULm{VSNi%^_7BYLkw4I}sd;y#OQX`mfh8(IktvD_M z4}e_H(aE6mduf$jc}rnP0x(5~Ldub?zPK)e8zzXCrx2~z7#u~3VE}bK3Hj)z2&m(S zS6#8L@6JI(gH1+)TG1s^bWIB`2jV3+p|$G5xeRtWg4K zv2o!7rH$<^Sz+HV)seQ)@spbN^_v$tt+54ga>7-t&Kg&KS0YKMkW!GomhpgY$xC6O zX)SRdCz5sw&8EV%_kb&DpO&_QbN7PBg=wTivwD|bIN*bH%ayMt!rkwPTC%EYUMl*^#!3k6(12?87+ zuyi26%yZB9HYrTqfQ(rYc}k1(H++Lsq4*LI-=$PNM8X$|DKx?&6gd0abnvBw1u&RF zMW$1`;`3I(k~?A+ARJXE=>tT$02C~a00W4F{eW`dB_&l!Ey_e0mGlvj)Gpd7!4RkI zxttTMOBKH90r1ppHlu4&Klv)a00afg@F}Bk#-XGhrX>p297P|&>CIT4SZckzo(S8 z$5jC87D7uy#5@a?HJV;GrbTnO_)i>4vUvs>6f-BqK7fKJyKu+FHCj)llU^x?H8Mji zNRFI(w^!o0n$ZjY231I^f}}`)5eDkuOBAO^@r+l=pbo520ilaxqh=~+8z~?)rl2qh zqeyaf^sBuXVG*4i32@QgE)&U%4o(_PQa;HhSx^)|Xr9Yg(`F)2fTSDrP`s=jk&-jg ztw3;_u&B$5mM%e|>*jLJ;0Z1UHen8!hLO4|q|m8Y(GzA3i7sgk;3elwyxh^)P#e$XOCOeT5$}}%p?C`lQ%!MT-QgOgchTtK>1Yd{= zD__Vcqw_!nS2AhG|5ztB45UJ4t?p$F;wt1MHSmNeEjC8Hp02%H<9m=A*-(RkKk^yU zh-j;@9A&_Ho~#vuXT|Wp`PWg3CtN(BlyAj!cwOjA-Ny&>w_8+Dz!ENW>E~8{627qj z<#?5lK?OKk5tEvvGXpV)%+|=~?%FF@hJ}_T?upXOK8|l}i>VA$y0R~pZ$;7XI-)nZ zdFVzc~3RwN&3M!oOYr!rFgbnhfgFZ{h0cO|?%?5qwgrsI^y5yAk zrtD~>6aWP3P>+ONE@%ykJZVo^R5+sxT*XSrUw=$4y#_EO0YifzN-gF$Qc{;JUDP?Y zw4DaJ7=|=<*aQZC{&OHOb~3Jpj&RT?`4=F~AqiKq(!@};U)YS^u4B9`OiuM`-te@y zO%-!toHTdB4)r0t8mg^MR~0>Gfd);E+axAtk!T^i2rmqpxFoRN)Y&ilrGIU!%9EiQ<_AyTk3a!7$RgfGB?5fvb5Y#ee* zy+?^a#)(=f0ZTjFsCP;FlMyGh!U&gp0rI8+SL%E*6^}Kd6-hdjeSa>!gd!lKjy|o_ zQe1Rd`}WZfzAYJ0vL@frUw8b&(O7bUAbD`s%(vtjHqJUJIApBz8?@SsIm28omUBJ| zL(T8e=wc+NJvPyq_z6r76ogBlKuvNC!#oD;V`QM1zWTtv`3^BGb#c0SL`uY9+Hx@Wo-Ag+P*CS_)sX#R4rUZSY#aXV{jiNNIUfQ-(EwC1WItqEGtt^Q5ei+)-ecv^hA0iMBAU zqbHDx!4bQ0To{v6>WY*ORz|^2)K!6`_NuVp9%=LqOq7&SoF5ctkTbXtLrBMn9vN7r zX6_{p6{pb`vv7t83dNWK9z#VdR7rBc1g--aW@1xVpCv8T)njhUFoPZ07A$69hP0yy z<46dyeDD|M2FXJLo6sF5g9lUquLHt{K!(>HLnz|&0K%9bH`Vm`Gj#HI2 zL@B8{15pR7h~A@%MjAJ(d(M|vL|=88=!8%8lO(F@ufRZ-0szOh z1Y59T#cHa_4<@%|qV~SIuJ=XpNly4mnbWdp1un@oGN%Pyu32kvuO4uYFt8hc49pmq zeoG{UoS|5;u>!v5-$JOwrNLN$Mf*=G^oF2d6k>-{VAMZ`ur5rEIa~=OuHkr{8mGb3 zM=`h}XCa_--C;-!kAS|0>7)RPX)?o2t)2O<4A^fAWqOVrhBXR;gk35{gaJ|RsT&j! zkr>wv@g!Qr=!T3ixM+$BK%u1SG-tp7KpL@)>xawnAgLA?6l+VMSc)(_E(7^yp2G^} zNZ3;dXJRlLTIeN0CTv>mGdzU~ZY%+s`f3zX1;!v9kZNUHh2aey@KbRunsypZ;$$o) zl(2wiDJR|mQyL}IU7nQbqGAlh^|}}!BBG9k#WnaS7g`IEATbSgC`3^0&-j#1Yyvl6 z=z=NYwc?4VXeSo+X}LfZU9=T?A(RMu1(}IJG_|=Xsi{k(xTH0^ z@jL?@5lGL>H_N=mpA%=ZmF2D3rt=MR3r!|eKfL5wUEbQ>-k1SJabeE~yBYq7#_H00fLtWw^0)^T`aYq}?;TC9g}xb{295qFcMi!3km{@a;H@ zyor3V1hY$|QQlt2C`{}PN!sj4C7u8){lDU&K90awRLk}1{AX*C>?b|Z3md#AW?5*E zl%ilPRyi1*Q{Wl3(eNkq#wq|5iNf?#e z6_4w+@C!#;CCc-a$-wn5q>Vn3hl5?g6vjfuHoB}tI4PuXDkw$3tHK%6O-6*QCIga8 zOwkKCk{osJK_E>hx{_%K$@V>}15ca0U#gpvC4$9AL)(Bs0~urEA}RnPT_;`%HGry^ zB!hbW@TV;j$%JH*928;+74LL)oj|(EbZvgE1Jj;eV1hZ!Ya3WmQ5(0^6gJf!RG)J! z=e-Z6CohN0rc65S%Pp^N^2zlA^Nb0GdC0kj*=4q|v9Z3%1vl;Mz--XS#T8d%EA($| znCwqxX!Qp}>$R*QC5XaORt#(VU-k<~K{d3iQ3q3F!5W zcU<)mxHB$nR!%e1GT8#Xk<|;K;xAwmSv0;<&P8yh=t zVRMYdo>9RMA(3_oh}{q;RuwV zDOJkklcJ1^`34HPP>615akb|DG*4qRuI~r#1VZ0BpB$=+*MTN z4OSV9)9^MY@$#~yV5TRaNXE`be!pgUW1V@1G?y8OIcVk;(c1b9!vf<3!*35T-7s~{ zHo2_wB5P%3duPjiKxaH=BBmrgO*@xsn`<)+v9U4R-r1h5uW1zE-8ba%+zZzt7cQ%q zyiB3gcu{J%9^48r=9wSYB&pHQ=e#GH8$U9O9Svo`i07zcGQy!T5|;47%nc3mshSGe zrA{oFnCXm$B?AznE$kW-Yf*?AS{V(vgLdfHJ9RncNC3QgHruH}rU;PO<*ap1sbHwa zgS@g$UjVr92fIcU<|!{eAr3LccWE!S{7Fy5pE!K6!#yQ>ZTBxVDxN4aBZNvNukf~| z&>nf!(P1C$go)MzUAh)1a4dy1c;Gbc_O)xlXi+H`y#eUwPidA2im6+&)k4DXqTfPt zQdMo)=9ot>Dw~CYESvE++M=05d{NAw7?F%y`9*?bjU@q?P44!zmZX+Oj0SLt0Z5%A zymbgPa#EB8u*tX9ne19=VIekAtSEVHAS8%$T*i8$6^11PJ+hnW66T^oD#wzdpGNYv zjgxFJHT=MlN8ke{I~2+DJ!+)lPB>_lRl73^xI`%2va_H)Ef5caBQGjp1VfUXr$7d( zG5}+w8eG83OI*?W=@CVhi^o}!Qeb8lLR?F>FxWAr8mC-w54k8xuWx;J_YG~D(B4}w z!S71()^UXk=3eNlrJ>MM2*VUQ7FHlxkdaL!U$3G^)^A8ei|VD~P{Y!Qpw2NgTH6IT zTLo5U;U^RYMxg?4_-vIq2(1;tCjBpj=T*|PpA^i|KiY@I7?Ekj1wNRJxzrzN zI3x~TTr)8++pLpEnP#v8lLFpkl3_x*v9)#c)-KmX*yP5Ioi9H9?Z5lo=l5^jcz1rm z_NN&<1`Z89;M0C(54efPp`CqHi-gp7U{v0uk0iSD*1uw4I%^;zLshC^;zf5s0w!>5CQX{OCmi$mnKFcsafydx42gxSWO1^d5Y<4Z9VpX*M&mRc z9OP2I&_DDTy_>m$UTEjcamZ%fo%EQt|DzO$~1rESaLdC8WFfC@pLU(zcgw>RsRQoW0b`pe8HdBS;F01H!L8gCC%9+n~2gxb;L&-|Mb{yjz57@#Xp5k z%P+ML@JTBKZHorLa5);TcxXYJR(pQ27!y-x=Ta_Pn09&Sdt-zBG&Tp8cD6S@`|X2^ zi~se1`_KOgsW0DNnlCVP;(;UHN#0}0gZs>PUi}jQ%kQ|@vT6t_uW_n$Mmmx*Ae`d~ zaiUmw@B4v|L>x>%DVm?44J6(GRhg-z*%^x_+^OmzY&3g)6TI;_HM%e|Y>{}bewh67 zWTrEy1c_gQLmeCg$UU?nFLJ7-$`^Dc%L#QKrlT?%fAc`5OCOA3`mTl5=KBXjpBd?n5`37$s2|-Dxd0QqYxJR9HaN(G9%q)fc(V!qYK28tqG}(hM%xMt! zFJ4f3FmxDBQH}&y8JYjGdFEhmv;gi`0R>YFIx6|%*h=LX(gXXU9TTm(SG(yR34$&-s0aj>LC<;v=&Y~AEi;aYKx$75_>2UtfId~5 zcF$;3+c*RUhqgAj&ji zxH*6(_Na?9t1jnH=>WOR)WT4Tvtm}%LRwan`8F<1G*ljV*csy*CU8|SXyO;s8CT)3 z53h(|%{-=-$WEfrrNc*Bl2!nx6%5d{aZGGDA5+UhM2XTh%p(kpSc2w(7hq6avy`7S zC~UY)*!XxM>bxhdGT?Ag7-zHaQr`{y~aU zMGP3;P0eUfJmM$*DQ()Iqa^&I=!MyF2#{pz3DFg(N%`9WyiM$c9jZV*suGKUy@-W47q7_o8Q=mIF422er zP~%}v9RPhJPw4h?grJCsS>`cjseyyFMNxX#lj4BH!Cs^4TQLXYI7t$rYzGB8F`&86 z%NPWR(IBefv7bUopPC9=!bsA7k4=>NpRWw33;Hz1J0k`-SA1GS0E+Zd0H;(J*@daB z;aG!Jj9e3e9NkJt2kKA35EJmAL?w(u5A5SMF8#r%0pK9b0st*`_J`CEk6Ib&Yx*hp`U5K)-=TU zsWfGD4bJKUs5F}ZwBORw>BZaQlyDKsE$x+P`d{O29b`}G=Phq7$U(>A2Egu ziA*WNR7!y9oicHz37XzwMnXc;mT#hi3*Z_2yrzdq)QI$2jwn+b=f^zN2nY3+pz8V& zRVxyw1PMBlgS~2#`4GmnjKz&i>ABiM2;@}2K)GwS<`30rjL?P;YcD#Tds2(8damcc+IQYeyL9PG7>R|#>6HlR~+rlSeZCa9!0pMot;AzQx2 zfHJ7wR6LE?>Y%_W!4_nqLnVshGRL;VGB2Prl{499{?S=Qfdh!a&;|a-+KSnWN+>ANRAdzxR%I(3uGA`K~;AWJ^bQ&B$b%= zp9laI3jokuQAj8Ps6r_yPE|9#m60I8x-=S`=}{EOdL+BYT1E}iOypOKWCtBxyGyx2msVf4?A4!65S_yH30&9m?Tk@!hz?UtHx2sY2H|}oB#ZLbF?F| z7_wc~XdHpfYCbOj93>e-2lAbVG}t&5SQyv8T+j)cxFM9M3g5L1O3Xcwp~LAfQL7Ta z%;#U&Gt3iB%Nj;~w3?ZCV`-x!q`&`)hZvBoP;erJt3uLYyoTUR35G*$rm>)7VdR?| zGD!8A`NO4;MQ@^Xw@kpbE&c(#4lKE})8bk|XZs%#X@R1#OOMFZsWh6+D&`#u6(WV) z$NqqhhjpD(Ou480+r9gU? zP}eq2VqjU1S>VX)E*u4{{1iHubTNW~9)%#1S50$eKi&KWJ_gm8AceTRq?=QYlv+c@ z(++SDau&PilzvN5t|km>8bG0_ii7gle-fhb3#Oa`Q#z>(snX(s|G!k!CelJb3IVJl znGM4yV;}N?Dw$xI7bq&8-ob?vx`)#zsXz;u2L?C1%t5KhYfPC#1tO&R#m7Bj9G%H^ zwS;XTs68>be8D#*86!rpt?>wG%?4`|O!jhvZx7667XkU;pNj&i0~>EfWE%V=@{RfU zne)m_sstR^0hD>Sy%ki!V5FoOINpSTcY*<2;Ux&SJy8p6$srJs|Y(_|z_Bq_ti7 z7BMkF@?_UF;8yi8Rv-rj_fPu&Tm!Ph3#OICinjX2*=ul&Iu&(69YhkY4_KQ7k;uYT zGDg8hJ!RgZo}z}}#wYIoIMm{Aj_oRnZnZ}~gbvnDJKSKxk<*c+8W;3R)80WpwsDyW zTti1%n%XShk=-mqV=3C5z{1xABUSD~ifayad7d?he9?uYLH6!~ci}m%AeoCB!I^Lj z*s|3Q5T<=m+ND;Rg5-+~p}ByG?kFW+dO6S)#6gx*+ar!f(EznKxmCB4YZP%i@=g~ zG3(UtQZ{y_Ld_fEU^jSUzn z%ARpdr)k9Vgpjd1CYyQ$sRq4-C&Wc;fH`X9L!rq?Ml>?N7KunwPk{vW9t8?fTSRsL zoHk{CPmz(#O$$&Bg#v&$vOld;sT@4ZSwjuF6=cJvcWOu@c4 zgB6{Cz#KNFn*kO8jOBR_A*x{G84b0J3p&Ddt@_EO?Fi*qHuw19q=-imQW(a^7}27b zirjJ0?)4Brozmd0<%ZU+$Rc@_(=j7NU8&MERHv4L60rA5g^JUf0b$Q#f{9FR@%U$p z)H2H8QZj!Q{Hn8Jpoc;xRzydJ_{xc5G+&{N7VTdnM7{Kb%Xd`$YLC}3K0;1qW5djt zmKumvj9Ft8OiT?6$X2g0R3R2`X=X_1RBs`|ut5{Ar(MW+4tZ_OoJAX@tQ_;DtP6afMqmmzTUsf$)J^ zE^@{`0!HAJc%mS~q5!6lxSy>sIHpuDP}++n1FxJBkYOv9+qe!Ud6En_N-;}>=-z=C5TQ3wh7ua6RnZ7;6@6Ry>WqQ;aJbj%Uh&K#|vs6qD^6wpner4}NK`d@(kK8RtM+@5BtKPeif|6>^X z!&xYvGMn-Bo!Ja)#5&&RQ$3xhccq;nLZfR61SWXLn3N2J_4K5;;QgV4xEa?cC;EuC z@n^D-NF79_y!2#4K{-RvbSdM4!a{?`ZE>W-4?hBcCqeOXRo-n((ByI`$%4>I$u%6Z zYH*h;D1s_)c=d6;FA0+nPFfHc=iLm3E4PImj6zVkf#g`$9&@I_MAbeX z=m#cn9ENeMjOy@nNR>7614${Pa%l^7dJ6{q7P<;EH4L*XP^f^#M2FjSIz`Ppmmp<@ zj0+Yds0fmA=i?(Krn2z5ek|+J9czTw*jdN2LQ4q}vOu|R`L|IovPy8$H&l|$us=u` zLCkgeTBf4B z0rM%!6dL%Xv4S9%4oaKaP?uJU?U9GxR*bZi(99%+l=V@(CT>K5JUIO;ZpnhE9XJ7u z>D_yAYF%E8xbzU4G$@wP>|KMP%z&Xd;-NdYR|~8_P-WaiJb6Zzv=Kv*WFo7XDAiD9 zLQz%_VRh-f8K#-xP2gyvPnN(Wca)ikA)~cqxESNfKrl~p84db0st(gBxr7|N?7d0j zg40enL;#5Bsyu(X<3-~?y9|$A!y{ak?yp|n=fRmu{r3yT@-Y%_oc7V-nm%|&L|lbC zwWH><1!EF0jwJuGRtr3#jg_ea=O8sWbsFkpZ_FPM=qN;gVBfD7J`Y zoB!n?Tk2Fy2ecT~GUOQmmM!V2%nF732iYjXTr=km74HH_2+~Ch$XA-kLQd{Qv$F=6 z+9m+$Qk0|+2&!-0n%%$>tC&%>^Zh5A<|-T6%??dVwjTHjY4?Vs;#6 z42zQFH6&kt6`vAU@JZmUyoKVnk6t0t0l$O<2-`Jn^j(q*P3*L$C5; zYM=vT;3q~kuV4{LQZ4yhppV9WKw%L}?{#n@Y1OSy)g4 zmGvc5w5YQSC?!}VA!;49a2iNI_AJx!&HrF=xC-o8LtL0tF^&wy))^a6usbGdU6PR` z0VW{uNKb_#WNXtAA(%03O~;kXW89(g8#jcC5neQ#1Dywh7Ih|y^i5ty!oPIBF_VrQ zIwlfYI-~q)7m(bZL9y#y3;v5nTCUd@0KZf*gOjjw6b7rsnARnPUKpu1v2A3qdUNuM zu>5f~11+{z9Bxq3uRi6-^HIKn;(}aE7-3|diRFkbxd7r~C67(ZRc3@8CL}F~C-wl8 zXCaB@Hy2!#vOGu}#&blUtDd?d7vS)9MYrM;mR)MY5r6rgLRcf9s|Igp`e7rt_gz4c zzj+xDV(|PtcW8BWLWj$4weU(=MIgwSK^F?2^NY**e2%8EmhIs4AS7bB3ezfnGY)2} zzK6l(jj@s^GCQv5HL4(>!IYCDtf|mN;WNuL$5`HV0YRRPMDTaKeZXl(6Z?1o27wM| z2rnOl-)l5>*Fs0)%zR8n5x$MI$r0Ives5eUfFkJ`iU+vH0b{Ju3ygYcNa>^n5SP`V z8m6I>76G8MsB;P~Eq3Gy2)U%Cf^m%S5^sBfspqs@8946E(!#xP@FTT1=lOHxz~H6O zpFVmq5!Vg21&iZ#E>)aHMdb({j&PwIcpW;5WLd*b$%59{pMLjo!WIGWaIBZ(j##xE3rqQKwz-2a^aSs?GT)#s4r)M#dp-SV_XAMvaL#E@DI*paNy;t>gKvm(wgDy13Vgr&JD6u9)X?Ir1 zIA!q)26YD=cc-{HMrkaXD6*LbO2(DhB=T)DBNC5!duz3GK=;d;h707_!Hz++vJV!a0FPpn63Cc{1347t6kVAOU_(M9nZOLIG2fWPGk=V+ z2-Gi>&M|Z@DvPP5M2fc%gi|y|qP8OnI%-DSuZXnw&m0Rcofrp5TNq&4jdJ~%hH;?X z^ondqumGB^u5E5^sjSc5=qHx`*eOVQ?bWMSPaZ#c{rZH53bwYkcDA=Z_~3(Ew{8NU z|D!xc4prk!M4zT#_0=vU>ec1ri}RybM?2fwpdTF`kKIiMDn;ToC5VpY_1#FM{?HBH*@#;`~3p1bIc9OaD3pCP* zeOH({R#z_0&w1vXX$Re2AHN0+0}@?q^37JEvjlT^U7TO=0y^GD_vYe~T}F23&<8m) z418wSr}TXJ56RG2=3B%wn8j|uz_b%p;Wb|j0xX4~L_y!|z{?HhFyfu9vrFEe3L=RQ zg?L>Gmq2T)e60afA_fHg-cQ~nfn6c8AgF@PG}ETJi?D5DJCjagK2(lt{}7?g_V22r0m&smK_KXGV>O(s1*D zM(4kv&(Qg=P^+xv}wg47vRgT400+9i5P^`aT0=d5cCw& zm?)Az*=@VXC4FJ>A;bLV@%1b0{;_F{os*vch%E>(kT5-)SnlyDPVE>}+mq9_}9=9v|Jj zeQRg?=Gn<>-Um)(^3Da{53;nv9I*O!Wqo7DqN^`<7a0y{cFM*4U~>AQ9OG=9G-gN}T|xG>O9n z0D~+8+)=_GmsEKO8;+c#4m5-Dc6EJ&O9_&60tesuXlbJe5(ZG@*L2G7`aJYyAn}~G zgwczALbHvxl!(k<0?4Y=0@x*AKvXU@}>7^n@Ax&UUXlbx0!o39N3I>Pa@55u*IhWC{V4Nyq zKz=2}l}AjZf{5joYtjy^b6-GfmO5JUNU zs&HwK3aTq^l#xon+C^Pw)0K*;HAUe3)22QSYO9y;reTbfYf7ytH3~*xI5?!<_$?kT z4_SmJ05zIA`sHZyI^<&K$Ws|mpiDUin0q!cwZ$C2gC;?UE zDM93lbR*l;Q-D#xwM#6sVxFfe>^RyXDd#a*VTWlyEu!1=FD}|6p1)3^1lx7J{a;i6 zsgT+zl4DnESF{tm`DTZ0F{a%_2L5769tSWm4N);I4p{QY;nDG{moHyEe}1%aaDH~e zBmpMho(0aE+mXR>;PNCxh>#<4{i6Qzi|5b3{_dNHKRkT=<0F2WjMufz=jULsm$iHI z=Et9WjOooj{P+Sd0(Ed2BT^>vNri9TFwgz|_rL$)haU(9UZsuW&o3@Me(>Sn{>Q)j z=);e=R%pbmuOaH&!=t0`zWw&$_un&pF>P$jwhj&tUcWwm@ZiC}{hPnJbN4PG%nXF$ z#LCgp;lqbN%+Js7-nsMH=bvwHZ9aSS^u?6@Qh^@6%dEvfu!R~LaBi`WvikQSv#zD)CAT)erYg?$4C4>yz; zej6KPWJgp&++)^RRtRUf#C=vKc95U0=Mwk2#f-))q1bY~Slim2Dye90hnQ=r?v5_Dk^hn4ync| z1jRrS3SF=Q%&Q`oqtk2?IRVi13mX9!U2v0I*Q{sh?VeKhw0n#HW zAtHzPh@Ob8YyyjA4vRKXrz*gL8P$2nNdbMN#X2w~#gp8`ay0(4K8hY8eu{U!@n6a4 zXShsu`6<_N&K^x`N}*XibAQL5Ghq8m+tPyhG^8%^=L;_9?rO5RL|50{KFr3C9pr&U zya6EyG7v$eLR2s&FGA)o;K3=I-q~taf*A z?c#IxY55}r!C}>+SO&)$uQ_=7^x0QmfA#R;_hfwLMdad&iOKI{%7-5j=Cnr}M#Sao zqoYSZ{_xdTe>^-nRX5ANT8@ZkeyB8*0|MkJ<@H^IJEz87xjb+{voYTrj1W%jv?2F!4gb!UP2~rvV(p1fh_Qnko(vQwf#le5XY3G+0+-EvEIXOOheR6!l-WQ^spPfE?{_M%)r=NZC1(OgDf4^IQ z_nO)7_;_#c;N|nZSIc`(o;}%|@oohlTEnPE$H%ws+`>+Z{FNBf59)l0RylffBf;}^wg0YQp_~FH+Gm!x3{-( zP(272W;w1M_l^#!m?ylD^*Rj6`N#g@>e?2ukF6PM=ebG7kkXO}nf)bLV}x8B=niAz zT0wr!Xw)(Zo5mXTb}cHL!Wzdx%KFTEUPw}%$Pxymv>cXvauhd>Womn$D$XAHx-kBfUsFJQa_@fTYZw$^;TaNHn&B^P*+sQwwE^rePqHa7kA*Qj~BV0O*m1 zfs3*b0LHqADWbfIF9c2~S4eYP3u?D+?lM1p`st_l?%l(iSQ1_+@#59q z*WZ70ba?0-)(c0xv$}fg*6lAo|6*fn0}C))A08dD^T{O^*Rt$z`R1#&Rjvp(HaD0g zxS+(*ZY18k2{SZ8aFpT*mukm!qB_?KVDw@)D#Q<0hmgW>ipMz-MK2LW8cdGXP^KM8 zK(qO)C=fBy0*}eSr57+P01Jx3^umHLy7trnWJoRGV)KgDbb#QwRIyDW%oF4swQ)=! zIJ|V57pBfwY_B1n7h7)G877epbjQm;FJ2Zj}m#HLaT05H8(3`hYmC`q<93y zKs;BFl>kBkN)VwrlESz)zAgZ`4%Kor#8j$=tR%Horc-ANgdzU)Pt<2ofdES9J|0zY zEzl!A42G(cK@syn0(hMZB|HM^kVlFo6P?%%)1mM9|OYjCR+)@b2WecwODY&1ap)!shWi~d4jzFZ7V zE}FS6yRox_DVRe*M+JauK%ezUNSCauA}+<;3^egWmKNkR9C-@n;sP@j7d4fPk~>?z zJT}uUX^J>N9K+mLFtaBQJBI;G6Vf}$c!GXD3S6Ew@zN@G%c&(80`QCq+%mN%<|D*-`lb0yK{85pyVU$N-~pVoJ8QDq97T z&}H;-{h-UrNV;0XWJP{Q1Z!T)C|&s2xu^-BLZidaA({Ex@r@ZQg$Bjxr87iSu!VwI zqqv6R_pUN;%37DmS8vF%Clx8P@8Byf)wH08A(1RKtmIiiG=+4C3;f3#0QrEAgz0fb zBC*BYbV>bH3ffKa@axhyXD3=Y>PIcN;Gj)Ym~2#@I;!&6U)=1&0@oks&5Ughr|2eZ z<7BYJJ)EVrw>$3jd$Wf<2QC*9Jst0{amGqH8(%I$@DNWu@wCaC(~}dfl(?KCYjAgm zzKt0_m+dYMjL1TX3KKdN%IVqZixE?P@OU$eU%(Ki z1*Z~)>H*-Bf`sUHTeJ(KfxG8lAFZi0si2miXgTq5^;UF7=y*@d8^5dhgzxo6F zpA5z~XK$GH5gL>6>*h5VuD-GJ_~i8U7w2Do_`o|=P$QBElM->rnl)0gs6z~hbIkm= z#ANfwKm38MQNoEilItfXjT^hWzy0Ksk3aelf`T^7zI%6keDwIylUMuu_>r}W*Vil{ ze2fYe1cMzCGV?Jj@9gjswQO?oVE-D6Cg$VA!z1QexNmK4+~8TtO#pDgC3~&xqar2i zsq^_6YapKN$E2i6)L}eCDQ{|MCTAmKpRH8(8VMLI%#J0qAWne-0%R<3G3b)5X(f&h zok1xUuWF(g2QXnWHrytn~K78O8<*o}5*D@Q^-vxt+lPg;Zt zWMc}gYCGB3yrL{{Cc@R<5S1rYPJz58fZ~KXJphnZ*!C2_D?D$FS(G7yQ!(T75gaDk zFf`NSUtnYr2Ot%7xbZKtS`U(PQ;RMNRY>BHM&;zs$8wc1Eis0|RXKibN7Z9+DEdtg zR&Qd7&3vN_f3;G!WkLN{-U9&%jaF=g@~qqmNPD#SBGCC+e9 zFn>)#X=(;b=0*Uy*sXb=w(?zBWD2g>ysw_?60d2vjSm)y_0=oxYaD>Gv1uZ9{hPbl z+{@=`2dTiqByeY5gCKJxmpgoC==QCf{9|EF?t1ld@95<-CKnzeAU8gH_VUrA?~jiD z7IYrn#tzW0U?_%N5S740T)yA~z1i~Y=UU!& z&ShN1THe29Hqc2+OFa60dUDF1*z41itj)O}4xSHIyxf0Ds#HI(>%AtUetCgU{H|UW z-L5>3ULRp}>h`O!N9(#kVJ{7r1t0KE^nh$F+il6EsF3MYmC= zK1-V%<+`4mnbvDDGn>#yRQ?nnnw#pr>{e#1AzWw%x zAAdk@e*PR^pP!y+{dQV3Tugwt9XF*@GW`c*v|nl=Ga(-8=XGm;dr#2-%DVm!;m!a`~w?SPS)7)a>u? z`$Yarp2(lwy?2-3!Md1^*^YURvbDXH$6#DlWYpkjq#QE=qVj#OXV0E<-;-ucN05Wg z5qH1_`vaWVk0@FXo0*d|<&<+{VCgg5usY-wo6}hPIy*Q5#Ar|m-z>$s+k!k;Y!Cl4 zlX)jfYPqIx!_BR^FgG>JS2S}mUJj0*WK6O&mO>uq7Lql=8n-~z9~3b6`?V^Eln<~b z;i@3txFD67dnUovPN1mQ!SJxB6%_(LCFW5UyD7qJ4Iy(>N-azaMwU+h-^d#+e|2uf~UFNPEH~xqXP5aty{N{h-oJ4JFgnx*|VfFc0OhsEL<%-%cSv~GumKY`w#z~BLfGEy?FZM>p#4G^6>j} zW~uen`N_rOC(pR=^6^Ixn5LO=Spl;!=Yevr=6DvI%P1#~<+nUNLI1ni(&_QZ{>uZ> zILxr_&D*z$;iH2Ca_GtH(??GqonFp&vu5=AoVz<{$W+0_0TU9BUcBXbHl9$Qt!>Oq zQ6w;2+1OrPJ2>H~idW2_e8rLEi6%N%-?}V-5ymGyuwuxAISx2GNP;UXTbsPUb>;Q( z3DYVma)VvF%Xh~|M{N9U?d)vb*!f^{(~;;jy@VVj%?v`5G~*jteb^A^QkIvv)Ap2$ zxcz;jXm$1V*(p~Wb3){J|KRA5-kI4Zf_M(|hSxap46lvB*C zV_|r)79)%x%lzPzV8}|WChr=vx*Q4Bfi2-YyP=CgVVt4@VdlKT6W1gR#Nd%40F{4< zZ)md|#M*2&u)u-`kG;-FQ%KBL?woP`{y)Suft{ur_E^|l%35-He{cWsqaUGs@7`U=^03^~=P$_K<^mG-Cb@u3HMdJRH)yCZ z(zq5}U;W^N2RCouWdDlXL7qQ5KH7irjBPyT8*=`fi-Pl0z9`D{N?}9m& z<_EhsRIpgS%fK>y#&gfy_TjROiw3^e^YL$gdwhDrl?va%WxH?haNiqGe7n@gue6fy zePhYUji^<&N6z_mA5!G}Ez^}V3NCc*&(~Bh*jjw^a(|EQBc8ul^YO!VMowPG;N>jQ zju=>y>sK#@SQ}=~lZ!cYzIFQ!Ns}D<;^iw0cTaq&ghFw6e?$~36HQC%bG*ciXQ0?u7 zH_r$`uHbO4(w)1F$Q`y7Uv+v(YU2A(n8eFy?{ytxX|CDGM+k`vPUi@$rBBXI=%<~K znDITN*^Dc#*Jno;>l<7}WuxrWOFs7%y|05rxiU8GkaLKCK6z;HaR1=d!7HBVTRJ>o z%5l5(oGZ&}2ftx%zJ2>POEIDpBST?t{}oTwGw4{~2*Svt?cH6R$aKgs$4xkg z#6nnlz0cB?C^WM>4QVBxNcWls`?x+pU90MgQz#gOOHx-!xn*bDIR^ZFE}^!zj(HPLzQoQYkxL-<&E}-gQ?&c=!v~*y`ssJyz5DUe zBOaTE0vlbulp(~UBXL58RAs-Zo>%^`d&1jsc=Bz9_Y&|F^3D!(19qBixJfeYK>B^s zufF+)$LJpZ_#;t>oTMSs4bQ*LFNt`nzR%$Jh}=3`T4r`;PWti5W0OCZo3nN7z@8Yf zi;Kv!E){zq{Bvz?j}>XB1Zfro?Yh}%Hf)l_*o*& z{pL3hwpg5RZ<8P%KY6nE>J{Kn+8BDr=$&pv~sP%&Oh+$j@9Auf3o69I8gSGF^^r5uv?HzDxS!<#Zkl>5xy|hUS zkQetaI1CHHKY8*vhl5EiEz<_I{F^!c3dGLD1$a!QKs<>+4$ZqcGF~+Kx|hk^BAFMocRHQj6=2& zP)@eAY|=rX7gGZvvKE5v|HEr(ACms+9W8Ax%^K)v83SQcQKEceivKFctHVe_VBZqWLWWxQd(?F$AfZ2e4C6~fj>460jnZyqs;xrg0 z;e@Rw@V#;Q%9VffZ~p%BrORy02jAW{yK=gAPM>CpqkI$0O|k99o)~*%bnhgB^LuR$ z1t+Q|S$tvS@Jm?D=g;po*K?6#$SJt97*`Q7ZXsc^h>nh5OEh2%zOc-3fcSD&E?t&D zY&)l)9+8;BQ8R?N5q%)~)-sSO8=SzcSWzPAPyk}ku~jWq5T8+|586b7OxvP#`}VCT zkDlK8<{K^l=*K^XaK;Yc^Kag*a6Ux#xJrb>0_~NnSBY|>05iie(Z(;MtBeRG<52hk z6Sxc!7~0}O(to$Uh08Y{8P1n3eMAU47mjnmS1@G3D0NeM^hYirn3 z00u8t)*?f77qTKj&{YlP(mSf)aQoEho&7ztZ1_+}W_eo9wE24S_}lNky?X5`AH|4( zVZjniWQ?CMZz8pf1{^M7mgAg}n{Z;s@y^*Z25TU&mk-+jmJ6kYJ@XED(50}ciWuSO z;ll@Ce{+j{W4Z&@g6G>;?*wFD9a8&dP#D?NL^E1U%OsEK`ylB{XG9U@oSVCMzo)|W z3O{tI3fpJTutF9)kI9}X*Pc9EcV9Z`ir*&aij|0OYiPpv^hpvFmMLMxpp0d zM(7|BdSREL5nwYM^2?7z9XJ}FXJa&Tr^bu}7~Y}cpFDno%2JHiD`HR)(Z~0szw-zQVa|2wr2+SA~|T z{cC=jSCn6eu8=W*f6X4ItpxygK#0HcrI-1xQ`cyKMcMd;(*H=oO927~H^M%oBs!uD z)>9E!;V~R%%6gJs_MLgyS}|Hv(ba?yG!v_o5DTIZ{bV2VyonVorpG}jEwiqxXij2EROw^Cn zq`^FEAIyG|`sm5i7k9qBZ9 zW7=x2l4Vlhh!xv!NCtV;5-tYd*6rQ3zy_r>;+Qf@{@aT>-@=Bt;lSh!oi-{5FK z9@Gi5?C)Q^`RPq^P#hfhlFnu;LUk3o&BN7oFpSzmt8_ybu|o^2pOo*&hJ{$=^p1We z?6FYN#Mj23Lh6lVnqr1&-S~t(uUxrG@i?sS#WTBxAIkh+%NE))SjzR$w}8+ ziLmud6%>7V3t$eJ?7UMij4?*!%kG{%d-*cE>mxWz#pZ<(UdusqV@>CX#>*zqz%+-d z$u~W+Bpy9@@ai>unok|d=)+T-A8{KYaM@y?eWduOuM-^4Gs+CScBM*RF}4j4An69+Xwzr!QWJr63gw zacBE9sYjWJp_}8O_!TGQ5O!vfv7mbw(SR(59c*YP`YgtV*JlVmoLGn%nM(F0Kotsu`vPL*tCrj+z7B@V5_SDAa zL#s9$PPKU@bq$r{TXN69jWJuhA3~kVte-i{CWf1B!a^XzsW>E>V$YvHB`ZwVh4Xut zFYV)Vp%nG8G|ti#W+uT~p_M@R>d+)JHv|ugyJNm4ckFsrN~rj zTZKeF9zWs6{py#$f*&!CkFH$#Aj~2)8d@2)9c&)Gd39=QSD?(YnK+8|70nFW6q4!2qAT8Z|}3uKEHYM zQ%e_1uK72&ZaF|*d4*|JYX0$qhivxEv*)I+VV$Iy3x82B>!N>WKe=)J)YjGu1}|XW zX6uNxUbuK+YbzIliB0sr9(?wEO}5q5;`@rk38uU8LOU!O1|P1RhcheUqp*pEj~ z5TE`>Nw1*mf|6E7jePO@pFQK%Nd~#RzfV3;;k0`5o+=&8G(13c1va2@8kDhCQLdzw zgBQ-($}fEWq#?8%<17-;wbXv)+I1ebT4-_DpqW6ZvPr-C`WpygmW7J;V*&IS?;c-2 zf;f=SaPF8KHcrEN_z)^^tlnvEz8y<60rCMnP=?n5^{xBM@F z;ZFc2a;D#I&0N+T`xFfg2=$Ev-4#T!PsMTl(wK@A6(49$X83;67Nq1G`B$yMMQ7+) z<8sSWuoo_-XS&+W!-jJ^`Mc6NH{w6Ol)xj% ze(>z@G!w(b|3c>fj|E2;&d}V0l?Ay9*chcE0&W_hdiQdQv%_2tYxp zLTZ5lWuZ~j=xg&gj)*{!H7I3tVHCPiIKir+V9HTWfQxk$j5%66Nu#ui;4fM0J$upz z*d+@&z{0YnHwSl->qBre`6xPciGW2?=^fAP8v`eN3-BJipa(ZD?(cgj@cZcTlN3{H z0LKVe_4TbSDO?PTF;(RnoEPKjd4AC&M$(j62jk`^x_b->RC8=S*Sq#VJ&zaWu|`Ao#} zP39)tFnoY3j%D4Go}}1)PjK^tqf*GvxppRS%O$$LRGJAcp(ew#$U;XC(itM^iX7& zr~xMGQWs~CK~JAQeemEOb{_w86!tr2n{_)NoZY{CiDRn$CF2?6$cdFq&k7`D@QvpC zo?6;k<0H-J%tZ2ezW>WU{;c|fvOl)>mks|};Sc>Sh5oYWho}GO-@$P_pLn$J#g9-XU->0W`-s*f#i_@+ihO55zbmZNkEUTNfs;<8B-(8 z7cWK81Qf_hCAcFuKl{{DjdjJ}{_eMjuV11+OT@qY+0P$b*rQrd2M%GO;7UEA6ieI; z90v;sd}B{D#~u0IZ+~ZS*je-!n}NcBkOqpuPUlYu=4O2yTG)silhh`dkdQ&Nb9P(G zndCbrFoEcF9G-^zSFT+?w|ACwM%G9z7Mg32fSV9JvHnIN4#`0a-Ubf6h$ux*G|pl_ zfA-Ag#@f9*_fKta-emTE_8IBJD%-F`vVP)=&p$u7@`^U#)!n#$gLoQ&ZgSpIB!BqB zA3B$FtNkU;02R3cgEA#;6Aa$&vqG9bBToq#Xs%=2kYH?HG5wgb>vE7T?qB}>?|u(a z?ZSCcgy;Kj@9_FXc%&!EwP0@1+F%B*cJc*giUvgywE z-D(`lL%(3%W36@@i{P%L30pPH-eUd?GMO8blAt6 zEJPkjX(lh-)0)4qmrgZ`zL+UkP%?5HfczY-Lns|>Zk%?Qp(>OjwB*XJ9*p@Xpe;y# z^5O1zHVotlO4D`-Ktq|rCKa;6<$N^ucw84|#MH1I4VygCf95u18vnYO-fJ|EZHX@c zD2$oah4y04Vu_?G)UL=$GhgUO_*(2^S(2R7#TdIr29tf17=)WU8ULJja z?`%;~s9z8=pY!Wq@}Zru`p2q&^7W55-+x;Q`>_;K@xT45>LQ1x?w|WNb&ztH)wB<7 zqd6vOO1Je%z8*BbhZzN+CyebzwscnT8lF%dqjp~BSiumlM5hA8?M<(rXo7KP`fwl? zWh4CLC;#fpFTec#?|=97*^_U-{Z^1vmK1@=c?SZPP{t;3q_)6tc#z#8I>elrL=K^$ z=fC{rFD=7*^NbWM#)Uc7BU+K%LOsAJyb-@SmI|^iP=4XUMTt81zQ2ct zZh!NQENXVl>75<-Wi;pbzkGoT2ycXkIMY)ntog0I5nm+Ca{G#A!vtSwXUFEs{B9e} zm@*7iCAmS&)vH&pko?Gw$ViyJIbsM?qvFekN%TWC9>;a;DY38H-`utUCEr4dt)ojx zF|r#@bdg?w&DN;+Z-ubU5n^>EDvXf+x8##uyy%V?!$9E5q5?SudjitbN^Vm}O=Cl& zLnKMXmObQO$i^EUh}m6P@adRe|K_)X)ZAVMa?znjk5nh+ZE>{u=G^WsGm~+WWN6vr zB;6&28Uf=^I>}(kdpHC(|3=OTUR-%&m12$1RG8PrlLLYsAArGuRE}Ce7$QV{MI->j z%J*etgkLdLOI-=)LLw115l#q95|Lsq2C4t&TPCf7D*BSfbs)uw@@SFq0lUVeM@U2H zk^FMaN)Oy8t8|gk+mu0;S}V?*7-wig1qunF6b3!B^XSPb_7g4PIj^0FH=^YNHt~Jp zs3gNBEEY%&7SEUPsc*AwL=7lS*#mX0;s4?HzkBrHv3!WjACd zlVwHVYyaH~4)^9Jc_$3M%z~AC`F_LZ;lZWl1t*bMBXwY)JuXK^+i|M$yb149TS&BK z->w|Kd~FXIa|eQm1BvWF)mQqvDS-qNk`0RM#%NrTzLKs;+qF@bkBf{21H6!=1Z#kSOKEF6M3uixKF2xxMpc!KGYO z^Y0Lot`S%>NkNTV(?3q23>>J)E$1ByAJ1w+FG$d;eeCA(;p2yb1Uel+G&2dD$r%l! zztRUthSYOm_=FGG=xm@uHE?JZ+_%`W*pcymO4;`J1jP~wkEYm#K&qOXf;S(c~bzjV(jD=(c)h_ zS;!uMaX9KM@zM7~`%$fmzj?!C7T0omrg)12j`$M$z&|+H*0t?MX+P$R2Y2uChmfLJ z8*|-|nAWdf9s&dW_!D2s;Kn!YfqnDZPEp7J14q=w8~<28c!Z1vrcgW@r>yybM2e-7y5*h&TMC#Bt8IcIsG1_VUxM zQ&HiGBG;nKAAkG_Oi!zpOk_Qaz#Wp4U>3g9qrPyJ#oB>}1F3@BhQb0i3f!{d7R@V?MHmifkx+w*XJ^P9uiPI^&P;e(BTlMNUDsGq7h!VezAM)?*fOo^#ZCu0}^0h_vx<*aiVU+@v3`R@HE1d77SdI|&P!t$9@ zC%kkiT3uaz{!~izS)+hFGYdjy1Tb*W+e0?7egV>Yo=2S5m35@j*-J3r5R9$NLn93! z3?u7?ECa2C1Q;go5Q(vyVv65F6eAF8wr$Ra?juJBJp?^_9ZKDI=8t$25+etKc`idp zLkxz2YJj~CW70|RBX>c-!vz2Wnx+9T*dij_oFYX`D z5z;p(+~l>AczpEu@l*KG8fZ;{XaQ9y-2w%gI{+~CsPa^fwab&qD1AM8_*f7{EgOK$ z_10+&!A5-D+pZ2@aec@=APa;T8e|IVG@E)U-M2LbF`p(0qso#T)7-q+8-gekTw@JH zf}F(3-0IGX1qF+aSqbmgZz5p*kMKs-|J-ZJkBqPOKU+^}USVg2p2#ugt=D-Lb5mA3b}vI6cf`xP_74f(`n*%ifn!M&CWhhx%QI zoB#ko07*naRL>8-xpfOd4h}3K920!p#~05IG=zJ}4y&MKS(0_JAeoa`!n&~0&x$k* zKw$G|8jG=Dc)6htG+pDwkRYy?U12oePKje_%3qmU5|f#plEH; zgs`VyQTxy<+O*Q!2+_PYbpkL#-*j7f0woH??tIoecfL0)M8XS~wXX#P3NvltQgN;! z$@hY&rG>bvcvLH`R;}WNV`QJxX_wpP#_MN1supP5!q(>bcJk7NnASl|mLNVD)S*-l zK1cO*^zGgI_rx0vOyn8G_>a6DyoPN;K>y*YrL~k6TtlfAnkG-BW}G}}a$yg;%qHTo z{G)=x04E~gm58N0u!*38xiB{QCpas`(xOn_ZMvsd5w5mWxVm=qq6|I5T*wAjn}z%L z?tk^o*CviNg)<;<*|}X=8#!MzP*J;qxk-7|F^r^^i68?r zM8pqepN5!xTp;zbsIg;EgFt9z8i)Fn?BPF!^V9Ox{90raIN1u61v@;&akgg(MWoCkT7s9LYzK)5}+<);6{$ zWsq1d>t?H+F6Na0GzVPWA$SrJVdSQ8PQ@KEJ386vq}^6{owB*$)P1p6kT`0dA3!Hq zZk-pjhrM`@KR8TC!CObE$6fItVTQ(vyCexH@C|}5{MhIho3%aPEt*S zzKct+vy79>HZ(R7M9fGNTZjYLh*DT!v9lblL6gypH$J{0YxT-UA9!(9 zZr{31WS={KHqC&jd;c^_XSoKjHbLu!q=W=R^2yUjxr480)&m$|+A(YmllE$HKBkv< zkl;XEL|E~iprwrryhw{q*%YcIWpS;od?o*^)3UOF4csslgJ*~`t&qiXchhWYU&XjO z^VB&sCi4YM2z}ToYsOHN2-}&xFD1f5Y1ySq1Z2b4%Mg8?&h0cx!Udwz`dTX< z(%!w4U9HVgKw_H}pGNA<7iw=D|9of6J$i1m+j_ablFT~W>EY8SKmWzgfB)52M3=KZ zSqi{o8NjUr)YIiU080Z9cgXX`d}a#H0pw{m7^1~2s9|;y1`<-WpPJ4y<5a9=m<{Rj z`xh=q_2mFQk}# zBl@4HUnnG?2^KS4Y}ZTKpFO5;z~1S9TC~ywMvb$fGg*XV&x2f?xX9x&cMZ1TF4O7@ z6_@d3-);ayX{RXlXYzLLG!wa;~Rc{WR`0?~V{p{y#646~8bM4wS zI6SeVIm;?JK$?8+b@5I{C|GE+i*@f`zGyL3I%vz>HMpQ!2>(_ed2s72y`67D+wnT`Q6GJLe{$!(&pweU|F4fAuT(DUPD( z6vXNY`v$B8C@DqC?t$22HTPo7F$w}Zln+gAxv3)VB1useVY9x{juX4XPK*1;B2MT! zc>3_kn^&(l9lmCxqWH_M<%cZ`hK@?qoaNKI_Z_qHP_sayipXLZ$2kUzrPDk%6{bB2 zhdd0BB2#uFIZo1o-_{?&aC<@3fFW56Sc4Pqr^85iTf9DvBRMVOco;gibg3n6r_a5fn|UR&1Y*VT=VJi}yYx=1 zI!_D};3$Gv+>s1Nx5UB8)vKZeG+?Yta1du%Tgf2f6*HiZjhS7^zEchf7$$WN)vRN# zB!MjafCFv31tXjl)0hoU$U~kUbYVF#fBNZX`*@qP{kikA`IRwBbC# z*TGA#?>@NSD~7106HY5SL7U7n+m=9<&@YlQtC-RoYPpZk&+Hm2vJ;2Ysi~*b7&vAW zhMgi|yANMR!b4jQIT40Jz&aY+zQs$ItSp)iboI&`2-nxW!4Ll7(dd+hFyTEaf!?4O zj0lXpe-;J?IeGuS49u0|lhlKyi1WMWtzeoMw1eu9uF0142cz$r%#+(YJDaDs9SLXL zTx@tT8*C0Vz}|7ny92Idg$jpR!5te1LbS(}TIi=8cxFeNu*=PbL4!DkJl&^~0C{Y!e?IC|O`5r7)i# z?*G&O{z);z7fe*E_&~%yEs(wJ5cmIh^~2O9s(f3v`^2A(Kt&a7LW7w%B{AKvXNCw&=eznx)~{R^4tJqJ-%5VLFqAGi9VA z2u1RB3Yg_0)@6d(ww9&;#w%@vd=U;ei z2Fo<+EjwJz4Xvkvhwqk8WlWR#Qqp(pEkfAoNcYFy6^HuJ3g1oKt?k}pZcl~H+(`uI?g|;gqHp%ANm$C`nt4N z(LA5UPB4^8xZJO`uC$Chp#{Wdq+@&Zg_s=-L)mO(j?AtdrD>FiY{aFWryn=jAP9;) zs#%k6#t6`C+Qz#KP0a3S>ttsdz2Zc_E^hpm-U<8l*Uuh5k@|S!#tramJ%9Ab zX{2rYS72cQ0#qzPT}J2`(b3HfizU>!@GAet+9#qSM)YA87E{$vYS6F+FWWzM=8QB4 z{E=Ie>OZtJBw4`n5>X>kcY6CwI5HK&u4Tg&0DwkdvyS}>uEyWmicvtWc^m1q3y*M^u#zG+(4vcDjbz{|H8r4FhL}O;DP0iAUGg7t?qvGMc2T}l> z7|7|dW!H+g<5d#M_Pw9#c->BhmlF2!rGp&}%<6zfPBdIZ6O zHosKnoTL zN_6ka-yT%GWAHk2ff-BIcpz}DK`dlY3orpRq5qMi^{s&FfdP->5stKV;QymLckdZB zL1`?qWznE4B~yt@2^go?X#{y%Xo;wvKUG~jOM}|%GMu4V@LGIYGW!FDK+eMK8O?wW z+|%#!GlO)R)tXDQIilGg$($@y0)EzTl!py%ysz828VZG$a;e2$2jd|>F+##Y4=*#s)SAZ_qV0ZK8 zwd>bqyKBNdTHj}bbyn%|5!;KhK`JaQAS(anm804bOlwGQY_~j}XQ_)jckce?*A7HK z_-Oyq1-qYiJDfMo`}VD+4*Et|n&`KffyYU9G8DgXt+uyLKRfvhr}DtwykwmhibPy0 zHYlyo@p%tfRa@(u^61W;IeTw?P!yFnv{mc@*+d(8wXC|taghPUkfy9{?0%h#HZT)2BHfZ#x{i2vGM?3>`*$sss}ExcsGtWaaM-KkmMkwv z1J8n{2t()MSlWR`P3eNoP+#Q1Fr-$6M3VBvcOKr9(X}rFlG6I0(G{7=yolMG7Q)Rs z=_r`yl&&F$Ifg z9a_9Oa1c%ke2v12!8=3Dsju5k3IZ0drC^6`Qi$mF%M?4v1UTodGx9)M7^LI`JFOhy zk-fdGoionx0E4VwoMW-Xr|1FSP_)`u?8_nkN$lpB@NLlx<6yw70M>u07pWipzU_p) zKfJj2=%GMLI?N$;FP?9lrujEf(aF8vxs8E{42*ymn+cz?o0QG{kb+sAr)-GT|yiRu<2d%U4%7RLfF~POw8!#7GnH za3OULX>jSxEdV+S;p{m~b?<)0mOTXYqR1vXG=DlV%#M%(4^XDjfAV8wu~I#(+GL0#pSJ!AAJg)mHUCGjaZ5dzRz&Knd4g8|Vu?mKDyA3P z*q+CjWCw+2`8PJ!u(6ui5QYtklM=%%lL(}qymsySkG|~i=6m<<*LaS39 zI3;pTwRV?np0{p&V>hLPJ(#$g?>2DDkACzw7xpfA!GaBn5il}L zzBQ_T*zo{;hp(OaC1xlB134bSgT`G7k7ilIY3UoYsemgY%80gBJxWFHVEJ#gEai;0 z<1oTzF^zT0xo^r^Jh#gPh0l)R40jf{;p?q2M(;1(rw4n68H65ePG>2%jGB1rq|5>v5WPg@biP$Uf=3MVvflB(T{(0{rbnO9j7Im zOg?1)R?IWnplNnTH+6DOuyhX^$TP zOYg#1-`whT668JwDfB7ihtEqnO9Ok6!3&B(TBH5?;i1IeIQwNJ6(6%+{VcazTI`|s{~pMi zynglC&6_u6Aq?9^JoLv;{`N~LaLAin@;X|2#8M=RK*sbV3tDX7lv_Wu;YbyePLFhI zQVTPNYZ2(RtpXCcUtIO(-0Us%M{5GWY&y7Ujj?PxR^QQWre4#e3&YN<0{1V`!uEAM zr@G8Sfs36TN)xNKO#u0Y=eKLy!q0^S7L!T`(T8gx@(2uq=1$)}d|B@O@#hcs|NP5R zVEM!c@#}|1mhVgP`Toa`<>p^}vd2Dv_7_iZAc(WB+n0sx#7a#DnF42L&p87lF16r% z^`k3SKl*6GxOM>KJf)U4H_mRK$(>%TPS|&>7%6Cy9!h||E(P=c!~3G#=7(ly{F*QA z2QSap$`B{iiD3dhXC3Un^75Hup9H?2StNM!_>mx0w&rk7Oy76`Dk)ZK+%~9e@RE9v zK(}MDIBgRU=LyrGow8seJ>g(7f^;LXa)^n%a;P$2+ln@JliLKoX<)5nucSV+reh>5 z*zUPE*#6sF-)*jYo%0i?o=860JAd9j?OV5RK_@qnCuQT&mCKh=yS4e?6}0Fml;hpq zvm(*9ZDIj&xd(T?v*u&z^Q|}#ΠA73b@o@|Mr?+K3yu#ygMSJvN(XPVemPp1XMA zLOEfiG3641@VxW_|>yN+3lG!AliMd=LYq zI3UkDGllJs^vx~M7<{~WBP?_E`gM^U=o46wG^G)-ps)V$yZg@{i7T<7DGS+bB8&`_ zQhg`q5DXN!xAb_7Tn*eTWcs#L6I_5uh#Gfe@Go46%dg^j>Yw1bA`>(_ii8(D))Bn069j znDvOrP9W{@acM2K=`#m#Xx0x2>9olrRo%o)LqNP38;nX4m{WrF}Q6)iEPtk=oIm09&V86wc|@wg2UR{-3BXYF~d5GxgrXyqa`h zQgslOWz-8yHPZA?I`%N5Hd0;2>|#jkhryP4MPZCxqfAN9? zU>FZjX;X%LS@>mx>|WS~7}->x+`Msd|2&I88e41K=n|75FAh7(C|>+-bBnLAfBpJZ zb~vdaL;!62aCA+{Nr==i6=SE6U9^wXJ^B(rstQ*1T|MsfCfx3w9yq7_L9J(^!^ zY>V>I>Elvl4)p1fMPVNYZcl*tmQ`={^ckaksvYCL~Upd)wv_%8P z79pNuT&0aF@IWfK2BVk3#;rgvm4fk-U+$?K$aMBh|Gw(b1Xq1iNw)n(E z`0DGgy*z?f!)%w)duBF5VQDGf_RC5#G@`S+XTG?3^YhO>FU)m(%-ll{7tWrsl>Fr0 zLrNgZB~dV{ovOx0$PufhHLd~C3DWji$l@{!1dv7&;NU}yvvHNJsWe%DAl+P{={TEj zy|A1kyM=MNu5KeLQk7O|Hc#6BV_#wX+H#-+xjVC!>cac)c@6_>ja;L#D@tq?T>0pFO=L1oCvBJb1{buu!HA zvDAmNdS{%70)_aa{8V9!Am&$Xk#2oX0rHBU=CDz6ks2x7G%1LQsB*zdVsp9z>I1NZP6C%X7PXh_x~PZ z(~*P*#F>5M=tt54uNzU%9D~7W%>*2is&Xn-?vIMVbk3zl1zm(-_tITUB;E@NftSVX z;0eY}_M8?I0)~^uD=eDNIn4_f^rb%h3AIKaVj5_{fBFNgSv90(>&|G=4*;=QvznSk zRT}YIe*6A~b2qMEWfvQn0n^^vG%I;G(L=04H@LCER@+>)SW=f6b!a5-hMC~E&19#J zwt+T|DV61<iWXwCfn3wN0n?5_R&y?^)b{w?T4 zJc%}Ap?DY$#%QBDazxyBl`)k*(-+yoyAAa)5$2(_i zBPcuwl8-<61aX|-yKwcYA$Qzz)`n#HQ`_#4K~Tu=wzQKt^A@`CT)-^a&SZCK1Z_Cm z!*_!{I?+4Ra=Owm=!ElU4Z}A^NxSoS?RK~EvxOmuS2mZ`tXsEkNqq*1rbKXA+?HL8 zeFO}wh+Ui%5ZK$@y?E*3{kwNd>pgt)>frgy4pNg*V>OuKz>Rg{Cxr-Lq=^X~u*H)g zRb~5g)Xpa&2 z1elPGyo`)Ba)o(eycaI)p^-#YF08YVcxkw!_A{F3}^aI+AQiE$mPt9u~Z+CU$)b{S)o>K?S3pwxf z$P@-meDb6a_c>xn0N)u9E%%YL)S?im4f2BY-RQq4*fn-M;NA714>XbwK1qq)8gcBO z9$5`ugh4;aMtD=afbUGkI@{PUj7?T9_p7f+S`B^c15Kkqur>vVeyobr%rHz|SAwB` z%o*ZHK~YW>@H+u>EFXkYOEQ-i-|DhYjlzXDFz4sP(yCL-_N2Z3^rwF>@S?V{63wB| z>|=3cg9cWq!Q~}av7avBB@L4jaH9}rSn|fkO=N6ahL&_q7Rm$RRh&Jsx0^~AolCUQ zlB)EQO{N5q!%~4Yq(EdC!WJ#HW4k^TktO0zI31FSY0KtHBjYM3RN7pkGj;K+%ozfn z{=0Z#@9O2t_Z~ig^V2&W^a3SNLxI_+X+>SIRcH&66F+earF(JXUvGSJx)VLjESKT(caZq8a=hdU^1 z!Z4LEMcjtXx{e*}=1d1UNSq9XF^JI(_!1o6T)T0@AwBTID^0MX-s#KGM4y(GJkpnx zk>tX6f#dLba>YimID`j$?AZ(awwAqNmaCP;mTMsqKI3CmB@zY-#LiI*~ZBE2@C?rxMsX9uOL1G zAc~qxV3!RA7j3X~mf`C+BshPaRY;ws$_1s(F~Pn1I;sO=q}Oa3m#N}rB`#>))ir4^ zA7B5N$Tp9JI9s1ZB4ClxnpmVAyOO1IEhORo@-zI-<%_7Aw*v?~96USNI-Q+IQ8}&1 zq7gPcceV$LVHTDSs($lw#k=R9J$bsdN3u_D2>G9&&PK~RN1wJQ?yT>f+r`WFW;_?? zcfxPdB?4x?vG5QT5LBClyGXid4e(PiK_1P+sxk|#eHkUh&1}I(P$ePi=7<^3vO{!G zz&g=mkG(gHXP)T^v5vU_FK`s2iR#4hqgat3IVbod9M#HTxNJ{QK9J1w2uV5K44L%s7BmPK6hXD#V z!hjuV$!)jLiy@QKz?zU>oSzZVf`X|})in7z$oOPn*6z4w);wuo3R881gb@$M_MCrN ztR=(v&IbhZB2(ZggmO8wkyr+wbjcGqhh5wvL7d42vmQfFzFphey!_GS9%TMEK_m$n z(JSN52?y*KKC6i=1UYX_IPn5nSRt>nF!=y6G!f8FLvlkh^R6j4c3K9(3Bwq6;{RTB`^8eCW)Pn zvR`6}rkkeVywWSn7&X4og8FobIU#P^pwl#W?mhDvXbLNqSKbjbD;6jCY?Kit0gSwD zizT5ZffI;?^}q!3B=x|`B$I-!gi*CRllg9qj}aDWVGSXg+qZ5#xPMQ?Mr-Wr#Jcr{ z*~THMA>83IIWiqkmgDO8-Fx?c^P68IBy1<;L0$|ei3H&_IU;l8BuCa6*ypHs8V;>M zROkGQ9tq0KTq7HMkDx9CqHUy*Tl~jxkWRqQ!kC$*fFInuXVeR260$`x$w^Xy2++$QwHQndcjuASLQ13BnE>qDqqD#4I_6lszM56hS3`ww^A z)Q@4BSw{wotE_Lfd~1yZ8PK&nQ*VZR{E#YL@xnC@=juB%Zr9zr4g-Du+)IxDmQc;+ zt*?fFGh|X|%^{GY_cCd%_7!@kV;$gTu{l5$f-%t?_#xq%-?iiQEiXqgaUpTUYK@W> zM5(cpoYMn)C2`xI4hujf1sC*Hy9!ZHo@P`)bfu+IbXKL3aD3Zje|o7NVgshKZqb@{ zn@2OC3MyoSj03G9`fzj<@lnK)u4$r*Gy$2a$SaNOuv!>VIh|QW1zdrg-ZPO6()jhR zh1$_My`|6TG$o)fKU!x++n{bJ6AKv37;4`9GTauaTVe>zQJRzPW5uM8D1Gb zfTzW3jfd~j`^XB-_ysdeUnLg?NFQlawfgFRrAhcy$q1JPnHob1E(0g9G$>9z+#8eW z9o6E`dRs_2fOXQuGi!V~-L{^fTj&_Gfd>qP7kcr1?#GkMp}%@`Q$+w#K(4=+n=Vbc z2~hWaZ{GB4bAx@IC)RG<{CM|#o0AAW`bN0J7u;nWxEAMx?G(d%W#hY>Ueify#G)O* zd|E21`y2?u{fjR?7rYk~XhYJx(MRtbzMD5cp=VT~oLmcdnxdBe=5Kzqcm6__@#cr=z@1dHR3^!LLTW5?oFBE2AIWZXiz?PN?$N8BbIJHK}Q1h zq_qPH5&Q;rkE}U+GC(rjMO*V1skjM{y^DLayF>R_)Q$zW-}m6)3VCg?swn1@(2*X7 z>;#(4o5A)dR!v+C!p9q`NE**@QDa87_&$TIu;o)c=N>OjSp0>*3^FX z?6J^_#xY==7KzS;TDV*BhT{j^Qvl&H6e51YKVu~_fO#5HUhN!X)B=tn->SiAM@4s>;p2nFbtDk(bcJTt1 zUM5ao&IgZ4%kJx#WD8?xl{-ks`&R~rXSki0%oA3ArWGQ)oyTmoj2UF*ykq~@uU)Y*28h0aV9O9#PoB zjNvR)vCsqNRZ>?PBo>xOdS{B?fAGLlp}2OV%BnbRBLP!}j%lOboUCeT<7jKOSKn&Y zU`@9!z@Q!WN+je8i$sc@v(1(y1tyVv^^tYuh9E#}YetDP14Js-$gWZbkCrLI@f0kE zx!OP}{UuD(7&TA9AuME+c2EM+{1*cIQTJ~5!1!H6IV-?}+7g<@q&cA|WabI~a*s7| zr1@McY6t|Sdw3Hd`313c0F~oPjnXKK=ERy{vBAU0f=Y$7i)v(aqKjak?-ueq+2f#Z8E%obak*PKjy!4-pL zXz0t;*vEJF#bk>#Q!l|a*D^KO71kA_OSlf60KP|Ut0=7qsHo88(NKI}+#Dw+VD)G$ z80v%QZ1Ww25B?K*rptF-q;ON{$IfoKcjwOei~B$N;)`onuezyI(p)+#o{8YL*9hfl zfyXJBLn0K3T86_!wG~D$Ac2r4&z~R<6zoEuUbw&&FrS2!-cw*}>%usyfY3k|fPL2y z4d?UV#sN$Tz6!gjHNy>zo1cEVx4&m@g8$81@j8ehV3D57JmbQ6DNcltTIv+#ju*bRv$7>5C^QVE`YrMR#e|Bd6BsZ-W*jgz z!d9;Pu?FJh_pFC;e6PAmoBKYO7W&yR^D{iP42*+BrmtPQetvh)+qtaMUfjR*-S^+$ zySM77n>PoCYt-C|gI78$5}fD?D4bwGBpFTFs+;=47diT9Mj%}8@*s|1!*;u!EW|Qr z0qd^?$QF2)LGk?Hj?gz6e=`UrAURa$!*su#L;{zj_s@)zk+Y27PXgT)0+8DLRcwY1 z!I=Tmc#mAsdf3!L-)9Yg_X0)!o72M^7deJU1K5P8Lh7f?k{ZS}A)(ne>b6PclP-rv z1aEbXfJGGDrA-h=Qz{=#Fm%vIoW|mWZ*a9}N2df&$VkO{Uf&p9JUPrD;D^F6vBg7*cKz7Hh*GfD;^qz~O=Z_zas_9t!wZknE3Jw`VmoHv| z|CetMV8CRVzxL-pf5}JUju?wqKpmZUyZLUDw@Rs$iv98>nIY-<{+&BA8y5q`o-N^p zcImgwUiyJgPrc3vOVH}h`4T|JZi<*1)3h&<%Ij{ir?@H-YGssodMD?ejcughxB8}2 zeL>pHbsC_P;jQgE-iBnE_gOHOO}QjV1gHcex20@*a~mfktWD`hF{Z;TiZG}CvcuY$ zpeNj-Nu8!HH4A8nnHfEV%{eb7@1`( z&E##QBZI?Q$fB5L1cU-8V)&Fo3dq-Rk~pfhG}A!kX~Q%JEd~yph_xY#n~e`)u+8+W zf>bp;b)Z>2gcr*&BiD@j#IZLqm>d673I?H3{rZxQAU+0Q{eqAxt{^av7KBX9Rhgv8`uwyKCwnmnYNCR#aAr4GWYa^cVrDZl1 zX_^?O*Go53_*}k?#F80Es)!}2b9Cs5y5Ir%kY#!iR2o@slJ^P-BEg0-p^c@%j4&b0 z7g%N~9DoH%iE|iv+^gO?D9dFqnwG;y$Nud{)gG^xl@Z(QZ1iKv`aKgp_)`Iqdi}G1h z^2?w6t%{6vqb78q4n=B2jE8&h*{7dc$Fq*o@t{N}gdQJT@S9_ccP$RXiATHy;)#b1 zo|Z#)&Ju5&R34=5&-YK?XDE*(qM<0-Ek2wbyVCBhh#D-n6-Ag>9fJ@?<-!nEuqk4B z+rv5a96bY;Bt$#<01Gl;s4N9iRH{|M!N8y=`JpHEOA`QK4TnTLkP+= z!6pynFlk;@HF@PYmcWt{vU^F`$MHPE`GpE8lJyN|5=PFi*X*y}Ikj`(kmwh)0q*n} zQ9kNRPU~p`xa5Y+Okp!*uOpApE;$p4TQ*_5nopx3P$=w)iws0MkI`_b0Isj|tN1f^ z55Rdh_Ah?%3!y?P#C#!%BD$UM-TMO6O^3afopj2$%%JFb_0T?yvC(T@-^jF6;_fTp3E;f(l%WZBr4CN`ZbF6^)q10bN=FyrjTOG{z;?$C8 z@3hX=h^0)Gk!r5cQDU?+v1x!sj6oBLEeFf$xY7;15syI5mZzSfrTaQ0-g zn*$W+Wno&Do7iB$gqTJiT7rCFuelj->ZJGBd~) ze0r?SQaTjQM5jSPWWw?K=o_UQ?lSxt5m1(v2QA^*&k<4)moKW+Cc~>C2P(8JH_1eW zRY?$ZC0tzYTTS^_Kluqs#HaQY0P~b67l2k>ut=~jYimHNiZ6Q9-b|@k_RCLGq+e^5 zRA9w4F{i0!`;UIg00GqkiNZ@ep6STVX(U6{)g+{u#gV#Z6y=q;7%e8+U?vh$Kp2)Z zYxL7j%`TUU?&wEe&r?az4C;UXz=B33 z?M@sq&jWWZ?YQWqcV|nqrLt3svQ_esz}vud0s$!BMqq}u1|QF4;fiTDxv5-jDGsk9r zfA8U!PNVSRvV(&Mmdb1}VcCrTNAdy^MI&(3t|`;Sz_(wA{|C+fgrR!FRoiaf=!(;0 zp#?=MyTQlH5{gf%<*3l}=fU#o#G!nWY=7a0u~Q+FuPyN*(q38RxH#p8RN~>GZL=oF z0k$b41(57Qxg#t(gcQw;foK!jELsm;MXluWu&Sg%D`Za}KY>XZpa)p(}#B+96fnsZH;)a0oJxzTeHy> z-(84W;1!y@Wgvd!E3>5E9%BLSj3_rMC%RA|XG!!PfgvQHM3k%u3(Z7iIYQv2V+=63 z>4OO?5e}$%(VWI)jYnjdCMOjXj5X#~eqlp2T+@N>5+)gnQXWs$LHG5(AHt6fDh z#f|>y5zbMDImkAwcGP8n74y-<*(6il!SteRJc_qDDMkO9plP^oj8_DRnRQ@N=c{s6 zj=os0y(KHr%NRp~1Q}`zoIuH}0=$xT@JDuZt}Xe7#$%$7dq*&TiM1h zJecR0oT~E?E16yaSr>CP7W|QspXPDT|Gs5&B8*TXQa!2pJXxk$U>`?-C(n6I~cbc*qOdAERl?z>Y z)MSoo6e+rV`3e&vD-QEa!j)PBvQFKrM|V`Abq0p_UAy1V+bohpPkB$veA!UpX>Vc} z&lZeCgBE5S(NbhxFI_rS*q7yw@8-hb>Kk&VWLm%Vy5`i?Y!6{pcyPN^Hz>WRyWFhk zJZ+d)KC+L`o{Xzku1OA9PzSJ4hYLJ8L)uYezNnfw)`MXNOcmigVwecDyoqDS1&ROJOa!qsjdWkGkK`JE-XxM?6!HQyJ0=m3 z{59(0q_ZSrko3G0q{JS3rRGt=Mvqu4oEDS=PRfirp>OCIuivGX&e<8j+bK8C5ogMl zpjuOHOE?x^vptq}uWQ1CwYMoM>qnCt=??ElG$@uQ;b*S=Y{2xGJX*FlNVKEz&7tJEu0a*uigsa&`rgG8cz zW?s7W#QC0wfM$*P&shhs1YJld>Ybwndy(9klC~*eDqq%2W67K<&d(?@WYN_e40bqV zI5yfa7ePHTCjlAfw;hg46IT|v$wE}j&-hn=uD-|pq*$1AEupSREZJAG%Gl=mvSOrp z_mUHd^7qVTZ(W>aL>t%3Zyj@(u77v5c}b;lA-P zf%>%(vPMgg66wb-nA7-j>1&x#UpzvrKIc(4mYblTdw%Jn>0HVHaw)(3G?ZYSc7j=J z{9@X8!R<8a=7;K<8fkesqnuSx^c4`0;veef381A84W zB$uvR#!&46sGq@B;l^c91L6!}?$o1OqXu%e91diCn?e@h65}G477d};03R*Z&CJSF zeUHEaS4V!QfB}FKf6gpK~%D1TXb?4i-&@ZHL_)~pM=9RB8_KB zH^iztLO;@~_A)W8IJ!lgLH&c>^A0y7WrV1Narj$w7-0uyizwxL^iBasyu9n=@1rB3 zq%S`IT4A)xGxm~sF5@`%qyX+tjtMqsAX1tT2tn95=HJf5yLCq z2@!*HQerY^8TMFahLGX`ov}qa%lvCR1506E7ImnLP47%e_L!fOJPgOjI`G4>^K~j7 zxOK{qXTd2d;nDG`Or+^fIxQAbxxR(y!t-OBp|Hn&AVzBDv912*Thc^QGD?| zjScyKz=sjf&RWPoQcOHt6!gtp0gHq!3SDx(LKnJItg%J7!80YX8>u8xZwYO(@)Ojd z!oM+ss`W5OychjiXd_!%JBD1r*u4);;^N8|NgraWuGYXc#M}91{C>yqjcwadd$fpD zPpUOMrxLSrYO7gF!=!p94&tmaMSU>_tq+I|_dPcU0}|3+er5By%;++d8jZ@q&r1iv zBJmsYSc|F{<>;KEeA6m|bJyVNjmhRr6Ixn3TJ||2sLpG7=GfhO)#8MgYGq)UGE2e6 zvUELzsyqOp0w%eetTU~-ri(Z-f}d;dyzg#6@`!w?LC<}}wwef+hbk_0-xrws zs`z(&fNp1fkbKjiKK#<80!y)l8|UX#{^7of4E{gAD%Ty2%-5QO8Bk6O3%JYoWpqn* zSW*5xv~w%zX<1%`VMejQ73og@i53(AfLx9l`+P1S4>be`vTQPZ^FS$6IMBHjL?h38 z62a9-Gpjtp@;KH~sES=6%zVaZi;(E_sN5_%%rX5D`|uc+w!e$8*VK{<8q;U3(dzV6 zO~#7lkYaP<13zSFrb>`@8i<9?;>f%OoL)knKQum!9F5J*fBEcAxo}|A zWKP!^>oMzZKe-9#R0THobN9Sltka1nhnSr@b^XTm9=&}F!XOXca7?US-oJEidq-2U z9sKM~9D_#B!R)vPc^Mfnyk@&tZmu5%LUU92@}EZ#|j51`%gXuOKJ(kxrW;hzaP<|PPMdRc~due~gT)DUdy zqFZPNY35=?kOH8*n8FdQrwE)17+t=2(WymJD421|m*9Elx~S0*;ZKBOYU`F0>euBNQ2R!*;aZ44IG42lsq3M&_WgKNnhsLvXo z27QSiYF4)kt@>VfOJCagp($Uq_TeJSR9SGTPq+L3hdaL2&ikve&YeEpTONhqsfqH? zMcvL2ret8as_@6AuqbLC7M_*^8PbTq4_#F{Z1?XEm$JSu4f1-u*DPz>a-04F0K|jP z{)rSO9oy~7_s?AOtkK#XBj(eZ^Obe(g3|~s3O3@5U70>+sL_!<7IMU(tQMFT;t1p* zN_RFdoTvH)%(2ZDgAm_4|Hclm#7KxTWn&gpHOep(E&eoS_br}TnbQ}!>7l|DiHm~; zWHTO;Cw64~&3iWG2}gD8?e3NPFc}j{!UpStuJZm7a@lXG-r6gP-#-a}H64Ba`DeDN z8M1NFaJ(o^(g=YdQ?R*9qc=zM-Ao$CknFfVKmPIG`uY2>{$O4y0>xm`R-UWZuRD#< z*d;7Do>9yX45%xl!C0P3f@pVV=f<^bmYp0f=k$R$_r7PH7#A-EFcc&~WqD@n9Xnu4 z^#ukKZI#Gc)a!#+B74_vT=Vidt;z!dfZ-!iN6SkfZs!Uw>_7hUH$5O=7g8>*ymgI= z_kZ~$ZrL~LIif^vFrP-tEl*GInk0&>r~EP!O0hDnNN)Xvu&>{ub{=veDXv9L!Lb3D z!&ck4ltoJ8S(wx^l?Ir!nm?^VMk>z)mxTuD0TPnj3?w`Q(b2EVV|giY6A-7t)$LkK z_QtrO$#CeIa*9hF%6p1XGZAC!C}862j6r4{STv|*^koZ!86Ww9S>Q!ev#}>e&sYe& z1qcwf@TYmBPZ>C8gEGhzO>m$faLkEuevk)Jonhz0bez`dvkxl_AS~nw`IO*0V_rgdhr2Dbi^K7>Y}bfm@FcNrdqi6 zUkuE{FX;=1OT2B|vrevn!WvT`F8_e8I+(~|!pJkg%-_s-qgb$7zXq1n4)vTKy}1*W zSD~C!S7Vq7_iaG|NYiUkVE|A~Xak6;Pf!d^O845jauu~%K(dy@-Z$Hc0WPC~CM)4z zo=>RL7D^WOsnLb$qlFYl&M}Tm$LTAUo5IZ<^o*!7%H%lwYRj}zMS7^Ih&&0``fQ{q z*QId2@@ME9&r+u0L(A0Hgeev&)aw8iY}C$7jA*)m;pRCmTN{}j_vcaR8>28Z#;dt{ z?V9GsFA@7{LzO@9g#PF&ILE69-cfZmuZqiv_yNRRxy?MNOHxM z%R)vqWwpP&JIjsySGer_?X`{QS-|M@*6AD9Z|v_~eCbsRhc9;bc2AwL)Y2v!r@YHb z)$1v*%|3C05m07|=_>MJ2h}-AjJy%@mJ(GSNFerwmUZXAzAFbL?wpd9u+~diT=U^y zXoSL!>FDlUuX8U>th_(nAp{SahpgTgF*)x0i-)B=(nualtDD7y_1_3j355}6;;(e9wo)}AU6CvcXciAee1zx(aCJf-v8#~I354kdK{=l?6ibJy zA`M5a7^*@!QncOB#SpoGq3L!SD+CHwc_3J(WX?~?kLK2xMy*MvQH&l2%2cVW0&+79kgMz z50`KCR|~qD9%}keuWE;Zk#!ZHc^L8JmSCOVH8VK(lj*Xw8XBq!_VBa7{(c~XYLvOC zUQ$508F!2SQzq=auZ$r-fb^loAbww=){#jI^??m99_n>6j+Cz*1US@L9xq7xPvbBV z-Xf>Yg35gvDOq9If~wMumO=L$UPBKou^AACfQQg})!tdmrdM5_!@ny z1z{)W!)f{A_YZ05= zrS&urd8j=6xu7h-VJwU7{Y2}J+pPjdA@pfJT@=^bC{vk~2gShH{AmF^9%&711)P9n zzs_-c#pY&ffh~NfJ~=y4Mc|n$v&n60ct)o%fTS|pWMwzSR~O<%6r{p z9FQbn{}FOfZciVBLpv(WOf;zvggZ+Ail z;%G+2d8Kzu$_4Rqs>2(COgHlsi>2rA8|@fRS1rIJQvWCVr4v~Ol;*K!BW1(qhL6a$ z(*&vdx!@!^s#C1X&t0fY3&HPEGZU%3=Y2zuH^v^*Pr?dYB|_0~*8f79N~!E_cnqn7 z(~&QgLd6VefyBMR&pgx50>4I<(#!1uVj7r{zNhIARqG-Q zMJ|9@WY!N*4Zzc9CIYa~T8(4ra1g?iCi+!98Rwhgb4~Am_7U`@Q)Y!|rv{9s8-Xx& z`%)c*cmfC809A^XB-9> zO(`#KLNLAStOvg%0H+{#{*6cNlcy*N)U+moS!-$e2f@wesqvfl5PgcI;@}i1vhN9%UPl`k*fN?FgmYFi$!In%g^+wZL zkw5xo$V1r#1gywlnA$T39r)Y9@w^$jbb>nB5c(%DShm_sk(=%UFT~$MELvMGK!j=+ zbm-`-ufFnwv)k4FGYI6i@VMMj}dwIWNF9nxl-6`?|(*N7unJr0@8&|Y@7IB887V1St zlV~JOdXh~4|1UDrKX4lvXJJ$4-0MI_E>)G))5GD=5?vjM#Wy$rcL0Hi0E#!+fA-mb z{OZ^5e(}q6+8pU#(!cOaBP#nL*xE^QqJGDhIk3RK8-R3x#FCSSzN9wAEW}hVDeJ{a zHP!*}*Er8d; zqz4d#O%gPrhD&{*9u1xL&_cyf1NH#0R8m$1f-ra`5EQLF7Km0RiD@Y%AMId`tEoa- z8ps~xITG@iBG409F+;V>bY?L5$c@jIJA2gB%h3P&S2!lZ7{y{U6tmSCF%?<~5L4AB zVSl8;(6nRHE=*WZxB6y!&5B=wOdvDg#Z17!0~s(;SUMq3JbwKpJ%H|EujT50z})>+ zAIzC);7{Igi$Kj%f~1~7hZ&DiV@!A<6Oa)nY7~f$pwY2QNzP{D^ttoQ&_U5C{h4?0@D4rA)u+5A&KVLNhd%$#C+k0^>@z+h_YeQbgTSC8Hdj0@ z3IT?IKO;aKIjtg12P1~ZJJ)I`fn)E8!LkOWg9{8N2JH%WhBUpji?btjD%V6h%wNIa zn#6aUQ^}D~g(V>Kb#ylk+O)5?Cer6PB46aI}HLfga6| z}y@92+SB_{Hna;lxfwcHZ#v!~wgCPGf{}~8&Ru!p?nYJ=O(IuwP+pZW3 z*oH7&x=ppZJ{nZnnty-n`wYUEdX2D2b0k77aGLlJO3fz+Q<5SEzHny30CY6~9ml-G zI~Gq7(T+|y16Uk2q`|Vuoxq9(P^dy#!Sf&e!@q+1)gP)2nedFn*;8s;DV3st;K3HU z4uWD-P?ZFo5An+S$mw1U6A(E_ijGCxDli@4J>k7;j2b;qS*Oxr4-z*H{km_8oGjom zybl2b$-tjHml80HS3m%(tNYWj*=_(SD-FOF5LHGGb~Lp`<B|>I(Ge0*zBIUOJmaeK%OOL`OhmHoqVN<=sppX_q*s<=@OwzMO7F*8Ex3E>dU%} zT#ioiJb{gP&?nFeR@%DAFKv|J>ER@1OIZhi2nniSrSi0vOFAnF060t*X%+iQdR8H; zYV;Un$GG}}3qCmVp>QTJltWeZs~Cn5<)754B5Txukd!g1*u?=Bk$h*eh;D>^DH&g1 z`!&x7rA{C5=nr?;GgKHEs+_t&0_y=JLcm{5=++lUA#KS@8Zl!9Q_n;<0%b$43jmESMqr&&v$(M5k1V+ zJ}7VAa#G``EV`H}l84`NbJ^Vh`jmT)**b(m0x164_4yLF!Z8taMffQX9(~06FI`tV z)Go*n@)S26jlB#;3bQzuXrxGe`zL2#D{? z0F|dH>yPqycP$e+rLG~&MZ+WYfywDV2){58|)q31q+=$WP7k%p!5bH6itdmupv=0VBPABr^Id2g9Urt3Q+wb=r^$nQYs~ zxLl~KEvCt&%yNYz8;0bWseNwGsbNhA5Z5XvIJIKNrVCePs@`+3Vdq1rT(P(UZ1 z2@7v%w7Foyj7bg28e$x5i=;3QX*XI>p|lVNspvdKqgiqJAW_KBi>PRTI-@{ZlTR>y z#+Sk~L>&mg$xefzj^(UAX-$Y2k0yWz8K0|%lQfV{b8I3=u;&?dgG2+-S%C)tMX3;X z)iS}W=v37*6@p)CuOm(ciQCUI74J-4NP=Xd*+!E3z!#+si)ZNCHC_r&!KBiIe2l+D z@B1%DTH3_EVJEHZXp7eyjO*Ifq3%0=EqB|l?MfVAx`#XI@ivSn4`4Wp@`e`%oee>6 zp5k)oP1-h{NT!N(>foBme`x6V=#97`p3-8?#Ud>GN&@gaF%FAcoleR=;`|{ygJiHU zbG3%|`rP-nF`>f~vhVArkZ*9{W*c`i|Nft}EDjtDhS$tf?d9}y2Z%NHhk%jc|8IZu zdj^SJUz=eGY#FY0eoct8KL-`de+IeZkB-mDlaI3%$pF)Y0S5#22Ql+}R63?x0^uNU zeL3!-^EgZsY^GeZm!%x+9KzaDGIUdzlnN4&y8+5(qfOF{Hfjovi-$3x__BCF2!sRX z09Xe*WMN85!qS72S3HcI5LAq42MSg|+l3io7bE%#8p^@Zs3i&bqo1~j%>1Z4Y;jBy z6jMp>mO~j!=eaDq0<6`)?06-8J1LDs+FeM^Uz#!W5k+ek7Yq_c7_6|vXy&{uoUMuS z6EerwfJQ=Urd+Yo)F~{r9kGczFeuAkgR4}~O8-GCA;>Sq@xqG(MlgzkfqWB(E;2_h zWss$nAIqyJiTS54$@faHaMvt3{P2h5s2uI+5bPrMKT)C+Y_d>XuYZy`^_=$ zgT_l7O?*(Qe|yB}L9_x|N~foO(gNwa6boSW=t`zG>a7!dbCJ$osN>N*836 z!1e?1j3=BI@_f!y$e;U>G~5^dz~x+i>Gq5xx8d}4zYX!c8)?G6DX7nNq=PVm=mg!9$qobV$CvX%(WU_mSPcpiJ2tTxd}j^2so%8< z)a)Up{*PW zn$Vy#I~fZ}GJ7q^r$TrD6FO)~;iRIaQdOGHR6>=KMvd_lv1u%1vkwJ;oqHT9dZXeH za&I?|*eXqiKB0!1;&gw|&;W1&v6-wVe~K)_Sa>Q25W0&{(iTkJ#w}&>km`yw6&ET= zsSAo0n8wpe_F18%uWHawR0IMIWWM643__Hk9Axl6Xb44ZjZ!bfKuORcE{@B1+8SCa z8&2}(tT;U;>vTF2|DY&!>QOq;xSd{3WkS?)h$B@UU{GwPR^ECfkUw23I)h`#KncY; z-65j%(FnlB5(XLRPRL8|3^7U_U+ps^r0|qVP9Ujm10^ZznW~y78Dk$Z*lTP;)DM1axa=2(nt4&t}yd;Ego%6=plzLWvVU1LSRZ6+;u4dN@^)lF=EdHy$g?Sy1 zXT9-#hrX=PMFz_ZybkLuJGEb9^X*aTaSuXWT((pkS zGmdmLig^6O&Gjx$ja#2g1>%Qs)_SPGNDiR2hJuJ#2BL6fCXrx9X)22CiX@R)r^!&T z6C30ez^SmRFrB7zCYQ-l>_*T9g91<)XRWvuvED1laljPMs;bsfwutHCf<AM1at*+g1_XtWB%8klo`PpDgB5De0)BwPqg2!p zb=iFtc!uZZv8zzIK)g7)2c#6nJZ-#Fq|;c*hedxjS>U2Y>?K1a{~ZdOTh$nA3ZN#y zDGV~ux9snuVft?NvIV!@sgZ3LDLg6^A(;a6@e{tco@_4#Fs6m`cLFOmbg@#$63)`n zrQwJD;+(s?i1;IiJ@=b*C%7Psi1=oBJs9O|2mT#yh)qZHbzQl!$q!!~<~9`o{I^Jt zhtu-r75BP)SiYW1+sgDa;|p&F9F?aG!~|%%5t&^s$2MnYA4=#L~` zm#325{8H@)~4ZO1HEinyPv*2j0EU+D!#i$t0@YC$y43u)FSJ2oN!v|y&- z41SzXTq9U`8hv##ZBn;0OW9?UwQvHdeO4Naj5|BAjR8peAOIpK2VEfxGPrc}fRlQN zWrkkunl&p}XbO>K801wk+HfY|-4a~^6y%Ik3NJuhcDV#XO==;cfMX9vg@PNT5=$Tf zg3jC`Wnh6Mg!SQC?KzB0hb5jM10hyBRWwFJA$QuYIL=k78ilrg(1b5MP;7-Dn`OCG zDg|JOz?F%R-m94M0LF!&UT6k~JwquC(BW!LfFy!a@#wl6I}N1(qG>uLnQ2gKnqX-F zA|%D~V!4KT3=m8nEU1j*AFAoY?lhp%V9UKBq-gsf2Drp$(n$p=T)&G@7EmTa zD3``x%g(Obj&O~M@aonA`}TQ|bK6je$Z-yEGZiIE6y@7+cZ5Qwv2H^{#|Cw1Zh3`# zN~~+PtT4 zfw#vO+iQZY?--+vbVypC@|^NiYQ;Ae8Ile8wsGi%6`P0I6BmYU#5Z5R{R+THWZwrz zSjSQlq}|3yx5rGAq$_4%kVi|)g$OdTPI9}>Q@F$!vOZjb(;|ZK+yKk54R4V#$Uo;^ z)@QsV%wyEvz%ux{Ffd8RXT|EfoO0C$JxI$4I=TZ_WU&iE(BY6mo;k;%I6AVuTy(jq zu)Hxbvx-3VB(y(*Te}301;vteaVrhwnic5G-$bq zv90KHV|)(pYu2COqSf#NCGNunGNi~Zt_aYxD|BCohQMg0FX_w3SNj39OVBQMw|E`n zGOd=lqA^`{2yg|wrq*E)S{gxu9kMK9EzIFR1QM|$q3za8s_6t4G#f|bQFeh=;uCaPE_$)0I@Pe2ko zmc5qjWLkbI6`Vioddi>T>z~pUxFi;#fo!pv{3p@ka_8FobE;_qA$jnb{)naTvLI0O zrZZ^Qprr{g9igfYU3xaRay<;gls|XCH|Rl-=`j*j*6GLL@#n9+{n;f&d4!o;Na=!^ z3DC5V;IDzGEL21qM&`pi5B48g%N0VGD2=0F(|cG;xEDr~vr_XG%o zokPTPHQpGI1Ou0!v!z}hhUI*Nj}a-{)LC-xToOUuc`rc%T2I9_o*y2FNG+P*elsuy z3A|#gWPb~DCP@ltZh$eyIv8$-@#;dJ-}0*;ey1nC3A2z>f^l~ECHxyMheF3~924)V z;$6g}SZAo*-(`&Z0>q#6>!OSR5!^VU^B?3a5$tTX8Nl;TUX`Gyq&@9sj zF zU5#h^arCA8coeEY zKyd3FerMe-}pLf*WFO#!cAcaD-c>5w?W7QVIFN4z{86AKAd7r3|H(3M9q!A;6wv%$vaQjJ31vslMbQZb1K9M1tm>y z!ze3Zo+akt415QtafPF@*eUt~U78=OWy*(BNPr$~xcF_S1BD7}+nhN~D<@CT{p73>lw z>n~}np4U%1D*)8+ax^Xi9}(L0l9R9Yt=$G1!8H#0zhuAZAoNV;pH)*=X05 z7Ub+6KA(Ob(s9qTo=&|A_-QilUqbC^{0UTFEq&)9?RduJjO|XeIaZ^HuqA9Hz5YZ9 z#09`Z`D}8H6Ai?fsux(Rzpo`lw@FWo+t|W{_TuG$ri0kA84Ny#r5iifWbUfKh_EW$ zzLtIT7VN@^e&7|fTuy))K^?uT7%`x&Fjr02oD=p!x&ODg$LmepR_b=qfqm@H3982= zPU&wEyAYtekvo%!Rx$k?7jS+@?-CJ>Dn-BxdSGYvwpto3(gChea31draBNE^`;CW?$M@jR5Cn-;3%Ay2;luH&j;KaqM zL9n2}dwQS-aA=2(eyB;^6``f@Cqfd$3bXdpg8KfK5RRf>prIbKkh1FEI&30NeWt!A zqAO>V4&#(5!HN|43iCvxfD7Jv!$sLk$$`(00VNgM>JytolBW4~h(S5W zw(R%oAut=Av|BuQC!cf7Ld`sK*e>DjkxQ6vPEy8ZA$ObX0N7&e5I=8!%<;k)JNNE% zKrDnxwH0!i_!q>GiO3L@gh62%T)82u z>X&E)x5A>6PAFt*8Mp{3d0V3};|xkD@DeS$yBRBf01%2KQf)5MT!B)GNVT+(QsNaP zqPEfs`CcJ4j5^;7{k@P-zxE}qKo4*nsPMaKX}HH=-i`C#fK_Zogvd=zz*JlE=I5s-|#Jh9p(YlN~GzSc!qIW}DId3Zbnp|8ImC67L zza`zYV_VZ{h>{ei$PSBi_LBe+@!Y2kV>rFDrAT0U{+KE?qYVcty+2K7tAVdTX_S|}*w&b8B zGApzdjP$#A;7tn#Yb6gpSaOQXX(>4h93M_RLS>pl?zoCTn0$|LC+>$E$LN$$d#Cyd zy(W6NG}p#9v%g$v@&NUEfWMB~a6xb68sNp8a~V!TY(xO~%Ty;h+*Bo(oz_V`5*(K+09Iso`$Od%7xBhl-zGtm@%A`Qn z`Vy`v6Z<3ymra<^cRgDeA?XSHQMwIZvM(<hqF9=6X2N6d_%4vxkHQvXO+6AnFF9mo1A)Gw17fo~nEPI6hUBMt0WaLC zHW_m4Z6H`h7-xKc=5sioQsFQEsNLrkdmY0n0}HNR#$k8{8yUC<#&j75QKL1oPJ|sv zA{ktu({iu~04+Bl1b>JW7|{s@6Ea(HF2MriiqMd@OU!M~5Cn@1B%-u|-z@iRW^%cI zh+}$cnpk4r!L{`s?R*IOyEnf;Om!IgyHa$)JHmn<ew?-p02iw zCvvVsx@5fpy>-sZO#zO|LNVT^l{nV@=3hYPle? zz>#R85y^NX*Ai03-zaR54pgrZ4u#J~qvmr6?M=>9aQ-Z;IHw|WLH)f9Fb9M4h|DeM zeVt(#NoRCl3;^ru^DiLVOS1&mWMi}wH=jTG(fO^9=%6IOrCcGxjS?ZpP@30J(Rvj> z+PcVW{&58+08)6)Dre-Get9cie})8~_*rT#t6O@w7hIZ7%eU(?CUYap9Yl;fbzu+i z^Ut^|#upIY{^Z9${rS)T<)44{Gd&xoAA)Q~af=Jbt-p~i|J91lnhF`ltP`7^tK=ph zLl5hEf|ToWq9m11j#vJm7_wX_9Y>MOp;hPrZO~0NWJrzY3%cWP+)@1y?YT}r0Pq+0 zH&c)A`$wItR0S7=@fd(W7w< zjU}hhYM#=!VluiBG+(NGLZgR_Rlsk67UZa6ch+uE@GjzwP#N1T$<>`*2~JK{+SBQa z42$C0F5SnmKP=8+;Hh&@EwBll!^!P4wyDUV7aS}O?`)AM&7N7#@ArUwR$xG!fHw5A z5lHztIqp!seZ%)X|NU3L{_p?#&tLxX7oU9oqd)Og3r~D0N;`exI1z3EDB$R5EksDh zFCcs0;NlyNn$O`mCK;)NO79v3cM)fChl)M8E$h6xlr_20z)%o1@<5LVSIf) zt(bqy*EWjv_s`b*s|^*>rhYz9lh=okWsoy|n4x8n^b)7=$@0IQ?lkv=NQYvtNP`cn z_{5D^(9t}ivX){lm@gdEIvu&o$w6hu^%vLZ4oxEb)Wk^lh~(^H44q%T13ChW#s7pI zSsC-o`zb&+(q1340%4aQh+5LIocs%Vie9M7_5wy@I>+uBbV(jGJ0=pK^6EMCA@Z)c4weh z!SPpVh7BlIlmoKTv)V|;@VL2sE`CvP!xX2JwZDr?U<%)%;4sFvl7 zb%*w-mqv$Rf{Utlk51Y;32Twph*&w#Dv7lZke9bSi_Fsu9!UHRkEs3c|M1ESqCz#Ao0pAQ4EsH5SL83`Gw#K?484Rkbwho1N*ZAL?~00%KXN4Co)1WISJf)u4G zYL;YbgSfdTEK^M`yj>?Eh_7r6?dmGZmQ(%H4|sh;eKcaNY;+P$ymTof!#f7?CFt+7 z99zuEvFZFT2r|avDSLVYJ9=3N|*)>KTSXy_rx900H>ssPFIOIrq5JOl7TiI z?A&`H)z)*iE-(0m+2uutOOW6Jy-?Ub#v!(Cxg}dOmM#_%@|Xz4+l-v8w$f6t0xKjN zF7D+7XmhNcO?N`=7Xb^$2L^BLrw&i(d59}&FPL^HhI-Yg5|Q1A3MAH+AJNI77Jgt7 z*)W<1^IJaN_UC~xzm4O%gE|}*Jjz_KP}bs`lVI!+wx%M&2$31iXbX#u6zXn+irxg* zi74iRq1xNy*g$DgyQo&Eze!wN4s|>$!6&1`Rpv zYn7U@ivNm!>1%2Hdqt`??u7;}HP4joULC~uIm9yp_`x9(cGog-5dt7988xvXWb1B* zf%tWx$UdAhheneV;U}i^gk}M-TJ415E!u^Lo`X1{fgM?Ka5-M#yf^1Yz;l2EECg)0 z@D!UsDMy>lw8mb~f)69-T||O~6pwN$iq9t!R^J0zqePM`-GW`A!sC^NsT^$LbG#si zCq1arm!&T-`{qwPm>as(RjX)xa^wXAv+^m6ysR%@6N@>fQh;w*>zMl%p{A^r*CfV| zZ}xtp*YV^)0>?E!=Dl+99U~9QupwGLHC01oc$$GwQ5V4dzgTS^l)`6vU^*{>Qf}{B z;X_#6u%1sn!y7lv@WEHZ6!zQ;LeW%>?ve`X8_GKQ`UcXJMIDMr2301KOlkO{hpu0d%bON3p?TA`-!7+4<1Ak1$aL^ZTnxzx%8!@Z| zn;2|cWm^X3(r9yOrcSMw%wfPhO)ur)QH{PGI&%ejvpGvP*!i*+&lX9s`EAzSxJFVm z*JXPEX7FIePy+uBp=L!Xqoq|6dujq+?-kJJ1@5Nr;9d5TsQoiJd3or>n={vI0Rm4p zBixG=P9~w?7xiqWvj%|E5NG0PI5>cY)I&N;x}juU48Rb=G4E%3Sc5zXI^SK|VH^l^2hu{P7RJ<2W|U zgbWY9kl~>L>gMS<#{@40#NY7oC<0&Yr04nvD-YSFow*XYF|@3_hMqVqSUDmANdWxJ zfe#7Q)98wFf=}^FhPPu88$WiS41f#JjGBxcqBaig0x*ZpO*pk$L_4DjYR!Ihq- z|E8&Etgh+?b%RXg0`X8h9pI2dRB#M4p;8%J@h3<{SfMCST=d4zF*?Peu)b|P++gCR zgoePTXUR%lWVVPayfoJ5h@bEhs_bIXcDx}+uZzP`EWa6;MS>3`(e+u;)t%rboNMuN zn!Jz-P~u(3AP~-oeB^@$9+AzlVkv-(1eUtNEx0DVdgNLsL|kVPvMnVJ&9Vp)s|bEh z=Dqpm>v<6(ehku(phlvA%z))QIEhDiXyBoNhXx)R_?8-|7ty}1fTJc35#$$ z*OIa|b_jxRlpVPVdcbOQgKrwg9394(PG>ijVaSO_gjFQfs^=t=g zl}GLtbH#yp7yv-K9_D(aSrPZ9jHhElR^beFbCSXLgq4T@cK^(1ynvZ&4YbxSP{DHB1$hnA4Z9=va$%g^5BgUO*;;sg-}>B(cjEMFb==G17U5Umz5?_Xr^)Mb zgaqbxyOx$U$#p_4`ZBv@ZeOM0-$A3kFjgiRu2lf@66ko8wT=k_Ym2YH!nGIz!cjJi zI|*(2Az<9Il?4*8-1FzC9k)+%3x{Vg{VGrVnadOI znc($Yn3+3)DBdSN!kxb6w>(!}18>%LA(b_a>=TO5H!MhPdU}^dd*D)WI0A2x|DoiI z7>>GH;yivI8hB{np@D}6s)1Th;ybc6qanpMb7S7D*))rtbyd_owC+-E6q5KB;Y-dK zK7k*MbyqDKXo=*?${;7xa5DNKb6rpc#6?2`Cf{X5w%T0LXy0`yfv%b>PPpXq_?c_%NM^!EfOJrm=>9Z|wAZ z8YZ6C;RKA2gYy+6z`*7WIjkfIe*KyaH(2rJY|8>r?|@^#2?4ZxjgkaloQH)Ty-sc_ zPN_r_h(m!D6<0z@QNmXq8W(ay%LY)tn)6uSRDqA{Ljw;DJT!1i1Ix%_Krf1Wd&aBg zS}$@D>SjQkwN5CwnO0K8>a$-y9^-R0*lDsc(OPMiEWF0RwTep$i05S5c(-~gh-G7q z^$+EMRtUX=?nlX7osh`RSy-C0Tx2rB?X-viUQ)SVRZ;3-Dg)BAnk4I8NCH40$+bxl zfh5h_7c8XKfBE-ASJKDzp@D}69vb+eYoKleyILcERh@18mxIw3W9hNVp4oEQf=%Wg zJ73rS^N;wn0e;101g@l(ocl4pGHM>N1o<1!Jumm84NsV#n*H3!(b{9_xh4WsS0~+e zlR986#kE)*Z&7l&@@TFogl2AEQbdvsAaK06Uao$(q$2ZyW@dUUU~gk-yAgj zh(0v%(7;0j_iMm8yZ3Kiz}PUc-Cf_fzUY=(zC^{=RqpBQZGdvCE5KQu_#x1>BpEXf zwr)=5^$K%FS)v9PE69~A%MO#FC)I`3-8q&=o-$x96`s&Hyye5AbwtY*l~jmJ)_t_> pq&03St~4Rh0GgXMmbZzF{6D>@D?p^rP!a$D002ovPDHLkV1lOO155w_ literal 0 HcmV?d00001 diff --git a/docs/index.rst b/docs/index.rst index 20946b12..e91cddf9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,23 @@ Of course you're better off if your input is decoded properly and has no glitche ftfy will do everything it can to fix the problem. +ftfy is a heuristic that was designed (not machine-learned) by Robyn Speer, at Luminoso. If you use ftfy in research, including pre-processing your language model data, you need to cite it: see :ref:`cite`. + +.. toctree:: + :maxdepth: 1 + + config + explain + encodings + fixes + detect + avoid + cli + accuracy + modules + cite + + Some quick examples ------------------- @@ -22,7 +39,9 @@ ftfy can fix mojibake (encoding mix-ups), by detecting patterns of characters th >>> ftfy.fix_text('✔ No problems') '✔ No problems' -It can fix multiple layers of mojibake simultaneously: +Does this sound impossible? It's really not. UTF-8 is a well-designed encoding that makes it obvious when it's being misused, and a string of mojibake usually contains all the information we need to recover the original string. + +ftfy can fix multiple layers of mojibake simultaneously: >>> ftfy.fix_text('The Mona Lisa doesn’t have eyebrows.') "The Mona Lisa doesn't have eyebrows." @@ -51,416 +70,3 @@ The following text could be encoded in Windows-1252 and decoded in UTF-8, and it >>> ftfy.fix_text('IL Y MARQUÉ…') 'IL Y MARQUÉ…' - -Citing ftfy ------------ -ftfy has been used as a crucial data processing step in major NLP research. - -It's important to give credit appropriately to everyone whose work you build on -in research. This includes software, not just high-status contributions such as -mathematical models. All I ask when you use ftfy for research is that you cite -it. - -ftfy has a citable record on `Zenodo`_. A citation of ftfy may look like this: - - Robyn Speer. (2019). ftfy (Version 5.5). Zenodo. - http://doi.org/10.5281/zenodo.2591652 - -In BibTeX format, the citation is:: - - @misc{speer-2019-ftfy, - author = {Robyn Speer}, - title = {ftfy}, - note = {Version 5.5}, - year = 2019, - howpublished = {Zenodo}, - doi = {10.5281/zenodo.2591652}, - url = {https://doi.org/10.5281/zenodo.2591652} - } - -.. _Zenodo: https://zenodo.org/record/2591652 - -Basic usage ------------ - - - -Individual ways to fix text -=========================== - -.. automodule:: ftfy.fixes - :members: unescape_html, remove_terminal_escapes, uncurl_quotes, - fix_latin_ligatures, fix_character_width, fix_line_breaks, - fix_surrogates, remove_control_chars, remove_bom, decode_escapes, - fix_one_step_and_explain, apply_plan, restore_byte_a0, - replace_lossy_sequences, fix_partial_utf8_punct_in_1252 - - -Is ftfy an encoding detector? -============================= - -No. Encoding detectors have ended up being a bad idea, and they are largely responsible for _creating_ the problems that ftfy has to fix. - -The text that you put into ftfy should be Unicode that you've attempted to decode correctly. ftfy doesn't accept bytes as input. - -There is a lot of Unicode out there that has already been mangled by mojibake, even when decoded properly. That is, you might correctly interpret the text as UTF-8, and what the UTF-8 text really says is a mojibake string like "réflexion". This is when you need ftfy. - - -I really need to guess the encoding of some bytes -------------------------------------------------- - -I understand. Sometimes we can't have nice things. - -Though it's not part of the usual operation of ftfy, ftfy _does_ contain a byte-encoding-guesser that tries to be less terrible than other byte-encoding-guessers: - -.. autofunction:: ftfy.guess_bytes - - -How can I avoid producing mojibake? -=================================== - -- **Assume text is in UTF-8** unless you have a specific reason to believe it isn't. - -In the 2020s, [UTF-8 is everywhere](http://utf8everywhere.org/), especially in text meant to be transferred over the Internet. - -In Python 3, you should use the Unicode string type (`str`) for all operations. You should open text files in UTF-8: - - openfile = open(filename, encoding='utf-8', errors='replace') - -When you are specifically working with bytes and you need to turn them into text, you should decode them as UTF-8: - - text = bytebuffer.decode('utf-8', 'replace') - -The exceptions, the cases where you're not using UTF-8, are few but relevant. If you're interacting with C APIs, you'll need to represent your text as bytes in the format the API expects. Windows APIs in particular expect UTF-16. - -We're mostly past the dark days when encodings were "character maps" of 256 possible characters, one byte per character. An unfortunate thing that keeps them alive is Microsoft Excel, whose "Export" feature will pick a 256-character encoding *based on your computer's operating system and default language*. So: - -- **Don't export CSV files from Excel** if you have any other choice. - -Excel CSVs aren't even interoperable between different computers. Use Google Sheets to create CSVs, and keep Excel files in Excel format so the Unicode won't be mangled. - -If you must export a CSV-like file from Excel, you can find an option to tell Excel to export in "Unicode Text", and it will create a tab-separated UTF-16 file. This is not a very widely-used format, but at least it's not mojibake. - -You can follow `these unwieldy directions from a SalesForce help article ` to use Excel and Notepad to create a UTF-8 CSV. You can see why I don't recommend this process. - -- **Don't use chardet**. - -As described in the previous section, encoding detection is not a good idea. It was important in the '90s, during the rough transition to Unicode -- and it hasn't changed since the '90s. A heuristic designed before there was multilingual social media, before there were emoji, is not going to work correctly in the 2020s. - -chardet was built on the assumption that "encoding detection is language detection", which is no longer true. Web sites now contain text in multiple languages, and for the most part they use UTF-8 regardless of the language. - -When chardet sees the _correct_ UTF-8 encoding of an emoji, it will have no idea what it's looking at, and will guess that it's Turkish encoded in Windows-1254. On other reasonable text, it will guess the very unpopular "iso-8859-2" encoding. Because the bug is inherent to the design of chardet, it's not easily fixed. - - -Fixing mojibake and related problems -==================================== - -Mojibake --------- - -The most interesting kind of brokenness that ftfy will fix is when someone has -encoded Unicode with one standard and decoded it with a different one. This -often shows up as characters that turn into nonsense sequences (called -"mojibake"): - -- The word ``schön`` might appear as ``schön``. -- An em dash (``—``) might appear as ``—``. -- Text that was meant to be enclosed in quotation marks might end up - instead enclosed in ``“`` and ``â€<9d>``, where ``<9d>`` represents an - unprintable character. - -This causes your Unicode-aware code to end up with garbage text because someone -else (or maybe "someone else") made a mistake. - -This happens very often to real text. It's often the fault of software that -makes it difficult to use UTF-8 correctly, such as Microsoft Office and some -programming languages. The :func:`ftfy.fix_encoding` function will look for -evidence of mojibake and, when possible, it will undo the process that produced -it to get back the text that was supposed to be there. - -Does this sound impossible? It's really not. UTF-8 is a well-designed encoding -that makes it obvious when it's being misused, and a string of mojibake usually -contains all the information we need to recover the original string. - -When ftfy is tested on multilingual data from Twitter, it has a false positive -rate of less than 1 per million tweets. - - -Other fixes ------------ - -Any given text string might have other irritating properties, possibly even -interacting with the erroneous decoding. The main function of ftfy, -:func:`ftfy.fix_text`, will fix other problems along the way, such as: - -- The text could contain HTML entities such as ``&`` in place of certain - characters, when you would rather see what the characters actually are. - -- For that matter, it could contain instructions for a text terminal to - do something like change colors, but you are not sending the text to a - terminal, so those instructions are just going to look like ``^[[30m;`` - or something in the middle of the text. - -- The text could write words in non-standard ways for display purposes, - such as using the three characters ``fl`` ``o`` ``p`` for the word "flop". - This can happen when you copy text out of a PDF, for example. - -- It might not be in *NFC normalized* form. You generally want your text to be - NFC-normalized, to avoid situations where unequal sequences of codepoints - can represent exactly the same text. You can also opt for ftfy to use the - more aggressive *NFKC normalization*. - - -.. note:: - - Before version 4.0, ftfy used NFKC normalization by default. This covered a - lot of helpful fixes at once, such as expanding ligatures and replacing - "fullwidth" characters with their standard form. However, it also performed - transformations that lose information, such as converting `™` to `TM` and - `H₂O` to `H2O`. - - The default, starting in ftfy 4.0, is to use NFC normalization unless told - to use NFKC normalization (or no normalization at all). The more helpful - parts of NFKC are implemented as separate, limited fixes. - - -There are other interesting things that ftfy can do that aren't part of -the :func:`ftfy.fix_text` pipeline, such as: - -* :func:`ftfy.explain_unicode`: show you what's going on in a string, - for debugging purposes -* :func:`ftfy.fixes.decode_escapes`: do what everyone thinks the built-in - `unicode_escape` codec does, but this one doesn't *cause* mojibake - -Encodings ftfy can handle -------------------------- - -ftfy can't fix all possible mix-ups. Its goal is to cover the most common -encoding mix-ups while keeping false positives to a very low rate. - -ftfy can understand text that was decoded as any of these single-byte -encodings: - -- Latin-1 (ISO-8859-1) -- Windows-1252 (cp1252 -- used in Microsoft products) -- Windows-1251 (cp1251 -- the Russian version of cp1252) -- Windows-1250 (cp1250 -- the Eastern European version of cp1252) -- ISO-8859-2 (which is not quite the same as Windows-1250) -- MacRoman (used on Mac OS 9 and earlier) -- cp437 (used in MS-DOS and some versions of the Windows command prompt) - -when it was actually intended to be decoded as one of these variable-length -encodings: - -- UTF-8 -- CESU-8 (what some people think is UTF-8) - -It can also understand text that was intended as Windows-1252 but decoded as -Latin-1. That's the very common case where things like smart-quotes and -bullets turn into single weird control characters. - -However, ftfy cannot understand other mixups between single-byte encodings, -because it is extremely difficult to detect which mixup in particular is the -one that happened. - -We also can't handle the legacy encodings used for Chinese, Japanese, and -Korean, such as ``shift-jis`` and ``gb18030``. See `issue #34 -`_ for why this is so -hard. - -But remember that the input to ftfy is Unicode, so it handles actual -CJK *text* just fine. It just can't discover that a CJK *encoding* introduced -mojibake into the text. - - -Citing ftfy ------------ -ftfy has been used as a crucial data processing step in major NLP research. - -It's important to give credit appropriately to everyone whose work you build on -in research. This includes software, not just high-status contributions such as -mathematical models. All I ask when you use ftfy for research is that you cite -it. - -ftfy has a citable record on `Zenodo`_. A citation of ftfy may look like this: - - Robyn Speer. (2019). ftfy (Version 5.5). Zenodo. - http://doi.org/10.5281/zenodo.2591652 - -In BibTeX format, the citation is:: - - @misc{speer-2019-ftfy, - author = {Robyn Speer}, - title = {ftfy}, - note = {Version 5.5}, - year = 2019, - howpublished = {Zenodo}, - doi = {10.5281/zenodo.2591652}, - url = {https://doi.org/10.5281/zenodo.2591652} - } - -.. _Zenodo: https://zenodo.org/record/2591652 - - -Using ftfy ----------- - -The main function, :func:`ftfy.fix_text`, will run text through a sequence of -fixes. If the text changed, it will run them through again, so that you can be -sure the output ends up in a standard form that will be unchanged by -:func:`ftfy.fix_text`. - -All the fixes are on by default, but you can pass options to turn them off. -Check that the default fixes are appropriate for your use case. For example: - -- You should set `fix_entities` to False if the output is meant to be - interpreted as HTML. - -- You should set `fix_character_width` to False if you want to preserve the - spacing of CJK text. - -- You should set `uncurl_quotes` to False if you want to preserve quotation - marks with nice typography. You could even consider doing quite the opposite - of `uncurl_quotes`, running `smartypants`_ on the result to make all the - punctuation nice. - -.. _smartypants: http://pythonhosted.org/smartypants/ - -If the only fix you need is to detect and repair decoding errors (mojibake), then -you should use :func:`ftfy.fix_encoding` directly. - -.. versionchanged:: 4.0 - The default normalization was changed from `'NFKC'` to `'NFC'`. The options - *fix_latin_ligatures* and *fix_character_width* were added to implement some - of the less lossy parts of NFKC normalization on top of NFC. - -.. autofunction:: ftfy.fix_text - -.. autofunction:: ftfy.fix_text_segment - -.. autofunction:: ftfy.fix_encoding - -.. autofunction:: ftfy.fix_file - -.. autofunction:: ftfy.explain_unicode - - -A note on encoding detection ----------------------------- - -:func:`ftfy.fix_text` expects its input to be a Python 3 `str` (a Unicode -string). If you pass in `bytes` instead, ftfy will point you to the `Python -Unicode HOWTO`_. - -.. _`Python Unicode HOWTO`: http://docs.python.org/3/howto/unicode.html - -Now, you may know that your input is a mess of bytes in an unknown encoding, -and you might want a tool that can just statistically analyze those bytes and -predict what encoding they're in. - -ftfy is not that tool. The :func:`ftfy.guess_bytes` function it contains will -do this in very limited cases, but to support more encodings from around the -world, something more is needed. - -You may have heard of `chardet`. Chardet is admirable, but it doesn't -completely do the job either. Its heuristics are designed for multi-byte -encodings, such as UTF-8 and the language-specific encodings used in East Asian -languages. It works badly on single-byte encodings, to the point where it will -output wrong answers with high confidence. - -:func:`ftfy.guess_bytes` doesn't even try the East Asian encodings, so the -ideal thing would combine the simple heuristic of :func:`ftfy.guess_bytes` with -the multibyte character set detection of `chardet`. This ideal thing doesn't -exist yet. - - -Accuracy --------- -.. include:: accuracy.rst - - -Command-line tool ------------------ -ftfy can be used from the command line. By default, it takes UTF-8 input and -writes it to UTF-8 output, fixing problems in its Unicode as it goes. - -Here's the usage documentation for the `ftfy` command:: - - usage: ftfy [-h] [-o OUTPUT] [-g] [-e ENCODING] [-n NORMALIZATION] - [--preserve-entities] - [filename] - - ftfy (fixes text for you), version 5.0 - - positional arguments: - filename The file whose Unicode is to be fixed. Defaults to -, - meaning standard input. - - optional arguments: - -h, --help show this help message and exit - -o OUTPUT, --output OUTPUT - The file to output to. Defaults to -, meaning standard - output. - -g, --guess Ask ftfy to guess the encoding of your input. This is - risky. Overrides -e. - -e ENCODING, --encoding ENCODING - The encoding of the input. Defaults to UTF-8. - -n NORMALIZATION, --normalization NORMALIZATION - The normalization of Unicode to apply. Defaults to - NFC. Can be "none". - --preserve-entities Leave HTML entities as they are. The default is to - decode them, as long as no HTML tags have appeared in - the file. - - -Module documentation -==================== - -*ftfy.fixes*: how individual fixes are implemented --------------------------------------------------- - -.. automodule:: ftfy.fixes - :members: unescape_html, remove_terminal_escapes, uncurl_quotes, - fix_latin_ligatures, fix_character_width, fix_line_breaks, - fix_surrogates, remove_control_chars, remove_bom, decode_escapes, - fix_one_step_and_explain, apply_plan, restore_byte_a0, - replace_lossy_sequences, fix_partial_utf8_punct_in_1252 - - -*ftfy.badness*: measures the "badness" of text ----------------------------------------------- - -.. automodule:: ftfy.badness - :members: - - -*ftfy.bad_codecs*: support some encodings Python doesn't like -------------------------------------------------------------- - -.. automodule:: ftfy.bad_codecs - -"Sloppy" encodings -~~~~~~~~~~~~~~~~~~ -.. automodule:: ftfy.bad_codecs.sloppy - -Variants of UTF-8 -~~~~~~~~~~~~~~~~~ -.. automodule:: ftfy.bad_codecs.utf8_variants - - -*ftfy.formatting*: justify Unicode text in a monospaced terminal ----------------------------------------------------------------- -.. automodule:: ftfy.formatting - :members: - - -*ftfy.chardata* and *ftfy.build_data*: trivia about characters --------------------------------------------------------------- -These files load information about the character properties in Unicode 11.0. -Yes, even if your version of Python doesn't support Unicode 11.0. This ensures -that ftfy's behavior is consistent across versions. - -.. automodule:: ftfy.chardata - :members: - -.. autofunction:: ftfy.build_data.make_char_data_file diff --git a/docs/modules.rst b/docs/modules.rst new file mode 100644 index 00000000..67287ad1 --- /dev/null +++ b/docs/modules.rst @@ -0,0 +1,44 @@ +Module documentation +==================== + +*ftfy.fixes*: how individual fixes are implemented +-------------------------------------------------- + +.. automodule:: ftfy.fixes + :members: + + +*ftfy.badness*: measures the "badness" of text +---------------------------------------------- + +.. automodule:: ftfy.badness + :members: + + +*ftfy.bad_codecs*: support some encodings Python doesn't like +------------------------------------------------------------- + +.. automodule:: ftfy.bad_codecs + +"Sloppy" encodings +~~~~~~~~~~~~~~~~~~ +.. automodule:: ftfy.bad_codecs.sloppy + +Variants of UTF-8 +~~~~~~~~~~~~~~~~~ +.. automodule:: ftfy.bad_codecs.utf8_variants + + +*ftfy.formatting*: justify Unicode text in a monospaced terminal +---------------------------------------------------------------- +.. automodule:: ftfy.formatting + :members: + + +*ftfy.chardata*: trivia about characters +---------------------------------------- + + +.. automodule:: ftfy.chardata + :members: + diff --git a/docs/poem.txt b/docs/poem.txt new file mode 100644 index 00000000..c6e2da8d --- /dev/null +++ b/docs/poem.txt @@ -0,0 +1,17 @@ + Once there was a little o, + with an accent on top like só. + + It started out as UTF8, + (universal since '98), + but the program only knew latin1, + and changed little ó to "ó" for fun. + + A second program saw the "ó" + and said "I know HTML entity!" + So "ó" was smartened to "&ATILDE;&SUP3;" + and passed on through happily. + + Another program saw the tangle + (more precisely, ampersands to mangle) + and thus the humble "&ATILDE;&SUP3;" + became "&AMP;ATILDE;&AMP;SUP3;" diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 120dd0e1..74a1d3b7 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -6,7 +6,7 @@ """ import unicodedata -from typing import Optional, Tuple, NamedTuple, Union, List +from typing import List, NamedTuple, Optional, Tuple, Union from ftfy import bad_codecs from ftfy import chardata, fixes @@ -21,13 +21,25 @@ bad_codecs.ok() -class TextFixerConfig(NamedTuple): +class ExplainedText(NamedTuple): + """ + The return type from ftfy's functions that provide an "explanation" of which + steps it applied to fix the text, such as :func:`fix_and_explain()`. + + When the 'explain' option is disabled, these functions return the same + type, but the `explanation` will be None. """ + text: str + explanation: Optional[List[Tuple[str, str]]] + + +class TextFixerConfig(NamedTuple): + r""" A TextFixerConfig object stores configuration options for ftfy. It's implemented as a namedtuple with defaults, so you can instantiate it by providing the values to change from their defaults as keyword arguments. - For example, to disable 'unescape_html' and keep the rest of the defaults: + For example, to disable 'unescape_html' and keep the rest of the defaults:: TextFixerConfig(unescape_html=False) @@ -35,26 +47,26 @@ class TextFixerConfig(NamedTuple): - `unescape_html`: "auto" - Configures whether to replace HTML entities such as & with the character - they represent. "auto" says to do this by default, but disable it when a - literal < character appears, indicating that the input is actual HTML and - entities should be preserved. The value can be True, to always enable this - fixer, or False, to always disable it. + Configures whether to replace HTML entities such as & with the character + they represent. "auto" says to do this by default, but disable it when a + literal < character appears, indicating that the input is actual HTML and + entities should be preserved. The value can be True, to always enable this + fixer, or False, to always disable it. - `remove_terminal_escapes`: True - Removes "ANSI" terminal escapes, such as for changing the color of text in a - terminal window. + Removes "ANSI" terminal escapes, such as for changing the color of text in a + terminal window. - `fix_encoding`: True - Detect mojibake and attempt to fix it by decoding the text in a different - encoding standard. + Detect mojibake and attempt to fix it by decoding the text in a different + encoding standard. - The following four options affect `fix_encoding` works, and do nothing if - `fix_encoding` is False: + The following four options affect `fix_encoding` works, and do nothing if + `fix_encoding` is False: - - `restore_byte_a0`: True + - `restore_byte_a0`: True Allow a literal space (U+20) to be interpreted as a non-breaking space (U+A0) when that would make it part of a fixable mojibake string. @@ -64,13 +76,13 @@ class TextFixerConfig(NamedTuple): mojibake. Disabling `restore_byte_a0` is safer from false positives, but creates false negatives. - - `replace_lossy_sequences`: True + - `replace_lossy_sequences`: True Detect mojibake that has been partially replaced by the characters '�' or '?'. If the mojibake could be decoded otherwise, replace the detected sequence with '�'. - - `decode_inconsistent_utf8`: True + - `decode_inconsistent_utf8`: True When we see sequences that distinctly look like UTF-8 mojibake, but there's no consistent way to reinterpret the string in a new encoding, @@ -79,7 +91,7 @@ class TextFixerConfig(NamedTuple): This helps to decode strings that are concatenated from different encodings. - - `fix_c1_controls`: True + - `fix_c1_controls`: True Replace C1 control characters (the useless characters U+80 - U+9B that come from Latin-1) with their Windows-1252 equivalents, like HTML5 does, @@ -87,59 +99,59 @@ class TextFixerConfig(NamedTuple): - `fix_latin_ligatures`: True - Replace common Latin-alphabet ligatures, such as 'fi', with the - letters they're made of. + Replace common Latin-alphabet ligatures, such as ``fi``, with the + letters they're made of. - `fix_character_width`: True - Replace fullwidth Latin characters and halfwidth Katakana with - their more standard widths. + Replace fullwidth Latin characters and halfwidth Katakana with + their more standard widths. - `uncurl_quotes`: True - Replace curly quotes with straight quotes. + Replace curly quotes with straight quotes. - `fix_line_breaks`: True - Replace various forms of line breaks with the standard Unix line - break, '\n'. + Replace various forms of line breaks with the standard Unix line + break, ``\n``. - `fix_surrogates`: True - Replace sequences of UTF-16 surrogate codepoints with the character - they were meant to encode. This fixes text that was decoded with the - obsolete UCS-2 standard, and allows it to support high-numbered - codepoints such as emoji. + Replace sequences of UTF-16 surrogate codepoints with the character + they were meant to encode. This fixes text that was decoded with the + obsolete UCS-2 standard, and allows it to support high-numbered + codepoints such as emoji. - `remove_control_chars`: True - Remove certain control characters that have no displayed effect on text. + Remove certain control characters that have no displayed effect on text. - `normalization`: "NFC" - Choose what kind of Unicode normalization is applied. Usually, we apply - NFC normalization, so that letters followed by combining characters become - single combined characters. + Choose what kind of Unicode normalization is applied. Usually, we apply + NFC normalization, so that letters followed by combining characters become + single combined characters. - Changing this to "NFKC" applies more compatibility conversions, such as - replacing the 'micro sign' with a standard Greek lowercase mu, which looks - identical. However, some NFKC normalizations change the meaning of text, - such as converting "10³" to "103". + Changing this to "NFKC" applies more compatibility conversions, such as + replacing the 'micro sign' with a standard Greek lowercase mu, which looks + identical. However, some NFKC normalizations change the meaning of text, + such as converting "10³" to "103". `normalization` can be None, to apply no normalization. - `max_decode_length`: 1_000_000 - The maximum size of "segment" that ftfy will try to fix all at once. + The maximum size of "segment" that ftfy will try to fix all at once. - `explain`: True - Whether to compute 'explanations', lists describing what ftfy changed. - When this is False, the explanation will be None, and the code that - builds the explanation will be skipped, possibly saving time. + Whether to compute 'explanations', lists describing what ftfy changed. + When this is False, the explanation will be None, and the code that + builds the explanation will be skipped, possibly saving time. - Functions that accept TextFixerConfig and don't return an explanation - will automatically set `explain` to False. + Functions that accept TextFixerConfig and don't return an explanation + will automatically set `explain` to False. """ unescape_html: Union[str, bool] = "auto" remove_terminal_escapes: bool = True @@ -159,11 +171,6 @@ class TextFixerConfig(NamedTuple): explain: bool = True -class ExplainedText(NamedTuple): - text: str - explanation: Optional[List[Tuple[str, str]]] - - FIXERS = { "unescape_html": fixes.unescape_html, "remove_terminal_escapes": fixes.remove_terminal_escapes, @@ -227,22 +234,12 @@ def fix_text(text: str, config: Optional[TextFixerConfig] = None, **kwargs) -> s >>> fix_text('✔ No problems') '✔ No problems' - >>> fix_text('ünicode') - 'ünicode' - >>> print(fix_text("¯\\_(ã\x83\x84)_/¯")) ¯\_(ツ)_/¯ - >>> fix_text('Broken text… it’s flubberific!', - ... normalization='NFKC') + >>> fix_text('Broken text… it’s flubberific!') "Broken text... it's flubberific!" - >>> fix_text('HTML entities <3') - 'HTML entities <3' - - >>> fix_text('HTML entities <3') - 'HTML entities <3' - >>> fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ') 'LOUD NOISES' @@ -258,7 +255,7 @@ def fix_text(text: str, config: Optional[TextFixerConfig] = None, **kwargs) -> s fields of the TextFixerConfig object. For example, here are two ways to fix text but skip the "uncurl_quotes" - step: + step:: fix_text(text, TextFixerConfig(uncurl_quotes=False)) fix_text(text, uncurl_quotes=False) @@ -370,6 +367,18 @@ def fix_encoding_and_explain( This includes fixing text by encoding and decoding it in different encodings, as well as the subordinate fixes `restore_byte_a0`, `replace_lossy_sequences`, `decode_inconsistent_utf8`, and `fix_c1_controls`. + + Examples:: + + >>> fix_encoding_and_explain("só") + ExplainedText(text='só', explanation=[('encode', 'latin-1'), ('decode', 'utf-8')]) + + >>> result = fix_encoding_and_explain("voilà le travail") + >>> result.text + 'voilà le travail' + >>> result.explanation + [('encode', 'latin-1'), ('transcode', 'restore_byte_a0'), ('decode', 'utf-8')] + """ if config is None: config = TextFixerConfig() @@ -499,6 +508,11 @@ def fix_encoding(text: str, config: TextFixerConfig = None, **kwargs): """ Apply just the encoding-fixing steps of ftfy to this text. Returns the fixed text, discarding the explanation. + + >>> fix_encoding("ó") + 'ó' + >>> fix_encoding("&ATILDE;&SUP3;") + '&ATILDE;&SUP3;' """ if config is None: config = TextFixerConfig() @@ -618,7 +632,7 @@ def guess_bytes(bstring): return bstring.decode("sloppy-windows-1252"), "sloppy-windows-1252" -def apply_plan(text, plan): +def apply_plan(text: str, plan: List[Tuple[str, str]]): """ Apply a plan for fixing the encoding of text. @@ -631,8 +645,16 @@ def apply_plan(text, plan): - `'transcode'`: convert bytes to bytes, using the function named `arg` - `'apply'`: convert a string to a string, using the function named `arg` - The functions that can be applied by 'transcode' and 'apply' appear in - the dictionary named `FIXERS`. + The functions that can be applied by 'transcode' and 'apply' are + specifically those that appear in the dictionary named `FIXERS`. They + can also can be imported from the `ftfy.fixes` module. + + Example:: + + >>> mojibake = "schön" + >>> text, plan = fix_and_explain(mojibake) + >>> apply_plan(mojibake, plan) + 'schön' """ obj = text for operation, encoding in plan: @@ -651,7 +673,7 @@ def apply_plan(text, plan): return obj -def explain_unicode(text): +def explain_unicode(text: str): """ A utility method that's useful for debugging mysterious Unicode. diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 15b1fff8..662a9908 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -222,11 +222,11 @@ def _buffer_decode_surrogates(sup, input, errors, final): # Everything below here is boilerplate that matches the modules in the # built-in `encodings` package. -def encode(input: str, errors: str='strict') -> Tuple[bytes, int]: +def encode(input, errors='strict'): return IncrementalEncoder(errors).encode(input, final=True), len(input) -def decode(input: bytes, errors: str='strict') -> Tuple[str, int]: +def decode(input, errors='strict'): return IncrementalDecoder(errors).decode(input, final=True), len(input) diff --git a/ftfy/badness.py b/ftfy/badness.py index 1a09a1d1..89b3f54a 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -3,10 +3,325 @@ """ import warnings +import re from ftfy import chardata +def _char_range(start, end): + """ + Get a contiguous range of characters as a string. Useful for including + in a regex without making the representation _only_ usable in regexes. + """ + return "".join(chr(codept) for codept in range(start, end + 1)) + + +# There are only 403 characters that occur in known UTF-8 mojibake, and we can +# characterize them: + +MOJIBAKE_CATEGORIES = { + # Characters that appear in many different contexts. Sequences that contain + # them are not inherently mojibake + "common": ( + "\N{NO-BREAK SPACE}" + "\N{SOFT HYPHEN}" + "\N{MIDDLE DOT}" + "\N{ACUTE ACCENT}" + "\N{EN DASH}" + "\N{EM DASH}" + "\N{HORIZONTAL BAR}" + "\N{HORIZONTAL ELLIPSIS}" + "\N{RIGHT SINGLE QUOTATION MARK}" + ), + # the C1 control character range, which have no uses outside of mojibake anymore + "c1": _char_range(0x80, 0x9F), + # Characters that are nearly 100% used in mojibake + "bad": ( + "\N{BROKEN BAR}" + "\N{CURRENCY SIGN}" + "\N{DIAERESIS}" + "\N{NOT SIGN}" + "\N{MACRON}" + "\N{PILCROW SIGN}" + "\N{SECTION SIGN}" + "\N{CEDILLA}" + "\N{LATIN SMALL LETTER F WITH HOOK}" + "\N{MODIFIER LETTER CIRCUMFLEX ACCENT}" # it's not a modifier + "\N{CARON}" + "\N{BREVE}" + "\N{OGONEK}" + "\N{SMALL TILDE}" + "\N{DAGGER}" + "\N{DOUBLE DAGGER}" + "\N{PER MILLE SIGN}" + "\N{REVERSED NOT SIGN}" + "\N{LOZENGE}" + "\ufffd" + # Theoretically these would appear in 'numeric' contexts, but when they + # co-occur with other mojibake characters, it's not really ambiguous + "\N{FEMININE ORDINAL INDICATOR}" + "\N{MASCULINE ORDINAL INDICATOR}" + "\N{NUMERO SIGN}" + ), + "currency": ( + "\N{CENT SIGN}" + "\N{POUND SIGN}" + "\N{YEN SIGN}" + "\N{PESETA SIGN}" + "\N{EURO SIGN}" + ), + "start_punctuation": ( + "\N{INVERTED EXCLAMATION MARK}" + "\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}" + "\N{INVERTED QUESTION MARK}" + "\N{COPYRIGHT SIGN}" + "\N{GREEK TONOS}" + "\N{GREEK DIALYTIKA TONOS}" + "\N{LEFT SINGLE QUOTATION MARK}" + "\N{SINGLE LOW-9 QUOTATION MARK}" + "\N{LEFT DOUBLE QUOTATION MARK}" + "\N{DOUBLE LOW-9 QUOTATION MARK}" + "\N{BULLET}" + "\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}" + "\uf8ff" # OS-specific symbol, usually the Apple logo + ), + "end_punctuation": ( + "\N{REGISTERED SIGN}" + "\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}" + "\N{DOUBLE ACUTE ACCENT}" + "\N{RIGHT DOUBLE QUOTATION MARK}" + "\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}" + "\N{TRADE MARK SIGN}" + ), + "numeric": ( + "\N{SUPERSCRIPT TWO}" + "\N{SUPERSCRIPT THREE}" + "\N{SUPERSCRIPT ONE}" + "\N{PLUS-MINUS SIGN}" + "\N{VULGAR FRACTION ONE QUARTER}" + "\N{VULGAR FRACTION ONE HALF}" + "\N{VULGAR FRACTION THREE QUARTERS}" + "\N{MULTIPLICATION SIGN}" + "\N{MICRO SIGN}" + "\N{DIVISION SIGN}" + "\N{FRACTION SLASH}" + "\N{PARTIAL DIFFERENTIAL}" + "\N{INCREMENT}" + "\N{N-ARY PRODUCT}" + "\N{N-ARY SUMMATION}" + "\N{SQUARE ROOT}" + "\N{INFINITY}" + "\N{INTERSECTION}" + "\N{INTEGRAL}" + "\N{ALMOST EQUAL TO}" + "\N{NOT EQUAL TO}" + "\N{IDENTICAL TO}" + "\N{LESS-THAN OR EQUAL TO}" + "\N{GREATER-THAN OR EQUAL TO}" + ), + "kaomoji": ( + _char_range(0xD2, 0xD6) + + _char_range(0xD9, 0xDC) + + _char_range(0xF2, 0xF6) + + _char_range(0xF8, 0xFC) + + "\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}" + "\N{DEGREE SIGN}" + ), + "upper_accented": ( + _char_range(0xC0, 0xD1) + + # skip capital O's and U's that could be used in kaomoji, but + # include Ø because it's very common in Arabic mojibake: + "\N{LATIN CAPITAL LETTER O WITH STROKE}" + "\N{LATIN CAPITAL LETTER U WITH DIAERESIS}" + "\N{LATIN CAPITAL LETTER Y WITH ACUTE}" + "\N{LATIN CAPITAL LETTER A WITH BREVE}" + "\N{LATIN CAPITAL LETTER A WITH OGONEK}" + "\N{LATIN CAPITAL LETTER C WITH ACUTE}" + "\N{LATIN CAPITAL LETTER C WITH CARON}" + "\N{LATIN CAPITAL LETTER D WITH CARON}" + "\N{LATIN CAPITAL LETTER D WITH STROKE}" + "\N{LATIN CAPITAL LETTER E WITH OGONEK}" + "\N{LATIN CAPITAL LETTER E WITH CARON}" + "\N{LATIN CAPITAL LETTER G WITH BREVE}" + "\N{LATIN CAPITAL LETTER I WITH DOT ABOVE}" + "\N{LATIN CAPITAL LETTER L WITH ACUTE}" + "\N{LATIN CAPITAL LETTER L WITH CARON}" + "\N{LATIN CAPITAL LETTER L WITH STROKE}" + "\N{LATIN CAPITAL LETTER N WITH ACUTE}" + "\N{LATIN CAPITAL LETTER N WITH CARON}" + "\N{LATIN CAPITAL LIGATURE OE}" + "\N{LATIN CAPITAL LETTER R WITH CARON}" + "\N{LATIN CAPITAL LETTER S WITH ACUTE}" + "\N{LATIN CAPITAL LETTER S WITH CEDILLA}" + "\N{LATIN CAPITAL LETTER S WITH CARON}" + "\N{LATIN CAPITAL LETTER T WITH CEDILLA}" + "\N{LATIN CAPITAL LETTER T WITH CARON}" + "\N{LATIN CAPITAL LETTER U WITH RING ABOVE}" + "\N{LATIN CAPITAL LETTER U WITH DOUBLE ACUTE}" + "\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}" + "\N{LATIN CAPITAL LETTER Z WITH ACUTE}" + "\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}" + "\N{LATIN CAPITAL LETTER Z WITH CARON}" + "\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}" + ), + "lower_accented": ( + "\N{LATIN SMALL LETTER SHARP S}" + _char_range(0xE0, 0xF1) + + # skip o's and u's that could be used in kaomoji + "\N{LATIN SMALL LETTER A WITH BREVE}" + "\N{LATIN SMALL LETTER A WITH OGONEK}" + "\N{LATIN SMALL LETTER C WITH ACUTE}" + "\N{LATIN SMALL LETTER C WITH CARON}" + "\N{LATIN SMALL LETTER D WITH CARON}" + "\N{LATIN SMALL LETTER D WITH STROKE}" + "\N{LATIN SMALL LETTER E WITH OGONEK}" + "\N{LATIN SMALL LETTER E WITH CARON}" + "\N{LATIN SMALL LETTER G WITH BREVE}" + "\N{LATIN SMALL LETTER L WITH ACUTE}" + "\N{LATIN SMALL LETTER L WITH CARON}" + "\N{LATIN SMALL LETTER L WITH STROKE}" + "\N{LATIN SMALL LIGATURE OE}" + "\N{LATIN SMALL LETTER R WITH ACUTE}" + "\N{LATIN SMALL LETTER S WITH ACUTE}" + "\N{LATIN SMALL LETTER S WITH CEDILLA}" + "\N{LATIN SMALL LETTER S WITH CARON}" + "\N{LATIN SMALL LETTER T WITH CARON}" + "\N{LATIN SMALL LETTER Z WITH ACUTE}" + "\N{LATIN SMALL LETTER Z WITH DOT ABOVE}" + "\N{LATIN SMALL LETTER Z WITH CARON}" + "\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}" + "\N{LATIN SMALL LIGATURE FI}" + "\N{LATIN SMALL LIGATURE FL}" + ), + "upper_common": ( + "\N{LATIN CAPITAL LETTER THORN}" + + _char_range(0x391, 0x3A9) # Greek capital letters + # not included under 'accented' because these can commonly + # occur at ends of words, in positions where they'd be detected + # as mojibake + + "\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}" + "\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}" + "\N{GREEK CAPITAL LETTER ETA WITH TONOS}" + "\N{GREEK CAPITAL LETTER IOTA WITH TONOS}" + "\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}" + "\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}" + "\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}" + "\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}" + "\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}" + + _char_range(0x401, 0x42F) # Cyrillic capital letters + ), + "lower_common": ( + # lowercase thorn does not appear in mojibake + _char_range(0x3B1, 0x3C6) # Greek lowercase letters + + "\N{GREEK SMALL LETTER ALPHA WITH TONOS}" + "\N{GREEK SMALL LETTER EPSILON WITH TONOS}" + "\N{GREEK SMALL LETTER ETA WITH TONOS}" + "\N{GREEK SMALL LETTER IOTA WITH TONOS}" + "\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}" + + _char_range(0x430, 0x45F) # Cyrillic lowercase letters + ), + "box": ("─│┌┐┘├┤┬┼" + _char_range(0x2550, 0x256C) + "▀▄█▌▐░▒▓"), +} + + +# We can now build a regular expression that detects unlikely juxtapositions +# of characters, mostly based on their categories + +BADNESS_RE = re.compile( + r""" + [{c1}] + | + [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] + | + [{bad}] [{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] + | + [{lower_accented}{lower_common}{box}{end_punctuation}{currency}{numeric}] [{upper_accented}] + | + [{box}{end_punctuation}{currency}{numeric}] [{lower_accented}] + | + # leave out [upper_accented][currency] without further info, because it's used in some + # fancy leetspeak-esque writing + [{lower_accented}{box}{end_punctuation}] [{currency}] + | + \s [{upper_accented}] [{currency}] + | + [{upper_accented}{box}{end_punctuation}] [{numeric}] + | + [{lower_accented}{upper_accented}{currency}{numeric}{box}] [{end_punctuation}] [{start_punctuation}] + | + [{currency}{numeric}{box}] [{start_punctuation}] + | + [a-z] [{upper_accented}] [{start_punctuation}{currency}] + | + [{box}] [{kaomoji}] + | + [{lower_accented}{upper_accented}{currency}{numeric}{start_punctuation}{end_punctuation}] [{box}] + | + [{box}] [{end_punctuation}] + | + [{lower_accented}{upper_accented}] [{end_punctuation}] \\w + | + + # The ligature œ when not followed by an unaccented Latin letter + [Œœ][^A-Za-z] + | + + # Common Windows-1252 2-character mojibake that isn't covered by the cases above + [ÂÃÎÐ][€Šš¢£Ÿž\xa0\xad®©°·»{end_punctuation}–—´] + | + × [²³] + | + + # Windows-1252 mojibake that starts 3-character sequences for some South Asian + # alphabets + à[²µ¹¼½¾] + | + + # MacRoman mojibake that isn't covered by the cases above + √[±∂†≠®™´≤≥¥µø] + | + ≈[°¢] + | + ‚Ä[ìîïòôúùû†°¢π] + | + ‚[âó][àä°ê] + | + + # Windows-1251 mojibake of characters in the U+2000 range + †+ | + + # Windows-1251 mojibake of Latin-1 characters and/or the Cyrillic alphabet. + # Because the 2-character sequences involved here may be common, we require + # seeing a 3-character sequence. + [ВГРС][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°µ][ВГРС] + | + + # Windows-1252 encodings of 'à' and 'á' + Ã[\xa0¡] + | + [a-z]\s?Ã[ ] + | + ^Ã[ ] + | + + # Windows-1253 mojibake of characters in the U+2000 range + β€[™\xa0Ά\xad®°] + | + + # Windows-1253 mojibake of Latin-1 characters and/or the Greek alphabet + [ΒΓΞΟ][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°][ΒΓΞΟ] +""".format( + **MOJIBAKE_CATEGORIES + ), + re.VERBOSE, +) + + def sequence_weirdness(text): + """ + This was the name of the heuristic used in ftfy 2.x through 5.x. As an + attempt at compatibility with external code that calls the heuristic + directly, we redirect to our new heuristic, :func:`badness`. + """ warnings.warn( "`sequence_weirdness()` is an old heuristic, and the current " "closest equivalent is `ftfy.badness.badness()`" @@ -16,14 +331,19 @@ def sequence_weirdness(text): def badness(text): """ - Get the 'badness' of a sequence of text. A badness greater than 0 indicates - that some of it seems to be mojibake. + Get the 'badness' of a sequence of text, counting the number of unlikely + character sequences. A badness greater than 0 indicates that some of it + seems to be mojibake. """ - return len(chardata.BADNESS_RE.findall(text)) + return len(BADNESS_RE.findall(text)) def is_bad(text): """ Returns true iff the given text looks like it contains mojibake. + + This can be faster than `badness`, because it returns when the first match + is found to a regex instead of counting matches. Note that as strings get + longer, they have a higher chance of returning True for `is_bad(string)`. """ - return bool(chardata.BADNESS_RE.search(text)) + return bool(BADNESS_RE.search(text)) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 8647261a..9e1a6e0b 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -247,309 +247,6 @@ def _build_width_map(): WIDTH_MAP = _build_width_map() -def _char_range(start, end): - """ - Get a contiguous range of characters as a string. Useful for including - in a regex without making the representation _only_ usable in regexes. - """ - return "".join(chr(codept) for codept in range(start, end + 1)) - - -# There are only 403 characters that occur in known UTF-8 mojibake, and we can -# characterize them: - -MOJIBAKE_CATEGORIES = { - # Characters that appear in many different contexts. Sequences that contain - # them are not inherently mojibake - "common": ( - "\N{NO-BREAK SPACE}" - "\N{SOFT HYPHEN}" - "\N{MIDDLE DOT}" - "\N{ACUTE ACCENT}" - "\N{EN DASH}" - "\N{EM DASH}" - "\N{HORIZONTAL BAR}" - "\N{HORIZONTAL ELLIPSIS}" - "\N{RIGHT SINGLE QUOTATION MARK}" - ), - # the C1 control character range, which have no uses outside of mojibake anymore - "c1": _char_range(0x80, 0x9F), - # Characters that are nearly 100% used in mojibake - "bad": ( - "\N{BROKEN BAR}" - "\N{CURRENCY SIGN}" - "\N{DIAERESIS}" - "\N{NOT SIGN}" - "\N{MACRON}" - "\N{PILCROW SIGN}" - "\N{SECTION SIGN}" - "\N{CEDILLA}" - "\N{LATIN SMALL LETTER F WITH HOOK}" - "\N{MODIFIER LETTER CIRCUMFLEX ACCENT}" # it's not a modifier - "\N{CARON}" - "\N{BREVE}" - "\N{OGONEK}" - "\N{SMALL TILDE}" - "\N{DAGGER}" - "\N{DOUBLE DAGGER}" - "\N{PER MILLE SIGN}" - "\N{REVERSED NOT SIGN}" - "\N{LOZENGE}" - "\ufffd" - # Theoretically these would appear in 'numeric' contexts, but when they - # co-occur with other mojibake characters, it's not really ambiguous - "\N{FEMININE ORDINAL INDICATOR}" - "\N{MASCULINE ORDINAL INDICATOR}" - "\N{NUMERO SIGN}" - ), - "currency": ( - "\N{CENT SIGN}" - "\N{POUND SIGN}" - "\N{YEN SIGN}" - "\N{PESETA SIGN}" - "\N{EURO SIGN}" - ), - "start_punctuation": ( - "\N{INVERTED EXCLAMATION MARK}" - "\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}" - "\N{INVERTED QUESTION MARK}" - "\N{COPYRIGHT SIGN}" - "\N{GREEK TONOS}" - "\N{GREEK DIALYTIKA TONOS}" - "\N{LEFT SINGLE QUOTATION MARK}" - "\N{SINGLE LOW-9 QUOTATION MARK}" - "\N{LEFT DOUBLE QUOTATION MARK}" - "\N{DOUBLE LOW-9 QUOTATION MARK}" - "\N{BULLET}" - "\N{SINGLE LEFT-POINTING ANGLE QUOTATION MARK}" - "\uf8ff" # OS-specific symbol, usually the Apple logo - ), - "end_punctuation": ( - "\N{REGISTERED SIGN}" - "\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}" - "\N{DOUBLE ACUTE ACCENT}" - "\N{RIGHT DOUBLE QUOTATION MARK}" - "\N{SINGLE RIGHT-POINTING ANGLE QUOTATION MARK}" - "\N{TRADE MARK SIGN}" - ), - "numeric": ( - "\N{SUPERSCRIPT TWO}" - "\N{SUPERSCRIPT THREE}" - "\N{SUPERSCRIPT ONE}" - "\N{PLUS-MINUS SIGN}" - "\N{VULGAR FRACTION ONE QUARTER}" - "\N{VULGAR FRACTION ONE HALF}" - "\N{VULGAR FRACTION THREE QUARTERS}" - "\N{MULTIPLICATION SIGN}" - "\N{MICRO SIGN}" - "\N{DIVISION SIGN}" - "\N{FRACTION SLASH}" - "\N{PARTIAL DIFFERENTIAL}" - "\N{INCREMENT}" - "\N{N-ARY PRODUCT}" - "\N{N-ARY SUMMATION}" - "\N{SQUARE ROOT}" - "\N{INFINITY}" - "\N{INTERSECTION}" - "\N{INTEGRAL}" - "\N{ALMOST EQUAL TO}" - "\N{NOT EQUAL TO}" - "\N{IDENTICAL TO}" - "\N{LESS-THAN OR EQUAL TO}" - "\N{GREATER-THAN OR EQUAL TO}" - ), - "kaomoji": ( - _char_range(0xD2, 0xD6) - + _char_range(0xD9, 0xDC) - + _char_range(0xF2, 0xF6) - + _char_range(0xF8, 0xFC) - + "\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}" - "\N{DEGREE SIGN}" - ), - "upper_accented": ( - _char_range(0xC0, 0xD1) + - # skip capital O's and U's that could be used in kaomoji, but - # include Ø because it's very common in Arabic mojibake: - "\N{LATIN CAPITAL LETTER O WITH STROKE}" - "\N{LATIN CAPITAL LETTER U WITH DIAERESIS}" - "\N{LATIN CAPITAL LETTER Y WITH ACUTE}" - "\N{LATIN CAPITAL LETTER A WITH BREVE}" - "\N{LATIN CAPITAL LETTER A WITH OGONEK}" - "\N{LATIN CAPITAL LETTER C WITH ACUTE}" - "\N{LATIN CAPITAL LETTER C WITH CARON}" - "\N{LATIN CAPITAL LETTER D WITH CARON}" - "\N{LATIN CAPITAL LETTER D WITH STROKE}" - "\N{LATIN CAPITAL LETTER E WITH OGONEK}" - "\N{LATIN CAPITAL LETTER E WITH CARON}" - "\N{LATIN CAPITAL LETTER G WITH BREVE}" - "\N{LATIN CAPITAL LETTER I WITH DOT ABOVE}" - "\N{LATIN CAPITAL LETTER L WITH ACUTE}" - "\N{LATIN CAPITAL LETTER L WITH CARON}" - "\N{LATIN CAPITAL LETTER L WITH STROKE}" - "\N{LATIN CAPITAL LETTER N WITH ACUTE}" - "\N{LATIN CAPITAL LETTER N WITH CARON}" - "\N{LATIN CAPITAL LIGATURE OE}" - "\N{LATIN CAPITAL LETTER R WITH CARON}" - "\N{LATIN CAPITAL LETTER S WITH ACUTE}" - "\N{LATIN CAPITAL LETTER S WITH CEDILLA}" - "\N{LATIN CAPITAL LETTER S WITH CARON}" - "\N{LATIN CAPITAL LETTER T WITH CEDILLA}" - "\N{LATIN CAPITAL LETTER T WITH CARON}" - "\N{LATIN CAPITAL LETTER U WITH RING ABOVE}" - "\N{LATIN CAPITAL LETTER U WITH DOUBLE ACUTE}" - "\N{LATIN CAPITAL LETTER Y WITH DIAERESIS}" - "\N{LATIN CAPITAL LETTER Z WITH ACUTE}" - "\N{LATIN CAPITAL LETTER Z WITH DOT ABOVE}" - "\N{LATIN CAPITAL LETTER Z WITH CARON}" - "\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}" - ), - "lower_accented": ( - "\N{LATIN SMALL LETTER SHARP S}" + _char_range(0xE0, 0xF1) + - # skip o's and u's that could be used in kaomoji - "\N{LATIN SMALL LETTER A WITH BREVE}" - "\N{LATIN SMALL LETTER A WITH OGONEK}" - "\N{LATIN SMALL LETTER C WITH ACUTE}" - "\N{LATIN SMALL LETTER C WITH CARON}" - "\N{LATIN SMALL LETTER D WITH CARON}" - "\N{LATIN SMALL LETTER D WITH STROKE}" - "\N{LATIN SMALL LETTER E WITH OGONEK}" - "\N{LATIN SMALL LETTER E WITH CARON}" - "\N{LATIN SMALL LETTER G WITH BREVE}" - "\N{LATIN SMALL LETTER L WITH ACUTE}" - "\N{LATIN SMALL LETTER L WITH CARON}" - "\N{LATIN SMALL LETTER L WITH STROKE}" - "\N{LATIN SMALL LIGATURE OE}" - "\N{LATIN SMALL LETTER R WITH ACUTE}" - "\N{LATIN SMALL LETTER S WITH ACUTE}" - "\N{LATIN SMALL LETTER S WITH CEDILLA}" - "\N{LATIN SMALL LETTER S WITH CARON}" - "\N{LATIN SMALL LETTER T WITH CARON}" - "\N{LATIN SMALL LETTER Z WITH ACUTE}" - "\N{LATIN SMALL LETTER Z WITH DOT ABOVE}" - "\N{LATIN SMALL LETTER Z WITH CARON}" - "\N{CYRILLIC SMALL LETTER GHE WITH UPTURN}" - "\N{LATIN SMALL LIGATURE FI}" - "\N{LATIN SMALL LIGATURE FL}" - ), - "upper_common": ( - "\N{LATIN CAPITAL LETTER THORN}" - + _char_range(0x391, 0x3A9) # Greek capital letters - # not included under 'accented' because these can commonly - # occur at ends of words, in positions where they'd be detected - # as mojibake - + "\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}" - "\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}" - "\N{GREEK CAPITAL LETTER ETA WITH TONOS}" - "\N{GREEK CAPITAL LETTER IOTA WITH TONOS}" - "\N{GREEK CAPITAL LETTER OMICRON WITH TONOS}" - "\N{GREEK CAPITAL LETTER UPSILON WITH TONOS}" - "\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}" - "\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}" - "\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}" - + _char_range(0x401, 0x42F) # Cyrillic capital letters - ), - "lower_common": ( - # lowercase thorn does not appear in mojibake - _char_range(0x3B1, 0x3C6) # Greek lowercase letters - + "\N{GREEK SMALL LETTER ALPHA WITH TONOS}" - "\N{GREEK SMALL LETTER EPSILON WITH TONOS}" - "\N{GREEK SMALL LETTER ETA WITH TONOS}" - "\N{GREEK SMALL LETTER IOTA WITH TONOS}" - "\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}" - + _char_range(0x430, 0x45F) # Cyrillic lowercase letters - ), - "box": ("─│┌┐┘├┤┬┼" + _char_range(0x2550, 0x256C) + "▀▄█▌▐░▒▓"), -} - - -BADNESS_RE = re.compile( - r""" - [{c1}] - | - [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] - | - [{bad}] [{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] - | - [{lower_accented}{lower_common}{box}{end_punctuation}{currency}{numeric}] [{upper_accented}] - | - [{box}{end_punctuation}{currency}{numeric}] [{lower_accented}] - | - # leave out [upper_accented][currency] without further info, because it's used in some - # fancy leetspeak-esque writing - [{lower_accented}{box}{end_punctuation}] [{currency}] - | - \s [{upper_accented}] [{currency}] - | - [{upper_accented}{box}{end_punctuation}] [{numeric}] - | - [{lower_accented}{upper_accented}{currency}{numeric}{box}] [{end_punctuation}] [{start_punctuation}] - | - [{currency}{numeric}{box}] [{start_punctuation}] - | - [a-z] [{upper_accented}] [{start_punctuation}{currency}] - | - [{box}] [{kaomoji}] - | - [{lower_accented}{upper_accented}{currency}{numeric}{start_punctuation}{end_punctuation}] [{box}] - | - [{box}] [{end_punctuation}] - | - [{lower_accented}{upper_accented}] [{end_punctuation}] \\w - | - - # The ligature œ when not followed by an unaccented Latin letter - [Œœ][^A-Za-z] - | - - # Common Windows-1252 2-character mojibake that isn't covered by the cases above - [ÂÃÎÐ][€Šš¢£Ÿž\xa0\xad®©°·»{end_punctuation}–—´] - | - × [²³] - | - - # Windows-1252 mojibake sequences for some South Asian alphabets - à[²µ¹¼½¾] - | - - # MacRoman mojibake that isn't covered by the cases above - √[±∂†≠®™´≤≥¥µø] - | - ≈[°¢] - | - ‚Ä[ìîïòôúùû†°¢π] - | - ‚[âó][àä°ê] - | - - # Windows-1251 mojibake of characters in the U+2000 range - †- | - - # Windows-1251 mojibake of Latin-1 characters and/or the Cyrillic alphabet - [ВГРС][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°µ][ВГРС] - | - - # Windows-1252 encodings of 'à' and 'á' - Ã[\xa0¡] - | - [a-z]\s?Ã[ ] - | - ^Ã[ ] - | - - # Windows-1253 mojibake of characters in the U+2000 range - β€[™\xa0Ά\xad®°] - | - - # Windows-1253 mojibake of Latin-1 characters and/or the Greek alphabet - [ΒΓΞΟ][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°][ΒΓΞΟ] -""".format( - **MOJIBAKE_CATEGORIES - ), - re.VERBOSE, -) - - # Character classes that help us pinpoint embedded mojibake. These can # include common characters, because we'll also check them for 'badness'. UTF8_CLUES = { From e136e26e2bf4d1bf463e23052640a58ce9b63866 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 31 Mar 2021 18:03:12 -0400 Subject: [PATCH 472/502] v6 documentation is mostly ready --- docs/bad_encodings.rst | 13 ++++++++++ docs/cli.rst | 4 ++- docs/conf.py | 2 +- docs/fixes.rst | 1 + docs/index.rst | 5 ++-- docs/modules.rst | 44 -------------------------------- ftfy/bad_codecs/__init__.py | 3 ++- ftfy/bad_codecs/sloppy.py | 13 ++++++---- ftfy/bad_codecs/utf8_variants.py | 3 ++- ftfy/fixes.py | 20 +++++++-------- ftfy/formatting.py | 4 +-- 11 files changed, 44 insertions(+), 68 deletions(-) create mode 100644 docs/bad_encodings.rst delete mode 100644 docs/modules.rst diff --git a/docs/bad_encodings.rst b/docs/bad_encodings.rst new file mode 100644 index 00000000..0bbefe24 --- /dev/null +++ b/docs/bad_encodings.rst @@ -0,0 +1,13 @@ +Support for "bad" encodings +=========================== + +.. automodule:: ftfy.bad_codecs + +"Sloppy" encodings +------------------ +.. automodule:: ftfy.bad_codecs.sloppy + +Variants of UTF-8 +----------------- +.. automodule:: ftfy.bad_codecs.utf8_variants + diff --git a/docs/cli.rst b/docs/cli.rst index 0ed46d2e..92d06aaa 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -3,7 +3,9 @@ Command-line usage ftfy can be used from the command line. By default, it takes UTF-8 input and writes it to UTF-8 output, fixing problems in its Unicode as it goes. -Here's the usage documentation for the `ftfy` command:: +Here's the usage documentation for the `ftfy` command: + +.. code-block:: text usage: ftfy [-h] [-o OUTPUT] [-g] [-e ENCODING] [-n NORMALIZATION] [--preserve-entities] diff --git a/docs/conf.py b/docs/conf.py index 5ff68964..2a7f6aad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -121,7 +121,7 @@ # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -html_title = "ftfy v6 documentation" +html_title = "ftfy v6" # A shorter title for the navigation bar. Default is the same as html_title. html_short_title = "ftfy" diff --git a/docs/fixes.rst b/docs/fixes.rst index d9dd78a6..db8d19f0 100644 --- a/docs/fixes.rst +++ b/docs/fixes.rst @@ -2,3 +2,4 @@ Individual ways to fix text =========================== .. automodule:: ftfy.fixes + :members: diff --git a/docs/index.rst b/docs/index.rst index e91cddf9..afa2c42f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,15 +16,14 @@ ftfy is a heuristic that was designed (not machine-learned) by Robyn Speer, at L .. toctree:: :maxdepth: 1 - config explain + config encodings fixes detect avoid + bad_encodings cli - accuracy - modules cite diff --git a/docs/modules.rst b/docs/modules.rst deleted file mode 100644 index 67287ad1..00000000 --- a/docs/modules.rst +++ /dev/null @@ -1,44 +0,0 @@ -Module documentation -==================== - -*ftfy.fixes*: how individual fixes are implemented --------------------------------------------------- - -.. automodule:: ftfy.fixes - :members: - - -*ftfy.badness*: measures the "badness" of text ----------------------------------------------- - -.. automodule:: ftfy.badness - :members: - - -*ftfy.bad_codecs*: support some encodings Python doesn't like -------------------------------------------------------------- - -.. automodule:: ftfy.bad_codecs - -"Sloppy" encodings -~~~~~~~~~~~~~~~~~~ -.. automodule:: ftfy.bad_codecs.sloppy - -Variants of UTF-8 -~~~~~~~~~~~~~~~~~ -.. automodule:: ftfy.bad_codecs.utf8_variants - - -*ftfy.formatting*: justify Unicode text in a monospaced terminal ----------------------------------------------------------------- -.. automodule:: ftfy.formatting - :members: - - -*ftfy.chardata*: trivia about characters ----------------------------------------- - - -.. automodule:: ftfy.chardata - :members: - diff --git a/ftfy/bad_codecs/__init__.py b/ftfy/bad_codecs/__init__.py index 6483355c..c5486bd5 100644 --- a/ftfy/bad_codecs/__init__.py +++ b/ftfy/bad_codecs/__init__.py @@ -1,5 +1,6 @@ r""" -Give Python the ability to decode some common, flawed encodings. +The `ftfy.bad_codecs` module gives Python the ability to decode some common, +flawed encodings. Python does not want you to be sloppy with your text. Its encoders and decoders ("codecs") follow the relevant standards whenever possible, which means that diff --git a/ftfy/bad_codecs/sloppy.py b/ftfy/bad_codecs/sloppy.py index 2257a5c1..0503a55f 100644 --- a/ftfy/bad_codecs/sloppy.py +++ b/ftfy/bad_codecs/sloppy.py @@ -1,6 +1,9 @@ r""" -Decodes single-byte encodings, filling their "holes" in the same messy way that -everyone else does. +`ftfy.bad_codecs.sloppy` provides character-map encodings that fill their "holes" +in a messy but common way: by outputting the Unicode codepoints with the same +numbers. + +This is incredibly ugly, and it's also in the HTML5 standard. A single-byte encoding maps each byte to a Unicode character, except that some bytes are left unmapped. In the commonly-used Windows-1252 encoding, for @@ -45,10 +48,10 @@ Aliases such as "sloppy-cp1252" for "sloppy-windows-1252" will also be defined. -Only sloppy-windows-1251 and sloppy-windows-1252 are used by the rest of ftfy; -the rest are rather uncommon. +Five of these encodings (`sloppy-windows-1250` through `sloppy-windows-1254`) +are used within ftfy. -Here are some examples, using `ftfy.explain_unicode` to illustrate how +Here are some examples, using :func:`ftfy.explain_unicode` to illustrate how sloppy-windows-1252 merges Windows-1252 with Latin-1: >>> from ftfy import explain_unicode diff --git a/ftfy/bad_codecs/utf8_variants.py b/ftfy/bad_codecs/utf8_variants.py index 662a9908..566d2ee6 100644 --- a/ftfy/bad_codecs/utf8_variants.py +++ b/ftfy/bad_codecs/utf8_variants.py @@ -35,7 +35,8 @@ .. [1] In a pinch, you can decode CESU-8 in Python 2 using the UTF-8 codec: first decode the bytes (incorrectly), then encode them, then decode them - again, using UTF-8 as the codec every time. + again, using UTF-8 as the codec every time. But Python 2 is dead, so use + ftfy instead. """ import re diff --git a/ftfy/fixes.py b/ftfy/fixes.py index fb2a6465..2ee4003a 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -1,6 +1,7 @@ """ -This module contains the individual fixes that the main fix_text function -can perform. +The `ftfy.fixes` module contains the individual fixes that :func:`ftfy.fix_text` +can perform, and provides the functions that are named in "explanations" +such as the output of :func:`ftfy.fix_and_explain`. """ import codecs @@ -197,14 +198,13 @@ def fix_line_breaks(text): This will convert the following sequences into the standard \\n line break: - - CRLF (\\r\\n), used on Windows and in some communication - protocols - - CR (\\r), once used on Mac OS Classic, and now kept alive - by misguided software such as Microsoft Office for Mac - - LINE SEPARATOR (\\u2028) and PARAGRAPH SEPARATOR (\\u2029), - defined by Unicode and used to sow confusion and discord - - NEXT LINE (\\x85), a C1 control character that is certainly - not what you meant + - CRLF (\\r\\n), used on Windows and in some communication protocols + - CR (\\r), once used on Mac OS Classic, and now kept alive by misguided + software such as Microsoft Office for Mac + - LINE SEPARATOR (\\u2028) and PARAGRAPH SEPARATOR (\\u2029), defined by + Unicode and used to sow confusion and discord + - NEXT LINE (\\x85), a C1 control character that is certainly not what you + meant The NEXT LINE character is a bit of an odd case, because it usually won't show up if `fix_encoding` is also being run. diff --git a/ftfy/formatting.py b/ftfy/formatting.py index 59eaaf60..36a7847d 100644 --- a/ftfy/formatting.py +++ b/ftfy/formatting.py @@ -10,7 +10,7 @@ from wcwidth import wcswidth, wcwidth -def character_width(char): +def character_width(char: str) -> int: r""" Determine the width that a character is likely to be displayed as in a monospaced terminal. The width for a printable character will @@ -31,7 +31,7 @@ def character_width(char): return wcwidth(char) -def monospaced_width(text): +def monospaced_width(text: str) -> int: r""" Return the number of character cells that this string is likely to occupy when displayed in a monospaced, modern, Unicode-aware terminal emulator. From b08bd438af435607ad0e21de557db01fdc3a855d Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 31 Mar 2021 18:47:03 -0400 Subject: [PATCH 473/502] tune heuristic to fix the replacement character --- ftfy/badness.py | 2 ++ tests/test_cases.json | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/ftfy/badness.py b/ftfy/badness.py index 89b3f54a..25574731 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -245,6 +245,8 @@ def _char_range(start, end): | [{upper_accented}{box}{end_punctuation}] [{numeric}] | + [{lower_accented}{upper_accented}{box}{currency}{end_punctuation}] [{start_punctuation}] [{numeric}] + | [{lower_accented}{upper_accented}{currency}{numeric}{box}] [{end_punctuation}] [{start_punctuation}] | [{currency}{numeric}{box}] [{start_punctuation}] diff --git a/tests/test_cases.json b/tests/test_cases.json index 9b179e58..c95fb963 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -142,6 +142,12 @@ "fixed": "\"Handwerk bringt dich überall hin\": Von der YOU bis nach Monaco", "expect": "pass" }, + { + "label": "Latin-1 / Windows-1252 mixup of the replacement character", + "original": "Some comments may be republished on the website or in the newspaper � email addresses will not be published.", + "fixed": "Some comments may be republished on the website or in the newspaper � email addresses will not be published.", + "expect": "pass" + }, { "label": "CESU-8 / Windows-1252 emoji", "original": "Hi guys í ½í¸\u008d", From a96199faa29cb55c1dcbfa7254f47d1291da4f18 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 11:39:51 -0400 Subject: [PATCH 474/502] clean up regex ranges --- ftfy/badness.py | 90 +++++++++++++++++++++++++++------------- tests/test_characters.py | 10 ++++- 2 files changed, 71 insertions(+), 29 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 25574731..b250ae1f 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -1,5 +1,15 @@ """ -Heuristics to detect likely mojibake. +`ftfy.badness` contains a heuristic that detects likely mojibake. + +This heuristic signals to ftfy which segments of text need to be fixed, and +also indicates when the text can stop being fixed. + +The design of this heuristic is that we categorize the approximately 400 +Unicode characters that occur in UTF-8 mojibake, specifically the characters +that come from mixing up UTF-8 with the other encodings we support. We +identify sequences and contexts of these characters that are much more likely +to be mojibake than intended strings, such as lowercase accented letters +followed immediately by currency symbols. """ import warnings @@ -7,14 +17,6 @@ from ftfy import chardata -def _char_range(start, end): - """ - Get a contiguous range of characters as a string. Useful for including - in a regex without making the representation _only_ usable in regexes. - """ - return "".join(chr(codept) for codept in range(start, end + 1)) - - # There are only 403 characters that occur in known UTF-8 mojibake, and we can # characterize them: @@ -33,7 +35,7 @@ def _char_range(start, end): "\N{RIGHT SINGLE QUOTATION MARK}" ), # the C1 control character range, which have no uses outside of mojibake anymore - "c1": _char_range(0x80, 0x9F), + "c1": "\x80-\x9f", # Characters that are nearly 100% used in mojibake "bad": ( "\N{BROKEN BAR}" @@ -118,16 +120,24 @@ def _char_range(start, end): "\N{LESS-THAN OR EQUAL TO}" "\N{GREATER-THAN OR EQUAL TO}" ), + # Letters that might be used to make emoticon faces (kaomoji), and + # therefore might need to appear in more improbable-looking contexts. + # + # These are concatenated character ranges for use in a regex. I know + # they look like faces themselves. I think expressing the ranges like + # this helps to illustrate why we need to be careful with these + # characters. "kaomoji": ( - _char_range(0xD2, 0xD6) - + _char_range(0xD9, 0xDC) - + _char_range(0xF2, 0xF6) - + _char_range(0xF8, 0xFC) - + "\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}" + "Ò-Ö" + "Ù-Ü" + "ò-ö" + "ø-ü" + "\N{LATIN CAPITAL LETTER O WITH DOUBLE ACUTE}" "\N{DEGREE SIGN}" ), "upper_accented": ( - _char_range(0xC0, 0xD1) + + # LATIN CAPITAL LETTER A WITH GRAVE - LATIN CAPITAL LETTER N WITH TILDE + "\xc0-\xd1" # skip capital O's and U's that could be used in kaomoji, but # include Ø because it's very common in Arabic mojibake: "\N{LATIN CAPITAL LETTER O WITH STROKE}" @@ -164,7 +174,9 @@ def _char_range(start, end): "\N{CYRILLIC CAPITAL LETTER GHE WITH UPTURN}" ), "lower_accented": ( - "\N{LATIN SMALL LETTER SHARP S}" + _char_range(0xE0, 0xF1) + + "\N{LATIN SMALL LETTER SHARP S}" + # LATIN SMALL LETTER A WITH GRAVE - LATIN SMALL LETTER N WITH TILDE + "\xe0-\xf1" # skip o's and u's that could be used in kaomoji "\N{LATIN SMALL LETTER A WITH BREVE}" "\N{LATIN SMALL LETTER A WITH OGONEK}" @@ -193,11 +205,11 @@ def _char_range(start, end): ), "upper_common": ( "\N{LATIN CAPITAL LETTER THORN}" - + _char_range(0x391, 0x3A9) # Greek capital letters + "\N{GREEK CAPITAL LETTER ALPHA}-\N{GREEK CAPITAL LETTER OMEGA}" # not included under 'accented' because these can commonly # occur at ends of words, in positions where they'd be detected # as mojibake - + "\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}" + "\N{GREEK CAPITAL LETTER ALPHA WITH TONOS}" "\N{GREEK CAPITAL LETTER EPSILON WITH TONOS}" "\N{GREEK CAPITAL LETTER ETA WITH TONOS}" "\N{GREEK CAPITAL LETTER IOTA WITH TONOS}" @@ -206,24 +218,36 @@ def _char_range(start, end): "\N{GREEK CAPITAL LETTER OMEGA WITH TONOS}" "\N{GREEK CAPITAL LETTER IOTA WITH DIALYTIKA}" "\N{GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA}" - + _char_range(0x401, 0x42F) # Cyrillic capital letters + "\N{CYRILLIC CAPITAL LETTER IO}-\N{CYRILLIC CAPITAL LETTER YA}" ), "lower_common": ( # lowercase thorn does not appear in mojibake - _char_range(0x3B1, 0x3C6) # Greek lowercase letters - + "\N{GREEK SMALL LETTER ALPHA WITH TONOS}" + "\N{GREEK SMALL LETTER ALPHA}-\N{GREEK SMALL LETTER OMEGA}" + "\N{GREEK SMALL LETTER ALPHA WITH TONOS}" "\N{GREEK SMALL LETTER EPSILON WITH TONOS}" "\N{GREEK SMALL LETTER ETA WITH TONOS}" "\N{GREEK SMALL LETTER IOTA WITH TONOS}" "\N{GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS}" - + _char_range(0x430, 0x45F) # Cyrillic lowercase letters + "\N{CYRILLIC SMALL LETTER A}-\N{CYRILLIC SMALL LETTER DZHE}" + ), + "box": ( + # omit the single horizontal line, might be used in kaomoji + "│┌┐┘├┤┬┼" + "\N{BOX DRAWINGS DOUBLE HORIZONTAL}-\N{BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL}" + "▀▄█▌▐░▒▓" ), - "box": ("─│┌┐┘├┤┬┼" + _char_range(0x2550, 0x256C) + "▀▄█▌▐░▒▓"), } # We can now build a regular expression that detects unlikely juxtapositions -# of characters, mostly based on their categories +# of characters, mostly based on their categories. +# +# Another regular expression, which detects sequences that look more specifically +# like UTF-8 mojibake, appears in chardata.py. +# +# This is a verbose regular expression, with whitespace added for somewhat more +# readability. Remember that the only spaces that count as literal spaces in this +# expression are ones inside character classes (square brackets). BADNESS_RE = re.compile( r""" @@ -296,13 +320,23 @@ def _char_range(start, end): # seeing a 3-character sequence. [ВГРС][{c1}{bad}{start_punctuation}{end_punctuation}{currency}°µ][ВГРС] | + # A distinctive five-character sequence of Cyrillic letters, which can be + # Windows-1251 mojibake on top of Latin-1 mojibake of Windows-1252 characters. + # Require a Latin letter nearby. + ГўВЂВ.[A-Za-z ] + | - # Windows-1252 encodings of 'à' and 'á' + # Windows-1252 encodings of 'à' and 'á', as well as \xa0 itself Ã[\xa0¡] | - [a-z]\s?Ã[ ] + [a-z]\s?[ÃÂ][ ] + | + ^[ÃÂ][ ] | - ^Ã[ ] + + # Cases where  precedes a character as an encoding of exactly the same + # character, and the character is common enough + [a-z.,?!{end_punctuation}]  [ {start_punctuation}{end_punctuation}] | # Windows-1253 mojibake of characters in the U+2000 range diff --git a/tests/test_characters.py b/tests/test_characters.py index 3a004455..52318e40 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -1,4 +1,4 @@ -from ftfy import fix_encoding, fix_encoding_and_explain, apply_plan +from ftfy import fix_encoding, fix_encoding_and_explain, fix_text, apply_plan from ftfy.fixes import remove_control_chars, fix_surrogates from ftfy.chardata import possible_encoding from ftfy.badness import badness @@ -32,6 +32,14 @@ def test_welsh_flag(): assert remove_control_chars(text) == text +def test_ohio_flag(): + # I did not expect to find the "Flag of Ohio" emoji in the wild but there it is. + # Test that this emoji (which no emoji database believes has been implemented) + # passes through unchanged. + text = "#superman #ohio 🏴\U000e0075\U000e0073\U000e006f\U000e0068\U000e007f #cleveland #usa 🇺🇸" + assert fix_text(text) == text + + def test_surrogates(): assert fix_surrogates('\udbff\udfff') == '\U0010ffff' assert fix_surrogates('\ud800\udc00') == '\U00010000' From 427a0aaa573e62f69baeb13af56ee6545131ec42 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 12:42:39 -0400 Subject: [PATCH 475/502] more docs --- docs/avoid.rst | 22 +++++++++++++++++----- docs/conf.py | 2 +- docs/fixes.rst | 10 +++++++--- docs/heuristic.rst | 26 ++++++++++++++++++++++++++ docs/index.rst | 7 ++++--- ftfy/fixes.py | 14 ++++++++++---- 6 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 docs/heuristic.rst diff --git a/docs/avoid.rst b/docs/avoid.rst index 5e2159f5..3a4070c5 100644 --- a/docs/avoid.rst +++ b/docs/avoid.rst @@ -1,12 +1,19 @@ How can I avoid producing mojibake? =================================== +Read the Python Unicode HOWTO +----------------------------- + +The `Python Unicode HOWTO`_ is a useful introduction to how to use Unicode correctly in Python. If you find yourself confused about the difference between bytes and characters, or you need to unlearn bad habits from Python 2, it's a great place to start. + +.. _`Python Unicode HOWTO`: https://docs.python.org/3/howto/unicode.html + Assume UTF-8 ------------ **Assume text is in UTF-8** unless you have a specific reason to believe it isn't. -In the 2020s, `UTF-8 is everywhere`_, especially in text meant to be transferred over the Internet. +In the 2020s, `UTF-8 is everywhere`_, especially in text meant to be transferred over the Internet. Most mojibake comes from decoding correct UTF-8 as if it were some other encoding. .. _`UTF-8 is everywhere`: http://utf8everywhere.org/ @@ -55,14 +62,19 @@ ASCII isn't extended A sign that something is about to go wrong with encodings is if a developer is talking about "extended ASCII". -ASCII is a set of 128 character codes (95 of them displayable), and has not had any new characters added to it since the backslash was added in 1967. +ASCII is a set of 128 character codes (95 of them displayable). It has not had any new characters added to it since the backslash was added in 1967. -Because ASCII is a 7-bit encoding but our computers use 8-bit bytes, it seems clear that ASCII *could* be extended to assign a meaning to all 256 possible bytes. There are many different, incompatible encodings that have done so. Many developers refer to one of these encodings as "extended ASCII", whose colloquial meaning is "the encoding of 256 characters that I learned first". Its meaning is completely dependent on the country you were in and the operating system you were using when you started programming: +Because ASCII is a 7-bit encoding but our computers use 8-bit bytes, it seems clear that ASCII *could* be extended to assign a meaning to all 256 possible bytes. There are many different encodings that have done so, and they're all incompatible with one another, which is why treating bytes as characters as a bad idea and why we have Unicode now. + +Many developers refer to one of these encodings as "extended ASCII", whose colloquial meaning is "the encoding of 256 characters that I learned first". Its meaning is completely dependent on the country you were in and the operating system you were using when you started programming: - My "extended ASCII" when I learned to program was IBM codepage 437, the one that was used in US versions of MS-DOS. - To many people, "extended ASCII" is Windows codepage 1252, which they'd find in the Character Map of their Windows 9x computer, at least if they were in North America or Western Europe. -- Or it might be Latin-1, the common name for the ISO-8859-1 standard that became the first 256 characters of Unicode. +- To others in other countries, it could be a different Windows codepage, such as 1251 (which contains Cyrillic letters) or 1250 (which contains a different set of accented letters for Eastern European languages). +- Or it might be Latin-1, the common name for the ISO-8859-1 standard that became the first 256 characters of Unicode. Latin-1 is easy to implement by accident, such as when you see byte C2 and assume it means Unicode codepoint U+00C2 -- what you get by incorrectly running `chr()` on each byte. "Extended ASCII" doesn't specify which encoding you mean, and often indicates that you don't realize that different people are thinking of different sets of 256 characters. -Instead of "extended ASCII", say the name of the encoding such as "Latin-1", "Windows-1252", "Windows-1250", "codepage 437", or maybe "I don't know what it is but it looks right on my machine". And then revise things so that you use UTF-8. \ No newline at end of file +Instead of "extended ASCII", say the name of the encoding such as "Latin-1", "Windows-1252", "Windows-1250", "codepage 437", or maybe "I don't know what it is but it looks right on my machine". + +And then revise things so that you use UTF-8, which is still a superset of ASCII but can represent every Unicode character. diff --git a/docs/conf.py b/docs/conf.py index 2a7f6aad..1e8c0f6e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -121,7 +121,7 @@ # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -html_title = "ftfy v6" +html_title = "ftfy: fixes text for you" # A shorter title for the navigation bar. Default is the same as html_title. html_short_title = "ftfy" diff --git a/docs/fixes.rst b/docs/fixes.rst index db8d19f0..987eab7f 100644 --- a/docs/fixes.rst +++ b/docs/fixes.rst @@ -1,5 +1,9 @@ -Individual ways to fix text -=========================== +"Fixer" functions +================= .. automodule:: ftfy.fixes - :members: + :members: decode_escapes, decode_inconsistent_utf8, fix_c1_controls, + fix_character_width, fix_latin_ligatures, fix_line_breaks, + fix_surrogates, remove_control_chars, remove_terminal_escapes, + replace_lossy_sequences, restore_byte_a0, uncurl_quotes, + unescape_html diff --git a/docs/heuristic.rst b/docs/heuristic.rst new file mode 100644 index 00000000..245a7cdd --- /dev/null +++ b/docs/heuristic.rst @@ -0,0 +1,26 @@ +.. _heuristic: + +Heuristics for detecting mojibake +================================= + +The "badness" heuristic +----------------------- + +.. automodule:: ftfy.badness + :members: badness, is_bad + + +The "UTF-8 detector" heuristic +------------------------------ +A more narrow heuristic is defined in ``chardata.py`` as ``UTF8_DETECTOR_RE``. This heuristic looks for specific sequences of mojibake characters that come from the decoding of common UTF-8 sequences. + +Text that matches this regular expression can be partially fixed by :func:`ftfy.fixes.decode_inconsistent_utf8`, even when the string as a whole doesn't decode consistently. + +Because of this, the expression requires that the match isn't preceded by likely UTF-8 characters -- if this were allowed, then it might pick two or three characters out of a larger mess of mojibake to decode as another character while leaving the rest untouched. This makes the problem more confusing, doesn't really solve anything, and can even pile up recursively to decode as entirely arbitrary characters. + + +The "lossy UTF-8" heuristic +---------------------------- +``chardata.py`` also includes ``LOSSY_UTF8_RE``, which is used similarly to the "UTF-8 detector" heuristic. This regular expression matches sequences that look like they were incorrectly decoded from UTF-8, but with characters replaced by question marks or the Unicode replacement character `�`. + +Characters that match this heuristic will be replaced by `�` in the :func:`ftfy.fixes.replace_lossy_sequences` fixer. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index afa2c42f..f9734ca1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,15 +1,15 @@ ftfy: fixes text for you ======================== +*Version 6.0* + **ftfy** fixes Unicode that's broken in various ways. The goal of ftfy is to **take in bad Unicode and output good Unicode**, for use in your Unicode-aware code. This is different from taking in non-Unicode and outputting Unicode, which is not a goal of ftfy. It also isn't designed to protect you from having to write Unicode-aware code. ftfy helps those who help themselves. -Of course you're better off if your input is decoded properly and has no glitches. But you often don't have any control over your input; it's someone else's mistake, but it's your problem now. - -ftfy will do everything it can to fix the problem. +Of course you're better off if your input is decoded properly and has no glitches. But you often don't have any control over your input; it's someone else's mistake, but it's your problem now. ftfy will do everything it can to fix the problem. ftfy is a heuristic that was designed (not machine-learned) by Robyn Speer, at Luminoso. If you use ftfy in research, including pre-processing your language model data, you need to cite it: see :ref:`cite`. @@ -22,6 +22,7 @@ ftfy is a heuristic that was designed (not machine-learned) by Robyn Speer, at L fixes detect avoid + heuristic bad_encodings cli cite diff --git a/ftfy/fixes.py b/ftfy/fixes.py index 2ee4003a..d93cbebb 100644 --- a/ftfy/fixes.py +++ b/ftfy/fixes.py @@ -2,6 +2,12 @@ The `ftfy.fixes` module contains the individual fixes that :func:`ftfy.fix_text` can perform, and provides the functions that are named in "explanations" such as the output of :func:`ftfy.fix_and_explain`. + +Two of these functions are particularly useful on their own, as more robust +versions of functions in the Python standard library: + +- :func:`ftfy.fixes.decode_escapes` +- :func:`ftfy.fixes.unescape_html` """ import codecs @@ -345,6 +351,10 @@ def decode_escapes(text): Decode backslashed escape sequences, including \\x, \\u, and \\U character references, even in the presence of other Unicode. + This function has to be called specifically. It's not run automatically by + ftfy, because escaped text is not necessarily a mistake, and there is no + way to distinguish when it is. + This is what Python's "string-escape" and "unicode-escape" codecs were meant to do, but in contrast, this actually works. It will decode the string exactly the same way that the Python interpreter decodes its string @@ -365,10 +375,6 @@ def decode_escapes(text): represent escape sequences, and decodes them, leaving the rest alone. All valid escape sequences are made of ASCII characters, and this allows "unicode-escape" to work correctly. - - This fix cannot be automatically applied by the `ftfy.fix_text` function, - because escaped text is not necessarily a mistake, and there is no way - to distinguish text that's supposed to be escaped from text that isn't. """ def decode_match(match): From b0f76e17f9017883cc4d926d25c4ea16804908ff Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 13:59:12 -0400 Subject: [PATCH 476/502] update README to go with docs --- README.md | 100 ++++++++++++++++-------------------------------------- 1 file changed, 30 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 6ffb42c5..0b4b5952 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # ftfy: fixes text for you -[![Travis](https://img.shields.io/travis/LuminosoInsight/python-ftfy/master.svg?label=Travis%20CI)](https://travis-ci.org/LuminosoInsight/python-ftfy) [![PyPI package](https://badge.fury.io/py/ftfy.svg)](https://badge.fury.io/py/ftfy) [![Docs](https://readthedocs.org/projects/ftfy/badge/?version=latest)](https://ftfy.readthedocs.org/en/latest/) @@ -9,7 +8,7 @@ (ง'⌣')ง ``` -Full documentation: **https://ftfy.readthedocs.org** +The full documentation of ftfy is available at [ftfy.readthedocs.org](https://ftfy.readthedocs.org). It covers a lot more than this README. ## Testimonials @@ -31,90 +30,55 @@ Full documentation: **https://ftfy.readthedocs.org** ## Developed at Luminoso [Luminoso](https://www.luminoso.com) makes groundbreaking software for text -analytics that really understands what words mean, in many languages. Our -software is used by enterprise customers such as Sony, Intel, Mars, and Scotts, -and it's built on Python and open-source technologies. +analytics, built on open source technologies, that really understands what +words mean, in many languages. We use ftfy every day at Luminoso, because the first step in understanding text is making sure it has the correct characters in it! -Luminoso is growing fast and hiring. If you're interested in joining us, take a -look at [our careers page](https://luminoso.com/about/work-here). +If you're interested in joining us, take a look at [our careers page](https://luminoso.com/about/work-here). ## What it does -`ftfy` fixes Unicode that's broken in various ways. +Here are some examples (found in the real world) of what ftfy can do: -The goal of `ftfy` is to **take in bad Unicode and output good Unicode**, for use -in your Unicode-aware code. This is different from taking in non-Unicode and -outputting Unicode, which is not a goal of ftfy. It also isn't designed to -protect you from having to write Unicode-aware code. ftfy helps those who help -themselves. +ftfy can fix mojibake (encoding mix-ups), by detecting patterns of characters that were clearly meant to be UTF-8 but were decoded as something else: -Of course you're better off if your input is decoded properly and has no -glitches. But you often don't have any control over your input; it's someone -else's mistake, but it's your problem now. + >>> import ftfy + >>> ftfy.fix_text('✔ No problems') + '✔ No problems' -`ftfy` will do everything it can to fix the problem. +Does this sound impossible? It's really not. UTF-8 is a well-designed encoding that makes it obvious when it's being misused, and a string of mojibake usually contains all the information we need to recover the original string. -## Mojibake +ftfy can fix multiple layers of mojibake simultaneously: -The most interesting kind of brokenness that ftfy will fix is when someone has -encoded Unicode with one standard and decoded it with a different one. This -often shows up as characters that turn into nonsense sequences (called -"mojibake"): + >>> ftfy.fix_text('The Mona Lisa doesn’t have eyebrows.') + "The Mona Lisa doesn't have eyebrows." -- The word ``schön`` might appear as ``schön``. -- An em dash (``—``) might appear as ``—``. -- Text that was meant to be enclosed in quotation marks might end up - instead enclosed in ``“`` and ``â€<9d>``, where ``<9d>`` represents an - unprintable character. +It can fix mojibake that has had "curly quotes" applied on top of it, which cannot be consistently decoded until the quotes are uncurled: -ftfy uses heuristics to detect and undo this kind of mojibake, with a very -low rate of false positives. + >>> ftfy.fix_text("l’humanité") + "l'humanité" -This part of ftfy now has an unofficial Web implementation by simonw: -https://ftfy.now.sh/ +ftfy can fix mojibake that would have included the character U+A0 (non-breaking space), but the U+A0 was turned into an ASCII space and then combined with another following space: + >>> ftfy.fix_text('Ã\xa0 perturber la réflexion') + 'à perturber la réflexion' + >>> ftfy.fix_text('à perturber la réflexion') + 'à perturber la réflexion' -## Examples +ftfy can also decode HTML entities that appear outside of HTML, even in cases where the entity has been incorrectly capitalized: -`fix_text` is the main function of ftfy. This section is meant to give you a -taste of the things it can do. `fix_encoding` is the more specific function -that only fixes mojibake. + >>> # by the HTML 5 standard, only 'PÉREZ' is acceptable + >>> ftfy.fix_text('P&EACUTE;REZ') + 'PÉREZ' + +These fixes are not applied in all cases, because ftfy has a strongly-held goal of avoiding false positives -- it should never change correctly-decoded text to something else. -Please read [the documentation](https://ftfy.readthedocs.org) for more -information on what ftfy does, and how to configure it for your needs. +The following text could be encoded in Windows-1252 and decoded in UTF-8, and it would decode as 'MARQUɅ'. However, the original text is already sensible, so it is unchanged. - -```python - ->>> print(fix_text('This text should be in “quotesâ€\x9d.')) -This text should be in "quotes". - ->>> print(fix_text('ünicode')) -ünicode - ->>> print(fix_text('Broken text… it’s flubberific!', -... normalization='NFKC')) -Broken text... it's flubberific! - ->>> print(fix_text('HTML entities <3')) -HTML entities <3 - ->>> print(fix_text('HTML entities in HTML <3')) -HTML entities in HTML <3 - ->>> print(fix_text('\001\033[36;44mI’m blue, da ba dee da ba ' -... 'doo…\033[0m', normalization='NFKC')) -I'm blue, da ba dee da ba doo... - ->>> print(fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ')) -LOUD NOISES - ->>> print(fix_text('ＬＯＵＤ　ＮＯＩＳＥＳ', fix_character_width=False)) -ＬＯＵＤ　ＮＯＩＳＥＳ -``` + >>> ftfy.fix_text('IL Y MARQUÉ…') + 'IL Y MARQUÉ…' ## Installing @@ -126,10 +90,6 @@ ftfy is a Python 3 package that can be installed using `pip`: (Or use `pip3 install ftfy` on systems where Python 2 and 3 are both globally installed and `pip` refers to Python 2.) -If you're on Python 2.7, you can install an older version: - - pip install 'ftfy<5' - You can also clone this Git repository and install it with `python setup.py install`. From 9ad12dcaece4f69cbd15abf60d94491487c0a3be Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 14:10:27 -0400 Subject: [PATCH 477/502] add 'doc-requirements' that RTD might use --- doc-requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc-requirements.txt diff --git a/doc-requirements.txt b/doc-requirements.txt new file mode 100644 index 00000000..a95ae18b --- /dev/null +++ b/doc-requirements.txt @@ -0,0 +1 @@ +furo From ff3c20eea73ca698f8e328277b3a41a88178cb17 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 14:15:01 -0400 Subject: [PATCH 478/502] try some new RTD configuration --- .readthedocs.yaml | 8 ++++++++ doc-requirements.txt => docs/requirements.txt | 0 2 files changed, 8 insertions(+) create mode 100644 .readthedocs.yaml rename doc-requirements.txt => docs/requirements.txt (100%) diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..c16bfec5 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,8 @@ +version: 2 +sphinx: + configuration: docs/conf.py + +python: + version: 3.8 + install: + - requirements: docs/requirements.txt diff --git a/doc-requirements.txt b/docs/requirements.txt similarity index 100% rename from doc-requirements.txt rename to docs/requirements.txt From 618e00ce31e982d4e06a719daf27d26f9afb8e5a Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 14:23:25 -0400 Subject: [PATCH 479/502] update changelog --- CHANGELOG.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbfb9d50..cee7faf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,34 @@ +## Version 6.0 (April 2, 2021) + +- New function: `ftfy.fix_and_explain()` can describe all the transformations + that happen when fixing a string. This is similar to what + `ftfy.fixes.fix_encoding_and_explain()` did in previous versions, but it + can fix more than the encoding. + +- `fix_and_explain()` and `fix_encoding_and_explain()` are now in the top-level + ftfy module. + +- Changed the heuristic entirely. ftfy no longer needs to categorize every + Unicode character, but only characters that are expected to appear in + mojibake. + +- Because of the new heuristic, ftfy will no longer have to release a new + version for every new version of Unicode. It should also run faster and + use less RAM when imported. + +- The heuristic `ftfy.badness.is_bad(text)` can be used to determine whether + there apears to be mojibake in a string. Some users were already using + the old function `sequence_weirdness()` for that, but this one is actually + designed for that purpose. + +- Instead of a pile of named keyword arguments, ftfy functions now take in + a TextFixerConfig object. The keyword arguments still work, and become + settings that override the defaults in TextFixerConfig. + +- Added support for UTF-8 mixups with Windows-1253 and Windows-1254. + +- Overhauled the documentation: https://ftfy.readthedocs.org + ## Version 5.9 (February 10, 2021) This version is brought to you by the letter à and the number 0xC3. From a510f3c56306d42a3c1da0a5a3f1d4959cc4f4ce Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 15:25:38 -0400 Subject: [PATCH 480/502] Convenient doc links in README --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0b4b5952..a2021151 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,19 @@ (ง'⌣')ง ``` -The full documentation of ftfy is available at [ftfy.readthedocs.org](https://ftfy.readthedocs.org). It covers a lot more than this README. +The full documentation of ftfy is available at [ftfy.readthedocs.org](https://ftfy.readthedocs.org). The documentation covers a lot more than this README, so here are +some links into it: + +- [Fixing problems and getting explanations](https://ftfy.readthedocs.io/en/v6.0/explain.html) +- [Configuring ftfy](https://ftfy.readthedocs.io/en/v6.0/config.html) +- [Encodings ftfy can handle](https://ftfy.readthedocs.io/en/v6.0/encodings.html) +- [“Fixer” functions](https://ftfy.readthedocs.io/en/v6.0/fixes.html) +- [Is ftfy an encoding detector?](https://ftfy.readthedocs.io/en/v6.0/detect.html) +- [Heuristics for detecting mojibake](https://ftfy.readthedocs.io/en/v6.0/heuristics.html) +- [Support for “bad” encodings](https://ftfy.readthedocs.io/en/v6.0/bad_encodings.html) +- [Command-line usage](https://ftfy.readthedocs.io/en/v6.0/cli.html) +- [Citing ftfy](https://ftfy.readthedocs.io/en/v6.0/cite.html) + ## Testimonials From 1118425f5702c621fd7b9fe4aabf172ae1f1dd10 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 15:25:45 -0400 Subject: [PATCH 481/502] tweak Arabic, add a case we can't handle yet --- ftfy/badness.py | 5 +++++ ftfy/chardata.py | 3 ++- tests/test_cases.json | 7 +++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index b250ae1f..14b973bc 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -295,6 +295,11 @@ | × [²³] | + # Windows-1252 mojibake of Arabic words needs to include the 'common' characters. + # To compensate, we require four characters to be matched. + [ØÙ] [{common}{currency}{bad}{numeric}{start_punctuation}ŸŠ®°µ»] + [ØÙ] [{common}{currency}{bad}{numeric}{start_punctuation}ŸŠ®°µ»] + | # Windows-1252 mojibake that starts 3-character sequences for some South Asian # alphabets diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 9e1a6e0b..84d991a1 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -129,6 +129,7 @@ def _build_control_char_mapping(): # 0xc5 -> U+160 LATIN CAPITAL LETTER S WITH CARON # 0xce -> U+3A0 GREEK CAPITAL LETTER PI # 0xd0 -> U+420 CYRILLIC CAPITAL LETTER ER +# 0xd9 -> U+660 ARABIC-INDIC DIGIT ZERO # # In three-character sequences, we exclude some lead bytes in some cases. # @@ -154,7 +155,7 @@ def _build_control_char_mapping(): # I've seen it once, but the text still wasn't recoverable. ALTERED_UTF8_RE = re.compile( - b"[\xc2\xc3\xc5\xce\xd0][ ]" + b"[\xc2\xc3\xc5\xce\xd0\xd9][ ]" b"|[\xe2-\xe9\xeb\xec][ ][\x80-\xbf]" b"|[\xe0-\xef][\x80-\xbf][ ]" b"|[\xf0][ ][\x80-\xbf][\x80-\xbf]" diff --git a/tests/test_cases.json b/tests/test_cases.json index c95fb963..f2d14c0b 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -276,6 +276,13 @@ "fixed": "À tous mes frères et soeurs dans la syrienneté comme dans l'humanité, sans discrimination aucune, je vous souhaite bonne fête عيد سعيد.Que la paix, la liberté, l'égalité, la fraternité et la dignité soient avec vous.Pardonnez ce ton un peu ecclésiastique.", "expect": "pass" }, + { + "label": "Synthetic: Incomplete UTF-8 / Windows-1252 mixup in Arabic", + "comment": "I find text like this in OSCAR a fair amount, but couldn't isolate a good example that tested digits. The intended text means 'more than 100 countries'.", + "original": "أكثر من Ù Ù Ù¡ بلد", + "fixed": "أكثر من ٠٠١ بلد", + "expect": "fail" + }, { "label": "UTF-8 / sloppy Windows-1250 mixup in Romanian", "original": "vedere Ă®nceĹŁoĹźatÄ\u0083", From 2c91fbac6fd72a7008643a67d57dc70c3e36414c Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 15:29:20 -0400 Subject: [PATCH 482/502] remove obsolete accuracy stats -- working on new method --- docs/accuracy.rst | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 docs/accuracy.rst diff --git a/docs/accuracy.rst b/docs/accuracy.rst deleted file mode 100644 index 9714393b..00000000 --- a/docs/accuracy.rst +++ /dev/null @@ -1,37 +0,0 @@ -Accuracy -======== - -ftfy uses Twitter's streaming API as an endless source of realistic sample -data. Twitter is massively multilingual, and despite that it's supposed to be -uniformly UTF-8, in practice, any encoding mistake that someone can make will -be made by someone's Twitter client. - -We check what ftfy's :func:`fix_encoding` heuristic does to this data, and we -aim to have the rate of false positives be indistinguishable from zero. - -A pre-release version of ftfy 4.0 was evaluated on 30,880,000 tweets received -from Twitter's streaming API in April 2015. There was 1 false positive, and it -was due to a bug that has now been fixed. - -When looking at the changes ftfy makes, we found: - -- :func:`ftfy.fix_text`, with all default options, will change about 1 in 18 tweets. -- With stylistic changes (`fix_character_width` and `uncurl_quotes`) turned off, - :func:`ftfy.fix_text` will change about 1 in every 300 tweets. -- :func:`ftfy.fix_encoding` alone will change about 1 in every 8500 tweets. - -We sampled 1000 of these :func:`ftfy.fix_encoding` changes for further -evaluation, and found: - -- 980 of them correctly restored the text. -- 12 of them incompletely or incorrectly restored the text, when a sufficiently - advanced heuristic might have been able to fully recover the text. -- 8 of them represented text that had lost too much information to be fixed. -- None of those 1000 changed correct text to incorrect text (these would be - false positives). - -There is one string of real-world text that we have encountered that remains a -false positive in the current version: - - fix_encoding('├┤a┼┐a┼┐a┼┐a┼┐a') == 'ôaſaſaſaſa' - From ce5a3833dd1b95b8f2d01fca42b0bd14f24d50a8 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 15:32:12 -0400 Subject: [PATCH 483/502] don't include the poem inline, I don't know if I have permission --- docs/explain.rst | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/docs/explain.rst b/docs/explain.rst index c4f01900..cb78c057 100644 --- a/docs/explain.rst +++ b/docs/explain.rst @@ -4,33 +4,9 @@ Fixing problems and getting explanations Ode to a Shipping Label ----------------------- -There's a silly poem about mojibake by `@shyhoof`_ on Twitter, and it's going to serve as an excellent example for this section: +A `poem about mojibake by Twitter user @shyhoof`_ shows a shipping label that serves as an excellent example for this section, addressed to the surname `L&AMP;ATILDE;&AMP;SUP3;PEZ`. -.. code-block:: text - - Once there was a little o, - with an accent on top like só. - - It started out as UTF8, - (universal since '98), - but the program only knew latin1, - and changed little ó to "ó" for fun. - - A second program saw the "ó" - and said "I know HTML entity!" - So "ó" was smartened to "&ATILDE;&SUP3;" - and passed on through happily. - - Another program saw the tangle - (more precisely, ampersands to mangle) - and thus the humble "&ATILDE;&SUP3;" - became "&AMP;ATILDE;&AMP;SUP3;" - -.. image:: images/shipping-label.png - :width: 450 - :alt: A picture of the shipping label being described by the poem - -.. _`@shyhoof`: https://twitter.com/shyhoof/status/332621356338405376 +.. _`poem about mojibake by Twitter user @shyhoof`: https://twitter.com/shyhoof/status/332621356338405376 We can use ftfy not only to fix the text that was on the label, but to show us what happened to it (like the poem does):: From 11fc8bfb66f97c281c370ab2a9c7c45afdd07d74 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 15:32:48 -0400 Subject: [PATCH 484/502] we don't need this text file --- docs/poem.txt | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 docs/poem.txt diff --git a/docs/poem.txt b/docs/poem.txt deleted file mode 100644 index c6e2da8d..00000000 --- a/docs/poem.txt +++ /dev/null @@ -1,17 +0,0 @@ - Once there was a little o, - with an accent on top like só. - - It started out as UTF8, - (universal since '98), - but the program only knew latin1, - and changed little ó to "ó" for fun. - - A second program saw the "ó" - and said "I know HTML entity!" - So "ó" was smartened to "&ATILDE;&SUP3;" - and passed on through happily. - - Another program saw the tangle - (more precisely, ampersands to mangle) - and thus the humble "&ATILDE;&SUP3;" - became "&AMP;ATILDE;&AMP;SUP3;" From f15b50ec45eaefb3e3f3d5e989cae6f2d334e6f9 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 15:44:51 -0400 Subject: [PATCH 485/502] The numero sign is used next to punctuation --- ftfy/badness.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 14b973bc..bb202982 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -62,7 +62,6 @@ # co-occur with other mojibake characters, it's not really ambiguous "\N{FEMININE ORDINAL INDICATOR}" "\N{MASCULINE ORDINAL INDICATOR}" - "\N{NUMERO SIGN}" ), "currency": ( "\N{CENT SIGN}" @@ -119,6 +118,7 @@ "\N{IDENTICAL TO}" "\N{LESS-THAN OR EQUAL TO}" "\N{GREATER-THAN OR EQUAL TO}" + "\N{NUMERO SIGN}" ), # Letters that might be used to make emoticon faces (kaomoji), and # therefore might need to appear in more improbable-looking contexts. @@ -267,7 +267,7 @@ | \s [{upper_accented}] [{currency}] | - [{upper_accented}{box}{end_punctuation}] [{numeric}] + [{upper_accented}{box}] [{numeric}] | [{lower_accented}{upper_accented}{box}{currency}{end_punctuation}] [{start_punctuation}] [{numeric}] | From 778423f8ef1f2c038c615179cdd5e9d225f3fd2c Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 16:33:11 -0400 Subject: [PATCH 486/502] back off on fixing A0 in CJK ranges --- ftfy/chardata.py | 12 ++++++------ tests/halibote.txt | 2 +- tests/test_cases.json | 7 +++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ftfy/chardata.py b/ftfy/chardata.py index 84d991a1..8be84a52 100644 --- a/ftfy/chardata.py +++ b/ftfy/chardata.py @@ -138,10 +138,10 @@ def _build_control_char_mapping(): # # 0xe0 -> Samaritan script # 0xe1 -> Mongolian script (corresponds to Latin-1 'á' which is too common) -# 0xea -> Syloti Nagri or North Indic -# 0xed -> UTF-16 surrogates -# 0xee -> unassigned -# 0xef -> unassigned +# +# We accept 0xe2 and 0xe3, which cover many scripts. Bytes 0xe4 and +# higher point mostly to CJK characters, which we generally don't want to +# decode near Latin lowercase letters. # # In four-character sequences, the lead byte must be F0, because that accounts # for almost all of the usage of high-numbered codepoints (tag characters whose @@ -156,8 +156,8 @@ def _build_control_char_mapping(): ALTERED_UTF8_RE = re.compile( b"[\xc2\xc3\xc5\xce\xd0\xd9][ ]" - b"|[\xe2-\xe9\xeb\xec][ ][\x80-\xbf]" - b"|[\xe0-\xef][\x80-\xbf][ ]" + b"|[\xe2\xe3][ ][\x80-\x84\x86-\x9f\xa1-\xbf]" + b"|[\xe0-\xe3][\x80-\x84\x86-\x9f\xa1-\xbf][ ]" b"|[\xf0][ ][\x80-\xbf][\x80-\xbf]" b"|[\xf0][\x80-\xbf][ ][\x80-\xbf]" b"|[\xf0][\x80-\xbf][\x80-\xbf][ ]" diff --git a/tests/halibote.txt b/tests/halibote.txt index 0be33306..0070a07a 100644 --- a/tests/halibote.txt +++ b/tests/halibote.txt @@ -1 +1 @@ -ã€æ›´æ–°ã€‘ã€Šå“ˆåˆ©æ³¢ç‰¹ã€‹çŸ³å §åœæ‰æ–°å©šå¨¶åˆæˆ€ä»Šç— é€ +【更新】《哈利波特》石堧卜才新婚娶初戀今痠逝 diff --git a/tests/test_cases.json b/tests/test_cases.json index f2d14c0b..0002f33d 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -534,6 +534,13 @@ "fixed": "NESTLÉ® requiere contratar personal para diferentes areas a nivel nacional e internacional", "expect": "pass" }, + { + "label": "Mostly negative: we only need to fix C1 control characters", + "comment": "We should not decode 'é\u0085 ' as '酠'", + "original": "C'est vrai que nous n'en avons pas encore beaucoup parlé\u0085 Tu sais, ça fait de nombreuses années", + "fixed": "C'est vrai que nous n'en avons pas encore beaucoup parlé… Tu sais, ça fait de nombreuses années", + "expect": "pass" + }, { "label": "French example containing non-breaking spaces", "original": "ART TRIP Ã\u00a0 l'office de tourisme", From f2713871e78770632bd39972d389c0e29f1963f8 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 16:54:28 -0400 Subject: [PATCH 487/502] forgot some categories next to 'bad' --- ftfy/badness.py | 2 +- tests/test_cases.json | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index bb202982..40457b6e 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -253,7 +253,7 @@ r""" [{c1}] | - [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] + [{bad}{lower_accented}{upper_accented}{lower_common}{upper_common}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] | [{bad}] [{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] | diff --git a/tests/test_cases.json b/tests/test_cases.json index 0002f33d..c6a3fce2 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -547,6 +547,13 @@ "fixed": "ART TRIP à l'office de tourisme", "expect": "pass" }, + { + "label": "English example in UTF-8 / Windows-1251 with a ligature", + "original": "This is signiп¬Ѓcantly lower than the respective share", + "fixed-encoding": "This is significantly lower than the respective share", + "fixed": "This is significantly lower than the respective share", + "expect": "pass" + }, { "label": "Synthetic: we can recognize à in some cases when it's the only mojibake", "original": "voilà le travail", From 54952ff69c842aa2e1bc25649468bdcc8aa5fec4 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 17:58:22 -0400 Subject: [PATCH 488/502] lower_common and upper_common + bad need an adjacent ASCII letter --- ftfy/badness.py | 4 +++- tests/test_cases.json | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ftfy/badness.py b/ftfy/badness.py index 40457b6e..ce44be86 100644 --- a/ftfy/badness.py +++ b/ftfy/badness.py @@ -253,7 +253,9 @@ r""" [{c1}] | - [{bad}{lower_accented}{upper_accented}{lower_common}{upper_common}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] + [{bad}{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] [{bad}] + | + [a-zA-Z] [{lower_common}{upper_common}] [{bad}] | [{bad}] [{lower_accented}{upper_accented}{box}{start_punctuation}{end_punctuation}{currency}{numeric}] | diff --git a/tests/test_cases.json b/tests/test_cases.json index c6a3fce2..7736af0f 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -229,6 +229,12 @@ "fixed": "Tại sao giá hạt sầu riêng lại lên giá?", "expect": "pass" }, + { + "label": "Negative: using diaereses as quotation marks in Greek", + "original": "Η ¨ανατροφή¨ δυστυχώς από τους προπονητές", + "fixed": "Η ¨ανατροφή¨ δυστυχώς από τους προπονητές", + "expect": "pass" + }, { "label": "Science! Mid-word Greek letter gets fixed correctly", "original": "Humanized HLA-DR4.RagKO.IL2RγcKO.NOD (DRAG) mice sustain the complex vertebrate life cycle of Plasmodium falciparum malaria.", From 2c705e33388c0376c1503111f4983d5a407549e8 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Thu, 1 Apr 2021 18:11:53 -0400 Subject: [PATCH 489/502] trying to explain these requirements to RTD --- .readthedocs.yaml | 5 ++++- docs/requirements.txt | 1 - setup.py | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c16bfec5..fbabfe31 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -5,4 +5,7 @@ sphinx: python: version: 3.8 install: - - requirements: docs/requirements.txt + - method: pip + path: . + extra_requirements: + - docs diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index a95ae18b..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -furo diff --git a/setup.py b/setup.py index 210f815b..215dd22e 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,9 @@ 'ftfy = ftfy.cli:main' ] }, + extras_require={ + "docs": ["furo", "sphinx"] + }, project_urls={ 'Documentation': 'http://ftfy.readthedocs.io', } From 98acf18331c8ce08b301e5bfb92d0331edc74233 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Tue, 6 Apr 2021 15:23:59 -0400 Subject: [PATCH 490/502] fix credit on "Ode to a Shipping Label" --- docs/explain.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/explain.rst b/docs/explain.rst index cb78c057..fd37e0b5 100644 --- a/docs/explain.rst +++ b/docs/explain.rst @@ -4,9 +4,10 @@ Fixing problems and getting explanations Ode to a Shipping Label ----------------------- -A `poem about mojibake by Twitter user @shyhoof`_ shows a shipping label that serves as an excellent example for this section, addressed to the surname `L&AMP;ATILDE;&AMP;SUP3;PEZ`. +A `poem about mojibake`_, whose original author might be `Carlos Bueno on Facebook`_, shows a shipping label that serves as an excellent example for this section, addressed to the surname `L&AMP;ATILDE;&AMP;SUP3;PEZ`. -.. _`poem about mojibake by Twitter user @shyhoof`: https://twitter.com/shyhoof/status/332621356338405376 +.. _`poem about mojibake`: https://imgur.com/4J7Il0m +.. _`Carlos Bueno on Facebook`: https://www.facebook.com/cmb/posts/619241744770551:0 We can use ftfy not only to fix the text that was on the label, but to show us what happened to it (like the poem does):: From 7185141a164fd79fe2df6c7557349d4da4a0285c Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 7 Apr 2021 12:41:07 -0400 Subject: [PATCH 491/502] update docs --- README.md | 11 ----------- docs/cli.rst | 2 +- docs/detect.rst | 6 +++--- docs/explain.rst | 4 ++++ 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a2021151..62827b19 100644 --- a/README.md +++ b/README.md @@ -39,17 +39,6 @@ some links into it: - “9.2/10” — [pylint](https://bitbucket.org/logilab/pylint/) -## Developed at Luminoso - -[Luminoso](https://www.luminoso.com) makes groundbreaking software for text -analytics, built on open source technologies, that really understands what -words mean, in many languages. - -We use ftfy every day at Luminoso, because the first step in understanding text -is making sure it has the correct characters in it! - -If you're interested in joining us, take a look at [our careers page](https://luminoso.com/about/work-here). - ## What it does Here are some examples (found in the real world) of what ftfy can do: diff --git a/docs/cli.rst b/docs/cli.rst index 92d06aaa..995af62c 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -11,7 +11,7 @@ Here's the usage documentation for the `ftfy` command: [--preserve-entities] [filename] - ftfy (fixes text for you), version 5.0 + ftfy (fixes text for you), version 6.0 positional arguments: filename The file whose Unicode is to be fixed. Defaults to -, diff --git a/docs/detect.rst b/docs/detect.rst index 38882dec..f64faf6d 100644 --- a/docs/detect.rst +++ b/docs/detect.rst @@ -3,11 +3,11 @@ Is ftfy an encoding detector? No, it's a mojibake detector (and fixer). That makes its task much easier, because it doesn't have to guess the encoding of everything: it can leave correct-looking text as it is. -Encoding detectors have ended up being a bad idea, and they are largely responsible for _creating_ the problems that ftfy has to fix. +Encoding detectors have ended up being a bad idea, and they are largely responsible for *creating* the problems that ftfy has to fix. The text that you put into ftfy should be Unicode that you've attempted to decode correctly. ftfy doesn't accept bytes as input. -There is a lot of Unicode out there that has already been mangled by mojibake, even when decoded properly. That is, you might correctly interpret the text as UTF-8, and what the UTF-8 text really says is a mojibake string like "réflexion" that needs to be decoded _again_. This is when you need ftfy. +There is a lot of Unicode out there that has already been mangled by mojibake, even when decoded properly. That is, you might correctly interpret the text as UTF-8, and what the UTF-8 text really says is a mojibake string like "réflexion" that needs to be decoded *again*. This is when you need ftfy. I really need to guess the encoding of some bytes @@ -15,7 +15,7 @@ I really need to guess the encoding of some bytes I understand. Sometimes we can't have nice things. -Though it's not part of the usual operation of ftfy, ftfy _does_ contain a byte-encoding-guesser that tries to be less terrible than other byte-encoding-guessers in common cases. Instead of using probabilistic heuristics, it picks up on very strong signals like "having a UTF-16 byte-order mark" or "decoding successfully as UTF-8". +Though it's not part of the usual operation of ftfy, ftfy *does* contain a byte-encoding-guesser that tries to be less terrible than other byte-encoding-guessers in common cases. Instead of using probabilistic heuristics, it picks up on very strong signals like "having a UTF-16 byte-order mark" or "decoding successfully as UTF-8". This function won't solve everything. It can't solve everything. In particular, it has no capacity to guess non-Unicode CJK encodings such as Shift-JIS or Big5. diff --git a/docs/explain.rst b/docs/explain.rst index fd37e0b5..fca40f11 100644 --- a/docs/explain.rst +++ b/docs/explain.rst @@ -9,6 +9,10 @@ A `poem about mojibake`_, whose original author might be `Carlos Bueno on Facebo .. _`poem about mojibake`: https://imgur.com/4J7Il0m .. _`Carlos Bueno on Facebook`: https://www.facebook.com/cmb/posts/619241744770551:0 +.. image:: images/shipping-label.png + :width: 600 + :alt: A package addressed to a name including "L&AMP;ATILDE;&AMP;SUP3;PEZ" + We can use ftfy not only to fix the text that was on the label, but to show us what happened to it (like the poem does):: >>> from ftfy import fix_and_explain, apply_plan From a9235c583bfbe90009e87e34639c829c2f0aa79d Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 7 Apr 2021 12:42:13 -0400 Subject: [PATCH 492/502] yrrsinn.de is down and we don't need this many --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 62827b19..bb72f67b 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,6 @@ some links into it: — [@iancal](https://twitter.com/iancal) - “ftfy did the right thing right away, with no faffing about. Excellent work, solving a very tricky real-world (whole-world!) problem.” — Brennan Young -- “Hat mir die Tage geholfen. Im Übrigen bin ich der Meinung, dass wir keine komplexen Maschinen mit Computern bauen sollten solange wir nicht einmal Umlaute sicher verarbeiten können. :D” - — [Bruno Ranieri](https://yrrsinn.de/2012/09/17/gelesen-kw37/) - “I have no idea when I’m gonna need this, but I’m definitely bookmarking it.” — [/u/ocrow](https://reddit.com/u/ocrow) - “9.2/10” From cf7a552cb702ce0943acadfd864eed59aed9e4c5 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 7 Apr 2021 13:04:03 -0400 Subject: [PATCH 493/502] accept **kwargs in more functions --- ftfy/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 74a1d3b7..b52288a7 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -358,7 +358,7 @@ def fix_and_explain( def fix_encoding_and_explain( - text: str, config: Optional[TextFixerConfig] = None + text: str, config: Optional[TextFixerConfig] = None, **kwargs ) -> ExplainedText: """ Apply the steps of ftfy that detect mojibake and fix it. Returns the fixed @@ -384,6 +384,8 @@ def fix_encoding_and_explain( config = TextFixerConfig() if isinstance(text, bytes): raise UnicodeError(BYTES_ERROR_TEXT) + config = config._replace(**kwargs) + if not config.fix_encoding: # A weird trivial case: we're asked to fix the encoding, but skip # fixing the encoding @@ -537,7 +539,7 @@ def fix_text_segment(text: str, config: TextFixerConfig = None, **kwargs): return fixed -def fix_file(input_file, encoding=None, config=None): +def fix_file(input_file, encoding=None, config=None, **kwargs): """ Fix text that is found in a file. @@ -548,6 +550,10 @@ def fix_file(input_file, encoding=None, config=None): The output is a stream of fixed lines of text. """ + if config is None: + config = TextFixerConfig() + config = config._replace(**kwargs) + for line in input_file: if isinstance(line, bytes): if encoding is None: From 16156bd39471d61171117c642cc612fd2ed354ec Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Wed, 7 Apr 2021 13:10:02 -0400 Subject: [PATCH 494/502] update comments in test cases --- tests/test_cases.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_cases.json b/tests/test_cases.json index 7736af0f..2c93483f 100644 --- a/tests/test_cases.json +++ b/tests/test_cases.json @@ -409,7 +409,7 @@ }, { "label": "Negative: accents are sometimes used as quotes", - "comment": "The CESU-8 decoder will try to decode this, and then fail when it hits the end of the string", + "comment": "Under a previous heuristic, this tested the CESU-8 decoder, which would try to decode it and fail when it hit the end of the string", "original": "``toda produzida pronta pra assa aí´´", "fixed": "``toda produzida pronta pra assa aí´´", "expect": "pass" @@ -722,7 +722,7 @@ }, { "label": "Italian UTF-8 / MacRoman mojibake that looks like math", - "comment": "False negative: Dangerous to fix because of the following example", + "comment": "False negative: 'pi√π' is a bit too reasonable to fix", "original": "Sarai ricontattato dal nostro Esperto al pi√π presto.", "fixed": "Sarai ricontattato dal nostro Esperto al più presto.", "expect": "fail" @@ -902,7 +902,7 @@ }, { "label": "Synthetic, negative: Romanian word before a trademark sign", - "comment": "We would change 'DATÙ' to 'DATÙ', but getting windows-1250 involved makes it better to leave alone", + "comment": "We would change 'DATÙ' to 'DATÙ' if it passed the badness heuristic", "original": "NICIODATĂ™", "fixed": "NICIODATĂ™", "expect": "pass" @@ -923,7 +923,7 @@ }, { "label": "Synthetic: mojibake with trademark sign at the end of a word", - "comment": "I recall the correct version of this text from a sign in the movie Amélie. Unfortunately, we can't help her twin Amélie, who makes mojibaked signs.", + "comment": "I recall the correct version of this text from a sign in the movie Amélie. Now we can help her twin Amélie, who makes mojibaked signs.", "original": "OÙ ET QUAND?", "fixed": "OÙ ET QUAND?", "expect": "pass" From 23c95a637b380dcbfaa1bb8e0c913f99717c5c6a Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 9 Apr 2021 11:57:13 -0400 Subject: [PATCH 495/502] remove e-mail address that will stop working --- README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index bb72f67b..2a4809fb 100644 --- a/README.md +++ b/README.md @@ -95,14 +95,9 @@ You can also clone this Git repository and install it with ## Who maintains ftfy? -I'm Robyn Speer (rspeer@luminoso.com). I develop this tool as part of my -text-understanding company, [Luminoso](https://luminoso.com), where it has -proven essential. - -Luminoso provides ftfy as free, open source software under the extremely -permissive MIT license. - -You can report bugs regarding ftfy on GitHub and we'll handle them. +I'm Robyn Speer. You can find me [on GitHub](https://github.com/rspeer). +I created ftfy as part of my work at the text understanding company +[Luminoso](https://luminoso.com). ## Citing ftfy From 4f747786d60f7620133b61af7eefd725084dfd0a Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 9 Apr 2021 14:35:39 -0400 Subject: [PATCH 496/502] default to explain=False in fix_encoding --- ftfy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index b52288a7..0be4e35f 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -517,7 +517,7 @@ def fix_encoding(text: str, config: TextFixerConfig = None, **kwargs): '&ATILDE;&SUP3;' """ if config is None: - config = TextFixerConfig() + config = TextFixerConfig(explain=False) config = config._replace(**kwargs) fixed, _explan = fix_encoding_and_explain(text, config) return fixed From 862259815457c98d00ba7a0aa26386e6d5012641 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Fri, 9 Apr 2021 14:35:59 -0400 Subject: [PATCH 497/502] fix typo in readme --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cee7faf3..3fdc3b3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ use less RAM when imported. - The heuristic `ftfy.badness.is_bad(text)` can be used to determine whether - there apears to be mojibake in a string. Some users were already using + there appears to be mojibake in a string. Some users were already using the old function `sequence_weirdness()` for that, but this one is actually designed for that purpose. From f3073b3d598554f019b2a3d54b1bca0c833b015c Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 12 Apr 2021 11:19:51 -0400 Subject: [PATCH 498/502] update docs, adjust furo dark theme --- docs/conf.py | 13 +++++++++++-- docs/index.rst | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1e8c0f6e..d5a2ec9f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ # built documents. # # The short X.Y version. -version = '5.9' +version = '6.0' # The full version, including alpha/beta/rc tags. -release = '5.9' +release = '6.0.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -111,6 +111,15 @@ # I don't know why furo wants inline code to be so small, but don't let it "font-size--small--2": "100%", }, + "dark_css_variables": { + "color-brand-primary": "#9C6DFF", + "color-brand-content": "#9C6DFF", + "font-stack": "Source Sans Pro, sans-serif", + "font-stack--monospace": "Inconsolata", + "code-font-size": "18px", + + "font-size--small--2": "100%", + }, } html_css_files = [ 'css/custom.css', diff --git a/docs/index.rst b/docs/index.rst index f9734ca1..18082d8c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ ftfy: fixes text for you ======================== -*Version 6.0* +*Version 6.0.1* **ftfy** fixes Unicode that's broken in various ways. From 453787dd1dfff30383b50ed430356eba352ae297 Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 12 Apr 2021 11:24:32 -0400 Subject: [PATCH 499/502] make dark-mode links less saturated --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d5a2ec9f..2f98f740 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -112,8 +112,8 @@ "font-size--small--2": "100%", }, "dark_css_variables": { - "color-brand-primary": "#9C6DFF", - "color-brand-content": "#9C6DFF", + "color-brand-primary": "#AC8DFF", + "color-brand-content": "#AC8DFF", "font-stack": "Source Sans Pro, sans-serif", "font-stack--monospace": "Inconsolata", "code-font-size": "18px", From d324cf7771dd2baa0825fa8e85c6b9e4bad18fdf Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 12 Apr 2021 11:25:10 -0400 Subject: [PATCH 500/502] update version and python_requires --- CHANGELOG.md | 8 ++++++++ README.md | 18 +++++++++--------- ftfy/__init__.py | 2 +- setup.py | 4 ++-- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fdc3b3e..81d2558a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Version 6.0.1 (April 12, 2021) + +- ftfy uses specific features introduced in Python 3.6. I'd forgotten to specify + this version constraint in the setup.py of version 6.0, but version 6.0.1 + adds it. + +- Use a lighter link color when the docs are viewed in dark mode. + ## Version 6.0 (April 2, 2021) - New function: `ftfy.fix_and_explain()` can describe all the transformations diff --git a/README.md b/README.md index 2a4809fb..f6bd747e 100644 --- a/README.md +++ b/README.md @@ -11,15 +11,15 @@ The full documentation of ftfy is available at [ftfy.readthedocs.org](https://ftfy.readthedocs.org). The documentation covers a lot more than this README, so here are some links into it: -- [Fixing problems and getting explanations](https://ftfy.readthedocs.io/en/v6.0/explain.html) -- [Configuring ftfy](https://ftfy.readthedocs.io/en/v6.0/config.html) -- [Encodings ftfy can handle](https://ftfy.readthedocs.io/en/v6.0/encodings.html) -- [“Fixer” functions](https://ftfy.readthedocs.io/en/v6.0/fixes.html) -- [Is ftfy an encoding detector?](https://ftfy.readthedocs.io/en/v6.0/detect.html) -- [Heuristics for detecting mojibake](https://ftfy.readthedocs.io/en/v6.0/heuristics.html) -- [Support for “bad” encodings](https://ftfy.readthedocs.io/en/v6.0/bad_encodings.html) -- [Command-line usage](https://ftfy.readthedocs.io/en/v6.0/cli.html) -- [Citing ftfy](https://ftfy.readthedocs.io/en/v6.0/cite.html) +- [Fixing problems and getting explanations](https://ftfy.readthedocs.io/en/v6.0.1/explain.html) +- [Configuring ftfy](https://ftfy.readthedocs.io/en/v6.0.1/config.html) +- [Encodings ftfy can handle](https://ftfy.readthedocs.io/en/v6.0.1/encodings.html) +- [“Fixer” functions](https://ftfy.readthedocs.io/en/v6.0.1/fixes.html) +- [Is ftfy an encoding detector?](https://ftfy.readthedocs.io/en/v6.0.1/detect.html) +- [Heuristics for detecting mojibake](https://ftfy.readthedocs.io/en/v6.0.1/heuristics.html) +- [Support for “bad” encodings](https://ftfy.readthedocs.io/en/v6.0.1/bad_encodings.html) +- [Command-line usage](https://ftfy.readthedocs.io/en/v6.0.1/cli.html) +- [Citing ftfy](https://ftfy.readthedocs.io/en/v6.0.1/cite.html) ## Testimonials diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 0be4e35f..11b3a2ac 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -13,7 +13,7 @@ from ftfy.badness import is_bad from ftfy.formatting import display_ljust -__version__ = "6.0" +__version__ = "6.0.1" # Though this function does nothing, it lets linters know that we're using diff --git a/setup.py b/setup.py index 215dd22e..af9af78b 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name="ftfy", - version='6.0', + version='6.0.1', maintainer='Robyn Speer', maintainer_email='rspeer@luminoso.com', license="MIT", @@ -33,7 +33,7 @@ packages=['ftfy', 'ftfy.bad_codecs'], install_requires=['wcwidth'], tests_require=['pytest'], - python_requires='>=3.5', + python_requires='>=3.6', classifiers=[ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", From 9c82362126791ea8ebc68744dfb17be58985058e Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 12 Apr 2021 11:30:06 -0400 Subject: [PATCH 501/502] put back remove_terminal_escapes. Fixes #173. --- ftfy/__init__.py | 1 + tests/test_characters.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ftfy/__init__.py b/ftfy/__init__.py index 11b3a2ac..2c8be4f8 100644 --- a/ftfy/__init__.py +++ b/ftfy/__init__.py @@ -343,6 +343,7 @@ def fix_and_explain( "uncurl_quotes", "fix_line_breaks", "fix_surrogates", + "remove_terminal_escapes", "remove_control_chars", ]: text = _try_fix(fixer, text, config, steps) diff --git a/tests/test_characters.py b/tests/test_characters.py index 52318e40..91fa872b 100644 --- a/tests/test_characters.py +++ b/tests/test_characters.py @@ -1,4 +1,6 @@ -from ftfy import fix_encoding, fix_encoding_and_explain, fix_text, apply_plan +from ftfy import ( + fix_encoding, fix_encoding_and_explain, fix_text, fix_and_explain, apply_plan +) from ftfy.fixes import remove_control_chars, fix_surrogates from ftfy.chardata import possible_encoding from ftfy.badness import badness @@ -44,3 +46,12 @@ def test_surrogates(): assert fix_surrogates('\udbff\udfff') == '\U0010ffff' assert fix_surrogates('\ud800\udc00') == '\U00010000' + +def test_color_escapes(): + fixed, plan = fix_and_explain("\001\033[36;44mfoo") + print(plan) + assert fixed == "foo" + assert plan == [ + ("apply", "remove_terminal_escapes"), + ("apply", "remove_control_chars") + ] From acb11f877d0dfaaea8c1e30b19b11904e412a71b Mon Sep 17 00:00:00 2001 From: Robyn Speer Date: Mon, 12 Apr 2021 12:18:09 -0400 Subject: [PATCH 502/502] update changelog with new changes --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81d2558a..9cdcc0bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ ## Version 6.0.1 (April 12, 2021) -- ftfy uses specific features introduced in Python 3.6. I'd forgotten to specify - this version constraint in the setup.py of version 6.0, but version 6.0.1 - adds it. +- The `remove_terminal_escapes` step was accidentally not being used. This + version restores it. + +- Specified in setup.py that ftfy 6 requires Python 3.6 or later. - Use a lighter link color when the docs are viewed in dark mode.