fix compatibility with unreleased changes to stdlib tokenizer

mwhudson · mwhudson · commit 4f53bbaf58c0 · 2018-10-01T15:44:44.000+13:00
python/cpython@c4ef489 (not yet in any released version, but it's been backported to all versions of Python in git, even 2.7!) changed the behaviour in the stdlib's tokenize module to emit a synthetic NEWLINE token even if the file does not end with a newline. This was causing a spurious "mixed-line-endings" warning to be emitted, but luckily the synthetic token is easy to test for (the token text is "").
diff --git a/patsy/tokens.py b/patsy/tokens.py
@@ -32,7 +32,10 @@ def python_tokenize(code):
             if pytype == tokenize.ENDMARKER:
                 break
             origin = Origin(code, start, end)
-            assert pytype not in (tokenize.NL, tokenize.NEWLINE)
+            assert pytype != tokenize.NL
+            if pytype == tokenize.NEWLINE:
+                assert string == ""
+                continue
             if pytype == tokenize.ERRORTOKEN:
                 raise PatsyError("error tokenizing input "
                                  "(maybe an unclosed string?)",
@@ -98,7 +101,9 @@ def pretty_untokenize(typed_tokens):
     brackets = []
     for token_type, token in typed_tokens:
         assert token_type not in (tokenize.INDENT, tokenize.DEDENT,
-                                  tokenize.NEWLINE, tokenize.NL)
+                                  tokenize.NL)
+        if token_type == tokenize.NEWLINE:
+            continue
         if token_type == tokenize.ENDMARKER:
             continue
         if token_type in (tokenize.NAME, tokenize.NUMBER, tokenize.STRING):
