[pypy-commit] pypy py3.7: maybe fix the token errors

[cfbolz]

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.7
Changeset: r98463:f69a090e703c
Date: 2020-01-06 20:44 +0100
http://bitbucket.org/pypy/pypy/changeset/f69a090e703c/

Log:	maybe fix the token errors

diff --git a/pypy/module/token/moduledef.py b/pypy/module/token/moduledef.py
--- a/pypy/module/token/moduledef.py
+++ b/pypy/module/token/moduledef.py
@@ -21,11 +21,6 @@
     Module.interpleveldefs["tok_name"] = "space.wrap(%r)" % (tok_name,)
     Module.interpleveldefs["N_TOKENS"] = "space.wrap(%d)" % len(tok_name)
     all_names = Module.interpleveldefs.keys()
-    # obscure, but these names are not in CPython's token module
-    # so we remove them from 'token.__all__' otherwise they end up
-    # twice in 'tokenize.__all__'
-    all_names.remove('COMMENT')
-    all_names.remove('NL')
     Module.interpleveldefs["__all__"] = "space.wrap(%r)" % (all_names,)
 
 _init_tokens()
diff --git a/pypy/module/token/test/test_token.py b/pypy/module/token/test/test_token.py
--- a/pypy/module/token/test/test_token.py
+++ b/pypy/module/token/test/test_token.py
@@ -16,3 +16,7 @@
     def test_iseof(self):
         assert self.token.ISEOF(self.token.ENDMARKER)
         assert not self.token.ISEOF(self.token.NAME)
+
+    def test_nl_and_comment_exist_in_all(self):
+        assert "NL" in self.token.__all__
+        assert "COMMENT" in self.token.__all__