[Python-checkins] bpo-44667: Treat correctly lines ending with comments and no newlines in the Python tokenizer (GH-27499) (GH-27501)

[ambv]

https://github.com/python/cpython/commit/2d11797c81be3ae776e418a5ba507098356d357c
commit: 2d11797c81be3ae776e418a5ba507098356d357c
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: ambv <lukasz at langa.pl>
date: 2021-08-02T11:44:01+02:00
summary:

bpo-44667: Treat correctly lines ending with comments and no newlines in the Python tokenizer (GH-27499) (GH-27501)

(cherry picked from commit b6bde9fc42aecad5be0457198d17cfe7b481ad79)

Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com>

files:
A Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst
M Lib/test/test_tokenize.py
M Lib/tokenize.py

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 6de7aa87bb2f9..e05a07fab9fe7 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1457,6 +1457,16 @@ def test_pathological_trailing_whitespace(self):
         # See http://bugs.python.org/issue16152
         self.assertExactTypeEqual('@          ', token.AT)
 
+    def test_comment_at_the_end_of_the_source_without_newline(self):
+        # See http://bugs.python.org/issue44667
+        source = 'b = 1\n\n#test'
+        expected_tokens = [token.NAME, token.EQUAL, token.NUMBER, token.NEWLINE, token.NL, token.COMMENT]
+
+        tokens = list(tokenize(BytesIO(source.encode('utf-8')).readline))
+        self.assertEqual(tok_name[tokens[0].exact_type], tok_name[ENCODING])
+        for i in range(6):
+            self.assertEqual(tok_name[tokens[i + 1].exact_type], tok_name[expected_tokens[i]])
+        self.assertEqual(tok_name[tokens[-1].exact_type], tok_name[token.ENDMARKER])
 
 class UntokenizeTest(TestCase):
 
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1aee21b5e18fa..a782f6250dd24 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -602,7 +602,7 @@ def _tokenize(readline, encoding):
                 pos += 1
 
     # Add an implicit NEWLINE if the input doesn't end in one
-    if last_line and last_line[-1] not in '\r\n':
+    if last_line and last_line[-1] not in '\r\n' and not last_line.strip().startswith("#"):
         yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
     for indent in indents[1:]:                 # pop remaining indent levels
         yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
diff --git a/Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst b/Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst
new file mode 100644
index 0000000000000..5b7e20e0afdf5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst
@@ -0,0 +1,4 @@
+The :func:`tokenize.tokenize` doesn't incorrectly generate a ``NEWLINE``
+token if the source doesn't end with a new line character but the last line
+is a comment, as the function is already generating a ``NL`` token. Patch by
+Pablo Galindo