[Python-checkins] [3.11] gh-104719: IDLE - test existence of all tokenize references. (GH-104767) (#104845)
terryjreedy
webhook-mailer at python.org
Wed May 24 08:46:41 EDT 2023
https://github.com/python/cpython/commit/22618a86f4c42aaecf5057055b41552346ddd3aa
commit: 22618a86f4c42aaecf5057055b41552346ddd3aa
branch: 3.11
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: terryjreedy <tjreedy at udel.edu>
date: 2023-05-24T08:46:33-04:00
summary:
[3.11] gh-104719: IDLE - test existence of all tokenize references. (GH-104767) (#104845)
gh-104719: IDLE - test existence of all tokenize references. (GH-104767)
Class editor.IndentSearcher contains all editor references to tokenize module.
Module io tokenize reference cover those other modules.
(cherry picked from commit e561c09975bf67ad8bb67c56a81e30a9165bcc84)
Co-authored-by: Terry Jan Reedy <tjreedy at udel.edu>
Co-authored-by: Jelle Zijlstra <jelle.zijlstra at gmail.com>
files:
A Misc/NEWS.d/next/IDLE/2023-05-23-17-19-49.gh-issue-104719.rvYXH-.rst
M Lib/idlelib/NEWS.txt
M Lib/idlelib/editor.py
M Lib/idlelib/idle_test/test_editor.py
M Lib/idlelib/idle_test/test_iomenu.py
diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 89e86c13b25e..efc8bf3ce2d5 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -4,6 +4,9 @@ Released after 2022-10-24
=========================
+gh-104719: Remove IDLE's modification of tokenize.tabsize and test
+other uses of tokenize data and methods.
+
gh-104499: Fix completions for Tk Aqua 8.7 (currently blank).
gh-104486: Make About print both tcl and tk versions if different,
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index df36be876601..69b27d0683a1 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -1571,7 +1571,7 @@ def reindent_to(self, column):
# blocks are found).
def guess_indent(self):
- opener, indented = IndentSearcher(self.text, self.tabwidth).run()
+ opener, indented = IndentSearcher(self.text).run()
if opener and indented:
raw, indentsmall = get_line_indent(opener, self.tabwidth)
raw, indentlarge = get_line_indent(indented, self.tabwidth)
@@ -1609,15 +1609,10 @@ def get_line_indent(line, tabwidth):
class IndentSearcher:
+ "Manage initial indent guess, returned by run method."
- # .run() chews over the Text widget, looking for a block opener
- # and the stmt following it. Returns a pair,
- # (line containing block opener, line containing stmt)
- # Either or both may be None.
-
- def __init__(self, text, tabwidth):
+ def __init__(self, text):
self.text = text
- self.tabwidth = tabwidth
self.i = self.finished = 0
self.blkopenline = self.indentedline = None
@@ -1633,7 +1628,8 @@ def readline(self):
def tokeneater(self, type, token, start, end, line,
INDENT=tokenize.INDENT,
NAME=tokenize.NAME,
- OPENERS=('class', 'def', 'for', 'if', 'try', 'while')):
+ OPENERS=('class', 'def', 'for', 'if', 'match', 'try',
+ 'while', 'with')):
if self.finished:
pass
elif type == NAME and token in OPENERS:
@@ -1643,6 +1639,10 @@ def tokeneater(self, type, token, start, end, line,
self.finished = 1
def run(self):
+ """Return 2 lines containing block opener and and indent.
+
+ Either the indent line or both may be None.
+ """
try:
tokens = tokenize.generate_tokens(self.readline)
for token in tokens:
@@ -1654,6 +1654,7 @@ def run(self):
### end autoindent code ###
+
def prepstr(s):
"""Extract the underscore from a string.
diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py
index fdb47abf43fb..9296a6d235fb 100644
--- a/Lib/idlelib/idle_test/test_editor.py
+++ b/Lib/idlelib/idle_test/test_editor.py
@@ -1,10 +1,10 @@
-"Test editor, coverage 35%."
+"Test editor, coverage 53%."
from idlelib import editor
import unittest
from collections import namedtuple
from test.support import requires
-from tkinter import Tk
+from tkinter import Tk, Text
Editor = editor.EditorWindow
@@ -31,7 +31,7 @@ def test_init(self):
e._close()
-class TestGetLineIndent(unittest.TestCase):
+class GetLineIndentTest(unittest.TestCase):
def test_empty_lines(self):
for tabwidth in [1, 2, 4, 6, 8]:
for line in ['', '\n']:
@@ -181,6 +181,36 @@ def test_indent_and_newline_event(self):
eq(get('1.0', 'end'), ' def f1(self, a,\n \n return a + b\n')
+class IndentSearcherTest(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ requires('gui')
+ cls.root = Tk()
+ cls.root.withdraw()
+ cls.text = Text(cls.root)
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.root.destroy()
+ del cls.root
+
+ def test_searcher(self):
+ text = self.text
+ searcher = (self.text)
+ test_info = (# text, (block, indent))
+ ("", (None, None)),
+ ("[1,", (None, None)), # TokenError
+ ("if 1:\n", ('if 1:\n', None)),
+ ("if 1:\n 2\n 3\n", ('if 1:\n', ' 2\n')),
+ )
+ for code, expected_pair in test_info:
+ with self.subTest(code=code):
+ insert(text, code)
+ actual_pair = editor.IndentSearcher(text).run()
+ self.assertEqual(actual_pair, expected_pair)
+
+
class RMenuTest(unittest.TestCase):
@classmethod
diff --git a/Lib/idlelib/idle_test/test_iomenu.py b/Lib/idlelib/idle_test/test_iomenu.py
index 2fb836dba216..e0642cf0cabe 100644
--- a/Lib/idlelib/idle_test/test_iomenu.py
+++ b/Lib/idlelib/idle_test/test_iomenu.py
@@ -8,6 +8,12 @@
from idlelib import util
from idlelib.idle_test.mock_idle import Func
+# Fail if either tokenize.open and t.detect_encoding does not exist.
+# These are used in loadfile and encode.
+# Also used in pyshell.MI.execfile and runscript.tabnanny.
+from tokenize import open, detect_encoding
+# Remove when we have proper tests that use both.
+
class IOBindingTest(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/IDLE/2023-05-23-17-19-49.gh-issue-104719.rvYXH-.rst b/Misc/NEWS.d/next/IDLE/2023-05-23-17-19-49.gh-issue-104719.rvYXH-.rst
new file mode 100644
index 000000000000..3fbe04ba4f68
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2023-05-23-17-19-49.gh-issue-104719.rvYXH-.rst
@@ -0,0 +1,2 @@
+Remove IDLE's modification of tokenize.tabsize and test other uses of
+tokenize data and methods.
More information about the Python-checkins
mailing list