gh-104719: IDLE - test existence of all tokenize references. (#104767)

Class editor.IndentSearcher contains all editor references to tokenize module.
Module io tokenize reference cover those other modules.

Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>

Lib/idlelib/NEWS.txt

@@ -4,6 +4,9 @@ Released on 2023-10-02
 =========================
 
 
+gh-104719: Remove IDLE's modification of tokenize.tabsize and test
+other uses of tokenize data and methods.
+
 gh-104499: Fix completions for Tk Aqua 8.7 (currently blank).
 
 gh-104486: Make About print both tcl and tk versions if different,

Lib/idlelib/editor.py

@@ -1571,7 +1571,7 @@ class EditorWindow:
     # blocks are found).
 
     def guess_indent(self):
-        opener, indented = IndentSearcher(self.text, self.tabwidth).run()
+        opener, indented = IndentSearcher(self.text).run()
         if opener and indented:
             raw, indentsmall = get_line_indent(opener, self.tabwidth)
             raw, indentlarge = get_line_indent(indented, self.tabwidth)
@@ -1609,15 +1609,10 @@ def get_line_indent(line, tabwidth):
 
 
 class IndentSearcher:
+    "Manage initial indent guess, returned by run method."
 
-    # .run() chews over the Text widget, looking for a block opener
-    # and the stmt following it.  Returns a pair,
-    #     (line containing block opener, line containing stmt)
-    # Either or both may be None.
-
-    def __init__(self, text, tabwidth):
+    def __init__(self, text):
         self.text = text
-        self.tabwidth = tabwidth
         self.i = self.finished = 0
         self.blkopenline = self.indentedline = None
 
@@ -1633,7 +1628,8 @@ class IndentSearcher:
     def tokeneater(self, type, token, start, end, line,
                    INDENT=tokenize.INDENT,
                    NAME=tokenize.NAME,
-                   OPENERS=('class', 'def', 'for', 'if', 'try', 'while')):
+                   OPENERS=('class', 'def', 'for', 'if', 'match', 'try',
+                            'while', 'with')):
         if self.finished:
             pass
         elif type == NAME and token in OPENERS:
@@ -1643,6 +1639,10 @@ class IndentSearcher:
             self.finished = 1
 
     def run(self):
+        """Return 2 lines containing block opener and and indent.
+
+        Either the indent line or both may be None.
+        """
         try:
             tokens = tokenize.generate_tokens(self.readline)
             for token in tokens:
@@ -1654,6 +1654,7 @@ class IndentSearcher:
 
 ### end autoindent code ###
 
+
 def prepstr(s):
     """Extract the underscore from a string.
 

Lib/idlelib/idle_test/test_editor.py

@@ -1,10 +1,10 @@
-"Test editor, coverage 35%."
+"Test editor, coverage 53%."
 
 from idlelib import editor
 import unittest
 from collections import namedtuple
 from test.support import requires
-from tkinter import Tk
+from tkinter import Tk, Text
 
 Editor = editor.EditorWindow
 
@@ -31,7 +31,7 @@ class EditorWindowTest(unittest.TestCase):
         e._close()
 
 
-class TestGetLineIndent(unittest.TestCase):
+class GetLineIndentTest(unittest.TestCase):
     def test_empty_lines(self):
         for tabwidth in [1, 2, 4, 6, 8]:
             for line in ['', '\n']:
@@ -181,6 +181,36 @@ class IndentAndNewlineTest(unittest.TestCase):
         eq(get('1.0', 'end'), '  def f1(self, a,\n         \n    return a + b\n')
 
 
+class IndentSearcherTest(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        requires('gui')
+        cls.root = Tk()
+        cls.root.withdraw()
+        cls.text = Text(cls.root)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.root.destroy()
+        del cls.root
+
+    def test_searcher(self):
+        text = self.text
+        searcher = (self.text)
+        test_info = (# text, (block, indent))
+                     ("", (None, None)),
+                     ("[1,", (None, None)),  # TokenError
+                     ("if 1:\n", ('if 1:\n', None)),
+                     ("if 1:\n  2\n  3\n", ('if 1:\n', '  2\n')),
+                     )
+        for code, expected_pair in test_info:
+            with self.subTest(code=code):
+                insert(text, code)
+                actual_pair = editor.IndentSearcher(text).run()
+                self.assertEqual(actual_pair, expected_pair)
+
+
 class RMenuTest(unittest.TestCase):
 
     @classmethod

Lib/idlelib/idle_test/test_iomenu.py

@@ -8,6 +8,12 @@ from idlelib.editor import EditorWindow
 from idlelib import util
 from idlelib.idle_test.mock_idle import Func
 
+# Fail if either tokenize.open and t.detect_encoding does not exist.
+# These are used in loadfile and encode.
+# Also used in pyshell.MI.execfile and runscript.tabnanny.
+from tokenize import open, detect_encoding
+# Remove when we have proper tests that use both.
+
 
 class IOBindingTest(unittest.TestCase):
 

Misc/NEWS.d/next/IDLE/2023-05-23-17-19-49.gh-issue-104719.rvYXH-.rst

@@ -0,0 +1,2 @@
+Remove IDLE's modification of tokenize.tabsize and test other uses of
+tokenize data and methods.