[pypy-commit] pypy pyparser-improvements: improve error messages of ParseError

[cfbolz]

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: pyparser-improvements
Changeset: r93980:1c51dd151fee
Date: 2018-03-16 13:18 +0100
http://bitbucket.org/pypy/pypy/changeset/1c51dd151fee/

Log:	improve error messages of ParseError

	only covers very simple cases, but at least if tells you about
	forgotten ':' in the line starting a block, which is the syntax
	error I always make.

diff --git a/pypy/interpreter/pyparser/metaparser.py b/pypy/interpreter/pyparser/metaparser.py
--- a/pypy/interpreter/pyparser/metaparser.py
+++ b/pypy/interpreter/pyparser/metaparser.py
@@ -164,6 +164,13 @@
                 else:
                     gram.labels.append(gram.symbol_ids[label])
                     gram.symbol_to_label[label] = label_index
+                    first = self.first[label]
+                    if len(first) == 1:
+                        first, = first
+                        if not first[0].isupper():
+                            first = first.strip("\"'")
+                            assert label_index not in gram.token_to_error_string
+                            gram.token_to_error_string[label_index] = first
                     return label_index
             elif label.isupper():
                 token_index = gram.TOKENS[label]
@@ -185,7 +192,7 @@
                 else:
                     gram.labels.append(gram.KEYWORD_TOKEN)
                     gram.keyword_ids[value] = label_index
-                    return label_index
+                    result = label_index
             else:
                 try:
                     token_index = gram.OPERATOR_MAP[value]
@@ -196,7 +203,10 @@
                 else:
                     gram.labels.append(token_index)
                     gram.token_ids[token_index] = label_index
-                    return label_index
+                    result = label_index
+            assert result not in gram.token_to_error_string
+            gram.token_to_error_string[result] = value
+            return result
 
     def make_first(self, gram, name):
         original_firsts = self.first[name]
diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py
--- a/pypy/interpreter/pyparser/parser.py
+++ b/pypy/interpreter/pyparser/parser.py
@@ -17,6 +17,7 @@
         self.symbol_names = {}
         self.symbol_to_label = {}
         self.keyword_ids = {}
+        self.token_to_error_string = {}
         self.dfas = []
         self.labels = [0]
         self.token_ids = {}
@@ -193,7 +194,7 @@
 class ParseError(Exception):
 
     def __init__(self, msg, token_type, value, lineno, column, line,
-                 expected=-1):
+                 expected=-1, expected_str=None):
         self.msg = msg
         self.token_type = token_type
         self.value = value
@@ -201,6 +202,7 @@
         self.column = column
         self.line = line
         self.expected = expected
+        self.expected_str = expected_str
 
     def __str__(self):
         return "ParserError(%s, %r)" % (self.token_type, self.value)
@@ -293,10 +295,13 @@
                     # error.
                     if len(arcs) == 1:
                         expected = sym_id
+                        expected_str = self.grammar.token_to_error_string.get(
+                                arcs[0][0], None)
                     else:
                         expected = -1
+                        expected_str = None
                     raise ParseError("bad input", token_type, value, lineno,
-                                     column, line, expected)
+                                     column, line, expected, expected_str)
 
     def classify(self, token_type, value, lineno, column, line):
         """Find the label for a token."""
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -185,6 +185,9 @@
                 else:
                     new_err = error.SyntaxError
                     msg = "invalid syntax"
+                    if e.expected_str is not None:
+                        msg += " (expected '%s')" % e.expected_str
+
                 raise new_err(msg, e.lineno, e.column, e.line,
                               compile_info.filename)
             else:
diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py
--- a/pypy/interpreter/pyparser/test/test_parser.py
+++ b/pypy/interpreter/pyparser/test/test_parser.py
@@ -321,3 +321,12 @@
         assert isinstance(tree.get_child(1), parser.Nonterminal1)
 
 
+    def test_error_string(self):
+        p, gram = self.parser_for(
+            "foo: 'if' NUMBER '+' NUMBER"
+        )
+        info = py.test.raises(parser.ParseError, p.parse, "if 42")
+        info.value.expected_str is None
+        info = py.test.raises(parser.ParseError, p.parse, "if 42 42")
+        info.value.expected_str == '+'
+
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -165,3 +165,11 @@
         for linefeed in ["\r\n","\r"]:
             tree = self.parse(fmt % linefeed)
             assert expected_tree == tree
+
+    def test_error_forgotten_chars(self):
+        info = py.test.raises(SyntaxError, self.parse, "if 1\n    print 4")
+        assert "(expected ':')" in info.value.msg
+        info = py.test.raises(SyntaxError, self.parse, "for i in range(10)\n    print i")
+        assert "(expected ':')" in info.value.msg
+        info = py.test.raises(SyntaxError, self.parse, "def f:\n print 1")
+        assert "(expected '(')" in info.value.msg