[Jython-checkins] jython: Ignore lines with just comments, rather than treating as newline.

[frank.wierzbicki]

http://hg.python.org/jython/rev/a2ab729f5a16
changeset:   7314:a2ab729f5a16
user:        Peter Holloway <peter.holloway at diamond.ac.uk>
date:        Tue Jun 24 14:58:29 2014 +0000
summary:
  Ignore lines with just comments, rather than treating as newline.

files:
  Lib/test/test_codeop.py                     |   4 ++
  src/org/python/antlr/PythonTokenSource.java |  16 ++++++++-
  2 files changed, 18 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
--- a/Lib/test/test_codeop.py
+++ b/Lib/test/test_codeop.py
@@ -258,6 +258,10 @@
         ai("(x for x in")
         ai("(x for x in (")
 
+        # Merge test cases below upstream.
+        ai("def x():\n pass\n#comment")
+
+
     def test_invalid(self):
         ai = self.assertInvalid
         ai("a b")
diff --git a/src/org/python/antlr/PythonTokenSource.java b/src/org/python/antlr/PythonTokenSource.java
--- a/src/org/python/antlr/PythonTokenSource.java
+++ b/src/org/python/antlr/PythonTokenSource.java
@@ -252,15 +252,27 @@
     private List<Token> enqueueHiddens(Token t) {
         List<Token> newlines = new ArrayList<Token>();
         if (inSingle && t.getType() == Token.EOF) {
-            if (stream.size() > lastTokenAddedIndex + 1) {
-                Token hidden = stream.get(lastTokenAddedIndex + 1);
+            int k = 1;
+            while (stream.size() > lastTokenAddedIndex + k) {
+                Token hidden = stream.get(lastTokenAddedIndex + k);
                 if (hidden.getType() == PythonLexer.COMMENT) {
                     String text = hidden.getText();
                     int i = text.indexOf("\n");
+                    i = text.indexOf("\n", i + 1);
                     while(i != -1) {
                         newlines.add(hidden);
+                        lastTokenAddedIndex++;
                         i = text.indexOf("\n", i + 1);
                     }
+                    k++;
+                } else if (hidden.getType() == PythonLexer.NEWLINE) {
+                    generateNewline(hidden);
+                    lastTokenAddedIndex++;
+                    break;
+                } else if (hidden.getType() == PythonLexer.LEADING_WS) {
+                    k++;
+                } else {
+                    break;
                 }
             }
         }

-- 
Repository URL: http://hg.python.org/jython