DEDENT problem with comments in single mode fixed.

xiaoyinl · Jan 14, 2009 · 2ebbfef · 2ebbfef
1 parent 59862a4
commit 2ebbfef
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 3 deletions.
diff --git a/Lib/test/test_codeop_jy.py b/Lib/test/test_codeop_jy.py
@@ -102,6 +102,8 @@ def test_valid(self):
         # Failed for Jython 2.5a2.  See http://bugs.jython.org/issue1116.
         av("@a.b.c\ndef f():\n pass") 
 
+        av("def f():\n pass\n#foo")
+
     def test_incomplete(self):
         ai = self.assertIncomplete
 

diff --git a/src/org/python/antlr/PythonTokenSource.java b/src/org/python/antlr/PythonTokenSource.java
@@ -195,7 +195,7 @@ protected void insertImaginaryIndentDedentTokens() {
             t = stream.LT(1);
             stream.consume();
 
-            enqueueHiddens(t);
+            List<Token> commentedNewlines = enqueueHiddens(t);
 
             // compute cpos as the char pos of next non-WS token in line
             int cpos = t.getCharPositionInLine(); // column dictates indent/dedent
@@ -230,6 +230,9 @@ else if (cpos < lastIndent) { // they dedented
                 for(int i=1;i<newlines.length();i++) {
                     generateNewline(newline);
                 }
+                for (Token c : commentedNewlines) {
+                    generateNewline(c);
+                }
             }
 
             if (t.getType() != PythonLexer.LEADING_WS) { // discard WS
@@ -246,15 +249,16 @@ private void enqueue(Token t) {
         tokens.addElement(t);
     }
 
-    private void enqueueHiddens(Token t) {
+    private List<Token> enqueueHiddens(Token t) {
+        List<Token> newlines = new ArrayList<Token>();
         if (inSingle && t.getType() == Token.EOF) {
             if (stream.size() > lastTokenAddedIndex + 1) {
                 Token hidden = stream.get(lastTokenAddedIndex + 1);
                 if (hidden.getType() == PythonLexer.COMMENT) {
                     String text = hidden.getText();
                     int i = text.indexOf("\n");
                     while(i != -1) {
-                        generateNewline(hidden);
+                        newlines.add(hidden);
                         i = text.indexOf("\n", i + 1);
                     }
                 }
@@ -265,6 +269,7 @@ private void enqueueHiddens(Token t) {
             tokens.addAll(hiddenTokens);
         }
         lastTokenAddedIndex = t.getTokenIndex();
+        return newlines;
     }
 
     private void handleIndents(int cpos, CommonToken t) {