From 2ebbfefc0286dbc3204861c9742aebabded79a68 Mon Sep 17 00:00:00 2001
From: Frank Wierzbicki <fwierzbicki@gmail.com>
Date: Wed, 14 Jan 2009 23:56:22 +0000
Subject: [PATCH] DEDENT problem with comments in single mode fixed.

---
 Lib/test/test_codeop_jy.py                  |  2 ++
 src/org/python/antlr/PythonTokenSource.java | 11 ++++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_codeop_jy.py b/Lib/test/test_codeop_jy.py
index 8e697cea0..53cc96b89 100644
--- a/Lib/test/test_codeop_jy.py
+++ b/Lib/test/test_codeop_jy.py
@@ -102,6 +102,8 @@ def test_valid(self):
         # Failed for Jython 2.5a2.  See http://bugs.jython.org/issue1116.
         av("@a.b.c\ndef f():\n pass") 
 
+        av("def f():\n pass\n#foo")
+
     def test_incomplete(self):
         ai = self.assertIncomplete
 
diff --git a/src/org/python/antlr/PythonTokenSource.java b/src/org/python/antlr/PythonTokenSource.java
index 206444a0a..cd7352fcd 100644
--- a/src/org/python/antlr/PythonTokenSource.java
+++ b/src/org/python/antlr/PythonTokenSource.java
@@ -195,7 +195,7 @@ protected void insertImaginaryIndentDedentTokens() {
             t = stream.LT(1);
             stream.consume();
 
-            enqueueHiddens(t);
+            List<Token> commentedNewlines = enqueueHiddens(t);
 
             // compute cpos as the char pos of next non-WS token in line
             int cpos = t.getCharPositionInLine(); // column dictates indent/dedent
@@ -230,6 +230,9 @@ else if (cpos < lastIndent) { // they dedented
                 for(int i=1;i<newlines.length();i++) {
                     generateNewline(newline);
                 }
+                for (Token c : commentedNewlines) {
+                    generateNewline(c);
+                }
             }
 
             if (t.getType() != PythonLexer.LEADING_WS) { // discard WS
@@ -246,7 +249,8 @@ private void enqueue(Token t) {
         tokens.addElement(t);
     }
 
-    private void enqueueHiddens(Token t) {
+    private List<Token> enqueueHiddens(Token t) {
+        List<Token> newlines = new ArrayList<Token>();
         if (inSingle && t.getType() == Token.EOF) {
             if (stream.size() > lastTokenAddedIndex + 1) {
                 Token hidden = stream.get(lastTokenAddedIndex + 1);
@@ -254,7 +258,7 @@ private void enqueueHiddens(Token t) {
                     String text = hidden.getText();
                     int i = text.indexOf("\n");
                     while(i != -1) {
-                        generateNewline(hidden);
+                        newlines.add(hidden);
                         i = text.indexOf("\n", i + 1);
                     }
                 }
@@ -265,6 +269,7 @@ private void enqueueHiddens(Token t) {
             tokens.addAll(hiddenTokens);
         }
         lastTokenAddedIndex = t.getTokenIndex();
+        return newlines;
     }
 
     private void handleIndents(int cpos, CommonToken t) {