Merge commit '639ce04' into issue_9757

Author: Evan Wright

doc/source/whatsnew/v0.16.1.txt

@@ -96,3 +96,7 @@ Bug Fixes
 - Bug in ``FloatArrayFormatter`` where decision boundary for displaying "small" floats in decimal format is off by one order of magnitude for a given display.precision (:issue:`9764`)
 
 - Fixed bug where ``DataFrame.plot()`` raised an error when both ``color`` and ``style`` keywords were passed and there was no color symbol in the style strings (:issue:`9671`)
+
+- Bug in ``read_csv`` and ``read_table`` when using ``skip_rows`` parameter if blank lines are present. (:issue:`9832`)
+
+- Bug in csv parser causing lines with initial whitespace plus one non-space character to be skipped. (:issue:`9710`)

pandas/io/tests/test_parsers.py

@@ -839,6 +839,28 @@ def test_deep_skiprows(self):
         condensed_data = self.read_csv(StringIO(condensed_text))
         tm.assert_frame_equal(data, condensed_data)
 
+    def test_skiprows_blank(self):
+        # GH 9832
+        text = """#foo,a,b,c
+#foo,a,b,c
+
+#foo,a,b,c
+#foo,a,b,c
+
+1/1/2000,1.,2.,3.
+1/2/2000,4,5,6
+1/3/2000,7,8,9
+"""
+        data = self.read_csv(StringIO(text), skiprows=6, header=None,
+                              index_col=0, parse_dates=True)
+
+        expected = DataFrame(np.arange(1., 10.).reshape((3, 3)),
+                             columns=[1, 2, 3],
+                             index=[datetime(2000, 1, 1), datetime(2000, 1, 2),
+                                    datetime(2000, 1, 3)])
+        expected.index.name = 0
+        tm.assert_frame_equal(data, expected)
+
     def test_detect_string_na(self):
         data = """A,B
 foo,bar
@@ -2231,6 +2253,20 @@ def test_nrows_and_chunksize_raises_notimplemented(self):
         self.assertRaises(NotImplementedError, self.read_csv, StringIO(data),
                      nrows=10, chunksize=5)
 
+    def test_single_char_leading_whitespace(self):
+        # GH 9710
+        data = """\
+MyColumn
+   a
+   b
+   a
+   b\n"""
+
+        expected = DataFrame({'MyColumn' : list('abab')})
+
+        result = self.read_csv(StringIO(data), skipinitialspace=True)
+        tm.assert_frame_equal(result, expected)
+
 
 class TestPythonParser(ParserTests, tm.TestCase):
     def test_negative_skipfooter_raises(self):
@@ -3271,6 +3307,25 @@ def test_buffer_overflow(self):
             except Exception as cperr:
                 self.assertIn('Buffer overflow caught - possible malformed input file.', str(cperr))
 
+    def test_single_char_leading_whitespace(self):
+        # GH 9710
+        data = """\
+MyColumn
+   a
+   b
+   a
+   b\n"""
+
+        expected = DataFrame({'MyColumn' : list('abab')})
+
+        result = self.read_csv(StringIO(data), delim_whitespace=True,
+                               skipinitialspace=True)
+        tm.assert_frame_equal(result, expected)
+
+        result = self.read_csv(StringIO(data), lineterminator='\n',
+                               skipinitialspace=True)
+        tm.assert_frame_equal(result, expected)
+
 class TestCParserLowMemory(ParserTests, tm.TestCase):
 
     def read_csv(self, *args, **kwds):
@@ -3692,6 +3747,25 @@ def test_buffer_overflow(self):
             except Exception as cperr:
                 self.assertIn('Buffer overflow caught - possible malformed input file.', str(cperr))
 
+    def test_single_char_leading_whitespace(self):
+        # GH 9710
+        data = """\
+MyColumn
+   a
+   b
+   a
+   b\n"""
+
+        expected = DataFrame({'MyColumn' : list('abab')})
+
+        result = self.read_csv(StringIO(data), delim_whitespace=True,
+                               skipinitialspace=True)
+        tm.assert_frame_equal(result, expected)
+
+        result = self.read_csv(StringIO(data), lineterminator='\n',
+                               skipinitialspace=True)
+        tm.assert_frame_equal(result, expected)
+
 class TestMiscellaneous(tm.TestCase):
 
     # for tests that don't fit into any of the other classes, e.g. those that

pandas/src/parser/tokenizer.c

@@ -757,11 +757,9 @@ int tokenize_delimited(parser_t *self, size_t line_limit)
         case START_RECORD:
             // start of record
             if (skip_this_line(self, self->file_lines)) {
+                self->state = SKIP_LINE;
                 if (c == '\n') {
-                    END_LINE()
-                }
-                else {
-                    self->state = SKIP_LINE;
+                    END_LINE();
                 }
                 break;
             }
@@ -853,10 +851,11 @@ int tokenize_delimited(parser_t *self, size_t line_limit)
                 ;
             else { // backtrack
                 /* We have to use i + 1 because buf has been incremented but not i */
-                while (i + 1 > self->datapos && *buf != '\n') {
+                do {
                     --buf;
                     --i;
-                }
+                } while (i + 1 > self->datapos && *buf != '\n');
+
                 if (i + 1 > self->datapos) // reached a newline rather than the beginning
                 {
                     ++buf; // move pointer to first char after newline
@@ -1077,7 +1076,7 @@ int tokenize_delim_customterm(parser_t *self, size_t line_limit)
         // Next character in file
         c = *buf++;
 
-        TRACE(("Iter: %d Char: %c Line %d field_count %d, state %d\n",
+        TRACE(("tokenize_delim_customterm - Iter: %d Char: %c Line %d field_count %d, state %d\n",
                i, c, self->file_lines + 1, self->line_fields[self->lines],
                self->state));
 
@@ -1093,11 +1092,9 @@ int tokenize_delim_customterm(parser_t *self, size_t line_limit)
         case START_RECORD:
             // start of record
             if (skip_this_line(self, self->file_lines)) {
+                self->state = SKIP_LINE;
                 if (c == self->lineterminator) {
-                    END_LINE()
-                }
-                else {
-                    self->state = SKIP_LINE;
+                    END_LINE();
                 }
                 break;
             }
@@ -1172,10 +1169,11 @@ int tokenize_delim_customterm(parser_t *self, size_t line_limit)
                 ;
             else { // backtrack
                 /* We have to use i + 1 because buf has been incremented but not i */
-                while (i + 1 > self->datapos && *buf != self->lineterminator) {
+                do {
                     --buf;
                     --i;
-                }
+                } while (i + 1 > self->datapos && *buf != self->lineterminator);
+
                 if (i + 1 > self->datapos) // reached a newline rather than the beginning
                 {
                     ++buf; // move pointer to first char after newline
@@ -1342,7 +1340,7 @@ int tokenize_whitespace(parser_t *self, size_t line_limit)
         // Next character in file
         c = *buf++;
 
-        TRACE(("Iter: %d Char: %c Line %d field_count %d, state %d\n",
+        TRACE(("tokenize_whitespace - Iter: %d Char: %c Line %d field_count %d, state %d\n",
                i, c, self->file_lines + 1, self->line_fields[self->lines],
                self->state));
 
@@ -1391,11 +1389,9 @@ int tokenize_whitespace(parser_t *self, size_t line_limit)
         case START_RECORD:
             // start of record
             if (skip_this_line(self, self->file_lines)) {
+                self->state = SKIP_LINE;
                 if (c == '\n') {
-                    END_LINE()
-                }
-                else {
-                    self->state = SKIP_LINE;
+                    END_LINE();
                 }
                 break;
             } else  if (c == '\n') {