Test at all buffer sizes from 1 to 16. Fix the resulting bug when an embedded quote straddles two buffer blocks

Author: geoffleyland

lua/csv.lua

@@ -184,9 +184,15 @@ local function separated_values_iterator(file, parameters)
     local first, last, capture
     while true do
       first, last, capture = buffer:find(pattern, anchor_pos + offset - 1)
-      if not first then
+      -- if we found nothing, or the last character is at the end of the
+      -- buffer (and the match could potentially be longer) then read some
+      -- more.
+      if not first or last == #buffer then
         local s = file:read(buffer_size)
-        if not s then return end
+        -- if we read nothing from the file:
+        --  - and first is nil, then we found nothing.  This will return nil.
+        --  - and last == #buffer, then the capture we found above is good.
+        if not s then return first, last, capture end
         buffer = buffer..s
       else
         return first - anchor_pos + 1, last - anchor_pos + 1, capture

lua/test.lua

@@ -29,16 +29,20 @@ local function testhandle(handle, correct_result)
 end
 
 local function test(filename, correct_result, parameters)
-  local f = csv.open(filename, parameters)
-  local fileok = testhandle(f, correct_result)
+  parameters = parameters or {}
+  for i = 1, 16 do
+    parameters.buffer_size = i
+    local f = csv.open(filename, parameters)
+    local fileok = testhandle(f, correct_result)
 
-  if fileok then
-    f = io.open(filename, "r")
-    local data = f:read("*a")
-    f:close()
+    if fileok then
+      f = io.open(filename, "r")
+      local data = f:read("*a")
+      f:close()
 
-    f = csv.openstring(data, parameters)
-    testhandle(f, correct_result)
+      f = csv.openstring(data, parameters)
+      testhandle(f, correct_result)
+    end
   end
 end
 
@@ -50,26 +54,25 @@ newline!
 embedded
 newline,embedded
 newline,embedded
-newline!]], { buffer_size=4})
+newline!]])
 
 test("../test-data/embedded-quotes.csv", [[
 embedded "quotes",embedded "quotes",embedded "quotes"!
 embedded "quotes",embedded "quotes",embedded "quotes"!]])
 
 test("../test-data/header.csv", [[
 alpha:ONE,bravo:two,charlie:3!
-alpha:four,bravo:five,charlie:6!]], {header=true, buffer_size=5})
+alpha:four,bravo:five,charlie:6!]], {header=true})
 
 test("../test-data/header.csv", [[
 apple:one,charlie:30!
 apple:four,charlie:60!]],
-{ buffer_size = 7,
-  columns = {
+{ columns = {
   apple = { name = "ALPHA", transform = string.lower },
   charlie = { transform = function(x) return tonumber(x) * 10 end }}})
 
 test("../test-data/blank-line.csv", [[
-this,file,ends,with,a,blank,line!]], { buffer_size=11 })
+this,file,ends,with,a,blank,line!]])
 
 
 if errors == 0 then