Fix leading and trailing whitespace.

Author: klauspost

ndjson_test.go

@@ -1,7 +1,6 @@
 package simdjson
 
 import (
-	"bytes"
 	"encoding/binary"
 	"fmt"
 	"io/ioutil"
@@ -186,7 +185,7 @@ func TestDemoNdjson(t *testing.T) {
 }
 
 func TestNdjsonCountWhere(t *testing.T) {
-	ndjson := getPatchedNdjson("testdata/parking-citations-1M.json.zst")
+	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
 
 	pj := internalParsedJson{}
 	pj.initialize(len(ndjson) * 3 / 2)
@@ -199,7 +198,12 @@ func TestNdjsonCountWhere(t *testing.T) {
 }
 
 func TestNdjsonCountWhere2(t *testing.T) {
-	ndjson := getPatchedNdjson("testdata/RC_2009-01.json.zst")
+	ndjson := loadFile("testdata/RC_2009-01.json.zst")
+	// Test trimming
+	b := make([]byte, 0, len(ndjson)+4)
+	b = append(b, '\n', '\n')
+	b = append(b, ndjson...)
+	b = append(b, '\n', '\n')
 	pj, err := ParseND(ndjson, nil)
 	if err != nil {
 		t.Fatal(err)
@@ -210,13 +214,13 @@ func TestNdjsonCountWhere2(t *testing.T) {
 	}
 }
 
-func getPatchedNdjson(filename string) []byte {
+func loadFile(filename string) []byte {
 	if !strings.HasSuffix(filename, ".zst") {
 		ndjson, err := ioutil.ReadFile(filename)
 		if err != nil {
 			panic("Failed to load file")
 		}
-		return bytes.ReplaceAll(ndjson, []byte("\n"), []byte("{"))
+		return ndjson
 	}
 	var f *os.File
 	var err error
@@ -250,12 +254,12 @@ func getPatchedNdjson(filename string) []byte {
 	if err != nil {
 		panic("Failed to load file")
 	}
-	return bytes.ReplaceAll(ndjson, []byte("\n"), []byte("{"))
+	return ndjson
 }
 
 func BenchmarkNdjsonStage1(b *testing.B) {
 
-	ndjson := getPatchedNdjson("testdata/parking-citations-1M.json.zst")
+	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
 
 	pj := internalParsedJson{}
 
@@ -271,7 +275,7 @@ func BenchmarkNdjsonStage1(b *testing.B) {
 }
 
 func BenchmarkNdjsonStage2(b *testing.B) {
-	ndjson := getPatchedNdjson("testdata/parking-citations-1M.json.zst")
+	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
 	pj := internalParsedJson{}
 	pj.initialize(len(ndjson) * 3 / 2)
 
@@ -296,7 +300,7 @@ func count_raw_tape(tape []uint64) (count int) {
 
 func BenchmarkNdjsonColdCountStar(b *testing.B) {
 
-	ndjson := getPatchedNdjson("testdata/parking-citations-1M.json.zst")
+	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
 
 	b.SetBytes(int64(len(ndjson)))
 	b.ReportAllocs()
@@ -443,7 +447,7 @@ func countObjects(data ParsedJson) (count int) {
 }
 
 func BenchmarkNdjsonColdCountStarWithWhere(b *testing.B) {
-	ndjson := getPatchedNdjson("testdata/parking-citations-1M.json.zst")
+	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
 	const want = 110349
 	runtime.GC()
 	pj := internalParsedJson{}
@@ -482,7 +486,7 @@ func BenchmarkNdjsonColdCountStarWithWhere(b *testing.B) {
 }
 
 func BenchmarkNdjsonWarmCountStar(b *testing.B) {
-	ndjson := getPatchedNdjson("testdata/parking-citations-1M.json.zst")
+	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
 
 	pj := internalParsedJson{}
 	pj.initialize(len(ndjson) * 3 / 2)
@@ -498,7 +502,7 @@ func BenchmarkNdjsonWarmCountStar(b *testing.B) {
 }
 
 func BenchmarkNdjsonWarmCountStarWithWhere(b *testing.B) {
-	ndjson := getPatchedNdjson("testdata/parking-citations-1M.json.zst")
+	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
 
 	pj := internalParsedJson{}
 	pj.initialize(len(ndjson) * 3 / 2)

parsed_serialize_test.go

@@ -105,14 +105,14 @@ func BenchmarkDeSerialize(b *testing.B) {
 }
 
 func BenchmarkSerializeNDJSON(b *testing.B) {
-	ndjson := getPatchedNdjson("testdata/parking-citations-1M.json.zst")
-
-	pj := internalParsedJson{}
-	pj.initialize(len(ndjson) * 3 / 2)
-	pj.parseMessage(ndjson)
+	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
 
+	pj, err := ParseND(ndjson, nil)
+	if err != nil {
+		b.Fatal(err)
+	}
 	bench := func(b *testing.B, s *Serializer) {
-		output := s.Serialize(nil, pj.ParsedJson)
+		output := s.Serialize(nil, *pj)
 		if true {
 			b.Log(len(ndjson), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(ndjson)), "%")
 		}
@@ -121,7 +121,7 @@ func BenchmarkSerializeNDJSON(b *testing.B) {
 		b.ReportAllocs()
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			output = s.Serialize(output[:0], pj.ParsedJson)
+			output = s.Serialize(output[:0], *pj)
 		}
 	}
 	b.Run("default", func(b *testing.B) {
@@ -146,14 +146,14 @@ func BenchmarkSerializeNDJSON(b *testing.B) {
 }
 
 func BenchmarkDeSerializeNDJSON(b *testing.B) {
-	ndjson := getPatchedNdjson("testdata/parking-citations-1M.json.zst")
-
-	pj := internalParsedJson{}
-	pj.initialize(len(ndjson) * 3 / 2)
-	pj.parseMessage(ndjson)
+	ndjson := loadFile("testdata/parking-citations-1M.json.zst")
 
+	pj, err := ParseND(ndjson, nil)
+	if err != nil {
+		b.Fatal(err)
+	}
 	bench := func(b *testing.B, s *Serializer) {
-		output := s.Serialize(nil, pj.ParsedJson)
+		output := s.Serialize(nil, *pj)
 		if false {
 			b.Log(len(ndjson), "(JSON) ->", len(output), "(Serialized)", 100*float64(len(output))/float64(len(ndjson)), "%")
 		}

simdjson.go

@@ -2,6 +2,7 @@ package simdjson
 
 import (
 	"bufio"
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -52,6 +53,7 @@ func ParseND(b []byte, reuse *ParsedJson) (*ParsedJson, error) {
 	if reuse != nil {
 		pj.ParsedJson = *reuse
 	}
+	b = bytes.TrimSpace(b)
 	pj.initialize(len(b) * 3 / 2)
 
 	// FIXME(fwessels): We should not modify input.
@@ -123,11 +125,12 @@ func ParseNDStream(r io.Reader, res chan<- Stream, reuse <-chan *ParsedJson) {
 				tmp = append(tmp, b...)
 			}
 			// TODO: Do the parsing in several goroutines, but keep output in order.
-			if len(tmp) > 0 {
+			trimmed := bytes.TrimSpace(tmp)
+			if len(trimmed) > 0 {
 				// We cannot reuse the result since we share it
 				pj.ParsedJson = ParsedJson{}
-				pj.initialize(len(tmp) * 3 / 2)
-				parseErr := pj.parseMessageNdjson(tmp)
+				pj.initialize(len(trimmed) * 3 / 2)
+				parseErr := pj.parseMessageNdjson(trimmed)
 				if parseErr != nil {
 					res <- Stream{
 						Value: nil,