Skip to content

Commit 0480bf2

Browse files
klauspostklauspost
authored
Fix serializer & tests (#7)
* Fix serializer * REadd string deduplication. Fix stuff. * Offset 0 means un-set. * Fix pool write. Reduce string index size. * Remove debug file output. * Don't store end offsets. Fix values. * Panic on compression errors * Update compress * Fix test. * Document and add unexported stream serializer... * Make order a bit cleaner.
1 parent 1f98ba7 commit 0480bf2

38 files changed

+578
-338
lines changed

‎benchmarks_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import (
2525

2626
func benchmarkFromFile(b *testing.B, filename string) {
2727

28-
_, _, msg := loadCompressed(b, filename)
28+
msg := loadCompressed(b, filename)
2929

3030
b.SetBytes(int64(len(msg)))
3131
b.ReportAllocs()
@@ -62,7 +62,7 @@ func BenchmarkUpdate_center(b *testing.B) { benchmarkFromFile(b, "update-center
6262

6363
func benchmarkJsoniter(b *testing.B, filename string) {
6464

65-
_, _, msg := loadCompressed(b, filename)
65+
msg := loadCompressed(b, filename)
6666

6767
b.SetBytes(int64(len(msg)))
6868
b.ReportAllocs()
@@ -80,7 +80,7 @@ func benchmarkJsoniter(b *testing.B, filename string) {
8080

8181
func benchmarkEncodingJson(b *testing.B, filename string) {
8282

83-
_, _, msg := loadCompressed(b, filename)
83+
msg := loadCompressed(b, filename)
8484

8585
b.SetBytes(int64(len(msg)))
8686
b.ReportAllocs()

‎find_subroutines_amd64_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -315,7 +315,7 @@ func TestFindStructuralBitsWhitespacePadding(t *testing.T) {
315315
}
316316

317317
func TestFindStructuralBitsLoop(t *testing.T) {
318-
_, _, msg := loadCompressed(t, "twitter")
318+
msg := loadCompressed(t, "twitter")
319319

320320
prev_iter_ends_odd_backslash := uint64(0)
321321
prev_iter_inside_quote := uint64(0) // either all zeros or all ones
@@ -515,7 +515,7 @@ func TestFlattenBitsIncremental(t *testing.T) {
515515

516516
func BenchmarkFlattenBits(b *testing.B) {
517517

518-
_, _, msg := loadCompressed(b, "twitter")
518+
msg := loadCompressed(b, "twitter")
519519

520520
prev_iter_ends_odd_backslash := uint64(0)
521521
prev_iter_inside_quote := uint64(0) // either all zeros or all ones

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ go 1.13
44

55
require (
66
github.com/json-iterator/go v1.1.9
7-
github.com/klauspost/compress v1.9.8
7+
github.com/klauspost/compress v1.10.1
88
github.com/klauspost/cpuid v1.2.2
99
)

‎go.sum

+4
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGn
66
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
77
github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
88
github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
9+
github.com/klauspost/compress v1.10.0 h1:92XGj1AcYzA6UrVdd4qIIBrT8OroryvRvdmg/IfmC7Y=
10+
github.com/klauspost/compress v1.10.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
11+
github.com/klauspost/compress v1.10.1 h1:a/QY0o9S6wCi0XhxaMX/QmusicNUqCqFugR6WKPOSoQ=
12+
github.com/klauspost/compress v1.10.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
913
github.com/klauspost/cpuid v1.2.2 h1:1xAgYebNnsb9LKCdLOvFWtAxGU/33mjJtyOVbmUa0Us=
1014
github.com/klauspost/cpuid v1.2.2/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
1115
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=

‎ndjson_test.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,10 @@ func BenchmarkNdjsonStage2(b *testing.B) {
384384
b.ReportAllocs()
385385
b.ResetTimer()
386386
for i := 0; i < b.N; i++ {
387-
pj.parseMessage(ndjson)
387+
err := pj.parseMessageNdjson(ndjson)
388+
if err != nil {
389+
panic(err)
390+
}
388391
}
389392
}
390393

@@ -410,7 +413,7 @@ func BenchmarkNdjsonColdCountStar(b *testing.B) {
410413
b.ResetTimer()
411414

412415
for i := 0; i < b.N; i++ {
413-
pj.parseMessage(ndjson)
416+
pj.parseMessageNdjson(ndjson)
414417
count_raw_tape(pj.Tape)
415418
}
416419
}

‎parsed_json.go

+7
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
const GOLANG_NUMBER_PARSING = true
4444

4545
const JSONVALUEMASK = 0xffffffffffffff
46+
const JSONTAGMASK = 0xff << 56
4647
const STRINGBUFBIT = 0x80000000000000
4748
const STRINGBUFMASK = 0x7fffffffffffff
4849

@@ -882,6 +883,12 @@ const (
882883
TagEnd = Tag(0)
883884
)
884885

886+
var tagOpenToClose = [256]Tag{
887+
TagObjectStart: TagObjectEnd,
888+
TagArrayStart: TagArrayEnd,
889+
TagRoot: TagRoot,
890+
}
891+
885892
func (t Tag) String() string {
886893
return string([]byte{byte(t)})
887894
}

‎parsed_json_test.go

+9-30
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
package simdjson
1818

1919
import (
20-
"bytes"
2120
"encoding/binary"
2221
"encoding/json"
2322
"io/ioutil"
@@ -31,30 +30,11 @@ type tester interface {
3130
Fatal(args ...interface{})
3231
}
3332

34-
func loadCompressed(t tester, file string) (tape, sb, ref []byte) {
33+
func loadCompressed(t tester, file string) (ref []byte) {
3534
dec, err := zstd.NewReader(nil)
3635
if err != nil {
3736
t.Fatal(err)
3837
}
39-
tap, err := ioutil.ReadFile(filepath.Join("testdata", file+".tape.zst"))
40-
if err != nil {
41-
t.Fatal(err)
42-
}
43-
tap, err = dec.DecodeAll(tap, nil)
44-
// Our end-of-root has been incremented by one (past last element) for quick skipping of ndjson
45-
// So correct the initial root element to point to one position higher
46-
binary.LittleEndian.PutUint64(tap, binary.LittleEndian.Uint64(tap)+1)
47-
if err != nil {
48-
t.Fatal(err)
49-
}
50-
sb, err = ioutil.ReadFile(filepath.Join("testdata", file+".stringbuf.zst"))
51-
if err != nil {
52-
t.Fatal(err)
53-
}
54-
sb, err = dec.DecodeAll(sb, nil)
55-
if err != nil {
56-
t.Fatal(err)
57-
}
5838
ref, err = ioutil.ReadFile(filepath.Join("testdata", file+".json.zst"))
5939
if err != nil {
6040
t.Fatal(err)
@@ -64,7 +44,7 @@ func loadCompressed(t tester, file string) (tape, sb, ref []byte) {
6444
t.Fatal(err)
6545
}
6646

67-
return tap, sb, ref
47+
return ref
6848
}
6949

7050
var testCases = []struct {
@@ -185,31 +165,30 @@ func testCTapeCtoGoTapeCompare(t *testing.T, ctape []uint64, csbuf []byte, pj in
185165
}
186166

187167
func TestVerifyTape(t *testing.T) {
188-
168+
// FIXME: Does not have tapes any more.
189169
for _, tt := range testCases {
190170

191171
t.Run(tt.name, func(t *testing.T) {
192-
cbuf, csbuf, ref := loadCompressed(t, tt.name)
172+
ref := loadCompressed(t, tt.name)
193173

194174
pj := internalParsedJson{}
195175
if err := pj.parseMessage(ref); err != nil {
196176
t.Errorf("parseMessage failed: %v\n", err)
197177
return
198178
}
199179

200-
ctape := bytesToUint64(cbuf)
180+
//ctape := bytesToUint64(cbuf)
201181

202-
testCTapeCtoGoTapeCompare(t, ctape, csbuf, pj)
182+
//testCTapeCtoGoTapeCompare(t, ctape, csbuf, pj)
203183
})
204184
}
205185
}
206186

207187
func BenchmarkIter_MarshalJSONBuffer(b *testing.B) {
208188
for _, tt := range testCases {
209189
b.Run(tt.name, func(b *testing.B) {
210-
tap, sb, _ := loadCompressed(b, tt.name)
211-
212-
pj, err := loadTape(bytes.NewBuffer(tap), bytes.NewBuffer(sb))
190+
ref := loadCompressed(b, tt.name)
191+
pj, err := Parse(ref, nil)
213192
if err != nil {
214193
b.Fatal(err)
215194
}
@@ -236,7 +215,7 @@ func BenchmarkIter_MarshalJSONBuffer(b *testing.B) {
236215
func BenchmarkGoMarshalJSON(b *testing.B) {
237216
for _, tt := range testCases {
238217
b.Run(tt.name, func(b *testing.B) {
239-
_, _, ref := loadCompressed(b, tt.name)
218+
ref := loadCompressed(b, tt.name)
240219
var m interface{}
241220
m = map[string]interface{}{}
242221
if tt.array {

0 commit comments

Comments
 (0)