Unexport LoadTape. There is a serializer.

parsed_json.go

@@ -145,8 +145,8 @@ type Iter struct {
 	t Tag
 }
 
-// LoadTape will load the input from the supplied readers.
-func LoadTape(tape, strings io.Reader) (*ParsedJson, error) {
+// loadTape will load the input from the supplied readers.
+func loadTape(tape, strings io.Reader) (*ParsedJson, error) {
 	b, err := ioutil.ReadAll(tape)
 	if err != nil {
 		return nil, err

parsed_json_test.go

@@ -124,7 +124,7 @@ func TestLoadTape(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			pj, err := LoadTape(bytes.NewBuffer(tap), bytes.NewBuffer(sb))
+			pj, err := loadTape(bytes.NewBuffer(tap), bytes.NewBuffer(sb))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -198,7 +198,7 @@ func BenchmarkIter_MarshalJSONBuffer(b *testing.B) {
 		b.Run(tt.name, func(b *testing.B) {
 			tap, sb, _ := loadCompressed(b, tt.name)
 
-			pj, err := LoadTape(bytes.NewBuffer(tap), bytes.NewBuffer(sb))
+			pj, err := loadTape(bytes.NewBuffer(tap), bytes.NewBuffer(sb))
 			if err != nil {
 				b.Fatal(err)
 			}

parsed_serialize.go

@@ -9,8 +9,6 @@ import (
 	"sync"
 	"unsafe"
 
-	"github.com/klauspost/compress/fse"
-	"github.com/klauspost/compress/huff0"
 	"github.com/klauspost/compress/s2"
 	"github.com/klauspost/compress/zstd"
 )
@@ -21,10 +19,9 @@ const (
 	stringmask = stringSize - 1
 )
 
+// Serializer allows to serialize parsed json and read it back.
+// A Serializer can be reused, but not used concurrently.
 type Serializer struct {
-	tComp   fse.Scratch
-	sComp   huff0.Scratch
-	strings map[string]uint32
 	// Old -> new offset
 	stringIdxLUT []uint32
 	stringBuf    []byte
@@ -44,6 +41,7 @@ type Serializer struct {
 	reIndexStrings       bool
 }
 
+// NewSerializer will create and initialize a serializer.
 func NewSerializer() *Serializer {
 	s := Serializer{
 		compValues:     blockTypeS2,
@@ -53,6 +51,8 @@ func NewSerializer() *Serializer {
 	return &s
 }
 
+// Serialize the data in pj and return the data.
+// An optional destination can be provided.
 func (s *Serializer) Serialize(dst []byte, pj ParsedJson) []byte {
 	// Header: Version byte
 	// Varuint Strings size, uncompressed
@@ -225,6 +225,10 @@ func (s *Serializer) Serialize(dst []byte, pj ParsedJson) []byte {
 	return dst
 }
 
+// Deserialize the content in src.
+// Only basic sanity checks will be performed.
+// Slight corruption will likely go through unnoticed.
+// And optional destination can be provided.
 func (s *Serializer) Deserialize(src []byte, dst *ParsedJson) (*ParsedJson, error) {
 	br := bytes.NewBuffer(src)
 
@@ -453,13 +457,17 @@ func (s *Serializer) decBlock(br *bytes.Buffer, dst []byte, wg *sync.WaitGroup,
 
 // indexStringsLazy will deduplicate strings and populate
 // strings, stringsMap and stringBuf.
+// Returns false if unable to deduplicate.
 func (s *Serializer) indexStringsLazy(sb []byte) bool {
 	// Only possible on 64 bit platforms, so it will never trigger on 32 bit platforms.
-	if uint32(len(sb)) > math.MaxUint32 {
+	if uint32(len(sb)) >= math.MaxUint32 {
 		s.stringBuf = sb
 		// This would overflow our offset table.
 		return false
 	}
+
+	// Reset lookup table.
+	// Offsets are offset by 1, so 0 indicates an unfilled entry.
 	for i := range s.strings2[:] {
 		s.strings2[i] = 0
 	}

parsed_serialize_test.go

@@ -10,7 +10,7 @@ func BenchmarkSerialize(b *testing.B) {
 		s := NewSerializer()
 		b.Run(tt.name, func(b *testing.B) {
 			tap, sb, org := loadCompressed(b, tt.name)
-			pj, err := LoadTape(bytes.NewBuffer(tap), bytes.NewBuffer(sb))
+			pj, err := loadTape(bytes.NewBuffer(tap), bytes.NewBuffer(sb))
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -34,7 +34,7 @@ func BenchmarkDeSerialize(b *testing.B) {
 		s := NewSerializer()
 		b.Run(tt.name, func(b *testing.B) {
 			tap, sb, org := loadCompressed(b, tt.name)
-			pj, err := LoadTape(bytes.NewBuffer(tap), bytes.NewBuffer(sb))
+			pj, err := loadTape(bytes.NewBuffer(tap), bytes.NewBuffer(sb))
 			if err != nil {
 				b.Fatal(err)
 			}