UncompressedSize isn't optional.

klauspost · Jan 7, 2022 · 22da84c · 22da84c
1 parent fe4d285
commit 22da84c
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 16 deletions.
diff --git a/s2/README.md b/s2/README.md
@@ -141,7 +141,7 @@ Binaries can be downloaded on the [Releases Page](https://github.com/klauspost/c
 
 Installing then requires Go to be installed. To install them, use:
 
-`go install github.com/klauspost/compress/s2/cmd/s2c && go install github.com/klauspost/compress/s2/cmd/s2d`
+`go install github.com/klauspost/compress/s2/cmd/s2c@latest && go install github.com/klauspost/compress/s2/cmd/s2d@latest`
 
 To build binaries to the current folder use:
 
@@ -176,6 +176,8 @@ Options:
     	Compress faster, but with a minor compression loss
   -help
     	Display help
+  -index
+        Add seek index (default true)    	
   -o string
         Write output to another file. Single input file only
   -pad string
@@ -217,11 +219,15 @@ Options:
     	Display help
   -o string
         Write output to another file. Single input file only
-  -q	Don't write any output to terminal, except errors
+  -offset string
+        Start at offset. Examples: 92, 64K, 256K, 1M, 4M. Requires Index
+  -q    Don't write any output to terminal, except errors
   -rm
-    	Delete source file(s) after successful decompression
+        Delete source file(s) after successful decompression
   -safe
-    	Do not overwrite output files
+        Do not overwrite output files
+  -tail string
+        Return last of compressed file. Examples: 92, 64K, 256K, 1M, 4M. Requires Index
   -verify
     	Verify files, but do not write output                                      
 ```
@@ -730,7 +736,7 @@ with un-encoded value length of 64 bits, unless other limits are specified.
 | ID, `[1]byte`                                                           | Always 0x99.                                                                                                                  |
 | Data Length, `[3]byte`                                                  | 3 byte little-endian length of the chunk in bytes, following this.                                                            |
 | Header `[6]byte`                                                        | Header, must be `[115, 50, 105, 100, 120, 0]` or in text: "s2idx\x00".                                                        |
-| UncompressedSize, Varint                                                | Total Uncompressed size if known. Should be -1 if unknown.                                                                    |
+| UncompressedSize, Varint                                                | Total Uncompressed size.                                                                                                      |
 | CompressedSize, Varint                                                  | Total Compressed size if known. Should be -1 if unknown.                                                                      |
 | EstBlockSize, Varint                                                    | Block Size, used for guessing uncompressed offsets. Must be >= 0.                                                             |
 | Entries, Varint                                                         | Number of Entries in index, must be < 65536 and >=0.                                                                          |
@@ -756,9 +762,6 @@ In fact there is a maximum of 65536 block entries in an index.
 The writer can use any method to reduce the number of entries.
 An implicit block start at 0,0 can be assumed.
 
-It is strongly recommended adding `UncompressedSize`, 
-otherwise seeking from end-of-file (tailing for example) will not be possible. 
-
 ### Decoding entries:
 
 ```

diff --git a/s2/encode_test.go b/s2/encode_test.go
@@ -19,7 +19,7 @@ import (
 
 func testOptions(t testing.TB) map[string][]WriterOption {
 	var testOptions = map[string][]WriterOption{
-		"default": {},
+		"default": {WriterAddIndex()},
 		"better":  {WriterBetterCompression()},
 		"best":    {WriterBestCompression()},
 		"none":    {WriterUncompressed()},

diff --git a/s2/index.go b/s2/index.go
@@ -17,7 +17,7 @@ const (
 	maxIndexEntries = 1 << 16
 )
 
-// Index represents
+// Index represents an S2/Snappy index.
 type Index struct {
 	TotalUncompressed int64 // Total Uncompressed size if known. Will be -1 if unknown.
 	TotalCompressed   int64 // Total Compressed size if known. Will be -1 if unknown.
@@ -49,7 +49,7 @@ func (i *Index) allocInfos(n int) {
 }
 
 // add an uncompressed and compressed pair.
-// Entries should be sent in order.
+// Entries must be sent in order.
 func (i *Index) add(compressedOffset, uncompressedOffset int64) error {
 	if i == nil {
 		return nil
@@ -87,16 +87,16 @@ func (i *Index) add(compressedOffset, uncompressedOffset int64) error {
 // If offset from the end of the file is requested, but size is unknown,
 // ErrUnsupported will be returned.
 func (i *Index) Find(offset int64) (compressedOff, uncompressedOff int64, err error) {
+	if i.TotalUncompressed < 0 {
+		return 0, 0, ErrCorrupt
+	}
 	if offset < 0 {
-		if i.TotalUncompressed < 0 {
-			return 0, 0, ErrUnsupported
-		}
 		offset = i.TotalUncompressed + offset
 		if offset < 0 {
 			return 0, 0, io.ErrUnexpectedEOF
 		}
 	}
-	if i.TotalUncompressed >= 0 && offset > i.TotalUncompressed {
+	if offset > i.TotalUncompressed {
 		return 0, 0, io.ErrUnexpectedEOF
 	}
 	for _, info := range i.info {
@@ -232,7 +232,7 @@ func (i *Index) Load(b []byte) ([]byte, error) {
 	b = b[len(S2IndexHeader):]
 
 	// Total Uncompressed
-	if v, n := binary.Varint(b); n <= 0 {
+	if v, n := binary.Varint(b); n <= 0 || v < 0 {
 		return b, ErrCorrupt
 	} else {
 		i.TotalUncompressed = v