Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

s2: Add LZ4 block converter #748

Merged
merged 14 commits into from
Feb 17, 2023
Merged

s2: Add LZ4 block converter #748

Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
fe0c088
s2: Add LZ4 block converter
klauspost Feb 7, 2023
519a51c
Tweak tests.
klauspost Feb 7, 2023
b691e88
Return error when truncated.
klauspost Feb 7, 2023
b214753
Improve speed with inlining
klauspost Feb 14, 2023
0751108
Tweak benchmarks
klauspost Feb 14, 2023
95a0452
Add assembly converter
klauspost Feb 14, 2023
f082d16
Add asm function stub for other platforms.
klauspost Feb 15, 2023
66f7ba9
Merge branch 'master' into add-lz4-block-converter
klauspost Feb 15, 2023
80de799
Regen assembly.
klauspost Feb 16, 2023
ea57f35
Add fuzz tests.
klauspost Feb 16, 2023
55def3f
Combine write
klauspost Feb 16, 2023
7f1ea45
Add Snappy block converter. Why not I guess...
klauspost Feb 17, 2023
928a1c4
Fix lz4->snappy assembly.
klauspost Feb 17, 2023
25a9ae5
Add fuzz corpus
klauspost Feb 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Tweak tests.
  • Loading branch information
@klauspost
klauspost committed Feb 7, 2023
commit 519a51c3fc532f135f55580c7014c0a4f76f1350
16 changes: 11 additions & 5 deletions s2/lz4convert_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,19 @@ func TestLZ4Converter_ConvertBlock(t *testing.T) {
t.Errorf("output mismatch")
}

out2 := EncodeBetter(s2Dst[:0], data)
out2 := Encode(s2Dst[:0], data)
sz2 := len(out2) - hdr
t.Log("s2 (better) size:", sz2)
t.Log("s2 (default) size:", sz2)

out2 = EncodeBetter(s2Dst[:0], data)
sz3 := len(out2) - hdr
t.Log("s2 (better) size:", sz3)

t.Log("lz4 -> s2 bytes saved:", len(lz4Data)-sz)
t.Log("data -> s2 (better) bytes saved:", len(lz4Data)-sz2)
t.Log("direct data -> s2 better compared to converted from lz4:", sz-sz2)
t.Log("data -> s2 (default) bytes saved:", len(lz4Data)-sz2)
t.Log("data -> s2 (better) bytes saved:", len(lz4Data)-sz3)
t.Log("direct data -> s2 (default) compared to converted from lz4:", sz-sz2)
t.Log("direct data -> s2 (better) compared to converted from lz4:", sz-sz3)
})
}
}
Expand Down Expand Up @@ -110,7 +116,7 @@ func BenchmarkLZ4Converter_ConvertBlock(b *testing.B) {
}

func BenchmarkCompressBlockReference(b *testing.B) {
//b.Skip("Only reference for BenchmarkLZ4Converter_ConvertBlock")
b.Skip("Only reference for BenchmarkLZ4Converter_ConvertBlock")
for _, tf := range testFiles {
b.Run(tf.label, func(b *testing.B) {
if err := downloadBenchmarkFiles(b, tf.filename); err != nil {
Expand Down