Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(cmd/influx): support reading GZIP data in influx write #20763

Merged
merged 15 commits into from
Feb 19, 2021
Merged

feat(cmd/influx): support reading GZIP data in influx write #20763

Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
984a204
feat(ingest): add the ability to read compressed files to influx write
davidby-influx Jan 14, 2021
3ff0f8d
feat: enable GZIP auto-detection, and add tests
danxmoran Feb 4, 2021
97fe44a
refactor: remove unused flag, and clean up how readers are opened
danxmoran Feb 16, 2021
6108bbc
fix: adjust help text
danxmoran Feb 16, 2021
8116e70
chore: fix formatting
danxmoran Feb 16, 2021
e0f1368
refactor: use string flag for compression instead of bool
danxmoran Feb 16, 2021
03a3981
chore: more test cleanups.
danxmoran Feb 16, 2021
c52995f
chore: remove unused field
danxmoran Feb 16, 2021
fd8c4a4
test: add more write cases, and clean up constants
danxmoran Feb 16, 2021
1b7ad46
chore: update CHANGELOG
danxmoran Feb 18, 2021
f3e422e
fix: decode after decompressing
danxmoran Feb 18, 2021
033b862
chore: fix formatting
danxmoran Feb 18, 2021
3b62d94
test: fix reuse of gzip stdin contents
danxmoran Feb 18, 2021
50136d3
chore: fix formatting again
danxmoran Feb 18, 2021
bbcd966
fix: ensure original file name is reported in errors
danxmoran Feb 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test: fix reuse of gzip stdin contents
  • Loading branch information
@danxmoran
danxmoran committed Feb 19, 2021
commit 3b62d943f1126cb83997684bbc67a8ee40691c2c
37 changes: 24 additions & 13 deletions cmd/influx/write_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,27 +100,26 @@ func Test_writeFlags_dump(t *testing.T) {
func Test_writeFlags_createLineReader(t *testing.T) {
defer removeTempFiles()

gzipStdin := func(uncompressed string) io.Reader {
contents := &bytes.Buffer{}
writer := gzip.NewWriter(contents)
_, err := writer.Write([]byte(uncompressed))
require.NoError(t, err)
require.NoError(t, writer.Close())
return contents
}

lpContents := "f1 b=f2,c=f3,d=f4"
lpFile := createTempFile(t, "txt", []byte(lpContents), false)
gzipLpFile := createTempFile(t, "txt.gz", []byte(lpContents), true)
gzipLpFileNoExt := createTempFile(t, "lp", []byte(lpContents), true)
stdInLpContents := "stdin3 i=stdin1,j=stdin2,k=stdin4"
stdInLpGzipContents := &bytes.Buffer{}
stdInLpGzipWriter := gzip.NewWriter(stdInLpGzipContents)
_, err := stdInLpGzipWriter.Write([]byte(stdInLpContents))
require.NoError(t, err)
require.NoError(t, stdInLpGzipWriter.Close())

csvContents := "_measurement,b,c,d\nf1,f2,f3,f4"
csvFile := createTempFile(t, "csv", []byte(csvContents), false)
gzipCsvFile := createTempFile(t, "csv.gz", []byte(csvContents), true)
gzipCsvFileNoExt := createTempFile(t, "csv", []byte(csvContents), true)
stdInCsvContents := "i,j,_measurement,k\nstdin1,stdin2,stdin3,stdin4"
stdInCsvGzipContents := &bytes.Buffer{}
stdInCsvGzipWriter := gzip.NewWriter(stdInCsvGzipContents)
_, err = stdInCsvGzipWriter.Write([]byte(stdInCsvContents))
require.NoError(t, err)
require.NoError(t, stdInCsvGzipWriter.Close())

// use a test HTTP server to provide CSV data
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
Expand Down Expand Up @@ -235,7 +234,7 @@ func Test_writeFlags_createLineReader(t *testing.T) {
flags: writeFlagsBuilder{
Compression: inputCompressionGzip,
},
stdIn: stdInLpGzipContents,
stdIn: gzipStdin(stdInLpContents),
lines: []string{
stdInLpContents,
},
Expand All @@ -254,7 +253,7 @@ func Test_writeFlags_createLineReader(t *testing.T) {
flags: writeFlagsBuilder{
Compression: inputCompressionGzip,
},
stdIn: stdInLpGzipContents,
stdIn: gzipStdin(stdInLpContents),
arguments: []string{"-"},
lines: []string{
stdInLpContents,
Expand Down Expand Up @@ -388,7 +387,7 @@ func Test_writeFlags_createLineReader(t *testing.T) {
Format: inputFormatCsv,
Compression: inputCompressionGzip,
},
stdIn: stdInCsvGzipContents,
stdIn: gzipStdin(stdInCsvContents),
lines: []string{
stdInLpContents,
},
Expand All @@ -404,6 +403,18 @@ func Test_writeFlags_createLineReader(t *testing.T) {
stdInLpContents,
},
},
{
name: "read compressed CSV data from stdin using '-' argument + transform to line protocol",
flags: writeFlagsBuilder{
Format: inputFormatCsv,
Compression: inputCompressionGzip,
},
stdIn: gzipStdin(stdInCsvContents),
arguments: []string{"-"},
lines: []string{
stdInLpContents,
},
},
{
name: "read CSV data from 1st argument + transform to line protocol",
flags: writeFlagsBuilder{
Expand Down