-
Notifications
You must be signed in to change notification settings - Fork 5.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add influxdb gzip support #2978
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,7 @@ package client | |
|
||
import ( | ||
"bytes" | ||
"compress/gzip" | ||
"crypto/tls" | ||
"encoding/json" | ||
"fmt" | ||
|
@@ -95,8 +96,7 @@ type HTTPConfig struct { | |
HTTPProxy string | ||
|
||
// Gzip, if true, compresses each payload using gzip. | ||
// TODO | ||
// Gzip bool | ||
Gzip bool | ||
} | ||
|
||
// Response represents a list of statement results. | ||
|
@@ -233,15 +233,27 @@ func (c *httpClient) makeWriteRequest( | |
return nil, err | ||
} | ||
req.Header.Set("Content-Length", fmt.Sprint(contentLength)) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is going to complicate things... if we are using gzip we need to report the length of the body here after compression, but since we are now streaming the data it is trickier. I think we need to use I haven't ever done this with go, maybe we can use https://golang.org/pkg/net/http/httputil/#NewChunkedWriter? Or maybe it will be automatically handled if we don't set this header? |
||
// TODO | ||
// if gzip { | ||
// req.Header.Set("Content-Encoding", "gzip") | ||
// } | ||
if c.config.Gzip { | ||
req.Header.Set("Content-Encoding", "gzip") | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you move all the Header code into makeRequest, I'm not sure why they should be separate. |
||
return req, nil | ||
} | ||
|
||
func (c *httpClient) makeRequest(uri string, body io.Reader) (*http.Request, error) { | ||
req, err := http.NewRequest("POST", uri, body) | ||
var req *http.Request | ||
var err error | ||
if c.config.Gzip { | ||
// If gzip is set to true, then compress | ||
// the payload. | ||
buf := new(bytes.Buffer) | ||
buf.ReadFrom(body) | ||
compressed, err := compressWithGzip(buf.Bytes()) | ||
if err != nil { | ||
return nil, err | ||
} | ||
body = bytes.NewBuffer(compressed) | ||
} | ||
req, err = http.NewRequest("POST", uri, body) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
@@ -253,6 +265,18 @@ func (c *httpClient) makeRequest(uri string, body io.Reader) (*http.Request, err | |
return req, nil | ||
} | ||
|
||
func compressWithGzip(data []byte) ([]byte, error) { | ||
var buf bytes.Buffer | ||
gz := gzip.NewWriter(&buf) | ||
if _, err := gz.Write(data); err != nil { | ||
return nil, err | ||
} | ||
if err := gz.Close(); err != nil { | ||
return nil, err | ||
} | ||
return buf.Bytes(), nil | ||
} | ||
|
||
func (c *httpClient) Close() error { | ||
// Nothing to do. | ||
return nil | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Let's change the option to be
content_encoding = "gzip"
supporting onlygzip
, this will allow us to potentially add more encodings later and IMO reads better.