Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Converter processor: add support for large hexadecimal strings #9160

Merged
merged 7 commits into from
Apr 28, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions plugins/processors/converter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ Values that cannot be converted are dropped.
uniquely identifiable. Fields with the same series key (measurement + tags)
will overwrite one another.

**Note on large strings being converted to numeric types:** When converting a string value to a numeric type, precision may be lost if the number is too large. The largest numeric type this plugin supports is `float64`, and if a string 'number' exceeds its size limit, accuracy may be lost.

### Configuration
```toml
# Convert values to another metric value type
Expand Down
48 changes: 46 additions & 2 deletions plugins/processors/converter/converter.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
package converter

import (
"errors"
"fmt"
"math"
"math/big"
"strconv"
"strings"

"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/filter"
Expand Down Expand Up @@ -368,10 +371,19 @@ func toInteger(v interface{}) (int64, bool) {
result, err := strconv.ParseInt(value, 0, 64)

if err != nil {
result, err := strconv.ParseFloat(value, 64)
var result float64
var err error

if isHexadecimal(value) {
result, err = parseHexadecimal(value)
} else {
result, err = strconv.ParseFloat(value, 64)
}

if err != nil {
return 0, false
}

return toInteger(result)
}
return result, true
Expand Down Expand Up @@ -405,10 +417,19 @@ func toUnsigned(v interface{}) (uint64, bool) {
result, err := strconv.ParseUint(value, 0, 64)

if err != nil {
result, err := strconv.ParseFloat(value, 64)
var result float64
var err error

if isHexadecimal(value) {
result, err = parseHexadecimal(value)
} else {
result, err = strconv.ParseFloat(value, 64)
}

if err != nil {
return 0, false
}

return toUnsigned(result)
}
return result, true
Expand All @@ -430,6 +451,11 @@ func toFloat(v interface{}) (float64, bool) {
}
return 0.0, true
case string:
if isHexadecimal(value) {
result, err := parseHexadecimal(value)
return result, err == nil
}

result, err := strconv.ParseFloat(value, 64)
return result, err == nil
}
Expand All @@ -452,6 +478,24 @@ func toString(v interface{}) (string, bool) {
return "", false
}

func parseHexadecimal(value string) (float64, error) {
i := new(big.Int)

_, success := i.SetString(value, 0)
if !success {
return 0, errors.New("unable to parse string to big int")
}

f := new(big.Float).SetInt(i)
result, _ := f.Float64()
ivorybilled marked this conversation as resolved.
Show resolved Hide resolved

return result, nil
}

func isHexadecimal(value string) bool {
return len(value) >= 3 && strings.ToLower(value)[1] == 'x'
}

func init() {
processors.Add("converter", func() telegraf.Processor {
return &Converter{}
Expand Down
32 changes: 32 additions & 0 deletions plugins/processors/converter/converter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,38 @@ func TestConverter(t *testing.T) {
),
},
},
{
name: "from string field hexidecimal",
converter: &Converter{
Fields: &Conversion{
Integer: []string{"a"},
Unsigned: []string{"b"},
Float: []string{"c"},
},
},
input: testutil.MustMetric(
"cpu",
map[string]string{},
map[string]interface{}{
"a": "0x11826c",
"b": "0x11826c",
"c": "0x2139d19bb1c580ebe0",
},
time.Unix(0, 0),
),
expected: []telegraf.Metric{
testutil.MustMetric(
"cpu",
map[string]string{},
map[string]interface{}{
"a": int64(1147500),
"b": uint64(1147500),
"c": float64(612908836750534700000),
},
time.Unix(0, 0),
),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down