Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add new binary data parser #7030

Closed
wants to merge 64 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
64 commits
Select commit Hold shift + click to select a range
1a64cbe
Initial binary metric implementation
yolkhovyy Oct 13, 2019
2c1b466
Metric time
yolkhovyy Oct 13, 2019
72591fc
Metric time
yolkhovyy Oct 13, 2019
b76cea3
Added string type
yolkhovyy Oct 16, 2019
746e956
Re-factored and tests
yolkhovyy Oct 17, 2019
1365ce8
More tests
yolkhovyy Oct 17, 2019
bc81a4b
More tests
yolkhovyy Oct 18, 2019
7a511f2
Renamed to BinData
yolkhovyy Oct 18, 2019
5421933
Working bindata parser
yolkhovyy Oct 25, 2019
87ca271
Merge branch 'feature/fix_record_parser' into develop
yolkhovyy Oct 26, 2019
eda274e
Optional field size
yolkhovyy Nov 3, 2019
6f944e9
Added bindata parser description in README.md
yolkhovyy Nov 11, 2019
adfda86
README.md updated
yolkhovyy Dec 14, 2019
3e85c0b
Removed Protocol
yolkhovyy Dec 22, 2019
ec20ac2
Fixed typo
yolkhovyy Dec 22, 2019
c300d4f
Remove unused const
yolkhovyy Feb 2, 2020
7a79c8e
Merge branch 'master' into develop
yolkhovyy Feb 2, 2020
746ea4d
Updated README.md
yolkhovyy Feb 2, 2020
72f8137
Reworking string encoding
yolkhovyy Feb 5, 2020
4a1c812
Unit tests for string encoding
yolkhovyy Feb 5, 2020
7e9003c
Merge branch 'feature/new-string-encoding' into develop
yolkhovyy Feb 5, 2020
7b065d7
Padding test cleaned up
yolkhovyy Feb 6, 2020
e32d8a1
UTF-8 unit test
yolkhovyy Feb 6, 2020
ef749af
Unit tests cleaned up
yolkhovyy Feb 8, 2020
eb77521
Merge branch 'feature/utf8-encoding-test' into develop
yolkhovyy Feb 8, 2020
0a30031
Comments and commented out code
yolkhovyy Feb 8, 2020
c49fc04
README.md updated
yolkhovyy Feb 8, 2020
54a0cc8
../registry.go
yolkhovyy Feb 8, 2020
1ad2998
Added bindata factory, reworked unit tests
yolkhovyy Feb 8, 2020
b040ed3
Comments, etc
yolkhovyy Feb 8, 2020
7bdb452
Added unit test for default tags
yolkhovyy Feb 8, 2020
730f08d
Merge branch 'feature/bindata-factory' into develop
yolkhovyy Feb 8, 2020
5a2aee4
README.md updated
yolkhovyy Feb 8, 2020
e7c23c7
Duplicate field names check
yolkhovyy Feb 15, 2020
5b380cd
Merge branch 'develop'
yolkhovyy Feb 15, 2020
094174c
Removed bindata fields in config.go
yolkhovyy Feb 15, 2020
ccd4320
Merge branch 'master' into develop
yolkhovyy Feb 15, 2020
09622b4
Field removal
yolkhovyy Feb 15, 2020
4d3d355
Removed temporary Dockerfile
yolkhovyy Feb 15, 2020
984d7c6
Removed .vscode, reverted README.md
yolkhovyy Feb 16, 2020
2f222d5
Removed unnecassary change
yolkhovyy Feb 16, 2020
3fc716c
Formatting in bindata/paresr_test.go
yolkhovyy Feb 16, 2020
415e3cd
Merge branch 'master' of https://github.com/influxdata/telegraf
yolkhovyy Jun 20, 2020
499e9c3
Review remarks
yolkhovyy Dec 16, 2021
a31907b
Update plugins/parsers/bindata/README.md
yolkhovyy Jan 9, 2022
715a5d1
Update plugins/parsers/bindata/parser.go
yolkhovyy Jan 9, 2022
52e27b8
Update plugins/parsers/bindata/parser.go
yolkhovyy Jan 9, 2022
2abe97d
Update plugins/parsers/bindata/parser.go
yolkhovyy Jan 9, 2022
7fc1ba1
Removed new line
yolkhovyy Jan 9, 2022
f71923f
Removed new line
yolkhovyy Jan 9, 2022
f0c98e1
Update plugins/parsers/bindata/parser.go
yolkhovyy Jan 9, 2022
275858d
Update plugins/parsers/bindata/parser_test.go
yolkhovyy Jan 9, 2022
313c538
Update plugins/parsers/bindata/parser_test.go
yolkhovyy Jan 9, 2022
6be9cc2
Fixed unit test
yolkhovyy Jan 9, 2022
52a45a6
Removed new lines in tests
yolkhovyy Jan 9, 2022
016372b
Default endiannes
yolkhovyy Jan 9, 2022
013c539
Resolved merge conflicts
yolkhovyy Jan 23, 2022
d110b54
Compilation error fixe
yolkhovyy Jan 23, 2022
9ab9d4f
Reworked getParserConfig in config/config.go
yolkhovyy Jan 23, 2022
a8185c4
Lint errors fixed
yolkhovyy Jan 23, 2022
d127f83
Lint errors fixed
yolkhovyy Jan 23, 2022
0b154c2
Lint errors fixed
yolkhovyy Jan 23, 2022
9a93ace
Review comments addressed
yolkhovyy Jan 23, 2022
e5ad2d1
Review comments addressed
yolkhovyy Jan 23, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Unit tests for string encoding
  • Loading branch information
yolkhovyy committed Feb 5, 2020
commit 4a1c812b976bad4abc2989a06f49cf57f150dd5f
10 changes: 8 additions & 2 deletions plugins/parsers/bindata/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

const timeKey = "time"
const timezone = "UTC"
const stringEncoding = "UTF-8"

// Field is ...
type Field struct {
Expand All @@ -29,8 +30,8 @@ type BinData struct {
MetricName string
TimeFormat string
Endiannes string
Fields []Field
StringEncoding string
Fields []Field
// TagKeys []string
DefaultTags map[string]string
}
Expand Down Expand Up @@ -126,8 +127,13 @@ var fieldTypes = map[string]reflect.Type{

func (binData *BinData) validate() error {
if binData.StringEncoding == "" {
binData.StringEncoding = "utf-8"
binData.StringEncoding = stringEncoding
}
binData.StringEncoding = strings.ToUpper(binData.StringEncoding)
if binData.StringEncoding != stringEncoding {
return fmt.Errorf(`invalid string encoding %s`, binData.StringEncoding)
}

for i := 0; i < len(binData.Fields); i++ {
fieldType, ok := fieldTypes[strings.ToLower(binData.Fields[i].Type)]
if !ok {
Expand Down
49 changes: 22 additions & 27 deletions plugins/parsers/bindata/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,23 +52,6 @@ var fields = []Field{
Field{Name: "time", Type: "int32"},
}

var fieldsWithSizes = []Field{
Field{Name: "fieldBool0", Type: "bool", Size: 0},
Field{Name: "fieldBool1", Type: "bool", Size: 1},
Field{Name: "fieldUint8", Type: "uint8", Size: 2},
Field{Name: "fieldInt8", Type: "int8", Size: 3},
Field{Name: "fieldUint16", Type: "uint16", Size: 4},
Field{Name: "fieldInt16", Type: "int16", Size: 5},
Field{Name: "fieldUint32", Type: "uint32", Size: 6},
Field{Name: "fieldInt32", Type: "int32", Size: 7},
Field{Name: "fieldUint64", Type: "uint64", Size: 8},
Field{Name: "fieldInt64", Type: "int64", Size: 9},
Field{Name: "fieldFloat32", Type: "float32", Size: 10},
Field{Name: "fieldFloat64", Type: "float64", Size: 11},
Field{Name: "fieldString", Type: "string", Size: 20},
Field{Name: "time", Type: "int32", Size: 21},
}

var withOmittedFields = []Field{
Field{Type: "padding", Size: 1},
Field{Name: "fieldBool1", Type: "bool"},
Expand Down Expand Up @@ -140,23 +123,35 @@ func TestLittleEndian(t *testing.T) {
assert.Equal(t, expectedParseResult, metrics[0].Fields())
}

func TestFieldsWithSize(t *testing.T) {
func TestDefaultStringEncoding(t *testing.T) {

var withSize = BinData{
MetricName: "with_size",
Endiannes: "be",
TimeFormat: "unix",
Fields: fieldsWithSizes,
var defaultStringEncoding = BinData{
MetricName: "default_string_encoding",
Endiannes: "be",
TimeFormat: "unix",
StringEncoding: "utf-8",
Fields: fields,
}

metrics, err := withSize.Parse(binaryDataBigEndian)
metrics, err := defaultStringEncoding.Parse(binaryDataBigEndian)
require.NoError(t, err)
assert.Len(t, metrics, 1)
require.Equal(t, withSize.MetricName, metrics[0].Name())
assert.Equal(t, int64(0x5DA86C4C), metrics[0].Time().Unix())
assert.Equal(t, expectedParseResult, metrics[0].Fields())
}

func TestInvalidStringEncoding(t *testing.T) {

var invalidStringEncoding = BinData{
MetricName: "invalid_string_encoding",
Endiannes: "be",
TimeFormat: "unix",
StringEncoding: "utf-16",
Fields: fields,
}

metrics, err := invalidStringEncoding.Parse(binaryDataBigEndian)
require.Error(t, err)
assert.Len(t, metrics, 0)
}
func TestWithOmittedFields(t *testing.T) {

var withFieldsOmitted = BinData{
Expand Down