diff --git a/.circleci/config.yml b/.circleci/config.yml index b2043e1fa291c..027a529cb0385 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,6 +25,23 @@ executors: GOFLAGS: -p=8 commands: + generate-config: + parameters: + os: + type: string + default: "linux" + steps: + - checkout + - attach_workspace: + at: '/build' + - run: ./scripts/generate_config.sh << parameters.os >> + - store_artifacts: + path: './new-config' + destination: 'new-config' + - persist_to_workspace: + root: './new-config' + paths: + - '*' check-changed-files-or-halt: steps: - run: ./scripts/check-file-changes.sh @@ -109,9 +126,6 @@ commands: release: type: boolean default: false - nightly: - type: boolean - default: false type: type: string default: "" @@ -123,19 +137,13 @@ commands: - when: condition: << parameters.release >> steps: - - run: 'mips=1 mipsel=1 arm64=1 amd64=1 static=1 armel=1 armhf=1 s390x=1 ppc641e=1 i386=1 windows=1 darwin=1 make package' - - when: - condition: << parameters.nightly >> - steps: - - run: 'mips=1 mipsel=1 arm64=1 amd64=1 static=1 armel=1 armhf=1 s390x=1 ppc641e=1 i386=1 windows=1 darwin=1 NIGHTLY=1 make package' - - run: 'make upload-nightly' + - run: 'make package' - unless: condition: or: - - << parameters.nightly >> - << parameters.release >> steps: - - run: '<< parameters.type >>=1 make package' + - run: 'make package include_packages="$(make << parameters.type >>)"' - store_artifacts: path: './build/dist' destination: 'build/dist' @@ -215,11 +223,11 @@ jobs: steps: - package-build: type: i386 - ppc641e-package: - executor: go-1_17 + ppc64le-package: + executor: go-1_16 steps: - package-build: - type: ppc641e + type: ppc64le s390x-package: executor: go-1_17 steps: @@ -269,8 +277,17 @@ jobs: nightly: executor: go-1_17 steps: - - package-build: - nightly: true + - attach_workspace: + at: '/build' + - run: + command: | + aws s3 sync /build/dist s3://dl.influxdata.com/telegraf/nightlies/ \ + --exclude "*" \ + --include "*.tar.gz" \ + --include "*.deb" \ + --include "*.rpm" \ + --include "*.zip" \ + --acl public-read package-consolidate: executor: name: win/default @@ -335,6 +352,81 @@ jobs: PR=${CIRCLE_PULL_REQUEST##*/} printf -v payload '{ "pullRequestNumber": "%s" }' "$PR" curl -X POST "https://182c7jdgog.execute-api.us-east-1.amazonaws.com/prod/shareArtifacts" --data "$payload" + generate-config: + executor: go-1_17 + steps: + - generate-config + generate-config-win: + executor: + name: win/default + shell: bash.exe + steps: + - generate-config: + os: windows + update-config: + executor: go-1_17 + steps: + - checkout + - attach_workspace: + at: '/new-config' + - run: ./scripts/update_config.sh ${UPDATE_CONFIG_TOKEN} + +commonjobs: + - &test-awaiter + 'test-awaiter': + requires: + - 'test-go-1_16' + - 'test-go-1_16-386' + - 'test-go-1_17' + - 'test-go-1_17-386' + - &windows-package + 'windows-package': + requires: + - 'test-go-windows' + - &darwin-package + 'darwin-package': + requires: + - 'test-go-mac' + - &i386-package + 'i386-package': + requires: + - 'test-awaiter' + - &ppc64le-package + 'ppc64le-package': + requires: + - 'test-awaiter' + - &s390x-package + 's390x-package': + requires: + - 'test-awaiter' + - &armel-package + 'armel-package': + requires: + - 'test-awaiter' + - &amd64-package + 'amd64-package': + requires: + - 'test-awaiter' + - &arm64-package + 'arm64-package': + requires: + - 'test-awaiter' + - &armhf-package + 'armhf-package': + requires: + - 'test-awaiter' + - &static-package + 'static-package': + requires: + - 'test-awaiter' + - &mipsel-package + 'mipsel-package': + requires: + - 'test-awaiter' + - &mips-package + 'mips-package': + requires: + - 'test-awaiter' workflows: version: 2 @@ -376,52 +468,45 @@ workflows: filters: tags: only: /.*/ - - 'test-awaiter': - requires: - - 'test-go-1_16' - - 'test-go-1_16-386' - - 'test-go-1_17' - - 'test-go-1_17-386' - - 'windows-package': - requires: - - 'test-go-windows' - - 'darwin-package': - requires: - - 'test-go-mac' - - 'i386-package': - requires: - - 'test-awaiter' - - 'ppc641e-package': - requires: - - 'test-awaiter' - - 's390x-package': - requires: - - 'test-awaiter' - - 'armel-package': + - *test-awaiter + - *windows-package + - *darwin-package + - *i386-package + - *ppc64le-package + - *s390x-package + - *armel-package + - *amd64-package + - *arm64-package + - *armhf-package + - *static-package + - *mipsel-package + - *mips-package + - 'generate-config': requires: - - 'test-awaiter' - - 'amd64-package': - requires: - - 'test-awaiter' - - 'arm64-package': - requires: - - 'test-awaiter' - - 'armhf-package': - requires: - - 'test-awaiter' - - 'static-package': - requires: - - 'test-awaiter' - - 'mipsel-package': + - 'amd64-package' + filters: + branches: + only: + - master + - 'generate-config-win': requires: - - 'test-awaiter' - - 'mips-package': + - 'windows-package' + filters: + branches: + only: + - master + - 'update-config': requires: - - 'test-awaiter' + - 'generate-config-win' + - 'generate-config' + filters: + branches: + only: + - master - 'share-artifacts': requires: - 'i386-package' - - 'ppc641e-package' + - 'ppc64le-package' - 's390x-package' - 'armel-package' - 'amd64-package' @@ -479,14 +564,33 @@ workflows: - 'deps' - 'test-go-mac' - 'test-go-windows' - - 'nightly': + - *test-awaiter + - *windows-package + - *darwin-package + - *i386-package + - *ppc64le-package + - *s390x-package + - *armel-package + - *amd64-package + - *arm64-package + - *armhf-package + - *static-package + - *mipsel-package + - *mips-package + - nightly: requires: - - 'test-go-windows' - - 'test-go-mac' - - 'test-go-1_16' - - 'test-go-1_16-386' - - 'test-go-1_17' - - 'test-go-1_17-386' + - 'i386-package' + - 'ppc64le-package' + - 's390x-package' + - 'armel-package' + - 'amd64-package' + - 'mipsel-package' + - 'mips-package' + - 'darwin-package' + - 'windows-package' + - 'static-package' + - 'arm64-package' + - 'armhf-package' triggers: - schedule: cron: "0 7 * * *" diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1c717ddbb1a15..67b65a26247fb 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,7 +5,7 @@ show completion. --> - [ ] Updated associated README.md. - [ ] Wrote appropriate unit tests. -- [ ] Pull request title or commits are in [conventional commit format](https://www.conventionalcommits.org/en/v1.0.0/#summary) (e.g. feat: or fix:) +- [ ] Pull request title or commits are in [conventional commit format](https://www.conventionalcommits.org/en/v1.0.0/#summary) setting alias: {} name: {} id: {}'".format(alias, name, id)) + state["aliases"][id] = name + if "value" in metric.fields: + buildTopicTags(metric, topicFields) + buildNameTags(metric, name) + else: + output = None + + # Try to resolve the unresolved if any + if len(state["unresolved"]) > 0: + # Filter out the matching metrics and keep the rest as unresolved + log.debug(" unresolved") + unresolved = [("{}/{}".format(edgeid, m.fields["alias"]), m) for m in state["unresolved"]] + matching = [(mid, m) for mid, m in unresolved if mid == id] + state["unresolved"] = [m for mid, m in unresolved if mid != id] + + log.debug(" found {} matching unresolved metrics".format(len(matching))) + # Process the matching metrics and output - TODO - needs debugging + # for mid, m in matching: + # buildTopicTags(m,topicFields) + # buildNameTags(m) + # output = [m for _, m in matching] + [metric] + + elif DATA_TAG in topic: + log.debug(" metric msg_type: {} edgeid: {} topic: {}".format(DATA_TAG, edgeid, topic)) + if "alias" in metric.fields: + alias = metric.fields.get("alias") + + # Lookup the ID. If we know it, replace the name of the metric with the lookup value, + # otherwise we need to keep the metric for resolving later. + # This can happen if the messages are out-of-order for some reason... + id = "{}/{}".format(edgeid,alias) + if id in state["aliases"]: + name = state["aliases"][id] + log.debug(" found alias: {} name: {}".format(alias, name)) + buildTopicTags(metric,topicFields) + buildNameTags(metric,name) + else: + # We want to hold the metric until we get the corresponding birth message + log.debug(" id not found: {}".format(id)) + output = None + if len(state["unresolved"]) >= MAX_UNRESOLVED: + log.warn(" metric overflow, trimming {}".format(len(state["unresolved"]) - MAX_UNRESOLVED+1)) + # Release the unresolved metrics as raw and trim buffer + output = state["unresolved"][MAX_UNRESOLVED-1:] + state["unresolved"] = state["unresolved"][:MAX_UNRESOLVED-1] + log.debug(" --> keeping metric") + state["unresolved"].append(metric) + else: + output = None + + return output + diff --git a/plugins/serializers/json/README.md b/plugins/serializers/json/README.md index 08bb9d4f7c904..b33875578272a 100644 --- a/plugins/serializers/json/README.md +++ b/plugins/serializers/json/README.md @@ -19,6 +19,13 @@ The `json` output data format converts metrics into JSON documents. ## such as "1ns", "1us", "1ms", "10ms", "1s". Durations are truncated to ## the power of 10 less than the specified units. json_timestamp_units = "1s" + + ## The default timestamp format is Unix epoch time, subject to the + # resolution configured in json_timestamp_units. + # Other timestamp layout can be configured using the Go language time + # layout specification from https://golang.org/pkg/time/#Time.Format + # e.g.: json_timestamp_format = "2006-01-02T15:04:05Z07:00" + #json_timestamp_format = "" ``` ### Examples: diff --git a/plugins/serializers/json/json.go b/plugins/serializers/json/json.go index e2d7af3305117..6db2a43ee231a 100644 --- a/plugins/serializers/json/json.go +++ b/plugins/serializers/json/json.go @@ -8,18 +8,20 @@ import ( "github.com/influxdata/telegraf" ) -type serializer struct { - TimestampUnits time.Duration +type Serializer struct { + TimestampUnits time.Duration + TimestampFormat string } -func NewSerializer(timestampUnits time.Duration) (*serializer, error) { - s := &serializer{ - TimestampUnits: truncateDuration(timestampUnits), +func NewSerializer(timestampUnits time.Duration, timestampformat string) (*Serializer, error) { + s := &Serializer{ + TimestampUnits: truncateDuration(timestampUnits), + TimestampFormat: timestampformat, } return s, nil } -func (s *serializer) Serialize(metric telegraf.Metric) ([]byte, error) { +func (s *Serializer) Serialize(metric telegraf.Metric) ([]byte, error) { m := s.createObject(metric) serialized, err := json.Marshal(m) if err != nil { @@ -30,7 +32,7 @@ func (s *serializer) Serialize(metric telegraf.Metric) ([]byte, error) { return serialized, nil } -func (s *serializer) SerializeBatch(metrics []telegraf.Metric) ([]byte, error) { +func (s *Serializer) SerializeBatch(metrics []telegraf.Metric) ([]byte, error) { objects := make([]interface{}, 0, len(metrics)) for _, metric := range metrics { m := s.createObject(metric) @@ -48,7 +50,7 @@ func (s *serializer) SerializeBatch(metrics []telegraf.Metric) ([]byte, error) { return serialized, nil } -func (s *serializer) createObject(metric telegraf.Metric) map[string]interface{} { +func (s *Serializer) createObject(metric telegraf.Metric) map[string]interface{} { m := make(map[string]interface{}, 4) tags := make(map[string]string, len(metric.TagList())) @@ -71,7 +73,11 @@ func (s *serializer) createObject(metric telegraf.Metric) map[string]interface{} m["fields"] = fields m["name"] = metric.Name() - m["timestamp"] = metric.Time().UnixNano() / int64(s.TimestampUnits) + if s.TimestampFormat == "" { + m["timestamp"] = metric.Time().UnixNano() / int64(s.TimestampUnits) + } else { + m["timestamp"] = metric.Time().UTC().Format(s.TimestampFormat) + } return m } diff --git a/plugins/serializers/json/json_test.go b/plugins/serializers/json/json_test.go index 74d7f94166621..be939243904eb 100644 --- a/plugins/serializers/json/json_test.go +++ b/plugins/serializers/json/json_test.go @@ -30,7 +30,7 @@ func TestSerializeMetricFloat(t *testing.T) { } m := metric.New("cpu", tags, fields, now) - s, _ := NewSerializer(0) + s, _ := NewSerializer(0, "") var buf []byte buf, err := s.Serialize(m) assert.NoError(t, err) @@ -40,9 +40,10 @@ func TestSerializeMetricFloat(t *testing.T) { func TestSerialize_TimestampUnits(t *testing.T) { tests := []struct { - name string - timestampUnits time.Duration - expected string + name string + timestampUnits time.Duration + timestampFormat string + expected string }{ { name: "default of 1s", @@ -74,6 +75,11 @@ func TestSerialize_TimestampUnits(t *testing.T) { timestampUnits: 65 * time.Millisecond, expected: `{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":152547879512}`, }, + { + name: "timestamp format", + timestampFormat: "2006-01-02T15:04:05Z07:00", + expected: `{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":"2018-05-05T00:06:35Z"}`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -85,7 +91,7 @@ func TestSerialize_TimestampUnits(t *testing.T) { }, time.Unix(1525478795, 123456789), ) - s, _ := NewSerializer(tt.timestampUnits) + s, _ := NewSerializer(tt.timestampUnits, tt.timestampFormat) actual, err := s.Serialize(m) require.NoError(t, err) require.Equal(t, tt.expected+"\n", string(actual)) @@ -103,7 +109,7 @@ func TestSerializeMetricInt(t *testing.T) { } m := metric.New("cpu", tags, fields, now) - s, _ := NewSerializer(0) + s, _ := NewSerializer(0, "") var buf []byte buf, err := s.Serialize(m) assert.NoError(t, err) @@ -122,7 +128,7 @@ func TestSerializeMetricString(t *testing.T) { } m := metric.New("cpu", tags, fields, now) - s, _ := NewSerializer(0) + s, _ := NewSerializer(0, "") var buf []byte buf, err := s.Serialize(m) assert.NoError(t, err) @@ -142,7 +148,7 @@ func TestSerializeMultiFields(t *testing.T) { } m := metric.New("cpu", tags, fields, now) - s, _ := NewSerializer(0) + s, _ := NewSerializer(0, "") var buf []byte buf, err := s.Serialize(m) assert.NoError(t, err) @@ -161,7 +167,7 @@ func TestSerializeMetricWithEscapes(t *testing.T) { } m := metric.New("My CPU", tags, fields, now) - s, _ := NewSerializer(0) + s, _ := NewSerializer(0, "") buf, err := s.Serialize(m) assert.NoError(t, err) @@ -180,7 +186,7 @@ func TestSerializeBatch(t *testing.T) { ) metrics := []telegraf.Metric{m, m} - s, _ := NewSerializer(0) + s, _ := NewSerializer(0, "") buf, err := s.SerializeBatch(metrics) require.NoError(t, err) require.Equal(t, []byte(`{"metrics":[{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":0},{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":0}]}`), buf) @@ -199,7 +205,7 @@ func TestSerializeBatchSkipInf(t *testing.T) { ), } - s, err := NewSerializer(0) + s, err := NewSerializer(0, "") require.NoError(t, err) buf, err := s.SerializeBatch(metrics) require.NoError(t, err) @@ -218,7 +224,7 @@ func TestSerializeBatchSkipInfAllFields(t *testing.T) { ), } - s, err := NewSerializer(0) + s, err := NewSerializer(0, "") require.NoError(t, err) buf, err := s.SerializeBatch(metrics) require.NoError(t, err) diff --git a/plugins/serializers/registry.go b/plugins/serializers/registry.go index e67a9594dda73..b17364e66f0a6 100644 --- a/plugins/serializers/registry.go +++ b/plugins/serializers/registry.go @@ -88,6 +88,9 @@ type Config struct { // Timestamp units to use for JSON formatted output TimestampUnits time.Duration `toml:"timestamp_units"` + // Timestamp format to use for JSON formatted output + TimestampFormat string `toml:"timestamp_format"` + // Include HEC routing fields for splunkmetric output HecRouting bool `toml:"hec_routing"` @@ -123,7 +126,7 @@ func NewSerializer(config *Config) (Serializer, error) { case "graphite": serializer, err = NewGraphiteSerializer(config.Prefix, config.Template, config.GraphiteTagSupport, config.GraphiteTagSanitizeMode, config.GraphiteSeparator, config.Templates) case "json": - serializer, err = NewJSONSerializer(config.TimestampUnits) + serializer, err = NewJSONSerializer(config.TimestampUnits, config.TimestampFormat) case "splunkmetric": serializer, err = NewSplunkmetricSerializer(config.HecRouting, config.SplunkmetricMultiMetric) case "nowmetric": @@ -188,8 +191,8 @@ func NewWavefrontSerializer(prefix string, useStrict bool, sourceOverride []stri return wavefront.NewSerializer(prefix, useStrict, sourceOverride) } -func NewJSONSerializer(timestampUnits time.Duration) (Serializer, error) { - return json.NewSerializer(timestampUnits) +func NewJSONSerializer(timestampUnits time.Duration, timestampFormat string) (Serializer, error) { + return json.NewSerializer(timestampUnits, timestampFormat) } func NewCarbon2Serializer(carbon2format string, carbon2SanitizeReplaceChar string) (Serializer, error) { diff --git a/scripts/generate_config.sh b/scripts/generate_config.sh new file mode 100755 index 0000000000000..c85dd05172631 --- /dev/null +++ b/scripts/generate_config.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# This script is responsible for generating the Telegraf config found under the `etc` directory. +# This script is meant to be only ran in within the Circle CI pipeline so that the Tiger Bot can update them automatically. +# It supports Windows and Linux because the configs are different depending on the OS. + + +os=$1 # windows or linux +exe_path="/build/extracted" # Path will contain telegraf binary +config_name="telegraf.conf" + +if [ "$os" = "windows" ]; then + zip=$(/bin/find ./build/dist -maxdepth 1 -name "*windows_amd64.zip" -print) + exe_path="$PWD/build/extracted" + unzip "$zip" -d "$exe_path" + config_name="telegraf_windows.conf" + exe_path=$(/bin/find "$exe_path" -name telegraf.exe -type f -print) +else + tar_path=$(find /build/dist -maxdepth 1 -name "*linux_amd64.tar.gz" -print | grep -v ".*static.*") + mkdir "$exe_path" + tar --extract --file="$tar_path" --directory "$exe_path" + exe_path=$(find "$exe_path" -name telegraf -type f -print | grep ".*usr/bin/.*") +fi + +$exe_path config > $config_name + +mkdir ./new-config +mv $config_name ./new-config diff --git a/scripts/update_config.sh b/scripts/update_config.sh new file mode 100755 index 0000000000000..87cfe2620ab61 --- /dev/null +++ b/scripts/update_config.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# This script is responsible for triggering the Tiger Bot endpoint that will create the pull request with the newly generated configs. +# This script is meant to be only ran in within the Circle CI pipeline. + +token=$1 + +config_path="/new-config" + +if [ ! -f "$config_path/telegraf.conf" ]; then + echo "$config_path/telegraf.conf does not exist" + exit +fi +if [ ! -f "$config_path/telegraf_windows.conf" ]; then + echo "$config_path/telegraf_windows.conf does not exist" + exit +fi + +if cmp -s "$config_path/telegraf.conf" "etc/telegraf.conf" && cmp -s "$config_path/telegraf_windows.conf" "etc/telegraf_windows.conf"; then + echo "Both telegraf.conf and telegraf_windows.conf haven't changed" +fi + +curl -H "Authorization: Bearer $token" -X POST "https://182c7jdgog.execute-api.us-east-1.amazonaws.com/prod/updateConfig" diff --git a/testutil/tls.go b/testutil/tls.go index 68a244a8b1e74..686f327d06f49 100644 --- a/testutil/tls.go +++ b/testutil/tls.go @@ -2,7 +2,7 @@ package testutil import ( "fmt" - "io/ioutil" + "io" "os" "path" @@ -93,7 +93,7 @@ func readCertificate(filename string) string { if err != nil { panic(fmt.Sprintf("opening %q: %v", filename, err)) } - octets, err := ioutil.ReadAll(file) + octets, err := io.ReadAll(file) if err != nil { panic(fmt.Sprintf("reading %q: %v", filename, err)) }