From 477a074cba08defc4fbcfc5d3c92461a3cac7610 Mon Sep 17 00:00:00 2001 From: Paulin Todev Date: Fri, 12 Apr 2024 13:18:59 +0100 Subject: [PATCH] Cherry pick commits for v0.40.4 (#6854) * Updating dependencies to fix CVEs: * CVE-2024-27304 * CVE-2024-27289 * CVE-2024-28180 * CVE-2024-24786 * Use stackdriver exporter fork with a fix for histogram sum + count (#6720) * Update kafka exporter dependency (#6778) Co-authored-by: kgeckhart Co-authored-by: William Dumont Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- CHANGELOG.md | 30 +++++ component/prometheus/exporter/kafka/kafka.go | 30 ++++- .../prometheus/exporter/kafka/kafka_test.go | 14 ++- .../internal/build/kafka_exporter.go | 13 +++ .../components/prometheus.exporter.kafka.md | 62 ++++++---- .../integrations/kafka-exporter-config.md | 40 ++++++- go.mod | 40 ++++--- go.sum | 73 ++++++------ integration-tests/docker-compose.yaml | 27 +++++ .../tests/unix/unix_metrics_test.go | 1 - integration-tests/utils.go | 3 + .../configs/kafka/Dockerfile | 9 ++ .../integration-tests/configs/kafka/main.go | 109 ++++++++++++++++++ .../tests/kafka/config.river | 25 ++++ .../tests/kafka/kafka_metrics_test.go | 34 ++++++ pkg/integrations/gcp_exporter/gcp_exporter.go | 5 +- .../kafka_exporter/kafka_exporter.go | 63 +++++++++- 17 files changed, 485 insertions(+), 93 deletions(-) create mode 100644 internal/cmd/integration-tests/configs/kafka/Dockerfile create mode 100644 internal/cmd/integration-tests/configs/kafka/main.go create mode 100644 internal/cmd/integration-tests/tests/kafka/config.river create mode 100644 internal/cmd/integration-tests/tests/kafka/kafka_metrics_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b6a30c1c090..d060eea647a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,36 @@ This document contains a historical list of changes between releases. Only changes that impact end-user behavior are listed; changes to documentation or internal API changes are not present. +v0.40.4 (2024-04-12) +-------------------- + +### Security fixes + +- Fixes following vulnerabilities (@ptodev) + * [CVE-2024-27304](https://github.com/advisories/GHSA-mrww-27vc-gghv) + * [CVE-2024-27289](https://github.com/advisories/GHSA-m7wr-2xf7-cm9p) + * [CVE-2024-28180](https://github.com/advisories/GHSA-c5q2-7r4c-mv6g) + * [CVE-2024-24786](https://github.com/advisories/GHSA-8r3f-844c-mc37) + +### Enhancements + +- Update `prometheus.exporter.kafka` with the following functionalities (@wildum): + * GSSAPI config + * enable/disable PA_FX_FAST + * set a TLS server name + * show the offset/lag for all consumer group or only the connected ones + * set the minimum number of topics to monitor + * enable/disable auto-creation of requested topics if they don't already exist + * regex to exclude topics / groups + * added metric kafka_broker_info + +- In `prometheus.exporter.kafka`, the interpolation table used to compute estimated lag metrics is now pruned + on `metadata_refresh_interval` instead of `prune_interval_seconds`. (@wildum) + +### Bugfixes + +- Update gcp_exporter to a newer version with a patch for incorrect delta histograms (@kgeckhart) + v0.40.3 (2024-03-14) -------------------- diff --git a/component/prometheus/exporter/kafka/kafka.go b/component/prometheus/exporter/kafka/kafka.go index e57bb69cd5a1..c0cdc7094f0b 100644 --- a/component/prometheus/exporter/kafka/kafka.go +++ b/component/prometheus/exporter/kafka/kafka.go @@ -20,8 +20,12 @@ var DefaultArguments = Arguments{ AllowConcurrent: true, MaxOffsets: 1000, PruneIntervalSeconds: 30, + OffsetShowAll: true, + TopicWorkers: 100, TopicsFilter: ".*", + TopicsExclude: "^$", GroupFilter: ".*", + GroupExclude: "^$", } type Arguments struct { @@ -32,7 +36,9 @@ type Arguments struct { SASLUsername string `river:"sasl_username,attr,optional"` SASLPassword rivertypes.Secret `river:"sasl_password,attr,optional"` SASLMechanism string `river:"sasl_mechanism,attr,optional"` + SASLDisablePAFXFast bool `river:"sasl_disable_pafx_fast,attr,optional"` UseTLS bool `river:"use_tls,attr,optional"` + TlsServerName string `river:"tls_server_name,attr,optional"` CAFile string `river:"ca_file,attr,optional"` CertFile string `river:"cert_file,attr,optional"` KeyFile string `river:"key_file,attr,optional"` @@ -42,11 +48,21 @@ type Arguments struct { ZookeeperURIs []string `river:"zookeeper_uris,attr,optional"` ClusterName string `river:"kafka_cluster_name,attr,optional"` MetadataRefreshInterval string `river:"metadata_refresh_interval,attr,optional"` + ServiceName string `river:"gssapi_service_name,attr,optional"` + KerberosConfigPath string `river:"gssapi_kerberos_config_path,attr,optional"` + Realm string `river:"gssapi_realm,attr,optional"` + KeyTabPath string `river:"gssapi_key_tab_path,attr,optional"` + KerberosAuthType string `river:"gssapi_kerberos_auth_type,attr,optional"` + OffsetShowAll bool `river:"offset_show_all,attr,optional"` + TopicWorkers int `river:"topic_workers,attr,optional"` AllowConcurrent bool `river:"allow_concurrency,attr,optional"` + AllowAutoTopicCreation bool `river:"allow_auto_topic_creation,attr,optional"` MaxOffsets int `river:"max_offsets,attr,optional"` - PruneIntervalSeconds int `river:"prune_interval_seconds,attr,optional"` + PruneIntervalSeconds int `river:"prune_interval_seconds,attr,optional"` // deprecated - no-op TopicsFilter string `river:"topics_filter_regex,attr,optional"` + TopicsExclude string `river:"topics_exclude_regex,attr,optional"` GroupFilter string `river:"groups_filter_regex,attr,optional"` + GroupExclude string `river:"groups_exclude_regex,attr,optional"` } func init() { @@ -97,7 +113,9 @@ func (a *Arguments) Convert() *kafka_exporter.Config { SASLUsername: a.SASLUsername, SASLPassword: config.Secret(a.SASLPassword), SASLMechanism: a.SASLMechanism, + SASLDisablePAFXFast: a.SASLDisablePAFXFast, UseTLS: a.UseTLS, + TlsServerName: a.TlsServerName, CAFile: a.CAFile, CertFile: a.CertFile, KeyFile: a.KeyFile, @@ -107,10 +125,20 @@ func (a *Arguments) Convert() *kafka_exporter.Config { ZookeeperURIs: a.ZookeeperURIs, ClusterName: a.ClusterName, MetadataRefreshInterval: a.MetadataRefreshInterval, + ServiceName: a.ServiceName, + KerberosConfigPath: a.KerberosConfigPath, + Realm: a.Realm, + KeyTabPath: a.KeyTabPath, + KerberosAuthType: a.KerberosAuthType, + OffsetShowAll: a.OffsetShowAll, + TopicWorkers: a.TopicWorkers, AllowConcurrent: a.AllowConcurrent, + AllowAutoTopicCreation: a.AllowAutoTopicCreation, MaxOffsets: a.MaxOffsets, PruneIntervalSeconds: a.PruneIntervalSeconds, TopicsFilter: a.TopicsFilter, + TopicsExclude: a.TopicsExclude, GroupFilter: a.GroupFilter, + GroupExclude: a.GroupExclude, } } diff --git a/component/prometheus/exporter/kafka/kafka_test.go b/component/prometheus/exporter/kafka/kafka_test.go index 7529677dbef6..b1791a5b9608 100644 --- a/component/prometheus/exporter/kafka/kafka_test.go +++ b/component/prometheus/exporter/kafka/kafka_test.go @@ -18,7 +18,6 @@ func TestRiverUnmarshal(t *testing.T) { metadata_refresh_interval = "1m" allow_concurrency = true max_offsets = 1000 - prune_interval_seconds = 30 topics_filter_regex = ".*" groups_filter_regex = ".*" ` @@ -35,8 +34,12 @@ func TestRiverUnmarshal(t *testing.T) { AllowConcurrent: true, MaxOffsets: 1000, PruneIntervalSeconds: 30, + OffsetShowAll: true, + TopicWorkers: 100, TopicsFilter: ".*", GroupFilter: ".*", + TopicsExclude: "^$", + GroupExclude: "^$", } require.Equal(t, expected, args) } @@ -50,7 +53,6 @@ func TestUnmarshalInvalid(t *testing.T) { metadata_refresh_interval = "1m" allow_concurrency = true max_offsets = 1000 - prune_interval_seconds = 30 topics_filter_regex = ".*" groups_filter_regex = ".*" ` @@ -78,8 +80,12 @@ func TestRiverConvert(t *testing.T) { AllowConcurrent: true, MaxOffsets: 1000, PruneIntervalSeconds: 30, + OffsetShowAll: true, + TopicWorkers: 100, TopicsFilter: ".*", GroupFilter: ".*", + TopicsExclude: "^$", + GroupExclude: "^$", } converted := orig.Convert() expected := kafka_exporter.Config{ @@ -90,8 +96,12 @@ func TestRiverConvert(t *testing.T) { AllowConcurrent: true, MaxOffsets: 1000, PruneIntervalSeconds: 30, + OffsetShowAll: true, + TopicWorkers: 100, TopicsFilter: ".*", GroupFilter: ".*", + TopicsExclude: "^$", + GroupExclude: "^$", } require.Equal(t, expected, *converted) diff --git a/converter/internal/staticconvert/internal/build/kafka_exporter.go b/converter/internal/staticconvert/internal/build/kafka_exporter.go index 16be4275ddce..aff74de63f7b 100644 --- a/converter/internal/staticconvert/internal/build/kafka_exporter.go +++ b/converter/internal/staticconvert/internal/build/kafka_exporter.go @@ -14,13 +14,16 @@ func (b *IntegrationsConfigBuilder) appendKafkaExporter(config *kafka_exporter.C func toKafkaExporter(config *kafka_exporter.Config) *kafka.Arguments { return &kafka.Arguments{ + Instance: config.Instance, KafkaURIs: config.KafkaURIs, UseSASL: config.UseSASL, UseSASLHandshake: config.UseSASLHandshake, SASLUsername: config.SASLUsername, SASLPassword: rivertypes.Secret(config.SASLPassword), SASLMechanism: config.SASLMechanism, + SASLDisablePAFXFast: config.SASLDisablePAFXFast, UseTLS: config.UseTLS, + TlsServerName: config.TlsServerName, CAFile: config.CAFile, CertFile: config.CertFile, KeyFile: config.KeyFile, @@ -30,10 +33,20 @@ func toKafkaExporter(config *kafka_exporter.Config) *kafka.Arguments { ZookeeperURIs: config.ZookeeperURIs, ClusterName: config.ClusterName, MetadataRefreshInterval: config.MetadataRefreshInterval, + ServiceName: config.ServiceName, + KerberosConfigPath: config.KerberosConfigPath, + Realm: config.Realm, + KeyTabPath: config.KeyTabPath, + KerberosAuthType: config.KerberosAuthType, + OffsetShowAll: config.OffsetShowAll, + TopicWorkers: config.TopicWorkers, AllowConcurrent: config.AllowConcurrent, + AllowAutoTopicCreation: config.AllowAutoTopicCreation, MaxOffsets: config.MaxOffsets, PruneIntervalSeconds: config.PruneIntervalSeconds, TopicsFilter: config.TopicsFilter, + TopicsExclude: config.TopicsExclude, GroupFilter: config.GroupFilter, + GroupExclude: config.GroupExclude, } } diff --git a/docs/sources/flow/reference/components/prometheus.exporter.kafka.md b/docs/sources/flow/reference/components/prometheus.exporter.kafka.md index 1de06212f557..c3c5adef7cfd 100644 --- a/docs/sources/flow/reference/components/prometheus.exporter.kafka.md +++ b/docs/sources/flow/reference/components/prometheus.exporter.kafka.md @@ -12,7 +12,7 @@ title: prometheus.exporter.kafka # prometheus.exporter.kafka The `prometheus.exporter.kafka` component embeds -[kafka_exporter](https://github.com/davidmparrott/kafka_exporter) for collecting metrics from a kafka server. +[kafka_exporter](https://github.com/grafana/kafka_exporter) for collecting metrics from a kafka server. ## Usage @@ -27,30 +27,42 @@ prometheus.exporter.kafka "LABEL" { You can use the following arguments to configure the exporter's behavior. Omitted fields take their default values. -| Name | Type | Description | Default | Required | -| --------------------------- | --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -------- | -| `kafka_uris` | `array(string)` | Address array (host:port) of Kafka server. | | yes | -| `instance` | `string` | The`instance`label for metrics, default is the hostname:port of the first kafka_uris. You must manually provide the instance value if there is more than one string in kafka_uris. | | no | -| `use_sasl` | `bool` | Connect using SASL/PLAIN. | | no | -| `use_sasl_handshake` | `bool` | Only set this to false if using a non-Kafka SASL proxy. | `false` | no | -| `sasl_username` | `string` | SASL user name. | | no | -| `sasl_password` | `string` | SASL user password. | | no | -| `sasl_mechanism` | `string` | The SASL SCRAM SHA algorithm sha256 or sha512 as mechanism. | | no | -| `use_tls` | `bool` | Connect using TLS. | | no | -| `ca_file` | `string` | The optional certificate authority file for TLS client authentication. | | no | -| `cert_file` | `string` | The optional certificate file for TLS client authentication. | | no | -| `key_file` | `string` | The optional key file for TLS client authentication. | | no | -| `insecure_skip_verify` | `bool` | If set to true, the server's certificate will not be checked for validity. This makes your HTTPS connections insecure. | | no | -| `kafka_version` | `string` | Kafka broker version. | `2.0.0` | no | -| `use_zookeeper_lag` | `bool` | If set to true, use a group from zookeeper. | | no | -| `zookeeper_uris` | `array(string)` | Address array (hosts) of zookeeper server. | | no | -| `kafka_cluster_name` | `string` | Kafka cluster name. | | no | -| `metadata_refresh_interval` | `duration` | Metadata refresh interval. | `1m` | no | -| `allow_concurrency` | `bool` | If set to true, all scrapes trigger Kafka operations. Otherwise, they will share results. WARNING: Disable this on large clusters. | `true` | no | -| `max_offsets` | `int` | The maximum number of offsets to store in the interpolation table for a partition. | `1000` | no | -| `prune_interval_seconds` | `int` | How frequently should the interpolation table be pruned, in seconds. | `30` | no | -| `topics_filter_regex` | `string` | Regex filter for topics to be monitored. | `.*` | no | -| `groups_filter_regex` | `string` | Regex filter for consumer groups to be monitored. | `.*` | no | +| Name | Type | Description | Default | Required | +| ----------------------------- | --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -------- | +| `kafka_uris` | `array(string)` | Address array (host:port) of Kafka server. | | yes | +| `instance` | `string` | The`instance`label for metrics, default is the hostname:port of the first kafka_uris. You must manually provide the instance value if there is more than one string in kafka_uris. | | no | +| `use_sasl` | `bool` | Connect using SASL/PLAIN. | | no | +| `use_sasl_handshake` | `bool` | Only set this to false if using a non-Kafka SASL proxy. | `true` | no | +| `sasl_username` | `string` | SASL user name. | | no | +| `sasl_password` | `string` | SASL user password. | | no | +| `sasl_mechanism` | `string` | The SASL SCRAM SHA algorithm sha256 or sha512 as mechanism. | | no | +| `sasl_disable_pafx_fast` | `bool` | Configure the Kerberos client to not use PA_FX_FAST. | | no | +| `use_tls` | `bool` | Connect using TLS. | | no | +| `tls_server_name` | `string` | Used to verify the hostname on the returned certificates unless tls.insecure-skip-tls-verify is given. If you don't provide the Kafka server name, the hostname is taken from the URL. | | no | +| `ca_file` | `string` | The optional certificate authority file for TLS client authentication. | | no | +| `cert_file` | `string` | The optional certificate file for TLS client authentication. | | no | +| `key_file` | `string` | The optional key file for TLS client authentication. | | no | +| `insecure_skip_verify` | `bool` | If set to true, the server's certificate will not be checked for validity. This makes your HTTPS connections insecure. | | no | +| `kafka_version` | `string` | Kafka broker version. | `2.0.0` | no | +| `use_zookeeper_lag` | `bool` | If set to true, use a group from zookeeper. | | no | +| `zookeeper_uris` | `array(string)` | Address array (hosts) of zookeeper server. | | no | +| `kafka_cluster_name` | `string` | Kafka cluster name. | | no | +| `metadata_refresh_interval` | `duration` | Metadata refresh interval. | `1m` | no | +| `gssapi_service_name` | `string` | Service name when using Kerberos Authorization | | no | +| `gssapi_kerberos_config_path` | `string` | Kerberos config path. | | no | +| `gssapi_realm` | `string` | Kerberos realm. | | no | +| `gssapi_key_tab_path` | `string` | Kerberos keytab file path. | | no | +| `gssapi_kerberos_auth_type` | `string` | Kerberos auth type. Either 'keytabAuth' or 'userAuth'. | | no | +| `offset_show_all` | `bool` | If true, the broker may auto-create topics that we requested which do not already exist. | `true` | no | +| `topic_workers` | `int` | Minimum number of topics to monitor. | `100` | no | +| `allow_concurrency` | `bool` | If set to true, all scrapes trigger Kafka operations. Otherwise, they will share results. WARNING: Disable this on large clusters. | `true` | no | +| `allow_auto_topic_creation` | `bool` | If true, the broker may auto-create topics that we requested which do not already exist. | | no | +| `max_offsets` | `int` | The maximum number of offsets to store in the interpolation table for a partition. | `1000` | no | +| `prune_interval_seconds` | `int` | Deprecated (no-op), use `metadata_refresh_interval` instead. | `30` | no | +| `topics_filter_regex` | `string` | Regex filter for topics to be monitored. | `.*` | no | +| `topics_exclude_regex` | `string` | Regex that determines which topics to exclude. | `^$` | no | +| `groups_filter_regex` | `string` | Regex filter for consumer groups to be monitored. | `.*` | no | +| `groups_exclude_regex` | `string` | Regex that determines which consumer groups to exclude. | `^$` | no | ## Exported fields diff --git a/docs/sources/static/configuration/integrations/kafka-exporter-config.md b/docs/sources/static/configuration/integrations/kafka-exporter-config.md index 14c8e5e99075..3e0bf8017ee7 100644 --- a/docs/sources/static/configuration/integrations/kafka-exporter-config.md +++ b/docs/sources/static/configuration/integrations/kafka-exporter-config.md @@ -11,7 +11,7 @@ title: kafka_exporter_config # kafka_exporter_config The `kafka_exporter_config` block configures the `kafka_exporter` -integration, which is an embedded version of [`kafka_exporter`](https://github.com/davidmparrott/kafka_exporter). +integration, which is an embedded version of [`kafka_exporter`](https://github.com/grafana/kafka_exporter). This allows for the collection of Kafka Lag metrics and exposing them as Prometheus metrics. We strongly recommend that you configure a separate user for the Agent, and give it only the strictly mandatory @@ -78,9 +78,15 @@ Full reference of options: # The SASL SCRAM SHA algorithm sha256 or sha512 as mechanism [sasl_mechanism: ] + # Configure the Kerberos client to not use PA_FX_FAST. + [sasl_disable_pafx_fast: ] + # Connect using TLS [use_tls: ] + # Used to verify the hostname on the returned certificates unless tls.insecure-skip-tls-verify is given. If you don't provide the Kafka server name, the hostname is taken from the URL. + [tls_server_name: ] + # The optional certificate authority file for TLS client authentication [ca_file: ] @@ -108,19 +114,49 @@ Full reference of options: # Metadata refresh interval [metadata_refresh_interval: | default = "1m"] + # Service name when using kerberos Auth. + [gssapi_service_name: ] + + # Kerberos config path. + [gssapi_kerberos_config_path: ] + + # Kerberos realm. + [gssapi_realm: ] + + # Kerberos keytab file path. + [gssapi_key_tab_path: ] + + # Kerberos auth type. Either 'keytabAuth' or 'userAuth'. + [gssapi_kerberos_auth_type: ] + + # Whether show the offset/lag for all consumer group, otherwise, only show connected consumer groups. + [offset_show_all: | default = true] + + # Minimum number of topics to monitor. + [topic_workers: | default = 100] + # If true, all scrapes will trigger kafka operations otherwise, they will share results. WARN: This should be disabled on large clusters [allow_concurrency: | default = true] + # If true, the broker may auto-create topics that we requested which do not already exist. + [allow_auto_topic_creation: ] + # Maximum number of offsets to store in the interpolation table for a partition [max_offsets: | default = 1000] - # How frequently should the interpolation table be pruned, in seconds + # Deprecated (no-op), use metadata_refresh_interval instead. [prune_interval_seconds: | default = 30] # Regex filter for topics to be monitored [topics_filter_regex: | default = ".*"] + # Regex that determines which topics to exclude. + [topics_exclude_regex: | default = "^$"] + # Regex filter for consumer groups to be monitored [groups_filter_regex: | default = ".*"] + # Regex that determines which consumer groups to exclude. + [groups_exclude_regex: | default = "^$"] + ``` diff --git a/go.mod b/go.mod index 152232302a98..d19219a3fb82 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/IBM/sarama v1.42.1 github.com/Lusitaniae/apache_exporter v0.11.1-0.20220518131644-f9522724dab4 github.com/Masterminds/sprig/v3 v3.2.3 - github.com/PuerkitoBio/rehttp v1.1.0 + github.com/PuerkitoBio/rehttp v1.3.0 github.com/alecthomas/kingpin/v2 v2.4.0 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 github.com/aws/aws-sdk-go v1.45.25 @@ -24,7 +24,6 @@ require ( github.com/cilium/ebpf v0.12.3 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/coreos/go-systemd/v22 v22.5.0 - github.com/davidmparrott/kafka_exporter/v2 v2.0.1 github.com/docker/docker v24.0.7+incompatible github.com/docker/go-connections v0.4.0 github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 @@ -213,16 +212,16 @@ require ( go.uber.org/goleak v1.2.1 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.18.0 + golang.org/x/crypto v0.20.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa - golang.org/x/net v0.20.0 + golang.org/x/net v0.21.0 golang.org/x/oauth2 v0.16.0 - golang.org/x/sys v0.16.0 + golang.org/x/sys v0.17.0 golang.org/x/text v0.14.0 - golang.org/x/time v0.3.0 - google.golang.org/api v0.149.0 + golang.org/x/time v0.5.0 + google.golang.org/api v0.152.0 google.golang.org/grpc v1.61.0 - google.golang.org/protobuf v1.32.0 + google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools v2.2.0+incompatible @@ -320,7 +319,7 @@ require ( github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/danieljoos/wincred v1.2.0 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dennwc/btrfs v0.0.0-20230312211831-a1f570bd01a1 // indirect github.com/dennwc/ioctl v1.0.0 // indirect github.com/dennwc/varint v1.0.0 // indirect @@ -427,13 +426,13 @@ require ( github.com/influxdata/telegraf v1.16.3 // indirect github.com/ionos-cloud/sdk-go/v6 v6.1.9 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.14.0 // indirect + github.com/jackc/pgconn v1.14.3 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.3.2 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgtype v1.14.0 // indirect - github.com/jackc/pgx/v4 v4.18.1 // indirect + github.com/jackc/pgx/v4 v4.18.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect @@ -560,8 +559,8 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect - github.com/xdg/stringprep v1.0.0 // indirect + github.com/xdg/scram v1.0.3 // indirect + github.com/xdg/stringprep v1.0.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -582,8 +581,8 @@ require ( go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/sync v0.5.0 // indirect - golang.org/x/term v0.16.0 // indirect - golang.org/x/tools v0.15.0 // indirect + golang.org/x/term v0.17.0 // indirect + golang.org/x/tools v0.15.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect gonum.org/v1/gonum v0.14.0 // indirect @@ -607,9 +606,11 @@ require github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab // require ( connectrpc.com/connect v1.14.0 + github.com/Shopify/sarama v1.38.1 github.com/githubexporter/github-exporter v0.0.0-20231025122338-656e7dc33fe7 github.com/grafana/agent-remote-config v0.0.2 github.com/grafana/jfr-parser/pprof v0.0.0-20240126072739-986e71dc0361 + github.com/grafana/kafka_exporter v0.0.0-20240409084445-5e3488ad9f9a github.com/natefinch/atomic v1.0.1 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.87.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.87.0 @@ -625,7 +626,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.20.0 // indirect - github.com/Shopify/sarama v1.38.1 // indirect github.com/Showmax/go-fqdn v1.0.0 // indirect github.com/Workiva/go-datastructures v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/service/amp v1.23.0 // indirect @@ -642,7 +642,7 @@ require ( github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/log v0.1.0 // indirect github.com/drone/envsubst v1.0.3 // indirect - github.com/go-jose/go-jose/v3 v3.0.1 // indirect + github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/grafana/jfr-parser v0.8.0 // indirect @@ -762,3 +762,7 @@ exclude ( ) replace github.com/github/smimesign => github.com/grafana/smimesign v0.2.1-0.20220408144937-2a5adf3481d3 + +// Replacing for an internal for with a bugfix for delta histograms, https://github.com/grafana/stackdriver_exporter/pull/1 +// Moving back to upstream is being tracked in an internal issue +replace github.com/prometheus-community/stackdriver_exporter => github.com/grafana/stackdriver_exporter v0.0.0-20240228143257-3a2c9acef5a2 diff --git a/go.sum b/go.sum index 2665dfe3c18e..8af81ef44748 100644 --- a/go.sum +++ b/go.sum @@ -224,15 +224,14 @@ github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCv github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/rehttp v1.1.0 h1:JFZ7OeK+hbJpTxhNB0NDZT47AuXqCU0Smxfjtph7/Rs= -github.com/PuerkitoBio/rehttp v1.1.0/go.mod h1:LUwKPoDbDIA2RL5wYZCNsQ90cx4OJ4AWBmq6KzWZL1s= +github.com/PuerkitoBio/rehttp v1.3.0 h1:w54Pb72MQn2eJrSdPsvGqXlAfiK1+NMTGDrOJJ4YvSU= +github.com/PuerkitoBio/rehttp v1.3.0/go.mod h1:LUwKPoDbDIA2RL5wYZCNsQ90cx4OJ4AWBmq6KzWZL1s= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/SAP/go-hdb v0.12.0/go.mod h1:etBT+FAi1t5k3K3tf5vQTnosgYmhDkRi8jEnQqCnxF0= github.com/SermoDigital/jose v0.0.0-20180104203859-803625baeddc/go.mod h1:ARgCUhI1MHQH+ONky/PAtmVHQrP5JlGY0F3poXOp/fA= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.27.1/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II= -github.com/Shopify/sarama v1.27.2/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II= github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= @@ -536,8 +535,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davidmparrott/kafka_exporter/v2 v2.0.1 h1:nGn+MKV8z08NK4xqcYSa3fBCs/VPVesT/5kboFWJaiE= -github.com/davidmparrott/kafka_exporter/v2 v2.0.1/go.mod h1:n3ho8mZ5tZcmr8NAu/SjQHY61CDTqXtrACcEYwLXv4Y= github.com/denisenkom/go-mssqldb v0.0.0-20180620032804-94c9c97e8c9f/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/dennwc/btrfs v0.0.0-20230312211831-a1f570bd01a1 h1:ue4Es4Xzz255hWQ7NAWzZxuXG+YOV7URzzusLLSe0zU= @@ -711,8 +708,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= -github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= -github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= +github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= @@ -1050,6 +1047,8 @@ github.com/grafana/jfr-parser v0.8.0 h1:/uo2wZNXrxw7tKLFwP2omJ3EQGMkD9wzhPsRogVo github.com/grafana/jfr-parser v0.8.0/go.mod h1:M5u1ux34Qo47ZBWksbMYVk40s7dvU3WMVYpxweEu4R0= github.com/grafana/jfr-parser/pprof v0.0.0-20240126072739-986e71dc0361 h1:TtNajaiSRfM2Mz8N7ouFQDFlviXbIEk9Hts0yoZnhGM= github.com/grafana/jfr-parser/pprof v0.0.0-20240126072739-986e71dc0361/go.mod h1:P5406BrWxjahTzVF6aCSumNI1KPlZJc0zO0v+zKZ4gc= +github.com/grafana/kafka_exporter v0.0.0-20240409084445-5e3488ad9f9a h1:jqM4NNdx8LSquKo8bPx+XWn91S2b+sgNvEcFfSJQtHY= +github.com/grafana/kafka_exporter v0.0.0-20240409084445-5e3488ad9f9a/go.mod h1:ZXGGyeTUMenf/H1CDBK9lv3azjswfa0nVzLoQAYmnDc= github.com/grafana/loki v1.6.2-0.20231004111112-07cbef92268a h1:lvSHlNONeo/H+aWRk86QEfBpRDCEX1yoqpsCK0Tys+g= github.com/grafana/loki v1.6.2-0.20231004111112-07cbef92268a/go.mod h1:a5c5ZTC6FNufKkvF8NeDAb2nCWJpgkVDrejmV+O9hac= github.com/grafana/loki/pkg/push v0.0.0-20231212100434-384e5c2dc872 h1:6kPX7bngjBgUlHqADwZ6249UtzMaoQW5n0H8bOtnYeM= @@ -1080,6 +1079,8 @@ github.com/grafana/smimesign v0.2.1-0.20220408144937-2a5adf3481d3 h1:UPkAxuhlAcR github.com/grafana/smimesign v0.2.1-0.20220408144937-2a5adf3481d3/go.mod h1:iZiiwNT4HbtGRVqCQu7uJPEZCuEE5sfSSttcnePkDl4= github.com/grafana/snowflake-prometheus-exporter v0.0.0-20221213150626-862cad8e9538 h1:tkT0yha3JzB5S5VNjfY4lT0cJAe20pU8XGt3Nuq73rM= github.com/grafana/snowflake-prometheus-exporter v0.0.0-20221213150626-862cad8e9538/go.mod h1:VxVydRyq8f6w1qmX/5MSYIdSbgujre8rdFRLgU6u/RI= +github.com/grafana/stackdriver_exporter v0.0.0-20240228143257-3a2c9acef5a2 h1:xBGGPnQyQNK0Apz269BZoKTnFxKKxYhhXzI++N2phE0= +github.com/grafana/stackdriver_exporter v0.0.0-20240228143257-3a2c9acef5a2/go.mod h1:Ce7MjYSAUzZZeFb5jBNqSUUZ45w5IMdnNEKfz3jJRos= github.com/grafana/tail v0.0.0-20230510142333-77b18831edf0 h1:bjh0PVYSVVFxzINqPFYJmAmJNrWPgnVjuSdYJGHmtFU= github.com/grafana/tail v0.0.0-20230510142333-77b18831edf0/go.mod h1:7t5XR+2IA8P2qggOAHTj/GCZfoLBle3OvNSYh1VkRBU= github.com/grafana/vmware_exporter v0.0.4-beta h1:Tb8Edm/wDYh0Lvhm38HLNTlkflUrlPGB+jD+/hW4xHI= @@ -1313,8 +1314,8 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.14.0 h1:vrbA9Ud87g6JdFWkHTJXppVce58qPIdP7N8y0Ml/A7Q= -github.com/jackc/pgconn v1.14.0/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -1330,8 +1331,8 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.2 h1:7eY55bdBeCz1F2fTzSz69QC+pG46jYq9/jtSPiJ5nn0= -github.com/jackc/pgproto3/v2 v2.3.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= @@ -1346,12 +1347,11 @@ github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08 github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.18.1 h1:YP7G1KABtKpB5IHrO9vYwSrCOhs7p3uqhvhhQBptya0= -github.com/jackc/pgx/v4 v4.18.1/go.mod h1:FydWkUyadDmdNH/mHnGob881GawxeEm7TcMCzkb+qQE= +github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU= +github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jaegertracing/jaeger v1.50.0 h1:qsOcPeB3nAc3h8tx+gnZ3JODAZfqbYmQr45jPEwBd2w= github.com/jaegertracing/jaeger v1.50.0/go.mod h1:MVGvxf4+Pcn31gz9RnLo0097w3khKFwJIprIZHOt89s= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= @@ -1708,8 +1708,8 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector v0.87.0 h1:ArBXfq0KQ89DV9th/MU/snH205Uh6jFCnIiwd/wKp+s= github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector v0.87.0/go.mod h1:hN1ufLEIhE10FeG7L/yKMXMr9B0hcyrvqiZ3vR/qq/c= @@ -1914,8 +1914,6 @@ github.com/prometheus-community/go-runit v0.1.0 h1:uTWEj/Fn2RoLdfg/etSqwzgYNOYPr github.com/prometheus-community/go-runit v0.1.0/go.mod h1:AvJ9Jo3gAFu2lbM4+qfjdpq30FfiLDJZKbQ015u08IQ= github.com/prometheus-community/prom-label-proxy v0.6.0 h1:vRY29tUex8qI2MEimovTzJdieEwiSko+f7GuPCLjFkI= github.com/prometheus-community/prom-label-proxy v0.6.0/go.mod h1:XyAyskjjhqEx0qnbGUVeAkYSz3Wm9gStT7/wXFxD8n0= -github.com/prometheus-community/stackdriver_exporter v0.13.0 h1:4h7v28foRJ4/RuchNZCYsoDp+CkF4Mp9nebtPzgil3g= -github.com/prometheus-community/stackdriver_exporter v0.13.0/go.mod h1:ZFO015Mexz1xNHSvFjZFiIspYx6qhDg9Kre4LPUjO9s= github.com/prometheus-community/windows_exporter v0.24.1-0.20231127180936-5a872a227c2f h1:nEIgTweLXQk5ihBuKa84+l9WG/xrqA/1qX0jgbJ69OQ= github.com/prometheus-community/windows_exporter v0.24.1-0.20231127180936-5a872a227c2f/go.mod h1:wxKb/CTmvhDaZz4BokGt3btOn4aCrr5ruDQT7KxmJok= github.com/prometheus-operator/prometheus-operator v0.66.0 h1:Jj4mbGAkfBbTih6ait03f2vUjEHB7Kb4gnlAmWu7AJ0= @@ -1938,7 +1936,6 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.6.1-0.20200604110148-03575cad4e55/go.mod h1:25h+Uz1WvXDBZYwqGX8PAb71RBkcjxEVV/R5wGnsq4I= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= @@ -1962,7 +1959,6 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= @@ -1991,7 +1987,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= @@ -2263,10 +2258,12 @@ github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= +github.com/xdg/scram v1.0.3 h1:nTadYh2Fs4BK2xdldEa2g5bbaZp0/+1nJMMPtPxS/to= +github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -2498,8 +2495,9 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= golang.org/x/crypto/x509roots/fallback v0.0.0-20240208163226-62c9f1799c91 h1:Lyizcy9jX02jYR0ceBkL6S+jRys8Uepf7wt1vrz6Ras= golang.org/x/crypto/x509roots/fallback v0.0.0-20240208163226-62c9f1799c91/go.mod h1:kNa9WdvYnzFwC79zRpLRMJbdEFlhyM5RPFBBZp/wWH8= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2622,8 +2620,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20170807180024-9a379c6b3e95/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2734,7 +2732,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2785,8 +2782,8 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2797,8 +2794,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2824,8 +2821,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2955,8 +2952,8 @@ google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00 google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= -google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= +google.golang.org/api v0.152.0 h1:t0r1vPnfMc260S2Ci+en7kfCZaLOPs5KI0sVV/6jZrY= +google.golang.org/api v0.152.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -3090,8 +3087,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/integration-tests/docker-compose.yaml b/integration-tests/docker-compose.yaml index eb5650f75f7f..c01666e15390 100644 --- a/integration-tests/docker-compose.yaml +++ b/integration-tests/docker-compose.yaml @@ -10,6 +10,33 @@ services: - -config.file=/etc/mimir-config/mimir.yaml ports: - "9009:9009" + + zookeeper: + image: confluentinc/cp-zookeeper:latest + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + kafka: + image: confluentinc/cp-kafka:latest + depends_on: + - zookeeper + ports: + - "9094:9094" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:9094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + + kafka-gen: + depends_on: + - kafka + build: + dockerfile: ./internal/cmd/integration-tests/configs/kafka/Dockerfile + context: .. loki: image: grafana/loki:latest diff --git a/integration-tests/tests/unix/unix_metrics_test.go b/integration-tests/tests/unix/unix_metrics_test.go index 7456ac29706a..e85e1fc0891f 100644 --- a/integration-tests/tests/unix/unix_metrics_test.go +++ b/integration-tests/tests/unix/unix_metrics_test.go @@ -159,6 +159,5 @@ func TestUnixMetrics(t *testing.T) { "node_vmstat_pswpin", "node_vmstat_pswpout", } - // TODO(marctc): Report list of failed metrics instead of one by one. common.MimirMetricsTest(t, unixMetrics, []string{}, "unix_metrics") } diff --git a/integration-tests/utils.go b/integration-tests/utils.go index 28ad81b95e9a..f69683d15503 100644 --- a/integration-tests/utils.go +++ b/integration-tests/utils.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "sync" + "time" ) const ( @@ -39,6 +40,8 @@ func buildAgent() { func setupEnvironment() { executeCommand("docker-compose", []string{"up", "-d"}, "Setting up environment with Docker Compose") + fmt.Println("Sleep for 30 seconds to ensure that the env has time to initialize...") + time.Sleep(30 * time.Second) } func runSingleTest(testDir string, port int) { diff --git a/internal/cmd/integration-tests/configs/kafka/Dockerfile b/internal/cmd/integration-tests/configs/kafka/Dockerfile new file mode 100644 index 000000000000..074f2cf20a87 --- /dev/null +++ b/internal/cmd/integration-tests/configs/kafka/Dockerfile @@ -0,0 +1,9 @@ +FROM golang:1.22.1 as build +WORKDIR /app/ +COPY go.mod go.sum ./ +RUN go mod download +COPY ./internal/cmd/integration-tests/configs/kafka/ ./ +RUN CGO_ENABLED=0 go build -o main main.go +FROM alpine:3.18 +COPY --from=build /app/main /app/main +CMD ["/app/main"] diff --git a/internal/cmd/integration-tests/configs/kafka/main.go b/internal/cmd/integration-tests/configs/kafka/main.go new file mode 100644 index 000000000000..b3cfbd1edaec --- /dev/null +++ b/internal/cmd/integration-tests/configs/kafka/main.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "log" + "time" + + "github.com/Shopify/sarama" +) + +const ( + topicName = "test_topic" + brokerAddress = "kafka:9092" +) + +// This app sends and consumes messages via a Kafka topic. + +func main() { + go produceMessages() + consumeMessages() +} + +func produceMessages() { + config := sarama.NewConfig() + config.Producer.Return.Successes = true + + var producer sarama.SyncProducer + var err error + + for { + producer, err = sarama.NewSyncProducer([]string{brokerAddress}, config) + if err == nil { + break + } + log.Printf("Failed to start Sarama producer: %v, retrying in 5 seconds...", err) + time.Sleep(5 * time.Second) + } + + defer producer.Close() + log.Println("Sarama producer started successfully") + + for { + message := &sarama.ProducerMessage{ + Topic: topicName, + Value: sarama.StringEncoder("hello"), + } + + partition, offset, err := producer.SendMessage(message) + if err != nil { + log.Printf("Failed to send message: %v", err) + } else { + log.Printf("Message is stored in topic(%s)/partition(%d)/offset(%d)\n", topicName, partition, offset) + } + time.Sleep(time.Second) + } +} + +func consumeMessages() { + config := sarama.NewConfig() + config.Consumer.Return.Errors = true + config.Consumer.Offsets.Initial = sarama.OffsetOldest + + var consumerGroup sarama.ConsumerGroup + var err error + + for { + consumerGroup, err = sarama.NewConsumerGroup([]string{brokerAddress}, "test_consumer_group", config) + if err == nil { + break + } + log.Printf("Failed to start Sarama consumer group: %v, retrying in 5 seconds...", err) + time.Sleep(5 * time.Second) + } + + defer func() { + if err := consumerGroup.Close(); err != nil { + log.Fatalf("Failed to close consumer group: %v", err) + } + }() + log.Println("Sarama consumer group started successfully") + + ctx := context.Background() + consumer := Consumer{} + + go func() { + for err := range consumerGroup.Errors() { + log.Printf("Consumer group error: %v", err) + } + }() + + for { + topics := []string{topicName} + if err := consumerGroup.Consume(ctx, topics, &consumer); err != nil { + log.Fatalf("Error from consumer: %v", err) + } + } +} + +type Consumer struct{} + +func (consumer *Consumer) Setup(_ sarama.ConsumerGroupSession) error { return nil } +func (consumer *Consumer) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } +func (consumer *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for message := range claim.Messages() { + log.Printf("Message claimed: value = %s, timestamp = %v, topic = %s\n", string(message.Value), message.Timestamp, message.Topic) + session.MarkMessage(message, "") + } + return nil +} diff --git a/internal/cmd/integration-tests/tests/kafka/config.river b/internal/cmd/integration-tests/tests/kafka/config.river new file mode 100644 index 000000000000..7c5c72c605ab --- /dev/null +++ b/internal/cmd/integration-tests/tests/kafka/config.river @@ -0,0 +1,25 @@ +prometheus.exporter.kafka "kafka_metrics" { + kafka_uris = ["127.0.0.1:9094"] +} + +prometheus.scrape "kafka_metrics" { + targets = prometheus.exporter.kafka.kafka_metrics.targets + forward_to = [prometheus.remote_write.kafka_metrics.receiver] + scrape_interval = "1s" + scrape_timeout = "500ms" +} + +prometheus.remote_write "kafka_metrics" { + endpoint { + url = "http://localhost:9009/api/v1/push" + metadata_config { + send_interval = "1s" + } + queue_config { + max_samples_per_send = 100 + } + } + external_labels = { + test_name = "kafka_metrics", + } +} \ No newline at end of file diff --git a/internal/cmd/integration-tests/tests/kafka/kafka_metrics_test.go b/internal/cmd/integration-tests/tests/kafka/kafka_metrics_test.go new file mode 100644 index 000000000000..2a51a59b56c0 --- /dev/null +++ b/internal/cmd/integration-tests/tests/kafka/kafka_metrics_test.go @@ -0,0 +1,34 @@ +//go:build !windows + +package main + +import ( + "testing" + + "github.com/grafana/agent/integration-tests/common" +) + +func TestKafkaMetrics(t *testing.T) { + var kafkaMetrics = []string{ + "kafka_topic_partition_replicas", + "kafka_topic_partition_leader", + "kafka_exporter_build_info", + "kafka_consumergroup_current_offset", + "kafka_consumergroup_current_offset_sum", + // "kafka_consumer_lag_extrapolation", this one is not generated because it uses "kafka_consumer_lag_interpolation" instead + "kafka_consumer_lag_interpolation", + "kafka_broker_info", + "kafka_brokers", + "kafka_consumergroup_uncommitted_offsets_sum", + "kafka_topic_partition_current_offset", + "kafka_consumergroup_uncommitted_offsets", + "kafka_topic_partition_in_sync_replica", + "kafka_topic_partition_under_replicated_partition", + "kafka_topic_partition_leader_is_preferred", + "kafka_consumergroup_members", + "kafka_consumer_lag_millis", + "kafka_topic_partitions", + "kafka_topic_partition_oldest_offset", + } + common.MimirMetricsTest(t, kafkaMetrics, []string{}, "kafka_metrics") +} diff --git a/pkg/integrations/gcp_exporter/gcp_exporter.go b/pkg/integrations/gcp_exporter/gcp_exporter.go index 302fa172948c..d157410953ff 100644 --- a/pkg/integrations/gcp_exporter/gcp_exporter.go +++ b/pkg/integrations/gcp_exporter/gcp_exporter.go @@ -14,6 +14,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/dskit/multierror" "github.com/prometheus-community/stackdriver_exporter/collectors" + "github.com/prometheus-community/stackdriver_exporter/delta" "github.com/prometheus-community/stackdriver_exporter/utils" "github.com/prometheus/client_golang/prometheus" "golang.org/x/oauth2/google" @@ -105,8 +106,8 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error) AggregateDeltas: true, }, l, - collectors.NewInMemoryDeltaCounterStore(l, 30*time.Minute), - collectors.NewInMemoryDeltaDistributionStore(l, 30*time.Minute), + delta.NewInMemoryCounterStore(l, 30*time.Minute), + delta.NewInMemoryHistogramStore(l, 30*time.Minute), ) if err != nil { return nil, fmt.Errorf("failed to create monitoring collector: %w", err) diff --git a/pkg/integrations/kafka_exporter/kafka_exporter.go b/pkg/integrations/kafka_exporter/kafka_exporter.go index 5d50fe1452ea..52d3c5129535 100644 --- a/pkg/integrations/kafka_exporter/kafka_exporter.go +++ b/pkg/integrations/kafka_exporter/kafka_exporter.go @@ -6,11 +6,12 @@ import ( config_util "github.com/prometheus/common/config" "github.com/IBM/sarama" - kafka_exporter "github.com/davidmparrott/kafka_exporter/v2/exporter" "github.com/go-kit/log" + "github.com/grafana/agent/pkg/flow/logging/level" "github.com/grafana/agent/pkg/integrations" integrations_v2 "github.com/grafana/agent/pkg/integrations/v2" "github.com/grafana/agent/pkg/integrations/v2/metricsutils" + kafka_exporter "github.com/grafana/kafka_exporter/exporter" ) // DefaultConfig holds the default settings for the kafka_lag_exporter @@ -22,8 +23,12 @@ var DefaultConfig = Config{ AllowConcurrent: true, MaxOffsets: 1000, PruneIntervalSeconds: 30, + OffsetShowAll: true, + TopicWorkers: 100, TopicsFilter: ".*", + TopicsExclude: "^$", GroupFilter: ".*", + GroupExclude: "^$", } // Config controls kafka_exporter @@ -49,9 +54,15 @@ type Config struct { // The SASL SCRAM SHA algorithm sha256 or sha512 as mechanism SASLMechanism string `yaml:"sasl_mechanism,omitempty"` + // Configure the Kerberos client to not use PA_FX_FAST. + SASLDisablePAFXFast bool `yaml:"sasl_disable_pafx_fast,omitempty"` + // Connect using TLS UseTLS bool `yaml:"use_tls,omitempty"` + // Used to verify the hostname on the returned certificates unless tls.insecure-skip-tls-verify is given. The kafka server's name should be given. + TlsServerName string `yaml:"tls_server_name,omitempty"` + // The optional certificate authority file for TLS client authentication CAFile string `yaml:"ca_file,omitempty"` @@ -79,20 +90,50 @@ type Config struct { // Metadata refresh interval MetadataRefreshInterval string `yaml:"metadata_refresh_interval,omitempty"` + // Service name when using kerberos Auth. + ServiceName string `yaml:"gssapi_service_name,omitempty"` + + // Kerberos config path. + KerberosConfigPath string `yaml:"gssapi_kerberos_config_path,omitempty"` + + // Kerberos realm. + Realm string `yaml:"gssapi_realm,omitempty"` + + // Kerberos keytab file path. + KeyTabPath string `yaml:"gssapi_key_tab_path,omitempty"` + + // Kerberos auth type. Either 'keytabAuth' or 'userAuth'. + KerberosAuthType string `yaml:"gssapi_kerberos_auth_type,omitempty"` + + // Whether show the offset/lag for all consumer group, otherwise, only show connected consumer groups. + OffsetShowAll bool `yaml:"offset_show_all,omitempty"` + + // Number of topic workers. + TopicWorkers int `yaml:"topic_workers,omitempty"` + // If true, all scrapes will trigger kafka operations otherwise, they will share results. WARN: This should be disabled on large clusters AllowConcurrent bool `yaml:"allow_concurrency,omitempty"` + // If true, the broker may auto-create topics that we requested which do not already exist. + AllowAutoTopicCreation bool `yaml:"allow_auto_topic_creation,omitempty"` + // Maximum number of offsets to store in the interpolation table for a partition MaxOffsets int `yaml:"max_offsets,omitempty"` - // How frequently should the interpolation table be pruned, in seconds + // No-op (deprecated). Use metadata_refresh_interval instead. PruneIntervalSeconds int `yaml:"prune_interval_seconds,omitempty"` // Regex filter for topics to be monitored TopicsFilter string `yaml:"topics_filter_regex,omitempty"` + // Regex that determines which topics to exclude. + TopicsExclude string `yaml:"topics_exclude_regex,omitempty"` + // Regex filter for consumer groups to be monitored GroupFilter string `yaml:"groups_filter_regex,omitempty"` + + // Regex that determines which consumer groups to exclude. + GroupExclude string `yaml:"groups_exclude_regex,omitempty"` } // UnmarshalYAML implements yaml.Unmarshaler for Config @@ -147,6 +188,11 @@ func New(logger log.Logger, c *Config) (integrations.Integration, error) { return nil, fmt.Errorf("zookeeper lag is enabled but no zookeeper uri was provided") } + // 30 is the default value + if c.PruneIntervalSeconds != 30 { + level.Warn(logger).Log("msg", "prune_interval_seconds is not used anymore, use metadata_refresh_interval instead") + } + options := kafka_exporter.Options{ Uri: c.KafkaURIs, UseSASL: c.UseSASL, @@ -154,7 +200,9 @@ func New(logger log.Logger, c *Config) (integrations.Integration, error) { SaslUsername: c.SASLUsername, SaslPassword: string(c.SASLPassword), SaslMechanism: c.SASLMechanism, + SaslDisablePAFXFast: c.SASLDisablePAFXFast, UseTLS: c.UseTLS, + TlsServerName: c.TlsServerName, TlsCAFile: c.CAFile, TlsCertFile: c.CertFile, TlsKeyFile: c.KeyFile, @@ -164,12 +212,19 @@ func New(logger log.Logger, c *Config) (integrations.Integration, error) { UriZookeeper: c.ZookeeperURIs, Labels: c.ClusterName, MetadataRefreshInterval: c.MetadataRefreshInterval, + ServiceName: c.ServiceName, + KerberosConfigPath: c.KerberosConfigPath, + Realm: c.Realm, + KeyTabPath: c.KeyTabPath, + KerberosAuthType: c.KerberosAuthType, + OffsetShowAll: c.OffsetShowAll, + TopicWorkers: c.TopicWorkers, AllowConcurrent: c.AllowConcurrent, + AllowAutoTopicCreation: c.AllowAutoTopicCreation, MaxOffsets: c.MaxOffsets, - PruneIntervalSeconds: c.PruneIntervalSeconds, } - newExporter, err := kafka_exporter.New(logger, options, c.TopicsFilter, c.GroupFilter) + newExporter, err := kafka_exporter.New(logger, options, c.TopicsFilter, c.TopicsExclude, c.GroupFilter, c.GroupExclude) if err != nil { return nil, fmt.Errorf("could not instantiate kafka lag exporter: %w", err) }