diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6b4a999e0ec..cc8fb52396a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -453,6 +453,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Cloud Foundry metadata is cached to disk. {pull}20775[20775] - Add option to select the type of index template to load: legacy, component, index. {pull}21212[21212] - Release `add_cloudfoundry_metadata` as GA. {pull}21525[21525] +- Add support for OpenStack SSL metadata APIs in `add_cloud_metadata`. {pull}21590[21590] *Auditbeat* diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index de1a8063667..5376d563fca 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -26,6 +26,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/processors" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" @@ -53,6 +54,7 @@ type addCloudMetadata struct { type initData struct { fetchers []metadataFetcher timeout time.Duration + tlsConfig *tlscommon.TLSConfig overwrite bool } @@ -63,14 +65,24 @@ func New(c *common.Config) (processors.Processor, error) { return nil, errors.Wrap(err, "failed to unpack add_cloud_metadata config") } + tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS) + if err != nil { + return nil, errors.Wrap(err, "TLS configuration load") + } + initProviders := selectProviders(config.Providers, cloudMetaProviders) fetchers, err := setupFetchers(initProviders, c) if err != nil { return nil, err } p := &addCloudMetadata{ - initData: &initData{fetchers, config.Timeout, config.Overwrite}, - logger: logp.NewLogger("add_cloud_metadata"), + initData: &initData{ + fetchers: fetchers, + timeout: config.Timeout, + tlsConfig: tlsConfig, + overwrite: config.Overwrite, + }, + logger: logp.NewLogger("add_cloud_metadata"), } go p.init() diff --git a/libbeat/processors/add_cloud_metadata/config.go b/libbeat/processors/add_cloud_metadata/config.go index 08f4a241483..93a31137592 100644 --- a/libbeat/processors/add_cloud_metadata/config.go +++ b/libbeat/processors/add_cloud_metadata/config.go @@ -20,12 +20,15 @@ package add_cloud_metadata import ( "fmt" "time" + + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) type config struct { - Timeout time.Duration `config:"timeout"` // Amount of time to wait for responses from the metadata services. - Overwrite bool `config:"overwrite"` // Overwrite if cloud.* fields already exist. - Providers providerList `config:"providers"` // List of providers to probe + Timeout time.Duration `config:"timeout"` // Amount of time to wait for responses from the metadata services. + TLS *tlscommon.Config `config:"ssl"` // TLS configuration + Overwrite bool `config:"overwrite"` // Overwrite if cloud.* fields already exist. + Providers providerList `config:"providers"` // List of providers to probe } type providerList []string diff --git a/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc b/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc index 41c0dd6d9f3..44497f31539 100644 --- a/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc +++ b/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc @@ -52,12 +52,16 @@ List of names the `providers` setting supports: - "aws", or "ec2" for Amazon Web Services (enabled by default). - "gcp" for Google Copmute Enging (enabled by default). - "openstack", or "nova" for Openstack Nova (enabled by default). +- "openstack-ssl", or "nova-ssl" for Openstack Nova when SSL metadata APIs are enabled (enabled by default). - "tencent", or "qcloud" for Tencent Cloud (disabled by default). The third optional configuration setting is `overwrite`. When `overwrite` is `true`, `add_cloud_metadata` overwrites existing `cloud.*` fields (`false` by default). +The `add_cloud_metadata` processor supports SSL options to configure the http +client used to query cloud metadata. See <> for more information. + The metadata that is added to events varies by hosting provider. Below are examples for each of the supported providers. diff --git a/libbeat/processors/add_cloud_metadata/http_fetcher.go b/libbeat/processors/add_cloud_metadata/http_fetcher.go index e9b18478661..e337edd5be9 100644 --- a/libbeat/processors/add_cloud_metadata/http_fetcher.go +++ b/libbeat/processors/add_cloud_metadata/http_fetcher.go @@ -27,6 +27,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) type httpMetadataFetcher struct { @@ -127,9 +128,15 @@ func (f *httpMetadataFetcher) fetchRaw( // getMetadataURLs loads config and generates the metadata URLs. func getMetadataURLs(c *common.Config, defaultHost string, metadataURIs []string) ([]string, error) { + return getMetadataURLsWithScheme(c, "http", defaultHost, metadataURIs) +} + +// getMetadataURLsWithScheme loads config and generates the metadata URLs. +func getMetadataURLsWithScheme(c *common.Config, scheme string, defaultHost string, metadataURIs []string) ([]string, error) { var urls []string config := struct { - MetadataHostAndPort string `config:"host"` // Specifies the host and port of the metadata service (for testing purposes only). + MetadataHostAndPort string `config:"host"` // Specifies the host and port of the metadata service (for testing purposes only). + TLSConfig *tlscommon.Config `config:"ssl"` }{ MetadataHostAndPort: defaultHost, } @@ -138,7 +145,7 @@ func getMetadataURLs(c *common.Config, defaultHost string, metadataURIs []string return urls, errors.Wrap(err, "failed to unpack add_cloud_metadata config") } for _, uri := range metadataURIs { - urls = append(urls, "http://"+config.MetadataHostAndPort+uri) + urls = append(urls, scheme+"://"+config.MetadataHostAndPort+uri) } return urls, nil } diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go index 17bc5abf689..7c9a997e0e2 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go @@ -32,16 +32,24 @@ const ( // OpenStack Nova Metadata Service // Document https://docs.openstack.org/nova/latest/user/metadata-service.html var openstackNovaMetadataFetcher = provider{ - Name: "openstack-nova", + Name: "openstack-nova", + Local: true, + Create: buildOpenstackNovaCreate("http"), +} - Local: true, +var openstackNovaSSLMetadataFetcher = provider{ + Name: "openstack-nova-ssl", + Local: true, + Create: buildOpenstackNovaCreate("https"), +} - Create: func(provider string, c *common.Config) (metadataFetcher, error) { +func buildOpenstackNovaCreate(scheme string) func(provider string, c *common.Config) (metadataFetcher, error) { + return func(provider string, c *common.Config) (metadataFetcher, error) { osSchema := func(m map[string]interface{}) common.MapStr { return common.MapStr(m) } - urls, err := getMetadataURLs(c, metadataHost, []string{ + urls, err := getMetadataURLsWithScheme(c, scheme, metadataHost, []string{ osMetadataInstanceIDURI, osMetadataInstanceTypeURI, osMetadataHostnameURI, @@ -71,5 +79,5 @@ var openstackNovaMetadataFetcher = provider{ } fetcher := &httpMetadataFetcher{"openstack", nil, responseHandlers, osSchema} return fetcher, nil - }, + } } diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go index d5c38a8437b..0a63c026cde 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go @@ -29,8 +29,8 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -func initOpenstackNovaTestServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +func openstackNovaMetadataHandler() http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.RequestURI == osMetadataInstanceIDURI { w.Write([]byte("i-0000ffac")) return @@ -49,13 +49,13 @@ func initOpenstackNovaTestServer() *httptest.Server { } http.Error(w, "not found", http.StatusNotFound) - })) + }) } func TestRetrieveOpenstackNovaMetadata(t *testing.T) { logp.TestingSetup() - server := initOpenstackNovaTestServer() + server := httptest.NewServer(openstackNovaMetadataHandler()) defer server.Close() config, err := common.NewConfigFrom(map[string]interface{}{ @@ -66,6 +66,28 @@ func TestRetrieveOpenstackNovaMetadata(t *testing.T) { t.Fatal(err) } + assertOpenstackNova(t, config) +} + +func TestRetrieveOpenstackNovaMetadataWithHTTPS(t *testing.T) { + logp.TestingSetup() + + server := httptest.NewTLSServer(openstackNovaMetadataHandler()) + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "host": server.Listener.Addr().String(), + "ssl.verification_mode": "none", + }) + + if err != nil { + t.Fatal(err) + } + + assertOpenstackNova(t, config) +} + +func assertOpenstackNova(t *testing.T, config *common.Config) { p, err := New(config) if err != nil { t.Fatal(err) diff --git a/libbeat/processors/add_cloud_metadata/providers.go b/libbeat/processors/add_cloud_metadata/providers.go index f93e8cf78f4..ea03b64258a 100644 --- a/libbeat/processors/add_cloud_metadata/providers.go +++ b/libbeat/processors/add_cloud_metadata/providers.go @@ -51,17 +51,19 @@ type result struct { } var cloudMetaProviders = map[string]provider{ - "alibaba": alibabaCloudMetadataFetcher, - "ecs": alibabaCloudMetadataFetcher, - "azure": azureVMMetadataFetcher, - "digitalocean": doMetadataFetcher, - "aws": ec2MetadataFetcher, - "ec2": ec2MetadataFetcher, - "gcp": gceMetadataFetcher, - "openstack": openstackNovaMetadataFetcher, - "nova": openstackNovaMetadataFetcher, - "qcloud": qcloudMetadataFetcher, - "tencent": qcloudMetadataFetcher, + "alibaba": alibabaCloudMetadataFetcher, + "ecs": alibabaCloudMetadataFetcher, + "azure": azureVMMetadataFetcher, + "digitalocean": doMetadataFetcher, + "aws": ec2MetadataFetcher, + "ec2": ec2MetadataFetcher, + "gcp": gceMetadataFetcher, + "openstack": openstackNovaMetadataFetcher, + "nova": openstackNovaMetadataFetcher, + "openstack-ssl": openstackNovaSSLMetadataFetcher, + "nova-ssl": openstackNovaSSLMetadataFetcher, + "qcloud": qcloudMetadataFetcher, + "tencent": qcloudMetadataFetcher, } func selectProviders(configList providerList, providers map[string]provider) map[string]provider { @@ -138,6 +140,7 @@ func (p *addCloudMetadata) fetchMetadata() *result { Timeout: p.initData.timeout, KeepAlive: 0, }).DialContext, + TLSClientConfig: p.initData.tlsConfig.ToConfig(), }, }