Merge branch 'main' into fix_altdomain_dcname_overlap

.changelog/15654.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+cli: Adds new command - `consul services export` - for exporting a service to a peer or partition
+```

.changelog/16552.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+raft: Remove expensive reflection from raft/mesh hot path
+```

.changelog/16845.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+systemd: set service type to notify.
+```

.changelog/17038.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+agent: add new metrics to track cpu disk and memory usage for server hosts (defaults to: enabled)
+```

.changelog/17055.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+gateways: Fix an bug where targeting a virtual service defined by a service-resolver was broken for HTTPRoutes.
+```

.changelog/17231.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+acl: Fix an issue where the anonymous token was synthesized in non-primary datacenters which could cause permission errors when federating clusters with ACL replication enabled.
+```

.changelog/17415.txt

@@ -0,0 +1,7 @@
+```release-note:security
+extensions: Disable remote downstream proxy patching by Envoy Extensions other than AWS Lambda. Previously, an operator with service:write ACL permissions for an upstream service could modify Envoy proxy config for downstream services without equivalent permissions for those services. This issue only impacts the Lua extension. [[CVE-2023-2816](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-2816)]
+```
+
+```release-note:breaking-change
+extensions: The Lua extension now targets local proxy listeners for the configured service's upstreams, rather than remote downstream listeners for the configured service, when ListenerType is set to outbound in extension configuration. See [CVE-2023-2816](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-2816) changelog entry for more details.
+```

.changelog/17424.txt

@@ -0,0 +1,3 @@
+```release-note:breaking-change
+api: The `/v1/health/connect/` and `/v1/health/ingress/` endpoints now immediately return 403 "Permission Denied" errors whenever a token with insufficient `service:read` permissions is provided. Prior to this change, the endpoints returned a success code with an empty result list when a token with insufficient permissions was provided.
+```

.changelog/17426.txt

@@ -0,0 +1,5 @@
+```release-note:improvement
+ peering: gRPC queries for TrustBundleList, TrustBundleRead, PeeringList, and PeeringRead now support blocking semantics,
+ reducing network and CPU demand.
+ The HTTP APIs for Peering List and Read have been updated to support blocking.
+ ```

.changelog/17452.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+mesh: Support configuring JWT authentication in Envoy.
+```

.changelog/17456.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+peering: Fix issue where modifying the list of exported services did not correctly replicate changes for services that exist in a non-default namespace.
+```

.changelog/17460.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+hcp: Add new metrics sink to collect, aggregate and export server metrics to HCP in OTEL format.
+```

.changelog/17487.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+xds: Add `property-override` built-in Envoy extension that directly patches Envoy resources.
+```

.changelog/17495.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+xds: Add a built-in Envoy extension that inserts External Authorization (ext_authz) network and HTTP filters.
+```

.changelog/17505.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+xds: Add a built-in Envoy extension that inserts Wasm network filters.
+```

.changelog/17513.txt

@@ -0,0 +1,3 @@
+```release-note:security
+Update to UBI base image to 9.2. 
+```

.changelog/17525.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+http: accept query parameters `datacenter`, `ap` (enterprise-only), and `namespace` (enterprise-only). Both short-hand and long-hand forms of these query params are now supported via the HTTP API (dc/datacenter, ap/partition, ns/namespace).
+```

.changelog/4633.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+server: **(Enterprise Only)** added server side RPC requests IP based read/write rate-limiter.
+```

.changelog/5102.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+server: **(Enterprise Only)** allow automatic license utilization reporting.
+```

.github/workflows/backport-assistant.yml

@@ -19,7 +19,7 @@ jobs:
   backport:
     if: github.event.pull_request.merged
     runs-on: ubuntu-latest
-    container: hashicorpdev/backport-assistant:0.3.3
+    container: hashicorpdev/backport-assistant:0.3.4
     steps:
       - name: Run Backport Assistant for release branches
         run: |
@@ -40,4 +40,4 @@ jobs:
           curl -s -H "Authorization: token ${{ secrets.PR_COMMENT_TOKEN }}" \
             -X POST \
             -d "{ \"body\": \"${github_message}\"}" \
-            "https://api.github.com/repos/${GITHUB_REPOSITORY}/pull/${{ github.event.pull_request.number }}/comments"
+            "https://api.github.com/repos/${GITHUB_REPOSITORY}/pull/${{ github.event.pull_request.number }}/comments"

.github/workflows/build.yml

@@ -34,16 +34,16 @@ jobs:
         run: |
           CONSUL_DATE=$(build-support/scripts/build-date.sh)
           ## TODO: This assumes `make version` outputs 1.1.1+ent-prerel
-          echo "::set-output name=product-date::${CONSUL_DATE}"
+          echo "product-date=${CONSUL_DATE}" >> "$GITHUB_OUTPUT"
 
       - name: Set shared -ldflags
         id: shared-ldflags
         run: |
           T="github.com/hashicorp/consul/version"
-          echo "::set-output name=shared-ldflags::-X ${T}.GitCommit=${GITHUB_SHA::8} \
+          echo "shared-ldflags=-X ${T}.GitCommit=${GITHUB_SHA::8} \
           -X ${T}.GitDescribe=${{ steps.set-product-version.outputs.product-version }} \
           -X ${T}.BuildDate=${{ steps.get-product-version.outputs.product-date }} \
-          "
+          " >> "$GITHUB_OUTPUT"
   validate-outputs:
     needs: set-product-version
     runs-on: ubuntu-latest

.github/workflows/reusable-lint.yml

@@ -51,6 +51,7 @@ jobs:
           working-directory: ${{ matrix.directory }}
           version: v1.51.1
           args: --build-tags="${{ env.GOTAGS }}" -v
+          skip-cache: true
       - name: Notify Slack
         if: ${{ failure() }}
         run: .github/scripts/notify_slack.sh

.github/workflows/reusable-unit-split.yml

@@ -157,6 +157,8 @@ jobs:
           chmod +x /usr/local/bin/datadog-ci
 
       - name: upload coverage
+        # do not run on forks
+        if: ${{ env.DATADOG_API_KEY}}
         env:
           DD_ENV: ci
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" ${{env.TEST_RESULTS}}/gotestsum-report.xml

.github/workflows/reusable-unit.yml

@@ -125,6 +125,8 @@ jobs:
           chmod +x /usr/local/bin/datadog-ci
 
       - name: upload coverage
+        # do not run on forks
+        if: ${{ env.DATADOG_API_KEY}}
         env:
           DD_ENV: ci
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" ${{env.TEST_RESULTS}}/gotestsum-report.xml

.github/workflows/test-integrations.yml

@@ -121,6 +121,8 @@ jobs:
           chmod +x /usr/local/bin/datadog-ci
 
       - name: upload coverage
+        # do not run on forks
+        if: github.event.pull_request.head.repo.full_name == github.repository
         env:
           DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
           DD_ENV: ci
@@ -200,18 +202,24 @@ jobs:
           chmod +x /usr/local/bin/datadog-ci
 
       - name: upload coverage
+        # do not run on forks
+        if: github.event.pull_request.head.repo.full_name == github.repository
         env:
           DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
           DD_ENV: ci
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml"
 
       - name: upload leader coverage
+        # do not run on forks
+        if: github.event.pull_request.head.repo.full_name == github.repository
         env:
           DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
           DD_ENV: ci
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml"
 
       - name: upload agent coverage
+        # do not run on forks
+        if: github.event.pull_request.head.repo.full_name == github.repository
         env:
           DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
           DD_ENV: ci
@@ -338,6 +346,8 @@ jobs:
           chmod +x /usr/local/bin/datadog-ci
 
       - name: upload coverage
+        # do not run on forks
+        if: github.event.pull_request.head.repo.full_name == github.repository
         env:
           DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
           DD_ENV: ci
@@ -442,6 +452,8 @@ jobs:
           chmod +x /usr/local/bin/datadog-ci
 
       - name: upload coverage
+        # do not run on forks
+        if: github.event.pull_request.head.repo.full_name == github.repository
         env:
           DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
           DD_ENV: ci
@@ -452,6 +464,9 @@ jobs:
     needs:
       - setup
       - dev-build
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read
     strategy:
       fail-fast: false
       matrix:
@@ -527,10 +542,37 @@ jobs:
           COMPOSE_INTERACTIVE_NO_CLI: 1
           # tput complains if this isn't set to something.
           TERM: ansi
-      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Authenticate to Vault
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: vault-auth
+        run: vault-auth
+
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Fetch Secrets
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: secrets
+        uses: hashicorp/vault-action@v2.5.0
         with:
-          name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
-          path: ${{ env.TEST_RESULTS_DIR }}
+          url: ${{ steps.vault-auth.outputs.addr }}
+          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
+          token: ${{ steps.vault-auth.outputs.token }}
+          secrets: |
+              kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY;
+
+      - name: prepare datadog-ci
+        if: ${{ !endsWith(github.repository, '-enterprise') }}
+        run: |
+          curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci"
+          chmod +x /usr/local/bin/datadog-ci
+
+      - name: upload coverage
+        # do not run on forks
+        if: github.event.pull_request.head.repo.full_name == github.repository
+        env:
+          DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}"
+          DD_ENV: ci
+        run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
 
   test-integrations-success:
     needs: 

.release/linux/package/usr/lib/systemd/system/consul.service

@@ -6,6 +6,7 @@ After=network-online.target
 ConditionFileNotEmpty=/etc/consul.d/consul.hcl
 
 [Service]
+Type=notify
 EnvironmentFile=-/etc/consul.d/consul.env
 User=consul
 Group=consul

CHANGELOG.md

@@ -1,3 +1,61 @@
+## 1.15.3 (June 1, 2023)
+
+BREAKING CHANGES:
+
+* extensions: The Lua extension now targets local proxy listeners for the configured service's upstreams, rather than remote downstream listeners for the configured service, when ListenerType is set to outbound in extension configuration. See [CVE-2023-2816](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-2816) changelog entry for more details. [[GH-17415](https://github.com/hashicorp/consul/issues/17415)]
+
+SECURITY:
+
+* Update to UBI base image to 9.2. [[GH-17513](https://github.com/hashicorp/consul/issues/17513)]
+* Upgrade golang.org/x/net to address [CVE-2022-41723](https://nvd.nist.gov/vuln/detail/CVE-2022-41723) [[GH-16754](https://github.com/hashicorp/consul/issues/16754)]
+* Upgrade to use Go 1.20.4.
+This resolves vulnerabilities [CVE-2023-24537](https://github.com/advisories/GHSA-9f7g-gqwh-jpf5)(`go/scanner`),
+[CVE-2023-24538](https://github.com/advisories/GHSA-v4m2-x4rp-hv22)(`html/template`),
+[CVE-2023-24534](https://github.com/advisories/GHSA-8v5j-pwr7-w5f8)(`net/textproto`) and
+[CVE-2023-24536](https://github.com/advisories/GHSA-9f7g-gqwh-jpf5)(`mime/multipart`).
+Also, `golang.org/x/net` has been updated to v0.7.0 to resolve CVEs [CVE-2022-41721
+](https://github.com/advisories/GHSA-fxg5-wq6x-vr4w
+), [CVE-2022-27664](https://github.com/advisories/GHSA-69cg-p879-7622) and [CVE-2022-41723
+](https://github.com/advisories/GHSA-vvpx-j8f3-3w6h
+.) [[GH-17240](https://github.com/hashicorp/consul/issues/17240)]
+* extensions: Disable remote downstream proxy patching by Envoy Extensions other than AWS Lambda. Previously, an operator with service:write ACL permissions for an upstream service could modify Envoy proxy config for downstream services without equivalent permissions for those services. This issue only impacts the Lua extension. [[CVE-2023-2816](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-2816)] [[GH-17415](https://github.com/hashicorp/consul/issues/17415)]
+
+FEATURES:
+
+* hcp: Add new metrics sink to collect, aggregate and export server metrics to HCP in OTEL format. [[GH-17460](https://github.com/hashicorp/consul/issues/17460)]
+
+IMPROVEMENTS:
+
+* Fixes a performance issue in Raft where commit latency can increase by 100x or more when under heavy load. For more details see https://github.com/hashicorp/raft/pull/541. [[GH-17081](https://github.com/hashicorp/consul/issues/17081)]
+* agent: add a configurable maximimum age (default: 7 days) to prevent servers re-joining a cluster with stale data [[GH-17171](https://github.com/hashicorp/consul/issues/17171)]
+* agent: add new metrics to track cpu disk and memory usage for server hosts (defaults to: enabled) [[GH-17038](https://github.com/hashicorp/consul/issues/17038)]
+* connect: update supported envoy versions to 1.22.11, 1.23.8, 1.24.6, 1.25.4 [[GH-16889](https://github.com/hashicorp/consul/issues/16889)]
+* envoy: add `MaxEjectionPercent` and `BaseEjectionTime` to passive health check configs. [[GH-15979](https://github.com/hashicorp/consul/issues/15979)]
+* hcp: Add support for linking existing Consul clusters to HCP management plane. [[GH-16916](https://github.com/hashicorp/consul/issues/16916)]
+* logging: change snapshot log header from `agent.server.snapshot` to `agent.server.raft.snapshot` [[GH-17236](https://github.com/hashicorp/consul/issues/17236)]
+* peering: allow re-establishing terminated peering from new token without deleting existing peering first. [[GH-16776](https://github.com/hashicorp/consul/issues/16776)]
+* peering: gRPC queries for TrustBundleList, TrustBundleRead, PeeringList, and PeeringRead now support blocking semantics,
+ reducing network and CPU demand.
+ The HTTP APIs for Peering List and Read have been updated to support blocking. [[GH-17426](https://github.com/hashicorp/consul/issues/17426)]
+* raft: Remove expensive reflection from raft/mesh hot path [[GH-16552](https://github.com/hashicorp/consul/issues/16552)]
+* xds: rename envoy_hcp_metrics_bind_socket_dir to envoy_telemetry_collector_bind_socket_dir to remove HCP naming references. [[GH-17327](https://github.com/hashicorp/consul/issues/17327)]
+
+BUG FIXES:
+
+* Fix an bug where decoding some Config structs with unset pointer fields could fail with `reflect: call of reflect.Value.Type on zero Value`. [[GH-17048](https://github.com/hashicorp/consul/issues/17048)]
+* acl: **(Enterprise only)** Check permissions in correct partition/namespace when resolving service in non-default partition/namespace
+* acl: Fix an issue where the anonymous token was synthesized in non-primary datacenters which could cause permission errors when federating clusters with ACL replication enabled. [[GH-17231](https://github.com/hashicorp/consul/issues/17231)]
+* acls: Fix ACL bug that can result in sidecar proxies having incorrect endpoints.
+* connect: Fix multiple inefficient behaviors when querying service health. [[GH-17241](https://github.com/hashicorp/consul/issues/17241)]
+* gateways: Fix an bug where targeting a virtual service defined by a service-resolver was broken for HTTPRoutes. [[GH-17055](https://github.com/hashicorp/consul/issues/17055)]
+* grpc: ensure grpc resolver correctly uses lan/wan addresses on servers [[GH-17270](https://github.com/hashicorp/consul/issues/17270)]
+* namespaces: adjusts the return type from HTTP list API to return the `api` module representation of a namespace.
+This fixes an error with the `consul namespace list` command when a namespace has a deferred deletion timestamp.
+* peering: Fix issue where modifying the list of exported services did not correctly replicate changes for services that exist in a non-default namespace. [[GH-17456](https://github.com/hashicorp/consul/issues/17456)]
+* peering: Fix issue where peer streams could incorrectly deregister services in various scenarios. [[GH-17235](https://github.com/hashicorp/consul/issues/17235)]
+* peering: ensure that merged central configs of peered upstreams for partitioned downstreams work [[GH-17179](https://github.com/hashicorp/consul/issues/17179)]
+* xds: Fix possible panic that can when generating clusters before the root certificates have been fetched. [[GH-17185](https://github.com/hashicorp/consul/issues/17185)]
+
 ## 1.14.7 (May 16, 2023)
 
 SECURITY:

Dockerfile

@@ -201,7 +201,7 @@ CMD ["agent", "-dev", "-client", "0.0.0.0"]
 
 # Red Hat UBI-based image
 # This target is used to build a Consul image for use on OpenShift.
-FROM registry.access.redhat.com/ubi9-minimal:9.1.0 as ubi
+FROM registry.access.redhat.com/ubi9-minimal:9.2 as ubi
 
 ARG PRODUCT_NAME
 ARG PRODUCT_VERSION

GNUmakefile

@@ -165,13 +165,13 @@ dev-build:
 	cp ${MAIN_GOPATH}/bin/consul ./bin/consul
 
 
-dev-docker-dbg: dev-docker linux dev-build
+dev-docker-dbg: dev-docker
 	@echo "Pulling consul container image - $(CONSUL_IMAGE_VERSION)"
 	@docker pull consul:$(CONSUL_IMAGE_VERSION) >/dev/null
 	@echo "Building Consul Development container - $(CONSUL_DEV_IMAGE)"
 	@#  'consul-dbg:local' tag is needed to run the integration tests
 	@#  'consul-dev:latest' is needed by older workflows
-	@docker buildx use default && docker buildx build -t 'consul-dbg:local' -t '$(CONSUL_DEV_IMAGE)' \
+	@docker buildx use default && docker buildx build -t $(CONSUL_COMPAT_TEST_IMAGE)-dbg:local \
        --platform linux/$(GOARCH) \
 	   --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \
        --load \
@@ -384,6 +384,7 @@ codegen-tools:
 deep-copy: codegen-tools
 	@$(SHELL) $(CURDIR)/agent/structs/deep-copy.sh
 	@$(SHELL) $(CURDIR)/agent/proxycfg/deep-copy.sh
+	@$(SHELL) $(CURDIR)/agent/consul/state/deep-copy.sh
 
 version:
 	@echo -n "Version:                    "

agent/agent.go

@@ -597,6 +597,12 @@ func (a *Agent) Start(ctx context.Context) error {
 	// regular and on-demand state synchronizations (anti-entropy).
 	a.sync = ae.NewStateSyncer(a.State, c.AEInterval, a.shutdownCh, a.logger)
 
+	err = validateFIPSConfig(a.config)
+	if err != nil {
+		// Log warning, rather than force breaking
+		a.logger.Warn("FIPS 140-2 Compliance", "issue", err)
+	}
+
 	// create the config for the rpc server/client
 	consulCfg, err := newConsulConfig(a.config, a.logger)
 	if err != nil {
@@ -4565,13 +4571,13 @@ func (a *Agent) persistServerMetadata() {
 
 			f, err := consul.OpenServerMetadata(file)
 			if err != nil {
-				a.logger.Error("failed to open existing server metadata: %w", err)
+				a.logger.Error("failed to open existing server metadata", "error", err)
 				continue
 			}
 
 			if err := consul.WriteServerMetadata(f); err != nil {
 				f.Close()
-				a.logger.Error("failed to write server metadata: %w", err)
+				a.logger.Error("failed to write server metadata", "error", err)
 				continue
 			}