Merge remote-tracking branch 'upstream/main' into enhance-fuzz-logging

Signed-off-by: Kevin Baichoo <kbaichoo@google.com>

.azure-pipelines/bazel.yml

@@ -26,6 +26,9 @@ parameters:
   type: string
   default: ''
 
+- name: bazelConfigRBE
+  type: string
+  default: --config=remote-ci --config=rbe-google --jobs=$(RbeJobs)
 - name: cacheKeyBazel
   type: string
   default: $(cacheKeyBazel)
@@ -47,6 +50,13 @@ parameters:
 - name: bazelBuildExtraOptions
   type: string
   default: ""
+- name: bazelStartupExtraOptions
+  type: string
+  default: ""
+- name: bazelUseBES
+  displayName: "Upload bazel run data to BES"
+  type: boolean
+  default: true
 - name: envoyBuildFilterExample
   type: string
   default: ""
@@ -175,12 +185,17 @@ steps:
   displayName: "Enable IPv6"
   condition: ${{ parameters.managedAgent }}
 
-- script: ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}'
+- script: |
+    if [[ "${{ parameters.bazelUseBES }}" != 'true' ]]; then
+        unset GOOGLE_BES_PROJECT_ID
+    fi
+    ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}'
   workingDirectory: $(Build.SourcesDirectory)
   env:
     ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
     ENVOY_BUILD_FILTER_EXAMPLE: ${{ parameters.envoyBuildFilterExample }}
     GITHUB_TOKEN: "${{ parameters.authGithub }}"
+    BAZEL_STARTUP_EXTRA_OPTIONS: "${{ parameters.bazelStartupExtraOptions }}"
     ${{ if ne(parameters['cacheTestResults'], true) }}:
       BAZEL_NO_CACHE_TEST_RESULTS: 1
     ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
@@ -194,7 +209,7 @@ steps:
     ${{ if parameters.rbe }}:
       GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
       ENVOY_RBE: "1"
-      BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=rbe-google --jobs=$(RbeJobs) ${{ parameters.bazelBuildExtraOptions }}"
+      BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelConfigRBE }} ${{ parameters.bazelBuildExtraOptions }}"
     ${{ if eq(parameters.rbe, false) }}:
       BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci ${{ parameters.bazelBuildExtraOptions }}"
       BAZEL_REMOTE_CACHE: $(LocalBuildCache)
@@ -206,6 +221,20 @@ steps:
     echo "disk space at end of build:"
     df -h
 
+    for hprof in $(find "$(Build.StagingDirectory)" -name "*heapdump.hprof"); do
+        echo
+        mkdir -p $(Build.StagingDirectory)/envoy/hprof
+        echo "Copying ${hprof}"
+        cp -a $hprof $(Build.StagingDirectory)/envoy/hprof
+    done
+
+    cp -a "$(Build.StagingDirectory)/bazel_root/base/server/jvm.out" $(Build.StagingDirectory)/envoy
+
+    if [[ "${{ parameters.artifactSuffix }}" == ".arm64" ]]; then
+        # Dump bazel-remote logging (only required for arm/self-hosted).
+        sudo systemctl status --no-pager bazel-remote > $(Build.StagingDirectory)/envoy/br.status
+        sudo journalctl --no-pager -xu bazel-remote > $(Build.StagingDirectory)/envoy/br.journal
+    fi
     echo
     du -ch "$(Build.StagingDirectory)" | grep -E "[0-9]{2,}M|[0-9]G"
 

.azure-pipelines/pipelines.yml

@@ -47,7 +47,7 @@ variables:
 - name: cacheKeyName
   value: envoy
 - name: cacheKeyVersion
-  value: v0
+  value: v1
 - name: cacheKeyBazel
   value: '.bazelversion | ./WORKSPACE | **/*.bzl, !mobile/**, !envoy-docs/**'
 - name: cacheKeyDocker

.azure-pipelines/stage/checks.yml

@@ -102,23 +102,28 @@ jobs:
   # TODO(phlax): Make this configurable and ratchet down!
   timeoutInMinutes: 240
   pool: "envoy-x64-large"
+  variables:
+    BAZEL_STARTUP_EXTRA: ''
   strategy:
     maxParallel: 2
     matrix:
       coverage:
         CI_TARGET: "coverage"
+        BAZEL_STARTUP_EXTRA: --host_jvm_args=-Xmx20g --host_jvm_args=-XX:MaxDirectMemorySize=4G
       fuzz_coverage:
         CI_TARGET: "fuzz_coverage"
   steps:
   - template: ../bazel.yml
     parameters:
       managedAgent: false
       ciTarget: bazel.$(CI_TARGET)
-      rbe: false
+      rbe: true
       # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces
-      bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base"
+      bazelConfigRBE: --config=ci --config=rbe-google --jobs=$(RbeJobs)
+      bazelBuildExtraOptions: --sandbox_base=/tmp/sandbox_base --define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only
+      bazelStartupExtraOptions: $(BAZEL_STARTUP_EXTRA)
+      bazelUseBES: false
       cacheTestResults: ${{ parameters.cacheTestResults }}
-
   - script: ci/run_envoy_docker.sh 'ci/do_ci.sh $(CI_TARGET)-upload'
     displayName: "Upload $(CI_TARGET) Report to GCS"
     env:

.bazelrc

@@ -9,7 +9,6 @@
 # The number 3G is chosen heuristically to both support large VM and small VM with RBE.
 # Startup options cannot be selected via config.
 startup --host_jvm_args=-Xmx3g
-startup --host_jvm_args=-XX:MaxDirectMemorySize=512m
 
 fetch --color=yes
 run --color=yes

.github/workflows/_cache_docker.yml

@@ -37,7 +37,7 @@ jobs:
   docker:
     runs-on: ubuntu-22.04
     steps:
-    - uses: envoyproxy/toolshed/gh-actions/docker/cache/prime@actions-v0.0.10
+    - uses: envoyproxy/toolshed/gh-actions/docker/cache/prime@actions-v0.0.16
       name: Prime Docker cache (${{ inputs.image_repo }}:${{ inputs.image_tag }}@sha256:${{ inputs.image_sha }})
       with:
         image_tag: "${{ inputs.image_repo }}:${{ inputs.image_tag }}@sha256:${{ inputs.image_sha }}"

.github/workflows/_ci.yml

@@ -92,7 +92,7 @@ jobs:
     steps:
     - if: ${{ inputs.cache_build_image }}
       name: Restore Docker cache (${{ inputs.cache_build_image }})
-      uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@actions-v0.0.10
+      uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@actions-v0.0.16
       with:
         image_tag: ${{ inputs.cache_build_image }}
 
@@ -123,7 +123,7 @@ jobs:
 
     - if: ${{ inputs.run_pre }}
       name: Run pre action ${{ inputs.run_pre && format('({0})', inputs.run_pre) || '' }}
-      uses: envoyproxy/toolshed/gh-actions/using/recurse@actions-v0.0.10
+      uses: envoyproxy/toolshed/gh-actions/using/recurse@actions-v0.0.16
       with:
         uses: ${{ inputs.run_pre }}
         with: ${{ inputs.run_pre_with }}
@@ -147,7 +147,7 @@ jobs:
 
     - if: ${{ inputs.run_post }}
       name: Run post action ${{ inputs.run_pre && format('({0})', inputs.run_post) || '' }}
-      uses: envoyproxy/toolshed/gh-actions/using/recurse@actions-v0.0.10
+      uses: envoyproxy/toolshed/gh-actions/using/recurse@actions-v0.0.16
       with:
         uses: ${{ inputs.run_post }}
         with: ${{ inputs.run_post_with }}

.github/workflows/_workflow-start.yml

@@ -29,7 +29,7 @@ jobs:
 
     - if: ${{ steps.env.outputs.trusted != 'true' }}
       name: Start status check
-      uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.10
+      uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.16
       with:
         authToken: ${{ secrets.GITHUB_TOKEN }}
         context: ${{ inputs.workflow_name }}

.github/workflows/commands.yml

@@ -2,7 +2,8 @@ name: commands
 
 on:
   issue_comment:
-    types: [created]
+    types:
+    - created
 
 permissions:
   contents: read
@@ -21,7 +22,10 @@ jobs:
     permissions:
       pull-requests: write
       actions: write
+      checks: read
     steps:
-    - uses: envoyproxy/toolshed/gh-actions/retest@actions-v0.0.10
+    - uses: envoyproxy/toolshed/gh-actions/retest@actions-v0.0.16
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
+        azp_org: cncf
+        azp_token: ${{ secrets.AZP_TOKEN }}

.github/workflows/envoy-sync.yml

@@ -27,7 +27,7 @@ jobs:
         - envoy-filter-example
         - data-plane-api
     steps:
-    - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.0.10
+    - uses: envoyproxy/toolshed/gh-actions/dispatch@actions-v0.0.16
       with:
         repository: "envoyproxy/${{ matrix.downstream }}"
         ref: main

.github/workflows/workflow-complete.yml

@@ -53,7 +53,7 @@ jobs:
         echo "state=${STATE}" >> "$GITHUB_OUTPUT"
       id: job
     - name: Complete status check
-      uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.10
+      uses: envoyproxy/toolshed/gh-actions/status@actions-v0.0.16
       with:
         authToken: ${{ secrets.GITHUB_TOKEN }}
         context: Verify/examples

.gitignore

@@ -55,3 +55,5 @@ bazel.output.txt
 tools/dev/src
 distribution/custom
 examples/websocket/certs
+/contrib/golang/**/test_data/go.sum
+/contrib/golang/**/test_data/*/go.sum

CODEOWNERS

@@ -23,8 +23,8 @@ extensions/filters/common/original_src @klarose @mattklein123
 # cdn_loop extension
 /*/extensions/filters/http/cdn_loop @justin-mp @penguingao @alyssawilk
 # external processing filter
-/*/extensions/filters/http/ext_proc @gbrail @stevenzzzz @tyxia @mattklein123
-/*/extensions/filters/common/mutation_rules @gbrail @chaoqin-li1123 @tyxia @mattklein123
+/*/extensions/filters/http/ext_proc @gbrail @stevenzzzz @tyxia @mattklein123 @htuch
+/*/extensions/filters/common/mutation_rules @gbrail @chaoqin-li1123 @tyxia @mattklein123 @htuch
 # jwt_authn http filter extension
 /*/extensions/filters/http/jwt_authn @qiwzhang @lizan
 # grpc_field_extraction http filter extension
@@ -351,7 +351,7 @@ extensions/filters/http/oauth2 @derekargueta @mattklein123
 /contrib/client_ssl_auth/ @UNOWNED @UNOWNED
 /contrib/common/sqlutils/ @cpakulski @cpakulski
 /contrib/dynamo/ @UNOWNED @UNOWNED
-/contrib/golang/ @doujiang24 @wangfakang @StarryVae
+/contrib/golang/ @doujiang24 @wangfakang @StarryVae @spacewander @antJack
 /contrib/squash/ @yuval-k @alyssawilk
 /contrib/kafka/ @mattklein123 @adamkotwasinski
 /contrib/rocketmq_proxy/ @aaron-ai @lizhanhui @lizan

api/envoy/config/tap/v3/common.proto

@@ -4,6 +4,7 @@ package envoy.config.tap.v3;
 
 import "envoy/config/common/matcher/v3/matcher.proto";
 import "envoy/config/core/v3/base.proto";
+import "envoy/config/core/v3/extension.proto";
 import "envoy/config/core/v3/grpc_service.proto";
 import "envoy/config/route/v3/route_components.proto";
 
@@ -183,7 +184,7 @@ message OutputConfig {
 }
 
 // Tap output sink configuration.
-// [#next-free-field: 6]
+// [#next-free-field: 7]
 message OutputSink {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.service.tap.v2alpha.OutputSink";
@@ -259,6 +260,9 @@ message OutputSink {
     //   been configured to receive tap configuration from some other source (e.g., static
     //   file, XDS, etc.) configuring the buffered admin output type will fail.
     BufferedAdminSink buffered_admin = 5;
+
+    // Tap output filter will be defined by an extension type
+    core.v3.TypedExtensionConfig custom_sink = 6;
   }
 }
 

api/envoy/extensions/load_balancing_policies/cluster_provided/v3/cluster_provided.proto

@@ -17,7 +17,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // If this extension is configured, the target cluster must provide load balancer when the cluster
 // is created.
 //
-// ..note::
+// .. note::
 //   Cluster provided load balancing policy could not be used as sub-policy of other hierarchical
 //   load balancing policies, such as subset load balancing policy.
 message ClusterProvided {

api/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.proto

@@ -56,7 +56,7 @@ message RingHash {
   // If set to `true`, the cluster will use hostname instead of the resolved
   // address as the key to consistently hash to an upstream host. Only valid for StrictDNS clusters with hostnames which resolve to a single IP address.
   //
-  // ..note::
+  // .. note::
   //   This is deprecated and please use :ref:`consistent_hashing_lb_config
   //   <envoy_v3_api_field_extensions.load_balancing_policies.ring_hash.v3.RingHash.consistent_hashing_lb_config>` instead.
   bool use_hostname_for_hashing = 4
@@ -79,7 +79,7 @@ message RingHash {
   // This is an O(N) algorithm, unlike other load balancers. Using a lower `hash_balance_factor` results in more hosts
   // being probed, so use a higher value if you require better performance.
   //
-  // ..note::
+  // .. note::
   //   This is deprecated and please use :ref:`consistent_hashing_lb_config
   //   <envoy_v3_api_field_extensions.load_balancing_policies.ring_hash.v3.RingHash.consistent_hashing_lb_config>` instead.
   google.protobuf.UInt32Value hash_balance_factor = 5 [

bazel/repository_locations.bzl

@@ -1384,12 +1384,13 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "Bazel rust rules",
         project_desc = "Bazel rust rules (used by Wasm)",
         project_url = "https://github.com/bazelbuild/rules_rust",
-        version = "0.26.0",
-        sha256 = "9d04e658878d23f4b00163a72da3db03ddb451273eb347df7d7c50838d698f49",
-        urls = ["https://github.com/bazelbuild/rules_rust/releases/download/{version}/rules_rust-v{version}.tar.gz"],
+        version = "262b6a5ea17ec91b3c07e37e82af0eb15dd6ceef",
+        strip_prefix = "rules_rust-{version}",
+        sha256 = "e968b3d01c305282bd237bcdafe5e5141192e93eaefb22712e8bc4299e672b16",
+        urls = ["https://github.com/bazelbuild/rules_rust/archive/{version}.tar.gz"],
         use_category = ["dataplane_ext"],
         extensions = ["envoy.wasm.runtime.wasmtime"],
-        release_date = "2023-07-28",
+        release_date = "2023-08-11",
         cpe = "N/A",
         license = "Apache-2.0",
         license_url = "https://github.com/bazelbuild/rules_rust/blob/{version}/LICENSE.txt",

changelogs/current.yaml

@@ -2,9 +2,6 @@ date: Pending
 
 behavior_changes:
 # *Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required*
-- area: tracing
-  change: |
-    OpenTracing is now deprecated since the upstream project has been abandoned.
 
 - area: eds
   change: |
@@ -81,5 +78,12 @@ new_features:
 - area: redis
   change: |
     added support for lmove command.
+- area: tap
+  change: |
+    added :ref:`custom_sink <envoy_v3_api_field_config.tap.v3.OutputSink.custom_sink>` type to enable writing tap data
+    out to a custom sink extension.
 
 deprecated:
+- area: tracing
+  change: |
+    OpenTracing is deprecated and will be removed at version 1.30, since the upstream project has been abandoned.

ci/build_setup.sh

@@ -28,6 +28,7 @@ export ENVOY_BUILD_FILTER_EXAMPLE="${ENVOY_BUILD_FILTER_EXAMPLE:-0}"
 
 read -ra BAZEL_BUILD_EXTRA_OPTIONS <<< "${BAZEL_BUILD_EXTRA_OPTIONS:-}"
 read -ra BAZEL_EXTRA_TEST_OPTIONS <<< "${BAZEL_EXTRA_TEST_OPTIONS:-}"
+read -ra BAZEL_STARTUP_EXTRA_OPTIONS <<< "${BAZEL_STARTUP_EXTRA_OPTIONS:-}"
 read -ra BAZEL_OPTIONS <<< "${BAZEL_OPTIONS:-}"
 
 echo "ENVOY_SRCDIR=${ENVOY_SRCDIR}"
@@ -105,12 +106,15 @@ trap cleanup EXIT
 _bazel="$(which bazel)"
 
 BAZEL_STARTUP_OPTIONS=(
+    "${BAZEL_STARTUP_EXTRA_OPTIONS[@]}"
     "--output_user_root=${BUILD_DIR}/bazel_root"
     "--output_base=${BUILD_DIR}/bazel_root/base")
 
 bazel () {
-    # echo "RUNNING BAZEL (${PWD}): ${BAZEL_STARTUP_OPTIONS[*]} <> ${*}" >&2
-    "$_bazel" "${BAZEL_STARTUP_OPTIONS[@]}" "$@"
+    local startup_options
+    read -ra startup_options <<< "${BAZEL_STARTUP_OPTION_LIST:-}"
+    # echo "RUNNING BAZEL (${PWD}): ${startup_options[*]} <> ${*}" >&2
+    "$_bazel" "${startup_options[@]}" "$@"
 }
 
 export _bazel

ci/do_ci.sh

@@ -41,12 +41,14 @@ else
 fi
 
 function collect_build_profile() {
-  declare -g build_profile_count=${build_profile_count:-1}
-  mv -f \
-     "$(bazel info "${BAZEL_BUILD_OPTIONS[@]}" output_base)/command.profile.gz" \
-     "${ENVOY_BUILD_PROFILE}/${build_profile_count}-$1.profile.gz" \
-      || :
-  ((build_profile_count++))
+    local output_base
+    declare -g build_profile_count=${build_profile_count:-1}
+    output_base="$(bazel info "${BAZEL_BUILD_OPTIONS[@]}" output_base)"
+    mv -f \
+       "${output_base}/command.profile.gz" \
+       "${ENVOY_BUILD_PROFILE}/${build_profile_count}-$1.profile.gz" \
+        || :
+    ((build_profile_count++))
 }
 
 function bazel_with_collection() {
@@ -416,6 +418,7 @@ case $CI_TARGET in
         if [[ "$CI_TARGET" == "fuzz_coverage" ]]; then
             export FUZZ_COVERAGE=true
         fi
+        export BAZEL_GRPC_LOG="${ENVOY_BUILD_DIR}/grpc.log"
         "${ENVOY_SRCDIR}/test/run_envoy_bazel_coverage.sh" \
             "${COVERAGE_TEST_TARGETS[@]}"
         collect_build_profile coverage
@@ -578,9 +581,12 @@ case $CI_TARGET in
                   "${FETCH_TARGETS[@]}" \
                 && break
             n=$((n+1))
-            if [[ "$n" -ne 10 ]]; then
+            if [[ "$n" -lt 10 ]]; then
                 sleep 15
                 echo "Retrying fetch ..."
+            else
+                echo "Fetch failed"
+                exit 1
             fi
         done
         ;;