build: setup contrib (envoyproxy#17595)

Signed-off-by: Matt Klein <mklein@lyft.com>

.azure-pipelines/pipelines.yml

@@ -310,17 +310,33 @@ stages:
         itemPattern: "bazel.release/envoy_binary.tar.gz"
         downloadType: single
         targetPath: $(Build.StagingDirectory)
+    - task: DownloadBuildArtifacts@0
+      inputs:
+        buildType: current
+        artifactName: "bazel.release"
+        itemPattern: "bazel.release/envoy-contrib_binary.tar.gz"
+        downloadType: single
+        targetPath: $(Build.StagingDirectory)
     - task: DownloadBuildArtifacts@0
       inputs:
         buildType: current
         artifactName: "bazel.release.arm64"
         itemPattern: "bazel.release.arm64/envoy_binary.tar.gz"
         downloadType: single
         targetPath: $(Build.StagingDirectory)
+    - task: DownloadBuildArtifacts@0
+      inputs:
+        buildType: current
+        artifactName: "bazel.release.arm64"
+        itemPattern: "bazel.release.arm64/envoy-contrib_binary.tar.gz"
+        downloadType: single
+        targetPath: $(Build.StagingDirectory)
     - bash: |
         set -e
         mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64
+        tar zxf $(Build.StagingDirectory)/bazel.release/envoy-contrib_binary.tar.gz -C ./linux/amd64
         mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64
+        tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy-contrib_binary.tar.gz -C ./linux/arm64
         ci/docker_ci.sh
       workingDirectory: $(Build.SourcesDirectory)
       env:

.dockerignore

@@ -3,8 +3,8 @@
 !/ci
 !/configs/google-vrp
 !/configs/*yaml
-!/linux/amd64/build_release*
-!/linux/arm64/build_release*
+!/linux/amd64/build_*release*
+!/linux/arm64/build_*release*
 !/local
 !/test/config/integration/certs
 !/windows

BUILD

@@ -33,3 +33,10 @@ package_group(
         "//test/extensions/...",
     ],
 )
+
+package_group(
+    name = "contrib_library",
+    packages = [
+        "//contrib/...",
+    ],
+)

CODEOWNERS

@@ -119,7 +119,6 @@ extensions/filters/common/original_src @snowp @klarose
 /*/extensions/filters/http/router @alyssawilk @mattklein123 @snowp
 /*/extensions/filters/http/grpc_web @fengli79 @lizan
 /*/extensions/filters/http/grpc_stats @kyessenov @lizan
-/*/extensions/filters/http/squash @yuval-k @alyssawilk
 /*/extensions/filters/common/original_src @klarose @snowp
 /*/extensions/filters/listener/tls_inspector @ggreenway @asraa
 /*/extensions/grpc_credentials/example @wozz @htuch
@@ -201,3 +200,7 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp
 /*/extensions/matching/input_matchers/ip @aguinet @snowp
 # Kafka
 /*/extensions/filters/network/kafka @mattklein123 @adamkotwasinski
+
+# Contrib
+/contrib/exe/ @mattklein123 @lizan
+/contrib/squash/ @yuval-k @alyssawilk

CONTRIBUTING.md

@@ -262,7 +262,17 @@ Other changes will likely include
   * Editing [source/extensions/extensions_metadata.yaml](source/extensions/extensions_metadata.yaml) to include metadata for the new extensions
   * Editing [docs/root/api-v3/config/config.rst](docs/root/api-v3/config/config.rst) to add area/area
   * Adding `docs/root/api-v3/config/area/area.rst` to add a table of contents for the API docs
-  * Adding `source/extensions/area/well_known_names.h` for registered plugins
+
+# Adding contrib extensions
+
+See [EXTENSION_POLICY.md](EXTENSION_POLICY.md) for more information on contrib. Adding a contrib
+extension mostly mirrors adding a normal extension above. Some differences are noted here:
+
+* API files should be added in `api/contrib/envoy/`, but the protos' namespaces should still be as
+  in normal extensions (which will make file movement easier later if the extension gets promoted
+  to core).
+* Build config and metadata should be included in [contrib/contrib_build_config.bzl](contrib/contrib_build_config.bzl)
+  and [contrib/extensions_metadata.yaml](contrib/extensions_metadata.yaml).
 
 # DCO: Sign your work
 

EXTENSION_POLICY.md

@@ -137,3 +137,16 @@ case we need to install an extension point, which can be done as follows:
   3. Update [extending envoy](docs/root/extending/extending.rst) to list the new
      extension point and add any documentation explaining the extension point.
      At the very least this should link to the corresponding proto.
+
+## Contrib extensions
+
+As described in [this document](https://docs.google.com/document/d/1yl7GOZK1TDm_7vxQvt8UQEAu07UQFru1uEKXM6ZZg_g/edit#),
+Envoy allows an alternate path to adding extensions called `contrib/`. The barrier to entry for a
+contrib extension is lower than a core extension, with the tradeoff that contrib extensions are not
+included by default in the main image builds. Consumers need to pull directly from the contrib
+images described in the installation guide. Please read the linked document in detail to determine
+whether contrib extensions are the right choice for a newly proposed extension.
+
+**NOTE:** Contrib extensions are not eligible for Envoy security team coverage.
+**NOTE:** As per the linked Google Doc, contrib extensions generally should use `v3alpha` to avoid
+requiring API shepherd reviews.

REPO_LAYOUT.md

@@ -66,7 +66,6 @@ Not every directory within test is described below, but a few highlights:
 We maintain a very specific code and namespace layout for extensions. This aids in discovering
 code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNERS).
 
-
 * All extensions are either registered in [all_extensions.bzl](source/extensions/all_extensions.bzl)
   or [extensions_build_config.bzl](source/extensions/extensions_build_config.bzl). The former is
   for extensions that cannot be removed from the primary Envoy build. The latter is for extensions
@@ -124,3 +123,14 @@ code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNE
   code that is used by both HTTP and network filters. Common code used only by two HTTP filters
   would be found in `filters/http/common/`. Common code should be placed in a common namespace.
   E.g., `Envoy::Extensions::Filters::Common`.
+
+## [contrib](contrib/) layout
+
+This directory contains contrib extensions. See [EXTENSION_POLICY.md](EXTENSION_POLICY.md) for
+more information.
+
+* [contrib/exe/](contrib/exe/): The default executable for contrib. This is similar to the
+  `envoy-static` target but also includes all contrib extensions, and is used to produce the
+  contrib image targets.
+* [contrib/...](contrib/): The rest of this directory mirrors the [source/extensions](source/extensions/)
+  layout. Contrib extensions are placed here.

SECURITY.md

@@ -93,6 +93,8 @@ detect issues during their execution on ClusterFuzz. A soak period of 7 days pro
 guarantee, since we will invoke the security release process for medium or higher severity issues
 for these older bugs.
 
+**NOTE:** Contrib extensions are not eligible for Envoy security team coverage.
+
 ### Threat model
 
 See https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/threat_model.

api/BUILD

@@ -57,6 +57,7 @@ proto_library(
     name = "v3_protos",
     visibility = ["//visibility:public"],
     deps = [
+        "//contrib/envoy/extensions/filters/http/squash/v3:pkg",
         "//envoy/admin/v3:pkg",
         "//envoy/config/accesslog/v3:pkg",
         "//envoy/config/bootstrap/v3:pkg",
@@ -143,7 +144,6 @@ proto_library(
         "//envoy/extensions/filters/http/rbac/v3:pkg",
         "//envoy/extensions/filters/http/router/v3:pkg",
         "//envoy/extensions/filters/http/set_metadata/v3:pkg",
-        "//envoy/extensions/filters/http/squash/v3:pkg",
         "//envoy/extensions/filters/http/tap/v3:pkg",
         "//envoy/extensions/filters/http/wasm/v3:pkg",
         "//envoy/extensions/filters/listener/http_inspector/v3:pkg",

api/test/validate/BUILD

@@ -20,7 +20,6 @@ api_cc_test(
         "@envoy_api//envoy/extensions/filters/http/ip_tagging/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/http/lua/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/http/router/v3:pkg_cc_proto",
-        "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/network/mongo_proxy/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/network/redis_proxy/v3:pkg_cc_proto",

api/test/validate/pgv_test.cc

@@ -19,7 +19,6 @@
 #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.validate.h"
 #include "envoy/extensions/filters/http/lua/v3/lua.pb.validate.h"
 #include "envoy/extensions/filters/http/router/v3/router.pb.validate.h"
-#include "envoy/extensions/filters/http/squash/v3/squash.pb.validate.h"
 #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.validate.h"
 #include "envoy/extensions/filters/network/mongo_proxy/v3/mongo_proxy.pb.validate.h"
 #include "envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.pb.validate.h"

api/versioning/BUILD

@@ -9,6 +9,7 @@ proto_library(
     name = "active_protos",
     visibility = ["//visibility:public"],
     deps = [
+        "//contrib/envoy/extensions/filters/http/squash/v3:pkg",
         "//envoy/admin/v3:pkg",
         "//envoy/config/accesslog/v3:pkg",
         "//envoy/config/bootstrap/v3:pkg",
@@ -95,7 +96,6 @@ proto_library(
         "//envoy/extensions/filters/http/rbac/v3:pkg",
         "//envoy/extensions/filters/http/router/v3:pkg",
         "//envoy/extensions/filters/http/set_metadata/v3:pkg",
-        "//envoy/extensions/filters/http/squash/v3:pkg",
         "//envoy/extensions/filters/http/tap/v3:pkg",
         "//envoy/extensions/filters/http/wasm/v3:pkg",
         "//envoy/extensions/filters/listener/http_inspector/v3:pkg",

bazel/README.md

@@ -668,7 +668,10 @@ The following optional features can be enabled on the Bazel build command-line:
 
 Envoy uses a modular build which allows extensions to be removed if they are not needed or desired.
 Extensions that can be removed are contained in
-[extensions_build_config.bzl](../source/extensions/extensions_build_config.bzl).
+[extensions_build_config.bzl](../source/extensions/extensions_build_config.bzl). Contrib build
+extensions are contained in [contrib_build_config.bzl](../contrib/contrib_build_config.bzl). Note
+that contrib extensions are only included by default when building the contrib executable and in
+the default contrib images pushed to Docker Hub.
 
 The extensions disabled by default can be enabled by adding the following parameter to Bazel, for example to enable
 `envoy.filters.http.kill_request` extension, add `--//source/extensions/filters/http/kill_request:enabled`.
@@ -681,6 +684,13 @@ If you're building from a custom build repository, the parameters need to prefix
 
 You may persist those options in `user.bazelrc` in Envoy repo or your `.bazelrc`.
 
+Contrib extensions can be enabled and disabled similarly to above when building the contrib
+executable. For example:
+
+`bazel build //contrib/exe:envoy-static --//contrib/squash/filters/http/source:enabled=false`
+
+Will disable the squash extension when building the contrib executable.
+
 ## Customize extension build config
 
 You can also use the following procedure to customize the extensions for your build:
@@ -719,6 +729,11 @@ local_repository(
 ...
 ```
 
+When performing custom builds, it is acceptable to include contrib extensions as well. This can
+be done by including the desired Bazel paths from [contrib_build_config.bzl](../contrib/contrib_build_config.bzl)
+into the overriden `extensions_build_config.bzl`. (There is no need to specifically perform
+a contrib build to include a contrib extension.)
+
 ## Extra extensions
 
 If you are building your own Envoy extensions or custom Envoy builds and encounter visibility

bazel/api_binding.bzl

@@ -8,6 +8,7 @@ def _default_envoy_api_impl(ctx):
         "test",
         "tools",
         "versioning",
+        "contrib",
     ]
     for d in api_dirs:
         ctx.symlink(ctx.path(ctx.attr.envoy_root).dirname.get_child(ctx.attr.reldir).get_child(d), d)

bazel/envoy_build_system.bzl

@@ -6,6 +6,7 @@ load(":envoy_internal.bzl", "envoy_external_dep_path")
 load(
     ":envoy_library.bzl",
     _envoy_basic_cc_library = "envoy_basic_cc_library",
+    _envoy_cc_contrib_extension = "envoy_cc_contrib_extension",
     _envoy_cc_extension = "envoy_cc_extension",
     _envoy_cc_library = "envoy_cc_library",
     _envoy_cc_linux_library = "envoy_cc_linux_library",
@@ -50,8 +51,8 @@ load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 def envoy_package():
     native.package(default_visibility = ["//visibility:public"])
 
-def envoy_extension_package(enabled_default = True):
-    native.package(default_visibility = EXTENSION_PACKAGE_VISIBILITY)
+def envoy_extension_package(enabled_default = True, default_visibility = EXTENSION_PACKAGE_VISIBILITY):
+    native.package(default_visibility = default_visibility)
 
     bool_flag(
         name = "enabled",
@@ -63,6 +64,9 @@ def envoy_extension_package(enabled_default = True):
         flag_values = {":enabled": "True"},
     )
 
+def envoy_contrib_package():
+    envoy_extension_package(default_visibility = ["//:contrib_library"])
+
 # A genrule variant that can output a directory. This is useful when doing things like
 # generating a fuzz corpus mechanically.
 def _envoy_directory_genrule_impl(ctx):
@@ -220,6 +224,7 @@ envoy_cc_binary = _envoy_cc_binary
 # Library wrappers (from envoy_library.bzl)
 envoy_basic_cc_library = _envoy_basic_cc_library
 envoy_cc_extension = _envoy_cc_extension
+envoy_cc_contrib_extension = _envoy_cc_contrib_extension
 envoy_cc_library = _envoy_cc_library
 envoy_cc_linux_library = _envoy_cc_linux_library
 envoy_cc_posix_library = _envoy_cc_posix_library

bazel/envoy_library.bzl

@@ -71,6 +71,14 @@ def envoy_cc_extension(
         visibility = visibility,
     )
 
+def envoy_cc_contrib_extension(
+        name,
+        tags = [],
+        extra_visibility = [],
+        visibility = ["//:contrib_library"],
+        **kwargs):
+    envoy_cc_extension(name, tags, extra_visibility, visibility, **kwargs)
+
 # Envoy C++ library targets should be specified with this function.
 def envoy_cc_library(
         name,

ci/Dockerfile-envoy

@@ -13,11 +13,12 @@ RUN apt-get update && apt-get upgrade -y \
 
 RUN mkdir -p /etc/envoy
 
+ARG ENVOY_BINARY=envoy
 ARG ENVOY_BINARY_SUFFIX=_stripped
-ADD ${TARGETPLATFORM}/build_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/
+ADD ${TARGETPLATFORM}/build_${ENVOY_BINARY}_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/
 ADD configs/envoyproxy_io_proxy.yaml /etc/envoy/envoy.yaml
 
-ADD ${TARGETPLATFORM}/build_release/su-exec /usr/local/bin/
+ADD ${TARGETPLATFORM}/build_${ENVOY_BINARY}_release/su-exec /usr/local/bin/
 RUN chown root:root /usr/local/bin/su-exec && adduser --group --system envoy
 
 EXPOSE 10000

ci/Dockerfile-envoy-alpine

@@ -6,7 +6,7 @@ RUN apk add --no-cache shadow su-exec \
         && addgroup -S envoy && adduser --no-create-home -S envoy -G envoy
 
 ARG ENVOY_BINARY_SUFFIX=_stripped
-ADD linux/amd64/build_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/
+ADD linux/amd64/build_envoy_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/
 
 EXPOSE 10000
 

ci/Dockerfile-envoy-distroless

@@ -3,7 +3,7 @@ FROM gcr.io/distroless/base-debian10:nonroot
 ADD configs/envoyproxy_io_proxy.yaml /etc/envoy/envoy.yaml
 
 ARG ENVOY_BINARY_SUFFIX=_stripped
-ADD linux/amd64/build_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/
+ADD linux/amd64/build_envoy_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/
 
 EXPOSE 10000
 

ci/build_setup.sh

@@ -10,6 +10,8 @@ export PPROF_PATH=/thirdparty_build/bin/pprof
 [ -z "${ENVOY_SRCDIR}" ] && export ENVOY_SRCDIR=/source
 [ -z "${ENVOY_BUILD_TARGET}" ] && export ENVOY_BUILD_TARGET=//source/exe:envoy-static
 [ -z "${ENVOY_BUILD_DEBUG_INFORMATION}" ] && export ENVOY_BUILD_DEBUG_INFORMATION=//source/exe:envoy-static.dwp
+[ -z "${ENVOY_CONTRIB_BUILD_TARGET}" ] && export ENVOY_CONTRIB_BUILD_TARGET=//contrib/exe:envoy-static
+[ -z "${ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION}" ] && export ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION=//contrib/exe:envoy-static.dwp
 [ -z "${ENVOY_BUILD_ARCH}" ] && {
     ENVOY_BUILD_ARCH=$(uname -m)
     export ENVOY_BUILD_ARCH