[WPB-8764] Backport changes from q1-2024 to develop (Apr 23, 2024)

Makefile

@@ -7,7 +7,7 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed on internal kubernetes testing environments
-CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn k8ssandra-test-cluster
+CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana restund coturn k8ssandra-test-cluster
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)
@@ -18,7 +18,7 @@ fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\
 nginx-ingress-controller ingress-nginx-controller nginx-ingress-services reaper sftd restund coturn		\
-inbucket k8ssandra-test-cluster postgresql ldap-scim-bridge
+inbucket k8ssandra-test-cluster postgresql ldap-scim-bridge smallstep-accomp
 KIND_CLUSTER_NAME     := wire-server
 HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for all-parallel tests
 

changelog.d/2-features/WPB-6252

@@ -0,0 +1 @@
+Add Helm chart `smallstep-accomp` that provides a CRL endpoint proxy for federated E2EI

changelog.d/2-features/WPB-7025

@@ -0,0 +1 @@
+Remove Helm migrated charts webapp, team-settings, account-pages, sftd

changelog.d/2-features/claim-key-packages-rate-limit

@@ -0,0 +1 @@
+charts/nginz: Rate limiting claiming MLS key-pacakges by requesting and target user

changelog.d/2-features/one2one-rate-limit

@@ -0,0 +1 @@
+charts/nginz: Allow 3000 reqs/min on /conversations/one2one/:user_domain/:user

changelog.d/5-internal/smallstep-accomp-no-client-side-caching

@@ -0,0 +1,2 @@
+To ensure certificate revocations get active in a short time frame, disable
+caching of proxy results on client side by setting respective HTTP headers.

charts/elasticsearch-index/templates/migrate-data.yaml

@@ -22,8 +22,7 @@ spec:
     spec:
       restartPolicy: OnFailure
       containers:
-        # Creates index in elasticsearch only when it doesn't exist.
-        # Does nothing if the index exists.
+        # Reindexes all users when a new migration is detected.
         - name: brig-index
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }}

charts/nginz/templates/conf/_nginx.conf.tpl

@@ -125,6 +125,11 @@ http {
       0 "";
   }
 
+  map $rate_limit $rate_limited_by_zuser_path {
+      1 "$zauth_user$uri";
+      0 "";
+  }
+
   map $http_origin $cors_header {
       default "";
     {{ range $origin := .Values.nginx_conf.allowlisted_origins }}
@@ -275,6 +280,10 @@ http {
         limit_req zone={{ $location.specific_user_rate_limit }}{{ if hasKey $location "specific_user_rate_limit_burst" }} burst={{ $location.specific_user_rate_limit_burst }}{{ end }} nodelay;
             {{- end }}
 
+            {{- range $specific_limit := $location.specific_rate_limits }}
+        limit_req zone={{ $specific_limit.zone }}{{ if hasKey $specific_limit "burst" }} burst={{ $specific_limit.burst }}{{ end }} nodelay;
+            {{- end }}
+
         if ($request_method = 'OPTIONS') {
             add_header 'Access-Control-Allow-Methods' "GET, POST, PUT, DELETE, OPTIONS";
             add_header 'Access-Control-Allow-Headers' "$http_access_control_request_headers, DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type";

charts/nginz/values.yaml

@@ -67,6 +67,9 @@ nginx_conf:
   user_rate_limit_request_zones:
     - limit_req_zone $rate_limited_by_addr zone=reqs_per_addr_sso:12m rate=50r/s;
     - limit_req_zone $rate_limited_by_zuser zone=reqs_per_user_signatures:12m rate=10r/m;
+    - limit_req_zone $rate_limited_by_zuser zone=key_package_claims_per_user:12m rate=3000r/m;
+    - limit_req_zone $rate_limited_by_zuser_path zone=key_package_claims_per_user_per_target:12m rate=100r/m;
+    - limit_req_zone $rate_limited_by_zuser zone=one2one_conv:12m rate=3000r/m;
 
   # The origins from which we allow CORS requests. These are combined with
   # 'external_env_domain' and 'additional_external_env_domains' to form a full
@@ -209,6 +212,20 @@ nginx_conf:
     - path: /clients
       envs:
       - all
+    - path: /mls/key-packages/claim
+      envs:
+      - all
+      specific_rate_limits:
+      # This endpoint gets called for every user that will get added to a
+      # conversation. A lot of these are expected during conversation
+      # creation.
+      - zone: key_package_claims_per_user
+        burst: 600
+      - zone: key_package_claims_per_user_per_target
+        burst: 100
+      # The name is a little misleading, this just disables default rate
+      # limiting in favour of the specific one defined above.
+      unlimited_requests_endpoint: true
     - path: /mls/key-packages
       envs:
       - all
@@ -440,6 +457,15 @@ nginx_conf:
       - all
       max_body_size: 40m
       body_buffer_size: 256k
+    - path: /conversations/one2one/
+      envs:
+      - all
+      # During MLS migration, this endpoint gets called _a lot_.
+      specific_user_rate_limit: one2one_conv
+      specific_user_rate_limit_burst: 1000
+      # The name is a little misleading, this just disables default rate
+      # limiting in favour of the specific one defined above.
+      unlimited_requests_endpoint: true
     - path: /conversations/([^/]*)/([^/]*)/protocol
       envs:
       - all