Make Alert Manager API Concurrency configurable

Signed-off-by: Emmanuel Lodovice <lodovice@amazon.com>

Author: emanlodovice

CHANGELOG.md

@@ -16,6 +16,7 @@
 * [FEATURE] Query Frontend: Add `cortex_rejected_queries_total` metric for throttled queries. #5356
 * [FEATURE] Querier: Log query stats when querying store gateway. #5376
 * [FEATURE] Querier/StoreGateway: Allow the tenant shard sizes to be a percent of total instances. #5393
+* [FEATURE] Added the flag `-alertmanager.api-concurrency` to configure alert manager api concurrency limit. #5412
 * [ENHANCEMENT] Distributor/Ingester: Add span on push path #5319
 * [ENHANCEMENT] Support object storage backends for runtime configuration file. #5292
 * [ENHANCEMENT] Query Frontend: Reject subquery with too small step size. #5323

docs/configuration/config-file-reference.md

@@ -416,6 +416,10 @@ cluster:
 # CLI flag: -experimental.alertmanager.enable-api
 [enable_api: <boolean> | default = false]
 
+# Maximum number of concurrent GET API requests before returning 503.
+# CLI flag: -alertmanager.api-concurrency
+[api_concurrency: <int> | default = 0]
+
 alertmanager_client:
   # Timeout for downstream alertmanagers.
   # CLI flag: -alertmanager.alertmanager-client.remote-timeout

pkg/alertmanager/alertmanager.go

@@ -80,6 +80,7 @@ type Config struct {
 	Replicator        Replicator
 	Store             alertstore.AlertStore
 	PersisterConfig   PersisterConfig
+	APIConcurrency    int
 }
 
 // An Alertmanager manages the alerts for one user.
@@ -266,6 +267,7 @@ func New(cfg *Config, reg *prometheus.Registry) (*Alertmanager, error) {
 		GroupFunc: func(f1 func(*dispatch.Route) bool, f2 func(*types.Alert, time.Time) bool) (dispatch.AlertGroups, map[model.Fingerprint][]string) {
 			return am.dispatcher.Groups(f1, f2)
 		},
+		Concurrency: am.cfg.APIConcurrency,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to create api: %v", err)

pkg/alertmanager/multitenant.go

@@ -78,7 +78,8 @@ type MultitenantAlertmanagerConfig struct {
 
 	Cluster ClusterConfig `yaml:"cluster"`
 
-	EnableAPI bool `yaml:"enable_api"`
+	EnableAPI      bool `yaml:"enable_api"`
+	APIConcurrency int  `yaml:"api_concurrency"`
 
 	// For distributor.
 	AlertmanagerClient ClientConfig `yaml:"alertmanager_client"`
@@ -117,6 +118,7 @@ func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) {
 	f.DurationVar(&cfg.PollInterval, "alertmanager.configs.poll-interval", 15*time.Second, "How frequently to poll Cortex configs")
 
 	f.BoolVar(&cfg.EnableAPI, "experimental.alertmanager.enable-api", false, "Enable the experimental alertmanager config api.")
+	f.IntVar(&cfg.APIConcurrency, "alertmanager.api-concurrency", 0, "Maximum number of concurrent GET API requests before returning 503.")
 
 	f.BoolVar(&cfg.ShardingEnabled, "alertmanager.sharding-enabled", false, "Shard tenants across multiple alertmanager instances.")
 	f.Var(&cfg.EnabledTenants, "alertmanager.enabled-tenants", "Comma separated list of tenants whose alerts this alertmanager can process. If specified, only these tenants will be handled by alertmanager, otherwise this alertmanager can process alerts from all tenants.")
@@ -965,6 +967,7 @@ func (am *MultitenantAlertmanager) newAlertmanager(userID string, amConfig *amco
 		Store:             am.store,
 		PersisterConfig:   am.cfg.Persister,
 		Limits:            am.limits,
+		APIConcurrency:    am.cfg.APIConcurrency,
 	}, reg)
 	if err != nil {
 		return nil, fmt.Errorf("unable to start Alertmanager for user %v: %v", userID, err)