Skip to content

Commit

Permalink
Translate RateLimitInfra into K8s resources (#939)
Browse files Browse the repository at this point in the history 
* Translate RateLimitInfra into K8s resources

Relates to #670

Signed-off-by: Arko Dasgupta <arko@tetrate.io>
  • Loading branch information
@arkodg
arkodg authored Jan 31, 2023
1 parent 2d7e8c1 commit 6f8b6fb
Show file tree
Hide file tree
Showing 16 changed files with 540 additions and 165 deletions.
23 changes: 13 additions & 10 deletions internal/cmd/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,16 +155,19 @@ func setupRunners(cfg *config.Server) error {
return err
}

// Start the Global RateLimit Runner
// It subscribes to the xds Resources and translates it to Envoy Ratelimit Service
// infrastructure and configuration.
rateLimitRunner := ratelimitrunner.New(&ratelimitrunner.Config{
Server: *cfg,
XdsIR: xdsIR,
RateLimitInfraIR: rateLimitInfraIR,
})
if err := rateLimitRunner.Start(ctx); err != nil {
return err
// Start the global rateLimit runner if it has been enabled through the config
if cfg.EnvoyGateway.RateLimit != nil {
// Start the Global RateLimit Runner
// It subscribes to the xds Resources and translates it to Envoy Ratelimit Service
// infrastructure and configuration.
rateLimitRunner := ratelimitrunner.New(&ratelimitrunner.Config{
Server: *cfg,
XdsIR: xdsIR,
RateLimitInfraIR: rateLimitInfraIR,
})
if err := rateLimitRunner.Start(ctx); err != nil {
return err
}
}

// Wait until done
Expand Down
5 changes: 0 additions & 5 deletions internal/globalratelimit/runner/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,6 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) {
func (r *Runner) translate(xdsIRs []*ir.Xds) (*ir.RateLimitInfra, error) {
rlInfra := new(ir.RateLimitInfra)

// Return empty IR if ratelimit has not been enabled yet in the EnvoyGateway API
if r.EnvoyGateway.RateLimit == nil {
return nil, nil
}

for _, xdsIR := range xdsIRs {
for _, listener := range xdsIR.HTTP {
config := translator.BuildRateLimitServiceConfig(listener)
Expand Down
98 changes: 72 additions & 26 deletions internal/infrastructure/kubernetes/configmap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ var (
`"private_key":{"filename":"%s"}}}]}`, xdsTLSCertFilename, xdsTLSKeyFilename)
)

// expectedConfigMap returns the expected ConfigMap based on the provided infra.
func (i *Infra) expectedConfigMap(infra *ir.Infra) (*corev1.ConfigMap, error) {
// expectedProxyConfigMap returns the expected ConfigMap based on the provided infra.
func (i *Infra) expectedProxyConfigMap(infra *ir.Infra) (*corev1.ConfigMap, error) {
// Set the labels based on the owning gateway name.
labels := envoyLabels(infra.GetProxyInfra().GetProxyMetadata().Labels)
if len(labels[gatewayapi.OwningGatewayNamespaceLabel]) == 0 || len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 {
Expand All @@ -56,7 +56,7 @@ func (i *Infra) expectedConfigMap(infra *ir.Infra) (*corev1.ConfigMap, error) {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: i.Namespace,
Name: expectedConfigMapName(infra.Proxy.Name),
Name: expectedProxyConfigMapName(infra.Proxy.Name),
Labels: labels,
},
Data: map[string]string{
Expand All @@ -66,48 +66,99 @@ func (i *Infra) expectedConfigMap(infra *ir.Infra) (*corev1.ConfigMap, error) {
}, nil
}

// createOrUpdateConfigMap creates a ConfigMap in the Kube api server based on the provided
// createOrUpdateProxyConfigMap creates a ConfigMap in the Kube api server based on the provided
// infra, if it doesn't exist and updates it if it does.
func (i *Infra) createOrUpdateConfigMap(ctx context.Context, infra *ir.Infra) (*corev1.ConfigMap, error) {
cm, err := i.expectedConfigMap(infra)
func (i *Infra) createOrUpdateProxyConfigMap(ctx context.Context, infra *ir.Infra) error {
cm, err := i.expectedProxyConfigMap(infra)
if err != nil {
return nil, err
return err
}

return i.createOrUpdateConfigMap(ctx, cm)
}

// deleteProxyConfigMap deletes the Envoy ConfigMap in the kube api server, if it exists.
func (i *Infra) deleteProxyConfigMap(ctx context.Context, infra *ir.Infra) error {
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: i.Namespace,
Name: expectedProxyConfigMapName(infra.Proxy.Name),
},
}

return i.deleteConfigMap(ctx, cm)
}

func expectedProxyConfigMapName(proxyName string) string {
cMapName := utils.GetHashedName(proxyName)
return fmt.Sprintf("%s-%s", config.EnvoyPrefix, cMapName)
}

// expectedRateLimitConfigMap returns the expected ConfigMap based on the provided infra.
func (i *Infra) expectedRateLimitConfigMap(infra *ir.RateLimitInfra) *corev1.ConfigMap {
labels := rateLimitLabels()
data := make(map[string]string)

for _, config := range infra.Configs {
data[config.Name] = config.Config
}

return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: i.Namespace,
Name: rateLimitInfraName,
Labels: labels,
},
Data: data,
}
}

// createOrUpdateRateLimitConfigMap creates a ConfigMap in the Kube api server based on the provided
// infra, if it doesn't exist and updates it if it does.
func (i *Infra) createOrUpdateRateLimitConfigMap(ctx context.Context, infra *ir.RateLimitInfra) error {
cm := i.expectedRateLimitConfigMap(infra)
return i.createOrUpdateConfigMap(ctx, cm)
}

// deleteProxyConfigMap deletes the Envoy ConfigMap in the kube api server, if it exists.
func (i *Infra) deleteRateLimitConfigMap(ctx context.Context, _ *ir.RateLimitInfra) error {
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: i.Namespace,
Name: rateLimitInfraName,
},
}

return i.deleteConfigMap(ctx, cm)
}

func (i *Infra) createOrUpdateConfigMap(ctx context.Context, cm *corev1.ConfigMap) error {
current := &corev1.ConfigMap{}
key := types.NamespacedName{
Namespace: i.Namespace,
Name: expectedConfigMapName(infra.Proxy.Name),
Namespace: cm.Namespace,
Name: cm.Name,
}

if err := i.Client.Get(ctx, key, current); err != nil {
// Create if not found.
if kerrors.IsNotFound(err) {
if err := i.Client.Create(ctx, cm); err != nil {
return nil, fmt.Errorf("failed to create configmap %s/%s: %w", cm.Namespace, cm.Name, err)
return fmt.Errorf("failed to create configmap %s/%s: %w", cm.Namespace, cm.Name, err)
}
}
} else {
// Update if current value is different.
if !reflect.DeepEqual(cm.Data, current.Data) {
if err := i.Client.Update(ctx, cm); err != nil {
return nil, fmt.Errorf("failed to update configmap %s/%s: %w", cm.Namespace, cm.Name, err)
return fmt.Errorf("failed to update configmap %s/%s: %w", cm.Namespace, cm.Name, err)
}
}
}

return cm, nil
return nil
}

// deleteConfigMap deletes the Envoy ConfigMap in the kube api server, if it exists.
func (i *Infra) deleteConfigMap(ctx context.Context, infra *ir.Infra) error {
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: i.Namespace,
Name: expectedConfigMapName(infra.Proxy.Name),
},
}

func (i *Infra) deleteConfigMap(ctx context.Context, cm *corev1.ConfigMap) error {
if err := i.Client.Delete(ctx, cm); err != nil {
if kerrors.IsNotFound(err) {
return nil
Expand All @@ -117,8 +168,3 @@ func (i *Infra) deleteConfigMap(ctx context.Context, infra *ir.Infra) error {

return nil
}

func expectedConfigMapName(proxyName string) string {
cMapName := utils.GetHashedName(proxyName)
return fmt.Sprintf("%s-%s", config.EnvoyPrefix, cMapName)
}
28 changes: 17 additions & 11 deletions internal/infrastructure/kubernetes/configmap_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
corev1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"

"github.com/envoyproxy/gateway/internal/envoygateway"
Expand All @@ -22,7 +23,7 @@ import (
"github.com/envoyproxy/gateway/internal/ir"
)

func TestExpectedConfigMap(t *testing.T) {
func TestExpectedProxyConfigMap(t *testing.T) {
// Setup the infra.
cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build()
cfg, err := config.New()
Expand All @@ -35,13 +36,13 @@ func TestExpectedConfigMap(t *testing.T) {

// An infra without Gateway owner labels should trigger
// an error.
_, err = kube.expectedConfigMap(infra)
_, err = kube.expectedProxyConfigMap(infra)
require.NotNil(t, err)

infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default"
infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name

cm, err := kube.expectedConfigMap(infra)
cm, err := kube.expectedProxyConfigMap(infra)
require.NoError(t, err)

require.Equal(t, "envoy-test-74657374", cm.Name)
Expand All @@ -57,7 +58,7 @@ func TestExpectedConfigMap(t *testing.T) {
assert.True(t, apiequality.Semantic.DeepEqual(wantLabels, cm.Labels))
}

func TestCreateOrUpdateConfigMap(t *testing.T) {
func TestCreateOrUpdateProxyConfigMap(t *testing.T) {
cfg, err := config.New()
require.NoError(t, err)
kube := NewInfra(nil, cfg)
Expand Down Expand Up @@ -123,17 +124,22 @@ func TestCreateOrUpdateConfigMap(t *testing.T) {
} else {
kube.Client = fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).Build()
}
cm, err := kube.createOrUpdateConfigMap(context.Background(), infra)
err := kube.createOrUpdateProxyConfigMap(context.Background(), infra)
require.NoError(t, err)
require.Equal(t, tc.expect.Namespace, cm.Namespace)
require.Equal(t, tc.expect.Name, cm.Name)
assert.True(t, apiequality.Semantic.DeepEqual(tc.expect.Labels, cm.Labels))
require.Equal(t, tc.expect.Data, cm.Data)
actual := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: tc.expect.Namespace,
Name: tc.expect.Name,
},
}
require.NoError(t, kube.Client.Get(context.Background(), client.ObjectKeyFromObject(actual), actual))
require.Equal(t, tc.expect.Data, actual.Data)
assert.True(t, apiequality.Semantic.DeepEqual(tc.expect.Labels, actual.Labels))
})
}
}

func TestDeleteConfigMap(t *testing.T) {
func TestDeleteConfigProxyMap(t *testing.T) {
cfg, err := config.New()
require.NoError(t, err)

Expand Down Expand Up @@ -173,7 +179,7 @@ func TestDeleteConfigMap(t *testing.T) {
t.Parallel()
cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects(tc.current).Build()
kube := NewInfra(cli, cfg)
err := kube.deleteConfigMap(context.Background(), infra)
err := kube.deleteProxyConfigMap(context.Background(), infra)
require.NoError(t, err)
})
}
Expand Down
Loading

0 comments on commit 6f8b6fb

Please sign in to comment.