Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions cmd/galactic-gateway/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ func runCmd(cfg *config.GatewayConfig) error {
NodeName: nodeName,
SRv6Address: cfg.SRv6Address,
EgressAddress: cfg.EgressAddress,
EgressSID: cfg.EgressSID,
}).SetupWithManager(mgr); err != nil {
return fmt.Errorf("setup NetworkGateway controller: %w", err)
}
Expand All @@ -153,6 +154,17 @@ func runCmd(cfg *config.GatewayConfig) error {
return fmt.Errorf("setup NetworkRule controller: %w", err)
}

// Register NetworkEgressPolicy controller (one-time gateway-node
// assignment; see internal/controller/networkegresspolicy_controller.go,
// datum-cloud/enhancements#865).
if err := (&controller.NetworkEgressPolicyReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
NodeName: nodeName,
}).SetupWithManager(mgr); err != nil {
return fmt.Errorf("setup NetworkEgressPolicy controller: %w", err)
}

if err := mgr.Start(ctx); err != nil {
return fmt.Errorf("manager exited: %w", err)
}
Expand Down
57 changes: 57 additions & 0 deletions cmd/galactic-router/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,14 @@ func runCmd(cfg *config.RouterConfig) error {
}
}()

// Register EgressRoute controller and start its ticker goroutine
// (datum-cloud/enhancements#865, design plan §4.4/§7.1) -- extracted
// to its own function to keep this function's own cyclomatic
// complexity down (gocyclo), not for any reuse reason.
if err := startEgressRouteController(ctx, mgr, cfg, nodeName); err != nil {
return err
}

if err := mgr.Start(ctx); err != nil {
return fmt.Errorf("manager exited: %w", err)
}
Expand All @@ -296,6 +304,52 @@ func runCmd(cfg *config.RouterConfig) error {
return nil
}

// startEgressRouteController registers controller.EgressRouteReconciler and
// starts its ticker goroutine, reconciling tenant VRF ::/0 default routes
// toward each VPC's assigned gateway egress_sid (datum-cloud/
// enhancements#865, design plan §4.4/§7.1). Mirrors the GC ticker in
// runCmd exactly (including waiting for cache sync first, so the initial
// pass doesn't see an empty NetworkEgressPolicy list and remove every live
// egress route) -- reuses cfg.GCNamespace (both scan the same namespace's
// CRDs) but its own separate interval, see DefaultRouterEgressRouteInterval's
// doc comment for why.
func startEgressRouteController(
ctx context.Context, mgr ctrl.Manager, cfg *config.RouterConfig, nodeName string,
) error {
rec := &controller.EgressRouteReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Namespace: cfg.GCNamespace,
NodeName: nodeName,
Interval: cfg.EgressRouteInterval,
}
if err := rec.SetupWithManager(mgr); err != nil {
return fmt.Errorf("setup EgressRoute controller: %w", err)
}

go func() {
ticker := time.NewTicker(cfg.EgressRouteInterval)
defer ticker.Stop()

if !mgr.GetCache().WaitForCacheSync(ctx) {
log.Printf("EgressRoute: cache sync failed, skipping initial pass")
return
}
rec.RunOnce(ctx)

for {
select {
case <-ctx.Done():
return
case <-ticker.C:
rec.RunOnce(ctx)
}
}
}()
Comment on lines +330 to +348

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why's this use a separate ticker process for reconciling things instead of leveraging the normal controller runtime behavior?


return nil
}

// newRootCommand builds the root cobra command with all flags and the
// application startup logic.
func newRootCommand() *cobra.Command {
Expand Down Expand Up @@ -344,6 +398,9 @@ func newRootCommand() *cobra.Command {
cmd.Flags().DurationP("gc-interval", "",
config.DefaultRouterGCInterval,
"Cleanup interval")
cmd.Flags().DurationP("egress-route-interval", "",
config.DefaultRouterEgressRouteInterval,
"Egress default-route reconcile interval (datum-cloud/enhancements#865)")
Comment on lines +401 to +403

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The link to an enhancement here is a little odd.

cmd.Flags().Bool("webhook-enabled", false,
"Enable the NetworkRule admission webhook (requires TLS cert material; see config/webhook/)")
cmd.Flags().IntP("webhook-port", "",
Expand Down
35 changes: 26 additions & 9 deletions internal/config/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,14 @@ const (
DefaultRouterGCNamespace = "galactic-system"
DefaultRouterGCInterval = 5 * time.Minute

// DefaultRouterEgressRouteInterval is the default reconcile period for
// EgressRouteReconciler (datum-cloud/enhancements#865, design plan
// §4.4/§7.1), a separate tunable from DefaultRouterGCInterval despite
// sharing a default value: GC cleans up stale kernel/CRD state, this
// installs/removes live egress default routes -- conceptually distinct
// concerns that operators may want to tune independently.
DefaultRouterEgressRouteInterval = 5 * time.Minute

// DefaultRouterWebhookPort matches sigs.k8s.io/controller-runtime/pkg/webhook's
// own DefaultPort, named here so callers don't need that import just to
// read the default.
Expand All @@ -41,6 +49,11 @@ const (
EnvRouterGCNamespace = "GALACTIC_ROUTER_GC_NAMESPACE"
EnvRouterGCInterval = "GALACTIC_ROUTER_GC_INTERVAL"

// EnvRouterEgressRouteInterval configures EgressRouteReconciler's
// reconcile period -- see DefaultRouterEgressRouteInterval's doc
// comment for why this is a separate knob from EnvRouterGCInterval.
EnvRouterEgressRouteInterval = "GALACTIC_ROUTER_EGRESS_ROUTE_INTERVAL"

// EnvRouterWebhookEnabled gates the NetworkRule admission webhook
// (internal/webhook). Defaults to false: this is the first webhook in
// this codebase, and enabling it requires TLS cert material
Expand Down Expand Up @@ -72,15 +85,16 @@ type RouterConfig struct {
prefix string

// Resolved fields.
NodeName string
Mode string
Reflector bool
BGPListenPort int
BGPLocalAddr string
MetricsPort int
GRPCHealthPort int
GCNamespace string
GCInterval time.Duration
NodeName string
Mode string
Reflector bool
BGPListenPort int
BGPLocalAddr string
MetricsPort int
GRPCHealthPort int
GCNamespace string
GCInterval time.Duration
EgressRouteInterval time.Duration

// WebhookEnabled/WebhookPort/WebhookCertDir configure the NetworkRule
// admission webhook (internal/webhook) -- see
Expand All @@ -107,6 +121,7 @@ func NewRouterConfig() *RouterConfig {
v.SetDefault("grpc_health_port", DefaultRouterGRPCHealthPort)
v.SetDefault("gc_namespace", DefaultRouterGCNamespace)
v.SetDefault("gc_interval", DefaultRouterGCInterval.String())
v.SetDefault("egress_route_interval", DefaultRouterEgressRouteInterval.String())
v.SetDefault("webhook_enabled", false)
v.SetDefault("webhook_port", DefaultRouterWebhookPort)
v.SetDefault("webhook_cert_dir", "")
Expand Down Expand Up @@ -135,6 +150,7 @@ func (c *RouterConfig) BindFlags(flags *pflag.FlagSet) {
{"grpc-health-port", "grpc_health_port"},
{"gc-namespace", "gc_namespace"},
{"gc-interval", "gc_interval"},
{"egress-route-interval", "egress_route_interval"},
{"webhook-enabled", "webhook_enabled"},
{"webhook-port", "webhook_port"},
{"webhook-cert-dir", "webhook_cert_dir"},
Expand All @@ -161,6 +177,7 @@ func (c *RouterConfig) readFields() {
c.GRPCHealthPort = c.v.GetInt("grpc_health_port")
c.GCNamespace = c.v.GetString("gc_namespace")
c.GCInterval = c.v.GetDuration("gc_interval")
c.EgressRouteInterval = c.v.GetDuration("egress_route_interval")
c.WebhookEnabled = c.v.GetBool("webhook_enabled")
c.WebhookPort = c.v.GetInt("webhook_port")
c.WebhookCertDir = c.v.GetString("webhook_cert_dir")
Expand Down
7 changes: 7 additions & 0 deletions internal/config/router_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ func TestRouterConfigDefaults(t *testing.T) {
if cfg.GCInterval != DefaultRouterGCInterval {
t.Errorf("GCInterval = %v, want %v", cfg.GCInterval, DefaultRouterGCInterval)
}
if cfg.EgressRouteInterval != DefaultRouterEgressRouteInterval {
t.Errorf("EgressRouteInterval = %v, want %v", cfg.EgressRouteInterval, DefaultRouterEgressRouteInterval)
}
if cfg.Reflector {
t.Error("Reflector = true, want false")
}
Expand All @@ -62,6 +65,7 @@ func TestRouterConfigEnvOverride(t *testing.T) {
t.Setenv(EnvRouterGRPCHealthPort, "5179")
t.Setenv(EnvRouterGCNamespace, "custom-ns")
t.Setenv(EnvRouterGCInterval, "10m")
t.Setenv(EnvRouterEgressRouteInterval, "15m")
t.Setenv(EnvRouterWebhookEnabled, testBoolTrue)
t.Setenv(EnvRouterWebhookPort, "9444")
t.Setenv(EnvRouterWebhookCertDir, "/tmp/certs")
Expand Down Expand Up @@ -95,6 +99,9 @@ func TestRouterConfigEnvOverride(t *testing.T) {
if cfg.GCInterval != 10*time.Minute {
t.Errorf("GCInterval = %v, want 10m", cfg.GCInterval)
}
if cfg.EgressRouteInterval != 15*time.Minute {
t.Errorf("EgressRouteInterval = %v, want 15m", cfg.EgressRouteInterval)
}
if !cfg.WebhookEnabled {
t.Error("WebhookEnabled = false, want true")
}
Expand Down
90 changes: 90 additions & 0 deletions internal/controller/egressroute_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright 2026 Datum Cloud, Inc.
//
// SPDX-License-Identifier: AGPL-3.0-or-later

package controller

import (
"context"
"log/slog"
"time"

"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

"go.datum.net/galactic/internal/egressroute"
)

// EgressRouteReconciler runs periodic egress default-route reconciliation
// on this compute node (datum-cloud/enhancements#865, design plan
// §4.4/§7.1) — the ticker-driven wrapper around internal/egressroute.Run,
// mirroring GCReconciler's own split between a thin, time-driven
// controller here and the real logic in a sibling non-controller package
// (internal/gc). Unlike NetworkGatewayReconciler/NetworkEgressPolicyReconciler
// (gateway-node-scoped, registered from cmd/galactic-gateway), this runs
// from cmd/galactic-router's tenant-role process: it needs to see every
// compute node's own local VRF state, not just gateway nodes', and
// galactic-router (tenant role) already runs on every node that could have
// one.
//
// Time-driven, not object-driven, for the same reason GCReconciler is: a
// newly-created local VRF interface is pure kernel state with no
// corresponding Kubernetes watch event to react to, so this can only ever
// notice it on a periodic sweep, not a reactive one.
type EgressRouteReconciler struct {
client.Client
Scheme *runtime.Scheme
Namespace string
NodeName string
Interval time.Duration
}

// slogAdapter adapts log/slog's package-level functions to
// egressroute.Logger, matching GCReconciler's own choice of slog over
// logr.Logger for this package's plain log lines (see gc_controller.go's
// identical use of slog.Info/slog.Error at its own call sites).
type slogAdapter struct{}

func (slogAdapter) Info(msg string, keysAndValues ...any) { slog.Info(msg, keysAndValues...) }
func (slogAdapter) Error(err error, msg string, keysAndValues ...any) {
slog.Error(msg, append([]any{"err", err}, keysAndValues...)...)
}

// Reconcile runs an egress-route pass at the configured interval. It does
// not watch any Kubernetes resources — it is purely time-driven, same as
// GCReconciler.Reconcile.
func (r *EgressRouteReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) {
if r.Namespace == "" {
slog.Debug("EgressRoute: namespace not configured, skipping")
return ctrl.Result{RequeueAfter: r.Interval}, nil
}

result := r.RunOnce(ctx)
if result.Errors > 0 {
slog.Info("EgressRoute: completed with errors",
"routesInstalled", result.RoutesInstalled, "routesRemoved", result.RoutesRemoved, "errors", result.Errors)
} else if result.RoutesInstalled > 0 || result.RoutesRemoved > 0 {
slog.Info("EgressRoute: reconcile complete",
"routesInstalled", result.RoutesInstalled, "routesRemoved", result.RoutesRemoved)
}

return ctrl.Result{RequeueAfter: r.Interval}, nil
}

// SetupWithManager registers the EgressRouteReconciler with the manager.
// Like GCReconciler, it is started by a ticker goroutine launched from
// root.go where the manager's context is available.
func (r *EgressRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
if r.Interval == 0 {
r.Interval = 5 * time.Minute
}
return nil
}

// RunOnce runs a single egress-route reconcile pass in the given context.
// This is the public API for triggering it from outside the reconciler
// (e.g. root.go's ticker goroutine), mirroring GCReconciler.RunGC.
func (r *EgressRouteReconciler) RunOnce(ctx context.Context) egressroute.Result {
return egressroute.Run(ctx, r.Client, r.Namespace, r.NodeName, slogAdapter{})
}
Loading