Skip to content

Commit

Permalink
fix: listener on IPv6 first cluster
Browse files Browse the repository at this point in the history
Signed-off-by: zirain <zirain2009@gmail.com>
  • Loading branch information
zirain committed Oct 31, 2024
1 parent bb3bbdb commit 30d1b00
Show file tree
Hide file tree
Showing 84 changed files with 1,733 additions and 126 deletions.
4 changes: 4 additions & 0 deletions charts/gateway-helm/templates/envoy-gateway-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ spec:
- server
- --config-path=/config/envoy-gateway.yaml
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: ENVOY_GATEWAY_NAMESPACE
valueFrom:
fieldRef:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ envoyProxyForGatewayClass:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
5 changes: 3 additions & 2 deletions internal/cmd/envoy/shutdown_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,9 @@ func Shutdown(drainTimeout time.Duration, minDrainDuration time.Duration, exitAt

// postEnvoyAdminAPI sends a POST request to the Envoy admin API
func postEnvoyAdminAPI(path string) error {
// TODO: change bootstrap.AdminAddress() to localhost because there're in the same pod?

Check warning on line 173 in internal/cmd/envoy/shutdown_manager.go

View check run for this annotation

Codecov / codecov/patch

internal/cmd/envoy/shutdown_manager.go#L173

Added line #L173 was not covered by tests
if resp, err := http.Post(fmt.Sprintf("http://%s:%d/%s",
bootstrap.EnvoyAdminAddress, bootstrap.EnvoyAdminPort, path), "application/json", nil); err != nil {
bootstrap.AdminAddress(egv1a1.IPv4), bootstrap.EnvoyAdminPort, path), "application/json", nil); err != nil {

Check warning on line 175 in internal/cmd/envoy/shutdown_manager.go

View check run for this annotation

Codecov / codecov/patch

internal/cmd/envoy/shutdown_manager.go#L175

Added line #L175 was not covered by tests
return err
} else {
defer resp.Body.Close()
Expand All @@ -187,7 +188,7 @@ func postEnvoyAdminAPI(path string) error {
func getTotalConnections() (*int, error) {
// Send request to Envoy admin API to retrieve server.total_connections stat
if resp, err := http.Get(fmt.Sprintf("http://%s:%d//stats?filter=^server\\.total_connections$&format=json",
bootstrap.EnvoyAdminAddress, bootstrap.EnvoyAdminPort)); err != nil {
bootstrap.AdminAddress(egv1a1.IPv4), bootstrap.EnvoyAdminPort)); err != nil {

Check warning on line 191 in internal/cmd/envoy/shutdown_manager.go

View check run for this annotation

Codecov / codecov/patch

internal/cmd/envoy/shutdown_manager.go#L191

Added line #L191 was not covered by tests
return nil, err
} else {
defer resp.Body.Close()
Expand Down
4 changes: 4 additions & 0 deletions internal/envoygateway/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/envoyproxy/gateway/api/v1alpha1/validation"
"github.com/envoyproxy/gateway/internal/logging"
"github.com/envoyproxy/gateway/internal/utils/env"
"github.com/envoyproxy/gateway/internal/utils/net"
)

const (
Expand All @@ -38,6 +39,8 @@ type Server struct {
Logger logging.Logger
// Elected chan is used to signal what a leader is elected
Elected chan struct{}
// IPv6First is a flag to indicate if the server should prefer IPv6 addresses.
IPv6First bool
}

// New returns a Server with default parameters.
Expand All @@ -46,6 +49,7 @@ func New() (*Server, error) {
EnvoyGateway: egv1a1.DefaultEnvoyGateway(),
Namespace: env.Lookup("ENVOY_GATEWAY_NAMESPACE", DefaultNamespace),
DNSDomain: env.Lookup("KUBERNETES_CLUSTER_DOMAIN", DefaultDNSDomain),
IPv6First: net.IsIPv6FirstPod(),
// the default logger
Logger: logging.DefaultLogger(egv1a1.LogLevelInfo),
Elected: make(chan struct{}),
Expand Down
18 changes: 15 additions & 3 deletions internal/gatewayapi/listener.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@ import (
"github.com/envoyproxy/gateway/internal/ir"
"github.com/envoyproxy/gateway/internal/utils"
"github.com/envoyproxy/gateway/internal/utils/naming"
"github.com/envoyproxy/gateway/internal/utils/net"
)

const (
ipv4ListenerAddress = "0.0.0.0"
ipv6ListenerAddress = "::1"
)

var _ ListenersTranslator = (*Translator)(nil)
Expand Down Expand Up @@ -99,6 +105,12 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR resource
if !isReady {
continue
}

address := ipv4ListenerAddress
if net.PreferIPFamily(t.IPv6First, gateway.envoyProxy) == egv1a1.IPv6 {
address = ipv6ListenerAddress
}

Check warning on line 112 in internal/gatewayapi/listener.go

View check run for this annotation

Codecov / codecov/patch

internal/gatewayapi/listener.go#L111-L112

Added lines #L111 - L112 were not covered by tests

// Add the listener to the Xds IR
servicePort := &protocolPort{protocol: listener.Protocol, port: int32(listener.Port)}
containerPort := servicePortToContainerPort(int32(listener.Port), gateway.envoyProxy)
Expand All @@ -107,7 +119,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR resource
irListener := &ir.HTTPListener{
CoreListenerDetails: ir.CoreListenerDetails{
Name: irListenerName(listener),
Address: "0.0.0.0",
Address: address,
Port: uint32(containerPort),
Metadata: buildListenerMetadata(listener, gateway),
IPFamily: getIPFamily(gateway.envoyProxy),
Expand All @@ -134,7 +146,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR resource
irListener := &ir.TCPListener{
CoreListenerDetails: ir.CoreListenerDetails{
Name: irListenerName(listener),
Address: "0.0.0.0",
Address: address,
Port: uint32(containerPort),
IPFamily: getIPFamily(gateway.envoyProxy),
},
Expand All @@ -150,7 +162,7 @@ func (t *Translator) ProcessListeners(gateways []*GatewayContext, xdsIR resource
irListener := &ir.UDPListener{
CoreListenerDetails: ir.CoreListenerDetails{
Name: irListenerName(listener),
Address: "0.0.0.0",
Address: address,
Port: uint32(containerPort),
},
}
Expand Down
1 change: 1 addition & 0 deletions internal/gatewayapi/runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) {
Namespace: r.Namespace,
MergeGateways: gatewayapi.IsMergeGatewaysEnabled(resources),
WasmCache: r.wasmCache,
IPv6First: r.IPv6First,

Check warning on line 155 in internal/gatewayapi/runner/runner.go

View check run for this annotation

Codecov / codecov/patch

internal/gatewayapi/runner/runner.go#L155

Added line #L155 was not covered by tests
}

// If an extension is loaded, pass its supported groups/kinds to the translator
Expand Down
3 changes: 3 additions & 0 deletions internal/gatewayapi/translator.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ type Translator struct {

// WasmCache is the cache for Wasm modules.
WasmCache wasm.Cache

// IPv6First is true when IPv6 addresses should be preferred
IPv6First bool
}

type TranslateResult struct {
Expand Down
2 changes: 2 additions & 0 deletions internal/infrastructure/host/proxy_infra.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/envoyproxy/gateway/internal/infrastructure/common"
"github.com/envoyproxy/gateway/internal/ir"
"github.com/envoyproxy/gateway/internal/utils"
"github.com/envoyproxy/gateway/internal/utils/net"
"github.com/envoyproxy/gateway/internal/xds/bootstrap"
)

Expand Down Expand Up @@ -59,6 +60,7 @@ func (i *Infra) CreateOrUpdateProxyInfra(ctx context.Context, infra *ir.Infra) e
WasmServerPort: ptr.To(int32(0)),
AdminServerPort: ptr.To(int32(0)),
ReadyServerPort: ptr.To(int32(0)),
IPFamily: net.PreferIPFamily(false, proxyConfig),

Check warning on line 63 in internal/infrastructure/host/proxy_infra.go

View check run for this annotation

Codecov / codecov/patch

internal/infrastructure/host/proxy_infra.go#L63

Added line #L63 was not covered by tests
}

args, err := common.BuildProxyArgs(proxyInfra, proxyConfig.Spec.Shutdown, bootstrapConfigOptions, proxyName)
Expand Down
3 changes: 3 additions & 0 deletions internal/infrastructure/kubernetes/infra.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ type Infra struct {

// Client wrap k8s client.
Client *InfraClient

IPv6First bool
}

// NewInfra returns a new Infra.
Expand All @@ -67,6 +69,7 @@ func NewInfra(cli client.Client, cfg *config.Server) *Infra {
DNSDomain: cfg.DNSDomain,
EnvoyGateway: cfg.EnvoyGateway,
Client: New(cli),
IPv6First: cfg.IPv6First,
}
}

Expand Down
6 changes: 4 additions & 2 deletions internal/infrastructure/kubernetes/proxy/resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource"
"github.com/envoyproxy/gateway/internal/ir"
"github.com/envoyproxy/gateway/internal/utils"
"github.com/envoyproxy/gateway/internal/utils/net"
"github.com/envoyproxy/gateway/internal/xds/bootstrap"
)

Expand Down Expand Up @@ -83,8 +84,8 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
containerSpec *egv1a1.KubernetesContainerSpec,
shutdownConfig *egv1a1.ShutdownConfig,
shutdownManager *egv1a1.ShutdownManager,
namespace string,
dnsDomain string,
namespace string, dnsDomain string,
ipv6First bool,
) ([]corev1.Container, error) {
// Define slice to hold container ports
var ports []corev1.ContainerPort
Expand Down Expand Up @@ -135,6 +136,7 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
},
MaxHeapSizeBytes: maxHeapSizeBytes,
XdsServerHost: ptr.To(fmt.Sprintf("%s.%s.svc.%s", config.EnvoyGatewayServiceName, namespace, dnsDomain)),
IPFamily: net.PreferIPFamily(ipv6First, infra.Config),
}

args, err := common.BuildProxyArgs(infra, shutdownConfig, bootstrapConfigOptions, fmt.Sprintf("$(%s)", envoyPodEnvVar))
Expand Down
11 changes: 8 additions & 3 deletions internal/infrastructure/kubernetes/proxy/resource_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,17 @@ type ResourceRender struct {
DNSDomain string

ShutdownManager *egv1a1.ShutdownManager

IPv6First bool
}

func NewResourceRender(ns string, dnsDomain string, infra *ir.ProxyInfra, gateway *egv1a1.EnvoyGateway) *ResourceRender {
func NewResourceRender(ipv6First bool, ns string, dnsDomain string, infra *ir.ProxyInfra, gateway *egv1a1.EnvoyGateway) *ResourceRender {
return &ResourceRender{
Namespace: ns,
DNSDomain: dnsDomain,
infra: infra,
ShutdownManager: gateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().ShutdownManager,
IPv6First: ipv6First,
}
}

Expand Down Expand Up @@ -262,7 +265,8 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) {

proxyConfig := r.infra.GetProxyConfig()
// Get expected bootstrap configurations rendered ProxyContainers
containers, err := expectedProxyContainers(r.infra, deploymentConfig.Container, proxyConfig.Spec.Shutdown, r.ShutdownManager, r.Namespace, r.DNSDomain)
containers, err := expectedProxyContainers(r.infra, deploymentConfig.Container, proxyConfig.Spec.Shutdown,
r.ShutdownManager, r.Namespace, r.DNSDomain, r.IPv6First)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -364,7 +368,8 @@ func (r *ResourceRender) DaemonSet() (*appsv1.DaemonSet, error) {
proxyConfig := r.infra.GetProxyConfig()

// Get expected bootstrap configurations rendered ProxyContainers
containers, err := expectedProxyContainers(r.infra, daemonSetConfig.Container, proxyConfig.Spec.Shutdown, r.ShutdownManager, r.Namespace, r.DNSDomain)
containers, err := expectedProxyContainers(r.infra, daemonSetConfig.Container, proxyConfig.Spec.Shutdown,
r.ShutdownManager, r.Namespace, r.DNSDomain, r.IPv6First)
if err != nil {
return nil, err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,7 @@ func TestDeployment(t *testing.T) {
tc.infra.Proxy.Config.Spec.ExtraArgs = tc.extraArgs
}

r := NewResourceRender(cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
r := NewResourceRender(false, cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
dp, err := r.Deployment()
require.NoError(t, err)

Expand Down Expand Up @@ -993,7 +993,7 @@ func TestDaemonSet(t *testing.T) {
tc.infra.Proxy.Config.Spec.ExtraArgs = tc.extraArgs
}

r := NewResourceRender(cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
r := NewResourceRender(false, cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
ds, err := r.DaemonSet()
require.NoError(t, err)

Expand Down Expand Up @@ -1143,7 +1143,7 @@ func TestService(t *testing.T) {
provider.EnvoyService = tc.service
}

r := NewResourceRender(cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
r := NewResourceRender(false, cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
svc, err := r.Service()
require.NoError(t, err)

Expand Down Expand Up @@ -1186,7 +1186,7 @@ func TestConfigMap(t *testing.T) {

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
r := NewResourceRender(cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
r := NewResourceRender(false, cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
cm, err := r.ConfigMap()
require.NoError(t, err)

Expand Down Expand Up @@ -1229,7 +1229,7 @@ func TestServiceAccount(t *testing.T) {

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
r := NewResourceRender(cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
r := NewResourceRender(false, cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
sa, err := r.ServiceAccount()
require.NoError(t, err)

Expand Down Expand Up @@ -1285,7 +1285,7 @@ func TestPDB(t *testing.T) {

provider.GetEnvoyProxyKubeProvider()

r := NewResourceRender(cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
r := NewResourceRender(false, cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)

pdb, err := r.PodDisruptionBudget()
require.NoError(t, err)
Expand Down Expand Up @@ -1371,7 +1371,7 @@ func TestHorizontalPodAutoscaler(t *testing.T) {
}
provider.GetEnvoyProxyKubeProvider()

r := NewResourceRender(cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
r := NewResourceRender(false, cfg.Namespace, cfg.DNSDomain, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
hpa, err := r.HorizontalPodAutoscaler()
require.NoError(t, err)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ spec:
- name: envoy-gateway-proxy-ready-0.0.0.0-19001
address:
socket_address:
address: 0.0.0.0
address: '0.0.0.0'
port_value: 19001
protocol: TCP
filter_chains:
Expand Down
Loading

0 comments on commit 30d1b00

Please sign in to comment.