Skip to content

Commit dd4185b

Browse files
smirasmira
committed
feat: add KubeSpan extra endpoint configuration
Fixes #9174 Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
1 parent 3038ccf commit dd4185b

26 files changed

Lines changed: 632 additions & 108 deletions

File tree

‎api/resource/definitions/kubespan/kubespan.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ message ConfigSpec {
1919
uint32 mtu = 6;
2020
repeated string endpoint_filters = 7;
2121
bool harvest_extra_endpoints = 8;
22+
repeated common.NetIPPort extra_endpoints = 9;
2223
}
2324

2425
// EndpointSpec describes Endpoint state.

‎hack/release.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,12 @@ Talos Linux installer now never wipes the system disk on upgrades, which means t
229229
title = "Disk Management"
230230
description = """\
231231
Talos Linux now supports [configuration](https://www.talos.dev/v1.8/talos-guides/configuration/disk-management/#machine-configuration) for the `EPHEMERAL` volume.
232+
"""
233+
234+
[notes.kubespan]
235+
title = "KubeSpan"
236+
description = """\
237+
Extra announced endpoints can be added using the [`KubespanEndpointsConfig` document](https://www.talos.dev/v1.8/talos-guides/network/kubespan/#configuration).
232238
"""
233239

234240
[make_deps]

‎internal/app/machined/pkg/controllers/cluster/local_affiliate.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,13 @@ func (ctrl *LocalAffiliateController) Run(ctx context.Context, r controller.Runt
288288
spec.KubeSpan.Endpoints = xslices.Map(endpointIPs, func(addr netip.Addr) netip.AddrPort {
289289
return netip.AddrPortFrom(addr, constants.KubeSpanDefaultPort)
290290
})
291+
292+
// add extra announced endpoints, deduplicating on the way
293+
for _, addr := range kubespanConfig.TypedSpec().ExtraEndpoints {
294+
if !slices.Contains(spec.KubeSpan.Endpoints, addr) {
295+
spec.KubeSpan.Endpoints = append(spec.KubeSpan.Endpoints, addr)
296+
}
297+
}
291298
}
292299

293300
return nil

‎internal/app/machined/pkg/controllers/cluster/local_affiliate_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ func (suite *LocalAffiliateSuite) TestGeneration() {
7070
ksConfig := kubespan.NewConfig(config.NamespaceName, kubespan.ConfigID)
7171
ksConfig.TypedSpec().EndpointFilters = []string{"0.0.0.0/0", "!192.168.0.0/16", "2001::/16"}
7272
ksConfig.TypedSpec().AdvertiseKubernetesNetworks = true
73+
ksConfig.TypedSpec().ExtraEndpoints = []netip.AddrPort{netip.MustParseAddrPort("10.5.0.1:51820"), netip.MustParseAddrPort("1.2.3.4:5678")}
7374
suite.Require().NoError(suite.state.Create(suite.ctx, ksConfig))
7475

7576
// add KS address to the list of node addresses, it should be ignored in the endpoints
@@ -109,7 +110,6 @@ func (suite *LocalAffiliateSuite) TestGeneration() {
109110

110111
asrt.NotZero(spec.KubeSpan.PublicKey)
111112
asrt.NotZero(spec.KubeSpan.AdditionalAddresses)
112-
asrt.Len(spec.KubeSpan.Endpoints, 4)
113113

114114
asrt.Equal(ksIdentity.TypedSpec().Address.Addr(), spec.KubeSpan.Address)
115115
asrt.Equal(ksIdentity.TypedSpec().PublicKey, spec.KubeSpan.PublicKey)
@@ -120,6 +120,7 @@ func (suite *LocalAffiliateSuite) TestGeneration() {
120120
"10.5.0.1:51820",
121121
"1.1.1.1:51820",
122122
"[2001:123:4567::1]:51820",
123+
"1.2.3.4:5678",
123124
},
124125
xslices.Map(spec.KubeSpan.Endpoints, netip.AddrPort.String),
125126
)

‎internal/app/machined/pkg/controllers/kubespan/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ func NewConfigController() *ConfigController {
5151
res.TypedSpec().HarvestExtraEndpoints = c.Machine().Network().KubeSpan().HarvestExtraEndpoints()
5252
res.TypedSpec().MTU = c.Machine().Network().KubeSpan().MTU()
5353
res.TypedSpec().EndpointFilters = c.Machine().Network().KubeSpan().Filters().Endpoints()
54+
res.TypedSpec().ExtraEndpoints = c.KubespanConfig().ExtraAnnouncedEndpoints()
5455
}
5556

5657
return nil

‎internal/app/machined/pkg/controllers/kubespan/config_test.go‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
package kubespan_test
55

66
import (
7+
"fmt"
8+
"net/netip"
79
"testing"
810
"time"
911

@@ -14,6 +16,7 @@ import (
1416

1517
kubespanctrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/kubespan"
1618
"github.com/siderolabs/talos/pkg/machinery/config/container"
19+
"github.com/siderolabs/talos/pkg/machinery/config/types/network"
1720
"github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1"
1821
"github.com/siderolabs/talos/pkg/machinery/resources/config"
1922
"github.com/siderolabs/talos/pkg/machinery/resources/kubespan"
@@ -28,22 +31,30 @@ func (suite *ConfigSuite) TestReconcileConfig() {
2831

2932
suite.startRuntime()
3033

31-
cfg := config.NewMachineConfig(
32-
container.NewV1Alpha1(
33-
&v1alpha1.Config{
34-
ConfigVersion: "v1alpha1",
35-
MachineConfig: &v1alpha1.MachineConfig{
36-
MachineNetwork: &v1alpha1.NetworkConfig{
37-
NetworkKubeSpan: &v1alpha1.NetworkKubeSpan{
38-
KubeSpanEnabled: pointer.To(true),
39-
},
34+
ctr, err := container.New(
35+
&v1alpha1.Config{
36+
ConfigVersion: "v1alpha1",
37+
MachineConfig: &v1alpha1.MachineConfig{
38+
MachineNetwork: &v1alpha1.NetworkConfig{
39+
NetworkKubeSpan: &v1alpha1.NetworkKubeSpan{
40+
KubeSpanEnabled: pointer.To(true),
4041
},
4142
},
42-
ClusterConfig: &v1alpha1.ClusterConfig{
43-
ClusterID: "8XuV9TZHW08DOk3bVxQjH9ih_TBKjnh-j44tsCLSBzo=",
44-
ClusterSecret: "I+1In7fLnpcRIjUmEoeugZnSyFoTF6MztLxICL5Yu0s=",
45-
},
46-
}))
43+
},
44+
ClusterConfig: &v1alpha1.ClusterConfig{
45+
ClusterID: "8XuV9TZHW08DOk3bVxQjH9ih_TBKjnh-j44tsCLSBzo=",
46+
ClusterSecret: "I+1In7fLnpcRIjUmEoeugZnSyFoTF6MztLxICL5Yu0s=",
47+
},
48+
},
49+
&network.KubespanEndpointsConfigV1Alpha1{
50+
ExtraAnnouncedEndpointsConfig: []netip.AddrPort{
51+
netip.MustParseAddrPort("192.168.33.11:1001"),
52+
},
53+
},
54+
)
55+
suite.Require().NoError(err)
56+
57+
cfg := config.NewMachineConfig(ctr)
4758

4859
suite.Require().NoError(suite.state.Create(suite.ctx, cfg))
4960

@@ -61,6 +72,7 @@ func (suite *ConfigSuite) TestReconcileConfig() {
6172
suite.Assert().True(spec.ForceRouting)
6273
suite.Assert().False(spec.AdvertiseKubernetesNetworks)
6374
suite.Assert().False(spec.HarvestExtraEndpoints)
75+
suite.Assert().Equal("[\"192.168.33.11:1001\"]", fmt.Sprintf("%q", spec.ExtraEndpoints))
6476

6577
return nil
6678
},

‎internal/integration/api/discovery.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ package api
99
import (
1010
"context"
1111
"fmt"
12+
"math/rand/v2"
1213
"net/netip"
1314
"strings"
1415
"time"
@@ -23,6 +24,7 @@ import (
2324

2425
"github.com/siderolabs/talos/internal/integration/base"
2526
"github.com/siderolabs/talos/pkg/machinery/client"
27+
"github.com/siderolabs/talos/pkg/machinery/config/types/network"
2628
"github.com/siderolabs/talos/pkg/machinery/resources/cluster"
2729
"github.com/siderolabs/talos/pkg/machinery/resources/kubespan"
2830
)
@@ -277,6 +279,59 @@ func (suite *DiscoverySuite) TestKubeSpanPeers() {
277279
}
278280
}
279281

282+
// TestKubeSpanExtraEndpoints verifies that KubeSpan peer specs are updated with extra endpoints.
283+
func (suite *DiscoverySuite) TestKubeSpanExtraEndpoints() {
284+
if !suite.Capabilities().RunsTalosKernel {
285+
suite.T().Skip("not running Talos kernel")
286+
}
287+
288+
// check that cluster has KubeSpan enabled
289+
node := suite.RandomDiscoveredNodeInternalIP()
290+
suite.ClearConnectionRefused(suite.ctx, node)
291+
292+
nodeCtx := client.WithNode(suite.ctx, node)
293+
provider, err := suite.ReadConfigFromNode(nodeCtx)
294+
suite.Require().NoError(err)
295+
296+
if !provider.Machine().Network().KubeSpan().Enabled() {
297+
suite.T().Skip("KubeSpan is disabled")
298+
}
299+
300+
nodes := suite.DiscoverNodeInternalIPs(suite.ctx)
301+
302+
if len(nodes) < 2 {
303+
suite.T().Skip("need at least two nodes for this test")
304+
}
305+
306+
perm := rand.Perm(len(nodes))
307+
308+
checkNode := nodes[perm[0]]
309+
targetNode := nodes[perm[1]]
310+
311+
mockEndpoint := netip.MustParseAddrPort("169.254.121.121:5820")
312+
313+
// inject extra endpoint to target node
314+
cfgDocument := network.NewKubespanEndpointsV1Alpha1()
315+
cfgDocument.ExtraAnnouncedEndpointsConfig = []netip.AddrPort{mockEndpoint}
316+
317+
suite.T().Logf("injecting extra endpoint %s to node %s", mockEndpoint, targetNode)
318+
suite.PatchMachineConfig(client.WithNode(suite.ctx, targetNode), cfgDocument)
319+
320+
targetIdentity, err := safe.ReaderGetByID[*kubespan.Identity](client.WithNode(suite.ctx, targetNode), suite.Client.COSI, kubespan.LocalIdentity)
321+
suite.Require().NoError(err)
322+
323+
suite.T().Logf("checking extra endpoint %s on node %s", mockEndpoint, checkNode)
324+
rtestutils.AssertResources(client.WithNode(suite.ctx, checkNode), suite.T(), suite.Client.COSI, []string{targetIdentity.TypedSpec().PublicKey},
325+
func(peer *kubespan.PeerSpec, asrt *assert.Assertions) {
326+
asrt.Contains(peer.TypedSpec().Endpoints, mockEndpoint)
327+
},
328+
)
329+
330+
// the extra endpoints disappears with a timeout from the discovery service, so can't assert on that
331+
suite.T().Logf("removin extra endpoint %s from node %s", mockEndpoint, targetNode)
332+
suite.RemoveMachineConfigDocuments(client.WithNode(suite.ctx, targetNode), cfgDocument.MetaKind)
333+
}
334+
280335
func (suite *DiscoverySuite) getMembers(nodeCtx context.Context) []*cluster.Member {
281336
var result []*cluster.Member
282337

0 commit comments

Comments
 (0)