Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

use the entire installation spec object in node join response #1211

Merged
merged 10 commits into from
Sep 23, 2024
Merged

use the entire installation spec object in node join response #1211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
eb45543
use unified join command response
laverya Sep 20, 2024
e5a6e40
add kots image override
laverya Sep 20, 2024
23cf9d9
unit tests and preflights
laverya Sep 20, 2024
aa88ae3
update kotsadm image
laverya Sep 20, 2024
10d6e25
Merge branch 'main' into laverya/return-entire-installation-spec-object
laverya Sep 20, 2024
ab6761e
f
laverya Sep 20, 2024
4fb83ca
Spec -> InstallationSpec
laverya Sep 20, 2024
e22765b
updated adminconsole version
laverya Sep 23, 2024
fa99406
remove override image
laverya Sep 23, 2024
73f9470
Merge branch 'automation/update-makefile-adminconsole' into laverya/r…
laverya Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 48 additions & 47 deletions cmd/embedded-cluster/join.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,21 +31,17 @@ import (
"github.com/replicatedhq/embedded-cluster/pkg/netutils"
"github.com/replicatedhq/embedded-cluster/pkg/prompts"
"github.com/replicatedhq/embedded-cluster/pkg/spinner"
"github.com/replicatedhq/embedded-cluster/pkg/versions"
)

// JoinCommandResponse is the response from the kots api we use to fetch the k0s join token.
type JoinCommandResponse struct {
K0sJoinCommand string `json:"k0sJoinCommand"`
K0sToken string `json:"k0sToken"`
ClusterID uuid.UUID `json:"clusterID"`
K0sUnsupportedOverrides string `json:"k0sUnsupportedOverrides"`
EndUserK0sConfigOverrides string `json:"endUserK0sConfigOverrides"`
// MetricsBaseURL is the https://replicated.app endpoint url
MetricsBaseURL string `json:"metricsBaseURL"`
AirgapRegistryAddress string `json:"airgapRegistryAddress"`
Proxy *ecv1beta1.ProxySpec `json:"proxy"`
Network *ecv1beta1.NetworkSpec `json:"network"`
LocalArtifactMirrorPort int `json:"localArtifactMirrorPort,omitempty"`
K0sJoinCommand string `json:"k0sJoinCommand"`
K0sToken string `json:"k0sToken"`
ClusterID uuid.UUID `json:"clusterID"`
EmbeddedClusterVersion string `json:"embeddedClusterVersion"`
AirgapRegistryAddress string `json:"airgapRegistryAddress"`
InstallationSpec ecv1beta1.InstallationSpec `json:"installationSpec,omitempty"`
}

// extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that
Expand All @@ -69,13 +65,13 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map
// EndUserOverrides returns a dig.Mapping that can be applied on top of a k0s configuration.
// This patch is assembled based on the EndUserK0sConfigOverrides field.
func (j JoinCommandResponse) EndUserOverrides() (dig.Mapping, error) {
return j.extractK0sConfigOverridePatch([]byte(j.EndUserK0sConfigOverrides))
return j.extractK0sConfigOverridePatch([]byte(j.InstallationSpec.EndUserK0sConfigOverrides))
}

// EmbeddedOverrides returns a dig.Mapping that can be applied on top of a k0s configuration.
// This patch is assembled based on the K0sUnsupportedOverrides field.
func (j JoinCommandResponse) EmbeddedOverrides() (dig.Mapping, error) {
return j.extractK0sConfigOverridePatch([]byte(j.K0sUnsupportedOverrides))
return j.extractK0sConfigOverridePatch([]byte(j.InstallationSpec.Config.UnsupportedOverrides.K0s))
}

// getJoinToken issues a request to the kots api to get the actual join command
Expand Down Expand Up @@ -114,15 +110,15 @@ func startAndWaitForK0s(c *cli.Context, jcmd *JoinCommandResponse) error {
logrus.Debugf("starting %s service", binName)
if err := startK0sService(); err != nil {
err := fmt.Errorf("unable to start service: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

loading.Infof("Waiting for %s node to be ready", binName)
logrus.Debugf("waiting for k0s to be ready")
if err := waitForK0s(); err != nil {
err := fmt.Errorf("unable to wait for node: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

Expand Down Expand Up @@ -198,13 +194,18 @@ var joinCommand = &cli.Command{
return fmt.Errorf("unable to get join token: %w", err)
}

setProxyEnv(jcmd.Proxy)
proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.Proxy, networkInterface)
// check to make sure the version returned by the join token is the same as the one we are running
if jcmd.EmbeddedClusterVersion != versions.Version {
return fmt.Errorf("embedded cluster version mismatch - this binary is version %q, but the cluster is running version %q", versions.Version, jcmd.EmbeddedClusterVersion)
}

setProxyEnv(jcmd.InstallationSpec.Proxy)
proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.InstallationSpec.Proxy, networkInterface)
if err != nil {
return fmt.Errorf("failed to check proxy config for local IP: %w", err)
}
if !proxyOK {
return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.Proxy.NoProxy, localIP)
return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.InstallationSpec.Proxy.NoProxy, localIP)
}

isAirgap := c.String("airgap-bundle") != ""
Expand All @@ -216,25 +217,25 @@ var joinCommand = &cli.Command{
}
}

metrics.ReportJoinStarted(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID)
metrics.ReportJoinStarted(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID)
logrus.Debugf("materializing %s binaries", binName)
if err := materializeFiles(c); err != nil {
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

applier, err := getAddonsApplier(c, "", jcmd.Proxy)
applier, err := getAddonsApplier(c, "", jcmd.InstallationSpec.Proxy)
if err != nil {
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

// jcmd.MetricsBaseURL is the replicated.app endpoint url
replicatedAPIURL := jcmd.MetricsBaseURL
// jcmd.InstallationSpec.MetricsBaseURL is the replicated.app endpoint url
replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL
proxyRegistryURL := fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress)
if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.Proxy); err != nil {
if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy); err != nil {
err := fmt.Errorf("unable to run host preflights locally: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

Expand All @@ -246,54 +247,54 @@ var joinCommand = &cli.Command{
logrus.Debugf("saving token to disk")
if err := saveTokenToDisk(jcmd.K0sToken); err != nil {
err := fmt.Errorf("unable to save token to disk: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

logrus.Debugf("installing %s binaries", binName)
if err := installK0sBinary(); err != nil {
err := fmt.Errorf("unable to install k0s binary: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

if jcmd.AirgapRegistryAddress != "" {
if err := airgap.AddInsecureRegistry(jcmd.AirgapRegistryAddress); err != nil {
err := fmt.Errorf("unable to add insecure registry: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}
}

logrus.Debugf("creating systemd unit files")
localArtifactMirrorPort := defaults.LocalArtifactMirrorPort
if jcmd.LocalArtifactMirrorPort > 0 {
localArtifactMirrorPort = jcmd.LocalArtifactMirrorPort
if jcmd.InstallationSpec.LocalArtifactMirror != nil && jcmd.InstallationSpec.LocalArtifactMirror.Port > 0 {
localArtifactMirrorPort = jcmd.InstallationSpec.LocalArtifactMirror.Port
}
// both controller and worker nodes will have 'worker' in the join command
if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.Proxy, localArtifactMirrorPort); err != nil {
if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.InstallationSpec.Proxy, localArtifactMirrorPort); err != nil {
err := fmt.Errorf("unable to create systemd unit files: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

logrus.Debugf("overriding network configuration")
if err := applyNetworkConfiguration(jcmd, networkInterface); err != nil {
err := fmt.Errorf("unable to apply network configuration: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
}

logrus.Debugf("applying configuration overrides")
if err := applyJoinConfigurationOverrides(jcmd); err != nil {
err := fmt.Errorf("unable to apply configuration overrides: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

logrus.Debugf("joining node to cluster")
if err := runK0sInstallCommand(jcmd.K0sJoinCommand, networkInterface); err != nil {
err := fmt.Errorf("unable to join node to cluster: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

Expand All @@ -302,45 +303,45 @@ var joinCommand = &cli.Command{
}

if !strings.Contains(jcmd.K0sJoinCommand, "controller") {
metrics.ReportJoinSucceeded(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID)
metrics.ReportJoinSucceeded(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID)
logrus.Debugf("worker node join finished")
return nil
}

kcli, err := kubeutils.KubeClient()
if err != nil {
err := fmt.Errorf("unable to get kube client: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}
hostname, err := os.Hostname()
if err != nil {
err := fmt.Errorf("unable to get hostname: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}
if err := waitForNode(c.Context, kcli, hostname); err != nil {
err := fmt.Errorf("unable to wait for node: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}

if c.Bool("enable-ha") {
if err := maybeEnableHA(c.Context, kcli); err != nil {
err := fmt.Errorf("unable to enable high availability: %w", err)
metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
return err
}
}

metrics.ReportJoinSucceeded(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID)
metrics.ReportJoinSucceeded(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID)
logrus.Debugf("controller node join finished")
return nil
},
}

func applyNetworkConfiguration(jcmd *JoinCommandResponse, networkInterface string) error {
if jcmd.Network != nil {
if jcmd.InstallationSpec.Network != nil {
clusterSpec := config.RenderK0sConfig()
address, err := netutils.FirstValidAddress(networkInterface)
if err != nil {
Expand All @@ -350,13 +351,13 @@ func applyNetworkConfiguration(jcmd *JoinCommandResponse, networkInterface strin
clusterSpec.Spec.Storage.Etcd.PeerAddress = address
// NOTE: we should be copying everything from the in cluster config spec and overriding
// the node specific config from clusterSpec.GetClusterWideConfig()
clusterSpec.Spec.Network.PodCIDR = jcmd.Network.PodCIDR
clusterSpec.Spec.Network.ServiceCIDR = jcmd.Network.ServiceCIDR
if jcmd.Network.NodePortRange != "" {
clusterSpec.Spec.Network.PodCIDR = jcmd.InstallationSpec.Network.PodCIDR
clusterSpec.Spec.Network.ServiceCIDR = jcmd.InstallationSpec.Network.ServiceCIDR
if jcmd.InstallationSpec.Network.NodePortRange != "" {
if clusterSpec.Spec.API.ExtraArgs == nil {
clusterSpec.Spec.API.ExtraArgs = map[string]string{}
}
clusterSpec.Spec.API.ExtraArgs["service-node-port-range"] = jcmd.Network.NodePortRange
clusterSpec.Spec.API.ExtraArgs["service-node-port-range"] = jcmd.InstallationSpec.Network.NodePortRange
}
clusterSpecYaml, err := k8syaml.Marshal(clusterSpec)

Expand Down
11 changes: 9 additions & 2 deletions cmd/embedded-cluster/join_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/k0sproject/dig"
k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -101,8 +102,14 @@ func TestJoinCommandResponseOverrides(t *testing.T) {
t.Run(tname, func(t *testing.T) {
req := require.New(t)
join := JoinCommandResponse{
K0sUnsupportedOverrides: tt.EmbeddedOverrides,
EndUserK0sConfigOverrides: tt.EndUserOverrides,
InstallationSpec: ecv1beta1.InstallationSpec{
Config: &ecv1beta1.ConfigSpec{
UnsupportedOverrides: ecv1beta1.UnsupportedOverrides{
K0s: tt.EmbeddedOverrides,
},
},
EndUserK0sConfigOverrides: tt.EndUserOverrides,
},
}

embedded, err := join.EmbeddedOverrides()
Expand Down
18 changes: 12 additions & 6 deletions cmd/embedded-cluster/preflights.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/replicatedhq/embedded-cluster/pkg/defaults"
"github.com/replicatedhq/embedded-cluster/pkg/netutils"
"github.com/replicatedhq/embedded-cluster/pkg/versions"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
)
Expand Down Expand Up @@ -126,13 +127,18 @@ var joinRunPreflightsCommand = &cli.Command{
return fmt.Errorf("unable to get join token: %w", err)
}

setProxyEnv(jcmd.Proxy)
proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.Proxy, networkInterface)
// check to make sure the version returned by the join token is the same as the one we are running
if jcmd.EmbeddedClusterVersion != versions.Version {
return fmt.Errorf("embedded cluster version mismatch - this binary is version %q, but the cluster is running version %q", versions.Version, jcmd.EmbeddedClusterVersion)
}

setProxyEnv(jcmd.InstallationSpec.Proxy)
proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.InstallationSpec.Proxy, networkInterface)
if err != nil {
return fmt.Errorf("failed to check proxy config for local IP: %w", err)
}
if !proxyOK {
return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.Proxy.NoProxy, localIP)
return fmt.Errorf("no-proxy config %q does not allow access to local IP %q", jcmd.InstallationSpec.Proxy.NoProxy, localIP)
}

isAirgap := c.String("airgap-bundle") != ""
Expand All @@ -142,15 +148,15 @@ var joinRunPreflightsCommand = &cli.Command{
return err
}

applier, err := getAddonsApplier(c, "", jcmd.Proxy)
applier, err := getAddonsApplier(c, "", jcmd.InstallationSpec.Proxy)
if err != nil {
return err
}

logrus.Debugf("running host preflights")
replicatedAPIURL := jcmd.MetricsBaseURL
replicatedAPIURL := jcmd.InstallationSpec.MetricsBaseURL
proxyRegistryURL := fmt.Sprintf("https://%s", defaults.ProxyRegistryAddress)
if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.Proxy); err != nil {
if err := RunHostPreflights(c, applier, replicatedAPIURL, proxyRegistryURL, isAirgap, jcmd.InstallationSpec.Proxy); err != nil {
err := fmt.Errorf("unable to run host preflights locally: %w", err)
return err
}
Expand Down
18 changes: 9 additions & 9 deletions pkg/addons/adminconsole/static/metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,26 @@
# $ make buildtools
# $ output/bin/buildtools update addon <addon name>
#
version: 1.117.2
version: 1.117.3
location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/admin-console
images:
kotsadm:
repo: proxy.replicated.com/anonymous/kotsadm/kotsadm
tag:
amd64: v1.117.2-amd64@sha256:eb36a5a9af101164598f1e2f7f0745aad59bdb8d8870e09e2702d8c75c753380
arm64: v1.117.2-arm64@sha256:a60cf01f935a785b1a5e693d65a18bd6f4e8d7aac7be30a3311c4c17f3b8222f
amd64: v1.117.3-amd64@sha256:d47ac4df627ac357452efffb717776adb452c3ab9b466ef3ccaf808df722b7a6
arm64: v1.117.3-arm64@sha256:6b1b7bfc42fb3c7f21fe077b4645194fbdd1d497f92f4b12c00ceadb40969b8e
kotsadm-migrations:
repo: proxy.replicated.com/anonymous/kotsadm/kotsadm-migrations
tag:
amd64: v1.117.2-amd64@sha256:afb070977dfc16d3b85d2ae9e3cf68fc317e17e3ab05e6ad08d353201f1682ef
arm64: v1.117.2-arm64@sha256:cff8c43da1094d7969ef689eca3fd11c17aae936f45d1584fd00e25085a6112b
amd64: v1.117.3-amd64@sha256:56d2765497a57c06ef6e9f7705cf5218d21a978d197575a3c22fe7d84db07f0a
arm64: v1.117.3-arm64@sha256:25ff2b4697425be55d576f5f068480a3ab8fe6216341f2ec3b3c1962d3ac08ba
kurl-proxy:
repo: proxy.replicated.com/anonymous/kotsadm/kurl-proxy
tag:
amd64: v1.117.2-amd64@sha256:b2e4a7ee3ca4cedd1028fbac0678fcab091b1e0a81fd39d6b8d5d20f509b2304
arm64: v1.117.2-arm64@sha256:056b640c03ff824553adb2fc95fb4930e4cdea18c89b656ed81c1937d8ab3c73
amd64: v1.117.3-amd64@sha256:816bcbc273ec51255d7b459e49831ce09fd361db4a295d31f61d7af02177860f
arm64: v1.117.3-arm64@sha256:c4f5632ca9ea3fae2208b4ca7923900c6e958a0906b4a9dd4f6b5ebdab4b9d89
rqlite:
repo: proxy.replicated.com/anonymous/kotsadm/rqlite
tag:
amd64: 8.30.3-r0-amd64@sha256:2f0c6a90c462b8be8f08c5a1a83f71932be96b846beb3dfbd9fa0e3d13c66d63
arm64: 8.30.3-r0-arm64@sha256:4e96628236b7a1e9549bfdb16d44ff71ee7607b2450548b7e0f86f1f48b36c4e
amd64: 8.30.4-r0-amd64@sha256:884ac56b236e059e420858c94d90a083fe48b666c8b3433da612b9380906ce41
arm64: 8.30.4-r0-arm64@sha256:cfb55508de1fb59cfc5d14586f52d8a09b776e766ea612351df77d19ae2e6d9a
Loading