diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go index 0f87dd71f6..969a226e6c 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go @@ -186,6 +186,12 @@ func (*Sequencer) Boot(r runtime.Runtime) []runtime.Phase { ).Append( "saveConfig", SaveConfig, + ).Append( + "memorySizeCheck", + MemorySizeCheck, + ).Append( + "diskSizeCheck", + DiskSizeCheck, ).Append( "env", SetUserEnvVars, diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go index 5552f49df4..f507f704ca 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go @@ -26,8 +26,10 @@ import ( cgroupsv2 "github.com/containerd/cgroups/v2" "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/state" - multierror "github.com/hashicorp/go-multierror" + "github.com/dustin/go-humanize" + "github.com/hashicorp/go-multierror" "github.com/opencontainers/runtime-spec/specs-go" + pprocfs "github.com/prometheus/procfs" "github.com/siderolabs/go-blockdevice/blockdevice" "github.com/siderolabs/go-blockdevice/blockdevice/partition/gpt" "github.com/siderolabs/go-blockdevice/blockdevice/util" @@ -73,6 +75,7 @@ import ( resourcefiles "github.com/siderolabs/talos/pkg/machinery/resources/files" "github.com/siderolabs/talos/pkg/machinery/resources/k8s" resourceruntime "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/minimal" "github.com/siderolabs/talos/pkg/version" ) @@ -681,6 +684,82 @@ func ValidateConfig(seq runtime.Sequence, data interface{}) (runtime.TaskExecuti }, "validateConfig" } +// MemorySizeCheck represents the MemorySizeCheck task. +func MemorySizeCheck(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + if r.State().Platform().Mode() == runtime.ModeContainer { + log.Println("skipping memory size check in the container") + + return nil + } + + pc, err := pprocfs.NewDefaultFS() + if err != nil { + return fmt.Errorf("failed to open procfs: %w", err) + } + + info, err := pc.Meminfo() + if err != nil { + return fmt.Errorf("failed to read meminfo: %w", err) + } + + minimum, recommended, err := minimal.Memory(r.Config().Machine().Type()) + if err != nil { + return err + } + + switch memTotal := pointer.SafeDeref(info.MemTotal) * humanize.KiByte; { + case memTotal < minimum: + log.Println("WARNING: memory size is less than recommended") + log.Println("WARNING: Talos may not work properly") + log.Println("WARNING: minimum memory size is", minimum/humanize.MiByte, "MiB") + log.Println("WARNING: recommended memory size is", recommended/humanize.MiByte, "MiB") + log.Println("WARNING: current total memory size is", memTotal/humanize.MiByte, "MiB") + case memTotal < recommended: + log.Println("NOTE: recommended memory size is", recommended/humanize.MiByte, "MiB") + log.Println("NOTE: current total memory size is", memTotal/humanize.MiByte, "MiB") + default: + log.Println("memory size is OK") + log.Println("memory size is", memTotal/humanize.MiByte, "MiB") + } + + return nil + }, "memorySizeCheck" +} + +// DiskSizeCheck represents the DiskSizeCheck task. +func DiskSizeCheck(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { + return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { + if r.State().Platform().Mode() == runtime.ModeContainer { + log.Println("skipping disk size check in the container") + + return nil + } + + disk := r.State().Machine().Disk() // get ephemeral disk state + if disk == nil { + return fmt.Errorf("failed to get ephemeral disk state") + } + + diskSize, err := disk.Size() + if err != nil { + return fmt.Errorf("failed to get ephemeral disk size: %w", err) + } + + if minimum := minimal.DiskSize(); diskSize < minimum { + log.Println("WARNING: disk size is less than recommended") + log.Println("WARNING: Talos may not work properly") + log.Println("WARNING: minimum recommended disk size is", minimum/humanize.MiByte, "MiB") + log.Println("WARNING: current total disk size is", diskSize/humanize.MiByte, "MiB") + } else { + log.Println("disk size is OK") + log.Println("disk size is", diskSize/humanize.MiByte, "MiB") + } + + return nil + }, "diskSizeCheck" +} + // SetUserEnvVars represents the SetUserEnvVars task. func SetUserEnvVars(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) { return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) { diff --git a/pkg/cluster/check/default.go b/pkg/cluster/check/default.go index 405b7b4023..30b2fe63c3 100644 --- a/pkg/cluster/check/default.go +++ b/pkg/cluster/check/default.go @@ -43,6 +43,20 @@ func DefaultClusterChecks() []ClusterCheck { }, 5*time.Minute, 5*time.Second) }, + // wait for all nodes to report their memory size + func(cluster ClusterInfo) conditions.Condition { + return conditions.PollingCondition("all nodes memory sizes", func(ctx context.Context) error { + return AllNodesMemorySizes(ctx, cluster) + }, 5*time.Minute, 5*time.Second) + }, + + // wait for all nodes to report their disk size + func(cluster ClusterInfo) conditions.Condition { + return conditions.PollingCondition("all nodes disk sizes", func(ctx context.Context) error { + return AllNodesDiskSizes(ctx, cluster) + }, 5*time.Minute, 5*time.Second) + }, + // wait for kubelet to be healthy on all func(cluster ClusterInfo) conditions.Condition { return conditions.PollingCondition("kubelet to be healthy", func(ctx context.Context) error { diff --git a/pkg/cluster/check/nodes.go b/pkg/cluster/check/nodes.go new file mode 100644 index 0000000000..8150b13ef3 --- /dev/null +++ b/pkg/cluster/check/nodes.go @@ -0,0 +1,277 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package check + +import ( + "context" + "errors" + fmt "fmt" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/dustin/go-humanize" + "github.com/hashicorp/go-multierror" + "github.com/siderolabs/gen/slices" + "google.golang.org/grpc/codes" + + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" + "github.com/siderolabs/talos/pkg/minimal" +) + +// AllNodesMemorySizes checks that all nodes have enough memory. +func AllNodesMemorySizes(ctx context.Context, cluster ClusterInfo) error { + cl, err := cluster.Client() + if err != nil { + return fmt.Errorf("error getting client: %w", err) + } + + nodesIP, err := getNonContainerNodes( + client.WithNodes( + ctx, + mapIPsToStrings(mapNodeInfosToInternalIPs(cluster.Nodes()))..., + ), + cl, + ) + if err != nil { + return err + } + + if len(nodesIP) == 0 { + return nil + } + + resp, err := cl.Memory(client.WithNodes(ctx, nodesIP...)) + if err != nil { + return fmt.Errorf("error getting nodes memory: %w", err) + } + + var resultErr error + + nodeToType := getNodesTypes(cluster, machine.TypeInit, machine.TypeControlPlane, machine.TypeWorker) + + for _, msg := range resp.Messages { + if msg.Metadata == nil { + return fmt.Errorf("no metadata in the response") + } + + hostname := msg.Metadata.Hostname + + typ, ok := nodeToType[hostname] + if !ok { + return fmt.Errorf("unexpected node %q in response", hostname) + } + + minimum, _, err := minimal.Memory(typ) + if err != nil { + resultErr = multierror.Append(resultErr, err) + + continue + } + + if totalMemory := msg.Meminfo.Memtotal * humanize.KiByte; totalMemory < minimum { + resultErr = multierror.Append( + resultErr, + fmt.Errorf( + "node %q does not meet memory requirements: expected at least %d MiB, actual %d MiB", + hostname, + minimum/humanize.MiByte, + totalMemory/humanize.MiByte, + ), + ) + } + } + + return resultErr +} + +func getNodesTypes(cluster ClusterInfo, nodeTypes ...machine.Type) map[string]machine.Type { + result := map[string]machine.Type{} + + for _, typ := range nodeTypes { + for _, node := range cluster.NodesByType(typ) { + result[node.InternalIP.String()] = typ + } + } + + return result +} + +// AllNodesDiskSizes checks that all nodes have enough disk space. +// +//nolint:gocyclo +func AllNodesDiskSizes(ctx context.Context, cluster ClusterInfo) error { + cl, err := cluster.Client() + if err != nil { + return fmt.Errorf("error getting client: %w", err) + } + + nodesIP, err := getNonContainerNodes( + client.WithNodes( + ctx, + mapIPsToStrings(mapNodeInfosToInternalIPs(cluster.Nodes()))..., + ), + cl, + ) + if err != nil { + return err + } + + if len(nodesIP) == 0 { + return nil + } + + ctx = client.WithNodes(ctx, nodesIP...) + + nodesMounts, err := getNodesMounts(ctx, cl) + if err != nil { + return err + } + + var resultErr error + + for _, nodeIP := range nodesIP { + data, err := getEphemeralPartitionData(ctx, cl.COSI, nodeIP) + if errors.Is(err, ErrOldTalosVersion) { + continue + } else if err != nil { + resultErr = multierror.Append(resultErr, err) + + continue + } + + nodeMounts, ok := nodesMounts[nodeIP] + if !ok { + resultErr = multierror.Append(resultErr, fmt.Errorf("node %q not found in mounts", nodeIP)) + + continue + } + + idx := slices.IndexFunc(nodeMounts, func(mnt mntData) bool { return mnt.Filesystem == data.Source }) + if idx == -1 { + resultErr = multierror.Append(resultErr, fmt.Errorf("ephemeral partition %q not found for node %q", data.Source, nodeIP)) + + continue + } + + minimalDiskSize := minimal.DiskSize() + + // adjust by 1400 MiB to account for the size of system stuff + if actualDiskSize := nodeMounts[idx].Size + 1400*humanize.MiByte; actualDiskSize < minimal.DiskSize() { + resultErr = multierror.Append(resultErr, fmt.Errorf( + "ephemeral partition %q for node %q is too small, expected at least %s, actual %s", + data.Source, + nodeIP, + humanize.IBytes(minimalDiskSize), + humanize.IBytes(actualDiskSize), + )) + + continue + } + } + + return resultErr +} + +func getNonContainerNodes(ctx context.Context, cl *client.Client) ([]string, error) { + resp, err := cl.Version(ctx) + if err != nil { + return nil, fmt.Errorf("error getting version: %w", err) + } + + result := make([]string, 0, len(resp.Messages)) + + for _, msg := range resp.Messages { + if msg.Metadata == nil { + return nil, errors.New("got empty metadata") + } + + if msg.Platform.Mode == "container" { + continue + } + + result = append(result, msg.Metadata.Hostname) + } + + return result, nil +} + +type mountData struct { + Source string +} + +// ErrOldTalosVersion is returned when the node is running an old version of Talos. +var ErrOldTalosVersion = fmt.Errorf("old Talos version") + +func getEphemeralPartitionData(ctx context.Context, state state.State, nodeIP string) (mountData, error) { + items, err := safe.StateList[*runtime.MountStatus]( + client.WithNode(ctx, nodeIP), + state, + resource.NewMetadata(v1alpha1.NamespaceName, runtime.MountStatusType, "", resource.VersionUndefined), + ) + if err != nil { + if client.StatusCode(err) == codes.Unimplemented { + // old version of Talos without COSI API + return mountData{}, ErrOldTalosVersion + } + + return mountData{}, fmt.Errorf("error listing mounts for node %q: %w", nodeIP, err) + } + + for it := safe.IteratorFromList(items); it.Next(); { + mount := it.Value() + mountID := mount.Metadata().ID() + + if mountID == constants.EphemeralPartitionLabel { + return mountData{ + Source: mount.TypedSpec().Source, + }, nil + } + } + + return mountData{}, fmt.Errorf("no ephemeral partition found for node '%s'", nodeIP) +} + +type mntData struct { + Filesystem string + Size uint64 +} + +func getNodesMounts(ctx context.Context, cl *client.Client) (map[string][]mntData, error) { + diskResp, err := cl.Mounts(ctx) + if err != nil { + return nil, fmt.Errorf("error getting nodes mounts: %w", err) + } + + if len(diskResp.Messages) == 0 { + return nil, fmt.Errorf("no nodes with mounts found") + } + + nodesMnts := map[string][]mntData{} + + for _, msg := range diskResp.Messages { + switch { + case msg.Metadata == nil: + return nil, fmt.Errorf("no metadata in response") + case len(msg.GetStats()) == 0: + return nil, fmt.Errorf("no mounts found for node %q", msg.Metadata.Hostname) + } + + hostname := msg.Metadata.Hostname + + for _, mnt := range msg.GetStats() { + nodesMnts[hostname] = append(nodesMnts[hostname], mntData{ + Filesystem: mnt.Filesystem, + Size: mnt.Size, + }) + } + } + + return nodesMnts, nil +} diff --git a/pkg/minimal/limits.go b/pkg/minimal/limits.go new file mode 100644 index 0000000000..1e9ee25d12 --- /dev/null +++ b/pkg/minimal/limits.go @@ -0,0 +1,38 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package minimal provides the minimal/recommended limits for different machine types. +package minimal + +import ( + "fmt" + + "github.com/dustin/go-humanize" + + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1/machine" +) + +// Memory returns the minimal/recommended amount of memory required to run the node. +func Memory(typ machine.Type) (minimum, recommended uint64, err error) { + // We remove 100 MiB from the recommended memory to account for the kernel + switch typ { //nolint:exhaustive + case machine.TypeControlPlane, machine.TypeInit: + minimum = 2*humanize.GiByte - 150*humanize.MiByte + recommended = 4*humanize.GiByte - 150*humanize.MiByte + + case machine.TypeWorker: + minimum = 1*humanize.GiByte - 150*humanize.MiByte + recommended = 2*humanize.GiByte - 150*humanize.MiByte + + default: + return 0, 0, fmt.Errorf("unknown machine type %q", typ) + } + + return minimum, recommended, nil +} + +// DiskSize returns the minimal/recommended amount of disk space required to run the node. +func DiskSize() uint64 { + return 6 * humanize.GiByte +}