From c223481fbe8fb091dc912b05ab9de428ab7dbdad Mon Sep 17 00:00:00 2001 From: Diego Bonfigli Date: Fri, 25 Dec 2020 13:00:34 +0100 Subject: [PATCH] add Linode cloud provider --- cluster-autoscaler/README.md | 2 + .../cloudprovider/builder/builder_all.go | 6 +- .../cloudprovider/builder/builder_linode.go | 42 +++ .../cloudprovider/cloud_provider.go | 2 + .../cloudprovider/linode/README.md | 62 ++++ .../examples/cluster-autoscale-pdb.yaml | 9 + .../cluster-autoscaler-autodiscover.yaml | 169 +++++++++++ .../examples/cluster-autoscaler-secret.yaml | 26 ++ .../cloudprovider/linode/linode_api_client.go | 52 ++++ .../linode/linode_cloud_config.go | 164 ++++++++++ .../linode/linode_cloud_config_test.go | 160 ++++++++++ .../linode/linode_cloud_provider.go | 166 +++++++++++ .../linode/linode_cloud_provider_test.go | 188 ++++++++++++ .../cloudprovider/linode/linode_manager.go | 131 ++++++++ .../linode/linode_manager_test.go | 148 +++++++++ .../cloudprovider/linode/linode_node_group.go | 265 ++++++++++++++++ .../linode/linode_node_group_test.go | 282 ++++++++++++++++++ .../cloudprovider/linode/linode_utils_test.go | 43 +++ .../cloudprovider/linode/linodego/README.md | 5 + .../linode/linodego/linode_api_client_rest.go | 213 +++++++++++++ .../linodego/linode_api_client_rest_test.go | 126 ++++++++ 21 files changed, 2260 insertions(+), 1 deletion(-) create mode 100644 cluster-autoscaler/cloudprovider/builder/builder_linode.go create mode 100644 cluster-autoscaler/cloudprovider/linode/README.md create mode 100644 cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscale-pdb.yaml create mode 100644 cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-autodiscover.yaml create mode 100644 cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-secret.yaml create mode 100644 cluster-autoscaler/cloudprovider/linode/linode_api_client.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linode_cloud_config.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linode_cloud_config_test.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linode_cloud_provider.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linode_cloud_provider_test.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linode_manager.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linode_manager_test.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linode_node_group.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linode_node_group_test.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linode_utils_test.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linodego/README.md create mode 100644 cluster-autoscaler/cloudprovider/linode/linodego/linode_api_client_rest.go create mode 100644 cluster-autoscaler/cloudprovider/linode/linodego/linode_api_client_rest_test.go diff --git a/cluster-autoscaler/README.md b/cluster-autoscaler/README.md index 0bc4f0315ed9..0bec9a810d0c 100644 --- a/cluster-autoscaler/README.md +++ b/cluster-autoscaler/README.md @@ -21,6 +21,7 @@ You should also take a look at the notes and "gotchas" for your specific cloud p * [Packet](./cloudprovider/packet/README.md#notes) * [IonosCloud](./cloudprovider/ionoscloud/README.md) * [OVHcloud](./cloudprovider/ovhcloud/README.md) +* [Linode](./cloudprovider/linode/README.md) # Releases @@ -151,3 +152,4 @@ Supported cloud providers: * Exoscale https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/exoscale/README.md * Packet https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/packet/README.md * OVHcloud https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/ovhcloud/README.md +* Linode https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/linode/README.md diff --git a/cluster-autoscaler/cloudprovider/builder/builder_all.go b/cluster-autoscaler/cloudprovider/builder/builder_all.go index 6fa56bf0ed10..2c88a792024a 100644 --- a/cluster-autoscaler/cloudprovider/builder/builder_all.go +++ b/cluster-autoscaler/cloudprovider/builder/builder_all.go @@ -1,4 +1,4 @@ -// +build !gce,!aws,!azure,!kubemark,!alicloud,!magnum,!digitalocean,!clusterapi,!huaweicloud,!ionoscloud +// +build !gce,!aws,!azure,!kubemark,!alicloud,!magnum,!digitalocean,!clusterapi,!huaweicloud,!ionoscloud,!linode /* Copyright 2018 The Kubernetes Authors. @@ -31,6 +31,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/gce" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/huaweicloud" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/ionoscloud" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/linode" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/magnum" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/ovhcloud" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/packet" @@ -52,6 +53,7 @@ var AvailableCloudProviders = []string{ cloudprovider.OVHcloudProviderName, clusterapi.ProviderName, cloudprovider.IonoscloudProviderName, + cloudprovider.LinodeProviderName, } // DefaultCloudProvider is GCE. @@ -87,6 +89,8 @@ func buildCloudProvider(opts config.AutoscalingOptions, do cloudprovider.NodeGro return clusterapi.BuildClusterAPI(opts, do, rl) case cloudprovider.IonoscloudProviderName: return ionoscloud.BuildIonosCloud(opts, do, rl) + case cloudprovider.LinodeProviderName: + return linode.BuildLinode(opts, do, rl) } return nil } diff --git a/cluster-autoscaler/cloudprovider/builder/builder_linode.go b/cluster-autoscaler/cloudprovider/builder/builder_linode.go new file mode 100644 index 000000000000..8685b9b94056 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/builder/builder_linode.go @@ -0,0 +1,42 @@ +// +build linode + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/linode" + "k8s.io/autoscaler/cluster-autoscaler/config" +) + +// AvailableCloudProviders supported by the cloud provider builder. +var AvailableCloudProviders = []string{ + cloudprovider.LinodeProviderName, +} + +// DefaultCloudProvider for linode-only build is linode. +const DefaultCloudProvider = cloudprovider.LinodeProviderName + +func buildCloudProvider(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscoveryOptions, rl *cloudprovider.ResourceLimiter) cloudprovider.CloudProvider { + switch opts.CloudProviderName { + case cloudprovider.LinodeProviderName: + return linode.BuildLinode(opts, do, rl) + } + + return nil +} diff --git a/cluster-autoscaler/cloudprovider/cloud_provider.go b/cluster-autoscaler/cloudprovider/cloud_provider.go index 83c767167f06..4c0f5d5ca88e 100644 --- a/cluster-autoscaler/cloudprovider/cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/cloud_provider.go @@ -53,6 +53,8 @@ const ( IonoscloudProviderName = "ionoscloud" // OVHcloudProviderName gets the provider name of ovhcloud OVHcloudProviderName = "ovhcloud" + // LinodeProviderName gets the provider name of linode + LinodeProviderName = "linode" ) // CloudProvider contains configuration info and functions for interacting with diff --git a/cluster-autoscaler/cloudprovider/linode/README.md b/cluster-autoscaler/cloudprovider/linode/README.md new file mode 100644 index 000000000000..26166d376b54 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/README.md @@ -0,0 +1,62 @@ +# Cluster Autoscaler for Linode + +The cluster autoscaler for Linode scales nodes in a LKE cluster. + +## Linode Kubernetes Engine + +Linode Kubernetes Engine ([LKE](https://www.linode.com/docs/guides/deploy-and-manage-a-cluster-with-linode-kubernetes-engine-a-tutorial/)) is the managed K8s solution provided by Linode. + +LKE lets users create Node Pools, i.e. groups of nodes (also called Linodes) each of the same type. + +The size of a Node Pool can be configured at any moment. The user cannot select specific nodes to be deleted when downsizing a Node Pool, rather, Linode will randomly select nodes to be deleted to reach the defined size, even if a node is not healthy or has been manually deleted. + +Nodes in a Node Pool are considered disposable: they can be deleted and recreated at any moment, deleting a single node or using the *recycle* feature, on these cases the node will be recreated by Linode after a small amount of time. + +Node Pools do not support user defined tags or labels. + +There is no limitation on the number of Node Pool a LKE Cluster can have, limited to the maximum number of nodes an LKE Cluster can have. + +## Cluster Autoscaler + +A cluster autoscaler node group is composed of multiple LKE Node Pools of the same Linode type (e.g. g6-standard-1, g6-standard-2), each holding a *single* Linode. + +At every scan interval the cluster autoscaler reviews every LKE Node Pool and if: +* it is not among the ones to be excluded as defined in the configuration file; +* it holds a single Linode; + +then it becomes part of the node group holding LKE Node Pools of the Linode type it has, or a node group is created with this LKE Node Pool inside if there are no node groups with that Linode type at the moment. + +Scaling is achieved adding LKE Node Pools to node groups, *not* increasing the size of a LKE Node Pool, that must stay 1. The reason behind this is that Linode does not provide a way to selectively delete a Linode from a LKE Node Pool and decrease the size of the pool with it. + +This is also the reason we cannot use the standard `nodes` and `node-group-auto-discovery` cluster autoscaler flag (no labels could be used to select a specific node group), and the reason why there can be no node group of the same type. + +## Configuration + +The cluster autoscaler automatically select every LKE Node Pool that is part of a LKE cluster, so there is no need define the `node-group-auto-discovery` or `nodes` flags, see [examples/cluster-autoscaler-autodiscover.yaml](examples/cluster-autoscaler-autodiscover.yaml) for an example of a kubernetes deployment. + +It is mandatory to define the cloud configuration file `cloud-config`. +You can see an example of the cloud config file at [examples/cluster-autoscaler-secret.yaml](examples/cluster-autoscaler-secret.yaml), it is an INI file with the following fields: + +| Key | Value | Mandatory | Default | +|-----|-------|-----------|---------| +| global/linode-token | Linode API Token with Read/Write permission for Kubernetes and Linodes | yes | none | +| global/lke-cluster-id | ID of the LKE cluster (numeric of the form: 12345, you can get this via `linode-cli` or looking at the first number of a linode in a pool, e.g. for lke15989-19461-5fec9212fad2 the lke-cluster-id is "15989") | yes | none | +| global/defaut-min-size-per-linode-type | minimum size of a node group (must be > 0) | no | 1 | +| global/defaut-max-size-per-linode-type | maximum size of a node group | no | 254 | +| global/do-not-import-pool-id | Pool id (numeric of the form: 12345) that will be excluded from the pools managed by the cluster autoscaler; can be repeated | no | none +| nodegroup \"linode_type\"/min-size" | minimum size for a specific node group | no | global/defaut-min-size-per-linode-type | +| nodegroup \"linode_type\"/max-size" | maximum size for a specific node group | no | global/defaut-min-size-per-linode-type | + +Log levels of intertest for the Linode provider are: +* 1 (flag: ```--v=1```): basic logging at start; +* 2 (flag: ```--v=2```): logging of the node group composition at every scan; + +## Development + +Make sure you are inside the `cluster-autoscaler` path of the [autoscaler repository](https://github.com/kubernetes/autoscaler). + +Create the docker image: +``` +make container +``` +tag the generated docker image and push it to a registry. diff --git a/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscale-pdb.yaml b/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscale-pdb.yaml new file mode 100644 index 000000000000..e811124b651d --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscale-pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: cluster-autoscaler-pdb +spec: + minAvailable: 1 + selector: + matchLabels: + app: cluster-autoscaler \ No newline at end of file diff --git a/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-autodiscover.yaml b/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-autodiscover.yaml new file mode 100644 index 000000000000..0d42426c1887 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-autodiscover.yaml @@ -0,0 +1,169 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8085' + spec: + serviceAccountName: cluster-autoscaler + containers: + - image: k8s.gcr.io/autoscaling/cluster-autoscaler:latest + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=2 + - --cloud-provider=linode + - --cloud-config=/config/cloud-config + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + - name: cloud-config + mountPath: /config + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" + - name: cloud-config + secret: + secretName: cluster-autoscaler-cloud-config \ No newline at end of file diff --git a/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-secret.yaml b/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-secret.yaml new file mode 100644 index 000000000000..db7c8371ada2 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-secret.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: cluster-autoscaler-cloud-config + namespace: kube-system +type: Opaque +stringData: + cloud-config: |- + [global] + linode-token=55b4c52123462dk23sa165cc3dba1234123125666f1422c8 + lke-cluster-id=12345 + defaut-min-size-per-linode-type=1 + defaut-max-size-per-linode-type=10 + do-not-import-pool-id=19462 + do-not-import-pool-id=19763 + + [nodegroup "g6-standard-1"] + min-size=2 + max-size=5 + + [nodegroup "g6-standard-2"] + max-size=4 + + [nodegroup "g6-standard-4"] + max-size=2 \ No newline at end of file diff --git a/cluster-autoscaler/cloudprovider/linode/linode_api_client.go b/cluster-autoscaler/cloudprovider/linode/linode_api_client.go new file mode 100644 index 000000000000..d695bc3adf03 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linode_api_client.go @@ -0,0 +1,52 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linode + +import ( + "context" + "net/http" + "time" + + "golang.org/x/oauth2" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/linode/linodego" + "k8s.io/autoscaler/cluster-autoscaler/version" +) + +const ( + userAgent = "kubernetes/cluster-autoscaler/" + version.ClusterAutoscalerVersion +) + +// linodeAPIClient is the interface used to call linode API +type linodeAPIClient interface { + ListLKEClusterPools(ctx context.Context, clusterID int, opts *linodego.ListOptions) ([]linodego.LKEClusterPool, error) + CreateLKEClusterPool(ctx context.Context, clusterID int, createOpts linodego.LKEClusterPoolCreateOptions) (*linodego.LKEClusterPool, error) + DeleteLKEClusterPool(ctx context.Context, clusterID int, id int) error +} + +// buildLinodeAPIClient returns the struct ready to perform calls to linode API +func buildLinodeAPIClient(linodeToken string) linodeAPIClient { + tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: linodeToken}) + oauth2Client := &http.Client{ + Timeout: 60 * time.Second, + Transport: &oauth2.Transport{ + Source: tokenSource, + }, + } + client := linodego.NewClient(oauth2Client) + client.SetUserAgent(userAgent) + return &client +} diff --git a/cluster-autoscaler/cloudprovider/linode/linode_cloud_config.go b/cluster-autoscaler/cloudprovider/linode/linode_cloud_config.go new file mode 100644 index 000000000000..03098d690d62 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linode_cloud_config.go @@ -0,0 +1,164 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linode + +import ( + "fmt" + "io" + "strconv" + + "gopkg.in/gcfg.v1" +) + +const ( + // defaultMinSizePerLinodeType is the min size of the node groups + // if no other value is defined for a specific node group. + defaultMinSizePerLinodeType int = 1 + // defaultMaxSizePerLinodeType is the max size of the node groups + // if no other value is defined for a specific node group. + defaultMaxSizePerLinodeType int = 254 +) + +// nodeGroupConfig is the configuration for a specific node group. +type nodeGroupConfig struct { + minSize int + maxSize int +} + +// linodeConfig holds the configuration for the linode provider. +type linodeConfig struct { + clusterID int + token string + defaultMinSize int + defaultMaxSize int + excludedPoolIDs map[int]bool + nodeGroupCfg map[string]*nodeGroupConfig +} + +// GcfgGlobalConfig is the gcfg representation of the global section in the cloud config file for linode. +type GcfgGlobalConfig struct { + ClusterID string `gcfg:"lke-cluster-id"` + Token string `gcfg:"linode-token"` + DefaultMinSize string `gcfg:"defaut-min-size-per-linode-type"` + DefaultMaxSize string `gcfg:"defaut-max-size-per-linode-type"` + ExcludedPoolIDs []string `gcfg:"do-not-import-pool-id"` +} + +// GcfgNodeGroupConfig is the gcfg representation of the section in the cloud config file to change defaults for a node group. +type GcfgNodeGroupConfig struct { + MinSize string `gcfg:"min-size"` + MaxSize string `gcfg:"max-size"` +} + +// gcfgCloudConfig is the gcfg representation of the cloud config file for linode. +type gcfgCloudConfig struct { + Global GcfgGlobalConfig `gcfg:"global"` + NodeGroups map[string]*GcfgNodeGroupConfig `gcfg:"nodegroup"` +} + +// buildCloudConfig creates the configuration struct for the provider. +func buildCloudConfig(config io.Reader) (*linodeConfig, error) { + + // read the config and get the gcfg struct + var gcfgCloudConfig gcfgCloudConfig + if err := gcfg.ReadInto(&gcfgCloudConfig, config); err != nil { + return nil, err + } + + // get the clusterID + clusterID, err := strconv.Atoi(gcfgCloudConfig.Global.ClusterID) + if err != nil { + return nil, fmt.Errorf("LKE Cluster ID %q is not a number: %v", gcfgCloudConfig.Global.ClusterID, err) + } + + // get the linode token + token := gcfgCloudConfig.Global.Token + if len(gcfgCloudConfig.Global.Token) == 0 { + return nil, fmt.Errorf("linode token not present in global section") + } + + // get the list of LKE pools that must not be imported + excludedPoolIDs := make(map[int]bool) + for _, excludedPoolIDStr := range gcfgCloudConfig.Global.ExcludedPoolIDs { + excludedPoolID, err := strconv.Atoi(excludedPoolIDStr) + if err != nil { + return nil, fmt.Errorf("excluded pool id %q is not a number: %v", excludedPoolIDStr, err) + } + excludedPoolIDs[excludedPoolID] = true + } + + // get the default min and max size as defined in the global section of the config file + defaultMinSize, defaultMaxSize, err := getSizeLimits( + gcfgCloudConfig.Global.DefaultMinSize, + gcfgCloudConfig.Global.DefaultMaxSize, + defaultMinSizePerLinodeType, + defaultMaxSizePerLinodeType) + if err != nil { + return nil, fmt.Errorf("cannot get default size values in global section: %v", err) + } + + // get the specific configuration of a node group + nodeGroupCfg := make(map[string]*nodeGroupConfig) + for nodeType, gcfgNodeGroup := range gcfgCloudConfig.NodeGroups { + minSize, maxSize, err := getSizeLimits(gcfgNodeGroup.MinSize, gcfgNodeGroup.MaxSize, defaultMinSize, defaultMaxSize) + if err != nil { + return nil, fmt.Errorf("cannot get size values for node group %q: %v", nodeType, err) + } + ngc := &nodeGroupConfig{ + maxSize: maxSize, + minSize: minSize, + } + nodeGroupCfg[nodeType] = ngc + } + + return &linodeConfig{ + clusterID: clusterID, + token: token, + defaultMinSize: defaultMinSize, + defaultMaxSize: defaultMaxSize, + excludedPoolIDs: excludedPoolIDs, + nodeGroupCfg: nodeGroupCfg, + }, nil +} + +// getSizeLimits takes the max, min size of a node group as strings (empty if no values are provided) +// and default sizes, validates them an returns them as integer, or an error if such occurred +func getSizeLimits(minStr string, maxStr string, defaultMin int, defaultMax int) (int, int, error) { + var err error + min := defaultMin + if len(minStr) != 0 { + min, err = strconv.Atoi(minStr) + if err != nil { + return 0, 0, fmt.Errorf("could not parse min size for node group: %v", err) + } + } + if min < 1 { + return 0, 0, fmt.Errorf("min size for node group cannot be < 1") + } + max := defaultMax + if len(maxStr) != 0 { + max, err = strconv.Atoi(maxStr) + if err != nil { + return 0, 0, fmt.Errorf("could not parse max size for node group: %v", err) + } + } + if min > max { + return 0, 0, fmt.Errorf("min size for a node group must be less than its max size (got min: %d, max: %d)", + min, max) + } + return min, max, nil +} diff --git a/cluster-autoscaler/cloudprovider/linode/linode_cloud_config_test.go b/cluster-autoscaler/cloudprovider/linode/linode_cloud_config_test.go new file mode 100644 index 000000000000..42e2c8238aa6 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linode_cloud_config_test.go @@ -0,0 +1,160 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linode + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCludConfig_getSizeLimits(t *testing.T) { + _, _, err := getSizeLimits("3", "2", 1, 2) + assert.Error(t, err, "no errors on minSize > maxSize") + + _, _, err = getSizeLimits("4", "", 2, 3) + assert.Error(t, err, "no errors on minSize > maxSize using defaults") + + _, _, err = getSizeLimits("", "4", 5, 10) + assert.Error(t, err, "no errors on minSize > maxSize using defaults") + + _, _, err = getSizeLimits("-1", "4", 5, 10) + assert.Error(t, err, "no errors on minSize <= 0") + + _, _, err = getSizeLimits("1", "4a", 5, 10) + assert.Error(t, err, "no error on malformed integer string") + + _, _, err = getSizeLimits("1.0", "4", 5, 10) + assert.Error(t, err, "no error on malformed integer string") + + min, max, err := getSizeLimits("", "", 1, 2) + assert.Equal(t, 1, min) + assert.Equal(t, 2, max) + + min, max, err = getSizeLimits("", "3", 1, 2) + assert.Equal(t, 1, min) + assert.Equal(t, 3, max) + + min, max, err = getSizeLimits("6", "8", 1, 2) + assert.Equal(t, 6, min) + assert.Equal(t, 8, max) +} + +func TestCludConfig_buildCloudConfig(t *testing.T) { + cfg := strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +defaut-min-size-per-linode-type=2 +defaut-max-size-per-linode-type=10 +do-not-import-pool-id=888 +do-not-import-pool-id=999 + +[nodegroup "g6-standard-1"] +min-size=1 +max-size=2 + +[nodegroup "g6-standard-2"] +min-size=4 +max-size=5 +`) + config, err := buildCloudConfig(cfg) + assert.NoError(t, err) + assert.Equal(t, "123123123", config.token) + assert.Equal(t, 456456, config.clusterID) + assert.Equal(t, 2, config.defaultMinSize) + assert.Equal(t, 10, config.defaultMaxSize) + assert.Equal(t, true, config.excludedPoolIDs[999]) + assert.Equal(t, true, config.excludedPoolIDs[888]) + assert.Equal(t, 1, config.nodeGroupCfg["g6-standard-1"].minSize) + assert.Equal(t, 2, config.nodeGroupCfg["g6-standard-1"].maxSize) + assert.Equal(t, 4, config.nodeGroupCfg["g6-standard-2"].minSize) + assert.Equal(t, 5, config.nodeGroupCfg["g6-standard-2"].maxSize) + + cfg = strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +defaut-max-size-per-linode-type=20 + +[nodegroup "g6-standard-1"] +max-size=10 + +[nodegroup "g6-standard-2"] +min-size=3 +`) + config, err = buildCloudConfig(cfg) + assert.NoError(t, err) + assert.Equal(t, 1, config.nodeGroupCfg["g6-standard-1"].minSize) + assert.Equal(t, 10, config.nodeGroupCfg["g6-standard-1"].maxSize) + assert.Equal(t, 3, config.nodeGroupCfg["g6-standard-2"].minSize) + assert.Equal(t, 20, config.nodeGroupCfg["g6-standard-2"].maxSize) + + cfg = strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +defaut-max-size-per-linode-type=20 + +[nodegroup "g6-standard-1"] +max-size=10a +`) + config, err = buildCloudConfig(cfg) + assert.Error(t, err, "no error on size of a specific node group is not an integer string") + + cfg = strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +do-not-import-pool-id=99f +`) + config, err = buildCloudConfig(cfg) + assert.Error(t, err, "no error on excluded pool id is not an integer string") + + cfg = strings.NewReader(` +[global] +lke-cluster-id=456456 +`) + config, err = buildCloudConfig(cfg) + assert.Error(t, err, "no error on missing linode token") + + cfg = strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456aaa +`) + config, err = buildCloudConfig(cfg) + assert.Error(t, err, "no error when lke cluster id is not an integer string") + + cfg = strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +defaut-max-size-per-linode-type=20.0 +`) + config, err = buildCloudConfig(cfg) + assert.Error(t, err, "no error when default max size in global section is not an integer string") + + cfg = strings.NewReader(` +[gglobal] +linode-token=123123123 +lke-cluster-id=456456 +`) + config, err = buildCloudConfig(cfg) + assert.Error(t, err, "no error when config has no global section") +} diff --git a/cluster-autoscaler/cloudprovider/linode/linode_cloud_provider.go b/cluster-autoscaler/cloudprovider/linode/linode_cloud_provider.go new file mode 100644 index 000000000000..b14e97f151ee --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linode_cloud_provider.go @@ -0,0 +1,166 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linode + +import ( + "fmt" + "io" + "os" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/config" + "k8s.io/autoscaler/cluster-autoscaler/utils/errors" + klog "k8s.io/klog/v2" +) + +// linodeCloudProvider implements CloudProvider interface. +type linodeCloudProvider struct { + manager *manager + resourceLimiter *cloudprovider.ResourceLimiter +} + +// Name returns name of the cloud provider. +func (l *linodeCloudProvider) Name() string { + return cloudprovider.LinodeProviderName +} + +// NodeGroups returns all node groups configured for this cloud provider. +func (l *linodeCloudProvider) NodeGroups() []cloudprovider.NodeGroup { + nodeGroups := make([]cloudprovider.NodeGroup, len(l.manager.nodeGroups)) + i := 0 + for _, ng := range l.manager.nodeGroups { + nodeGroups[i] = ng + i++ + } + return nodeGroups +} + +// NodeGroupForNode returns the node group for the given node, nil if the node +// should not be processed by cluster autoscaler, or non-nil error if such +// occurred. Must be implemented. +func (l *linodeCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.NodeGroup, error) { + for _, ng := range l.manager.nodeGroups { + pool, err := ng.findLKEPoolForNode(node) + if err != nil { + return nil, err + } + if pool != nil { + return ng, nil + } + } + return nil, nil +} + +// Pricing returns pricing model for this cloud provider or error if not available. +// Implementation optional. +func (l *linodeCloudProvider) Pricing() (cloudprovider.PricingModel, errors.AutoscalerError) { + return nil, cloudprovider.ErrNotImplemented +} + +// GetAvailableMachineTypes get all machine types that can be requested from the cloud provider. +// Implementation optional. +func (l *linodeCloudProvider) GetAvailableMachineTypes() ([]string, error) { + return []string{}, cloudprovider.ErrNotImplemented +} + +// NewNodeGroup builds a theoretical node group based on the node definition provided. The node group is not automatically +// created on the cloud provider side. The node group is not returned by NodeGroups() until it is created. +// Implementation optional. +func (l *linodeCloudProvider) NewNodeGroup(machineType string, labels map[string]string, systemLabels map[string]string, + taints []apiv1.Taint, extraResources map[string]resource.Quantity) (cloudprovider.NodeGroup, error) { + return nil, cloudprovider.ErrNotImplemented +} + +// GetResourceLimiter returns struct containing limits (max, min) for resources (cores, memory etc.). +func (l *linodeCloudProvider) GetResourceLimiter() (*cloudprovider.ResourceLimiter, error) { + return l.resourceLimiter, nil +} + +// GPULabel returns the label added to nodes with GPU resource. +func (l *linodeCloudProvider) GPULabel() string { + return "" +} + +// GetAvailableGPUTypes return all available GPU types cloud provider supports. +func (l *linodeCloudProvider) GetAvailableGPUTypes() map[string]struct{} { + return nil +} + +// Cleanup cleans up open resources before the cloud provider is destroyed, i.e. go routines etc. +func (l *linodeCloudProvider) Cleanup() error { + return nil +} + +// BuildLinode builds the BuildLinode cloud provider. +func BuildLinode( + opts config.AutoscalingOptions, + do cloudprovider.NodeGroupDiscoveryOptions, + rl *cloudprovider.ResourceLimiter, +) cloudprovider.CloudProvider { + + // the cloud provider automatically uses all node pools in linode. + // This means we don't use the cloudprovider.NodeGroupDiscoveryOptions + // flags (which can be set via '--node-group-auto-discovery' or '-nodes') + + if opts.CloudConfig == "" { + klog.Fatalf("No config file provided, please specify it via the --cloud-config flag") + } + configFile, err := os.Open(opts.CloudConfig) + if err != nil { + klog.Fatalf("Could not open cloud provider configuration file %q, error: %v", opts.CloudConfig, err) + } + defer configFile.Close() + lcp, err := newLinodeCloudProvider(configFile, rl) + if err != nil { + klog.Fatalf("Could not create linode cloud provider: %v", err) + } + return lcp +} + +// Refresh is called before every main loop and can be used to dynamically update cloud provider state. +// In particular the list of node groups returned by NodeGroups can change as a result of CloudProvider.Refresh(). +func (l *linodeCloudProvider) Refresh() error { + return l.manager.refresh() +} + +func newLinodeCloudProvider(config io.Reader, rl *cloudprovider.ResourceLimiter) (cloudprovider.CloudProvider, error) { + m, err := newManager(config) + if err != nil { + return nil, fmt.Errorf("could not create linode manager: %v", err) + } + + err = m.refresh() + if err != nil { + klog.V(1).Infof("Error on first import of LKE node pools: %v", err) + } + klog.V(1).Infof("First import of existing LKE node pools ended") + if len(m.nodeGroups) == 0 { + klog.V(1).Infof("Could not import any LKE node pool in any node group") + } else { + klog.V(1).Infof("imported LKE node pools:") + for _, ng := range m.nodeGroups { + klog.V(1).Infof("%s", ng.extendedDebug()) + } + } + + return &linodeCloudProvider{ + manager: m, + resourceLimiter: rl, + }, nil +} diff --git a/cluster-autoscaler/cloudprovider/linode/linode_cloud_provider_test.go b/cluster-autoscaler/cloudprovider/linode/linode_cloud_provider_test.go new file mode 100644 index 000000000000..7889415866bc --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linode_cloud_provider_test.go @@ -0,0 +1,188 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linode + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + apiv1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/linode/linodego" +) + +func TestCloudProvider_newLinodeCloudProvider(t *testing.T) { + // test ok on correctly creating a linode provider + rl := &cloudprovider.ResourceLimiter{} + cfg := strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +`) + _, err := newLinodeCloudProvider(cfg, rl) + assert.NoError(t, err) + + // test error on creating a linode provider when config is bad + cfg = strings.NewReader(` +[globalxxx] +linode-token=123123123 +lke-cluster-id=456456 +`) + _, err = newLinodeCloudProvider(cfg, rl) + assert.Error(t, err) +} + +func TestCloudProvider_NodeGroups(t *testing.T) { + // test ok on getting the correct nodes when calling NodeGroups() + cfg := strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +`) + m, _ := newManager(cfg) + m.nodeGroups = map[string]*NodeGroup{ + "g6-standard-1": {id: "g6-standard-1"}, + "g6-standard-2": {id: "g6-standard-2"}, + } + lcp := &linodeCloudProvider{manager: m} + ng := lcp.NodeGroups() + assert.Equal(t, 2, len(ng)) + assert.Contains(t, ng, m.nodeGroups["g6-standard-1"]) + assert.Contains(t, ng, m.nodeGroups["g6-standard-2"]) +} + +func TestCloudProvider_NodeGroupForNode(t *testing.T) { + cfg := strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +`) + m, _ := newManager(cfg) + ng1 := &NodeGroup{ + id: "g6-standard-1", + lkePools: map[int]*linodego.LKEClusterPool{ + 1: {ID: 1, + Count: 2, + Type: "g6-standard-1", + Linodes: []linodego.LKEClusterPoolLinode{ + {InstanceID: 111}, + {InstanceID: 222}, + }, + }, + 2: {ID: 2, + Count: 2, + Type: "g6-standard-1", + Linodes: []linodego.LKEClusterPoolLinode{ + {InstanceID: 333}, + }, + }, + 3: {ID: 3, + Count: 2, + Type: "g6-standard-1", + Linodes: []linodego.LKEClusterPoolLinode{ + {InstanceID: 444}, + {InstanceID: 555}, + }, + }, + }, + } + ng2 := &NodeGroup{ + id: "g6-standard-2", + lkePools: map[int]*linodego.LKEClusterPool{ + 4: {ID: 4, + Count: 2, + Type: "g6-standard-2", + Linodes: []linodego.LKEClusterPoolLinode{ + {InstanceID: 666}, + {InstanceID: 777}, + }, + }, + }, + } + m.nodeGroups = map[string]*NodeGroup{ + "g6-standard-1": ng1, + "g6-standard-2": ng2, + } + lcp := &linodeCloudProvider{manager: m} + + // test ok on getting the right node group for an apiv1.Node + node := &apiv1.Node{ + Spec: apiv1.NodeSpec{ + ProviderID: "linode://555", + }, + } + ng, err := lcp.NodeGroupForNode(node) + assert.NoError(t, err) + assert.Equal(t, ng1, ng) + + // test ok on getting the right node group for an apiv1.Node + node = &apiv1.Node{ + Spec: apiv1.NodeSpec{ + ProviderID: "linode://666", + }, + } + ng, err = lcp.NodeGroupForNode(node) + assert.NoError(t, err) + assert.Equal(t, ng2, ng) + + // test ok on getting nil when looking for a apiv1.Node we do not manage + node = &apiv1.Node{ + Spec: apiv1.NodeSpec{ + ProviderID: "linode://999", + }, + } + ng, err = lcp.NodeGroupForNode(node) + assert.NoError(t, err) + assert.Nil(t, ng) + + // test error on looking for a apiv1.Node with a bad providerID + node = &apiv1.Node{ + Spec: apiv1.NodeSpec{ + ProviderID: "linode://aaa", + }, + } + ng, err = lcp.NodeGroupForNode(node) + assert.Error(t, err) + +} + +func TestCloudProvider_others(t *testing.T) { + cfg := strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +`) + m, _ := newManager(cfg) + resourceLimiter := &cloudprovider.ResourceLimiter{} + lcp := &linodeCloudProvider{ + manager: m, + resourceLimiter: resourceLimiter, + } + assert.Equal(t, cloudprovider.LinodeProviderName, lcp.Name()) + _, err := lcp.GetAvailableMachineTypes() + assert.Error(t, err) + _, err = lcp.NewNodeGroup("", nil, nil, nil, nil) + assert.Error(t, err) + rl, err := lcp.GetResourceLimiter() + assert.Equal(t, resourceLimiter, rl) + assert.Equal(t, "", lcp.GPULabel()) + assert.Nil(t, lcp.GetAvailableGPUTypes()) + assert.Nil(t, lcp.Cleanup()) + _, err2 := lcp.Pricing() + assert.Error(t, err2) +} diff --git a/cluster-autoscaler/cloudprovider/linode/linode_manager.go b/cluster-autoscaler/cloudprovider/linode/linode_manager.go new file mode 100644 index 000000000000..0f74d59472d8 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linode_manager.go @@ -0,0 +1,131 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linode + +import ( + "context" + "fmt" + "io" + + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/linode/linodego" + klog "k8s.io/klog/v2" +) + +// manager handles Linode communication and holds information about +// the node groups (LKE pools with a single linode each) +type manager struct { + client linodeAPIClient + config *linodeConfig + nodeGroups map[string]*NodeGroup // key: NodeGroup.id +} + +func newManager(config io.Reader) (*manager, error) { + cfg, err := buildCloudConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to parse config: %v", err) + } + client := buildLinodeAPIClient(cfg.token) + m := &manager{ + client: client, + config: cfg, + nodeGroups: make(map[string]*NodeGroup), + } + return m, nil +} + +func (m *manager) refresh() error { + nodeGroups := make(map[string]*NodeGroup) + lkeClusterPools, err := m.client.ListLKEClusterPools(context.Background(), m.config.clusterID, nil) + if err != nil { + return fmt.Errorf("failed to get list of LKE pools from linode API: %v", err) + } + + for i, pool := range lkeClusterPools { + // skip this pool if it is among the ones to be excluded as defined in the config + _, found := m.config.excludedPoolIDs[pool.ID] + if found { + continue + } + // check if the nodes in the pool are more than 1, if so skip it + if pool.Count > 1 { + klog.V(2).Infof("The LKE pool %d has more than one node (current nodes in pool: %d), will exclude it from the node groups", + pool.ID, pool.Count) + continue + } + // add the LKE pool to the node groups map + linodeType := pool.Type + ng, found := nodeGroups[linodeType] + if found { + // if a node group for the node type of this pool already exists, add it to the related node group + // TODO if node group size is exceeded better to skip it or add it anyway? here we are adding it + ng.lkePools[pool.ID] = &lkeClusterPools[i] + } else { + // create a new node group with this pool in it + ng := buildNodeGroup(&lkeClusterPools[i], m.config, m.client) + nodeGroups[linodeType] = ng + } + } + + // show some debug info + klog.V(2).Infof("LKE node group after refresh:") + for _, ng := range nodeGroups { + klog.V(2).Infof("%s", ng.extendedDebug()) + } + for _, ng := range nodeGroups { + currentSize := len(ng.lkePools) + if currentSize > ng.maxSize { + klog.V(2).Infof("imported node pools in node group %q are > maxSize (current size: %d, min size: %d, max size: %d)", + ng.id, currentSize, ng.minSize, ng.maxSize) + } + if currentSize < ng.minSize { + klog.V(2).Infof("imported node pools in node group %q are < minSize (current size: %d, min size: %d, max size: %d)", + ng.id, currentSize, ng.minSize, ng.maxSize) + } + } + + m.nodeGroups = nodeGroups + return nil +} + +func buildNodeGroup(pool *linodego.LKEClusterPool, cfg *linodeConfig, client linodeAPIClient) *NodeGroup { + // get specific min and max size for a node group, if defined in the config + minSize := cfg.defaultMinSize + maxSize := cfg.defaultMaxSize + nodeGroupCfg, found := cfg.nodeGroupCfg[pool.Type] + if found { + minSize = nodeGroupCfg.minSize + maxSize = nodeGroupCfg.maxSize + } + // create the new node group with this single LKE pool inside + lkePools := make(map[int]*linodego.LKEClusterPool) + lkePools[pool.ID] = pool + poolOpts := linodego.LKEClusterPoolCreateOptions{ + Count: 1, + Type: pool.Type, + Disks: pool.Disks, + } + ng := &NodeGroup{ + client: client, + lkePools: lkePools, + poolOpts: poolOpts, + lkeClusterID: cfg.clusterID, + minSize: minSize, + maxSize: maxSize, + id: pool.Type, + } + return ng +} diff --git a/cluster-autoscaler/cloudprovider/linode/linode_manager_test.go b/cluster-autoscaler/cloudprovider/linode/linode_manager_test.go new file mode 100644 index 000000000000..c66fff2f2368 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linode_manager_test.go @@ -0,0 +1,148 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linode + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/linode/linodego" +) + +func TestManager_newManager(t *testing.T) { + cfg := strings.NewReader(` +[globalxxx] +linode-token=123123123 +lke-cluster-id=456456 +`) + _, err := newManager(cfg) + assert.Error(t, err) + + cfg = strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +`) + _, err = newManager(cfg) + assert.NoError(t, err) +} + +func TestManager_refresh(t *testing.T) { + + cfg := strings.NewReader(` +[global] +linode-token=123123123 +lke-cluster-id=456456 +defaut-min-size-per-linode-type=2 +defaut-max-size-per-linode-type=10 +do-not-import-pool-id=888 +do-not-import-pool-id=999 + +[nodegroup "g6-standard-1"] +min-size=1 +max-size=2 + +[nodegroup "g6-standard-2"] +min-size=4 +max-size=5 +`) + m, err := newManager(cfg) + assert.NoError(t, err) + + client := linodeClientMock{} + m.client = &client + ctx := context.Background() + + // test multiple pools with same type + client.On( + "ListLKEClusterPools", ctx, 456456, nil, + ).Return( + []linodego.LKEClusterPool{ + {ID: 1, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{ID: "aaa", InstanceID: 123}}}, + {ID: 2, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{ID: "bbb", InstanceID: 345}}}, + {ID: 3, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{ID: "ccc", InstanceID: 678}}}, + }, + nil, + ).Once() + err = m.refresh() + assert.NoError(t, err) + assert.Equal(t, 1, len(m.nodeGroups)) + assert.Equal(t, 3, len(m.nodeGroups["g6-standard-1"].lkePools)) + + // test skip pools with count > 1 + client.On( + "ListLKEClusterPools", ctx, 456456, nil, + ).Return( + []linodego.LKEClusterPool{ + {ID: 1, Count: 1, Type: "g6-standard-1"}, + {ID: 2, Count: 1, Type: "g6-standard-1"}, + {ID: 3, Count: 2, Type: "g6-standard-1"}, + }, + nil, + ).Once() + err = m.refresh() + assert.NoError(t, err) + assert.Equal(t, 1, len(m.nodeGroups)) + assert.Equal(t, 2, len(m.nodeGroups["g6-standard-1"].lkePools)) + + // test multiple pools with different types + client.On( + "ListLKEClusterPools", ctx, 456456, nil, + ).Return( + []linodego.LKEClusterPool{ + {ID: 1, Count: 1, Type: "g6-standard-1"}, + {ID: 2, Count: 1, Type: "g6-standard-1"}, + {ID: 3, Count: 1, Type: "g6-standard-2"}, + }, + nil, + ).Once() + err = m.refresh() + assert.NoError(t, err) + assert.Equal(t, 2, len(m.nodeGroups)) + assert.Equal(t, 2, len(m.nodeGroups["g6-standard-1"].lkePools)) + assert.Equal(t, 1, len(m.nodeGroups["g6-standard-2"].lkePools)) + + // test avoid import of specific pools + client.On( + "ListLKEClusterPools", ctx, 456456, nil, + ).Return( + []linodego.LKEClusterPool{ + {ID: 1, Count: 1, Type: "g6-standard-1"}, + {ID: 888, Count: 1, Type: "g6-standard-1"}, + {ID: 999, Count: 1, Type: "g6-standard-1"}, + }, + nil, + ).Once() + err = m.refresh() + assert.NoError(t, err) + assert.Equal(t, 1, len(m.nodeGroups)) + assert.Equal(t, 1, len(m.nodeGroups["g6-standard-1"].lkePools)) + + // test api error + client.On( + "ListLKEClusterPools", ctx, 456456, nil, + ).Return( + []linodego.LKEClusterPool{}, + fmt.Errorf("error on API call"), + ).Once() + err = m.refresh() + assert.Error(t, err) + +} diff --git a/cluster-autoscaler/cloudprovider/linode/linode_node_group.go b/cluster-autoscaler/cloudprovider/linode/linode_node_group.go new file mode 100644 index 000000000000..c673a0fe5f19 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linode_node_group.go @@ -0,0 +1,265 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linode + +import ( + "context" + "fmt" + "strconv" + "strings" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/linode/linodego" + klog "k8s.io/klog/v2" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" +) + +const ( + providerIDPrefix = "linode://" +) + +// NodeGroup implements cloudprovider.NodeGroup interface. NodeGroup contains +// configuration info and functions to control a set of nodes that have the +// same capacity and set of labels. +// +// Receivers assume all fields are initialized (i.e. not nil). +// +// We cannot use an LKE pool as node group because LKE does not provide a way to +// delete a specific linode in a pool, but provides an API to selectively delete +// a LKE pool. To get around this issue, we build a NodeGroup with multiple LKE pools, +// each with a single linode in them. +type NodeGroup struct { + client linodeAPIClient + lkePools map[int]*linodego.LKEClusterPool // key: LKEClusterPool.ID + poolOpts linodego.LKEClusterPoolCreateOptions + lkeClusterID int + minSize int + maxSize int + id string // this is a LKEClusterPool Type +} + +// MaxSize returns maximum size of the node group. +func (n *NodeGroup) MaxSize() int { + return n.maxSize +} + +// MinSize returns minimum size of the node group. +func (n *NodeGroup) MinSize() int { + return n.minSize +} + +// TargetSize returns the current target size of the node group. It is possible +// that the number of nodes in Kubernetes is different at the moment but should +// be equal to Size() once everything stabilizes (new nodes finish startup and +// registration or removed nodes are deleted completely). Implementation +// required. +func (n *NodeGroup) TargetSize() (int, error) { + return len(n.lkePools), nil +} + +// IncreaseSize increases the size of the node group. To delete a node you need +// to explicitly name it and use DeleteNode. This function should wait until +// node group size is updated. Implementation required. +func (n *NodeGroup) IncreaseSize(delta int) error { + if delta <= 0 { + return fmt.Errorf("delta must be positive, have: %d", delta) + } + + currentSize := len(n.lkePools) + targetSize := currentSize + delta + if targetSize > n.MaxSize() { + return fmt.Errorf("size increase is too large. current: %d desired: %d max: %d", + currentSize, targetSize, n.MaxSize()) + } + + for i := 0; i < delta; i++ { + err := n.addNewLKEPool() + if err != nil { + return err + } + } + + return nil +} + +// DeleteNodes deletes nodes from this node group (and also increasing the size +// of the node group with that). Error is returned either on failure or if the +// given node doesn't belong to this node group. This function should wait +// until node group size is updated. Implementation required. +func (n *NodeGroup) DeleteNodes(nodes []*apiv1.Node) error { + for _, node := range nodes { + pool, err := n.findLKEPoolForNode(node) + if err != nil { + return err + } + if pool == nil { + return fmt.Errorf("Failed to delete node %q with provider ID %q: cannot find this node in the node group", + node.Name, node.Spec.ProviderID) + } + err = n.deleteLKEPool(pool.ID) + if err != nil { + return fmt.Errorf("Failed to delete node %q with provider ID %q: %v", + node.Name, node.Spec.ProviderID, err) + } + } + return nil +} + +// DecreaseTargetSize decreases the target size of the node group. This function +// doesn't permit to delete any existing node and can be used only to reduce the +// request for new nodes that have not been yet fulfilled. Delta should be negative. +// It is assumed that cloud provider will not delete the existing nodes when there +// is an option to just decrease the target. Implementation required. +func (n *NodeGroup) DecreaseTargetSize(delta int) error { + // requests for new nodes are always fulfilled so we cannot + // decrease the size without actually deleting nodes + return cloudprovider.ErrNotImplemented +} + +// Id returns an unique identifier of the node group. +func (n *NodeGroup) Id() string { + return n.id +} + +// Debug returns a string containing all information regarding this node group. +func (n *NodeGroup) Debug() string { + return fmt.Sprintf("node group ID: %s (min:%d max:%d)", n.Id(), n.MinSize(), n.MaxSize()) +} + +// extendendDebug returns a string containing detailed information regarding this node group. +func (n *NodeGroup) extendedDebug() string { + lkePoolsList := make([]string, 0) + for _, p := range n.lkePools { + linodesList := make([]string, 0) + for _, l := range p.Linodes { + linode := fmt.Sprintf("ID: %q, instanceID: %d", l.ID, l.InstanceID) + linodesList = append(linodesList, linode) + } + linodes := strings.Join(linodesList, "; ") + lkePool := fmt.Sprintf("{ poolID: %d, count: %d, type: %s, associated linodes: [%s] }", + p.ID, p.Count, p.Type, linodes) + lkePoolsList = append(lkePoolsList, lkePool) + } + lkePools := strings.Join(lkePoolsList, ", ") + return fmt.Sprintf("node group ID %s := min: %d, max: %d, LKEClusterID: %d, poolOpts: %+v, associated LKE pools: %s", + n.id, n.minSize, n.maxSize, n.lkeClusterID, n.poolOpts, lkePools) +} + +// Nodes returns a list of all nodes that belong to this node group. It is +// required that Instance objects returned by this method have Id field set. +// Other fields are optional. +func (n *NodeGroup) Nodes() ([]cloudprovider.Instance, error) { + nodes := make([]cloudprovider.Instance, 0) + for _, pool := range n.lkePools { + linodesCount := len(pool.Linodes) + if linodesCount != 1 { + klog.V(2).Infof("Number of linodes in LKE pool %d is not exactly 1 (count: %d)", pool.ID, linodesCount) + } + for _, linode := range pool.Linodes { + instance := cloudprovider.Instance{ + Id: "linode://" + strconv.Itoa(linode.InstanceID), + } + nodes = append(nodes, instance) + } + } + return nodes, nil +} + +// TemplateNodeInfo returns a schedulerframework.NodeInfo structure of an empty +// (as if just started) node. This will be used in scale-up simulations to +// predict what would a new node look like if a node group was expanded. The +// returned NodeInfo is expected to have a fully populated Node object, with +// all of the labels, capacity and allocatable information as well as all pods +// that are started on the node by default, using manifest (most likely only +// kube-proxy). Implementation optional. +func (n *NodeGroup) TemplateNodeInfo() (*schedulerframework.NodeInfo, error) { + return nil, cloudprovider.ErrNotImplemented +} + +// Exist checks if the node group really exists on the cloud provider side. +// Allows to tell the theoretical node group from the real one. Implementation +// required. +func (n *NodeGroup) Exist() bool { + return true +} + +// Create creates the node group on the cloud provider side. Implementation +// optional. +func (n *NodeGroup) Create() (cloudprovider.NodeGroup, error) { + return nil, cloudprovider.ErrNotImplemented +} + +// Delete deletes the node group on the cloud provider side. This will be +// executed only for autoprovisioned node groups, once their size drops to 0. +// Implementation optional. +func (n *NodeGroup) Delete() error { + return cloudprovider.ErrNotImplemented +} + +// Autoprovisioned returns true if the node group is autoprovisioned. An +// autoprovisioned group was created by CA and can be deleted when scaled to 0. +func (n *NodeGroup) Autoprovisioned() bool { + return false +} + +// addNewLKEPool creates a new LKE Pool with a single linode in it and add it +// to the pools of this node group +func (n *NodeGroup) addNewLKEPool() error { + ctx := context.Background() + newPool, err := n.client.CreateLKEClusterPool(ctx, n.lkeClusterID, n.poolOpts) + if err != nil { + return fmt.Errorf("error on creating new LKE pool for LKE clusterID: %d", n.lkeClusterID) + } + n.lkePools[newPool.ID] = newPool + return nil +} + +// deleteLKEPool deletes a pool given its pool id and remove it from the pools +// of this node group +func (n *NodeGroup) deleteLKEPool(id int) error { + _, ok := n.lkePools[id] + if !ok { + return fmt.Errorf("cannot delete LKE pool %d, this pool is not one we are managing", id) + } + ctx := context.Background() + err := n.client.DeleteLKEClusterPool(ctx, n.lkeClusterID, id) + if err != nil { + return fmt.Errorf("error on deleting LKE pool %d, Linode API said: %v", id, err) + } + delete(n.lkePools, id) + return nil +} + +// findLKEPoolForNode returns the LKE pool where this node is, nil otherwise +func (n *NodeGroup) findLKEPoolForNode(node *apiv1.Node) (*linodego.LKEClusterPool, error) { + providerID := node.Spec.ProviderID + instanceIDStr := strings.TrimPrefix(providerID, providerIDPrefix) + instanceID, err := strconv.Atoi(instanceIDStr) + if err != nil { + return nil, fmt.Errorf("Cannot convert ProviderID %q to linode istance id (must be of type %s)", + providerID, providerIDPrefix) + } + for _, pool := range n.lkePools { + for _, linode := range pool.Linodes { + if linode.InstanceID == instanceID { + return pool, nil + } + } + } + return nil, nil +} diff --git a/cluster-autoscaler/cloudprovider/linode/linode_node_group_test.go b/cluster-autoscaler/cloudprovider/linode/linode_node_group_test.go new file mode 100644 index 000000000000..44a6dd519c31 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linode_node_group_test.go @@ -0,0 +1,282 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linode + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + apiv1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/linode/linodego" +) + +func TestNodeGroup_IncreaseSize(t *testing.T) { + client := linodeClientMock{} + ctx := context.Background() + poolOpts := linodego.LKEClusterPoolCreateOptions{ + Count: 1, + Type: "g6-standard-1", + } + ng := NodeGroup{ + lkePools: map[int]*linodego.LKEClusterPool{ + 1: {ID: 1, Count: 1, Type: "g6-standard-1"}, + 2: {ID: 2, Count: 1, Type: "g6-standard-1"}, + 3: {ID: 3, Count: 1, Type: "g6-standard-1"}, + }, + poolOpts: poolOpts, + client: &client, + lkeClusterID: 111, + minSize: 1, + maxSize: 7, + id: "g6-standard-1", + } + client.On( + "CreateLKEClusterPool", ctx, ng.lkeClusterID, poolOpts, + ).Return( + &linodego.LKEClusterPool{ID: 4, Count: 1, Type: "g6-standard-1"}, nil, + ).Once().On( + "CreateLKEClusterPool", ctx, ng.lkeClusterID, poolOpts, + ).Return( + &linodego.LKEClusterPool{ID: 5, Count: 1, Type: "g6-standard-1"}, nil, + ).Once().On( + "CreateLKEClusterPool", ctx, ng.lkeClusterID, poolOpts, + ).Return( + &linodego.LKEClusterPool{ID: 6, Count: 1, Type: "g6-standard-1"}, nil, + ).Once().On( + "CreateLKEClusterPool", ctx, ng.lkeClusterID, poolOpts, + ).Return( + &linodego.LKEClusterPool{ID: 6, Count: 1, Type: "g6-standard-1"}, fmt.Errorf("error on API call"), + ).Once() + + // test error on bad delta value + err := ng.IncreaseSize(0) + assert.Error(t, err) + + // test error on bad delta value + err = ng.IncreaseSize(-1) + assert.Error(t, err) + + // test error on a too large increase of nodes + err = ng.IncreaseSize(5) + assert.Error(t, err) + + // test ok to add a node + err = ng.IncreaseSize(1) + assert.NoError(t, err) + assert.Equal(t, 4, len(ng.lkePools)) + + // test ok to add multiple nodes + err = ng.IncreaseSize(2) + assert.NoError(t, err) + assert.Equal(t, 6, len(ng.lkePools)) + + // test error on linode API call error + err = ng.IncreaseSize(1) + assert.Error(t, err, "no error on injected API call error") +} + +func TestNodeGroup_DecreaseTargetSize(t *testing.T) { + ng := &NodeGroup{} + err := ng.DecreaseTargetSize(-1) + assert.Error(t, err) +} + +func TestNodeGroup_DeleteNodes(t *testing.T) { + client := linodeClientMock{} + ctx := context.Background() + poolOpts := linodego.LKEClusterPoolCreateOptions{ + Count: 1, + Type: "g6-standard-1", + } + ng := NodeGroup{ + lkePools: map[int]*linodego.LKEClusterPool{ + 1: {ID: 1, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{InstanceID: 123}}}, + 2: {ID: 2, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{InstanceID: 223}}}, + 3: {ID: 3, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{InstanceID: 323}}}, + 4: {ID: 4, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{InstanceID: 423}}}, + 5: {ID: 5, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{InstanceID: 523}}}, + }, + poolOpts: poolOpts, + client: &client, + lkeClusterID: 111, + minSize: 1, + maxSize: 6, + id: "g6-standard-1", + } + client.On( + "DeleteLKEClusterPool", ctx, ng.lkeClusterID, 1, + ).Return(nil).On( + "DeleteLKEClusterPool", ctx, ng.lkeClusterID, 2, + ).Return(nil).On( + "DeleteLKEClusterPool", ctx, ng.lkeClusterID, 3, + ).Return(nil).On( + "DeleteLKEClusterPool", ctx, ng.lkeClusterID, 4, + ).Return(fmt.Errorf("error on API call")).On( + "DeleteLKEClusterPool", ctx, ng.lkeClusterID, 5, + ).Return(nil) + + nodes := []*apiv1.Node{ + {Spec: apiv1.NodeSpec{ProviderID: "linode://123"}}, + {Spec: apiv1.NodeSpec{ProviderID: "linode://223"}}, + {Spec: apiv1.NodeSpec{ProviderID: "linode://523"}}, + } + + // test of on deleting nodes + err := ng.DeleteNodes(nodes) + assert.NoError(t, err) + assert.Equal(t, 2, len(ng.lkePools)) + assert.NotNil(t, ng.lkePools[3]) + assert.NotNil(t, ng.lkePools[4]) + + // test error on deleting a node with a malformed providerID + nodes = []*apiv1.Node{ + {Spec: apiv1.NodeSpec{ProviderID: "linode://aaa"}}, + } + err = ng.DeleteNodes(nodes) + assert.Error(t, err) + + // test error on deleting a node we are not managing + nodes = []*apiv1.Node{ + {Spec: apiv1.NodeSpec{ProviderID: "linode://555"}}, + } + err = ng.DeleteNodes(nodes) + assert.Error(t, err) + + // test error on deleting a node when the linode API call fails + nodes = []*apiv1.Node{ + {Spec: apiv1.NodeSpec{ProviderID: "linode://423"}}, + } + err = ng.DeleteNodes(nodes) + assert.Error(t, err) +} + +func TestNodeGroup_deleteLKEPool(t *testing.T) { + client := linodeClientMock{} + ctx := context.Background() + poolOpts := linodego.LKEClusterPoolCreateOptions{ + Count: 1, + Type: "g6-standard-1", + } + ng := NodeGroup{ + lkePools: map[int]*linodego.LKEClusterPool{ + 1: {ID: 1, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{InstanceID: 123}}}, + 2: {ID: 2, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{InstanceID: 223}}}, + 3: {ID: 3, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{InstanceID: 323}}}, + 4: {ID: 4, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{InstanceID: 423}}}, + 5: {ID: 5, Count: 1, Type: "g6-standard-1", Linodes: []linodego.LKEClusterPoolLinode{{InstanceID: 523}}}, + }, + poolOpts: poolOpts, + client: &client, + lkeClusterID: 111, + minSize: 1, + maxSize: 6, + id: "g6-standard-1", + } + client.On( + "DeleteLKEClusterPool", ctx, ng.lkeClusterID, 3, + ).Return(nil) + + // test ok on deleting a pool from a node group + err := ng.deleteLKEPool(3) + assert.NoError(t, err) + assert.Nil(t, ng.lkePools[3]) + + // test error on deleting a pool from a node group that does not contain it + err = ng.deleteLKEPool(6) + assert.Error(t, err) +} + +func TestNosdeGroup_Nodes(t *testing.T) { + + client := linodeClientMock{} + poolOpts := linodego.LKEClusterPoolCreateOptions{ + Count: 1, + Type: "g6-standard-1", + } + ng := NodeGroup{ + lkePools: map[int]*linodego.LKEClusterPool{ + 4: {ID: 4, + Count: 1, + Type: "g6-standard-1", + Linodes: []linodego.LKEClusterPoolLinode{ + {InstanceID: 423}, + }, + }, + 5: {ID: 5, + Count: 2, + Type: "g6-standard-1", + Linodes: []linodego.LKEClusterPoolLinode{ + {InstanceID: 523}, {InstanceID: 623}, + }, + }, + }, + poolOpts: poolOpts, + client: &client, + lkeClusterID: 111, + minSize: 1, + maxSize: 6, + id: "g6-standard-1", + } + + // test nodes returned from Nodes() are only the ones we are expecting + instancesList, err := ng.Nodes() + assert.NoError(t, err) + assert.Equal(t, 3, len(instancesList)) + assert.Contains(t, instancesList, cloudprovider.Instance{Id: "linode://423"}) + assert.Contains(t, instancesList, cloudprovider.Instance{Id: "linode://523"}) + assert.Contains(t, instancesList, cloudprovider.Instance{Id: "linode://623"}) + assert.NotContains(t, instancesList, cloudprovider.Instance{Id: "423"}) +} + +func TestNodeGroup_Others(t *testing.T) { + client := linodeClientMock{} + poolOpts := linodego.LKEClusterPoolCreateOptions{ + Count: 1, + Type: "g6-standard-1", + } + ng := NodeGroup{ + lkePools: map[int]*linodego.LKEClusterPool{ + 1: {ID: 1, Count: 1, Type: "g6-standard-1"}, + 2: {ID: 2, Count: 1, Type: "g6-standard-1"}, + 3: {ID: 3, Count: 1, Type: "g6-standard-1"}, + }, + poolOpts: poolOpts, + client: &client, + lkeClusterID: 111, + minSize: 1, + maxSize: 7, + id: "g6-standard-1", + } + assert.Equal(t, 1, ng.MinSize()) + assert.Equal(t, 7, ng.MaxSize()) + ts, err := ng.TargetSize() + assert.NoError(t, err) + assert.Equal(t, 3, ts) + assert.Equal(t, "g6-standard-1", ng.Id()) + assert.Equal(t, "node group ID: g6-standard-1 (min:1 max:7)", ng.Debug()) + assert.Equal(t, true, ng.Exist()) + assert.Equal(t, false, ng.Autoprovisioned()) + _, err = ng.TemplateNodeInfo() + assert.Error(t, err) + _, err = ng.Create() + assert.Error(t, err) + err = ng.Delete() + assert.Error(t, err) +} diff --git a/cluster-autoscaler/cloudprovider/linode/linode_utils_test.go b/cluster-autoscaler/cloudprovider/linode/linode_utils_test.go new file mode 100644 index 000000000000..3875881de52e --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linode_utils_test.go @@ -0,0 +1,43 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linode + +import ( + "context" + + "github.com/stretchr/testify/mock" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/linode/linodego" +) + +type linodeClientMock struct { + mock.Mock +} + +func (l *linodeClientMock) ListLKEClusterPools(ctx context.Context, clusterID int, opts *linodego.ListOptions) ([]linodego.LKEClusterPool, error) { + args := l.Called(ctx, clusterID, nil) + return args.Get(0).([]linodego.LKEClusterPool), args.Error(1) +} + +func (l *linodeClientMock) CreateLKEClusterPool(ctx context.Context, clusterID int, createOpts linodego.LKEClusterPoolCreateOptions) (*linodego.LKEClusterPool, error) { + args := l.Called(ctx, clusterID, createOpts) + return args.Get(0).(*linodego.LKEClusterPool), args.Error(1) +} + +func (l *linodeClientMock) DeleteLKEClusterPool(ctx context.Context, clusterID int, id int) error { + args := l.Called(ctx, clusterID, id) + return args.Error(0) +} diff --git a/cluster-autoscaler/cloudprovider/linode/linodego/README.md b/cluster-autoscaler/cloudprovider/linode/linodego/README.md new file mode 100644 index 000000000000..81fcf5de1cad --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linodego/README.md @@ -0,0 +1,5 @@ +# Linode REST API Client + +This package implements a minimal REST API client for the Linode REST v4 API. + +This client only implements the endpoints usend on the Linode cloud provider, it is intended to be a drop-in replacement for the [linodego](https://github.com/linode/linodego) official go client. \ No newline at end of file diff --git a/cluster-autoscaler/cloudprovider/linode/linodego/linode_api_client_rest.go b/cluster-autoscaler/cloudprovider/linode/linodego/linode_api_client_rest.go new file mode 100644 index 000000000000..0ddd5689e24e --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linodego/linode_api_client_rest.go @@ -0,0 +1,213 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linodego + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + + klog "k8s.io/klog/v2" +) + +// LKELinodeStatus constants reflect the current status of an node in a LKE Pool +const ( + LKELinodeReady LKELinodeStatus = "ready" + LKELinodeNotReady LKELinodeStatus = "not_ready" +) + +// LKELinodeStatus represents the status of a node in a LKE Pool +type LKELinodeStatus string + +// PageOptions are the pagination parameters for List endpoints +type PageOptions struct { + Page int `json:"page"` + Pages int `json:"pages"` + Results int `json:"results"` +} + +// ListOptions are the pagination and filtering parameters for endpoints +type ListOptions struct { + *PageOptions + PageSize int + Filter string +} + +// LKEClusterPoolResponse is the struct for unmarshaling response from the list of LKE pools API call +type LKEClusterPoolResponse struct { + Pools []LKEClusterPool `json:"data"` + PageOptions +} + +// LKEClusterPool represents a LKE Pool +type LKEClusterPool struct { + ID int `json:"id"` + Count int `json:"count"` + Type string `json:"type"` + Disks []LKEClusterPoolDisk `json:"disks"` + Linodes []LKEClusterPoolLinode `json:"nodes"` +} + +// LKEClusterPoolDisk represents a node disk in an LKEClusterPool object +type LKEClusterPoolDisk struct { + Size int `json:"size"` + Type string `json:"type"` +} + +// LKEClusterPoolLinode represents a node in a LKE Pool +type LKEClusterPoolLinode struct { + ID string `json:"id"` + InstanceID int `json:"instance_id"` + Status LKELinodeStatus `json:"status"` +} + +// LKEClusterPoolCreateOptions fields are those accepted by CreateLKEClusterPool +type LKEClusterPoolCreateOptions struct { + Count int `json:"count"` + Type string `json:"type"` + Disks []LKEClusterPoolDisk `json:"disks"` +} + +// SetUserAgent sets a custom user-agent for HTTP requests +func (c *Client) SetUserAgent(ua string) *Client { + c.userAgent = ua + return c +} + +// SetBaseURL sets the base URL of the Linode v4 API (https://api.linode.com/v4) +func (c *Client) SetBaseURL(url string) *Client { + c.baseURL = url + return c +} + +// NewClient factory to create new Client struct +func NewClient(hc *http.Client) (client Client) { + return Client{ + hc: hc, + baseURL: "https://api.linode.com/v4", + userAgent: "kubernetes/cluster-autoscaler", + } +} + +// Client is the struct to perform API calls +type Client struct { + hc *http.Client + baseURL string + userAgent string +} + +// request performs is used to perform a generic call to the Linode API and returns +// the body of the response, or error if such occurs or the response is not valid +func (c *Client) request(ctx context.Context, method, url string, jsonData []byte) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := c.hc.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to perform request: %w", err) + } + + defer func() { + if err := resp.Body.Close(); err != nil { + klog.Errorf("failed to close response body: %v", err) + } + }() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + ct := resp.Header.Get("Content-Type") + if !strings.HasPrefix(ct, "application/json") { + return nil, fmt.Errorf("Unexpected Content-Type: %s with status: %s", ct, resp.Status) + } + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return body, nil + } + return nil, fmt.Errorf("%v %v: %d %v", req.Method, req.URL, resp.StatusCode, string(body)) + +} + +// CreateLKEClusterPool creates a LKE Pool for for a LKE Cluster +func (c *Client) CreateLKEClusterPool(ctx context.Context, clusterID int, createOpts LKEClusterPoolCreateOptions) (*LKEClusterPool, error) { + bodyReq, err := json.Marshal(createOpts) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + url := fmt.Sprintf("%s/lke/clusters/%d/pools", c.baseURL, clusterID) + bodyResp, err := c.request(ctx, "POST", url, bodyReq) + if err != nil { + return nil, err + } + newPool := &LKEClusterPool{} + err = json.Unmarshal(bodyResp, newPool) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %w", err) + } + return newPool, nil +} + +// DeleteLKEClusterPool deletes the LKE Pool with the specified id +func (c *Client) DeleteLKEClusterPool(ctx context.Context, clusterID, id int) error { + url := fmt.Sprintf("%s/lke/clusters/%d/pools/%d", c.baseURL, clusterID, id) + _, err := c.request(ctx, "DELETE", url, []byte{}) + return err +} + +// listLKEClusterPoolsPaginated lists LKE Pools in a paginated request +// and the total number of pages the complete response is composed of +func (c *Client) listLKEClusterPoolsPaginated(ctx context.Context, clusterID int, opts *ListOptions, page int) ([]LKEClusterPool, int, error) { + url := fmt.Sprintf("%s/lke/clusters/%d/pools?pages=%d", c.baseURL, clusterID, page) + body, err := c.request(ctx, "GET", url, []byte{}) + if err != nil { + return nil, 0, err + } + poolResp := &LKEClusterPoolResponse{} + err = json.Unmarshal(body, poolResp) + if err != nil { + return nil, 0, fmt.Errorf("failed to unmarshal response: %w", err) + } + return poolResp.Pools, poolResp.PageOptions.Pages, nil +} + +// ListLKEClusterPools lists LKE Pools +func (c *Client) ListLKEClusterPools(ctx context.Context, clusterID int, opts *ListOptions) ([]LKEClusterPool, error) { + // get the first response with the number of pages in it + pools, pages, err := c.listLKEClusterPoolsPaginated(ctx, clusterID, opts, 1) + if err != nil { + return nil, err + } + // call again the API to get the results in the other pages + for p := 2; p <= pages; p++ { + poolsForPage, _, err := c.listLKEClusterPoolsPaginated(ctx, clusterID, opts, p) + if err != nil { + return nil, err + } + pools = append(pools, poolsForPage...) + } + return pools, nil +} diff --git a/cluster-autoscaler/cloudprovider/linode/linodego/linode_api_client_rest_test.go b/cluster-autoscaler/cloudprovider/linode/linodego/linode_api_client_rest_test.go new file mode 100644 index 000000000000..f2b1e04773ef --- /dev/null +++ b/cluster-autoscaler/cloudprovider/linode/linodego/linode_api_client_rest_test.go @@ -0,0 +1,126 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linodego + +import ( + "context" + "net/http" + "strconv" + "testing" + + . "k8s.io/autoscaler/cluster-autoscaler/utils/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +const createLKEClusterPoolResponse1 = ` +{"id": 19933, "type": "g6-standard-1", "count": 1, "nodes": [{"id": "19933-5ff4aabd3176", "instance_id": null, "status": "not_ready"}], "disks": []} +` + +const deleteLKEClusterPoolResponse1 = ` +{} +` + +const listLKEClusterPoolsResponse1 = ` +{"data": [{"id": 19930, "type": "g6-standard-1", "count": 2, "nodes": [{"id": "19930-5ff4a5cdc29d", "instance_id": 23810703, "status": "not_ready"}, {"id": "19930-5ff4a5ce2b64", "instance_id": 23810705, "status": "not_ready"}], "disks": []}, {"id": 19931, "type": "g6-standard-2", "count": 2, "nodes": [{"id": "19931-5ff4a5ce8f24", "instance_id": 23810707, "status": "not_ready"}, {"id": "19931-5ff4a5cef13e", "instance_id": 23810704, "status": "not_ready"}], "disks": []}], "page": 1, "pages": 1, "results": 2} +` + +const listLKEClusterPoolsResponse2 = ` +{"data": [{"id": 19930, "type": "g6-standard-1", "count": 2, "nodes": [{"id": "19930-5ff4a5cdc29d", "instance_id": 23810703, "status": "not_ready"}, {"id": "19930-5ff4a5ce2b64", "instance_id": 23810705, "status": "not_ready"}], "disks": []}, {"id": 19931, "type": "g6-standard-2", "count": 2, "nodes": [{"id": "19931-5ff4a5ce8f24", "instance_id": 23810707, "status": "not_ready"}, {"id": "19931-5ff4a5cef13e", "instance_id": 23810704, "status": "not_ready"}], "disks": []}], "page": 1, "pages": 3, "results": 4} +` + +const listLKEClusterPoolsResponse3 = ` +{"data": [{"id": 19932, "type": "g6-standard-1", "count": 1, "nodes": [{"id": "19932-5ff4a5cdc29f", "instance_id": 23810705, "status": "not_ready"}], "disks": []}], "page": 2, "pages": 3, "results": 4} +` + +const listLKEClusterPoolsResponse4 = ` +{"data": [{"id": 19933, "type": "g6-standard-1", "count": 1, "nodes": [{"id": "19932-5ff4a5cdc29a", "instance_id": 23810706, "status": "not_ready"}], "disks": []}], "page": 3, "pages": 3, "results": 4} +` + +func TestApiClientRest_CreateLKEClusterPool(t *testing.T) { + server := NewHttpServerMockWithContentType() + defer server.Close() + + client := NewClient(&http.Client{}) + client.SetBaseURL(server.URL) + + clusterID := 16293 + ctx := context.Background() + createOpts := LKEClusterPoolCreateOptions{ + Count: 1, + Type: "g6-standard-1", + Disks: []LKEClusterPoolDisk{}, + } + requestPath := "/lke/clusters/" + strconv.Itoa(clusterID) + "/pools" + server.On("handleWithContentType", requestPath).Return("application/json", createLKEClusterPoolResponse1).Once() + pool, err := client.CreateLKEClusterPool(ctx, clusterID, createOpts) + + assert.NoError(t, err) + assert.Equal(t, 1, pool.Count) + assert.Equal(t, "g6-standard-1", pool.Type) + assert.Equal(t, 1, len(pool.Linodes)) + + mock.AssertExpectationsForObjects(t, server) +} + +func TestApiClientRest_DeleteLKEClusterPool(t *testing.T) { + server := NewHttpServerMockWithContentType() + defer server.Close() + + client := NewClient(&http.Client{}) + client.SetBaseURL(server.URL) + + clusterID := 111 + poolID := 222 + ctx := context.Background() + requestPath := "/lke/clusters/" + strconv.Itoa(clusterID) + "/pools/" + strconv.Itoa(poolID) + server.On("handleWithContentType", requestPath).Return("application/json", deleteLKEClusterPoolResponse1).Once() + err := client.DeleteLKEClusterPool(ctx, clusterID, poolID) + assert.NoError(t, err) + + mock.AssertExpectationsForObjects(t, server) +} + +func TestApiClientRest_ListLKEClusterPools(t *testing.T) { + server := NewHttpServerMockWithContentType() + defer server.Close() + + client := NewClient(&http.Client{}) + client.SetBaseURL(server.URL) + + clusterID := 16293 + ctx := context.Background() + requestPath := "/lke/clusters/" + strconv.Itoa(clusterID) + "/pools" + server.On("handleWithContentType", requestPath).Return("application/json", listLKEClusterPoolsResponse1).Once().On("handleWithContentType", requestPath).Return("application/json", listLKEClusterPoolsResponse2).Once().On("handleWithContentType", requestPath).Return("application/json", listLKEClusterPoolsResponse3).Once().On("handleWithContentType", requestPath).Return("application/json", listLKEClusterPoolsResponse4).Once() + + pools, err := client.ListLKEClusterPools(ctx, clusterID, nil) + assert.NoError(t, err) + assert.Equal(t, 2, len(pools)) + assert.Equal(t, 19930, pools[0].ID) + assert.Equal(t, 19931, pools[1].ID) + + pools, err = client.ListLKEClusterPools(ctx, clusterID, nil) + assert.NoError(t, err) + assert.Equal(t, 4, len(pools)) + assert.Equal(t, 19930, pools[0].ID) + assert.Equal(t, 19931, pools[1].ID) + assert.Equal(t, 19932, pools[2].ID) + assert.Equal(t, 19933, pools[3].ID) + + mock.AssertExpectationsForObjects(t, server) +}