Skip to content

Commit

Permalink
Advertisement operator deployment
Browse files Browse the repository at this point in the history
- added command-line flags to set the kubeconfig
- added creation.go file with utility function to create a client to a cluster
- added comments to methods
  • Loading branch information
fraborg committed Feb 17, 2020
1 parent 9936cc8 commit 9f93c23
Show file tree
Hide file tree
Showing 10 changed files with 179 additions and 66 deletions.
3 changes: 0 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,3 @@ bin
*.swp
*.swo
*~

## yaml files for the virtual-kubelet
data
19 changes: 4 additions & 15 deletions cmd/advertisement-operator/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@ limitations under the License.
package main

import (
"context"
"errors"
"flag"
"os"

Expand All @@ -29,7 +27,6 @@ import (

protocolv1beta1 "github.com/netgroup-polito/dronev2/api/v1beta1"
"github.com/netgroup-polito/dronev2/internal/advertisement-operator"
pkg "github.com/netgroup-polito/dronev2/pkg/advertisement-operator"
// +kubebuilder:scaffold:imports
)

Expand All @@ -46,19 +43,15 @@ func init() {
}

func main() {
var metricsAddr, foreignKubeconfig string
var metricsAddr, foreignKubeconfig, configMapName string
var enableLeaderElection bool
flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
flag.StringVar(&foreignKubeconfig, "foreign-kubeconfig", "", "The kubeconfig of the foreign cluster.")
flag.StringVar(&foreignKubeconfig, "foreign-kubeconfig", "", "The path to the kubeconfig of the foreign cluster.")
flag.StringVar(&configMapName, "configMap-name", "foreign-kubeconfig", "The name of the configMap which contains the kubeconfig of the foreign cluster")
flag.Parse()

if foreignKubeconfig == "" {
setupLog.Error(errors.New("Foreign kubeconfig not provided"), "")
os.Exit(1)
}

ctrl.SetLogger(zap.New(func(o *zap.Options) {
o.Development = true
}))
Expand All @@ -84,11 +77,7 @@ func main() {
}
// +kubebuilder:scaffold:builder

err = pkg.CreateFromFile(mgr.GetClient(), context.Background(), setupLog, foreignKubeconfig)
if err != nil {
setupLog.Error(err, "Unable to create configMap for foreign kubeconfig")
}
go advertisement_operator.GenerateAdvertisement(mgr.GetClient(), foreignKubeconfig)
go advertisement_operator.GenerateAdvertisement(mgr.GetClient(), foreignKubeconfig, configMapName)

setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
Expand Down
1 change: 0 additions & 1 deletion data/foreignKubeconfig

This file was deleted.

4 changes: 2 additions & 2 deletions data/foreignKubeconfig_cm.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: remote-kubeconfig
name: foreign-kubeconfig
namespace: default
data:
remote: |
# put here your remote kubeconfig
# put here the kubeconfig of the foreign cluster
2 changes: 1 addition & 1 deletion data/vk_deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,5 @@ spec:
name: vk-config
- name: remote-config
configMap:
name: remote-kubeconfig
name: foreign-kubeconfig
status: {}
58 changes: 17 additions & 41 deletions internal/advertisement-operator/broadcaster.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package advertisement_operator

import (
"context"
"github.com/netgroup-polito/dronev2/pkg/advertisement-operator"
"os"
"runtime"
"time"

Expand All @@ -13,41 +11,48 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sruntime "k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"

ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

protocolv1beta1 "github.com/netgroup-polito/dronev2/api/v1beta1"
pkg "github.com/netgroup-polito/dronev2/pkg/advertisement-operator"
)

// generate an advertisement message every 10 minutes and post it to remote clusters
func GenerateAdvertisement(client client.Client, foreignKubeconfigPath string) {
// parameters
// - localClient: a client to the local kubernetes
// - foreignKubeconfigPath: the path to a kubeconfig file. If set, this file is used to create a client to the foreign cluster
// - configMapName: the name of the configMap containing the kubeconfig to the foreign cluster. If foreignKubeconfigPath is set it is ignored
// IMPORTANT: the data in the configMap must be named "remote"
func GenerateAdvertisement(localClient client.Client, foreignKubeconfigPath string, configMapName string) {
//TODO: recovering logic if errors occurs

log := ctrl.Log.WithName("advertisement-broadcaster")
// give time to the cache to be started
time.Sleep(5*time.Second)

remoteClient, err := newCRDClient(foreignKubeconfigPath)
log := ctrl.Log.WithName("advertisement-broadcaster")
log.Info("starting broadcaster")
remoteClient, err := pkg.NewCRDClient(foreignKubeconfigPath, configMapName, localClient)
if err != nil {
log.Error(err, "Unable to create client to remote cluster")
}
log.Info("created client to remote cluster" )

for {
var nodes v1.NodeList
err := client.List(context.Background(), &nodes)
err := localClient.List(context.Background(), &nodes)
if err != nil {
//TODO
log.Error(err, "Unable to list nodes")
}
//TODO: filter nodes (e.g. prune all virtual-kubelet)

adv := CreateAdvertisement(nodes.Items)
err = advertisement_operator.CreateOrUpdate(remoteClient, context.Background(), log, adv)
err = pkg.CreateOrUpdate(remoteClient, context.Background(), log, adv)
if err != nil {
log.Error(err, "Unable to create advertisement on remote cluster")
}
log.Info("correctly created advertisement on remote cluster" )
time.Sleep(10 * time.Minute)
}
}
Expand Down Expand Up @@ -173,32 +178,3 @@ func getDockerImages() []v1.ContainerImage {

return images
}

// create a client to a cluster given its kubeconfig
func newCRDClient(configPath string) (client.Client, error) {
var config *rest.Config

// Check if the kubeConfig file exists.
if _, err := os.Stat(configPath); !os.IsNotExist(err) {
// Get the kubeconfig from the filepath.
config, err = clientcmd.BuildConfigFromFlags("", configPath)
if err != nil {
return nil, err
}
} else {
return nil, err
}

scheme := k8sruntime.NewScheme()
_ = clientgoscheme.AddToScheme(scheme)
_ = protocolv1beta1.AddToScheme(scheme)

remoteClient, err := client.New(config, client.Options{
Scheme: scheme,
})
if err != nil {
return nil, err
}

return remoteClient, nil
}
4 changes: 3 additions & 1 deletion internal/advertisement-operator/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func (r *AdvertisementReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
"vkubelet-cfg.json": `
{
"virtual-kubelet": {
"remoteKubeconfig" : "/app/kubeconfig/remote",
"remoteKubeconfig": "/app/kubeconfig/remote",
"namespace": "drone-v2",
"cpu": "` + adv.Spec.Availability.Cpu().String() + `",
"memory": "` + adv.Spec.Availability.Memory().String() + `",
Expand Down Expand Up @@ -100,6 +100,8 @@ func (r *AdvertisementReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
return ctrl.Result{}, err
}

log.Info("launching virtual-kubelet")

return ctrl.Result{}, nil
}

Expand Down
150 changes: 150 additions & 0 deletions pkg/advertisement-operator/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
package advertisement_operator

import (
"context"
"fmt"
"os"

protocolv1beta1 "github.com/netgroup-polito/dronev2/api/v1beta1"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sruntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"

"sigs.k8s.io/controller-runtime/pkg/client"
)

// get config to create a client
// parameters:
// - path: the path to the kubeconfig file
// - configMapName: the name of the configMap containing the kubeconfig to the foreign cluster
// if path is specified create a config from a kubeconfig file, otherwise create or a inCluster config or read the kubeconfig from a configMap
func GetConfig(path string, configMapName string, crdClient client.Client) (*rest.Config, error) {
var config *rest.Config
var err error

if path == "" && configMapName == "" {
config, err = rest.InClusterConfig()
if err != nil {
return nil, err
}
} else if path == "" && configMapName != "" {
// Get the kubeconfig from configMap
if crdClient == nil {
c, err := NewK8sClient(path, configMapName)
if err != nil {
return nil, err
}
kubeconfigGetter := GetKubeconfigFromConfigMap(configMapName, c)
config, err = clientcmd.BuildConfigFromKubeconfigGetter("", kubeconfigGetter)
if err != nil {
return nil, err
}
} else {
kubeconfigGetter := GetKubeconfigFromConfigMapWithCRDClient(configMapName, crdClient)
config, err = clientcmd.BuildConfigFromKubeconfigGetter("", kubeconfigGetter)
if err != nil {
return nil, err
}
}
} else if path != "" {
if _, err := os.Stat(path); !os.IsNotExist(err) {
// Get the kubeconfig from the filepath.
config, err = clientcmd.BuildConfigFromFlags("", path)
if err != nil {
return nil, err
}
}
}

return config, err
}

// create a standard K8s client -> to access use client.CoreV1().<resource>(<namespace>).<method>())
func NewK8sClient(path string, configMapName string) (*kubernetes.Clientset, error) {
config, err := GetConfig(path, configMapName, nil)
if err != nil {
return nil, err
}
return kubernetes.NewForConfig(config)
}

// create a crd client (kubebuilder-like) -> to access use client.<method>(context, <NamespacedName>, <resource>)
func NewCRDClient(path string, configMapName string, c client.Client) (client.Client, error) {
config, err := GetConfig(path, configMapName, c)
if err != nil {
return nil, err
}

scheme := k8sruntime.NewScheme()
_ = clientgoscheme.AddToScheme(scheme)
_ = protocolv1beta1.AddToScheme(scheme)

remoteClient, err := client.New(config, client.Options{
Scheme: scheme,
})
if err != nil {
return nil, err
}

return remoteClient, nil
}

// extract kubeconfig from a configMap.
// parameters:
// - configMapName: the name of the configMap
// - client: the k8s client to the local cluster
func GetKubeconfigFromConfigMap(configMapName string, client *kubernetes.Clientset) clientcmd.KubeconfigGetter {
return func() (*clientcmdapi.Config, error) {
// Get the namespace this is running in from the env variable.
namespace := os.Getenv("POD_NAMESPACE")
if namespace == "" {
namespace = "default"
}

data := []byte{}
if configMapName != "" {
cm, err := client.CoreV1().ConfigMaps(namespace).Get(configMapName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("error in fetching configMap: %s", err)
}
data = []byte(cm.Data["remote"])
}
return clientcmd.Load(data)
}
}

// extract kubeconfig from a configMap
// parameters:
// - configMapName: the name of the configMap
// - crdClient: the kubebuilder-like client to the local cluster
func GetKubeconfigFromConfigMapWithCRDClient(configMapName string, crdClient client.Client) clientcmd.KubeconfigGetter {
return func() (*clientcmdapi.Config, error) {
// Get the namespace this is running in from the env variable.
namespace := os.Getenv("POD_NAMESPACE")
if namespace == "" {
namespace = "default"
}

data := []byte{}
var cm v1.ConfigMap
if configMapName != "" {
err := crdClient.Get(context.Background(), types.NamespacedName{
Namespace: namespace,
Name: configMapName,
}, &cm)
if err != nil {
return nil, fmt.Errorf("error in fetching configMap: %s", err)
}
data = []byte(cm.Data["remote"])
}
return clientcmd.Load(data)
}
}

4 changes: 2 additions & 2 deletions pkg/advertisement-operator/creation.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func CreateFromFile(c client.Client, ctx context.Context, log logr.Logger, filen

remoteKubeConfig := v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "remote-kubeconfig",
Name: "foreign-kubeconfig",
Namespace: "default",
},
Data: map[string]string{
Expand All @@ -223,4 +223,4 @@ func CreateFromFile(c client.Client, ctx context.Context, log logr.Logger, filen
}

return nil
}
}

0 comments on commit 9f93c23

Please sign in to comment.