Skip to content

Commit

Permalink
[FEATURE] - Run Jobs in central namespace (kubevela#295)
Browse files Browse the repository at this point in the history 
* - updating the chart to support job namespaces

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>

* - updating the controllers to support the ability to run the jobs in a central namespace and avoiding the need to copy global secrets around

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>

* - updating the cmd to support passing the job namespace and support localizing to a single namespace if required

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>

* adding the resource limits

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>

* - trying to fix the linting and unit tests

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>

* - improving the test coverage

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>

* - changing the job-namespace command line flag to --controller-namespace
- changing the variable name in controller and meta to reflect changes i.e JobNamespace -> ControllerNamespace
- updated the helm chart to reflect the changes

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>

* - uncommenting the tests

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>

* - commenting out the test for a moment

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>

* - fixing up the issue highlighted in the review

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>

* - rebasing master onto the branch and fixing the units tests / ensuring it works

Signed-off-by: Rohith Jayawardene <gambol99@gmail.com>
  • Loading branch information
@gambol99
gambol99 authored Jul 25, 2022
1 parent 88aa8b9 commit 4b5a5eb
Show file tree
Hide file tree
Showing 12 changed files with 249 additions and 86 deletions.
4 changes: 4 additions & 0 deletions chart/templates/terraform_controller.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ spec:
- name: terraform-controller
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
args:
{{- if .Values.controllerNamespace }}
- --controller-namespace={{ .Values.controllerNamespace }}
{{- end }}
env:
- name: CONTROLLER_NAMESPACE
valueFrom:
Expand Down
1 change: 1 addition & 0 deletions chart/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ image:
gitImage: alpine/git:latest
busyboxImage: busybox:latest
terraformImage: oamdev/docker-terraform:1.1.2
controllerNamespace: ""

# "{\"nat\": \"true\"}"
jobNodeSelector: ""
Expand Down
119 changes: 78 additions & 41 deletions controllers/configuration_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,14 @@ const (
)

const (
// TerraformStateNameInSecret is the key name to store Terraform state
TerraformStateNameInSecret = "tfstate"
// TFInputConfigMapName is the CM name for Terraform Input Configuration
TFInputConfigMapName = "tf-%s"
// TFVariableSecret is the Secret name for variables, including credentials from Provider
TFVariableSecret = "variable-%s"
// TFBackendSecret is the Secret name for Kubernetes backend
TFBackendSecret = "tfstate-%s-%s"
)

// TerraformExecutionType is the type for Terraform execution
Expand All @@ -95,9 +99,10 @@ const (
// ConfigurationReconciler reconciles a Configuration object.
type ConfigurationReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
ProviderName string
Log logr.Logger
ControllerNamespace string
ProviderName string
Scheme *runtime.Scheme
}

// +kubebuilder:rbac:groups=terraform.core.oam.dev,resources=configurations,verbs=get;list;watch;create;update;patch;delete
Expand All @@ -113,6 +118,22 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
}

meta := initTFConfigurationMeta(req, configuration)
if r.ControllerNamespace != "" {
uid := string(configuration.GetUID())
// @step: since we are using a single namespace to run these, we must ensure the names
// are unique across the namespace
meta.ApplyJobName = uid + "-" + string(TerraformApply)
meta.ConfigurationCMName = fmt.Sprintf(TFInputConfigMapName, uid)
meta.DestroyJobName = uid + "-" + string(TerraformDestroy)
meta.ControllerNamespace = r.ControllerNamespace
meta.TerraformBackendNamespace = r.ControllerNamespace
meta.VariableSecretName = fmt.Sprintf(TFVariableSecret, uid)

configuration.Spec.Backend = &v1beta2.Backend{
InClusterConfig: true,
SecretSuffix: uid,
}
}

// add finalizer
var isDeleting = !configuration.ObjectMeta.DeletionTimestamp.IsZero()
Expand All @@ -131,7 +152,7 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
}

var tfExecutionJob = &batchv1.Job{}
if err := r.Client.Get(ctx, client.ObjectKey{Name: meta.ApplyJobName, Namespace: meta.Namespace}, tfExecutionJob); err == nil {
if err := r.Client.Get(ctx, client.ObjectKey{Name: meta.ApplyJobName, Namespace: meta.ControllerNamespace}, tfExecutionJob); err == nil {
if !meta.EnvChanged && tfExecutionJob.Status.Succeeded == int32(1) {
if err := meta.updateApplyStatus(ctx, r.Client, types.Available, types.MessageCloudResourceDeployed); err != nil {
return ctrl.Result{}, err
Expand All @@ -143,15 +164,15 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
// terraform destroy
klog.InfoS("performing Configuration Destroy", "Namespace", req.Namespace, "Name", req.Name, "JobName", meta.DestroyJobName)

_, err := terraform.GetTerraformStatus(ctx, meta.Namespace, meta.DestroyJobName, terraformContainerName, terraformInitContainerName)
_, err := terraform.GetTerraformStatus(ctx, meta.Namespace, meta.DestroyJobName, meta.ControllerNamespace, terraformContainerName, terraformInitContainerName)
if err != nil {
klog.ErrorS(err, "Terraform destroy failed")
if updateErr := meta.updateDestroyStatus(ctx, r.Client, types.ConfigurationDestroyFailed, err.Error()); updateErr != nil {
return ctrl.Result{}, updateErr
}
}

if err := r.terraformDestroy(ctx, req.Namespace, configuration, meta); err != nil {
if err := r.terraformDestroy(ctx, configuration, meta); err != nil {
if err.Error() == types.MessageDestroyJobNotCompleted {
return ctrl.Result{RequeueAfter: 3 * time.Second}, nil
}
Expand Down Expand Up @@ -179,12 +200,14 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
}
return ctrl.Result{RequeueAfter: 3 * time.Second}, errors.Wrap(err, "failed to create/update cloud resource")
}
state, err := terraform.GetTerraformStatus(ctx, meta.Namespace, meta.ApplyJobName, terraformContainerName, terraformInitContainerName)
state, err := terraform.GetTerraformStatus(ctx, meta.Namespace, meta.ApplyJobName, meta.ControllerNamespace, terraformContainerName, terraformInitContainerName)
if err != nil {
klog.ErrorS(err, "Terraform apply failed")
if updateErr := meta.updateApplyStatus(ctx, r.Client, state, err.Error()); updateErr != nil {
return ctrl.Result{}, updateErr
}

return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}

return ctrl.Result{}, nil
Expand All @@ -194,6 +217,7 @@ func (r *ConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
type TFConfigurationMeta struct {
Name string
Namespace string
ControllerNamespace string
ConfigurationType types.ConfigurationType
CompleteConfiguration string
RemoteGit string
Expand All @@ -212,14 +236,14 @@ type TFConfigurationMeta struct {
Credentials map[string]string

Backend backend.Backend

// JobNodeSelector Expose the node selector of job to the controller level
JobNodeSelector map[string]string

// TerraformImage is the Terraform image which can run `terraform init/plan/apply`
TerraformImage string
BusyboxImage string
GitImage string
TerraformImage string
TerraformBackendNamespace string
BusyboxImage string
GitImage string

// Resources series Variables are for Setting Compute Resources required by this container
ResourcesLimitsCPU string
Expand All @@ -234,6 +258,7 @@ type TFConfigurationMeta struct {

func initTFConfigurationMeta(req ctrl.Request, configuration v1beta2.Configuration) *TFConfigurationMeta {
var meta = &TFConfigurationMeta{
ControllerNamespace: req.Namespace,
Namespace: req.Namespace,
Name: req.Name,
ConfigurationCMName: fmt.Sprintf(TFInputConfigMapName, req.Name),
Expand Down Expand Up @@ -283,7 +308,7 @@ func (r *ConfigurationReconciler) terraformApply(ctx context.Context, namespace
tfExecutionJob batchv1.Job
)

if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ApplyJobName, Namespace: namespace}, &tfExecutionJob); err != nil {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ApplyJobName, Namespace: meta.ControllerNamespace}, &tfExecutionJob); err != nil {
if kerrors.IsNotFound(err) {
return meta.assembleAndTriggerJob(ctx, k8sClient, TerraformApply)
}
Expand Down Expand Up @@ -311,7 +336,7 @@ func (r *ConfigurationReconciler) terraformApply(ctx context.Context, namespace
return nil
}

func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, namespace string, configuration v1beta2.Configuration, meta *TFConfigurationMeta) error {
func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, configuration v1beta2.Configuration, meta *TFConfigurationMeta) error {
var (
destroyJob batchv1.Job
k8sClient = r.Client
Expand All @@ -325,7 +350,7 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, namespac
deleteConfigurationDirectly := deletable || !meta.DeleteResource

if !deleteConfigurationDirectly {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.DestroyJobName, Namespace: meta.Namespace}, &destroyJob); err != nil {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.DestroyJobName, Namespace: meta.ControllerNamespace}, &destroyJob); err != nil {
if kerrors.IsNotFound(err) {
if err := r.Client.Get(ctx, client.ObjectKey{Name: configuration.Name, Namespace: configuration.Namespace}, &v1beta2.Configuration{}); err == nil {
if err = meta.assembleAndTriggerJob(ctx, k8sClient, TerraformDestroy); err != nil {
Expand All @@ -347,22 +372,22 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, namespac

if configuration.Spec.ForceDelete != nil && *configuration.Spec.ForceDelete {
// Try to clean up more sub-resources as possible. Ignore the issues if it hit any.
if err := r.cleanUpSubResources(ctx, namespace, configuration, meta); err != nil {
if err := r.cleanUpSubResources(ctx, meta.ControllerNamespace, configuration, meta); err != nil {
klog.Warningf("Failed to clean up sub-resources, but it's ignored as the resources are being forced to delete: %s", err)
}
return nil
}
// When the deletion Job process succeeded, clean up work is starting.
if !deleteConfigurationDirectly {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.DestroyJobName, Namespace: meta.Namespace}, &destroyJob); err != nil {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.DestroyJobName, Namespace: meta.ControllerNamespace}, &destroyJob); err != nil {
return err
}
if destroyJob.Status.Succeeded == int32(1) {
return r.cleanUpSubResources(ctx, namespace, configuration, meta)
return r.cleanUpSubResources(ctx, meta.ControllerNamespace, configuration, meta)
}
}
if deleteConfigurationDirectly {
return r.cleanUpSubResources(ctx, namespace, configuration, meta)
return r.cleanUpSubResources(ctx, meta.ControllerNamespace, configuration, meta)
}

return errors.New(types.MessageDestroyJobNotCompleted)
Expand Down Expand Up @@ -548,13 +573,13 @@ func (r *ConfigurationReconciler) preCheck(ctx context.Context, configuration *v
}

var variableInSecret v1.Secret
err = k8sClient.Get(ctx, client.ObjectKey{Name: meta.VariableSecretName, Namespace: meta.Namespace}, &variableInSecret)
err = k8sClient.Get(ctx, client.ObjectKey{Name: meta.VariableSecretName, Namespace: meta.ControllerNamespace}, &variableInSecret)
switch {
case kerrors.IsNotFound(err):
var secret = v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: meta.VariableSecretName,
Namespace: meta.Namespace,
Namespace: meta.ControllerNamespace,
},
TypeMeta: metav1.TypeMeta{Kind: "Secret"},
Data: meta.VariableSecretData,
Expand All @@ -578,8 +603,7 @@ func (r *ConfigurationReconciler) preCheck(ctx context.Context, configuration *v
return err
}

// Apply ClusterRole
return createTerraformExecutorClusterRole(ctx, k8sClient, fmt.Sprintf("%s-%s", meta.Namespace, ClusterRoleName))
return createTerraformExecutorClusterRole(ctx, k8sClient, fmt.Sprintf("%s-%s", meta.ControllerNamespace, ClusterRoleName))
}

func (meta *TFConfigurationMeta) updateApplyStatus(ctx context.Context, k8sClient client.Client, state types.ConfigurationState, message string) error {
Expand Down Expand Up @@ -622,14 +646,15 @@ func (meta *TFConfigurationMeta) updateDestroyStatus(ctx context.Context, k8sCli

func (meta *TFConfigurationMeta) assembleAndTriggerJob(ctx context.Context, k8sClient client.Client, executionType TerraformExecutionType) error {
// apply rbac
if err := createTerraformExecutorServiceAccount(ctx, k8sClient, meta.Namespace, ServiceAccountName); err != nil {
if err := createTerraformExecutorServiceAccount(ctx, k8sClient, meta.ControllerNamespace, ServiceAccountName); err != nil {
return err
}
if err := createTerraformExecutorClusterRoleBinding(ctx, k8sClient, meta.Namespace, fmt.Sprintf("%s-%s", meta.Namespace, ClusterRoleName), ServiceAccountName); err != nil {
if err := createTerraformExecutorClusterRoleBinding(ctx, k8sClient, meta.ControllerNamespace, fmt.Sprintf("%s-%s", meta.ControllerNamespace, ClusterRoleName), ServiceAccountName); err != nil {
return err
}

job := meta.assembleTerraformJob(executionType)

return k8sClient.Create(ctx, job)
}

Expand All @@ -646,7 +671,7 @@ func (meta *TFConfigurationMeta) updateTerraformJobIfNeeded(ctx context.Context,
}
}
var s v1.Secret
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.VariableSecretName, Namespace: meta.Namespace}, &s); err == nil {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.VariableSecretName, Namespace: meta.ControllerNamespace}, &s); err == nil {
if deleteErr := k8sClient.Delete(ctx, &s); deleteErr != nil {
return deleteErr
}
Expand Down Expand Up @@ -775,14 +800,19 @@ func (meta *TFConfigurationMeta) assembleTerraformJob(executionType TerraformExe
container.Resources = resourceRequirements
}

name := meta.ApplyJobName
if executionType == TerraformDestroy {
name = meta.DestroyJobName
}

return &batchv1.Job{
TypeMeta: metav1.TypeMeta{
Kind: "Job",
APIVersion: "batch/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: meta.Name + "-" + string(executionType),
Namespace: meta.Namespace,
Name: name,
Namespace: meta.ControllerNamespace,
},
Spec: batchv1.JobSpec{
Parallelism: &parallelism,
Expand Down Expand Up @@ -1013,7 +1043,7 @@ func getTerraformJSONVariable(tfVariables *runtime.RawExtension) (map[string]int

func (meta *TFConfigurationMeta) deleteConfigMap(ctx context.Context, k8sClient client.Client) error {
var cm v1.ConfigMap
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ConfigurationCMName, Namespace: meta.Namespace}, &cm); err == nil {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ConfigurationCMName, Namespace: meta.ControllerNamespace}, &cm); err == nil {
if err := k8sClient.Delete(ctx, &cm); err != nil {
return err
}
Expand All @@ -1038,25 +1068,32 @@ func deleteConnectionSecret(ctx context.Context, k8sClient client.Client, name,

func (meta *TFConfigurationMeta) createOrUpdateConfigMap(ctx context.Context, k8sClient client.Client, data map[string]string) error {
var gotCM v1.ConfigMap
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ConfigurationCMName, Namespace: meta.Namespace}, &gotCM); err != nil {
if kerrors.IsNotFound(err) {
cm := v1.ConfigMap{
TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
ObjectMeta: metav1.ObjectMeta{
Name: meta.ConfigurationCMName,
Namespace: meta.Namespace,
},
Data: data,
}
err := k8sClient.Create(ctx, &cm)
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ConfigurationCMName, Namespace: meta.ControllerNamespace}, &gotCM); err != nil {
if !kerrors.IsNotFound(err) {
return err
}
cm := v1.ConfigMap{
TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
ObjectMeta: metav1.ObjectMeta{
Name: meta.ConfigurationCMName,
Namespace: meta.ControllerNamespace,
},
Data: data,
}

if err := k8sClient.Create(ctx, &cm); err != nil {
return errors.Wrap(err, "failed to create TF configuration ConfigMap")
}
return err

return nil
}

if !reflect.DeepEqual(gotCM.Data, data) {
gotCM.Data = data

return errors.Wrap(k8sClient.Update(ctx, &gotCM), "failed to update TF configuration ConfigMap")
}

return nil
}

Expand All @@ -1081,7 +1118,7 @@ func (meta *TFConfigurationMeta) storeTFConfiguration(ctx context.Context, k8sCl
// CheckWhetherConfigurationChanges will check whether configuration is changed
func (meta *TFConfigurationMeta) CheckWhetherConfigurationChanges(ctx context.Context, k8sClient client.Client, configurationType types.ConfigurationType) error {
var cm v1.ConfigMap
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ConfigurationCMName, Namespace: meta.Namespace}, &cm); err != nil {
if err := k8sClient.Get(ctx, client.ObjectKey{Name: meta.ConfigurationCMName, Namespace: meta.ControllerNamespace}, &cm); err != nil {
return err
}

Expand Down
Loading

0 comments on commit 4b5a5eb

Please sign in to comment.