Skip to content

Commit

Permalink
Add support for volume and volume mounts for runners
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrei-trandafir
andrei-trandafir committed Jul 19, 2023
1 parent 42cebe3 commit 17315da
Show file tree
Hide file tree
Showing 9 changed files with 8,028 additions and 2,736 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ test-setup-ci:

e2e: deploy
kubectl create configmap crocodile-stress-test --from-file e2e/test.js
kubectl apply -f e2e/test.yaml
kubectl apply -f e2e/test.yaml -f e2e/test-initcontainer.yaml

e2e-cleanup:
kubectl delete configmap crocodile-stress-test
kubectl delete -f e2e/test.yaml
kubectl delete -f e2e/test.yaml -f e2e/test-initcontainer.yaml

# Build manager binary
manager: generate fmt vet
Expand Down
16 changes: 10 additions & 6 deletions api/v1alpha1/k6_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,19 @@ type Pod struct {
ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty"`
LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"`
InitContainers []InitContainer `json:"initContainers,omitempty"`
Volumes []corev1.Volume `json:"volumes,omitempty"`
VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
}

type InitContainer struct {
Image string `json:"image,omitempty"`
Env []corev1.EnvVar `json:"env,omitempty"`
EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
Command []string `json:"command,omitempty"`
Args []string `json:"args,omitempty"`
WorkingDir string `json:"workingDir,omitempty"`
Name string `json:"name,omitempty"`
Image string `json:"image,omitempty"`
Env []corev1.EnvVar `json:"env,omitempty"`
EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
Command []string `json:"command,omitempty"`
Args []string `json:"args,omitempty"`
WorkingDir string `json:"workingDir,omitempty"`
VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
}

type K6Scuttle struct {
Expand Down
21 changes: 21 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10,491 changes: 7,770 additions & 2,721 deletions config/crd/bases/k6.io_k6s.yaml
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions config/rbac/role.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand Down
15 changes: 12 additions & 3 deletions pkg/resources/jobs/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@ package jobs

import (
"fmt"
"github.com/grafana/k6-operator/pkg/types"
"strconv"

"github.com/grafana/k6-operator/pkg/types"

"github.com/grafana/k6-operator/api/v1alpha1"
corev1 "k8s.io/api/core/v1"
)
Expand Down Expand Up @@ -121,15 +122,23 @@ func getInitContainers(k6Spec *v1alpha1.K6Spec, script *types.Script) []corev1.C
var initContainers []corev1.Container

for i, k6InitContainer := range k6Spec.Runner.InitContainers {

name := fmt.Sprintf("k6-init-%d", i)
if k6InitContainer.Name != "" {
name = k6InitContainer.Name
}

volumeMounts := append(script.VolumeMount(), k6InitContainer.VolumeMounts...)

initContainer := corev1.Container{
Name: fmt.Sprintf("k6-init-%d", i),
Name: name,
Image: k6InitContainer.Image,
Command: k6InitContainer.Command,
Args: k6InitContainer.Args,
WorkingDir: k6InitContainer.WorkingDir,
EnvFrom: k6InitContainer.EnvFrom,
Env: k6InitContainer.Env,
VolumeMounts: script.VolumeMount(),
VolumeMounts: volumeMounts,
ImagePullPolicy: k6Spec.Runner.ImagePullPolicy,
}
initContainers = append(initContainers, initContainer)
Expand Down
10 changes: 8 additions & 2 deletions pkg/resources/jobs/initializer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,12 @@ func NewInitializerJob(k6 *v1alpha1.K6, argLine string) (*batchv1.Job, error) {

env := append(newIstioEnvVar(k6.Spec.Scuttle, istioEnabled), k6.Spec.Initializer.Env...)

volumes := script.Volume()
volumes = append(volumes, k6.Spec.Initializer.Volumes...)

volumeMounts := script.VolumeMount()
volumeMounts = append(volumeMounts, k6.Spec.Initializer.VolumeMounts...)

var zero32 int32
return &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -115,11 +121,11 @@ func NewInitializerJob(k6 *v1alpha1.K6, argLine string) (*batchv1.Job, error) {
Command: command,
Env: env,
Resources: k6.Spec.Initializer.Resources,
VolumeMounts: script.VolumeMount(),
VolumeMounts: volumeMounts,
Ports: ports,
},
},
Volumes: script.Volume(),
Volumes: volumes,
},
},
},
Expand Down
10 changes: 8 additions & 2 deletions pkg/resources/jobs/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,12 @@ func NewRunnerJob(k6 *v1alpha1.K6, index int, token string) (*batchv1.Job, error

env = append(env, k6.Spec.Runner.Env...)

volumes := script.Volume()
volumes = append(volumes, k6.Spec.Runner.Volumes...)

volumeMounts := script.VolumeMount()
volumeMounts = append(volumeMounts, k6.Spec.Runner.VolumeMounts...)

job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Expand Down Expand Up @@ -166,14 +172,14 @@ func NewRunnerJob(k6 *v1alpha1.K6, index int, token string) (*batchv1.Job, error
Command: command,
Env: env,
Resources: k6.Spec.Runner.Resources,
VolumeMounts: script.VolumeMount(),
VolumeMounts: volumeMounts,
Ports: ports,
EnvFrom: k6.Spec.Runner.EnvFrom,
LivenessProbe: generateProbe(k6.Spec.Runner.LivenessProbe),
ReadinessProbe: generateProbe(k6.Spec.Runner.ReadinessProbe),
}},
TerminationGracePeriodSeconds: &zero,
Volumes: script.Volume(),
Volumes: volumes,
},
},
},
Expand Down
196 changes: 196 additions & 0 deletions pkg/resources/jobs/runner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1387,3 +1387,199 @@ func TestNewRunnerJobWithInitContainer(t *testing.T) {
t.Errorf("NewRunnerJob returned unexpected data, diff: %s", diff)
}
}

func TestNewRunnerJobWithVolume(t *testing.T) {
script := &types.Script{
Name: "test",
Filename: "thing.js",
Type: "ConfigMap",
}

var zero int64 = 0
automountServiceAccountToken := true

expectedLabels := map[string]string{
"app": "k6",
"k6_cr": "test",
"runner": "true",
"label1": "awesome",
}

expectedVolumes := append(script.Volume(), corev1.Volume{
Name: "k6-test-volume",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
})

expectedVolumeMounts := append(script.VolumeMount(), corev1.VolumeMount{
Name: "k6-test-volume",
MountPath: "/test/location",
})

expectedOutcome := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: "test-1",
Namespace: "test",
Labels: expectedLabels,
Annotations: map[string]string{
"awesomeAnnotation": "dope",
},
},
Spec: batchv1.JobSpec{
BackoffLimit: new(int32),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: expectedLabels,
Annotations: map[string]string{
"awesomeAnnotation": "dope",
},
},
Spec: corev1.PodSpec{
Hostname: "test-1",
RestartPolicy: corev1.RestartPolicyNever,
SecurityContext: &corev1.PodSecurityContext{},
Affinity: nil,
NodeSelector: nil,
Tolerations: nil,
ServiceAccountName: "default",
AutomountServiceAccountToken: &automountServiceAccountToken,
InitContainers: []corev1.Container{
{
Name: "k6-init-0",
Image: "busybox:1.28",
Command: []string{"sh", "-c", "cat /test/test.js"},
VolumeMounts: expectedVolumeMounts,
ImagePullPolicy: corev1.PullNever,
EnvFrom: []corev1.EnvFromSource{
{
ConfigMapRef: &corev1.ConfigMapEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "env",
},
},
},
},
},
},
Containers: []corev1.Container{{
Image: "ghcr.io/grafana/k6-operator:latest-runner",
ImagePullPolicy: corev1.PullNever,
Name: "k6",
Command: []string{"k6", "run", "--quiet", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1", "--tag", "job_name=test-1"},
Env: []corev1.EnvVar{},
Resources: corev1.ResourceRequirements{},
VolumeMounts: expectedVolumeMounts,
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
EnvFrom: []corev1.EnvFromSource{
{
ConfigMapRef: &corev1.ConfigMapEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "env",
},
},
},
},
LivenessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Port: intstr.IntOrString{IntVal: 6565},
Scheme: "HTTP",
},
},
},
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/v1/status",
Port: intstr.IntOrString{IntVal: 6565},
Scheme: "HTTP",
},
},
},
}},
TerminationGracePeriodSeconds: &zero,
Volumes: expectedVolumes,
},
},
},
}
k6 := &v1alpha1.K6{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
Spec: v1alpha1.K6Spec{
Script: v1alpha1.K6Script{
ConfigMap: v1alpha1.K6Configmap{
Name: "test",
File: "test.js",
},
},
Runner: v1alpha1.Pod{
Metadata: v1alpha1.PodMetadata{
Labels: map[string]string{
"label1": "awesome",
},
Annotations: map[string]string{
"awesomeAnnotation": "dope",
},
},
EnvFrom: []corev1.EnvFromSource{
{
ConfigMapRef: &corev1.ConfigMapEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "env",
},
},
},
},
ImagePullPolicy: corev1.PullNever,
InitContainers: []v1alpha1.InitContainer{
{
Image: "busybox:1.28",
Command: []string{"sh", "-c", "cat /test/test.js"},
EnvFrom: []corev1.EnvFromSource{
{
ConfigMapRef: &corev1.ConfigMapEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "env",
},
},
},
},
VolumeMounts: []corev1.VolumeMount{
corev1.VolumeMount{
Name: "k6-test-volume",
MountPath: "/test/location",
},
},
},
},
Volumes: []corev1.Volume{
corev1.Volume{
Name: "k6-test-volume",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
},
VolumeMounts: []corev1.VolumeMount{
corev1.VolumeMount{
Name: "k6-test-volume",
MountPath: "/test/location",
},
},
},
},
}

job, err := NewRunnerJob(k6, 1, "")
if err != nil {
t.Errorf("NewRunnerJob errored, got: %v", err)
}
if diff := deep.Equal(job, expectedOutcome); diff != nil {
t.Errorf("NewRunnerJob returned unexpected data, diff: %s", diff)
}
}

0 comments on commit 17315da

Please sign in to comment.