Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement set-security-context feature for affinity assistant containers #8182

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/additional-configs.md
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,8 @@ Out-of-the-box, Tekton Pipelines Controller is configured for relatively small-s

To allow TaskRuns and PipelineRuns to run in namespaces with [restricted pod security standards](https://kubernetes.io/docs/concepts/security/pod-security-standards/),
set the "set-security-context" feature flag to "true" in the [feature-flags configMap](#customizing-the-pipelines-controller-behavior). This configuration option applies a [SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)
to any containers injected into TaskRuns by the Pipelines controller. This SecurityContext may not be supported in all Kubernetes implementations (for example, OpenShift).
to any containers injected into TaskRuns by the Pipelines controller. If the [Affinity Assistants](affinityassistants.md) feature is enabled, the SecurityContext is also applied to those containers.
This SecurityContext may not be supported in all Kubernetes implementations (for example, OpenShift).

**Note**: running TaskRuns and PipelineRuns in the "tekton-pipelines" namespace is discouraged.

Expand Down
6 changes: 6 additions & 0 deletions pkg/internal/affinityassistant/affinityassistant_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,9 @@ func GetAffinityAssistantBehavior(ctx context.Context) (AffinityAssistantBehavio

return "", fmt.Errorf("unknown combination of disable-affinity-assistant: %v and coschedule: %v", disableAA, coschedule)
}

// ContainerConfig defines AffinityAssistant container configuration
type ContainerConfig struct {
Image string
SetSecurityContext bool
}
20 changes: 10 additions & 10 deletions pkg/pod/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ const (
// SpiffeCsiDriver is the CSI storage plugin needed for injection of SPIFFE workload api.
SpiffeCsiDriver = "csi.spiffe.io"

// osSelectorLabel is the label Kubernetes uses for OS-specific workloads (https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-os)
osSelectorLabel = "kubernetes.io/os"
// OsSelectorLabel is the label Kubernetes uses for OS-specific workloads (https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-os)
OsSelectorLabel = "kubernetes.io/os"

// TerminationReasonTimeoutExceeded indicates a step execution timed out.
TerminationReasonTimeoutExceeded = "TimeoutExceeded"
Expand Down Expand Up @@ -132,10 +132,10 @@ var (
allowPrivilegeEscalation = false
runAsNonRoot = true

// The following security contexts allow init containers to run in namespaces
// LinuxSecurityContext allow init containers to run in namespaces
// with "restricted" pod security admission
// See https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
linuxSecurityContext = &corev1.SecurityContext{
LinuxSecurityContext = &corev1.SecurityContext{
AllowPrivilegeEscalation: &allowPrivilegeEscalation,
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{"ALL"},
Expand All @@ -145,7 +145,7 @@ var (
Type: corev1.SeccompProfileTypeRuntimeDefault,
},
}
windowsSecurityContext = &corev1.SecurityContext{
WindowsSecurityContext = &corev1.SecurityContext{
RunAsNonRoot: &runAsNonRoot,
}
)
Expand Down Expand Up @@ -607,9 +607,9 @@ func entrypointInitContainer(image string, steps []v1.Step, setSecurityContext,
command = append(command, StepName(s.Name, i))
}
volumeMounts := []corev1.VolumeMount{binMount, internalStepsMount}
securityContext := linuxSecurityContext
securityContext := LinuxSecurityContext
if windows {
securityContext = windowsSecurityContext
securityContext = WindowsSecurityContext
}

// Rewrite steps with entrypoint binary. Append the entrypoint init
Expand Down Expand Up @@ -679,9 +679,9 @@ func createResultsSidecar(taskSpec v1.TaskSpec, image string, setSecurityContext
Image: image,
Command: command,
}
securityContext := linuxSecurityContext
securityContext := LinuxSecurityContext
if windows {
securityContext = windowsSecurityContext
securityContext = WindowsSecurityContext
}
if setSecurityContext {
sidecar.SecurityContext = securityContext
Expand All @@ -696,7 +696,7 @@ func usesWindows(tr *v1.TaskRun) bool {
if tr.Spec.PodTemplate == nil || tr.Spec.PodTemplate.NodeSelector == nil {
return false
}
osSelector := tr.Spec.PodTemplate.NodeSelector[osSelectorLabel]
osSelector := tr.Spec.PodTemplate.NodeSelector[OsSelectorLabel]
return osSelector == "windows"
}

Expand Down
12 changes: 6 additions & 6 deletions pkg/pod/pod_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2162,7 +2162,7 @@ _EOF_
{Name: "tekton-internal-bin", ReadOnly: true, MountPath: "/tekton/bin"},
{Name: "tekton-internal-run-0", ReadOnly: true, MountPath: "/tekton/run/0"},
}, implicitVolumeMounts...),
SecurityContext: linuxSecurityContext,
SecurityContext: LinuxSecurityContext,
}},
Volumes: append(implicitVolumes, binVolume, runVolume(0), downwardVolume, corev1.Volume{
Name: "tekton-creds-init-home-0",
Expand Down Expand Up @@ -2403,7 +2403,7 @@ _EOF_
{Name: "tekton-internal-bin", ReadOnly: true, MountPath: "/tekton/bin"},
{Name: "tekton-internal-run-0", ReadOnly: true, MountPath: "/tekton/run/0"},
}, implicitVolumeMounts...),
SecurityContext: linuxSecurityContext,
SecurityContext: LinuxSecurityContext,
}},
Volumes: append(implicitVolumes, binVolume, runVolume(0), downwardVolume, corev1.Volume{
Name: "tekton-creds-init-home-0",
Expand Down Expand Up @@ -3424,7 +3424,7 @@ func TestPrepareInitContainers(t *testing.T) {
WorkingDir: "/",
Command: []string{"/ko-app/entrypoint", "init", "/ko-app/entrypoint", entrypointBinary, "step-foo", "step-bar"},
VolumeMounts: []corev1.VolumeMount{binMount, internalStepsMount},
SecurityContext: linuxSecurityContext,
SecurityContext: LinuxSecurityContext,
},
}, {
name: "nothing-special-two-steps-windows",
Expand Down Expand Up @@ -3456,7 +3456,7 @@ func TestPrepareInitContainers(t *testing.T) {
WorkingDir: "/",
Command: []string{"/ko-app/entrypoint", "init", "/ko-app/entrypoint", entrypointBinary, "step-foo", "step-bar"},
VolumeMounts: []corev1.VolumeMount{binMount, internalStepsMount},
SecurityContext: windowsSecurityContext,
SecurityContext: WindowsSecurityContext,
},
}}
for _, tc := range tcs {
Expand Down Expand Up @@ -3485,13 +3485,13 @@ func TestUsesWindows(t *testing.T) {
}, {
name: "uses linux",
taskRun: &v1.TaskRun{Spec: v1.TaskRunSpec{PodTemplate: &pod.Template{NodeSelector: map[string]string{
osSelectorLabel: "linux",
OsSelectorLabel: "linux",
}}}},
want: false,
}, {
name: "uses windows",
taskRun: &v1.TaskRun{Spec: v1.TaskRunSpec{PodTemplate: &pod.Template{NodeSelector: map[string]string{
osSelectorLabel: "windows",
OsSelectorLabel: "windows",
}}}},
want: true,
}}
Expand Down
4 changes: 2 additions & 2 deletions pkg/pod/script.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,13 @@ func convertScripts(shellImageLinux string, shellImageWin string, steps []v1.Ste
shellImage := shellImageLinux
shellCommand := "sh"
shellArg := "-c"
securityContext := linuxSecurityContext
securityContext := LinuxSecurityContext
// Set windows variants for Image, Command and Args
if requiresWindows {
shellImage = shellImageWin
shellCommand = "pwsh"
shellArg = "-Command"
securityContext = windowsSecurityContext
securityContext = WindowsSecurityContext
}

placeScriptsInit := corev1.Container{
Expand Down
14 changes: 7 additions & 7 deletions pkg/pod/script_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ _EOF_
/tekton/bin/entrypoint decode-script "${scriptfile}"
`},
VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount},
SecurityContext: linuxSecurityContext,
SecurityContext: LinuxSecurityContext,
}
want := []corev1.Container{{
Image: "step-1",
Expand Down Expand Up @@ -465,7 +465,7 @@ fi
debug-fail-continue-heredoc-randomly-generated-6nl7g
`},
VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount, debugScriptsVolumeMount},
SecurityContext: linuxSecurityContext,
SecurityContext: LinuxSecurityContext,
},
wantSteps: []corev1.Container{{
Image: "step-1",
Expand Down Expand Up @@ -608,7 +608,7 @@ fi
debug-beforestep-fail-continue-heredoc-randomly-generated-6nl7g
`},
VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount, debugScriptsVolumeMount},
SecurityContext: linuxSecurityContext},
SecurityContext: LinuxSecurityContext},
wantSteps: []corev1.Container{{
Name: "step-1",
Image: "step-1",
Expand Down Expand Up @@ -690,7 +690,7 @@ _EOF_
/tekton/bin/entrypoint decode-script "${scriptfile}"
`},
VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount},
SecurityContext: linuxSecurityContext,
SecurityContext: LinuxSecurityContext,
}
want := []corev1.Container{{
Image: "step-1",
Expand Down Expand Up @@ -777,7 +777,7 @@ no-shebang
"@ | Out-File -FilePath /tekton/scripts/script-3-mssqb.cmd
`},
VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount},
SecurityContext: windowsSecurityContext,
SecurityContext: WindowsSecurityContext,
}
want := []corev1.Container{{
Image: "step-1",
Expand Down Expand Up @@ -860,7 +860,7 @@ sidecar-1
"@ | Out-File -FilePath /tekton/scripts/sidecar-script-0-mssqb
`},
VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount},
SecurityContext: windowsSecurityContext,
SecurityContext: WindowsSecurityContext,
}
want := []corev1.Container{{
Image: "step-1",
Expand Down Expand Up @@ -922,7 +922,7 @@ sidecar-1
"@ | Out-File -FilePath /tekton/scripts/sidecar-script-0-9l9zj
`},
VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount},
SecurityContext: windowsSecurityContext,
SecurityContext: WindowsSecurityContext,
}
want := []corev1.Container{{
Image: "step-1",
Expand Down
4 changes: 2 additions & 2 deletions pkg/pod/workingdir_init.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@ func workingDirInit(workingdirinitImage string, stepContainers []corev1.Containe
// There are no workingDirs to initialize.
return nil
}
securityContext := linuxSecurityContext
securityContext := LinuxSecurityContext
if windows {
securityContext = windowsSecurityContext
securityContext = WindowsSecurityContext
}

c := &corev1.Container{
Expand Down
4 changes: 2 additions & 2 deletions pkg/pod/workingdir_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func TestWorkingDirInit(t *testing.T) {
Args: []string{"/workspace/bbb", "aaa", "zzz"},
WorkingDir: pipeline.WorkspaceDir,
VolumeMounts: implicitVolumeMounts,
SecurityContext: linuxSecurityContext,
SecurityContext: LinuxSecurityContext,
},
}, {
desc: "workingDirs are unique and sorted, absolute dirs are ignored, uses windows",
Expand Down Expand Up @@ -144,7 +144,7 @@ func TestWorkingDirInit(t *testing.T) {
Args: []string{"/workspace/bbb", "aaa", "zzz"},
WorkingDir: pipeline.WorkspaceDir,
VolumeMounts: implicitVolumeMounts,
SecurityContext: windowsSecurityContext,
SecurityContext: WindowsSecurityContext,
},
}} {
t.Run(c.desc, func(t *testing.T) {
Expand Down
24 changes: 20 additions & 4 deletions pkg/reconciler/pipelinerun/affinity_assistant.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
"github.com/tektoncd/pipeline/pkg/internal/affinityassistant"
aa "github.com/tektoncd/pipeline/pkg/internal/affinityassistant"
pipelinePod "github.com/tektoncd/pipeline/pkg/pod"
"github.com/tektoncd/pipeline/pkg/reconciler/volumeclaim"
"github.com/tektoncd/pipeline/pkg/workspace"
appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -139,7 +140,12 @@ func (c *Reconciler) createOrUpdateAffinityAssistant(ctx context.Context, affini
if err != nil {
return []error{err}
}
affinityAssistantStatefulSet := affinityAssistantStatefulSet(aaBehavior, affinityAssistantName, pr, claimTemplates, claimNames, c.Images.NopImage, cfg.Defaults.DefaultAAPodTemplate)
affinityAssistantContainerConfig := aa.ContainerConfig{
Image: c.Images.NopImage,
SetSecurityContext: cfg.FeatureFlags.SetSecurityContext,
}

affinityAssistantStatefulSet := affinityAssistantStatefulSet(aaBehavior, affinityAssistantName, pr, claimTemplates, claimNames, affinityAssistantContainerConfig, cfg.Defaults.DefaultAAPodTemplate)
_, err = c.KubeClientSet.AppsV1().StatefulSets(pr.Namespace).Create(ctx, affinityAssistantStatefulSet, metav1.CreateOptions{})
if err != nil {
errs = append(errs, fmt.Errorf("failed to create StatefulSet %s: %w", affinityAssistantName, err))
Expand Down Expand Up @@ -281,7 +287,7 @@ func getStatefulSetLabels(pr *v1.PipelineRun, affinityAssistantName string) map[
// with the given AffinityAssistantTemplate applied to the StatefulSet PodTemplateSpec.
// The VolumeClaimTemplates and Volume of StatefulSet reference the PipelineRun WorkspaceBinding VolumeClaimTempalte and the PVCs respectively.
// The PVs created by the StatefulSet are scheduled to the same availability zone which avoids PV scheduling conflict.
func affinityAssistantStatefulSet(aaBehavior aa.AffinityAssistantBehavior, name string, pr *v1.PipelineRun, claimTemplates []corev1.PersistentVolumeClaim, claimNames []string, affinityAssistantImage string, defaultAATpl *pod.AffinityAssistantTemplate) *appsv1.StatefulSet {
func affinityAssistantStatefulSet(aaBehavior aa.AffinityAssistantBehavior, name string, pr *v1.PipelineRun, claimTemplates []corev1.PersistentVolumeClaim, claimNames []string, containerConfig aa.ContainerConfig, defaultAATpl *pod.AffinityAssistantTemplate) *appsv1.StatefulSet {
// We want a singleton pod
replicas := int32(1)

Expand All @@ -296,9 +302,18 @@ func affinityAssistantStatefulSet(aaBehavior aa.AffinityAssistantBehavior, name
mounts = append(mounts, corev1.VolumeMount{Name: claimTemplate.Name, MountPath: claimTemplate.Name})
}

securityContext := &corev1.SecurityContext{}
if containerConfig.SetSecurityContext {
securityContext = pipelinePod.LinuxSecurityContext

if tpl.NodeSelector[pipelinePod.OsSelectorLabel] == "windows" {
securityContext = pipelinePod.WindowsSecurityContext
}
}

containers := []corev1.Container{{
Name: "affinity-assistant",
Image: affinityAssistantImage,
Image: containerConfig.Image,
Args: []string{"tekton_run_indefinitely"},

// Set requests == limits to get QoS class _Guaranteed_.
Expand All @@ -314,7 +329,8 @@ func affinityAssistantStatefulSet(aaBehavior aa.AffinityAssistantBehavior, name
"memory": resource.MustParse("100Mi"),
},
},
VolumeMounts: mounts,
SecurityContext: securityContext,
VolumeMounts: mounts,
}}

var volumes []corev1.Volume
Expand Down
Loading