diff --git a/README.md b/README.md index bf106c00..f9365d14 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,7 @@ Auditors can also be run individually. | `image` | Finds containers which do not use the desired version of an image (via the tag) or use an image without a tag. | [docs](docs/auditors/image.md) | | `limits` | Finds containers which exceed the specified CPU and memory limits or do not specify any. | [docs](docs/auditors/limits.md) | | `mountds` | Finds containers that have docker socket mounted. | [docs](docs/auditors/mountds.md) | +| `mounts` | Finds containers that have sensitive host paths mounted. | [docs](docs/auditors/mountds.md) | | `netpols` | Finds namespaces that do not have a default-deny network policy. | [docs](docs/auditors/netpols.md) | | `nonroot` | Finds containers running as root. | [docs](docs/auditors/nonroot.md) | | `privesc` | Finds containers that allow privilege escalation. | [docs](docs/auditors/privesc.md) | diff --git a/auditors/all/all.go b/auditors/all/all.go index 3fb72b76..cf5acbeb 100644 --- a/auditors/all/all.go +++ b/auditors/all/all.go @@ -12,6 +12,7 @@ import ( "github.com/Shopify/kubeaudit/auditors/image" "github.com/Shopify/kubeaudit/auditors/limits" "github.com/Shopify/kubeaudit/auditors/mountds" + "github.com/Shopify/kubeaudit/auditors/mounts" "github.com/Shopify/kubeaudit/auditors/netpols" "github.com/Shopify/kubeaudit/auditors/nonroot" "github.com/Shopify/kubeaudit/auditors/privesc" @@ -30,6 +31,7 @@ var AuditorNames = []string{ hostns.Name, image.Name, limits.Name, + mounts.Name, mountds.Name, netpols.Name, nonroot.Name, @@ -71,6 +73,8 @@ func initAuditor(name string, conf config.KubeauditConfig) (kubeaudit.Auditable, return image.New(conf.GetAuditorConfigs().Image), nil case limits.Name: return limits.New(conf.GetAuditorConfigs().Limits) + case mounts.Name: + return mounts.New(conf.GetAuditorConfigs().Mounts), nil case mountds.Name: return mountds.New(), nil case netpols.Name: diff --git a/auditors/mounts/config.go b/auditors/mounts/config.go new file mode 100644 index 00000000..bc4b3511 --- /dev/null +++ b/auditors/mounts/config.go @@ -0,0 +1,12 @@ +package mounts + +type Config struct { + SensitivePaths []string `yaml:"denyPathsList"` +} + +func (config *Config) GetSensitivePaths() []string { + if config == nil || len(config.SensitivePaths) == 0 { + return DefaultSensitivePaths + } + return config.SensitivePaths +} diff --git a/auditors/mounts/fixtures/docker-sock-mounted.yml b/auditors/mounts/fixtures/docker-sock-mounted.yml new file mode 100644 index 00000000..17802aa6 --- /dev/null +++ b/auditors/mounts/fixtures/docker-sock-mounted.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod + namespace: docker-sock-mounted +spec: + containers: + - name: container + image: scratch + volumeMounts: + - mountPath: /var/run/docker.sock + name: docker-sock-volume + volumes: + - name: docker-sock-volume + hostPath: + path: /var/run/docker.sock diff --git a/auditors/mounts/fixtures/proc-mounted-allowed-multi-containers-multi-labels.yml b/auditors/mounts/fixtures/proc-mounted-allowed-multi-containers-multi-labels.yml new file mode 100644 index 00000000..95e517cb --- /dev/null +++ b/auditors/mounts/fixtures/proc-mounted-allowed-multi-containers-multi-labels.yml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod + labels: + name: pod + container.audit.kubernetes.io/container1.allow-host-path-mount-proc-volume: "SomeReason" + container.audit.kubernetes.io/container2.allow-host-path-mount-proc-volume: "SomeReason" + namespace: proc-mounted-allowed-multi-containers-multi-labels +spec: + containers: + - name: container1 + image: scratch + volumeMounts: + - mountPath: /host/proc + name: proc-volume + - name: container2 + image: scratch + volumeMounts: + - mountPath: /host/proc + name: proc-volume + volumes: + - name: proc-volume + hostPath: + path: /proc diff --git a/auditors/mounts/fixtures/proc-mounted-allowed-multi-containers-single-label.yml b/auditors/mounts/fixtures/proc-mounted-allowed-multi-containers-single-label.yml new file mode 100644 index 00000000..f6a99664 --- /dev/null +++ b/auditors/mounts/fixtures/proc-mounted-allowed-multi-containers-single-label.yml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod + labels: + name: pod + container.audit.kubernetes.io/container1.allow-host-path-mount-proc-volume: "SomeReason" + namespace: proc-mounted-allowed-multi-containers-single-label +spec: + containers: + - name: container1 + image: scratch + volumeMounts: + - mountPath: /host/proc + name: proc-volume + - name: container2 + image: scratch + volumeMounts: + - mountPath: /host/proc + name: proc-volume + volumes: + - name: proc-volume + hostPath: + path: /proc diff --git a/auditors/mounts/fixtures/proc-mounted-allowed.yml b/auditors/mounts/fixtures/proc-mounted-allowed.yml new file mode 100644 index 00000000..527a93ee --- /dev/null +++ b/auditors/mounts/fixtures/proc-mounted-allowed.yml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod + labels: + name: pod + audit.kubernetes.io/pod.allow-host-path-mount-proc-volume: "SomeReason" + namespace: proc-mounted-allowed +spec: + containers: + - name: container + image: scratch + volumeMounts: + - mountPath: /host/proc + name: proc-volume + volumes: + - name: proc-volume + hostPath: + path: /proc diff --git a/auditors/mounts/fixtures/proc-mounted.yml b/auditors/mounts/fixtures/proc-mounted.yml new file mode 100644 index 00000000..a2879968 --- /dev/null +++ b/auditors/mounts/fixtures/proc-mounted.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod + namespace: proc-mounted +spec: + containers: + - name: container + image: scratch + volumeMounts: + - mountPath: /host/proc + name: proc-volume + volumes: + - name: proc-volume + hostPath: + path: /proc diff --git a/auditors/mounts/mounts.go b/auditors/mounts/mounts.go new file mode 100644 index 00000000..d855a31c --- /dev/null +++ b/auditors/mounts/mounts.go @@ -0,0 +1,123 @@ +package mounts + +import ( + "fmt" + "github.com/Shopify/kubeaudit" + "github.com/Shopify/kubeaudit/internal/k8s" + "github.com/Shopify/kubeaudit/internal/override" + "github.com/Shopify/kubeaudit/k8stypes" + v1 "k8s.io/api/core/v1" +) + +const Name = "mounts" + +const ( + // SensitivePathsMounted occurs when a container has sensitive host paths mounted + SensitivePathsMounted = "SensitivePathsMounted" +) + +// DefaultSensitivePaths is the default list of sensitive mount paths (from Falco rule: https://github.com/falcosecurity/falco/blob/master/rules/k8s_audit_rules.yaml#L155) +var DefaultSensitivePaths = []string{"/proc", "/var/run/docker.sock", "/", "/etc", "/root", "/var/run/crio/crio.sock", "/home/admin", "/var/lib/kubelet", "/var/lib/kubelet/pki", "/etc/kubernetes", "/etc/kubernetes/manifests"} + +const overrideLabelPrefix = "allow-host-path-mount-" + +const ( + MountNameMetadataKey = "MountName" + MountPathMetadataKey = "MountPath" + MountReadOnlyMetadataKey = "MountReadOnly" + MountVolumeNameKey = "MountVolume" + MountVolumeHostPathKey = "MountVolumeHostPath" +) + +// SensitivePathMounts implements Auditable +type SensitivePathMounts struct { + sensitivePaths map[string]bool +} + +func New(config Config) *SensitivePathMounts { + paths := make(map[string]bool) + for _, path := range config.GetSensitivePaths() { + paths[path] = true + } + return &SensitivePathMounts{ + sensitivePaths: paths, + } +} + +// Audit checks that the container does not have any sensitive host path +func (sensitive *SensitivePathMounts) Audit(resource k8stypes.Resource, _ []k8stypes.Resource) ([]*kubeaudit.AuditResult, error) { + var auditResults []*kubeaudit.AuditResult + + spec := k8s.GetPodSpec(resource) + if spec == nil { + return auditResults, nil + } + + sensitiveVolumes := auditPodVolumes(spec, sensitive.sensitivePaths) + + if len(sensitiveVolumes) == 0 { + return auditResults, nil + } + + for _, container := range k8s.GetContainers(resource) { + for _, auditResult := range auditContainer(container, sensitiveVolumes) { + auditResult = override.ApplyOverride(auditResult, container.Name, resource, getOverrideLabel(auditResult.Metadata[MountNameMetadataKey])) + if auditResult != nil { + auditResults = append(auditResults, auditResult) + } + } + } + + return auditResults, nil +} + +func auditPodVolumes(podSpec *k8stypes.PodSpecV1, sensitivePaths map[string]bool) map[string]v1.Volume { + if podSpec.Volumes == nil { + return nil + } + + found := make(map[string]v1.Volume) + for _, volume := range podSpec.Volumes { + if volume.HostPath == nil { + continue + } + + if _, ok := sensitivePaths[volume.HostPath.Path]; ok { + found[volume.Name] = volume + } + } + + return found +} + +func auditContainer(container *k8stypes.ContainerV1, sensitiveVolumes map[string]v1.Volume) []*kubeaudit.AuditResult { + if container.VolumeMounts == nil { + return nil + } + + var auditResults []*kubeaudit.AuditResult + + for _, mount := range container.VolumeMounts { + if volume, ok := sensitiveVolumes[mount.Name]; ok { + auditResults = append(auditResults, &kubeaudit.AuditResult{ + Name: SensitivePathsMounted, + Severity: kubeaudit.Error, + Message: fmt.Sprintf("Sensitive path mounted as volume: %s (hostPath: %s). It should be removed from the container's mounts list.", mount.Name, volume.HostPath.Path), + Metadata: kubeaudit.Metadata{ + "Container": container.Name, + MountNameMetadataKey: mount.Name, + MountPathMetadataKey: mount.MountPath, + MountReadOnlyMetadataKey: fmt.Sprintf("%t", mount.ReadOnly), + MountVolumeNameKey: volume.Name, + MountVolumeHostPathKey: volume.HostPath.Path, + }, + }) + } + } + + return auditResults +} + +func getOverrideLabel(mountName string) string { + return overrideLabelPrefix + mountName +} diff --git a/auditors/mounts/mounts_test.go b/auditors/mounts/mounts_test.go new file mode 100644 index 00000000..8be78a18 --- /dev/null +++ b/auditors/mounts/mounts_test.go @@ -0,0 +1,34 @@ +package mounts + +import ( + "github.com/Shopify/kubeaudit/internal/override" + "strings" + "testing" + + "github.com/Shopify/kubeaudit/internal/test" +) + +const fixtureDir = "fixtures" + +func TestSensitivePathsMounted(t *testing.T) { + cases := []struct { + file string + fixtureDir string + expectedErrors []string + }{ + {"docker-sock-mounted.yml", fixtureDir, []string{SensitivePathsMounted}}, + {"proc-mounted.yml", fixtureDir, []string{SensitivePathsMounted}}, + {"proc-mounted-allowed.yml", fixtureDir, []string{override.GetOverriddenResultName(SensitivePathsMounted)}}, + {"proc-mounted-allowed-multi-containers-multi-labels.yml", fixtureDir, []string{override.GetOverriddenResultName(SensitivePathsMounted)}}, + {"proc-mounted-allowed-multi-containers-single-label.yml", fixtureDir, []string{SensitivePathsMounted, override.GetOverriddenResultName(SensitivePathsMounted)}}, + } + + config := Config{} + + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + test.AuditManifest(t, tc.fixtureDir, tc.file, New(config), tc.expectedErrors) + test.AuditLocal(t, tc.fixtureDir, tc.file, New(config), strings.Split(tc.file, ".")[0], tc.expectedErrors) + }) + } +} diff --git a/cmd/commands/all.go b/cmd/commands/all.go index 220e7cd1..b86fd2f6 100644 --- a/cmd/commands/all.go +++ b/cmd/commands/all.go @@ -47,6 +47,10 @@ func setConfigFromFlags(cmd *cobra.Command, conf config.KubeauditConfig) config. conf.AuditorConfig.Capabilities.AllowAddList = capabilitiesConfig.AllowAddList } + if flagset.Changed(sensitivePathsFlagName) { + conf.AuditorConfig.Mounts.SensitivePaths = mountsConfig.SensitivePaths + } + return conf } @@ -88,4 +92,5 @@ func init() { setImageFlags(auditAllCmd) setLimitsFlags(auditAllCmd) setCapabilitiesFlags(auditAllCmd) + setPathsFlags(auditAllCmd) } diff --git a/cmd/commands/mounts.go b/cmd/commands/mounts.go new file mode 100644 index 00000000..94e6ddab --- /dev/null +++ b/cmd/commands/mounts.go @@ -0,0 +1,48 @@ +package commands + +import ( + "bytes" + "fmt" + "github.com/Shopify/kubeaudit/auditors/mounts" + "github.com/spf13/cobra" + "strings" +) + +const sensitivePathsFlagName = "denyPathsList" + +var mountsConfig mounts.Config + +var mountsCmd = &cobra.Command{ + Use: "mounts", + Short: "Audit containers that mount sensitive paths", + Long: fmt.Sprintf(`This command determines which containers mount sensitive host paths. If no paths list is provided, the following +paths are used: +%s + +A WARN result is generated when a container mounts one or more paths specified with the '--denyPathsList' argument. + +Example usage: +kubeaudit mounts --denyPathsList "%s"`, formatPathsList(), strings.Join(mounts.DefaultSensitivePaths[:3], ",")), + Run: func(cmd *cobra.Command, args []string) { + runAudit(mounts.New(mountsConfig))(cmd, args) + }, +} + +func init() { + RootCmd.AddCommand(mountsCmd) + setPathsFlags(mountsCmd) +} + +func setPathsFlags(cmd *cobra.Command) { + cmd.Flags().StringSliceVarP(&mountsConfig.SensitivePaths, sensitivePathsFlagName, "d", mounts.DefaultSensitivePaths, + "List of sensitive paths that shouldn't be mounted") +} + +func formatPathsList() string { + var buffer bytes.Buffer + for _, path := range mounts.DefaultSensitivePaths { + buffer.WriteString("\n- ") + buffer.WriteString(path) + } + return buffer.String() +} diff --git a/config/config.go b/config/config.go index f6459f02..c0761618 100644 --- a/config/config.go +++ b/config/config.go @@ -1,6 +1,7 @@ package config import ( + "github.com/Shopify/kubeaudit/auditors/mounts" "io" "io/ioutil" @@ -54,4 +55,5 @@ type AuditorConfig struct { Capabilities capabilities.Config `yaml:"capabilities"` Image image.Config `yaml:"image"` Limits limits.Config `yaml:"limits"` + Mounts mounts.Config `yaml:"mounts"` } diff --git a/config/config.yaml b/config/config.yaml index d3d14350..ed81448a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -9,6 +9,7 @@ enabledAuditors: image: true limits: true mountds: true + mounts: true netpols: true nonroot: true privesc: true @@ -17,10 +18,12 @@ enabledAuditors: seccomp: true auditors: capabilities: - # add capabilities needed to the add list, so kubeaudit won't report errors + # add capabilities needed to the add list, so kubeaudit won't report errors add: ["AUDIT_WRITE", "CHOWN", "KILL"] image: image: "myimage:mytag" limits: cpu: "750m" memory: "500m" + mounts: + denyPathsList: ["/proc", "/var/run/docker.sock", "/", "/etc", "/root", "/var/run/crio/crio.sock", "/home/admin", "/var/lib/kubelet", "/var/lib/kubelet/pki", "/etc/kubernetes", "/etc/kubernetes/manifests"] diff --git a/docs/auditors/mounts.md b/docs/auditors/mounts.md new file mode 100644 index 00000000..50c3cee0 --- /dev/null +++ b/docs/auditors/mounts.md @@ -0,0 +1,276 @@ +# Sensitive Host Path Mounted Auditor (mounts) + +Finds containers that have sensitive host paths mounted. + +## General Usage + +``` +kubeaudit mounts [flags] +``` + +### Flags + +| Short | Long | Description | Default | +| :------ | :---------------- | :------------------------------------------------------------------- | :----------------------------------------------------------------------- | +| -d | --denyPathsList | List of sensitive paths that shouldn't be mounted. | [default sensitive host paths list](#Default-sensitive-host-paths-list) | + +Also see [Global Flags](/README.md#global-flags) + +#### Default sensitive host paths list + +| Host path | Description | +| :------------------------ | :--------------------------------------------------------------------------------------- | +| /proc | Pseudo-filesystem which provides an interface to kernel data structures | +| /var/run/docker.sock | Unix socket used to communicate with Docker daemon | +| / | Filesystem's root | +| /etc | Directory that usually contains all system related configurations files | +| /root | Home directory of the `root` user | +| /var/run/crio/crio.sock | Unix socket used to communicate with the CRI-O Container Engine | +| /home/admin | Home directory of the `admin` user | +| /var/lib/kubelet | Directory for Kublet-related configuration | +| /var/lib/kubelet/pki | Directory containing the certificate and private key of the kublet | +| /etc/kubernetes | Directory containing Kubernetes related configuration | +| /etc/kubernetes/manifests | Directory containing manifest of Kubernetes components | + +## Examples + +``` +$ kubeaudit mounts -f auditors/mounts/fixtures/proc-mounted.yml + +---------------- Results for --------------- + + apiVersion: v1 + kind: Pod + metadata: + name: pod + namespace: proc-mounted + +-------------------------------------------- + +-- [error] SensitivePathsMounted + Message: Sensitive path mounted as volume: proc-volume (/proc -> /host/proc, readOnly: false). It should be removed from the container's mounts list. + Metadata: + Container: container + MountName: proc-volume + MountPath: /host/proc + MountReadOnly: false + MountVolume: proc-volume + MountVolumeHostPath: /proc + +``` + +### Example with Config File + +If you don't want kubeaudit to raise errors for all the paths in the default list (`DefaultSensitivePaths`), you can +provide a custom paths list in the config file. See [docs](docs/all.md) for more information. That way kubeaudit will +only raise errors for those specific paths listed in the config file. + +`config.yaml` + +```yaml +--- +enabledAuditors: + mounts: true +auditors: + mounts: + denyPathsList: ["/etc", "/var/run/docker.sock"] +``` + +`manifest.yaml` + +```yaml +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: deployment + namespace: example-namespace +spec: + template: + spec: + containers: + - name: container + image: scratch + volumeMounts: + - mountPath: /host/etc + name: etc-volume + - mountPath: /var/run/docker.sock + name: docker-socket-volume + volumes: + - name: etc-volume + hostPath: + path: /etc + - name: docker-socket-volume + hostPath: + path: /var/run/docker.sock +``` + +```shell +$ kubeaudit all --kconfig "config.yaml" -f "manifest.yaml" + +---------------- Results for --------------- + + apiVersion: apps/v1beta2 + kind: Deployment + metadata: + name: deployment + namespace: example-namespace + +-------------------------------------------- + +-- [error] SensitivePathsMounted + Message: Sensitive path mounted as volume: etc-volume (hostPath: /etc). It should be removed from the container's mounts list. + Metadata: + Container: container + MountName: etc-volume + MountPath: /host/etc + MountReadOnly: false + MountVolume: etc-volume + MountVolumeHostPath: /etc + +-- [error] SensitivePathsMounted + Message: Sensitive path mounted as volume: docker-socket-volume (hostPath: /var/run/docker.sock). It should be removed from the container's mounts list. + Metadata: + MountReadOnly: false + MountVolume: docker-socket-volume + MountVolumeHostPath: /var/run/docker.sock + Container: container + MountName: docker-socket-volume + MountPath: /var/run/docker.sock +``` + +### Example with Custom Paths List + +A custom paths list can be provided as a comma separated value list of paths using the `--denyPathsList` flag. These are +the host paths you'd like to have kubeaudit raise an error when they are mounted in a container. + +`manifest.yaml` (example manifest) + +```yaml +volumes: + - name: etc-volume + hostPath: + path: /etc + - name: docker-socket-volume + hostPath: + path: /var/run/docker.sock +``` + +```shell +$ kubeaudit mounts --denyPathsList "/etc,/var/run/docker.sock" -f "manifest.yaml" +---------------- Results for --------------- + + apiVersion: apps/v1beta2 + kind: Deployment + metadata: + name: deployment + namespace: example-namespace + +-------------------------------------------- + +-- [error] SensitivePathsMounted + Message: Sensitive path mounted as volume: etc-volume (hostPath: /etc). It should be removed from the container's mounts list. + Metadata: + Container: container + MountName: etc-volume + MountPath: /host/etc + MountReadOnly: false + MountVolume: etc-volume + MountVolumeHostPath: /etc + +-- [error] SensitivePathsMounted + Message: Sensitive path mounted as volume: docker-socket-volume (hostPath: /var/run/docker.sock). It should be removed from the container's mounts list. + Metadata: + Container: container + MountName: docker-socket-volume + MountPath: /var/run/docker.sock + MountReadOnly: false + MountVolume: docker-socket-volume + MountVolumeHostPath: /var/run/docker.sock +``` + +## Explanation + +Mounting some sensitive host paths (like `/etc`, `/proc`, or `/var/run/docker.sock`) may allow a container to access +sensitive information from the host like credentials or to spy on other workloads' activity. + +These sensitive paths should not be mounted. + +Example of a resource which **fails** the `mounts` audit: + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: container + image: scratch + volumeMounts: + - mountPath: /host/proc + name: proc-volume + volumes: + - name: proc-volume + hostPath: + path: /proc +``` + +## Override Errors + +First, see the [Introduction to Override Errors](/README.md#override-errors). + +The override identifier has the format `allow-host-path-mount-[mount name]` which allows for each mount to be +individually overridden. + +Example of resource with `mounts` overridden for a specific container: + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + template: #PodTemplateSpec + metadata: + labels: + container.audit.kubernetes.io/container2.allow-host-path-mount-proc-volume: "SomeReason" + spec: #PodSpec + containers: + - name: container1 + image: scratch + - name: container2 + image: scratch + volumeMounts: + - mountPath: /host/proc + name: proc-volume + volumes: + - name: proc-volume + hostPath: + path: /proc +``` + +Example of resource with `mounts` overridden for a whole pod: + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + template: #PodTemplateSpec + metadata: + labels: + audit.kubernetes.io/pod.allow-host-path-mount-proc-volume: "SomeReason" + spec: #PodSpec + containers: + - name: container1 + image: scratch + volumeMounts: + - mountPath: /host/proc + name: proc-volume + - name: container2 + image: scratch + volumeMounts: + - mountPath: /host/proc + name: proc-volume + volumes: + - name: proc-volume + hostPath: + path: /proc +```