diff --git a/deploy/kubernetes-1.18-test/README.md b/deploy/kubernetes-1.18-test/README.md new file mode 100644 index 000000000..751095bc9 --- /dev/null +++ b/deploy/kubernetes-1.18-test/README.md @@ -0,0 +1,6 @@ +The deployment for Kubernetes 1.18 uses CSIDriver v1 and +thus is incompatible with Kubernetes < 1.18. + +It uses separate pods and service accounts for each sidecar. This is +not how they would normally be deployed. It gets done this way to test +that the individual RBAC rules are correct. diff --git a/deploy/kubernetes-1.18-test/deploy.sh b/deploy/kubernetes-1.18-test/deploy.sh new file mode 120000 index 000000000..589c43f62 --- /dev/null +++ b/deploy/kubernetes-1.18-test/deploy.sh @@ -0,0 +1 @@ +../util/deploy-hostpath.sh \ No newline at end of file diff --git a/deploy/kubernetes-1.18-test/destroy.sh b/deploy/kubernetes-1.18-test/destroy.sh new file mode 120000 index 000000000..c66894352 --- /dev/null +++ b/deploy/kubernetes-1.18-test/destroy.sh @@ -0,0 +1 @@ +../util/destroy-hostpath.sh \ No newline at end of file diff --git a/deploy/kubernetes-1.18/hostpath/csi-hostpath-attacher.yaml b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-attacher.yaml similarity index 100% rename from deploy/kubernetes-1.18/hostpath/csi-hostpath-attacher.yaml rename to deploy/kubernetes-1.18-test/hostpath/csi-hostpath-attacher.yaml diff --git a/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-driverinfo.yaml b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-driverinfo.yaml new file mode 100644 index 000000000..c8cf666a4 --- /dev/null +++ b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-driverinfo.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: hostpath.csi.k8s.io + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: hostpath.csi.k8s.io + app.kubernetes.io/component: csi-driver +spec: + # Supports persistent and ephemeral inline volumes. + volumeLifecycleModes: + - Persistent + - Ephemeral + # To determine at runtime which mode a volume uses, pod info and its + # "csi.storage.k8s.io/ephemeral" entry are needed. + podInfoOnMount: true diff --git a/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-plugin.yaml b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-plugin.yaml new file mode 100644 index 000000000..50d500c77 --- /dev/null +++ b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-plugin.yaml @@ -0,0 +1,167 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpathplugin + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: plugin +spec: + serviceName: "csi-hostpathplugin" + # One replica only: + # Host path driver only works when everything runs + # on a single node. We achieve that by starting it once and then + # co-locate all other pods via inter-pod affinity + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: plugin + template: + metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: plugin + spec: + serviceAccountName: csi-external-health-monitor-controller + containers: + - name: hostpath + image: k8s.gcr.io/sig-storage/hostpathplugin:v1.6.2 + args: + - "--drivername=hostpath.csi.k8s.io" + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + ports: + - containerPort: 9898 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins-dir + - mountPath: /csi-data-dir + name: csi-data-dir + - mountPath: /dev + name: dev-dir + + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: k8s.gcr.io/sig-storage/livenessprobe:v2.2.0 + args: + - --csi-address=/csi/csi.sock + - --health-port=9898 + + - name: csi-external-health-monitor-agent + image: k8s.gcr.io/sig-storage/csi-external-health-monitor-agent:v0.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + + - name: csi-external-health-monitor-controller + image: k8s.gcr.io/sig-storage/csi-external-health-monitor-controller:v0.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + + - name: node-driver-registrar + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1 + args: + - --v=5 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /registration + name: registration-dir + - mountPath: /csi-data-dir + name: csi-data-dir + + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins + type: Directory + name: plugins-dir + - hostPath: + # 'path' is where PV data is persisted on host. + # using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot + path: /var/lib/csi-hostpath-data/ + type: DirectoryOrCreate + name: csi-data-dir + - hostPath: + path: /dev + type: Directory + name: dev-dir diff --git a/deploy/kubernetes-1.18/hostpath/csi-hostpath-provisioner.yaml b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-provisioner.yaml similarity index 100% rename from deploy/kubernetes-1.18/hostpath/csi-hostpath-provisioner.yaml rename to deploy/kubernetes-1.18-test/hostpath/csi-hostpath-provisioner.yaml diff --git a/deploy/kubernetes-1.18/hostpath/csi-hostpath-resizer.yaml b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-resizer.yaml similarity index 100% rename from deploy/kubernetes-1.18/hostpath/csi-hostpath-resizer.yaml rename to deploy/kubernetes-1.18-test/hostpath/csi-hostpath-resizer.yaml diff --git a/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-snapshotclass.yaml b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-snapshotclass.yaml new file mode 100644 index 000000000..dbf51d96d --- /dev/null +++ b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-snapshotclass.yaml @@ -0,0 +1,11 @@ +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshotClass +metadata: + name: csi-hostpath-snapclass + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-snapclass + app.kubernetes.io/component: volumesnapshotclass +driver: hostpath.csi.k8s.io #csi-hostpath +deletionPolicy: Delete diff --git a/deploy/kubernetes-1.18/hostpath/csi-hostpath-snapshotter.yaml b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-snapshotter.yaml similarity index 100% rename from deploy/kubernetes-1.18/hostpath/csi-hostpath-snapshotter.yaml rename to deploy/kubernetes-1.18-test/hostpath/csi-hostpath-snapshotter.yaml diff --git a/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-testing.yaml b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-testing.yaml new file mode 100644 index 000000000..4e4eb1354 --- /dev/null +++ b/deploy/kubernetes-1.18-test/hostpath/csi-hostpath-testing.yaml @@ -0,0 +1,83 @@ +# WARNING: this is only for testing purposes. Do not install in a production +# cluster. +# +# This exposes the hostpath's Unix domain csi.sock as a TCP port to the +# outside world. The mapping from Unix domain socket to TCP is done +# by socat. +# +# This is useful for testing with csi-sanity or csc. + +apiVersion: v1 +kind: Service +metadata: + name: hostpath-service + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-socat + app.kubernetes.io/component: socat +spec: + type: NodePort + selector: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-socat + app.kubernetes.io/component: socat + ports: + - port: 10000 # fixed port inside the pod, dynamically allocated port outside +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-socat + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-socat + app.kubernetes.io/component: socat +spec: + serviceName: "csi-hostpath-socat" + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-socat + app.kubernetes.io/component: socat + template: + metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-socat + app.kubernetes.io/component: socat + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/instance + operator: In + values: + - hostpath.csi.k8s.io + topologyKey: kubernetes.io/hostname + containers: + - name: socat + image: alpine/socat:1.0.3 + args: + - tcp-listen:10000,fork,reuseaddr + - unix-connect:/csi/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/deploy/kubernetes-1.18-test/test-driver.yaml b/deploy/kubernetes-1.18-test/test-driver.yaml new file mode 100644 index 000000000..d523dfa1f --- /dev/null +++ b/deploy/kubernetes-1.18-test/test-driver.yaml @@ -0,0 +1,24 @@ +# This file describes how to test this deployment of the CSI hostpath driver +# using the Kubernetes 1.17 E2E test suite. For details see: +# https://github.com/kubernetes/kubernetes/tree/v1.17.0/test/e2e/storage/external + +StorageClass: + FromName: true +SnapshotClass: + FromName: true +DriverInfo: + Name: hostpath.csi.k8s.io + SupportedSizeRange: + Min: 1Mi + Capabilities: + block: true + controllerExpansion: true + exec: true + multipods: true + nodeExpansion: true + persistence: true + singleNodeVolume: true + snapshotDataSource: true + topology: true +InlineVolumes: +- shared: true diff --git a/deploy/kubernetes-1.18/hostpath/csi-hostpath-plugin.yaml b/deploy/kubernetes-1.18/hostpath/csi-hostpath-plugin.yaml index 6496cc55d..4bfd9613d 100644 --- a/deploy/kubernetes-1.18/hostpath/csi-hostpath-plugin.yaml +++ b/deploy/kubernetes-1.18/hostpath/csi-hostpath-plugin.yaml @@ -1,3 +1,191 @@ +# All of the individual sidecar RBAC roles get bound +# to this account. +kind: ServiceAccount +apiVersion: v1 +metadata: + name: csi-hostpathplugin-sa + namespace: default + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: serviceaccount +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: attacher-cluster-role + name: csi-hostpathplugin-attacher-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-attacher-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: health-monitor-controller-cluster-role + name: csi-hostpathplugin-health-monitor-controller-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-health-monitor-controller-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: provisioner-cluster-role + name: csi-hostpathplugin-provisioner-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-provisioner-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: resizer-cluster-role + name: csi-hostpathplugin-resizer-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-resizer-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: snapshotter-cluster-role + name: csi-hostpathplugin-snapshotter-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-snapshotter-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: attacher-role + name: csi-hostpathplugin-attacher-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-attacher-cfg +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: health-monitor-controller-role + name: csi-hostpathplugin-health-monitor-controller-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-health-monitor-controller-cfg +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: provisioner-role + name: csi-hostpathplugin-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: resizer-role + name: csi-hostpathplugin-resizer-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-resizer-cfg +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: snapshotter-role + name: csi-hostpathplugin-snapshotter-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-snapshotter-leaderelection +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa +--- kind: StatefulSet apiVersion: apps/v1 metadata: @@ -11,8 +199,7 @@ spec: serviceName: "csi-hostpathplugin" # One replica only: # Host path driver only works when everything runs - # on a single node. We achieve that by starting it once and then - # co-locate all other pods via inter-pod affinity + # on a single node. replicas: 1 selector: matchLabels: @@ -28,8 +215,51 @@ spec: app.kubernetes.io/name: csi-hostpathplugin app.kubernetes.io/component: plugin spec: - serviceAccountName: csi-external-health-monitor-controller + serviceAccountName: csi-hostpathplugin-sa containers: + - name: hostpath + image: k8s.gcr.io/sig-storage/hostpathplugin:v1.6.2 + args: + - "--drivername=hostpath.csi.k8s.io" + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + ports: + - containerPort: 9898 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins-dir + - mountPath: /csi-data-dir + name: csi-data-dir + - mountPath: /dev + name: dev-dir + - name: csi-external-health-monitor-agent image: k8s.gcr.io/sig-storage/csi-external-health-monitor-agent:v0.2.0 args: @@ -59,6 +289,7 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi + - name: node-driver-registrar image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1 args: @@ -84,48 +315,6 @@ spec: - mountPath: /csi-data-dir name: csi-data-dir - - name: hostpath - image: k8s.gcr.io/sig-storage/hostpathplugin:v1.6.2 - args: - - "--drivername=hostpath.csi.k8s.io" - - "--v=5" - - "--endpoint=$(CSI_ENDPOINT)" - - "--nodeid=$(KUBE_NODE_NAME)" - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - securityContext: - privileged: true - ports: - - containerPort: 9898 - name: healthz - protocol: TCP - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 2 - volumeMounts: - - mountPath: /csi - name: socket-dir - - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - name: mountpoint-dir - - mountPath: /var/lib/kubelet/plugins - mountPropagation: Bidirectional - name: plugins-dir - - mountPath: /csi-data-dir - name: csi-data-dir - - mountPath: /dev - name: dev-dir - name: liveness-probe volumeMounts: - mountPath: /csi @@ -135,6 +324,63 @@ spec: - --csi-address=/csi/csi.sock - --health-port=9898 + - name: csi-attacher + image: k8s.gcr.io/sig-storage/csi-attacher:v3.1.0 + args: + - --v=5 + - --csi-address=/csi/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + + - name: csi-provisioner + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0 + args: + - -v=5 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + + - name: csi-resizer + image: k8s.gcr.io/sig-storage/csi-resizer:v1.1.0 + args: + - -v=5 + - -csi-address=/csi/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + + - name: csi-snapshotter + image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.3 + args: + - -v=5 + - --csi-address=/csi/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: - hostPath: path: /var/lib/kubelet/plugins/csi-hostpath diff --git a/deploy/kubernetes-1.19-test b/deploy/kubernetes-1.19-test new file mode 120000 index 000000000..ebd2bf630 --- /dev/null +++ b/deploy/kubernetes-1.19-test @@ -0,0 +1 @@ +kubernetes-1.18-test \ No newline at end of file diff --git a/deploy/kubernetes-1.20-test/README.md b/deploy/kubernetes-1.20-test/README.md new file mode 100644 index 000000000..be8fde129 --- /dev/null +++ b/deploy/kubernetes-1.20-test/README.md @@ -0,0 +1,7 @@ +The deployment for Kubernetes 1.20 uses the CSI snapshotter sidecar +4.x and thus is incompatible with Kubernetes clusters where older +snapshotter CRDs are installed. + +It uses separate pods and service accounts for each sidecar. This is +not how they would normally be deployed. It gets done this way to test +that the individual RBAC rules are correct. diff --git a/deploy/kubernetes-1.20-test/deploy.sh b/deploy/kubernetes-1.20-test/deploy.sh new file mode 120000 index 000000000..589c43f62 --- /dev/null +++ b/deploy/kubernetes-1.20-test/deploy.sh @@ -0,0 +1 @@ +../util/deploy-hostpath.sh \ No newline at end of file diff --git a/deploy/kubernetes-1.20-test/destroy.sh b/deploy/kubernetes-1.20-test/destroy.sh new file mode 120000 index 000000000..c66894352 --- /dev/null +++ b/deploy/kubernetes-1.20-test/destroy.sh @@ -0,0 +1 @@ +../util/destroy-hostpath.sh \ No newline at end of file diff --git a/deploy/kubernetes-1.20/hostpath/csi-hostpath-attacher.yaml b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-attacher.yaml similarity index 100% rename from deploy/kubernetes-1.20/hostpath/csi-hostpath-attacher.yaml rename to deploy/kubernetes-1.20-test/hostpath/csi-hostpath-attacher.yaml diff --git a/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-driverinfo.yaml b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-driverinfo.yaml new file mode 100644 index 000000000..c8cf666a4 --- /dev/null +++ b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-driverinfo.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: hostpath.csi.k8s.io + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: hostpath.csi.k8s.io + app.kubernetes.io/component: csi-driver +spec: + # Supports persistent and ephemeral inline volumes. + volumeLifecycleModes: + - Persistent + - Ephemeral + # To determine at runtime which mode a volume uses, pod info and its + # "csi.storage.k8s.io/ephemeral" entry are needed. + podInfoOnMount: true diff --git a/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-plugin.yaml b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-plugin.yaml new file mode 100644 index 000000000..50d500c77 --- /dev/null +++ b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-plugin.yaml @@ -0,0 +1,167 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpathplugin + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: plugin +spec: + serviceName: "csi-hostpathplugin" + # One replica only: + # Host path driver only works when everything runs + # on a single node. We achieve that by starting it once and then + # co-locate all other pods via inter-pod affinity + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: plugin + template: + metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: plugin + spec: + serviceAccountName: csi-external-health-monitor-controller + containers: + - name: hostpath + image: k8s.gcr.io/sig-storage/hostpathplugin:v1.6.2 + args: + - "--drivername=hostpath.csi.k8s.io" + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + ports: + - containerPort: 9898 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins-dir + - mountPath: /csi-data-dir + name: csi-data-dir + - mountPath: /dev + name: dev-dir + + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: k8s.gcr.io/sig-storage/livenessprobe:v2.2.0 + args: + - --csi-address=/csi/csi.sock + - --health-port=9898 + + - name: csi-external-health-monitor-agent + image: k8s.gcr.io/sig-storage/csi-external-health-monitor-agent:v0.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + + - name: csi-external-health-monitor-controller + image: k8s.gcr.io/sig-storage/csi-external-health-monitor-controller:v0.2.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + + - name: node-driver-registrar + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1 + args: + - --v=5 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /registration + name: registration-dir + - mountPath: /csi-data-dir + name: csi-data-dir + + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins + type: Directory + name: plugins-dir + - hostPath: + # 'path' is where PV data is persisted on host. + # using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot + path: /var/lib/csi-hostpath-data/ + type: DirectoryOrCreate + name: csi-data-dir + - hostPath: + path: /dev + type: Directory + name: dev-dir diff --git a/deploy/kubernetes-1.20/hostpath/csi-hostpath-provisioner.yaml b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-provisioner.yaml similarity index 100% rename from deploy/kubernetes-1.20/hostpath/csi-hostpath-provisioner.yaml rename to deploy/kubernetes-1.20-test/hostpath/csi-hostpath-provisioner.yaml diff --git a/deploy/kubernetes-1.20/hostpath/csi-hostpath-resizer.yaml b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-resizer.yaml similarity index 100% rename from deploy/kubernetes-1.20/hostpath/csi-hostpath-resizer.yaml rename to deploy/kubernetes-1.20-test/hostpath/csi-hostpath-resizer.yaml diff --git a/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-snapshotclass.yaml b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-snapshotclass.yaml new file mode 100644 index 000000000..0382f2f67 --- /dev/null +++ b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-snapshotclass.yaml @@ -0,0 +1,13 @@ +# Usage of the v1 API implies that the cluster must have +# external-snapshotter v4.x installed. +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: csi-hostpath-snapclass + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-snapclass + app.kubernetes.io/component: volumesnapshotclass +driver: hostpath.csi.k8s.io #csi-hostpath +deletionPolicy: Delete diff --git a/deploy/kubernetes-1.20/hostpath/csi-hostpath-snapshotter.yaml b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-snapshotter.yaml similarity index 100% rename from deploy/kubernetes-1.20/hostpath/csi-hostpath-snapshotter.yaml rename to deploy/kubernetes-1.20-test/hostpath/csi-hostpath-snapshotter.yaml diff --git a/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-testing.yaml b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-testing.yaml new file mode 100644 index 000000000..4e4eb1354 --- /dev/null +++ b/deploy/kubernetes-1.20-test/hostpath/csi-hostpath-testing.yaml @@ -0,0 +1,83 @@ +# WARNING: this is only for testing purposes. Do not install in a production +# cluster. +# +# This exposes the hostpath's Unix domain csi.sock as a TCP port to the +# outside world. The mapping from Unix domain socket to TCP is done +# by socat. +# +# This is useful for testing with csi-sanity or csc. + +apiVersion: v1 +kind: Service +metadata: + name: hostpath-service + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-socat + app.kubernetes.io/component: socat +spec: + type: NodePort + selector: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-socat + app.kubernetes.io/component: socat + ports: + - port: 10000 # fixed port inside the pod, dynamically allocated port outside +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-socat + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-socat + app.kubernetes.io/component: socat +spec: + serviceName: "csi-hostpath-socat" + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-socat + app.kubernetes.io/component: socat + template: + metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpath-socat + app.kubernetes.io/component: socat + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/instance + operator: In + values: + - hostpath.csi.k8s.io + topologyKey: kubernetes.io/hostname + containers: + - name: socat + image: alpine/socat:1.0.3 + args: + - tcp-listen:10000,fork,reuseaddr + - unix-connect:/csi/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/deploy/kubernetes-1.20-test/kubernetes-latest-test b/deploy/kubernetes-1.20-test/kubernetes-latest-test new file mode 120000 index 000000000..f6e4776fe --- /dev/null +++ b/deploy/kubernetes-1.20-test/kubernetes-latest-test @@ -0,0 +1 @@ +kubernetes-latest-test \ No newline at end of file diff --git a/deploy/kubernetes-1.20-test/test-driver.yaml b/deploy/kubernetes-1.20-test/test-driver.yaml new file mode 100644 index 000000000..d523dfa1f --- /dev/null +++ b/deploy/kubernetes-1.20-test/test-driver.yaml @@ -0,0 +1,24 @@ +# This file describes how to test this deployment of the CSI hostpath driver +# using the Kubernetes 1.17 E2E test suite. For details see: +# https://github.com/kubernetes/kubernetes/tree/v1.17.0/test/e2e/storage/external + +StorageClass: + FromName: true +SnapshotClass: + FromName: true +DriverInfo: + Name: hostpath.csi.k8s.io + SupportedSizeRange: + Min: 1Mi + Capabilities: + block: true + controllerExpansion: true + exec: true + multipods: true + nodeExpansion: true + persistence: true + singleNodeVolume: true + snapshotDataSource: true + topology: true +InlineVolumes: +- shared: true diff --git a/deploy/kubernetes-1.20/hostpath/csi-hostpath-plugin.yaml b/deploy/kubernetes-1.20/hostpath/csi-hostpath-plugin.yaml index 6496cc55d..e57925354 100644 --- a/deploy/kubernetes-1.20/hostpath/csi-hostpath-plugin.yaml +++ b/deploy/kubernetes-1.20/hostpath/csi-hostpath-plugin.yaml @@ -1,3 +1,191 @@ +# All of the individual sidecar RBAC roles get bound +# to this account. +kind: ServiceAccount +apiVersion: v1 +metadata: + name: csi-hostpathplugin-sa + namespace: default + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: serviceaccount +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: attacher-cluster-role + name: csi-hostpathplugin-attacher-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-attacher-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: health-monitor-controller-cluster-role + name: csi-hostpathplugin-health-monitor-controller-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-health-monitor-controller-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: provisioner-cluster-role + name: csi-hostpathplugin-provisioner-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-provisioner-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: resizer-cluster-role + name: csi-hostpathplugin-resizer-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-resizer-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: snapshotter-cluster-role + name: csi-hostpathplugin-snapshotter-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-snapshotter-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: attacher-role + name: csi-hostpathplugin-attacher-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-attacher-cfg +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: health-monitor-controller-role + name: csi-hostpathplugin-health-monitor-controller-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-health-monitor-controller-cfg +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: provisioner-role + name: csi-hostpathplugin-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: resizer-role + name: csi-hostpathplugin-resizer-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-resizer-cfg +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: snapshotter-role + name: csi-hostpathplugin-snapshotter-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-snapshotter-leaderelection +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa +--- kind: StatefulSet apiVersion: apps/v1 metadata: @@ -11,8 +199,7 @@ spec: serviceName: "csi-hostpathplugin" # One replica only: # Host path driver only works when everything runs - # on a single node. We achieve that by starting it once and then - # co-locate all other pods via inter-pod affinity + # on a single node. replicas: 1 selector: matchLabels: @@ -28,8 +215,51 @@ spec: app.kubernetes.io/name: csi-hostpathplugin app.kubernetes.io/component: plugin spec: - serviceAccountName: csi-external-health-monitor-controller + serviceAccountName: csi-hostpathplugin-sa containers: + - name: hostpath + image: k8s.gcr.io/sig-storage/hostpathplugin:v1.6.2 + args: + - "--drivername=hostpath.csi.k8s.io" + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + ports: + - containerPort: 9898 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins-dir + - mountPath: /csi-data-dir + name: csi-data-dir + - mountPath: /dev + name: dev-dir + - name: csi-external-health-monitor-agent image: k8s.gcr.io/sig-storage/csi-external-health-monitor-agent:v0.2.0 args: @@ -59,6 +289,7 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi + - name: node-driver-registrar image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1 args: @@ -84,48 +315,6 @@ spec: - mountPath: /csi-data-dir name: csi-data-dir - - name: hostpath - image: k8s.gcr.io/sig-storage/hostpathplugin:v1.6.2 - args: - - "--drivername=hostpath.csi.k8s.io" - - "--v=5" - - "--endpoint=$(CSI_ENDPOINT)" - - "--nodeid=$(KUBE_NODE_NAME)" - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - securityContext: - privileged: true - ports: - - containerPort: 9898 - name: healthz - protocol: TCP - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 2 - volumeMounts: - - mountPath: /csi - name: socket-dir - - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - name: mountpoint-dir - - mountPath: /var/lib/kubelet/plugins - mountPropagation: Bidirectional - name: plugins-dir - - mountPath: /csi-data-dir - name: csi-data-dir - - mountPath: /dev - name: dev-dir - name: liveness-probe volumeMounts: - mountPath: /csi @@ -135,6 +324,63 @@ spec: - --csi-address=/csi/csi.sock - --health-port=9898 + - name: csi-attacher + image: k8s.gcr.io/sig-storage/csi-attacher:v3.1.0 + args: + - --v=5 + - --csi-address=/csi/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + + - name: csi-provisioner + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0 + args: + - -v=5 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + + - name: csi-resizer + image: k8s.gcr.io/sig-storage/csi-resizer:v1.1.0 + args: + - -v=5 + - -csi-address=/csi/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + + - name: csi-snapshotter + image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0 + args: + - -v=5 + - --csi-address=/csi/csi.sock + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: - hostPath: path: /var/lib/kubelet/plugins/csi-hostpath diff --git a/deploy/kubernetes-latest-test b/deploy/kubernetes-latest-test new file mode 120000 index 000000000..99f49b7d4 --- /dev/null +++ b/deploy/kubernetes-latest-test @@ -0,0 +1 @@ +kubernetes-1.20-test \ No newline at end of file diff --git a/deploy/util/deploy-hostpath.sh b/deploy/util/deploy-hostpath.sh index ead296cf7..39be1dc69 100755 --- a/deploy/util/deploy-hostpath.sh +++ b/deploy/util/deploy-hostpath.sh @@ -82,6 +82,12 @@ function rbac_version () { image="$2" update_rbac="$3" + if ! [ -f "$yaml" ]; then + # Fall back to csi-hostpath-plugin.yaml for those deployments which do not + # have individual pods for the sidecars. + yaml="$(dirname "$yaml")/csi-hostpath-plugin.yaml" + fi + # get version from `image: quay.io/k8scsi/csi-attacher:v1.0.1`, ignoring comments version="$(sed -e 's/ *#.*$//' "$yaml" | grep "image:.*$image" | sed -e 's/ *#.*//' -e 's/.*://')" @@ -239,15 +245,30 @@ for i in $(ls ${BASE_DIR}/hostpath/*.yaml | sort); do fi done -# Wait until all pods are running. We have to make some assumptions -# about the deployment here, otherwise we wouldn't know what to wait -# for: the expectation is that we run attacher, provisioner, -# snapshotter, resizer, socat and hostpath plugin in the default namespace. -expected_running_pods=6 +check_statefulset () ( + ready=$(kubectl get "statefulset/$1" -o jsonpath="{.status.readyReplicas}") + if [ "$ready" ] && [ "$ready" -gt 0 ]; then + return 0 + fi + return 1 +) + +check_statefulsets () ( + while [ "$#" -gt 0 ]; do + if ! check_statefulset "$1"; then + return 1 + fi + shift + done + return 0 +) + +# Wait until all StatefulSets of the deployment are ready. +# The assumption is that we use one or more of those. +statefulsets="$(kubectl get statefulsets -l app.kubernetes.io/instance=hostpath.csi.k8s.io -o jsonpath='{range .items[*]}{" "}{.metadata.name}{end}')" cnt=0 -while [ $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) -lt ${expected_running_pods} ]; do +while ! check_statefulsets $statefulsets; do if [ $cnt -gt 30 ]; then - echo "Expecting $expected_running_pods, have $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l)." echo "Deployment:" (set +e; set -x; kubectl describe all,role,clusterrole,rolebinding,clusterrolebinding,serviceaccount,storageclass,csidriver --all-namespaces -l app.kubernetes.io/instance=hostpath.csi.k8s.io) echo @@ -277,5 +298,6 @@ if [ "${CSI_PROW_TEST_DRIVER}" ]; then # doesn't handle the case when the "wrong" node is chosen and gets # stuck permanently with: # error generating accessibility requirements: no topology key found on CSINode csi-prow-worker2 - echo >>"${CSI_PROW_TEST_DRIVER}" "ClientNodeName: $(kubectl get pods/csi-hostpath-provisioner-0 -o jsonpath='{.spec.nodeName}')" + node="$(kubectl get pods/csi-hostpathplugin-0 -o jsonpath='{.spec.nodeName}')" + echo >>"${CSI_PROW_TEST_DRIVER}" "ClientNodeName: $node" fi