diff --git a/deploy/kubernetes/with-host-iscsid/leader-election/csi-datera-latest.yaml b/deploy/kubernetes/with-host-iscsid/leader-election/csi-datera-latest.yaml new file mode 100644 index 0000000..7ec901a --- /dev/null +++ b/deploy/kubernetes/with-host-iscsid/leader-election/csi-datera-latest.yaml @@ -0,0 +1,397 @@ +# Datera CSI Driver Deployment Yaml +# +# Documentation on how to write this insanity: +# https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface.md#recommended-mechanism-for-deploying-csi-drivers-on-kubernetes +# +# Pieces +# ------ +# * datera storage class +# CONTROLLER +# * csi-provisioner +# . provisioner service account +# . provisioner cluster role +# . provisioner cluster role binding +# . provisioner service +# . provisioner statefulset +# * csi-attacher +# * csi-snapshotter +# * datera-plugin (controller) +# NODE +# * driver-registrar +# . driver-registrar service account +# . driver-registrar cluster role +# . driver-registrar cluster role binding +# . driver-registrar service? +# . driver-registrar daemonset +# * iscsid +# * datera-plugin (node/identity) +# +# +# ################################# +# StorageClass Supported Parameters +# ################################# +# Name Default +# ------------- ------------ +# replica_count 3 +# placement_mode hybrid +# ip_pool default +# template "" +# round_robin false +# read_iops_max 0 +# write_iops_max 0 +# total_iops_max 0 +# read_bandwidth_max 0 +# write_bandwidth_max 0 +# total_bandwidth_max 0 +# iops_per_gb 0 +# bandwidth_per_gb 0 +# fs_type ext4 +# fs_args -E lazy_itable_init=0,lazy_journal_init=0,nodiscard -F +# delete_on_unmount false +# +# +# Currently datera UDC environment variables are used to initialize the driver +# (This may change in the future) +# +# DAT_MGMT +# DAT_USER +# DAT_PASS +# DAT_TENANT +# DAT_API +# +# These need to be set for both the controller StatefulSet and node DaemonSet + + +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: dat-block-storage + namespace: kube-system + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: dsp.csi.daterainc.io +parameters: + replica_count: "3" + +###### Controller ###### +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-provisioner + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["list", "watch", "create", "update", "delete", "get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner + namespace: kube-system +subjects: + - kind: ServiceAccount + name: csi-provisioner + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-provisioner + apiGroup: rbac.authorization.k8s.io +--- +# needed for StatefulSet +kind: Service +apiVersion: v1 +metadata: + name: csi-provisioner + namespace: kube-system + labels: + app: csi-provisioner +spec: + selector: + app: csi-provisioner + ports: + - name: dummy + port: 12345 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-provisioner + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-provisioner + serviceName: "csi-provisioner" + replicas: 2 + template: + metadata: + labels: + app: csi-provisioner + spec: + serviceAccount: csi-provisioner + hostNetwork: true + containers: + - name: csi-provisioner + imagePullPolicy: Always + image: quay.io/k8scsi/csi-provisioner:v1.0.1 + args: + - "--provisioner=dsp.csi.daterainc.io" + - "--csi-address=$(DAT_SOCKET)" + - "--enable-leader-election=true" + - "--v=5" + env: + - name: DAT_SOCKET + value: unix:///var/lib/csi/dsp.csi.daterainc.io/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/ + - name: csi-attacher + imagePullPolicy: Always + image: quay.io/k8scsi/csi-attacher:v1.0.0 + args: + - "--csi-address=$(DAT_SOCKET)" + - "--leader-election=true" + - "--leader-election-namespace=kube-system" + - "--leader-election-identity=dat-csi-attacher" + - "--v=5" + env: + - name: DAT_SOCKET + # value: unix:///var/lib/csi/dsp.csi.daterainc.io/csi.sock + value: /var/lib/csi/dsp.csi.daterainc.io/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/ + # TODO: Enabled snapshotter in leader-election cases when support merges + #- name: csi-snapshotter + # imagePullPolicy: Always + # image: quay.io/k8scsi/csi-snapshotter:v1.0.1 + # args: + # - "--csi-address=$(DAT_SOCKET)" + # - "--v=5" + # env: + # - name: DAT_SOCKET + # #value: unix:///var/lib/csi/dsp.csi.daterainc.io/csi.sock + # value: /var/lib/csi/dsp.csi.daterainc.io/csi.sock + # imagePullPolicy: "IfNotPresent" + # volumeMounts: + # - name: socket-dir + # mountPath: /var/lib/csi/ + - name: dat-csi-plugin-controller + imagePullPolicy: Always + image: dateraiodev/dat-csi-plugin:latest + command: [ "/bin/sh", "-c", "/bin/dat-csi-plugin 2>&1 | tee /var/log/driver.log" ] + env: + - name: DAT_TYPE + value: conident + - name: DAT_SOCKET + value: unix:///var/lib/csi/dsp.csi.daterainc.io/csi.sock + - name: DAT_MGMT + value: 1.1.1.1 + - name: DAT_USER + value: admin + - name: DAT_PASS + value: password + - name: DAT_TENANT + value: /root + - name: DAT_API + value: "2.2" + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/ + volumes: + - name: socket-dir + emptyDir: {} + +###### NODE/IDENTITY ###### +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-node + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-node + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-node + namespace: kube-system +subjects: + - kind: ServiceAccount + name: csi-node + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-node + apiGroup: rbac.authorization.k8s.io + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-node + template: + metadata: + labels: + app: csi-node + spec: + serviceAccount: csi-node + hostNetwork: true + containers: + - name: node-driver-registrar + imagePullPolicy: Always + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + args: + - "--v=5" + - "--csi-address=$(DAT_SOCKET)" + - "--kubelet-registration-path=$(REG_SOCKET)" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/dsp.csi.daterainc.io /registration/dsp.csi.daterainc.io-reg.sock"] + env: + - name: DAT_SOCKET + #value: unix:///var/lib/csi/dsp.csi.daterainc.io/csi.sock + value: /csi/csi.sock + - name: REG_SOCKET + value: /var/lib/kubelet/plugins/dsp.csi.daterainc.io/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + - name: dat-csi-plugin-node + imagePullPolicy: Always + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: dateraiodev/dat-csi-plugin:latest + # The sleep is to allow the iscsid sidecar to start Running iscsiadm + # -m session initializes /etc/iscsi/initiatorname.iscsi in this + # container + command: [ "/bin/sh", "-c", "sleep 2; iscsiadm -m session; /bin/dat-csi-plugin 2>&1 | tee /var/log/driver.log" ] + env: + - name: DAT_TYPE + value: nodeident + - name: DAT_SOCKET + value: unix:///csi/csi.sock + - name: DAT_MGMT + value: 1.1.1.1 + - name: DAT_USER + value: admin + - name: DAT_PASS + value: password + - name: DAT_TENANT + value: /root + - name: DAT_API + value: "2.2" + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi/ + - name: devices + mountPath: /dev + - name: iscsi-socket + mountPath: /iscsi-socket/iscsi.sock + - name: pods-mount-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/dsp.csi.daterainc.io/ + type: DirectoryOrCreate + - name: iscsi-socket + hostPath: + path: /tmp/csi-iscsi.sock + type: FileOrCreate + - name: devices + hostPath: + path: /dev + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory + diff --git a/deploy/kubernetes/with-host-iscsid/leader-election/csi-datera-secrets-latest.yaml b/deploy/kubernetes/with-host-iscsid/leader-election/csi-datera-secrets-latest.yaml new file mode 100644 index 0000000..2d63a99 --- /dev/null +++ b/deploy/kubernetes/with-host-iscsid/leader-election/csi-datera-secrets-latest.yaml @@ -0,0 +1,408 @@ +# Datera CSI Driver Deployment Yaml +# +# Documentation on how to write this insanity: +# https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface.md#recommended-mechanism-for-deploying-csi-drivers-on-kubernetes +# +# Pieces +# ------ +# * datera storage class +# CONTROLLER +# * csi-provisioner +# . provisioner service account +# . provisioner cluster role +# . provisioner cluster role binding +# . provisioner service +# . provisioner statefulset +# * csi-attacher +# * csi-snapshotter +# * datera-plugin (controller) +# NODE +# * driver-registrar +# . driver-registrar service account +# . driver-registrar cluster role +# . driver-registrar cluster role binding +# . driver-registrar service? +# . driver-registrar daemonset +# * iscsid +# * datera-plugin (node/identity) +# +# +# ################################# +# StorageClass Supported Parameters +# ################################# +# Name Default +# ------------- ------------ +# replica_count 3 +# placement_mode hybrid +# ip_pool default +# template "" +# round_robin false +# read_iops_max 0 +# write_iops_max 0 +# total_iops_max 0 +# read_bandwidth_max 0 +# write_bandwidth_max 0 +# total_bandwidth_max 0 +# iops_per_gb 0 +# bandwidth_per_gb 0 +# fs_type ext4 +# fs_args -E lazy_itable_init=0,lazy_journal_init=0,nodiscard -F +# delete_on_unmount false +# +# +# Currently datera UDC environment variables are used to initialize the driver +# (This may change in the future) +# +# DAT_MGMT +# DAT_USER +# DAT_PASS +# DAT_TENANT +# DAT_API +# +# These need to be set for both the controller StatefulSet and node DaemonSet + + +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: dat-block-storage + namespace: kube-system + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: dsp.csi.daterainc.io +parameters: + replica_count: "3" + +###### Controller ###### +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-provisioner + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["list", "watch", "create", "update", "delete", "get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner + namespace: kube-system +subjects: + - kind: ServiceAccount + name: csi-provisioner + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-provisioner + apiGroup: rbac.authorization.k8s.io +--- +# needed for StatefulSet +kind: Service +apiVersion: v1 +metadata: + name: csi-provisioner + namespace: kube-system + labels: + app: csi-provisioner +spec: + selector: + app: csi-provisioner + ports: + - name: dummy + port: 12345 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-provisioner + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-provisioner + serviceName: "csi-provisioner" + replicas: 2 + template: + metadata: + labels: + app: csi-provisioner + spec: + serviceAccount: csi-provisioner + hostNetwork: true + containers: + - name: csi-provisioner + imagePullPolicy: Always + image: quay.io/k8scsi/csi-provisioner:v1.0.1 + args: + - "--provisioner=dsp.csi.daterainc.io" + - "--csi-address=$(DAT_SOCKET)" + - "--enable-leader-election=true" + - "--v=5" + env: + - name: DAT_SOCKET + value: unix:///var/lib/csi/dsp.csi.daterainc.io/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/ + - name: csi-attacher + imagePullPolicy: Always + image: quay.io/k8scsi/csi-attacher:v1.0.0 + args: + - "--csi-address=$(DAT_SOCKET)" + - "--leader-election=true" + - "--leader-election-namespace=kube-system" + - "--leader-election-identity=dat-csi-attacher" + - "--v=5" + env: + - name: DAT_SOCKET + # value: unix:///var/lib/csi/dsp.csi.daterainc.io/csi.sock + value: /var/lib/csi/dsp.csi.daterainc.io/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/ + # TODO: Enabled snapshotter in leader-election cases when support merges + #- name: csi-snapshotter + # imagePullPolicy: Always + # image: quay.io/k8scsi/csi-snapshotter:v1.0.1 + # args: + # - "--csi-address=$(DAT_SOCKET)" + # - "--v=5" + # env: + # - name: DAT_SOCKET + # #value: unix:///var/lib/csi/dsp.csi.daterainc.io/csi.sock + # value: /var/lib/csi/dsp.csi.daterainc.io/csi.sock + # imagePullPolicy: "IfNotPresent" + # volumeMounts: + # - name: socket-dir + # mountPath: /var/lib/csi/ + - name: dat-csi-plugin-controller + imagePullPolicy: Always + image: dateraiodev/dat-csi-plugin:latest + command: [ "/bin/sh", "-c", "/bin/dat-csi-plugin 2>&1 | tee /var/log/driver.log" ] + env: + - name: DAT_TYPE + value: conident + - name: DAT_SOCKET + value: unix:///var/lib/csi/dsp.csi.daterainc.io/csi.sock + - name: DAT_MGMT + value: 1.1.1.1 + - name: DAT_USER + valueFrom: + secretKeyRef: + name: datera-secret + key: username + - name: DAT_PASS + valueFrom: + secretKeyRef: + name: datera-secret + key: password + - name: DAT_TENANT + value: /root + - name: DAT_API + value: "2.2" + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/ + volumes: + - name: socket-dir + emptyDir: {} + +###### NODE/IDENTITY ###### +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-node + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-node + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-node + namespace: kube-system +subjects: + - kind: ServiceAccount + name: csi-node + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-node + apiGroup: rbac.authorization.k8s.io + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-node + template: + metadata: + labels: + app: csi-node + spec: + serviceAccount: csi-node + hostNetwork: true + containers: + - name: node-driver-registrar + imagePullPolicy: Always + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + args: + - "--v=5" + - "--csi-address=$(DAT_SOCKET)" + - "--kubelet-registration-path=$(REG_SOCKET)" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/dsp.csi.daterainc.io /registration/dsp.csi.daterainc.io-reg.sock"] + env: + - name: DAT_SOCKET + #value: unix:///var/lib/csi/dsp.csi.daterainc.io/csi.sock + value: /csi/csi.sock + - name: REG_SOCKET + value: /var/lib/kubelet/plugins/dsp.csi.daterainc.io/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + - name: dat-csi-plugin-node + imagePullPolicy: Always + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: dateraiodev/dat-csi-plugin:latest + # The sleep is to allow the iscsid sidecar to start Running iscsiadm + # -m session initializes /etc/iscsi/initiatorname.iscsi in this + # container + command: [ "/bin/sh", "-c", "sleep 2; iscsiadm -m session; /bin/dat-csi-plugin 2>&1 | tee /var/log/driver.log" ] + env: + - name: DAT_TYPE + value: nodeident + - name: DAT_SOCKET + value: unix:///csi/csi.sock + - name: DAT_MGMT + value: 1.1.1.1 + - name: DAT_USER + valueFrom: + secretKeyRef: + name: datera-secret + key: username + - name: DAT_PASS + valueFrom: + secretKeyRef: + name: datera-secret + key: password + - name: DAT_TENANT + value: /root + - name: DAT_API + value: "2.2" + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi/ + - name: devices + mountPath: /dev + - name: iscsi-socket + mountPath: /iscsi-socket/iscsi.sock + - name: pods-mount-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/dsp.csi.daterainc.io/ + type: DirectoryOrCreate + - name: iscsi-socket + hostPath: + path: /tmp/csi-iscsi.sock + type: FileOrCreate + - name: devices + hostPath: + path: /dev + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory