diff --git a/deploy/sample/fio.yaml b/deploy/sample/fio.yaml index 041273be1..49f5a72cc 100644 --- a/deploy/sample/fio.yaml +++ b/deploy/sample/fio.yaml @@ -8,6 +8,9 @@ parameters: compression: "on" dedup: "on" thinprovision: "yes" + #encryption: "on" + #keyformat: "raw" + #keylocation: "file:///home/pawan/key" poolname: "zfspv-pool" provisioner: openebs.io/zfs volumeBindingMode: WaitForFirstConsumer diff --git a/deploy/sample/percona.yaml b/deploy/sample/percona.yaml new file mode 100644 index 000000000..104b0eb49 --- /dev/null +++ b/deploy/sample/percona.yaml @@ -0,0 +1,131 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: openebs-zfspv +allowVolumeExpansion: true +parameters: + blocksize: "4k" + compression: "on" + dedup: "on" + thinprovision: "yes" + poolname: "zfspv-pool" +provisioner: openebs.io/zfs +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: csi-zfspv +spec: + storageClassName: openebs-zfspv + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + name: sqltest + namespace: default +data: + sql-test.sh: | + #!/bin/bash + + DB_PREFIX="Inventory" + DB_SUFFIX=`echo $(mktemp) | cut -d '.' -f 2` + DB_NAME="${DB_PREFIX}_${DB_SUFFIX}" + + + echo -e "\nWaiting for mysql server to start accepting connections.." + retries=10;wait_retry=30 + for i in `seq 1 $retries`; do + mysql -uroot -pk8sDem0 -e 'status' > /dev/null 2>&1 + rc=$? + [ $rc -eq 0 ] && break + sleep $wait_retry + done + + if [ $rc -ne 0 ]; + then + echo -e "\nFailed to connect to db server after trying for $(($retries * $wait_retry))s, exiting\n" + exit 1 + fi + mysql -uroot -pk8sDem0 -e "CREATE DATABASE $DB_NAME;" + mysql -uroot -pk8sDem0 -e "CREATE TABLE Hardware (id INTEGER, name VARCHAR(20), owner VARCHAR(20),description VARCHAR(20));" $DB_NAME + mysql -uroot -pk8sDem0 -e "INSERT INTO Hardware (id, name, owner, description) values (1, "dellserver", "basavaraj", "controller");" $DB_NAME + mysql -uroot -pk8sDem0 -e "DROP DATABASE $DB_NAME;" +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: percona + labels: + name: percona +spec: + replicas: 1 + selector: + matchLabels: + name: percona + template: + metadata: + labels: + name: percona + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - gke-pawan-zfspv-default-pool-26f2b9a9-5fqd + containers: + - resources: + name: percona + image: openebs/tests-custom-percona:latest + imagePullPolicy: IfNotPresent + args: + - "--ignore-db-dir" + - "lost+found" + env: + - name: MYSQL_ROOT_PASSWORD + value: k8sDem0 + ports: + - containerPort: 3306 + name: percona + volumeMounts: + - mountPath: /var/lib/mysql + name: demo-vol1 + - mountPath: /sql-test.sh + subPath: sql-test.sh + name: sqltest-configmap + livenessProbe: + exec: + command: ["bash", "sql-test.sh"] + initialDelaySeconds: 30 + periodSeconds: 1 + timeoutSeconds: 10 + volumes: + - name: demo-vol1 + persistentVolumeClaim: + claimName: csi-zfspv + - name: sqltest-configmap + configMap: + name: sqltest + +--- +apiVersion: v1 +kind: Service +metadata: + name: percona-mysql + labels: + name: percona-mysql +spec: + ports: + - port: 3306 + targetPort: 3306 + selector: + name: percona diff --git a/deploy/zfs-operator.yaml b/deploy/zfs-operator.yaml index 86fc5faf1..a86c7cb71 100644 --- a/deploy/zfs-operator.yaml +++ b/deploy/zfs-operator.yaml @@ -396,6 +396,8 @@ spec: mountPath: /plugin - name: device-dir mountPath: /dev + - name: encr-keys + mountPath: /home/keys - name: zfs-bin mountPath: /sbin/zfs - name: libzpool @@ -418,6 +420,10 @@ spec: hostPath: path: /dev type: Directory + - name: encr-keys + hostPath: + path: /home/keys + type: DirectoryOrCreate - name: zfs-bin hostPath: path: /sbin/zfs diff --git a/pkg/apis/openebs.io/core/v1alpha1/zfsvolume.go b/pkg/apis/openebs.io/core/v1alpha1/zfsvolume.go index 337872f42..b70630dd1 100644 --- a/pkg/apis/openebs.io/core/v1alpha1/zfsvolume.go +++ b/pkg/apis/openebs.io/core/v1alpha1/zfsvolume.go @@ -91,9 +91,21 @@ type VolumeInfo struct { Compression string `json:"compression"` // Dedup specifies the deduplication - // should be enabledd on the zvol + // should be enabled on the zvol Dedup string `json:"dedup"` + // Encryption specifies the encryption + // should be enabled on the zvol + Encryption string `json:"encryption"` + + // KeyLocation is the location of key + // for the encryption + KeyLocation string `json:"keylocation"` + + // KeyFormat specifies format of the + // encryption key + KeyFormat string `json:"keyformat"` + // Thinprovision specifies if we should // thin provisioned the volume or not ThinProvision string `json:"thinProvison"` diff --git a/pkg/builder/build.go b/pkg/builder/build.go index 506a2f806..e47151c43 100644 --- a/pkg/builder/build.go +++ b/pkg/builder/build.go @@ -54,7 +54,7 @@ func BuildFrom(volume *apis.ZFSVolume) *Builder { } } -// WithNamespace sets the namespace of csi volume +// WithNamespace sets the namespace of ZFSVolume func (b *Builder) WithNamespace(namespace string) *Builder { if namespace == "" { b.errs = append( @@ -69,7 +69,7 @@ func (b *Builder) WithNamespace(namespace string) *Builder { return b } -// WithName sets the name of csi volume +// WithName sets the name of ZFSVolume func (b *Builder) WithName(name string) *Builder { if name == "" { b.errs = append( @@ -100,42 +100,44 @@ func (b *Builder) WithCapacity(capacity string) *Builder { return b } -// WithCompression sets compression of CStorVolumeClaim -func (b *Builder) WithCompression(compression string) *Builder { +// WithEncryption sets the encryption on ZFSVolume +func (b *Builder) WithEncryption(encr string) *Builder { + b.volume.Object.Spec.Encryption = encr + return b +} - comp := "off" - if compression == "on" { - comp = "on" - } - b.volume.Object.Spec.Compression = comp +// WithKeyLocation sets the encryption key location on ZFSVolume +func (b *Builder) WithKeyLocation(kl string) *Builder { + b.volume.Object.Spec.KeyLocation = kl return b } -// WithDedup sets compression of CStorVolumeClaim -func (b *Builder) WithDedup(dedup string) *Builder { +// WithKeyFormat sets the encryption key format on ZFSVolume +func (b *Builder) WithKeyFormat(kf string) *Builder { + b.volume.Object.Spec.KeyFormat = kf + return b +} - dp := "off" - if dedup == "on" { - dp = "on" - } - b.volume.Object.Spec.Dedup = dp +// WithCompression sets compression of ZFSVolume +func (b *Builder) WithCompression(compression string) *Builder { + b.volume.Object.Spec.Compression = compression return b } -// WithThinProv sets compression of CStorVolumeClaim -func (b *Builder) WithThinProv(thinprov string) *Builder { +// WithDedup sets dedup property of ZFSVolume +func (b *Builder) WithDedup(dedup string) *Builder { + b.volume.Object.Spec.Dedup = dedup + return b +} - tp := "no" - if thinprov == "yes" { - tp = "yes" - } - b.volume.Object.Spec.ThinProvision = tp +// WithThinProv sets if ZFSVolume needs to be thin provisioned +func (b *Builder) WithThinProv(thinprov string) *Builder { + b.volume.Object.Spec.ThinProvision = thinprov return b } -// WithBlockSize sets blocksize of CStorVolumeClaim +// WithBlockSize sets blocksize of ZFSVolume func (b *Builder) WithBlockSize(blockSize string) *Builder { - bs := "4k" if len(blockSize) > 0 { bs = blockSize diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 5f87c26fa..eaa433bed 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -73,6 +73,9 @@ func (cs *controller) CreateVolume( bs := req.GetParameters()["blocksize"] compression := req.GetParameters()["compression"] dedup := req.GetParameters()["dedup"] + encr := req.GetParameters()["encryption"] + kf := req.GetParameters()["keyformat"] + kl := req.GetParameters()["keylocation"] pool := req.GetParameters()["poolname"] tp := req.GetParameters()["thinprovision"] @@ -82,6 +85,9 @@ func (cs *controller) CreateVolume( WithBlockSize(bs). WithPoolName(pool). WithDedup(dedup). + WithEncryption(encr). + WithKeyFormat(kf). + WithKeyLocation(kl). WithThinProv(tp). WithCompression(compression).Build() diff --git a/pkg/zfs/zfs_util.go b/pkg/zfs/zfs_util.go index adff3a030..12f40e8e7 100644 --- a/pkg/zfs/zfs_util.go +++ b/pkg/zfs/zfs_util.go @@ -18,14 +18,18 @@ package zfs import ( "os" + "os/exec" "github.com/Sirupsen/logrus" apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/core/v1alpha1" - "k8s.io/kubernetes/pkg/util/mount" ) const ( - ZFS_DEVPATH = "/dev/zvol/" + ZFS_DEVPATH = "/dev/zvol/" + ZFSVolCmd = "zfs" + ZFSCreateArg = "create" + ZFSDestroyArg = "destroy" + ZFSSetArg = "set" ) func PropertyChanged(oldVol *apis.ZFSVolume, newVol *apis.ZFSVolume) bool { @@ -34,38 +38,102 @@ func PropertyChanged(oldVol *apis.ZFSVolume, newVol *apis.ZFSVolume) bool { oldVol.Spec.Capacity != newVol.Spec.Capacity } +// builldVolumeCreateArgs returns zvol create command along with attributes as a string array +func buildVolumeCreateArgs(vol *apis.ZFSVolume) []string { + var ZFSVolCmd []string + + zvol := vol.Spec.PoolName + "/" + vol.Name + + ZFSVolCmd = append(ZFSVolCmd, ZFSCreateArg) + + if vol.Spec.ThinProvision == "yes" { + ZFSVolCmd = append(ZFSVolCmd, "-s") + } + if len(vol.Spec.Capacity) != 0 { + ZFSVolCmd = append(ZFSVolCmd, "-V", vol.Spec.Capacity) + } + if len(vol.Spec.BlockSize) != 0 { + ZFSVolCmd = append(ZFSVolCmd, "-b", vol.Spec.BlockSize) + } + if len(vol.Spec.Dedup) != 0 { + dedupProperty := "dedup=" + vol.Spec.Dedup + ZFSVolCmd = append(ZFSVolCmd, "-o", dedupProperty) + } + if len(vol.Spec.Compression) != 0 { + compressionProperty := "compression=" + vol.Spec.Compression + ZFSVolCmd = append(ZFSVolCmd, "-o", compressionProperty) + } + if len(vol.Spec.Encryption) != 0 { + encryptionProperty := "encryption=" + vol.Spec.Encryption + ZFSVolCmd = append(ZFSVolCmd, "-o", encryptionProperty) + } + if len(vol.Spec.KeyLocation) != 0 { + keyLocation := "keylocation=" + vol.Spec.KeyLocation + ZFSVolCmd = append(ZFSVolCmd, "-o", keyLocation) + } + if len(vol.Spec.KeyFormat) != 0 { + keyFormat := "keyformat=" + vol.Spec.KeyFormat + ZFSVolCmd = append(ZFSVolCmd, "-o", keyFormat) + } + + ZFSVolCmd = append(ZFSVolCmd, zvol) + + return ZFSVolCmd +} + +// builldVolumeSetArgs returns zvol set command along with attributes as a string array +// TODO(pawan) need to find a way to identify which property has changed +func buildVolumeSetArgs(vol *apis.ZFSVolume) []string { + var ZFSVolCmd []string + + zvol := vol.Spec.PoolName + "/" + vol.Name + + ZFSVolCmd = append(ZFSVolCmd, ZFSSetArg) + + if len(vol.Spec.Capacity) != 0 { + volsize := "volsize=" + vol.Spec.Capacity + ZFSVolCmd = append(ZFSVolCmd, volsize) + } + if len(vol.Spec.Dedup) != 0 { + dedupProperty := "dedup=" + vol.Spec.Dedup + ZFSVolCmd = append(ZFSVolCmd, dedupProperty) + } + if len(vol.Spec.Compression) != 0 { + compressionProperty := "compression=" + vol.Spec.Compression + ZFSVolCmd = append(ZFSVolCmd, compressionProperty) + } + + ZFSVolCmd = append(ZFSVolCmd, zvol) + + return ZFSVolCmd +} + +// builldVolumeDestroyArgs returns zvol destroy command along with attributes as a string array +func buildVolumeDestroyArgs(vol *apis.ZFSVolume) []string { + var ZFSVolCmd []string + + zvol := vol.Spec.PoolName + "/" + vol.Name + + ZFSVolCmd = append(ZFSVolCmd, ZFSDestroyArg, "-R", zvol) + + return ZFSVolCmd +} + // createZvol creates the zvol and returns the corresponding diskPath // of the volume which gets created on the node func createZvol(vol *apis.ZFSVolume) (string, error) { - var out []byte zvol := vol.Spec.PoolName + "/" + vol.Name devicePath := ZFS_DEVPATH + zvol if _, err := os.Stat(devicePath); os.IsNotExist(err) { - if vol.Spec.ThinProvision == "yes" { - out, err = mount.NewOsExec().Run( - "zfs", "create", - "-s", - "-V", vol.Spec.Capacity, - "-b", vol.Spec.BlockSize, - "-o", "compression="+vol.Spec.Compression, - "-o", "dedup="+vol.Spec.Dedup, - zvol, - ) - } else { - out, err = mount.NewOsExec().Run( - "zfs", "create", - "-V", vol.Spec.Capacity, - "-b", vol.Spec.BlockSize, - "-o", "compression="+vol.Spec.Compression, - "-o", "dedup="+vol.Spec.Dedup, - zvol, - ) - } + + args := buildVolumeCreateArgs(vol) + cmd := exec.Command(ZFSVolCmd, args...) + out, err := cmd.CombinedOutput() if err != nil { logrus.Errorf( - "zfs: could not create zvol %v vol %v error: %s", zvol, vol, string(out), + "zfs: could not create zvol %v cmd %v error: %s", zvol, args, string(out), ) return "", err } @@ -81,24 +149,18 @@ func createZvol(vol *apis.ZFSVolume) (string, error) { // SetZvolProp sets the zvol property func SetZvolProp(vol *apis.ZFSVolume) error { - var out []byte var err error zvol := vol.Spec.PoolName + "/" + vol.Name devicePath := ZFS_DEVPATH + zvol if _, err = os.Stat(devicePath); err == nil { - // TODO(pawan) need to find a way to identify - // which property has changed - out, err = mount.NewOsExec().Run( - "zfs", "set", - "volsize="+vol.Spec.Capacity, - "compression="+vol.Spec.Compression, - "dedup="+vol.Spec.Dedup, - zvol, - ) + args := buildVolumeSetArgs(vol) + cmd := exec.Command(ZFSVolCmd, args...) + out, err := cmd.CombinedOutput() + if err != nil { logrus.Errorf( - "zfs: could not set property on zvol %v vol %v error: %s", zvol, vol, string(out), + "zfs: could not set property on zvol %v cmd %v error: %s", zvol, args, string(out), ) return err } @@ -110,19 +172,17 @@ func SetZvolProp(vol *apis.ZFSVolume) error { // DestroyZvol deletes the zvol func DestroyZvol(vol *apis.ZFSVolume) error { - var out []byte zvol := vol.Spec.PoolName + "/" + vol.Name devicePath := ZFS_DEVPATH + zvol if _, err := os.Stat(devicePath); err == nil { - out, err = mount.NewOsExec().Run( - "zfs", "destroy", - "-R", - zvol, - ) + args := buildVolumeDestroyArgs(vol) + cmd := exec.Command(ZFSVolCmd, args...) + out, err := cmd.CombinedOutput() + if err != nil { logrus.Errorf( - "zfs: could not destroy zvol %v vol %v error: %s", zvol, vol, string(out), + "zfs: could not destroy zvol %v cmd %v error: %s", zvol, args, string(out), ) return err }