cephfs CSI plugin

.gitignore

@@ -4,7 +4,8 @@
 /_output
 
 # docker build
-/deploy/docker/rbdplugin
+/deploy/rbd/docker/rbdplugin
+/deploy/cephfs/docker/cephfsplugin
 
 # rbdplugin executable
 rbdplugin

Makefile

@@ -14,10 +14,13 @@
 
 .PHONY: all rbdplugin
 
-IMAGE_NAME=quay.io/cephcsi/rbdplugin
-IMAGE_VERSION=v0.2.0
+RBD_IMAGE_NAME=quay.io/cephcsi/rbdplugin
+RBD_IMAGE_VERSION=v0.2.0
 
-all: rbdplugin
+CEPHFS_IMAGE_NAME=quay.io/cephcsi/cephfsplugin
+CEPHFS_IMAGE_VERSION=v0.2.0
+
+all: rbdplugin cephfsplugin
 
 test:
 	go test github.com/ceph/ceph-csi/pkg/... -cover
@@ -27,11 +30,25 @@ rbdplugin:
 	if [ ! -d ./vendor ]; then dep ensure; fi
 	CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o  _output/rbdplugin ./rbd
 
-container: rbdplugin 
-	docker build -t $(IMAGE_NAME):$(IMAGE_VERSION) .
+rbdplugin-container: rbdplugin 
+	cp _output/rbdplugin  deploy/rbd/docker
+	docker build -t $(IMAGE_NAME):$(IMAGE_VERSION) deploy/rbd/docker
+
+cephfsplugin:
+	if [ ! -d ./vendor ]; then dep ensure; fi
+	CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o  _output/cephfsplugin ./cephfs
+
+cephfsplugin-container: cephfsplugin 
+	cp _output/cephfsplugin deploy/cephfs/docker
+	docker build -t $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION) deploy/cephfs/docker
+
+push-rbdplugin-container: rbdplugin-container
+	docker push $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION)
+
+push-cephfsplugin-container: cephfsplugin-container
+	docker push $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION)
 
-push-container: container
-	docker push $(IMAGE_NAME):$(IMAGE_VERSION)
 clean:
 	go clean -r -x
-	-rm -rf _output
+	rm -f deploy/rbd/docker/rbdplugin
+	rm -f deploy/cephfs/docker/rbdplugin

README.md

@@ -2,21 +2,22 @@
 
 ## Overview
 
-RBD CSI plugin implements an interface between  CSI enabled Container
+Ceph CSI plugins implement an interface between CSI enabled Container
 Orchestrator and CEPH cluster. It allows dynamically provision CEPH
 volumes and attach it to workloads.
-Current implementation of CSI RBD plugin was tested in Kubernetes environment,
-but its code does not rely on any Kubernetes specific calls (WIP to make it k8s agnostic)
+Current implementation of Ceph CSI plugins was tested in Kubernetes environment (requires Kubernetes 1.10+),
+but the code does not rely on any Kubernetes specific calls (WIP to make it k8s agnostic)
 and should be able to run with any CSI enabled CO (Containers Orchestration).
 
+[Container Storage Interface (CSI)](https://github.com/container-storage-interface/) driver, provisioner, and attacher for Ceph RBD and CephFS
+
+## RBD Plugin
+
 An RBD CSI plugin is available to help simplify storage management.
 Once user creates PVC with the reference to a RBD storage class, rbd image and 
 corresponding PV object gets dynamically created and becomes ready to be used by
 workloads.
 
-[Container Storage Interface (CSI)](https://github.com/container-storage-interface/) driver, provisioner, and attacher for Ceph RBD and CephFS
-
-## RBD Plugin
 ### Configuration Requirements
 
 * Secret object with the authentication key for ceph cluster
@@ -44,7 +45,7 @@ $ make rbdplugin
 
 To build a container:
 ```
-$ make container
+$ make container-rbdplugin
 ```
 By running:
 ```
@@ -73,29 +74,29 @@ Enable features `MountPropagation=true,CSIPersistentVolume=true` and runtime con
 
 #### Step 1: Create Secret 
 ```
-$ kubectl create -f ./deploy/kubernetes/rbd-secrets.yaml 
+$ kubectl create -f ./deploy/rbd/kubernetes/rbd-secrets.yaml 
 ```
 **Important:** rbd-secrets.yaml, must be customized to match your ceph environment.
 
 #### Step 2: Create StorageClass
 ```
-$ kubectl create -f ./deploy/kubernetes/rbd-storage-class.yaml
+$ kubectl create -f ./deploy/rbd/kubernetes/rbd-storage-class.yaml
 ```
 **Important:** rbd-storage-class.yaml, must be customized to match your ceph environment.
 
 #### Step 3: Start CSI CEPH RBD plugin
 ```
-$ kubectl create -f ./deploy/kubernetes/rbdplugin.yaml
+$ kubectl create -f ./deploy/rbd/kubernetes/rbdplugin.yaml
 ```
 
 #### Step 4: Start CSI External Attacher
 ```
-$ kubectl create -f ./deploy/kubernetes/csi-attacher.yaml
+$ kubectl create -f ./deploy/rbd/kubernetes/csi-attacher.yaml
 ```
 
 #### Step 5: Start CSI External Provisioner  
 ```
-$ kubectl create -f ./deploy/kubernetes/csi-provisioner.yaml
+$ kubectl create -f ./deploy/rbd/kubernetes/csi-provisioner.yaml
 ```
 **Important:** Deployment yaml files includes required Service Account definitions and
 required RBAC rules.
@@ -116,7 +117,7 @@ default       csi-provisioner-0                                   1/1       Runn
 
 #### Step 7: Create PVC 
 ```
-$ kubectl create -f ./deploy/kubernetes/pvc.yaml
+$ kubectl create -f ./deploy/rbd/kubernetes/pvc.yaml
 ```
 
 #### Step 8: Check status of provisioner PV  
@@ -152,12 +153,155 @@ Source:
 #### Step 9: Create a test pod
 
 ```bash
-# kubectl create -f ./deploy/pod.yaml
+# kubectl create -f ./deploy/rbd/pod.yaml
 ```
 
 ## CephFS plugin
 
-TODO 
+A CephFS CSI plugin is available to help simplify storage management.
+Once user creates PVC with the reference to a CephFS CSI storage class, corresponding
+PV object gets dynamically created and becomes ready to be used by workloads.
+
+### Configuration Requirements
+
+* Secret object with the authentication user ID `userID` and key `userKey` for ceph cluster
+* StorageClass with csi-cephfsplugin (default CSI CephFS plugin name) as a provisioner name
+  and information about ceph cluster (monitors, pool, rootPath, ...)
+* Service Accounts with required RBAC permissions
+
+Mounter options: specifies whether to use FUSE or ceph kernel client for mounting. By default, the plugin will probe for `ceph-fuse`. If this fails, the kernel client will be used instead. Command line argument `--volumemounter=[fuse|kernel]` overrides this behaviour.
+
+StorageClass options:
+* `provisionVolume: "bool"`: if set to true, the plugin will provision and mount a new volume. Admin credentials `adminID` and `adminKey` are required in the secret object, since this also creates a dedicated RADOS user used for mounting the volume.
+* `rootPath: /path-in-cephfs`: required field if `provisionVolume=true`. CephFS is mounted from the specified path. User credentials `userID` and `userKey` are required in the secret object.
+* `mounter: "kernel" or "fuse"`: (optional) per-StorageClass mounter configuration. Overrides the default mounter.
+
+### Feature Status
+
+### 1.10: Alpha
+
+**Important:** `CSIPersistentVolume` and `MountPropagation`
+[feature gates must be enabled starting in 1.9](#enabling-the-alpha-feature-gates).
+Also API server must run with running config set to: `storage.k8s.io/v1alpha1`
+
+* `kube-apiserver` must be launched with `--feature-gates=CSIPersistentVolume=true,MountPropagation=true`
+  and `--runtime-config=storage.k8s.io/v1alpha1=true`
+* `kube-controller-manager` must be launched with `--feature-gates=CSIPersistentVolume=true`
+* `kubelet` must be launched with `--feature-gates=CSIPersistentVolume=true,MountPropagation=true`
+
+### Compiling
+CSI CephFS plugin can be compiled in a form of a binary file or in a form of a container. When compiled
+as a binary file, it gets stored in \_output folder with the name cephfsplugin. When compiled as a container,
+the resulting image is stored in a local docker's image store. 
+
+To compile just a binary file:
+```
+$ make cephfsplugin
+```
+
+To build a container:
+```
+$ make cephfsplugin-container
+```
+By running:
+```
+$ docker images | grep cephfsplugin
+```
+You should see the following line in the output:
+```
+quay.io/cephcsi/cephfsplugin              v0.2.0                            79482e644593        4 minutes ago       305MB
+```
+
+### Testing
+
+#### Prerequisite
+
+##### Enable Mount Propagation in Docker 
+
+Comment out `MountFlags=slave` in docker systemd service then restart docker service.
+```
+# systemctl daemon-reload
+# systemctl restart docker
+```
+
+##### Enable Kubernetes Feature Gates
+
+Enable features `MountPropagation=true,CSIPersistentVolume=true` and runtime config `storage.k8s.io/v1alpha1=true`
+
+#### Step 1: Create Secret 
+```
+$ kubectl create -f ./deploy/cephfs/kubernetes/secret.yaml
+```
+**Important:** secret.yaml, must be customized to match your ceph environment.
+
+#### Step 2: Create StorageClass
+```
+$ kubectl create -f ./deploy/cephfs/kubernetes/cephfs-storage-class.yaml
+```
+**Important:** cephfs-storage-class.yaml, must be customized to match your ceph environment.
+
+#### Step 3: Start CSI CEPH CephFS plugin
+```
+$ kubectl create -f ./deploy/cephfs/kubernetes/cephfsplugin.yaml
+```
+
+#### Step 4: Start CSI External Attacher
+```
+$ kubectl create -f ./deploy/cephfs/kubernetes/csi-attacher.yaml
+```
+
+#### Step 5: Start CSI External Provisioner  
+```
+$ kubectl create -f ./deploy/cephfs/kubernetes/csi-provisioner.yaml
+```
+**Important:** Deployment yaml files includes required Service Account definitions and
+required RBAC rules.
+
+#### Step 6: Check status of CSI CephFS plugin  
+```
+$ kubectl get pods | grep csi 
+csi-attacher-0           1/1       Running   0          6m
+csi-cephfsplugin-hmqpk   2/2       Running   0          6m
+csi-provisioner-0        1/1       Running   0          6m
+```
+
+#### Step 7: Create PVC 
+```
+$ kubectl create -f ./deploy/cephfs/kubernetes/pvc.yaml
+```
+
+#### Step 8: Check status of provisioner PV  
+```
+$ kubectl get pv
+NAME                                     CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS    CLAIM                    STORAGECLASS   REASON    AGE
+kubernetes-dynamic-pv-715cef0b30d811e8   5Gi        RWX            Delete           Bound     default/csi-cephfs-pvc   csi-cephfs               5s
+```
+
+```
+$ kubectl describe pv kubernetes-dynamic-pv-715cef0b30d811e8
+Name:            kubernetes-dynamic-pv-715cef0b30d811e8
+Labels:          <none>
+Annotations:     pv.kubernetes.io/provisioned-by=csi-cephfsplugin
+StorageClass:    csi-cephfs
+Status:          Bound
+Claim:           default/csi-cephfs-pvc
+Reclaim Policy:  Delete
+Access Modes:    RWX
+Capacity:        5Gi
+Message:         
+Source:
+    Type:    CSI (a Container Storage Interface (CSI) volume source)
+    Driver:      ReadOnly:  %v
+
+    VolumeHandle:                                                                    csi-cephfsplugin
+%!(EXTRA string=csi-cephfs-7182b779-30d8-11e8-bf01-5254007d7491, bool=false)Events:  <none>
+```
+
+#### Step 9: Create a test pod
+
+```
+$ kubectl create -f ./deploy/cephfs/kubernetes/pod.yaml
+```
 
 ## Troubleshooting
 

cephfs/main.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"os"
+	"path"
+
+	// "github.com/ceph/ceph-csi/pkg/cephfs"
+	"github.com/gman0/ceph-csi/pkg/cephfs"
+	"github.com/golang/glog"
+)
+
+func init() {
+	flag.Set("logtostderr", "true")
+}
+
+var (
+	endpoint      = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
+	driverName    = flag.String("drivername", "csi-cephfsplugin", "name of the driver")
+	nodeId        = flag.String("nodeid", "", "node id")
+	volumeMounter = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := createPersistentStorage(path.Join(cephfs.PluginFolder, "controller")); err != nil {
+		glog.Errorf("failed to create persistent storage for controller: %v", err)
+		os.Exit(1)
+	}
+
+	if err := createPersistentStorage(path.Join(cephfs.PluginFolder, "node")); err != nil {
+		glog.Errorf("failed to create persistent storage for node: %v", err)
+		os.Exit(1)
+	}
+
+	driver := cephfs.NewCephFSDriver()
+	driver.Run(*driverName, *nodeId, *endpoint, *volumeMounter)
+
+	os.Exit(0)
+}
+
+func createPersistentStorage(persistentStoragePath string) error {
+	return os.MkdirAll(persistentStoragePath, os.FileMode(0755))
+}

deploy.sh → deploy-cephfs.sh

@@ -2,5 +2,5 @@
 
 if [ "${TRAVIS_BRANCH}" == "master" ] && [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
     docker login -u "${QUAY_IO_USERNAME}" -p "${QUAY_IO_PASSWORD}" quay.io 
-    make push-container
+    make push-cephfs-container
 fi

deploy-rbd.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+if [ "${TRAVIS_BRANCH}" == "master" ] && [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
+    docker login -u "${QUAY_IO_USERNAME}" -p "${QUAY_IO_PASSWORD}" quay.io 
+    make push-rbdplugin-container
+fi

deploy/cephfs/docker/Dockerfile

@@ -0,0 +1,18 @@
+FROM ubuntu:16.04
+LABEL maintainers="Kubernetes Authors"
+LABEL description="CephFS CSI Plugin"
+
+ENV CEPH_VERSION "luminous"
+
+RUN apt-get update && apt-get install -y wget && \
+    wget -q -O- 'https://download.ceph.com/keys/release.asc' | apt-key add - && \
+    echo "deb http://download.ceph.com/debian-$CEPH_VERSION/ xenial main" | tee /etc/apt/sources.list.d/ceph-$CEPH_VERSION.list && \
+    apt-get update && apt-get install -y kmod ceph-common ceph-fuse attr --no-install-recommends && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY cephfsplugin /cephfsplugin
+
+RUN chmod +x /cephfsplugin && \
+    mkdir -p /var/log/ceph
+
+ENTRYPOINT ["/cephfsplugin"]