rwx: implement test_encrypted_rwx_volume

Longhorn 7045

Signed-off-by: Derek Su <derek.su@suse.com>

manager/integration/tests/common.py

@@ -1536,31 +1536,33 @@ def finalizer():
 
 @pytest.fixture
 def crypto_secret(request):
-    manifest = {
-        'apiVersion': 'v1',
-        'kind': 'Secret',
-        'metadata': {
-            'name': 'longhorn-crypto',
-            'namespace': 'longhorn-system',
-        },
-        'stringData': {
-            'CRYPTO_KEY_VALUE': 'simple',
-            'CRYPTO_KEY_PROVIDER': 'secret'
+    def get_crypto_secret(namespace=LONGHORN_NAMESPACE):
+        crypto_secret.manifest = {
+            'apiVersion': 'v1',
+            'kind': 'Secret',
+            'metadata': {
+                'name': 'longhorn-crypto',
+                'namespace': namespace,
+            },
+            'stringData': {
+                'CRYPTO_KEY_VALUE': 'simple',
+                'CRYPTO_KEY_PROVIDER': 'secret'
+            }
         }
-    }
+        return crypto_secret.manifest
 
     def finalizer():
         api = get_core_api_client()
         try:
             api.delete_namespaced_secret(
-                name=manifest['metadata']['name'],
-                namespace=manifest['metadata']['namespace'])
+                name=crypto_secret.manifest['metadata']['name'],
+                namespace=crypto_secret.manifest['metadata']['namespace'])
         except ApiException as e:
             assert e.status == 404
 
     request.addfinalizer(finalizer)
 
-    return manifest
+    return get_crypto_secret
 
 
 @pytest.fixture
@@ -3828,9 +3830,9 @@ def wait_statefulset(statefulset_manifest):
     assert s_set.status.ready_replicas == replicas
 
 
-def create_crypto_secret(secret_manifest):
+def create_crypto_secret(secret_manifest, namespace=LONGHORN_NAMESPACE):
     api = get_core_api_client()
-    api.create_namespaced_secret(namespace=LONGHORN_NAMESPACE,
+    api.create_namespaced_secret(namespace,
                                  body=secret_manifest)
 
 

manager/integration/tests/test_csi.py

@@ -280,7 +280,8 @@ def test_csi_encrypted_block_volume(client, core_api, storage_class, crypto_secr
     7. Validate the data in `pod2` is consistent with `test_data`
     """
 
-    create_crypto_secret(crypto_secret)
+    secret = crypto_secret(LONGHORN_NAMESPACE)
+    create_crypto_secret(secret)
 
     storage_class['reclaimPolicy'] = 'Retain'
     storage_class['parameters']['csi.storage.k8s.io/provisioner-secret-name'] = 'longhorn-crypto'  # NOQA

manager/integration/tests/test_rwx.py

@@ -6,6 +6,8 @@
 from common import create_and_wait_pod, read_volume_data
 from common import get_apps_api_client, wait_statefulset
 from common import create_and_wait_deployment, delete_and_wait_pod
+from common import delete_and_wait_deployment
+from common import delete_and_wait_pvc
 from common import prepare_pod_with_data_in_mb, DATA_SIZE_IN_MB_1
 from common import create_snapshot, wait_for_backup_completion
 from common import find_backup, Gi, volume_name, csi_pv, pod_make  # NOQA
@@ -20,6 +22,8 @@
 from common import EXPANDED_VOLUME_SIZE
 from common import expand_and_wait_for_pvc, wait_for_volume_expansion
 from common import wait_deployment_replica_ready, wait_for_volume_healthy
+from common import crypto_secret, storage_class  # NOQA
+from common import create_crypto_secret, create_storage_class
 from backupstore import set_random_backupstore # NOQA
 from multiprocessing import Pool
 
@@ -637,3 +641,48 @@ def test_rwx_offline_expansion(client, core_api, pvc, make_deployment_with_pvc):
                                            pod_name,
                                            'default')
     assert int(data_size_in_pod)/1024/1024 == data_size_in_mb
+
+
+def test_encrypted_rwx_volume(core_api, statefulset, storage_class, crypto_secret, pvc, make_deployment_with_pvc):  # NOQA
+    """
+    Test creating encrypted rwx volume and use the secret in
+    non longhorn-system namespace.
+
+    1. Create crypto secret in non longhorn-system namespace.
+    2. Create a storage class.
+    3. Create a deployment with a PVC and the pods should be able to running.
+    """
+
+    namespace = 'default'
+    # Create crypto secret
+    secret = crypto_secret(namespace)
+    create_crypto_secret(secret, namespace)
+
+    # Create storage class
+    storage_class['reclaimPolicy'] = 'Delete'
+    storage_class['parameters']['csi.storage.k8s.io/provisioner-secret-name'] = 'longhorn-crypto'  # NOQA
+    storage_class['parameters']['csi.storage.k8s.io/provisioner-secret-namespace'] = namespace  # NOQA
+    storage_class['parameters']['csi.storage.k8s.io/node-publish-secret-name'] = 'longhorn-crypto'  # NOQA
+    storage_class['parameters']['csi.storage.k8s.io/node-publish-secret-namespace'] = namespace  # NOQA
+    storage_class['parameters']['csi.storage.k8s.io/node-stage-secret-name'] = 'longhorn-crypto'  # NOQA
+    storage_class['parameters']['csi.storage.k8s.io/node-stage-secret-namespace'] = namespace  # NOQA
+    create_storage_class(storage_class)
+
+    # Create deployment with PVC
+    pvc_name = 'pvc-deployment-with-encrypted-rwx-volume'
+    pvc['metadata']['name'] = pvc_name
+    pvc['spec']['storageClassName'] = storage_class['metadata']['name']
+    pvc['spec']['accessModes'] = ['ReadWriteMany']
+
+    core_api.create_namespaced_persistent_volume_claim(
+        body=pvc, namespace='default')
+
+    deployment = make_deployment_with_pvc(
+        'pvc-deployment-with-encrypted-rwx-volume', pvc_name, replicas=3)
+
+    apps_api = get_apps_api_client()
+    create_and_wait_deployment(apps_api, deployment)
+
+    # Clean up deployment and volume
+    delete_and_wait_deployment(apps_api, deployment["metadata"]["name"])
+    delete_and_wait_pvc(core_api, pvc_name)