test(negative): add fs stress operations

Signed-off-by: Yang Chiu <yang.chiu@suse.com>
longhorn · Apr 17, 2024 · 73d37e7 · 73d37e7
1 parent 1d42c1c
commit 73d37e7
Show file tree

Hide file tree

Showing 10 changed files with 150 additions and 10 deletions.
diff --git a/e2e/keywords/stress.resource b/e2e/keywords/stress.resource
@@ -1,7 +1,10 @@
 *** Settings ***
 Documentation       Stress Node Keywords
 
+Library             ../libs/keywords/common_keywords.py
 Library             ../libs/keywords/stress_keywords.py
+Library             ../libs/keywords/workload_keywords.py
+Library             ../libs/keywords/volume_keywords.py
 
 *** Keywords ***
 Stress CPU of all ${role} nodes
@@ -23,3 +26,18 @@ Stress memory of node with volume ${volume_id}
 
 Stress memory of volume nodes
     stress_node_memory_of_all_volumes
+
+Stress filesystem of volume ${volume_id} volume node
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${node_name} =    get_volume_node    ${volume_name}
+    stress_nodes_filesystem    ${node_name}
+
+Stress filesystem of statefulset ${statefulset_id} volume node
+    ${workload_name} =   generate_name_with_suffix    statefulset    ${statefulset_id}
+    ${volume_name} =    get_workload_volume_name    ${workload_name}
+    ${node_name} =    get_volume_node    ${volume_name}
+    stress_nodes_filesystem   ${node_name}
+
+Stress filesystem of all worker nodes
+    ${worker_nodes} =    get_worker_nodes
+    stress_nodes_filesystem   ${worker_nodes}
diff --git a/e2e/libs/keywords/common_keywords.py b/e2e/libs/keywords/common_keywords.py
@@ -1,5 +1,5 @@
 from node_exec import NodeExec
-
+from node import Node
 from utility.utility import init_k8s_api_client
 from utility.utility import generate_name_with_suffix
 
@@ -21,3 +21,6 @@ def cleanup_node_exec(self):
 
     def generate_name_with_suffix(self, kind, suffix):
         return generate_name_with_suffix(kind, suffix)
+
+    def get_worker_nodes(self):
+        return Node().list_node_names_by_role("worker")
diff --git a/e2e/libs/keywords/stress_keywords.py b/e2e/libs/keywords/stress_keywords.py
@@ -54,3 +54,9 @@ def stress_node_memory_of_all_volumes(self):
 
         logging(f'Stressing node memory for all volumes {volume_names}')
         self.stress_node_memory_by_volumes(volume_names)
+
+    def stress_nodes_filesystem(self, nodes):
+        logging(f'Stressing {nodes} nodes filesystem')
+        if isinstance(nodes, str):
+            nodes = [nodes]
+        self.stress.filesystem(nodes)
diff --git a/e2e/libs/node/constant.py b/e2e/libs/node/constant.py
@@ -1,6 +1,8 @@
 NODE_STRESS_CPU_LOAD_PERCENTAGE = 100
 NODE_STRESS_MEM_LOAD_PERCENTAGE = 100
 NODE_STRESS_MEM_VM_WORKERS = 1
+NODE_STRESS_FILESYSTEM_HDD_WORKERS = 8
+NODE_STRESS_FILESYSTEM_LOAD_PERCENTAGE = 80
 NODE_STRESS_TIMEOUT_SECOND = 60 * 60  # 1 hour
 
 STRESS_HELPER_LABEL = "longhorn-stress-helper"

diff --git a/e2e/libs/node/stress.py b/e2e/libs/node/stress.py
@@ -1,9 +1,10 @@
 from kubernetes.client.rest import ApiException
-
 from node import Node
 from node.constant import NODE_STRESS_CPU_LOAD_PERCENTAGE
 from node.constant import NODE_STRESS_MEM_LOAD_PERCENTAGE
 from node.constant import NODE_STRESS_MEM_VM_WORKERS
+from node.constant import NODE_STRESS_FILESYSTEM_HDD_WORKERS
+from node.constant import NODE_STRESS_FILESYSTEM_LOAD_PERCENTAGE
 from node.constant import NODE_STRESS_TIMEOUT_SECOND
 from node.constant import STRESS_HELPER_LABEL
 from node.constant import STRESS_HELPER_POD_NAME_PREFIX
@@ -91,3 +92,34 @@ def memory(self, node_names):
             pod_name = manifest['metadata']['name']
             logging(f"Creating memory stress pod {pod_name} on {node_name}")
             create_pod(manifest, is_wait_for_pod_running=True)
+
+    def filesystem(self, node_names):
+        for node_name in node_names:
+            pod_name = f"{STRESS_HELPER_POD_NAME_PREFIX}{node_name}"
+
+            # If the helper pod creation is called inside of a test case loop,
+            # we need to check if the pod already running.
+            try:
+                pod = get_pod(pod_name)
+                if pod and pod.status.phase != "Running":
+                    logging(f"Deleting stress pod {pod_name} in phase {pod.status.phase}")
+                    delete_pod(pod_name)
+                elif pod:
+                    logging(f"Stress pod {pod_name} already running")
+                    continue
+            except ApiException as e:
+                assert e.status == 404
+
+            manifest = new_pod_manifest(
+                pod_name=pod_name,
+                image=IMAGE_LITMUX,
+                command=["stress-ng"],
+                args=['--hdd', str(NODE_STRESS_FILESYSTEM_HDD_WORKERS),
+					  '--hdd-bytes', f"{NODE_STRESS_FILESYSTEM_LOAD_PERCENTAGE}%"],
+                node_name=node_name,
+                labels={'app': STRESS_HELPER_LABEL}
+            )
+
+            pod_name = manifest['metadata']['name']
+            logging(f"Creating filesystem stress pod {pod_name} on {node_name}")
+            create_pod(manifest, is_wait_for_pod_running=True)
diff --git a/e2e/libs/workload/pod.py b/e2e/libs/workload/pod.py
@@ -143,11 +143,13 @@ def wait_for_pod_status(name, status, namespace='default'):
     for i in range(retry_count):
         pod = get_pod(name, namespace)
 
-        logging(f"Waiting for pod {name} status {status}, current status {pod.status.phase} ({i}) ...")
-
-        if pod.status.phase == status:
-            is_running = True
-            break
+        try:
+            logging(f"Waiting for pod {name} status {status}, current status {pod.status.phase} ({i}) ...")
+            if pod.status.phase == status:
+                is_running = True
+                break
+        except Exception as e:
+            logging(e)
 
         time.sleep(retry_interval)
 

diff --git a/e2e/libs/workload/workload.py b/e2e/libs/workload/workload.py
@@ -76,6 +76,7 @@ def get_workload_persistent_volume_claim_names(workload_name, namespace="default
 
     for item in claim.items:
         claim_names.append(item.metadata.name)
+    claim_names.sort()
 
     assert len(claim_names) > 0, f"Failed to get PVC names for workload {workload_name}"
     return claim_names

diff --git a/e2e/tests/stress_cpu.robot b/e2e/tests/stress_cpu.robot
@@ -40,7 +40,7 @@ Stress Volume Node CPU When Volume Is Detaching and Attaching
         When Stress CPU of node with volume 0
         And Detach volume 0
         And Attach volume 0
-
+        And Wait for volume 0 healthy
         Then Check volume 0 data is intact
     END
 
@@ -52,8 +52,8 @@ Stress Volume Node CPU When Volume Is Online Expanding
         And Stress CPU of volume nodes
 
         When Expand statefulset 0 volume by 100 MiB
-
         Then Wait for statefulset 0 volume size expanded
+
         And Check statefulset 0 data in file 0.txt is intact
     END
 
@@ -70,5 +70,6 @@ Stress Volume Node CPU When Volume Is Offline Expanding
         Then Wait for statefulset 0 volume size expanded
         And Wait for statefulset 0 volume detached
         And Scale up statefulset 0 to attach volume
+        And Wait for volume of statefulset 0 healthy
         And Check statefulset 0 data in file 0.txt is intact
     END
diff --git a/e2e/tests/stress_filesystem.robot b/e2e/tests/stress_filesystem.robot
@@ -0,0 +1,74 @@
+*** Settings ***
+Documentation    Negative Test Cases
+
+Resource    ../keywords/common.resource
+Resource    ../keywords/persistentvolumeclaim.resource
+Resource    ../keywords/statefulset.resource
+Resource    ../keywords/stress.resource
+Resource    ../keywords/volume.resource
+Resource    ../keywords/workload.resource
+
+Test Setup    Set test environment
+Test Teardown    Cleanup test resources
+
+*** Variables ***
+${LOOP_COUNT}    1
+${RETRY_COUNT}    300
+${RETRY_INTERVAL}    1
+
+*** Test Cases ***
+
+Stress Volume Node Filesystem When Replica Is Rebuilding
+    Given Create volume 0 with 5 GB and 3 replicas
+    And Attach volume 0
+    And Write data to volume 0
+    And Stress filesystem of volume 0 volume node
+
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        When Delete volume 0 replica on volume node
+        And Wait until volume 0 replica rebuilding started on volume node
+
+        Then Wait until volume 0 replica rebuilding completed on volume node
+        And Check volume 0 data is intact
+    END
+
+Stress Volume Node Filesystem When Volume Is Detaching and Attaching
+    Given Create volume 0 with 5 GB and 3 replicas
+    And Attach volume 0
+    And Write data to volume 0
+    And Stress filesystem of volume 0 volume node
+
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        And Detach volume 0
+        And Attach volume 0
+        And Wait for volume 0 healthy
+        And Check volume 0 data is intact
+    END
+
+Stress Volume Node Filesystem When Volume Is Online Expanding
+    Given Create statefulset 0 using RWO volume
+    And Write 1024 MB data to file data.txt in statefulset 0
+    And Stress filesystem of statefulset 0 volume node
+
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        When Expand statefulset 0 volume by 100 MiB
+        Then Wait for statefulset 0 volume size expanded
+
+        And Check statefulset 0 data in file data.txt is intact
+    END
+
+Stress Volume Node Filesystem When Volume Is Offline Expanding
+    Given Create statefulset 0 using RWO volume
+    And Write 1024 MB data to file data.txt in statefulset 0
+    And Stress filesystem of all worker nodes
+
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        And Scale down statefulset 0 to detach volume
+
+        When Expand statefulset 0 volume by 100 MiB
+        Then Wait for statefulset 0 volume size expanded
+
+        And Scale up statefulset 0 to attach volume
+        And Wait for volume of statefulset 0 healthy
+        And Check statefulset 0 data in file data.txt is intact
+    END
diff --git a/e2e/tests/stress_memory.robot b/e2e/tests/stress_memory.robot
@@ -42,7 +42,7 @@ Stress Volume Node Memory When Volume Is Detaching and Attaching
 
         And Detach volume 0
         And Attach volume 0
-
+        And Wait for volume 0 healthy
         Then Check volume 0 data is intact
     END
 
@@ -72,5 +72,6 @@ Stress Volume Node Memory When Volume Is Offline Expanding
         Then Wait for statefulset 0 volume size expanded
         And Wait for statefulset 0 volume detached
         And Scale up statefulset 0 to attach volume
+        And Wait for volume of statefulset 0 healthy
         And Check statefulset 0 data in file 0.txt is intact
     END