From 73d37e77803b90bbad8192d42ad3c8700a87bc7f Mon Sep 17 00:00:00 2001 From: Yang Chiu Date: Tue, 9 Apr 2024 17:17:28 +0800 Subject: [PATCH] test(negative): add fs stress operations Signed-off-by: Yang Chiu --- e2e/keywords/stress.resource | 18 +++++++ e2e/libs/keywords/common_keywords.py | 5 +- e2e/libs/keywords/stress_keywords.py | 6 +++ e2e/libs/node/constant.py | 2 + e2e/libs/node/stress.py | 34 ++++++++++++- e2e/libs/workload/pod.py | 12 +++-- e2e/libs/workload/workload.py | 1 + e2e/tests/stress_cpu.robot | 5 +- e2e/tests/stress_filesystem.robot | 74 ++++++++++++++++++++++++++++ e2e/tests/stress_memory.robot | 3 +- 10 files changed, 150 insertions(+), 10 deletions(-) create mode 100644 e2e/tests/stress_filesystem.robot diff --git a/e2e/keywords/stress.resource b/e2e/keywords/stress.resource index a432b3bfd4..363719ad12 100644 --- a/e2e/keywords/stress.resource +++ b/e2e/keywords/stress.resource @@ -1,7 +1,10 @@ *** Settings *** Documentation Stress Node Keywords +Library ../libs/keywords/common_keywords.py Library ../libs/keywords/stress_keywords.py +Library ../libs/keywords/workload_keywords.py +Library ../libs/keywords/volume_keywords.py *** Keywords *** Stress CPU of all ${role} nodes @@ -23,3 +26,18 @@ Stress memory of node with volume ${volume_id} Stress memory of volume nodes stress_node_memory_of_all_volumes + +Stress filesystem of volume ${volume_id} volume node + ${volume_name} = generate_name_with_suffix volume ${volume_id} + ${node_name} = get_volume_node ${volume_name} + stress_nodes_filesystem ${node_name} + +Stress filesystem of statefulset ${statefulset_id} volume node + ${workload_name} = generate_name_with_suffix statefulset ${statefulset_id} + ${volume_name} = get_workload_volume_name ${workload_name} + ${node_name} = get_volume_node ${volume_name} + stress_nodes_filesystem ${node_name} + +Stress filesystem of all worker nodes + ${worker_nodes} = get_worker_nodes + stress_nodes_filesystem ${worker_nodes} diff --git a/e2e/libs/keywords/common_keywords.py b/e2e/libs/keywords/common_keywords.py index 0a64d102a9..ed3bfdd942 100644 --- a/e2e/libs/keywords/common_keywords.py +++ b/e2e/libs/keywords/common_keywords.py @@ -1,5 +1,5 @@ from node_exec import NodeExec - +from node import Node from utility.utility import init_k8s_api_client from utility.utility import generate_name_with_suffix @@ -21,3 +21,6 @@ def cleanup_node_exec(self): def generate_name_with_suffix(self, kind, suffix): return generate_name_with_suffix(kind, suffix) + + def get_worker_nodes(self): + return Node().list_node_names_by_role("worker") diff --git a/e2e/libs/keywords/stress_keywords.py b/e2e/libs/keywords/stress_keywords.py index afb3826716..8c296db388 100644 --- a/e2e/libs/keywords/stress_keywords.py +++ b/e2e/libs/keywords/stress_keywords.py @@ -54,3 +54,9 @@ def stress_node_memory_of_all_volumes(self): logging(f'Stressing node memory for all volumes {volume_names}') self.stress_node_memory_by_volumes(volume_names) + + def stress_nodes_filesystem(self, nodes): + logging(f'Stressing {nodes} nodes filesystem') + if isinstance(nodes, str): + nodes = [nodes] + self.stress.filesystem(nodes) diff --git a/e2e/libs/node/constant.py b/e2e/libs/node/constant.py index b7dc738512..48de3c6d79 100644 --- a/e2e/libs/node/constant.py +++ b/e2e/libs/node/constant.py @@ -1,6 +1,8 @@ NODE_STRESS_CPU_LOAD_PERCENTAGE = 100 NODE_STRESS_MEM_LOAD_PERCENTAGE = 100 NODE_STRESS_MEM_VM_WORKERS = 1 +NODE_STRESS_FILESYSTEM_HDD_WORKERS = 8 +NODE_STRESS_FILESYSTEM_LOAD_PERCENTAGE = 80 NODE_STRESS_TIMEOUT_SECOND = 60 * 60 # 1 hour STRESS_HELPER_LABEL = "longhorn-stress-helper" diff --git a/e2e/libs/node/stress.py b/e2e/libs/node/stress.py index 2aacb1041d..43ff7bd5eb 100644 --- a/e2e/libs/node/stress.py +++ b/e2e/libs/node/stress.py @@ -1,9 +1,10 @@ from kubernetes.client.rest import ApiException - from node import Node from node.constant import NODE_STRESS_CPU_LOAD_PERCENTAGE from node.constant import NODE_STRESS_MEM_LOAD_PERCENTAGE from node.constant import NODE_STRESS_MEM_VM_WORKERS +from node.constant import NODE_STRESS_FILESYSTEM_HDD_WORKERS +from node.constant import NODE_STRESS_FILESYSTEM_LOAD_PERCENTAGE from node.constant import NODE_STRESS_TIMEOUT_SECOND from node.constant import STRESS_HELPER_LABEL from node.constant import STRESS_HELPER_POD_NAME_PREFIX @@ -91,3 +92,34 @@ def memory(self, node_names): pod_name = manifest['metadata']['name'] logging(f"Creating memory stress pod {pod_name} on {node_name}") create_pod(manifest, is_wait_for_pod_running=True) + + def filesystem(self, node_names): + for node_name in node_names: + pod_name = f"{STRESS_HELPER_POD_NAME_PREFIX}{node_name}" + + # If the helper pod creation is called inside of a test case loop, + # we need to check if the pod already running. + try: + pod = get_pod(pod_name) + if pod and pod.status.phase != "Running": + logging(f"Deleting stress pod {pod_name} in phase {pod.status.phase}") + delete_pod(pod_name) + elif pod: + logging(f"Stress pod {pod_name} already running") + continue + except ApiException as e: + assert e.status == 404 + + manifest = new_pod_manifest( + pod_name=pod_name, + image=IMAGE_LITMUX, + command=["stress-ng"], + args=['--hdd', str(NODE_STRESS_FILESYSTEM_HDD_WORKERS), + '--hdd-bytes', f"{NODE_STRESS_FILESYSTEM_LOAD_PERCENTAGE}%"], + node_name=node_name, + labels={'app': STRESS_HELPER_LABEL} + ) + + pod_name = manifest['metadata']['name'] + logging(f"Creating filesystem stress pod {pod_name} on {node_name}") + create_pod(manifest, is_wait_for_pod_running=True) diff --git a/e2e/libs/workload/pod.py b/e2e/libs/workload/pod.py index 2393c34143..aeae7be702 100644 --- a/e2e/libs/workload/pod.py +++ b/e2e/libs/workload/pod.py @@ -143,11 +143,13 @@ def wait_for_pod_status(name, status, namespace='default'): for i in range(retry_count): pod = get_pod(name, namespace) - logging(f"Waiting for pod {name} status {status}, current status {pod.status.phase} ({i}) ...") - - if pod.status.phase == status: - is_running = True - break + try: + logging(f"Waiting for pod {name} status {status}, current status {pod.status.phase} ({i}) ...") + if pod.status.phase == status: + is_running = True + break + except Exception as e: + logging(e) time.sleep(retry_interval) diff --git a/e2e/libs/workload/workload.py b/e2e/libs/workload/workload.py index 0a7096dc34..c270406927 100644 --- a/e2e/libs/workload/workload.py +++ b/e2e/libs/workload/workload.py @@ -76,6 +76,7 @@ def get_workload_persistent_volume_claim_names(workload_name, namespace="default for item in claim.items: claim_names.append(item.metadata.name) + claim_names.sort() assert len(claim_names) > 0, f"Failed to get PVC names for workload {workload_name}" return claim_names diff --git a/e2e/tests/stress_cpu.robot b/e2e/tests/stress_cpu.robot index 293d459c90..35dbc9fcdc 100644 --- a/e2e/tests/stress_cpu.robot +++ b/e2e/tests/stress_cpu.robot @@ -40,7 +40,7 @@ Stress Volume Node CPU When Volume Is Detaching and Attaching When Stress CPU of node with volume 0 And Detach volume 0 And Attach volume 0 - + And Wait for volume 0 healthy Then Check volume 0 data is intact END @@ -52,8 +52,8 @@ Stress Volume Node CPU When Volume Is Online Expanding And Stress CPU of volume nodes When Expand statefulset 0 volume by 100 MiB - Then Wait for statefulset 0 volume size expanded + And Check statefulset 0 data in file 0.txt is intact END @@ -70,5 +70,6 @@ Stress Volume Node CPU When Volume Is Offline Expanding Then Wait for statefulset 0 volume size expanded And Wait for statefulset 0 volume detached And Scale up statefulset 0 to attach volume + And Wait for volume of statefulset 0 healthy And Check statefulset 0 data in file 0.txt is intact END diff --git a/e2e/tests/stress_filesystem.robot b/e2e/tests/stress_filesystem.robot new file mode 100644 index 0000000000..36591645dc --- /dev/null +++ b/e2e/tests/stress_filesystem.robot @@ -0,0 +1,74 @@ +*** Settings *** +Documentation Negative Test Cases + +Resource ../keywords/common.resource +Resource ../keywords/persistentvolumeclaim.resource +Resource ../keywords/statefulset.resource +Resource ../keywords/stress.resource +Resource ../keywords/volume.resource +Resource ../keywords/workload.resource + +Test Setup Set test environment +Test Teardown Cleanup test resources + +*** Variables *** +${LOOP_COUNT} 1 +${RETRY_COUNT} 300 +${RETRY_INTERVAL} 1 + +*** Test Cases *** + +Stress Volume Node Filesystem When Replica Is Rebuilding + Given Create volume 0 with 5 GB and 3 replicas + And Attach volume 0 + And Write data to volume 0 + And Stress filesystem of volume 0 volume node + + FOR ${i} IN RANGE ${LOOP_COUNT} + When Delete volume 0 replica on volume node + And Wait until volume 0 replica rebuilding started on volume node + + Then Wait until volume 0 replica rebuilding completed on volume node + And Check volume 0 data is intact + END + +Stress Volume Node Filesystem When Volume Is Detaching and Attaching + Given Create volume 0 with 5 GB and 3 replicas + And Attach volume 0 + And Write data to volume 0 + And Stress filesystem of volume 0 volume node + + FOR ${i} IN RANGE ${LOOP_COUNT} + And Detach volume 0 + And Attach volume 0 + And Wait for volume 0 healthy + And Check volume 0 data is intact + END + +Stress Volume Node Filesystem When Volume Is Online Expanding + Given Create statefulset 0 using RWO volume + And Write 1024 MB data to file data.txt in statefulset 0 + And Stress filesystem of statefulset 0 volume node + + FOR ${i} IN RANGE ${LOOP_COUNT} + When Expand statefulset 0 volume by 100 MiB + Then Wait for statefulset 0 volume size expanded + + And Check statefulset 0 data in file data.txt is intact + END + +Stress Volume Node Filesystem When Volume Is Offline Expanding + Given Create statefulset 0 using RWO volume + And Write 1024 MB data to file data.txt in statefulset 0 + And Stress filesystem of all worker nodes + + FOR ${i} IN RANGE ${LOOP_COUNT} + And Scale down statefulset 0 to detach volume + + When Expand statefulset 0 volume by 100 MiB + Then Wait for statefulset 0 volume size expanded + + And Scale up statefulset 0 to attach volume + And Wait for volume of statefulset 0 healthy + And Check statefulset 0 data in file data.txt is intact + END diff --git a/e2e/tests/stress_memory.robot b/e2e/tests/stress_memory.robot index d69329c393..d119ad0e9e 100644 --- a/e2e/tests/stress_memory.robot +++ b/e2e/tests/stress_memory.robot @@ -42,7 +42,7 @@ Stress Volume Node Memory When Volume Is Detaching and Attaching And Detach volume 0 And Attach volume 0 - + And Wait for volume 0 healthy Then Check volume 0 data is intact END @@ -72,5 +72,6 @@ Stress Volume Node Memory When Volume Is Offline Expanding Then Wait for statefulset 0 volume size expanded And Wait for statefulset 0 volume detached And Scale up statefulset 0 to attach volume + And Wait for volume of statefulset 0 healthy And Check statefulset 0 data in file 0.txt is intact END