Added normal, stress and chaos test.

Author: pantierra

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added documentation for cloud-native bucket access [#364](https://github.com/developmentseed/eoapi-k8s/pull/364)
 - Removed unused testing variable and unused deploy script [#369](https://github.com/developmentseed/eoapi-k8s/pull/369)
+- Added load testing scripts [#373](https://github.com/developmentseed/eoapi-k8s/pull/373)
 
 ### Added
 

scripts/lib/common.sh

@@ -270,11 +270,108 @@ cleanup_on_exit() {
 
 trap cleanup_on_exit EXIT
 
+get_base_url() {
+    local namespace="${1:-eoapi}"
+
+    # Try localhost first (most common in local dev)
+    if curl -s -f -m 3 "http://localhost/stac" >/dev/null 2>&1; then
+        echo "http://localhost"
+        return 0
+    fi
+
+    # Try ingress if configured
+    local host
+    host=$(kubectl get ingress -n "$namespace" -o jsonpath='{.items[0].spec.rules[0].host}' 2>/dev/null || echo "")
+    if [[ -n "$host" ]] && curl -s -f -m 3 "http://$host/stac" >/dev/null 2>&1; then
+        echo "http://$host"
+        return 0
+    fi
+
+    return 1
+}
+
 # Export functions
+validate_autoscaling_environment() {
+    local namespace="$1"
+
+    validate_cluster || return 1
+    validate_namespace "$namespace" || return 1
+
+    # Check HPA exists
+    if ! kubectl get hpa -n "$namespace" >/dev/null 2>&1 || [[ $(kubectl get hpa -n "$namespace" --no-headers 2>/dev/null | wc -l) -eq 0 ]]; then
+        log_error "No HPA resources found. Deploy with autoscaling enabled."
+        return 1
+    fi
+
+    # Check metrics server
+    if ! kubectl get deployment -A | grep -q metrics-server; then
+        log_error "metrics-server required for autoscaling tests"
+        return 1
+    fi
+
+    return 0
+}
+
 export -f log_info log_success log_warn log_error log_debug
 export -f command_exists validate_tools check_requirements validate_cluster
-export -f is_ci validate_namespace
+export -f is_ci validate_namespace get_base_url
 export -f detect_release_name detect_namespace
-export -f wait_for_pods validate_eoapi_deployment
+export -f wait_for_pods validate_eoapi_deployment validate_autoscaling_environment
 export -f preflight_deploy preflight_ingest preflight_test
+# Python dependency management
+validate_python_environment() {
+    if ! command_exists python3; then
+        log_error "python3 is required but not found"
+        log_info "Install python3 to continue"
+        return 1
+    fi
+
+    log_debug "Python3 environment validated"
+    return 0
+}
+
+install_python_requirements() {
+    local requirements_file="$1"
+    local project_root="${2:-}"
+
+    # Resolve the full path to requirements file
+    local full_path="$requirements_file"
+    if [[ -n "$project_root" ]]; then
+        full_path="$project_root/$requirements_file"
+    fi
+
+    if [[ ! -f "$full_path" ]]; then
+        log_error "Requirements file not found: $full_path"
+        return 1
+    fi
+
+    log_info "Installing Python test dependencies from $requirements_file..."
+
+    if python3 -m pip install --user -r "$full_path" >/dev/null 2>&1; then
+        log_debug "Python requirements installed successfully"
+        return 0
+    else
+        log_warn "Could not install test dependencies automatically"
+        log_info "Try manually: pip install -r $requirements_file"
+        return 1
+    fi
+}
+
+validate_python_with_requirements() {
+    local requirements_file="${1:-}"
+    local project_root="${2:-}"
+
+    validate_python_environment || return 1
+
+    if [[ -n "$requirements_file" ]]; then
+        install_python_requirements "$requirements_file" "$project_root" || {
+            log_warn "Python requirements installation failed, but continuing..."
+            return 0  # Don't fail the entire operation
+        }
+    fi
+
+    return 0
+}
+
+export -f validate_python_environment install_python_requirements validate_python_with_requirements
 export -f show_standard_options

scripts/load.sh

@@ -21,7 +21,7 @@ USAGE:
 
 COMMANDS:
     baseline        Low load, verify monitoring works
-    autoscaling     Test HPA scaling under load
+    autoscaling     Delegate to autoscaling.sh for HPA tests
     normal          Realistic scenario
     stress          Find breaking points
     chaos           Kill pods during load, test resilience
@@ -37,33 +37,33 @@ EXAMPLES:
     # Run baseline load test
     $(basename "$0") baseline
 
-    # Test individual services
-    $(basename "$0") services --debug
-
     # Test autoscaling behavior
     $(basename "$0") autoscaling --debug
 
+    # Find breaking points
+    $(basename "$0") stress --debug
+
     # Run all load tests
     $(basename "$0") all
 EOF
 }
 
-get_base_url() {
-    # Try localhost first (most common in local dev)
-    if curl -s -f -m 3 "http://localhost/stac" >/dev/null 2>&1; then
-        echo "http://localhost"
-        return 0
-    fi
 
-    # Try ingress if configured
-    local host
-    host=$(kubectl get ingress -n "$NAMESPACE" -o jsonpath='{.items[0].spec.rules[0].host}' 2>/dev/null || echo "")
-    if [[ -n "$host" ]] && curl -s -f -m 3 "http://$host/stac" >/dev/null 2>&1; then
-        echo "http://$host"
-        return 0
-    fi
+wait_for_services() {
+    local base_url="$1"
+
+    # Wait for deployments to be available
+    for service in stac raster vector; do
+        kubectl wait --for=condition=Available deployment/"${RELEASE_NAME}-${service}" -n "$NAMESPACE" --timeout=60s 2>/dev/null || \
+            log_warn "Service $service may not be ready"
+    done
 
-    return 1
+    # Test basic connectivity
+    for endpoint in "$base_url/stac" "$base_url/raster/healthz" "$base_url/vector/healthz"; do
+        if ! curl -s -f -m 5 "$endpoint" >/dev/null 2>&1; then
+            log_warn "Endpoint not responding: $endpoint"
+        fi
+    done
 }
 
 test_endpoint() {
@@ -99,17 +99,13 @@ load_baseline() {
     validate_namespace "$NAMESPACE" || return 1
 
     local base_url
-    if ! base_url=$(get_base_url); then
+    if ! base_url=$(get_base_url "$NAMESPACE"); then
         log_error "Cannot reach eoAPI endpoints"
         return 1
     fi
     log_info "Using base URL: $base_url"
 
-    # Wait for deployments
-    for service in stac raster vector; do
-        kubectl wait --for=condition=Available deployment/"${RELEASE_NAME}-${service}" -n "$NAMESPACE" --timeout=60s 2>/dev/null || \
-            log_warn "Service $service may not be ready"
-    done
+    wait_for_services "$base_url"
 
     log_info "Running light load tests..."
     log_info "Monitor pods: kubectl get pods -n $NAMESPACE -w"
@@ -141,79 +137,135 @@ load_services() {
 load_autoscaling() {
     log_info "Running autoscaling tests..."
 
-    validate_cluster || return 1
-    validate_namespace "$NAMESPACE" || return 1
+    validate_autoscaling_environment "$NAMESPACE" || return 1
 
-    # Check HPA exists
-    if ! kubectl get hpa -n "$NAMESPACE" >/dev/null 2>&1 || [[ $(kubectl get hpa -n "$NAMESPACE" --no-headers 2>/dev/null | wc -l) -eq 0 ]]; then
-        log_error "No HPA resources found. Deploy with autoscaling enabled."
-        return 1
-    fi
+    validate_python_with_requirements "tests/requirements.txt" "${SCRIPT_DIR}/.." || return 1
 
-    # Check metrics server
-    if ! kubectl get deployment -A | grep -q metrics-server; then
-        log_error "metrics-server required for autoscaling tests"
+    # Wait for deployments
+    for service in stac raster vector; do
+        kubectl wait --for=condition=Available deployment/"${RELEASE_NAME}-${service}" -n "$NAMESPACE" --timeout=90s || return 1
+    done
+
+    # Get ingress host
+    local ingress_host
+    ingress_host=$(kubectl get ingress -n "$NAMESPACE" -o jsonpath='{.items[0].spec.rules[0].host}' 2>/dev/null || echo "localhost")
+
+    # Set environment for Python tests
+    export STAC_ENDPOINT="http://$ingress_host/stac"
+    export RASTER_ENDPOINT="http://$ingress_host/raster"
+    export VECTOR_ENDPOINT="http://$ingress_host/vector"
+
+    log_info "Running Python autoscaling tests..."
+    cd "${SCRIPT_DIR}/.."
+
+    local cmd="python3 -m pytest tests/autoscaling"
+    [[ "$DEBUG_MODE" == "true" ]] && cmd="$cmd -v --tb=short"
+
+    if eval "$cmd"; then
+        log_success "Autoscaling tests passed"
+    else
+        log_error "Autoscaling tests failed"
         return 1
     fi
+}
+
+load_normal() {
+    log_info "Running normal load test scenario..."
+
+    validate_cluster || return 1
+    validate_namespace "$NAMESPACE" || return 1
+    validate_python_with_requirements "tests/requirements.txt" "${SCRIPT_DIR}/.." || return 1
 
     local base_url
-    if ! base_url=$(get_base_url); then
+    if ! base_url=$(get_base_url "$NAMESPACE"); then
         log_error "Cannot reach eoAPI endpoints"
         return 1
     fi
-    log_info "Using base URL: $base_url"
 
-    # Wait for services
-    for service in stac raster vector; do
-        kubectl wait --for=condition=Available deployment/"${RELEASE_NAME}-${service}" -n "$NAMESPACE" --timeout=90s || return 1
-    done
+    wait_for_services "$base_url"
+
+    log_info "Running Python normal load test..."
+    cd "${SCRIPT_DIR}/.."
 
-    log_info "Current HPA status:"
-    kubectl get hpa -n "$NAMESPACE"
-
-    log_info "Generating sustained load to trigger autoscaling..."
-
-    # Generate load that should trigger HPA (10 min, 15 concurrent)
-    if command_exists hey; then
-        log_info "Starting sustained load test (10 minutes)..."
-        hey -z 600s -c 15 "$base_url/stac/search" -m POST \
-            -H "Content-Type: application/json" -d '{"limit":100}' &
-        local load_pid=$!
-
-        # Monitor HPA changes every 30s
-        log_info "Monitoring HPA scaling..."
-        for i in {1..20}; do
-            sleep 30
-            log_info "HPA status after ${i}x30s:"
-            kubectl get hpa -n "$NAMESPACE" --no-headers | awk '{print $1 ": " $6 "/" $7 " replicas, CPU: " $3}'
-        done
-
-        # Stop load test
-        kill $load_pid 2>/dev/null || true
-        wait $load_pid 2>/dev/null || true
-
-        log_info "Final HPA status:"
-        kubectl get hpa -n "$NAMESPACE"
-        log_success "Autoscaling test completed"
+    local cmd="python3 -m tests.load.load_tester normal --base-url $base_url"
+    [[ "$DEBUG_MODE" == "true" ]] && cmd="$cmd --duration 30 --users 5"
+
+    log_debug "Running: $cmd"
+
+    if eval "$cmd"; then
+        log_success "Normal load test completed"
     else
-        log_error "hey required for autoscaling tests"
+        log_error "Normal load test failed"
         return 1
     fi
 }
 
-load_normal() {
-    log_info "Running normal load test scenario..."
-    # TODO: Implement realistic mixed scenario
-}
-
 load_stress() {
     log_info "Running stress test to find breaking points..."
-    # TODO: Implement stress testing
+
+    validate_cluster || return 1
+    validate_namespace "$NAMESPACE" || return 1
+
+    validate_python_with_requirements "tests/requirements.txt" "${SCRIPT_DIR}/.." || return 1
+
+    local base_url
+    if ! base_url=$(get_base_url "$NAMESPACE"); then
+        log_error "Cannot reach eoAPI endpoints"
+        return 1
+    fi
+
+    wait_for_services "$base_url"
+
+    log_info "Running Python stress test module..."
+    cd "${SCRIPT_DIR}/.."
+
+    local cmd="python3 -m tests.load.load_tester --base-url $base_url"
+    [[ "$DEBUG_MODE" == "true" ]] && cmd="$cmd --test-duration 5 --max-workers 20"
+
+    log_debug "Running: $cmd"
+
+    if eval "$cmd"; then
+        log_success "Stress test completed"
+    else
+        log_error "Stress test failed"
+        return 1
+    fi
 }
 
 load_chaos() {
     log_info "Running chaos testing with pod failures..."
-    # TODO: Implement chaos testing
+
+    validate_cluster || return 1
+    validate_namespace "$NAMESPACE" || return 1
+    validate_python_with_requirements "tests/requirements.txt" "${SCRIPT_DIR}/.." || return 1
+
+    if ! command_exists kubectl; then
+        log_error "kubectl required for chaos testing"
+        return 1
+    fi
+
+    local base_url
+    if ! base_url=$(get_base_url "$NAMESPACE"); then
+        log_error "Cannot reach eoAPI endpoints"
+        return 1
+    fi
+
+    wait_for_services "$base_url"
+
+    log_info "Running Python chaos test..."
+    cd "${SCRIPT_DIR}/.."
+
+    local cmd="python3 -m tests.load.load_tester chaos --base-url $base_url --namespace $NAMESPACE"
+    [[ "$DEBUG_MODE" == "true" ]] && cmd="$cmd --duration 60 --kill-interval 30"
+
+    log_debug "Running: $cmd"
+
+    if eval "$cmd"; then
+        log_success "Chaos test completed"
+    else
+        log_error "Chaos test failed"
+        return 1
+    fi
 }
 
 load_all() {
@@ -285,7 +337,7 @@ main() {
             load_autoscaling
             ;;
         normal)
-            load_mixed
+            load_normal
             ;;
         stress)
             load_stress