Fix autopilot scenario failures (hashicorp#21025)

* Fix autopilot scenario failures

Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com>
Signed-off-by: Mike Baum <mike.baum@hashicorp.com>

* use bash instead of sh in create logs dir shell script
* ensure to only enable the file audit device in the upgrade cluster of the autopilot scenario if the variable is enabled

---------

Signed-off-by: Jaymala Sinha <jaymala@hashicorp.com>
Signed-off-by: Mike Baum <mike.baum@hashicorp.com>
Co-authored-by: Mike Baum <mike.baum@hashicorp.com>

enos/enos-scenario-autopilot.hcl

@@ -182,7 +182,7 @@ scenario "autopilot" {
     }
 
     variables {
-      vault_instances   = step.create_vault_cluster_targets.hosts
+      vault_instances   = step.create_vault_cluster.target_hosts
       vault_install_dir = local.vault_install_dir
       vault_root_token  = step.create_vault_cluster.root_token
     }
@@ -203,7 +203,7 @@ scenario "autopilot" {
     variables {
       leader_public_ip  = step.get_vault_cluster_ips.leader_public_ip
       leader_private_ip = step.get_vault_cluster_ips.leader_private_ip
-      vault_instances   = step.create_vault_cluster_targets.hosts
+      vault_instances   = step.create_vault_cluster.target_hosts
       vault_install_dir = local.vault_install_dir
       vault_root_token  = step.create_vault_cluster.root_token
     }
@@ -269,6 +269,7 @@ scenario "autopilot" {
       storage_node_prefix         = "upgrade_node"
       target_hosts                = step.create_vault_cluster_upgrade_targets.hosts
       unseal_method               = matrix.seal
+      enable_file_audit_device    = var.vault_enable_file_audit_device
     }
   }
 
@@ -286,7 +287,7 @@ scenario "autopilot" {
 
     variables {
       vault_install_dir = local.vault_install_dir
-      vault_instances   = step.create_vault_cluster_upgrade_targets.hosts
+      vault_instances   = step.upgrade_vault_cluster_with_autopilot.target_hosts
     }
   }
 
@@ -303,7 +304,7 @@ scenario "autopilot" {
 
     variables {
       vault_install_dir = local.vault_install_dir
-      vault_instances   = step.create_vault_cluster_upgrade_targets.hosts
+      vault_instances   = step.upgrade_vault_cluster_with_autopilot.target_hosts
       vault_root_token  = step.upgrade_vault_cluster_with_autopilot.root_token
     }
   }
@@ -324,7 +325,7 @@ scenario "autopilot" {
       vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
       vault_autopilot_upgrade_status  = "await-server-removal"
       vault_install_dir               = local.vault_install_dir
-      vault_instances                 = step.create_vault_cluster_upgrade_targets.hosts
+      vault_instances                 = step.create_vault_cluster_upgrade_targets.target_hosts
       vault_root_token                = step.upgrade_vault_cluster_with_autopilot.root_token
     }
   }
@@ -343,11 +344,11 @@ scenario "autopilot" {
     }
 
     variables {
-      vault_instances       = step.create_vault_cluster_targets.hosts
+      vault_instances       = step.create_vault_cluster.target_hosts
       vault_install_dir     = local.vault_install_dir
       vault_root_token      = step.create_vault_cluster.root_token
       node_public_ip        = step.get_vault_cluster_ips.leader_public_ip
-      added_vault_instances = step.create_vault_cluster_targets.hosts
+      added_vault_instances = step.upgrade_vault_cluster_with_autopilot.target_hosts
     }
   }
 
@@ -386,7 +387,7 @@ scenario "autopilot" {
 
     variables {
       operator_instance      = step.get_updated_vault_cluster_ips.leader_public_ip
-      remove_vault_instances = step.create_vault_cluster_targets.hosts
+      remove_vault_instances = step.create_vault_cluster.target_hosts
       vault_install_dir      = local.vault_install_dir
       vault_instance_count   = 3
       vault_root_token       = step.create_vault_cluster.root_token
@@ -405,7 +406,7 @@ scenario "autopilot" {
     }
 
     variables {
-      old_vault_instances  = step.create_vault_cluster_targets.hosts
+      old_vault_instances  = step.create_vault_cluster.target_hosts
       vault_instance_count = 3
     }
   }
@@ -427,7 +428,7 @@ scenario "autopilot" {
       vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
       vault_autopilot_upgrade_status  = "idle"
       vault_install_dir               = local.vault_install_dir
-      vault_instances                 = step.create_vault_cluster_upgrade_targets.hosts
+      vault_instances                 = step.upgrade_vault_cluster_with_autopilot.target_hosts
       vault_root_token                = step.create_vault_cluster.root_token
     }
   }
@@ -448,7 +449,7 @@ scenario "autopilot" {
 
     variables {
       vault_install_dir = local.vault_install_dir
-      vault_instances   = step.create_vault_cluster_upgrade_targets.hosts
+      vault_instances   = step.upgrade_vault_cluster_with_autopilot.target_hosts
       vault_root_token  = step.create_vault_cluster.root_token
     }
   }

enos/modules/vault_cluster/main.tf

@@ -61,7 +61,7 @@ locals {
       path    = "vault"
     })
   ]
-  audit_device_file_path = "/var/log/vault_audit.log"
+  audit_device_file_path = "/var/log/vault/vault_audit.log"
   vault_service_user     = "vault"
   enable_audit_device    = var.enable_file_audit_device && var.initialize_cluster
 }

enos/modules/vault_cluster/scripts/create_audit_log_dir.sh

@@ -1,8 +1,30 @@
-#!/bin/env sh
+#!/usr/bin/env bash
 
 set -eux
 
 LOG_DIR=$(dirname "$LOG_FILE_PATH")
 
+function retry {
+  local retries=$1
+  shift
+  local count=0
+
+  until "$@"; do
+    exit=$?
+    wait=10
+    count=$((count + 1))
+
+    if [ "$count" -lt "$retries" ]; then
+      sleep "$wait"
+    else
+      return "$exit"
+    fi
+  done
+
+  return 0
+}
+
+retry 7 id -a "$SERVICE_USER"
+
 sudo mkdir -p "$LOG_DIR"
-sudo chown "$SERVICE_USER":"$SERVICE_USER" "$LOG_DIR"
+sudo chown -R "$SERVICE_USER":"$SERVICE_USER" "$LOG_DIR"