feat: ability to download multiple artifacts

Modify `download_ci_artifacts` to accept artifacts paths
as input and download by passing these paths to
`ensure_ci_artifacts`. `ensure_ci_artifacts` will default
to latest s3 artifacts if no args were provided.

Signed-off-by: Egor Lazarchuk <yegorlz@amazon.co.uk>

Author: ShadowCurse

.buildkite/common.py

@@ -376,6 +376,11 @@ def to_json(self):
         """Serialize the pipeline to JSON"""
         return json.dumps(self.to_dict(), indent=4, sort_keys=True, ensure_ascii=False)
 
+    def devtool_download_artifacts(self, artifacts_s3_url):
+        """Generate a `devtool download_ci_artifacts` command"""
+        parts = ["./tools/devtool -y download_ci_artifacts"]
+
+
     def devtool_test(self, devtool_opts=None, pytest_opts=None):
         """Generate a `devtool test` command"""
         cmds = []

tools/devtool

@@ -426,9 +426,10 @@ cmd_help() {
     echo "        Builds the rootfs and guest kernel artifacts we use for our CI."
     echo "        Run './tools/devtool build_ci_artifacts help' for more details about the available commands."
     echo ""
-    echo "    download_ci_artifacts [--force]"
+    echo "    download_ci_artifacts [--force] [artifact1 artifact2 ...]"
     echo "        Downloads the CI artifacts used for testing from our S3 bucket. If --force is passed, purges any existing"
-    echo "        artifacts first. Useful for refreshing local artifacts after an update, or if something got messed up."
+    echo "        artifacts first. If specific artifacts are listed, only those will be downloaded; otherwise all artifacts"
+    echo "        are downloaded. Useful for refreshing local artifacts after an update, or if something got messed up."
     echo ""
 
     cat <<EOF
@@ -577,20 +578,45 @@ cmd_distclean() {
 }
 
 cmd_download_ci_artifacts() {
-    if [ "$1" = "--force" ]; then
+    local force=false
+    local artifacts=()
+
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            --force)
+                force=true
+                ;;
+            *)
+                artifacts+=("$1")
+                ;;
+        esac
+        shift
+    done
+
+    if [ "$force" = true ]; then
         rm -rf $ARTIFACTS_DIR
     fi
 
-    ensure_ci_artifacts
+    if [ ${#artifacts[@]} -eq 0 ]; then
+      say "No artifacts were specified"
+    fi
+
+    for artifact in "${artifacts[@]}"; do
+      ensure_ci_artifacts $artifact
+    done
 }
 
 ensure_ci_artifacts() {
-    if ! command -v aws >/dev/null; then
-      die "AWS CLI not installed, which is required for downloading artifacts for integration tests."
+    local artifacts=$1
+
+    if [ -z $artifacts ]; then
+        local default_artifacts=$(get_newest_s3_artifacts)
+        say "No specific artifacts are defined. Using default artifacts: " $default_artifacts
+        artifacts=$default_artifacts
     fi
 
     # Fetch all the artifacts so they are local
-    local artifacts_s3_url=$(get_newest_s3_artifacts)
+    local artifacts_s3_url=$artifacts
     local artifacts_s3_url_arch=$artifacts_s3_url/$(uname -m)
     local artifacts_local_path=$(get_local_artifacts_path $artifacts_s3_url)/$(uname -m)
 
@@ -710,6 +736,7 @@ cmd_test() {
     do_build=1
     do_archive=1
     do_kvm_check=1
+    do_ci_artifacts_check=1
     # Parse any command line args.
     while [ $# -gt 0 ]; do
         case "$1" in
@@ -737,6 +764,9 @@ cmd_test() {
             "--no-kvm-check")
                 do_kvm_check=0
                 ;;
+            "--no-ci-artifacts-check")
+                do_ci_artifacts_check=0
+                ;;
             "--")               { shift; break;     } ;;
             *)
                 die "Unknown argument: $1. Please use --help for help."
@@ -749,7 +779,7 @@ cmd_test() {
     [ $do_kvm_check != 0 ] && ensure_kvm
     ensure_devctr
     ensure_build_dir
-    ensure_ci_artifacts
+    [ $do_ci_artifacts_check != 0 ] && ensure_ci_artifacts
     if [ $do_build != 0 ]; then
       cmd_build --release
       if [ -n "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" ]; then