[release] Hello world test for Azure

ci/ray_ci/oss_config.yaml

@@ -7,6 +7,7 @@ release_byod:
   byod_ecr_region: us-west-2
   aws_cr: 029272617770.dkr.ecr.us-west-2.amazonaws.com
   gcp_cr: us-west1-docker.pkg.dev/anyscale-oss-ci
+  azure_cr: rayreleasetest.azurecr.io
   aws2gce_credentials: release/aws2gce_iam.json
 ci_pipeline:
   premerge:

release/hello_world_tests/hello_world_compute_config_azure.yaml

@@ -0,0 +1,8 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+head_node_type:
+    name: head_node
+    instance_type: 4CPU-16GB
+
+worker_node_types: []

release/ray_release/buildkite/step.py

@@ -12,12 +12,12 @@
     get_test_project_id,
 )
 from ray_release.env import DEFAULT_ENVIRONMENT, load_environment
+from ray_release.template import get_test_env_var
+from ray_release.util import DeferredEnvVar
 from ray_release.custom_byod_build_init_helper import (
     generate_custom_build_step_key,
     get_prerequisite_step,
 )
-from ray_release.template import get_test_env_var
-from ray_release.util import DeferredEnvVar
 
 DEFAULT_ARTIFACTS_DIR_HOST = "/tmp/ray_release_test_artifacts"
 

release/ray_release/command_runner/_anyscale_job_wrapper.py

@@ -18,6 +18,7 @@
 from urllib.parse import urlparse
 from typing import Optional, List, Tuple
 
+AZURE_STORAGE_ACCOUNT = "rayreleasetests"
 OUTPUT_JSON_FILENAME = "output.json"
 AWS_CP_TIMEOUT = 300
 TIMEOUT_RETURN_CODE = 124  # same as bash timeout
@@ -69,6 +70,8 @@ def run_storage_cp(source: str, target: str):
         return False
 
     storage_service = urlparse(target).scheme
+    if target.startswith(f"https://{AZURE_STORAGE_ACCOUNT}.dfs.core.windows.net"):
+        storage_service = "azure_blob"
     cp_cmd_args = []
     if storage_service == "s3":
         cp_cmd_args = [
@@ -88,7 +91,7 @@ def run_storage_cp(source: str, target: str):
             source,
             target,
         ]
-    elif storage_service == "abfss":
+    elif storage_service == "azure_blob":
         subprocess.run(["azcopy", "login", "--identity"], check=True)
         cp_cmd_args = [
             "azcopy",

release/ray_release/configs/global_config.py

@@ -14,6 +14,7 @@ class GlobalConfig(TypedDict):
     byod_ecr_region: str
     byod_aws_cr: str
     byod_gcp_cr: str
+    byod_azure_cr: str
     state_machine_pr_aws_bucket: str
     state_machine_branch_aws_bucket: str
     state_machine_disabled: bool
@@ -83,6 +84,10 @@ def _init_global_config(config_file: str):
             config_content.get("byod", {}).get("gcp_cr")
             or config_content.get("release_byod", {}).get("gcp_cr")
         ),
+        byod_azure_cr=(
+            config_content.get("byod", {}).get("azure_cr")
+            or config_content.get("release_byod", {}).get("azure_cr")
+        ),
         aws2gce_credentials=(
             config_content.get("credentials", {}).get("aws2gce")
             or config_content.get("release_byod", {}).get("aws2gce_credentials")

release/ray_release/environments/azure.env

@@ -0,0 +1,5 @@
+ANYSCALE_HOST=https://console.anyscale-staging.com
+RELEASE_AWS_ANYSCALE_SECRET_ARN="arn:aws:secretsmanager:us-west-2:029272617770:secret:release-automation/anyscale-staging-token20231008005227440600000001-JTgxb0"
+RELEASE_DEFAULT_CLOUD_ID="cld_5nnv7pt2jn2312x2e5v72z53n2"
+RELEASE_DEFAULT_PROJECT="prj_y8syktydl7ltabhz5axdelwnce"
+ANYSCALE_CLOUD_STORAGE_PROVIDER=abfss

release/ray_release/test.py

@@ -593,6 +593,8 @@ def get_byod_ecr(self) -> str:
         """
         if self.is_gce() or self.is_kuberay():
             return get_global_config()["byod_gcp_cr"]
+        if self.is_azure():
+            return get_global_config()["byod_azure_cr"]
         byod_ecr = get_global_config()["byod_aws_cr"]
         if byod_ecr:
             return byod_ecr

release/release_tests.yaml

@@ -95,6 +95,11 @@
 
   variations:
     - __suffix__: aws
+    - __suffix__: azure
+      env: azure
+      frequency: manual
+      cluster:
+        cluster_compute: hello_world_compute_config_azure.yaml
 
 - name: hello_world_custom_byod
   team: reef