[Databricks E2E] Refactor code and log messages to stderr (#962)

* initial changes

* disable debug

* typo

* reference common.sh in init_environment and remove when not needed

* remove commented code

* remove commented code

e2e_samples/parking_sensors/.devcontainer/Dockerfile

@@ -22,6 +22,7 @@ RUN az config set extension.use_dynamic_install=yes_without_prompt
 RUN az config set extension.dynamic_install_allow_preview=true
 RUN az extension add --name application-insights
 RUN az extension add --name azure-devops
+RUN az bicep upgrade
 
 # Install jq & makepasswd for some frequently used utility
 RUN apt-get update \

e2e_samples/parking_sensors/deploy.sh

@@ -19,9 +19,7 @@
 set -o errexit
 set -o pipefail
 set -o nounset
-# set -o xtrace # For debugging
 
-. ./scripts/common.sh
 . ./scripts/init_environment.sh
 . ./scripts/verify_prerequisites.sh
 
@@ -69,11 +67,11 @@ DEV_DATAFACTORY_NAME=$DEV_DATAFACTORY_NAME \
 
 ####
 
-print_style "DEPLOYMENT SUCCESSFUL
+log "DEPLOYMENT SUCCESSFUL
 Details of the deployment can be found in local .env.* files.\n\n" "success"
 
-print_style "IMPORTANT:
+log "IMPORTANT:
 This script has updated your local Azure Pipeline YAML definitions to point to your Github repo.
 ACTION REQUIRED: Commit and push up these changes to your Github repo before proceeding.\n\n" "warning"
 
-echo "See README > Setup and Deployment for more details and next steps." 
+log "See README > Setup and Deployment for more details and next steps." 

e2e_samples/parking_sensors/scripts/clean_up.sh

@@ -19,9 +19,8 @@
 set -o errexit
 set -o pipefail
 set -o nounset
-# set -o xtrace # For debugging
 
-. ./scripts/common.sh
+. ./scripts/init_environment.sh
 . ./scripts/verify_prerequisites.sh
 
 ###################
@@ -35,77 +34,77 @@ delete_all(){
     local prefix=$1
     local DEPLOYMENT_ID=${2:-}
 
-    echo "!! WARNING: !!"
-    echo "THIS SCRIPT WILL DELETE RESOURCES PREFIXED WITH $prefix AND HAVING DEPLOYMENT_ID $DEPLOYMENT_ID!!"
+    log "!! WARNING: !!" "danger"
+    log "THIS SCRIPT WILL DELETE RESOURCES PREFIXED WITH $prefix AND HAVING DEPLOYMENT_ID $DEPLOYMENT_ID!!" "danger"
 
-    printf "\nDEVOPS PIPELINES:\n"
+    log "\nDEVOPS PIPELINES:\n"
     az pipelines list -o tsv --only-show-errors --query "[?contains(name,'$prefix')].name"
 
-    printf "\nDEVOPS VARIABLE GROUPS:\n"
+    log "\nDEVOPS VARIABLE GROUPS:\n"
     az pipelines variable-group list -o tsv --only-show-errors --query "[?contains(name, '$prefix')].name"
 
-    printf "\nDEVOPS SERVICE CONNECTIONS:\n"
+    log "\nDEVOPS SERVICE CONNECTIONS:\n"
     az devops service-endpoint list -o tsv --only-show-errors --query "[?contains(name, '$prefix')].name"
 
-    printf "\nENTRA SERVICE PRINCIPALS:\n"
+    log "\nENTRA SERVICE PRINCIPALS:\n"
     if [[ -z $DEPLOYMENT_ID ]] 
     then
         az ad sp list -o tsv --show-mine --query "[?contains(appDisplayName,'$prefix')].displayName"
     else
         az ad sp list -o tsv --show-mine --query "[?contains(appDisplayName,'$prefix') && contains(appDisplayName,'$DEPLOYMENT_ID')].displayName"
     fi
 
-    printf "\nRESOURCE GROUPS:\n"
+    log "\nRESOURCE GROUPS:\n"
     if [[ -z $DEPLOYMENT_ID ]] 
     then
         az group list -o tsv --query "[?contains(name,'$prefix') && ! contains(name,'dbw')].name"
     else
         az group list -o tsv --query "[?contains(name,'$prefix-$DEPLOYMENT_ID') && ! contains(name,'dbw')].name"
     fi
 
-    printf "\nEND OF SUMMARY\n"
+    log "\nEND OF SUMMARY\n"
 
     read -r -p "Do you wish to DELETE above? [y/N] " response
     case "$response" in
         [yY][eE][sS]|[yY]) 
-            echo "Delete pipelines that start with '$prefix' in name..."
+            log "Deleting pipelines that start with '$prefix' in name..."
             [[ -n $prefix ]] &&
                 az pipelines list -o tsv |
                 { grep "$prefix" || true; } |
                 awk '{print $4}' |
                 xargs -r -I % az pipelines delete --id % --yes
 
-            echo "Delete variable groups that start with '$prefix' in name..."
+            log "Deleting variable groups that start with '$prefix' in name..."
             [[ -n $prefix ]] &&
                 az pipelines variable-group list -o tsv --query "[?contains(name, '$prefix')].id" |
                 xargs -r -I % az pipelines variable-group delete --id % --yes
 
-            echo "Delete service connections that start with '$prefix' in name..."
+            log "Deleting service connections that start with '$prefix' in name..."
             [[ -n $prefix ]] &&
                 az devops service-endpoint list -o tsv --query "[?contains(name, '$prefix')].id" |
                 xargs -r -I % az devops service-endpoint delete --id % --yes
 
             if [[ -z $DEPLOYMENT_ID ]]
             then
-                echo "Delete service principal that contain '$prefix' in name, created by yourself..."
+                log "Deleting service principal that contain '$prefix' in name, created by yourself..."
                 [[ -n $prefix ]] &&
                     az ad sp list --query "[?contains(appDisplayName,'$prefix')].appId" -o tsv --show-mine | 
                     xargs -r -I % az ad sp delete --id %
             else
-                echo "Delete service principal that contain '$prefix' and $DEPLOYMENT_ID in name, created by yourself..."
+                log "Deleting service principal that contain '$prefix' and $DEPLOYMENT_ID in name, created by yourself..."
                 [[ -n $prefix ]] &&
                     az ad sp list --query "[?contains(appDisplayName,'$prefix') && contains(appDisplayName,'$DEPLOYMENT_ID')].appId" -o tsv --show-mine | 
                     xargs -r -I % az ad sp delete --id %
             fi
 
             if [[ -z $DEPLOYMENT_ID ]]
             then
-                echo "Delete resource group that comtain '$prefix' in name..."
+                log "Deleting resource groups that comtain '$prefix' in name..."
                 [[ -n $prefix ]] &&
                     az group list --query "[?contains(name,'$prefix') && ! contains(name,'dbw')].name" -o tsv |
                     xargs -I % az group delete --verbose --name % -y
             else
-                echo "Delete resource group that contain '$prefix-$DEPLOYMENT_ID' in name..."
+                log "Deleting resource groups that contain '$prefix-$DEPLOYMENT_ID' in name..."
                 [[ -n $prefix ]] &&
                     az group list --query "[?contains(name,'$prefix-$DEPLOYMENT_ID') && ! contains(name,'dbw')].name" -o tsv |
                     xargs -I % az group delete --verbose --name % -y
@@ -120,19 +119,19 @@ delete_all(){
 
 if [[ -z "$DEPLOYMENT_ID" ]]
 then 
-    echo "No deployment id [DEPLOYMENT_ID] specified. You will only be able to delete by prefix $prefix..."
+    log "No deployment id [DEPLOYMENT_ID] specified. You will only be able to delete by prefix $prefix..."
     response=3
 else
     read -r -p "Do you wish to DELETE by"$'\n'"  1) ONLY BY PREFIX ($prefix)?"$'\n'"  2) PREFIX ($prefix) AND DEPLOYMENT_ID ($DEPLOYMENT_ID)?"$'\n'" Choose 1 or 2: " response
 fi
 
 case "$response" in
     1) 
-        echo "Delete by prefix..."
+        log "Deleting by prefix..."
         delete_all $prefix
         ;;
     2)
-        echo "Delete by deployment id..."
+        log "Deleting by deployment id..."
         delete_all $prefix $DEPLOYMENT_ID
         ;;
     3)
@@ -147,7 +146,7 @@ case "$response" in
         esac
         ;;
     *)
-        echo "Invalid choice. Exiting..."
+        log "Invalid choice. Exiting..." "warning"
         exit
         ;;
 esac

e2e_samples/parking_sensors/scripts/common.sh

@@ -14,52 +14,73 @@ random_str() {
 }
 
 print_style () {
+    case "$2" in
+        "info")
+            COLOR="96m"
+            ;;
+        "success")
+            COLOR="92m"
+            ;;
+        "warning")
+            COLOR="93m"
+            ;;
+        "danger")
+            COLOR="91m"
+            ;;
+        "action")
+            COLOR="32m"
+            ;;
+        *)
+            COLOR="0m"
+            ;;
+    esac
 
-    if [ "$2" == "info" ] ; then
-        COLOR="96m";
-    elif [ "$2" == "success" ] ; then
-        COLOR="92m";
-    elif [ "$2" == "warning" ] ; then
-        COLOR="93m";
-    elif [ "$2" == "danger" ] ; then
-        COLOR="91m";
-    else #default color
-        COLOR="0m";
-    fi
-
-    STARTCOLOR="\e[$COLOR";
-    ENDCOLOR="\e[0m";
+    STARTCOLOR="\e[$COLOR"
+    ENDCOLOR="\e[0m"
 
-    printf "$STARTCOLOR%b$ENDCOLOR" "$1";
+    printf "$STARTCOLOR%b$ENDCOLOR" "$1"
 }
 
+log() {
+    # This function takes a string as an argument and prints it to the console to stderr
+    # if a second argument is provided, it will be used as the style of the message
+    # Usage: log "message" "style"
+    # Example: log "Hello, World!" "info"
+    local message=$1
+    local style=${2:-}
 
-deletePipelineIfExists() {
-    declare pipeline_name=$1
-    full_pipeline_name=$PROJECT-$pipeline_name
+    if [[ -z "$style" ]]; then
+        echo -e "$(print_style "$message" "default")" >&2
+    else
+        echo -e "$(print_style "$message" "$style")" >&2
+    fi
+}
+
+delete_azdo_pipeline_if_exists() {
+    declare full_pipeline_name=$1
 
     ## when returning a pipeline that does exist, delete.
 
     pipeline_output=$(az pipelines list --query "[?name=='$full_pipeline_name']" --output json)
     pipeline_id=$(echo "$pipeline_output" | jq -r '.[0].id')
 
     if [[ -z "$pipeline_id" || "$pipeline_id" == "null" ]]; then
-        echo "Pipeline $full_pipeline_name does not exist.Creating..."
+        log "No Deployment pipeline with name $full_pipeline_name found."
     else
         az pipelines delete --id "$pipeline_id" --yes 1>/dev/null
-        echo "Deleted existing pipeline: $full_pipeline_name (Pipeline ID: $pipeline_id)"
-
+        log "Deleted existing pipeline: $full_pipeline_name (Pipeline ID: $pipeline_id)"
     fi
 }
 
-createPipeline ()
+create_azdo_pipeline ()
 {
     declare pipeline_name=$1
     declare pipeline_description=$2
     full_pipeline_name=$PROJECT-$pipeline_name
 
+    delete_azdo_pipeline_if_exists "$full_pipeline_name"
+    log "Creating deployment pipeline: $full_pipeline_name"
 
-
     pipeline_id=$(az pipelines create \
         --name "$full_pipeline_name" \
         --description "$pipeline_description" \
@@ -70,5 +91,40 @@ createPipeline ()
         --skip-first-run true \
         --output json | jq -r '.id')
     echo "$pipeline_id"
+}
+
+databricks_cluster_exists () {
+    declare cluster_name="$1"
+    declare cluster=$(databricks clusters list | tr -s " " | cut -d" " -f2 | grep ^${cluster_name}$)
+    if [[ -n $cluster ]]; then
+        return 0; # cluster exists
+    else
+        return 1; # cluster does not exists
+    fi
+}
 
+
+create_adf_linked_service () {
+    declare name=$1
+    log "Creating ADF LinkedService: $name"
+    adfLsUrl="${adfFactoryBaseUrl}/linkedservices/${name}?api-version=${apiVersion}"
+    az rest --method put --uri "$adfLsUrl" --body @"${ADF_DIR}"/linkedService/"${name}".json
+}
+create_adf_dataset () {
+    declare name=$1
+    log "Creating ADF Dataset: $name"
+    adfDsUrl="${adfFactoryBaseUrl}/datasets/${name}?api-version=${apiVersion}"
+    az rest --method put --uri "$adfDsUrl" --body @"${ADF_DIR}"/dataset/"${name}".json
+}
+create_adf_pipeline () {
+    declare name=$1
+    log "Creating ADF Pipeline: $name"
+    adfPUrl="${adfFactoryBaseUrl}/pipelines/${name}?api-version=${apiVersion}"
+    az rest --method put --uri "$adfPUrl" --body @"${ADF_DIR}"/pipeline/"${name}".json
+}
+create_adf_trigger () {
+    declare name=$1
+    log "Creating ADF Trigger: $name"
+    adfTUrl="${adfFactoryBaseUrl}/triggers/${name}?api-version=${apiVersion}"
+    az rest --method put --uri "$adfTUrl" --body @"${ADF_DIR}"/trigger/"${name}".json
 }