Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Databricks E2E] Refactor code and log messages to stderr #962

Merged
merged 6 commits into from
Dec 18, 2024
+197 −187
Merged

[Databricks E2E] Refactor code and log messages to stderr #962

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d3115d2
initial changes
elenaterenzi Dec 17, 2024
c2dda96
disable debug
elenaterenzi Dec 17, 2024
fa31377
typo
elenaterenzi Dec 17, 2024
2533b09
reference common.sh in init_environment and remove when not needed
elenaterenzi Dec 18, 2024
2d533e4
remove commented code
elenaterenzi Dec 18, 2024
d52a076
remove commented code
elenaterenzi Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions e2e_samples/parking_sensors/.devcontainer/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ RUN az config set extension.use_dynamic_install=yes_without_prompt
RUN az config set extension.dynamic_install_allow_preview=true
RUN az extension add --name application-insights
RUN az extension add --name azure-devops
RUN az bicep upgrade
elenaterenzi marked this conversation as resolved.
Show resolved Hide resolved

# Install jq & makepasswd for some frequently used utility
RUN apt-get update \
Expand Down
7 changes: 3 additions & 4 deletions e2e_samples/parking_sensors/deploy.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ set -o pipefail
set -o nounset
# set -o xtrace # For debugging
elenaterenzi marked this conversation as resolved.
Show resolved Hide resolved

. ./scripts/common.sh
. ./scripts/init_environment.sh
. ./scripts/verify_prerequisites.sh

Expand Down Expand Up @@ -69,11 +68,11 @@ DEV_DATAFACTORY_NAME=$DEV_DATAFACTORY_NAME \

####

print_style "DEPLOYMENT SUCCESSFUL
log "DEPLOYMENT SUCCESSFUL
Details of the deployment can be found in local .env.* files.\n\n" "success"

print_style "IMPORTANT:
log "IMPORTANT:
This script has updated your local Azure Pipeline YAML definitions to point to your Github repo.
ACTION REQUIRED: Commit and push up these changes to your Github repo before proceeding.\n\n" "warning"

echo "See README > Setup and Deployment for more details and next steps."
log "See README > Setup and Deployment for more details and next steps."
40 changes: 20 additions & 20 deletions e2e_samples/parking_sensors/scripts/clean_up.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ set -o pipefail
set -o nounset
# set -o xtrace # For debugging
elenaterenzi marked this conversation as resolved.
Show resolved Hide resolved

. ./scripts/common.sh
. ./scripts/init_environment.sh
. ./scripts/verify_prerequisites.sh

###################
Expand All @@ -35,77 +35,77 @@ delete_all(){
local prefix=$1
local DEPLOYMENT_ID=${2:-}

echo "!! WARNING: !!"
echo "THIS SCRIPT WILL DELETE RESOURCES PREFIXED WITH $prefix AND HAVING DEPLOYMENT_ID $DEPLOYMENT_ID!!"
log "!! WARNING: !!" "danger"
log "THIS SCRIPT WILL DELETE RESOURCES PREFIXED WITH $prefix AND HAVING DEPLOYMENT_ID $DEPLOYMENT_ID!!" "danger"

printf "\nDEVOPS PIPELINES:\n"
log "\nDEVOPS PIPELINES:\n"
az pipelines list -o tsv --only-show-errors --query "[?contains(name,'$prefix')].name"

printf "\nDEVOPS VARIABLE GROUPS:\n"
log "\nDEVOPS VARIABLE GROUPS:\n"
az pipelines variable-group list -o tsv --only-show-errors --query "[?contains(name, '$prefix')].name"

printf "\nDEVOPS SERVICE CONNECTIONS:\n"
log "\nDEVOPS SERVICE CONNECTIONS:\n"
az devops service-endpoint list -o tsv --only-show-errors --query "[?contains(name, '$prefix')].name"

printf "\nENTRA SERVICE PRINCIPALS:\n"
log "\nENTRA SERVICE PRINCIPALS:\n"
if [[ -z $DEPLOYMENT_ID ]]
then
az ad sp list -o tsv --show-mine --query "[?contains(appDisplayName,'$prefix')].displayName"
else
az ad sp list -o tsv --show-mine --query "[?contains(appDisplayName,'$prefix') && contains(appDisplayName,'$DEPLOYMENT_ID')].displayName"
fi

printf "\nRESOURCE GROUPS:\n"
log "\nRESOURCE GROUPS:\n"
if [[ -z $DEPLOYMENT_ID ]]
then
az group list -o tsv --query "[?contains(name,'$prefix') && ! contains(name,'dbw')].name"
else
az group list -o tsv --query "[?contains(name,'$prefix-$DEPLOYMENT_ID') && ! contains(name,'dbw')].name"
fi

printf "\nEND OF SUMMARY\n"
log "\nEND OF SUMMARY\n"

read -r -p "Do you wish to DELETE above? [y/N] " response
case "$response" in
[yY][eE][sS]|[yY])
echo "Delete pipelines that start with '$prefix' in name..."
log "Deleting pipelines that start with '$prefix' in name..."
[[ -n $prefix ]] &&
az pipelines list -o tsv |
{ grep "$prefix" || true; } |
awk '{print $4}' |
xargs -r -I % az pipelines delete --id % --yes

echo "Delete variable groups that start with '$prefix' in name..."
log "Deleting variable groups that start with '$prefix' in name..."
[[ -n $prefix ]] &&
az pipelines variable-group list -o tsv --query "[?contains(name, '$prefix')].id" |
xargs -r -I % az pipelines variable-group delete --id % --yes

echo "Delete service connections that start with '$prefix' in name..."
log "Deleting service connections that start with '$prefix' in name..."
[[ -n $prefix ]] &&
az devops service-endpoint list -o tsv --query "[?contains(name, '$prefix')].id" |
xargs -r -I % az devops service-endpoint delete --id % --yes

if [[ -z $DEPLOYMENT_ID ]]
then
echo "Delete service principal that contain '$prefix' in name, created by yourself..."
log "Deleting service principal that contain '$prefix' in name, created by yourself..."
[[ -n $prefix ]] &&
az ad sp list --query "[?contains(appDisplayName,'$prefix')].appId" -o tsv --show-mine |
xargs -r -I % az ad sp delete --id %
else
echo "Delete service principal that contain '$prefix' and $DEPLOYMENT_ID in name, created by yourself..."
log "Deleting service principal that contain '$prefix' and $DEPLOYMENT_ID in name, created by yourself..."
[[ -n $prefix ]] &&
az ad sp list --query "[?contains(appDisplayName,'$prefix') && contains(appDisplayName,'$DEPLOYMENT_ID')].appId" -o tsv --show-mine |
xargs -r -I % az ad sp delete --id %
fi

if [[ -z $DEPLOYMENT_ID ]]
then
echo "Delete resource group that comtain '$prefix' in name..."
log "Deleting resource groups that comtain '$prefix' in name..."
[[ -n $prefix ]] &&
az group list --query "[?contains(name,'$prefix') && ! contains(name,'dbw')].name" -o tsv |
xargs -I % az group delete --verbose --name % -y
else
echo "Delete resource group that contain '$prefix-$DEPLOYMENT_ID' in name..."
log "Deleting resource groups that contain '$prefix-$DEPLOYMENT_ID' in name..."
[[ -n $prefix ]] &&
az group list --query "[?contains(name,'$prefix-$DEPLOYMENT_ID') && ! contains(name,'dbw')].name" -o tsv |
xargs -I % az group delete --verbose --name % -y
Expand All @@ -120,19 +120,19 @@ delete_all(){

if [[ -z "$DEPLOYMENT_ID" ]]
then
echo "No deployment id [DEPLOYMENT_ID] specified. You will only be able to delete by prefix $prefix..."
log "No deployment id [DEPLOYMENT_ID] specified. You will only be able to delete by prefix $prefix..."
response=3
else
read -r -p "Do you wish to DELETE by"$'\n'" 1) ONLY BY PREFIX ($prefix)?"$'\n'" 2) PREFIX ($prefix) AND DEPLOYMENT_ID ($DEPLOYMENT_ID)?"$'\n'" Choose 1 or 2: " response
fi

case "$response" in
1)
echo "Delete by prefix..."
log "Deleting by prefix..."
delete_all $prefix
;;
2)
echo "Delete by deployment id..."
log "Deleting by deployment id..."
delete_all $prefix $DEPLOYMENT_ID
;;
3)
Expand All @@ -147,7 +147,7 @@ case "$response" in
esac
;;
*)
echo "Invalid choice. Exiting..."
log "Invalid choice. Exiting..." "warning"
exit
;;
esac
102 changes: 79 additions & 23 deletions e2e_samples/parking_sensors/scripts/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,52 +14,73 @@ random_str() {
}

print_style () {
case "$2" in
"info")
COLOR="96m"
;;
"success")
COLOR="92m"
;;
"warning")
COLOR="93m"
;;
"danger")
COLOR="91m"
;;
"action")
COLOR="32m"
;;
*)
COLOR="0m"
;;
esac

if [ "$2" == "info" ] ; then
COLOR="96m";
elif [ "$2" == "success" ] ; then
COLOR="92m";
elif [ "$2" == "warning" ] ; then
COLOR="93m";
elif [ "$2" == "danger" ] ; then
COLOR="91m";
else #default color
COLOR="0m";
fi

STARTCOLOR="\e[$COLOR";
ENDCOLOR="\e[0m";
STARTCOLOR="\e[$COLOR"
ENDCOLOR="\e[0m"

printf "$STARTCOLOR%b$ENDCOLOR" "$1";
printf "$STARTCOLOR%b$ENDCOLOR" "$1"
}

log() {
# This function takes a string as an argument and prints it to the console to stderr
# if a second argument is provided, it will be used as the style of the message
# Usage: log "message" "style"
# Example: log "Hello, World!" "info"
local message=$1
local style=${2:-}

deletePipelineIfExists() {
declare pipeline_name=$1
full_pipeline_name=$PROJECT-$pipeline_name
if [[ -z "$style" ]]; then
echo -e "$(print_style "$message" "default")" >&2
else
echo -e "$(print_style "$message" "$style")" >&2
fi
}

delete_azdo_pipeline_if_exists() {
declare full_pipeline_name=$1

## when returning a pipeline that does exist, delete.

pipeline_output=$(az pipelines list --query "[?name=='$full_pipeline_name']" --output json)
pipeline_id=$(echo "$pipeline_output" | jq -r '.[0].id')

if [[ -z "$pipeline_id" || "$pipeline_id" == "null" ]]; then
echo "Pipeline $full_pipeline_name does not exist.Creating..."
log "No Deployment pipeline with name $full_pipeline_name found."
else
az pipelines delete --id "$pipeline_id" --yes 1>/dev/null
echo "Deleted existing pipeline: $full_pipeline_name (Pipeline ID: $pipeline_id)"

log "Deleted existing pipeline: $full_pipeline_name (Pipeline ID: $pipeline_id)"
fi
}

createPipeline ()
create_azdo_pipeline ()
{
declare pipeline_name=$1
declare pipeline_description=$2
full_pipeline_name=$PROJECT-$pipeline_name

delete_azdo_pipeline_if_exists "$full_pipeline_name"
log "Creating deployment pipeline: $full_pipeline_name"


pipeline_id=$(az pipelines create \
--name "$full_pipeline_name" \
--description "$pipeline_description" \
Expand All @@ -70,5 +91,40 @@ createPipeline ()
--skip-first-run true \
--output json | jq -r '.id')
echo "$pipeline_id"
}

databricks_cluster_exists () {
declare cluster_name="$1"
declare cluster=$(databricks clusters list | tr -s " " | cut -d" " -f2 | grep ^${cluster_name}$)
if [[ -n $cluster ]]; then
return 0; # cluster exists
else
return 1; # cluster does not exists
fi
}


create_adf_linked_service () {
declare name=$1
log "Creating ADF LinkedService: $name"
adfLsUrl="${adfFactoryBaseUrl}/linkedservices/${name}?api-version=${apiVersion}"
az rest --method put --uri "$adfLsUrl" --body @"${ADF_DIR}"/linkedService/"${name}".json
}
create_adf_dataset () {
declare name=$1
log "Creating ADF Dataset: $name"
adfDsUrl="${adfFactoryBaseUrl}/datasets/${name}?api-version=${apiVersion}"
az rest --method put --uri "$adfDsUrl" --body @"${ADF_DIR}"/dataset/"${name}".json
}
create_adf_pipeline () {
declare name=$1
log "Creating ADF Pipeline: $name"
adfPUrl="${adfFactoryBaseUrl}/pipelines/${name}?api-version=${apiVersion}"
az rest --method put --uri "$adfPUrl" --body @"${ADF_DIR}"/pipeline/"${name}".json
}
create_adf_trigger () {
declare name=$1
log "Creating ADF Trigger: $name"
adfTUrl="${adfFactoryBaseUrl}/triggers/${name}?api-version=${apiVersion}"
az rest --method put --uri "$adfTUrl" --body @"${ADF_DIR}"/trigger/"${name}".json
}
Loading