-
Notifications
You must be signed in to change notification settings - Fork 32
/
Copy pathvdbench_save.yml
206 lines (183 loc) · 9.79 KB
/
vdbench_save.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
# Copyright (C) Microsoft Corporation. All rights reserved.
# https://aka.ms/yaml
parameters:
- name: vdbench_config
type: string
steps:
- script: |
export PATH=$PATH:${BUILD_SOURCESDIRECTORY}/test/utils
az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID &&
az account set -s $ARM_SUBSCRIPTION_ID
check_rc.sh $? "Login to Azure and set default subscription" || exit 0
# Wait for the vdbench.py script to complete.
cmdokmsg="CMD_OK_$(date -u '+%m%dx%H%M%S')"
cmdlog=/tmp/vm_runcmd.log
polling_secs=60
while [ 1 ]; do
az vmss run-command invoke \
--resource-group ${RESOURCE_GROUP_NAME}-vmss \
--name "vmss" \
--instance-id 0 \
--command-id RunShellScript \
--scripts "runuser -l azureuser -c 'tmux ls' || echo ${cmdokmsg}" \
--query 'value[0].message' --output tsv |& tee ${cmdlog}
check_rc.sh ${PIPESTATUS[0]} "Checking for any tmux session " || exit
grep -q ${cmdokmsg} ${cmdlog}
if [ $? -ne 0 ]; then
echo "INFO: tmux session still active. Re-check in ${polling_secs} seconds."
sleep ${polling_secs}
continue
else
echo "INFO: tmux session no longer active"
break
fi
done
cmdokmsg="CMD_OK_$(date -u '+%m%dx%H%M%S')"
cmdlog=/tmp/vm_runcmd_tail.log
az vmss run-command invoke \
--resource-group ${RESOURCE_GROUP_NAME}-vmss \
--name "vmss" \
--instance-id 0 \
--command-id RunShellScript \
--scripts "runuser -l azureuser -c 'tail -25 /home/azureuser/vdb.log' && echo ${cmdokmsg}" \
--query 'value[0].message' --output tsv |& tee ${cmdlog}
grep -q ${cmdokmsg} ${cmdlog} || check_rc.sh 1 "Tail vdbench.log"
grep -q 'Vdbench execution completed successfully' ${cmdlog}
grep_rc=$?
check_rc.sh ${grep_rc} "vdbench.py success check"
displayName: "Check for vdbench.py completion"
timeoutInMinutes: 45
condition: not(or(failed(), canceled()))
env:
ARM_CLIENT_ID: $(AZURE-CLIENT-ID)
ARM_CLIENT_SECRET: $(AZURE-CLIENT-SECRET)
ARM_TENANT_ID: $(AZURE-TENANT-ID)
ARM_SUBSCRIPTION_ID: $(AZURE-SUBSCRIPTION-ID)
- bash: |
export PATH=$PATH:${BUILD_SOURCESDIRECTORY}/test/utils
az network vnet subnet update \
--resource-group ${RESOURCE_GROUP_NAME}-network \
--vnet-name rendervnet \
--name render_clients1 \
--disable-private-endpoint-network-policies true
check_rc.sh $? "Disable private endpoint network policies on render_clients1 subnet" || exit 0
export STOR_ACCT_RESOURCE_ID="/subscriptions/${ARM_SUBSCRIPTION_ID}/resourceGroups/${AZURE_STORAGE_ACCOUNT_RG_NAME}/providers/Microsoft.Storage/storageAccounts/${AZURE_STORAGE_ACCOUNT_NAME}"
echo "##vso[task.setvariable variable=STOR_ACCT_RESOURCE_ID]${STOR_ACCT_RESOURCE_ID}"
az deployment group create \
-f ${BUILD_SOURCESDIRECTORY}/templates/private_endpoint.json \
--resource-group ${RESOURCE_GROUP_NAME}-network \
-p location=${REGION} \
-p privateEndpointName=${RESOURCE_GROUP_NAME}-privendpt \
-p privateLinkResourceId=${STOR_ACCT_RESOURCE_ID} \
-p subnet=render_clients1 \
-p vnet=rendervnet \
-p subscriptionId=${ARM_SUBSCRIPTION_ID}
check_rc.sh $? "Create Private Endpoint" || exit 0
displayName: "Setup Private Endpoint for uploading vdbench results to Storage Account."
condition: not(or(failed(), canceled()))
env:
ARM_CLIENT_ID: $(AZURE-CLIENT-ID)
ARM_CLIENT_SECRET: $(AZURE-CLIENT-SECRET)
ARM_TENANT_ID: $(AZURE-TENANT-ID)
ARM_SUBSCRIPTION_ID: $(AZURE-SUBSCRIPTION-ID)
AZURE_STORAGE_ACCOUNT_NAME: $(AZURE-STORAGE-ACCOUNT-NAME)
AZURE_STORAGE_ACCOUNT_RG_NAME: $(AZURE-STORAGE-ACCOUNT-RG-NAME)
- script: |
export PATH=$PATH:${BUILD_SOURCESDIRECTORY}/test/utils
end_timestamp=$(date -u -d "60 minutes" '+%Y-%m-%dT%H:%MZ')
dest_storacct_key=$(az storage account keys list \
--account-name ${AZURE_STORAGE_ACCOUNT_NAME} \
--resource-group ${AZURE_STORAGE_ACCOUNT_RG_NAME} \
--subscription $ARM_SUBSCRIPTION_ID --query "[1].value" --output tsv)
rc=$?
echo "##vso[task.setvariable variable=dest_storacct_key;issecret=true]${dest_storacct_key}"
check_rc.sh $rc "Get destination Storage Account key" || exit 0
dest_container_sas=$(az storage container generate-sas \
--account-key "${dest_storacct_key}" \
--account-name ${AZURE_STORAGE_ACCOUNT_NAME} \
--name ${AZURE_STORAGE_ACCOUNT_CONTAINER_NAME} \
--expiry "${end_timestamp}" \
--permissions clrw --https-only --output tsv)
rc=$?
echo "##vso[task.setvariable variable=dest_container_sas;issecret=true]${dest_container_sas}"
check_rc.sh $rc "Generate destination SAS URL" || exit 0
sas_url="https://URI_TO_AZURE_STORAGE_ACCOUNT/${AZURE_STORAGE_ACCOUNT_CONTAINER_NAME}?${dest_container_sas}"
echo "##vso[task.setvariable variable=sas_url;issecret=true]${sas_url}"
cmdokmsg="CMD_OK_$(date -u '+%m%dx%H%M%S')"
cmdlog=/tmp/vm_runcmd_install_azcopy.log
az vmss run-command invoke \
--resource-group ${RESOURCE_GROUP_NAME}-vmss \
--name "vmss" \
--instance-id 0 \
--command-id RunShellScript \
--scripts "runuser -l azureuser -c 'wget https://aka.ms/downloadazcopy-v10-linux && tar -xvf downloadazcopy-v10-linux && sudo cp ./azcopy_linux_amd64_*/azcopy /usr/bin/' && echo ${cmdokmsg}" \
--query 'value[0].message' --output tsv |& tee ${cmdlog}
grep -q ${cmdokmsg} ${cmdlog} || check_rc.sh 1 "Install azcopy"
cmdokmsg="CMD_OK_$(date -u '+%m%dx%H%M%S')"
cmdlog=/tmp/vm_runcmd_tar.log
az vmss run-command invoke \
--resource-group ${RESOURCE_GROUP_NAME}-vmss \
--name "vmss" \
--instance-id 0 \
--command-id RunShellScript \
--scripts "runuser -l azureuser -c 'tar -c -f ${RESOURCE_GROUP_NAME}-${{ parameters.vdbench_config }}.tar /home/azureuser/vdbench/output/' && echo ${cmdokmsg}" \
--query 'value[0].message' --output tsv |& tee ${cmdlog}
grep -q ${cmdokmsg} ${cmdlog} || check_rc.sh 1 "Tar vdbench output directory"
cmdokmsg="CMD_OK_$(date -u '+%m%dx%H%M%S')"
cmdlog=/tmp/vm_runcmd_azcopy.log
az vmss run-command invoke \
--resource-group ${RESOURCE_GROUP_NAME}-vmss \
--name "vmss" \
--instance-id 0 \
--command-id RunShellScript \
--scripts "azcopy copy '/home/azureuser/${RESOURCE_GROUP_NAME}-${{ parameters.vdbench_config }}.tar' '${sas_url}' && echo ${cmdokmsg}" \
--query 'value[0].message' --output tsv |& tee ${cmdlog}
grep -q ${cmdokmsg} ${cmdlog} || check_rc.sh 1 "Run azcopy .tar to storage account"
az resource lock delete -n DO_NOT_DELETE --resource ${STOR_ACCT_RESOURCE_ID}
check_rc.sh $rc "Delete Lock on storage account" || exit 0
# Private endpoint removal needed so terraform destory will work.
az network private-endpoint dns-zone-group delete \
--endpoint-name ${RESOURCE_GROUP_NAME}-privendpt \
--name "privatelink.blob.core.windows.net" \
--resource-group ${RESOURCE_GROUP_NAME}-network
check_rc.sh $? "DNS-Zone delete" || exit 0
# TODO: This assumes that there is only 1 lock. If there are more, this will fail from timeout.
while [ 1 ]; do
cmdlog=/tmp/private_endpt_delete.log
# Private endpoint removal needed so terraform destory will work.
az network private-endpoint delete --name ${RESOURCE_GROUP_NAME}-privendpt \
--resource-group ${RESOURCE_GROUP_NAME}-network |& tee ${cmdlog}
rc=${PIPESTATUS[0]}
grep -q "remove the lock" ${cmdlog}
if [ $? -eq 0 ]; then
echo "Delete Lock"
az resource lock delete -n DO_NOT_DELETE --resource ${STOR_ACCT_RESOURCE_ID}
check_rc.sh $rc "Delete Lock on storage account" || exit 0
continue
else
check_rc.sh ${rc} "Private Endpoint delete" || exit 0
break
fi
done
az resource lock create --lock-type CanNotDelete \
-n DO_NOT_DELETE \
--resource ${STOR_ACCT_RESOURCE_ID} \
--notes "Delete created by Pipeline BuildID: ${BUILD_BUILDID}"
check_rc.sh $? "Create Lock on storage account" || exit 0
az network vnet subnet update \
--resource-group ${RESOURCE_GROUP_NAME}-network \
--vnet-name rendervnet \
--name render_clients1 \
--disable-private-endpoint-network-policies false
check_rc.sh $? "Enable private endpoint network policies on render_clients1 subnet" || exit 0
displayName: "Add vdbench .tar to Storage Account."
timeoutInMinutes: 30
condition: not(or(failed(), canceled()))
env:
ARM_SUBSCRIPTION_ID: $(AZURE-SUBSCRIPTION-ID)
AZURE_STORAGE_ACCOUNT_NAME: $(AZURE-STORAGE-ACCOUNT-NAME)
AZURE_STORAGE_ACCOUNT_RG_NAME: $(AZURE-STORAGE-ACCOUNT-RG-NAME)
AZURE_STORAGE_ACCOUNT_CONTAINER_NAME: $(AZURE-STORAGE-ACCOUNT-CONTAINER-NAME)
- bash: echo "##vso[build.updatebuildnumber]${BUILD_BUILDNUMBER} (vdbench, ${RESOURCE_GROUP_NAME}-${{ parameters.vdbench_config }}.tar)"
displayName: "Add .tar name with vdbench config "
condition: not(or(failed(), canceled()))