update

scripts/ci/sync_extensions.py

@@ -10,6 +10,8 @@
 import json
 import subprocess
 
+from util import run_az_cmd
+
 DEFAULT_TARGET_INDEX_URL = os.getenv('AZURE_EXTENSION_TARGET_INDEX_URL')
 STORAGE_ACCOUNT = os.getenv('AZURE_EXTENSION_TARGET_STORAGE_ACCOUNT')
 STORAGE_CONTAINER = os.getenv('AZURE_EXTENSION_TARGET_STORAGE_CONTAINER')
@@ -48,7 +50,6 @@ def download_file(url, file_path):
             if chunk:  # ignore keep-alive new chunks
                 f.write(chunk)
 
-
 def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir):
     download_url = ext['downloadUrl']
     whl_file = download_url.split('/')[-1]
@@ -62,29 +63,22 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir):
     if not overwrite:
         cmd = ['az', 'storage', 'blob', 'exists', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
                f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login']
-        try:
-            result = subprocess.run(cmd, capture_output=True, check=True)
-        except subprocess.CalledProcessError as ex:
-            raise Exception(f"Failed to check if '{blob_name}' exists in the storage") from ex
+        message = f"Checking if '{blob_name}' exists in the storage"
+        result = run_az_cmd(cmd, message=message, raise_error=True)
         if json.loads(result.stdout)['exists']:
             print("Skipping '{}' as it already exists...".format(whl_file))
             return
 
     cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
            f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--file', f'{os.path.abspath(whl_path)}',
            '--auth-mode', 'login', '--overwrite']
-    try:
-        subprocess.run(cmd, capture_output=True, check=True)
-    except subprocess.CalledProcessError as ex:
-        raise Exception(f"Failed to upload '{blob_name}' to the storage") from ex
+    message = f"uploading '{blob_name}' to the storage"
+    run_az_cmd(cmd, message=message, raise_error=True)
 
     cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
            f'{STORAGE_ACCOUNT}', '--name',  f'{blob_name}', '--auth-mode', 'login']
-    try:
-        result = subprocess.run(cmd, capture_output=True, check=True)
-        url = json.loads(result.stdout)
-    except subprocess.CalledProcessError as ex:
-        raise Exception(f"Failed to get the URL for '{blob_name}'") from ex
+    message = f"Getting the URL for '{blob_name}'"
+    result = run_az_cmd(cmd, message=message, raise_error=True)
 
     updated_index = ext
     updated_index['downloadUrl'] = url
@@ -160,10 +154,8 @@ def main():
         cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
                f'{STORAGE_ACCOUNT}', '--name', f'{backup_index_name}',
                '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login', '--overwrite']
-        try:
-            subprocess.run(cmd, capture_output=True, check=True)
-        except subprocess.CalledProcessError as ex:
-            raise Exception(f"Failed to upload '{backup_index_name}' to the storage account") from ex
+        message = f"Uploading '{backup_index_name}' to the storage"
+        run_az_cmd(cmd, message=message, raise_error=True)
 
         # start with an empty index.json to sync all extensions
         initial_index = {"extensions": {}, "formatVersion": "1"}
@@ -189,10 +181,8 @@ def main():
     cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
            f'{STORAGE_ACCOUNT}', '--name', f'{index_name}', '--file', f'{os.path.abspath(target_index_path)}',
            '--auth-mode', 'login', '--overwrite']
-    try:
-        subprocess.run(cmd, capture_output=True, check=True)
-    except subprocess.CalledProcessError as ex:
-        raise Exception(f"Failed to upload '{index_name}' to the storage account") from ex
+    message = f"Uploading '{index_name}' to the storage"
+    run_az_cmd(cmd, message=message, raise_error=True)
 
     print("\nSync finished.")
     if updated_indexes:

scripts/ci/update_ext_cmd_tree.py

@@ -6,13 +6,14 @@
 import filecmp
 import json
 import os
-import subprocess
 import sys
+
 from azure.cli.core import get_default_cli
 from azure.cli.core._session import Session
 from azure.cli.core.commands import _load_extension_command_loader
 from azure.cli.core.extension import get_extension_modname, get_extension_path
 from sync_extensions import download_file
+from util import run_az_cmd
 
 STORAGE_ACCOUNT = os.getenv('AZURE_EXTENSION_CMD_TREE_STORAGE_ACCOUNT')
 STORAGE_CONTAINER = os.getenv('AZURE_EXTENSION_CMD_TREE_STORAGE_CONTAINER')
@@ -83,18 +84,14 @@ def upload_cmd_tree():
     cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
            f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--file', f'{file_path}', '--auth-mode', 'login',
            '--overwrite']
-    try:
-        subprocess.run(cmd, capture_output=True, check=True)
-    except subprocess.CalledProcessError as ex:
-        raise Exception(f"Failed to upload '{blob_file_name}' to the storage account") from ex
+    message = f"Uploading '{blob_file_name}' to the storage"
+    run_az_cmd(cmd, message=message, raise_error=True)
 
     cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name',
            f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--auth-mode', 'login']
-    try:
-        result = subprocess.run(cmd, capture_output=True, check=True)
-        url = json.loads(result.stdout)
-    except subprocess.CalledProcessError as ex:
-        raise Exception(f"Failed to get the URL for '{blob_file_name}'") from ex
+    message = f"Getting the URL for '{blob_file_name}'"
+    result = run_az_cmd(cmd, message=message, raise_error=True)
+    url = json.loads(result.stdout)
 
     download_file_path = os.path.expanduser(os.path.join('~', '.azure', downloaded_file_name))
     download_file(url, download_file_path)

scripts/ci/util.py

@@ -3,15 +3,14 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
+import json
 import logging
 import os
 import re
 import shlex
-import json
+import subprocess
 import zipfile
 
-from subprocess import check_output
-
 logger = logging.getLogger(__name__)
 
 # copy from wheel==0.30.0
@@ -136,7 +135,7 @@ def diff_code(start, end):
 
         # If running in Travis CI, only run tests for edited extensions
         commit_range = os.environ.get('TRAVIS_COMMIT_RANGE')
-        if commit_range and not check_output(
+        if commit_range and not subprocess.check_output(
                 ['git', '--no-pager', 'diff', '--name-only', commit_range, '--', src_d_full]):
             continue
 
@@ -154,7 +153,7 @@ def diff_code(start, end):
             else:
                 cmd = cmd_tpl.format(start=start, end=end,
                                      code_dir=src_d_full)
-                if not check_output(shlex.split(cmd)):
+                if not subprocess.check_output(shlex.split(cmd)):
                     continue
 
         diff_ref.append((pkg_name, src_d_full))
@@ -163,3 +162,23 @@ def diff_code(start, end):
                    f'end: {end}, '
                    f'diff_ref: {diff_ref}.')
     return diff_ref
+
+def run_az_cmd(cmd, message=False, raise_error=True):
+    """
+    :param cmd: The entire command line to run.
+    :param message: A custom message to print, or True (bool) to use a default.
+    :param raise_error: Whether to raise an exception if the command fails.
+    """
+     # use default message if custom not provided
+    if message is True:
+        print(f'Running: {cmd}')
+
+    if message:
+        print(f'{message}')
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, check=True)
+    except subprocess.CalledProcessError as ex:
+        if raise_error:
+            raise Exception(f"Failed to run command: {cmd}") from ex
+        return result