{Package} Migrate CLI packaging system to PEP420 (Azure#14372)

* init pep420 change

* fix linter error

* fix another linter error

* elegant way to handle __main__.py

* remove nspkg

* refactor azure_bdist_wheel to fit wheel 0.34.2

* add namespace package __init__.py to sdist

* fix merge conflict

* fix generic update problem

* fix merge conflict

* upgrade telemetry version

* upgrade telemetry version

* rollback some useless change

* build yum/deb/homebrew etx

* use template to generate formula

* update build.cmd

* update build scripts

* upgrade azure-mutliapi-storage to 0.3.6

* fix debian build script

* fix merge conflict

* init pep420 change

* fix linter error

* fix another linter error

* elegant way to handle __main__.py

* remove nspkg

* refactor azure_bdist_wheel to fit wheel 0.34.2

* add namespace package __init__.py to sdist

* fix merge conflict

* fix generic update problem

* fix merge conflict

* upgrade telemetry version

* upgrade telemetry version

* rollback some useless change

* build yum/deb/homebrew etx

* use template to generate formula

* update build.cmd

* update build scripts

* upgrade azure-mutliapi-storage to 0.3.6

* fix debian build script

* fix merge conflict

* fix merge conflict

* fix generic update test problem

* remove extra __init__.py

* remove __init__.pyc file in debian and rpm

* update build wheel stratages

* update version

* update build scripts

* fix webapp test error

* add __main__.py in sdist

* add test homebrew package scripts

* update version value in __main__.py

* make test_homebrew_package.sh executable

* fix homebrew permission problem

* fix error

* skip some acr tests

* fix style error

* skip config tests

* fix config test MAC error

* fix role config test error

* fix test error

* fix test error

* workaround for vsts bug

* fix test error

* fix homebrew test error

* fix configure test error

* update homebrew package job timeout value

* use released azdev

* rollback pipelines

* rollback pipelines 2

* refactor config test

.azure-pipelines/templates/azdev_setup.yml

@@ -14,7 +14,7 @@ steps:
       chmod +x env/bin/activate
       . env/bin/activate
 
-      pip install -q azdev==0.1.24
+      pip install -q azdev==0.1.25
       azdev --version
 
       if [ -z "$CLI_EXT_REPO_PATH" ]; then

.snapcraft.yaml

@@ -28,7 +28,7 @@ parts:
       - libssl1.0.0
     requirements: requirements.txt
     install: |
-      python_packages="src/azure-cli-core src/azure-cli-nspkg src/azure-cli-command_modules-nspkg src/command_modules/azure-cli*"
+      python_packages="src/azure-cli-core src/command_modules/azure-cli*"
       export PYTHONUSERBASE=$SNAPCRAFT_PART_INSTALL
       export PYTHONHOME=$SNAPCRAFT_PART_INSTALL/usr
       echo $SNAPCRAFT_PART_INSTALL/usr/bin/python3

Dockerfile

@@ -50,7 +50,6 @@ COPY . /azure-cli
 
 # 1. Build packages and store in tmp dir
 # 2. Install the cli and the other command modules that weren't included
-# 3. Temporary fix - install azure-nspkg to remove import of pkg_resources in azure/__init__.py (to improve performance)
 RUN ./scripts/install_full.sh \
  && cat /azure-cli/az.completion > ~/.bashrc \
  && runDeps="$( \

Dockerfile.nanoserver

@@ -78,7 +78,6 @@ foreach($m in $modules) \
 { \
     & pip install --no-cache-dir $m; \
 }\
-& pip install --no-cache-dir --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg; \
 Pop-Location; \
 Write-Host 'Done'; 
 

Dockerfile.spot

@@ -33,15 +33,13 @@ COPY . /azure-cli
 # jmespath-terminal - we include jpterm as a useful tool
 # 1. Build packages and store in tmp dir
 # 2. Install the cli and the other command modules that weren't included
-# 3. Temporary fix - install azure-nspkg to remove import of pkg_resources in azure/__init__.py (to improve performance)
 RUN /bin/bash -c 'TMP_PKG_DIR=$(mktemp -d); \
-    for d in src/azure-cli src/azure-cli-core src/azure-cli-nspkg src/azure-cli-command_modules-nspkg src/command_modules/azure-cli-*/; \
+    for d in src/azure-cli src/azure-cli-core src/command_modules/azure-cli-*/; \
     do cd $d; echo $d; python setup.py bdist_wheel -d $TMP_PKG_DIR; cd -; \
     done; \
     [ -d privates ] && cp privates/*.whl $TMP_PKG_DIR; \
     all_modules=`find $TMP_PKG_DIR -name "*.whl"`; \
-    pip install --no-cache-dir $all_modules; \
-    pip install --no-cache-dir --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg;' \
+    pip install --no-cache-dir $all_modules;' \
  && cat /azure-cli/az.completion > ~/.bashrc
 
 WORKDIR /

azure-cli.pyproj

@@ -23,8 +23,6 @@
     <VisualStudioVersion Condition=" '$(VisualStudioVersion)' == '' ">10.0</VisualStudioVersion>
   </PropertyGroup>
   <ItemGroup>
-    <Compile Include="azure-cli-command_modules-nspkg\azure\cli\command_modules\__init__.py" />
-    <Compile Include="azure-cli-command_modules-nspkg\setup.py" />
     <Compile Include="azure-cli-core\azure\cli\core\adal_authentication.py" />
     <Compile Include="azure-cli-core\azure\cli\core\azlogging.py" />
     <Compile Include="azure-cli-core\azure\cli\core\cloud.py" />
@@ -78,8 +76,6 @@
     <Compile Include="azure-cli-core\azure\cli\__init__.py" />
     <Compile Include="azure-cli-core\azure\__init__.py" />
     <Compile Include="azure-cli-core\setup.py" />
-    <Compile Include="azure-cli-nspkg\azure\cli\__init__.py" />
-    <Compile Include="azure-cli-nspkg\setup.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\base.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\checkers.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\decorators.py" />
@@ -785,10 +781,6 @@
     <Compile Include="command_modules\azure-cli-vm\setup.py" />
   </ItemGroup>
   <ItemGroup>
-    <Folder Include="azure-cli-command_modules-nspkg\" />
-    <Folder Include="azure-cli-command_modules-nspkg\azure\" />
-    <Folder Include="azure-cli-command_modules-nspkg\azure\cli\" />
-    <Folder Include="azure-cli-command_modules-nspkg\azure\cli\command_modules\" />
     <Folder Include="azure-cli-core\" />
     <Folder Include="azure-cli-core\azure\" />
     <Folder Include="azure-cli-core\azure\cli\" />
@@ -797,9 +789,6 @@
     <Folder Include="azure-cli-core\azure\cli\core\extensions\" />
     <Folder Include="azure-cli-core\azure\cli\core\profiles\" />
     <Folder Include="azure-cli-core\azure\cli\core\tests\" />
-    <Folder Include="azure-cli-nspkg\" />
-    <Folder Include="azure-cli-nspkg\azure\" />
-    <Folder Include="azure-cli-nspkg\azure\cli\" />
     <Folder Include="azure-cli-testsdk\" />
     <Folder Include="azure-cli-testsdk\azure\" />
     <Folder Include="azure-cli-testsdk\azure\cli\" />
@@ -1130,13 +1119,8 @@
     <Folder Include="command_modules\azure-cli-kusto\" />
   </ItemGroup>
   <ItemGroup>
-    <Content Include="azure-cli-command_modules-nspkg\HISTORY.rst" />
-    <Content Include="azure-cli-command_modules-nspkg\MANIFEST.in" />
-    <Content Include="azure-cli-command_modules-nspkg\README.rst" />
-    <Content Include="azure-cli-command_modules-nspkg\setup.cfg" />
     <Content Include="azure-cli-core\HISTORY.rst" />
     <Content Include="azure-cli-core\setup.cfg" />
-    <Content Include="azure-cli-nspkg\HISTORY.rst" />
     <Content Include="azure-cli-testsdk\HISTORY.rst" />
     <Content Include="azure-cli-testsdk\setup.cfg" />
     <Content Include="azure-cli\az.completion.sh" />

azure-cli2017.pyproj

@@ -23,8 +23,6 @@
     <VisualStudioVersion Condition=" '$(VisualStudioVersion)' == '' ">10.0</VisualStudioVersion>
   </PropertyGroup>
   <ItemGroup>
-    <Compile Include="azure-cli-command_modules-nspkg\azure\cli\command_modules\__init__.py" />
-    <Compile Include="azure-cli-command_modules-nspkg\setup.py" />
     <Compile Include="azure-cli-core\azure\cli\core\adal_authentication.py" />
     <Compile Include="azure-cli-core\azure\cli\core\api.py" />
     <Compile Include="azure-cli-core\azure\cli\core\azlogging.py" />
@@ -90,8 +88,6 @@
     <Compile Include="azure-cli-core\azure\cli\core\telemetry.py" />
     <Compile Include="azure-cli-core\azure\cli\core\__init__.py" />
     <Compile Include="azure-cli-core\setup.py" />
-    <Compile Include="azure-cli-nspkg\azure\cli\__init__.py" />
-    <Compile Include="azure-cli-nspkg\setup.py" />
     <Compile Include="azure-cli-telemetry\azure\cli\telemetry\components\records_collection.py" />
     <Compile Include="azure-cli-telemetry\azure\cli\telemetry\components\records_collection.py" />
     <Compile Include="azure-cli-telemetry\azure\cli\telemetry\components\telemetry_client.py" />
@@ -980,10 +976,6 @@
     <Compile Include="azure-cli\setup.py" />
   </ItemGroup>
   <ItemGroup>
-    <Folder Include="azure-cli-command_modules-nspkg\" />
-    <Folder Include="azure-cli-command_modules-nspkg\azure\" />
-    <Folder Include="azure-cli-command_modules-nspkg\azure\cli\" />
-    <Folder Include="azure-cli-command_modules-nspkg\azure\cli\command_modules\" />
     <Folder Include="azure-cli-core\" />
     <Folder Include="azure-cli-core\azure\" />
     <Folder Include="azure-cli-core\azure\cli\" />
@@ -997,9 +989,6 @@
     <Folder Include="azure-cli-core\azure\cli\core\profiles\" />
     <Folder Include="azure-cli-core\azure\cli\core\tests\" />
     <Folder Include="azure-cli-core\azure\cli\core\tests\data\" />
-    <Folder Include="azure-cli-nspkg\" />
-    <Folder Include="azure-cli-nspkg\azure\" />
-    <Folder Include="azure-cli-nspkg\azure\cli\" />
     <Folder Include="azure-cli-telemetry\" />
     <Folder Include="azure-cli-telemetry\azure\" />
     <Folder Include="azure-cli-telemetry\azure\" />
@@ -1217,10 +1206,6 @@
     <Folder Include="azure-cli\azure\cli\command_modules\vm\tests\latest\keyvault\" />
   </ItemGroup>
   <ItemGroup>
-    <Content Include="azure-cli-command_modules-nspkg\HISTORY.rst" />
-    <Content Include="azure-cli-command_modules-nspkg\MANIFEST.in" />
-    <Content Include="azure-cli-command_modules-nspkg\README.rst" />
-    <Content Include="azure-cli-command_modules-nspkg\setup.cfg" />
     <Content Include="azure-cli-core\azure\cli\core\auth_landing_pages\fail.html" />
     <Content Include="azure-cli-core\azure\cli\core\auth_landing_pages\fail.html" />
     <Content Include="azure-cli-core\azure\cli\core\auth_landing_pages\ok.html" />
@@ -1234,8 +1219,6 @@
     <Content Include="azure-cli-core\azure\cli\core\tests\sp_cert.pem" />
     <Content Include="azure-cli-core\HISTORY.rst" />
     <Content Include="azure-cli-core\setup.cfg" />
-    <Content Include="azure-cli-nspkg\HISTORY.rst" />
-    <Content Include="azure-cli-nspkg\setup.cfg" />
     <Content Include="azure-cli-telemetry\azure\cli\telemetry\tests\resources\cache" />
     <Content Include="azure-cli-telemetry\azure\cli\telemetry\tests\resources\cache.1" />
     <Content Include="azure-cli-telemetry\azure\cli\telemetry\tests\resources\cache.10" />

azure-pipelines.yml

@@ -572,6 +572,33 @@ jobs:
 
     displayName: 'Bash Script'
 
+- job: TestHomebrewPackage
+  displayName: Test Homebrew Package
+  timeoutInMinutes: 120
+  dependsOn: BuildHomebrewFormula
+  condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'))
+  pool:
+    vmImage: 'macOS-10.14'
+  steps:
+  - task: DownloadPipelineArtifact@1
+    displayName: 'Download Metadata'
+    inputs:
+      TargetPath: '$(Build.ArtifactStagingDirectory)/metadata'
+      artifactName: metadata
+
+
+  - task: DownloadPipelineArtifact@1
+    displayName: 'Download Build Artifacts'
+    inputs:
+      TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew'
+      artifactName: homebrew
+
+
+  - bash: ./scripts/release/homebrew/test_homebrew_package.sh
+
+
+    displayName: 'Test Homebrew Package'
+
 - job: BuildYumPackage
   displayName: Build Yum Package
 

build_scripts/windows/scripts/build.cmd

@@ -88,7 +88,7 @@ robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL
 
 set CLI_SRC=%REPO_ROOT%\src
 %BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --force-reinstall pycparser==2.18
-for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-nspkg %CLI_SRC%\azure-cli-telemetry) do (
+for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-telemetry) do (
    pushd %%a
    %BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --no-cache-dir --no-deps .
    popd
@@ -97,8 +97,6 @@ for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-nsp
 
 if %errorlevel% neq 0 goto ERROR
 
-%BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg
-
 pushd %BUILDING_DIR%
 %BUILDING_DIR%\python.exe %~dp0\patch_models_v2.py
 popd

doc/authoring_command_modules/README.md

@@ -55,10 +55,6 @@ Also, you can run `az` and if your command module contributes any commands, they
 If your commands aren't showing with `az`, use `az --debug` to help debug. There could have been an exception
 thrown whilst attempting to load your module.
 
-**Note for locally installed Azure SDKs:**  
-If you're receiving Azure import errors, run `pip install -I azure-nspkg==1.0.0` in your virtual environment
-after installing your SDK locally.
-
 
 <a name="heading_author_command_mod"></a>Authoring command modules
 ------

scripts/ci/version.sh

@@ -20,6 +20,14 @@ echo "Replace with version: $version"
 
 platform=`uname`
 
+for each in $(find src -name __main__.py); do
+    if [ "$platform" == "Darwin" ]; then
+        sed -i "" "s/^__version__ = [\"']\(.*\)[\"']/__version__ = \"\1.dev$version\"/" $each
+    else
+        sed -i "s/^__version__ = [\"']\(.*\)[\"']/__version__ = \"\1.dev$version\"/" $each
+    fi
+done
+
 for each in $(find src -name __init__.py); do
     if [ "$platform" == "Darwin" ]; then
         sed -i "" "s/^__version__ = [\"']\(.*\)[\"']/__version__ = \"\1.dev$version\"/" $each

scripts/curl_install_pypi/install.py

@@ -149,10 +149,6 @@ def install_cli(install_dir, tmp_dir):
     path_to_pip = os.path.join(install_dir, 'bin', 'pip')
     cmd = [path_to_pip, 'install', '--cache-dir', tmp_dir, 'azure-cli', '--upgrade']
     exec_command(cmd)
-    # Temporary fix to make sure that we have empty __init__.py files for the azure site-packages folder.
-    # (including the pkg_resources/declare namespace significantly impacts startup perf for the CLI)
-    fixupcmd = [path_to_pip, 'install', '--cache-dir', tmp_dir, '--upgrade', '--force-reinstall', 'azure-nspkg', 'azure-mgmt-nspkg']
-    exec_command(fixupcmd)
 
 
 def create_executable(exec_dir, install_dir):

scripts/dev_setup.py

@@ -66,7 +66,6 @@ def pip_command(command):
 pip_command('install -e ./tools')
 
 # command modules have dependency on azure-cli-core so install this first
-pip_command('install -e src/azure-cli-nspkg')
 pip_command('install -e src/azure-cli-telemetry')
 pip_command('install -e src/azure-cli-core')
 py_command('-m automation.setup.install_modules')
@@ -75,11 +74,6 @@ def pip_command(command):
 pip_command('install -e src/azure-cli')
 pip_command('install -e src/azure-cli-testsdk')
 
-# Ensure that the site package's azure/__init__.py has the old style namespace
-# package declaration by installing the old namespace package
-pip_command('install --force-reinstall azure-nspkg==1.0.0')
-pip_command('install --force-reinstall azure-mgmt-nspkg==1.0.0')
-
 print_support_message()
 print('Finished dev setup.')
 

scripts/release/debian/Dockerfile

@@ -29,7 +29,6 @@ RUN mkdir -p ./bin/pypi && \
     BUILD_STAGINGDIRECTORY=/azure-cli/bin/pypi ./scripts/release/pypi/build.sh && \
     if [ -d ./privates ]; then find ./privates -name '*.whl' | xargs pip install; fi && \
     find ./bin/pypi -name '*.whl' | xargs pip install && \
-    pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg && \
     pip install --force-reinstall urllib3==1.24.2
 
 ARG cli_version=0.0.0-dev

scripts/release/debian/pipeline.sh

@@ -8,7 +8,7 @@ set -exv
 : ${DISTRO:?"DISTRO is not set"}
 : ${DISTRO_BASE_IMAGE:?"DISTRO_BASE_IMAGE is not set"}
 
-CLI_VERSION=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
+CLI_VERSION=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
 
 docker run --rm \
            -v "$BUILD_SOURCESDIRECTORY":/mnt/repo \

scripts/release/docker/pipeline.sh

@@ -1,5 +1,5 @@
 IMAGE_NAME=clibuild$BUILD_BUILDNUMBER
-CLI_VERSION=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
+CLI_VERSION=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
 
 docker build --no-cache \
              --build-arg BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`" \

scripts/release/get_version.sh

@@ -1,7 +1,7 @@
-# Extract the version of the CLI from azure-cli package's __init__.py file.
+# Extract the version of the CLI from azure-cli package's __main__.py file.
 
 : "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set}"
 
-ver=`cat src/azure-cli/azure/cli/__init__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
+ver=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
 echo $ver > $BUILD_STAGINGDIRECTORY/version
 echo $ver > $BUILD_STAGINGDIRECTORY/azure-cli-${ver}.txt

scripts/release/homebrew/docker/formula_template.txt

@@ -6,13 +6,20 @@ class AzureCli < Formula
   url "{{ upstream_url }}"
   version "{{ cli_version }}"
   sha256 "{{ upstream_sha }}"
+  license "MIT"
   head "https://github.com/Azure/azure-cli.git"
 
 {{ bottle_hash }}
 
   depends_on "openssl@1.1"
   depends_on "python@3.8"
 
+  uses_from_macos "libffi"
+
+  on_linux do
+    depends_on "pkg-config" => :build
+  end
+
 {{ resources }}
 
   def install
@@ -28,8 +35,6 @@ class AzureCli < Formula
       buildpath/"src/azure-cli",
       buildpath/"src/azure-cli-telemetry",
       buildpath/"src/azure-cli-core",
-      buildpath/"src/azure-cli-nspkg",
-      buildpath/"src/azure-cli-command_modules-nspkg",
     ]
 
     # Install CLI

scripts/release/homebrew/docker/run.sh

@@ -10,4 +10,4 @@ pip install -r /mnt/src/azure-cli/requirements.py3.Darwin.txt
 
 pip list
 
-python $root/formula_generate.py
+python $root/formula_generate.py -b use_template

scripts/release/homebrew/test_homebrew_package.py

@@ -0,0 +1,34 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import sys
+import subprocess
+
+if len(sys.argv) < 2:
+    print('AZ_BASE or PYTHON_VERSION are not specified')
+    sys.exit(-1)
+
+az_base = sys.argv[1]
+python_version = sys.argv[2]
+
+root_dir = '{}/lib/{}/site-packages/azure/cli/command_modules'.format(az_base, python_version)
+mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__']
+
+pytest_base_cmd = 'PYTHONPATH={}/lib/{}/site-packages python -m pytest -x -v --boxed -p no:warnings --log-level=WARN'.format(az_base, python_version)
+pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd)
+
+for mod_name in mod_list:
+    if mod_name in ['botservice', 'network', 'configure']:
+        exit_code = subprocess.call(['{} --junit-xml ./azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_base_cmd, mod_name, mod_name)], shell=True)
+    else:
+        exit_code = subprocess.call(['{} --junit-xml ./azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_parallel_cmd, mod_name, mod_name)], shell=True)
+    if exit_code == 5:
+        print('No tests found for {}'.format(mod_name))
+    elif exit_code != 0:
+        sys.exit(exit_code)
+
+exit_code = subprocess.call(['{} --junit-xml ./azure_cli_test_result/azure-cli-core.xml --pyargs azure.cli.core'.format(pytest_base_cmd)], shell=True)
+sys.exit(exit_code)