Support local builds for all installers (Docker, MSI, Pip, Debian, RP…

…M) (Azure#4696)

- packaged_releases -> build_scripts
- Docker: build Docker image from src code directory, support private SDKs
- Debian: local builds
- MSI: local builds
- remove old scripts
- RPM: local builds

.dockerignore

@@ -2,6 +2,10 @@
 *
 # Make an exception for azure-cli source code
 !src/*
+# Make an exception for private SDKs if they exist
+!privates/*
+# Make an exception for tab completion script
+!az.completion
 # Make an exception for the license file
 !LICENSE.txt
 # Exclude build droppings, as mentioned in .gitignore

.github/CODEOWNERS

@@ -4,7 +4,6 @@
 
 /tools/ @troydai @derekbekoe
 /scripts/ @troydai @derekbekoe
-/packaged_releases/ @derekbekoe
 /src/azure-cli-testsdk/ @troydai
 /src/command_modules/azure-cli-acr/ @djyou
 /src/command_modules/azure-cli-acs/ @rjtsdl

.gitignore

@@ -14,7 +14,7 @@ env27/
 obj/
 dist/
 MANIFEST
-packaged_releases/windows/out
+build_scripts/windows/out
 
 # Result of running python setup.py install/pip install -e
 RECORD.txt

Dockerfile

@@ -3,12 +3,24 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 #---------------------------------------------------------------------------------------------
 
-# This Dockerfile uses the latest code from the Git repo.
-#   Clone the repo then run 'docker build' with this Dockerfile file to get the latest versions of
-#   *all* CLI modules as in the Git repo.
-
 FROM python:3.5.2-alpine
 
+ARG CLI_VERSION
+
+# Metadata as defined at http://label-schema.org
+ARG BUILD_DATE
+LABEL org.label-schema.schema-version="1.0" \
+      org.label-schema.vendor="Microsoft" \
+      org.label-schema.name="Azure CLI 2.0" \
+      org.label-schema.version=$CLI_VERSION \
+      org.label-schema.license="MIT" \
+      org.label-schema.description="The Azure CLI 2.0 is the new Azure CLI and is applicable when you use the Resource Manager deployment model." \
+      org.label-schema.url="https://docs.microsoft.com/en-us/cli/azure/overview" \
+      org.label-schema.usage="https://docs.microsoft.com/en-us/cli/azure/install-az-cli2#docker" \
+      org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.vcs-url="https://github.com/Azure/azure-cli.git" \
+      org.label-schema.docker.cmd="docker run -v ${HOME}:/root -it azuresdk/azure-cli-python:<version>"
+
 WORKDIR azure-cli
 COPY . /azure-cli
 # pip wheel - required for CLI packaging
@@ -29,26 +41,15 @@ RUN wget https://github.com/jmespath/jp/releases/download/0.1.2/jp-linux-amd64 -
 # 3. Temporary fix - install azure-nspkg to remove import of pkg_resources in azure/__init__.py (to improve performance)
 RUN /bin/bash -c 'TMP_PKG_DIR=$(mktemp -d); \
     for d in src/azure-cli src/azure-cli-core src/azure-cli-nspkg src/azure-cli-command_modules-nspkg src/command_modules/azure-cli-*/; \
-    do cd $d; python setup.py bdist_wheel -d $TMP_PKG_DIR; cd -; \
-    done; \
-    MODULE_NAMES=""; \
-    for f in $TMP_PKG_DIR/*; \
-    do MODULE_NAMES="$MODULE_NAMES $f"; \
+    do cd $d; echo $d; python setup.py bdist_wheel -d $TMP_PKG_DIR; cd -; \
     done; \
-    pip install --no-cache-dir $MODULE_NAMES; \
+    [ -d privates ] && cp privates/*.whl $TMP_PKG_DIR; \
+    all_modules=`find $TMP_PKG_DIR -name "*.whl"`; \
+    pip install --no-cache-dir $all_modules; \
     pip install --no-cache-dir --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg;'
 
 # Tab completion
-RUN echo -e "\
-_python_argcomplete() {\n\
-    local IFS='\v'\n\
-    COMPREPLY=( \$(IFS=\"\$IFS\"                   COMP_LINE=\"\$COMP_LINE\"                   COMP_POINT=\"\$COMP_POINT\"                   _ARGCOMPLETE_COMP_WORDBREAKS=\"\$COMP_WORDBREAKS\"                   _ARGCOMPLETE=1                   \"\$1\" 8>&1 9>&2 1>/dev/null 2>/dev/null) )\n\
-    if [[ \$? != 0 ]]; then\n\
-        unset COMPREPLY\n\
-    fi\n\
-}\n\
-complete -o nospace -F _python_argcomplete \"az\"\n\
-" > ~/.bashrc
+RUN cat /azure-cli/az.completion > ~/.bashrc
 
 WORKDIR /
 

packaged_releases/debian/README.md → build_scripts/debian/README.md

@@ -7,22 +7,16 @@ Updating the Debian package
 On a build machine (e.g. new Ubuntu 14.04 VM), run the build script.
 
 For example:
-
-First copy the build scripts onto the build machine.
-```
-> ~/debian_build.sh; editor ~/debian_build.sh
-> ~/debian_dir_creator.sh; editor ~/debian_dir_creator.sh
-chmod +x ~/debian_build.sh ~/debian_dir_creator.sh
-```
-
-Then execute it with the appropriate environment variable values.
 ```
+git clone https://github.com/azure/azure-cli
+cd azure-cli
 export CLI_VERSION=2.0.9 \
-  && export CLI_DOWNLOAD_SHA256=e74150b2db2975e8b17710eb7ef270ded16e6a8c27f77929544533f6b4c33b76 \
   && export BUILD_ARTIFACT_DIR=$(mktemp -d)\
-  && ~/debian_build.sh ~/debian_dir_creator.sh
+  && build_scripts/debian/build.sh $(pwd)
 ```
 
+Note: The paths above have to be full paths, not relative otherwise the build will fail.
+
 Now you have built the package, upload the package to the apt repository.
 
 

packaged_releases/debian/debian_build.sh → build_scripts/debian/build.sh

@@ -11,50 +11,32 @@ set -ex
 
 if [ -z "$1" ]
   then
-    echo "First argument should be path to executable debian directory creator."
+    echo "Argument should be path to local repo."
     exit 1
 fi
 
-local_repo=$2 
-if [ -z "$local_repo" ]
-  then
-    : "${CLI_DOWNLOAD_SHA256:?CLI_DOWNLOAD_SHA256 environment variable not set.}"
-fi
+local_repo=$1
 
 sudo apt-get update
 
-debian_directory_creator=$1
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+debian_directory_creator=$script_dir/dir_creator.sh
 
 # Install dependencies for the build
 sudo apt-get install -y libssl-dev libffi-dev python3-dev debhelper
 # Download, Extract, Patch, Build CLI
 tmp_pkg_dir=$(mktemp -d)
 working_dir=$(mktemp -d)
 cd $working_dir
-if [ -z "$local_repo" ]
+source_dir=$local_repo
+deb_file=$local_repo/../azure-cli_${CLI_VERSION}-1_all.deb
+az_completion_file=$source_dir/az.completion
+# clean up old build output
+if [ -d "$source_dir/debian" ]
   then
-    source_archive=$working_dir/azure-cli-${CLI_VERSION}.tar.gz
-    source_dir=$working_dir/azure-cli-${CLI_VERSION}
-    deb_file=$working_dir/azure-cli_${CLI_VERSION}-1_all.deb
-    az_completion_file=$source_dir/az.completion
-    wget https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_${CLI_VERSION}.tar.gz -qO $source_archive
-    echo "$CLI_DOWNLOAD_SHA256  $source_archive" | sha256sum -c -
-    mkdir $source_dir
-    # Extract archive
-    archive_extract_dir=$(mktemp -d)
-    tar -xvzf $source_archive -C $archive_extract_dir
-    cp -r $archive_extract_dir/azure-cli_packaged_${CLI_VERSION}/* $source_dir
-  else
-    source_dir=$local_repo
-    deb_file=$local_repo/../azure-cli_${CLI_VERSION}-1_all.deb
-    az_completion_file=$source_dir/packaged_releases/az.completion
-    # clean up old build output
-    if [ -d "$source_dir/debian" ]
-      then
-        rm -rf $source_dir/debian
-    fi
-    cp $local_repo/privates/*.whl $tmp_pkg_dir
+    rm -rf $source_dir/debian
 fi
+[ -d $local_repo/privates ] && cp $local_repo/privates/*.whl $tmp_pkg_dir
 
 # Build Python from source and include
 python_dir=$(mktemp -d)
@@ -74,7 +56,8 @@ make install
 # It does not affect the built .deb file though.
 $source_dir/python_env/bin/pip3 install wheel
 for d in $source_dir/src/azure-cli $source_dir/src/azure-cli-core $source_dir/src/azure-cli-nspkg $source_dir/src/azure-cli-command_modules-nspkg $source_dir/src/command_modules/azure-cli-*/; do cd $d; $source_dir/python_env/bin/python3 setup.py bdist_wheel -d $tmp_pkg_dir; cd -; done;
-$source_dir/python_env/bin/pip3 install azure-cli --find-links $tmp_pkg_dir
+all_modules=`find $tmp_pkg_dir -name "*.whl"`
+$source_dir/python_env/bin/pip3 install $all_modules
 $source_dir/python_env/bin/pip3 install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg
 # WORKAROUND: Newer versions of cryptography do not work on Bash on Windows / WSL - see https://github.com/Azure/azure-cli/issues/4154
 # If you *have* to use a newer version of cryptography in the future, verify that it works on WSL also.

packaged_releases/docker/README.md → build_scripts/docker/README.md

@@ -5,9 +5,9 @@ The Docker image is available at https://hub.docker.com/r/azuresdk/azure-cli-pyt
 
 Updating the Docker image
 -------------------------
-1. Run `docker build` with this Dockerfile.
+1. Run `docker build` with the Dockerfile.
     When tagging this Docker image, choose an appropriate version number.
-      e.g.: ``sudo docker build --no-cache --build-arg BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`" --build-arg CLI_VERSION=${CLI_VERSION} --build-arg CLI_DOWNLOAD_SHA256=${CLI_DOWNLOAD_SHA256} -f Dockerfile -t azuresdk/azure-cli-python:${CLI_VERSION} .``
+      e.g.: ``sudo docker build --no-cache --build-arg BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`" --build-arg CLI_VERSION=${CLI_VERSION} -f Dockerfile -t azuresdk/azure-cli-python:${CLI_VERSION} .``
 2. Push the image to the registry,
       e.g.: `sudo docker push azuresdk/azure-cli-python:${CLI_VERSION}`
 3. Create a PR to commit the Dockerfile changes back to the repository.

packaged_releases/rpm/README.md → build_scripts/rpm/README.md

@@ -7,31 +7,18 @@ Building the RPM package
 On a build machine (e.g. new CentOS 7 VM) run the following.
 
 Install dependencies required to build:
+Required for rpm build tools & required to build the CLI.
 ```
-# Required for rpm build tools.
-sudo yum install -y gcc rpm-build rpm-devel rpmlint make python bash coreutils diffutils patch rpmdevtools
-
-# Required to build the CLI.
-sudo yum install -y gcc python libffi-devel python-devel openssl-devel
-```
-
-Set up directory structure for build:
-```
-mkdir -p ~/rpmbuild/
-cd ~/rpmbuild/
-mkdir -p BUILD RPMS SOURCES SPECS SRPMS
-> SPECS/azure-cli.spec; vi SPECS/azure-cli.spec
+sudo yum install -y gcc git rpm-build rpm-devel rpmlint make bash coreutils diffutils patch rpmdevtools python libffi-devel python-devel openssl-devel
 ```
 
-Set the CLI version and SHA256 for the archive:
+Build example:
+Note: use the full path to the repo path, not a relative path.
 ```
+git clone https://github.com/azure/azure-cli
 export CLI_VERSION=2.0.16
-export CLI_DOWNLOAD_SHA256=22c048d2911c13738c6b901a741ea655f277e0d9eb756c4fb9aee6bb6c2b0109
-```
-
-RPM Build:
-```
-rpmbuild -v -bb --clean SPECS/azure-cli.spec
+export REPO_PATH=~/azure-cli
+rpmbuild -v -bb --clean azure-cli/build_scripts/rpm/azure-cli.spec
 ```
 
 Verification

packaged_releases/rpm/azure-cli.spec → build_scripts/rpm/azure-cli.spec

@@ -9,8 +9,7 @@
 %define name           azure-cli
 %define release        1%{?dist}
 %define version        %{getenv:CLI_VERSION}
-%define source_sha256  %{getenv:CLI_DOWNLOAD_SHA256}
-%define source_url     https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_%{version}.tar.gz
+%define repo_path      %{getenv:REPO_PATH}
 %define venv_url       https://pypi.python.org/packages/source/v/virtualenv/virtualenv-15.0.0.tar.gz
 %define venv_sha256    70d63fb7e949d07aeb37f6ecc94e8b60671edb15b890aa86dba5dfaf2225dc19
 %define cli_lib_dir    %{_libdir}/az
@@ -20,7 +19,6 @@ License:        MIT
 Name:           %{name}
 Version:        %{version}
 Release:        %{release}
-Source0:        %{source_url}
 Url:            https://docs.microsoft.com/en-us/cli/azure/install-azure-cli
 BuildArch:      x86_64
 Requires:       python
@@ -40,11 +38,6 @@ A great cloud needs great tools; we're excited to introduce Azure CLI 2.0,
 tmp_venv_archive=$(mktemp)
 tmp_source_archive=$(mktemp)
 
-# Download, Extract Source
-wget %{source_url} -qO $tmp_source_archive
-echo "%{source_sha256}  $tmp_source_archive" | sha256sum -c -
-tar -xvzf $tmp_source_archive -C %{_builddir}
-
 # Download, Extract Virtualenv
 wget %{venv_url} -qO $tmp_venv_archive
 echo "%{venv_sha256}  $tmp_venv_archive" | sha256sum -c -
@@ -55,13 +48,16 @@ tar -xvzf $tmp_venv_archive -C %{_builddir}
 python %{_builddir}/virtualenv-15.0.0/virtualenv.py --python python %{buildroot}%{cli_lib_dir}
 
 # Build the wheels from the source
-source_dir=%{_builddir}/azure-cli_packaged_%{version}
+source_dir=%{repo_path}
 dist_dir=$(mktemp -d)
 for d in $source_dir/src/azure-cli $source_dir/src/azure-cli-core $source_dir/src/azure-cli-nspkg $source_dir/src/azure-cli-command_modules-nspkg $source_dir/src/command_modules/azure-cli-*/; \
 do cd $d; %{buildroot}%{cli_lib_dir}/bin/python setup.py bdist_wheel -d $dist_dir; cd -; done;
 
+[ -d $source_dir/privates ] && cp $source_dir/privates/*.whl $dist_dir
+
 # Install the CLI
-%{buildroot}%{cli_lib_dir}/bin/pip install azure-cli --find-links $dist_dir
+all_modules=`find $dist_dir -name "*.whl"`
+%{buildroot}%{cli_lib_dir}/bin/pip install $all_modules
 %{buildroot}%{cli_lib_dir}/bin/pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg
 
 # Fix up %{buildroot} appearing in some files...

build_scripts/wheels/README.md

@@ -0,0 +1,9 @@
+Wheel Builds
+============
+
+For example:
+```
+whl_dir=$(mktemp -d)
+python build_scripts/wheels/build.py $whl_dir .
+ls -la $whl_dir
+```

packaged_releases/windows/Product.wxs → build_scripts/windows/Product.wxs

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
 
-  <?define ProductVersion="$(env.CLIVERSION)" ?>
+  <?define ProductVersion="$(env.CLI_VERSION)" ?>
 
   <?define ProductName = "Microsoft CLI 2.0 for Azure" ?>
   <?define ProductDescription = "Command-line tools for Azure." ?>

packaged_releases/windows/README.md → build_scripts/windows/README.md

@@ -18,6 +18,11 @@ Prerequisites
 4. Install 'Microsoft Build Tools 2015'.
     https://www.microsoft.com/en-us/download/details.aspx?id=48159
 
+5. Install Python 3.6
+    - https://www.python.org/ftp/python/3.6.1/python-3.6.1.exe
+    - python-installer.exe /quiet InstallAllUsers=0 TargetDir=%PYTHON_DIR% PrependPath=0 AssociateFiles=0 CompileAll=1 Shortcuts=0 Include_test=0 Include_doc=0 Include_dev=0 Include_launcher=0 Include_tcltk=0 Include_tools=0
+    - %PYTHON_DIR% should be on PATH.
+
 5. Clone the repository.
     - git clone https://github.com/azure/azure-cli
 
@@ -26,9 +31,8 @@ Note: The above can be done on a Windows 10 VM.
 Building
 --------
 
-1. Set the `CLIVERSION` environment variable.
+1. Set the `CLI_VERSION` environment variable.
 
-2. Run `build.cmd`.
-    - Can `cd packaged_releases\windows` first.
+2. Run `build_scripts\windows\scripts\build.cmd`.
 
 3. The unsigned MSI will be in the `.\out` folder.