merge conflicts with public master

Azure · yugangw-msft · Sep 22, 2017 · Aug 4, 2017 · Aug 4, 2017 · Aug 7, 2017
commit 2aa24b8e1c9a7c3346e2ab2730a1f431485ae8eb
diff --git a/.travis.yml b/.travis.yml
@@ -13,6 +13,7 @@ jobs:
   include:
     - stage: build
       script: ./scripts/ci/build.sh
+      python: 3.6
     - stage: verify
       env: PURPOSE='Automation'
       script: ./scripts/ci/test_automation.sh

diff --git a/README.md b/README.md
@@ -19,7 +19,7 @@ A list of common install issues and their resolutions are available at [install
 ### Developer Installation (see below)
 
 - [Docker](#docker)
-- [Nightly Builds](#nightly-builds)
+- [Edge Builds](#edge-builds)
 - [Developer Setup](#developer-setup)
 
 ## Usage
@@ -131,30 +131,26 @@ $ docker run -v ${HOME}:/root -it azuresdk/azure-cli-python:<version>
 
 See our [Docker tags](https://hub.docker.com/r/azuresdk/azure-cli-python/tags/) for available versions.
 
-### Nightly Builds
+### Edge Builds
 
-Install nightly builds with pip in a virtual environment.
+If you want to get the latest build from the master branch, you can use our "edge" builds feed. Here's an example of 
+installing edge dev builds with pip in a virtual environment.
 
 ```bash
-$ pip install --pre azure-cli --extra-index-url https://azureclinightly.blob.core.windows.net/packages
+$ virtualenv env
+$ . env/bin/activate
+$ pip install --pre azure-cli --extra-index-url https://azurecliprod.blob.core.windows.net/edge
 ```
 
-To upgrade your current nightly build pass the `--upgrade` option.
+To upgrade your current edge build pass the `--upgrade` option. The `--no-cache-dir` option is also recommended since
+the feed is frequently updated.
 
 ```bash
-$ pip install --upgrade --pre azure-cli --extra-index-url https://azureclinightly.blob.core.windows.net/packages
+$ pip install --upgrade --pre azure-cli --extra-index-url https://azurecliprod.blob.core.windows.net/edge --no-cache-dir
 ```
 
-- Builds happen at 21:00:00 PDT each night. They are published shortly afterwards.
-- Whilst all command modules are built each night, not all are included on install.
-- Install additional components with:  
-    ```
-    $ export AZURE_COMPONENT_PACKAGE_INDEX_URL=https://azureclinightly.blob.core.windows.net/packages
-    ```  
-    ```
-    $ az component update --add <component_name> --private
-    ```
-- To view the list of installed packages, run ``az component list``
+The edge build is generated for each push to the `master` branch as a part of the Travis CI build. The version of the edge build follows 
+
 
 ## Developer Setup
 If you would like to setup a development environment and contribute to the CLI, see 

diff --git a/azure-cli.pyproj b/azure-cli.pyproj
@@ -497,20 +497,6 @@
     </Compile>
     <Compile Include="command_modules\azure-cli-role\azure\cli\command_modules\role\_validators.py" />
     <Compile Include="command_modules\azure-cli-role\azure_bdist_wheel.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\cluster_auth.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\commands.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\config.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\custom.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\test_sf_app.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\test_sf_node.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\test_sf_scenarios.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\test_sf_select.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\test_sf_service.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\_factory.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\_help.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\_params.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\__init__.py" />
-    <Compile Include="command_modules\azure-cli-sf\setup.py" />
     <Compile Include="command_modules\azure-cli-sql\azure\cli\command_modules\sql\commands.py" />
     <Compile Include="command_modules\azure-cli-sql\azure\cli\command_modules\sql\custom.py" />
     <Compile Include="command_modules\azure-cli-sql\azure\cli\command_modules\sql\params.py" />
@@ -836,12 +822,6 @@
     <Folder Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\" />
     <Folder Include="command_modules\azure-cli-role\azure\cli\command_modules\role\tests\" />
     <Folder Include="command_modules\azure-cli-role\tests\" />
-    <Folder Include="command_modules\azure-cli-sf\" />
-    <Folder Include="command_modules\azure-cli-sf\azure\" />
-    <Folder Include="command_modules\azure-cli-sf\azure\cli\" />
-    <Folder Include="command_modules\azure-cli-sf\azure\cli\command_modules\" />
-    <Folder Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\" />
-    <Folder Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\" />
     <Folder Include="command_modules\azure-cli-sql\" />
     <Folder Include="command_modules\azure-cli-sql\azure\" />
     <Folder Include="command_modules\azure-cli-sql\azure\cli\" />
@@ -1023,7 +1003,6 @@
     <Content Include="command_modules\azure-cli-resource\azure\cli\command_modules\resource\tests\test-template.json" />
     <Content Include="command_modules\azure-cli-resource\HISTORY.rst" />
     <Content Include="command_modules\azure-cli-role\HISTORY.rst" />
-    <Content Include="command_modules\azure-cli-sf\HISTORY.rst" />
     <Content Include="command_modules\azure-cli-sql\HISTORY.rst" />
     <Content Include="command_modules\azure-cli-rdbms\HISTORY.rst" />
     <Content Include="command_modules\azure-cli-storage\HISTORY.rst" />

diff --git a/azure-cli2017.pyproj b/azure-cli2017.pyproj
@@ -488,20 +488,6 @@
     <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\__init__.py" />
     <Compile Include="command_modules\azure-cli-servicefabric\azure\__init__.py" />
     <Compile Include="command_modules\azure-cli-servicefabric\setup.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\cluster_auth.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\commands.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\config.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\custom.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\test_sf_app.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\test_sf_node.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\test_sf_scenarios.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\test_sf_select.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\test_sf_service.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\_factory.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\_help.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\_params.py" />
-    <Compile Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\__init__.py" />
-    <Compile Include="command_modules\azure-cli-sf\setup.py" />
     <Compile Include="command_modules\azure-cli-sql\azure\cli\command_modules\sql\commands.py" />
     <Compile Include="command_modules\azure-cli-sql\azure\cli\command_modules\sql\custom.py" />
     <Compile Include="command_modules\azure-cli-sql\azure\cli\command_modules\sql\params.py" />
@@ -832,12 +818,6 @@
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\Template\Linux\" />
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\Template\Windows\" />
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\tests\" />
-    <Folder Include="command_modules\azure-cli-sf\" />
-    <Folder Include="command_modules\azure-cli-sf\azure\" />
-    <Folder Include="command_modules\azure-cli-sf\azure\cli\" />
-    <Folder Include="command_modules\azure-cli-sf\azure\cli\command_modules\" />
-    <Folder Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\" />
-    <Folder Include="command_modules\azure-cli-sf\azure\cli\command_modules\sf\tests\" />
     <Folder Include="command_modules\azure-cli-sql\" />
     <Folder Include="command_modules\azure-cli-sql\azure\" />
     <Folder Include="command_modules\azure-cli-sql\azure\cli\" />
@@ -1040,10 +1020,6 @@
     <Content Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\Template\Windows\template.json" />
     <Content Include="command_modules\azure-cli-servicefabric\HISTORY.rst" />
     <Content Include="command_modules\azure-cli-servicefabric\README.rst" />
-    <Content Include="command_modules\azure-cli-sf\HISTORY.rst" />
-    <Content Include="command_modules\azure-cli-sf\MANIFEST.in" />
-    <Content Include="command_modules\azure-cli-sf\README.rst" />
-    <Content Include="command_modules\azure-cli-sf\setup.cfg" />
     <Content Include="command_modules\azure-cli-sql\HISTORY.rst" />
     <Content Include="command_modules\azure-cli-storage\HISTORY.rst" />
     <Content Include="command_modules\azure-cli-taskhelp\HISTORY.rst" />

diff --git a/doc/authoring_command_modules/README.md b/doc/authoring_command_modules/README.md
@@ -178,11 +178,21 @@ This will be the release notes for the next release.
 
 e.g.:  
 ```
+.. :changelog:
+
+Release History
+===============
+
 unreleased
-----------
+++++++++++
 
 * This is my change.
 
+0.0.1 (YYYY-MM-DD)
+++++++++++++++++++
+
+* This is the changelog from a prev. release.
+
 ```
 
 Packaging/Publishing

diff --git a/doc/sphinx/azhelpgen/doc_source_map.json b/doc/sphinx/azhelpgen/doc_source_map.json
@@ -38,7 +38,6 @@
     "redis": "src/command_modules/azure-cli-redis/azure/cli/command_modules/redis/_help.py",
     "resource": "src/command_modules/azure-cli-resource/azure/cli/command_modules/resource/_help.py",
     "role": "src/command_modules/azure-cli-role/azure/cli/command_modules/role/_help.py",
-    "sf": "src/command_modules/azure-cli-sf/azure/cli/command_modules/sf/_help.py",
     "sql": "src/command_modules/azure-cli-sql/azure/cli/command_modules/sql/_help.py",
     "storage": "src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/_help.py",
     "tag": "src/command_modules/azure-cli-resource/azure/cli/command_modules/resource/_help.py",

diff --git a/packaged_releases/README.md b/packaged_releases/README.md
@@ -38,7 +38,7 @@ python -m automation.release.packaged --version 0.2.3 -f ~/cli-components.json
 OR
 
 ```
-python -m automation.release.packaged --version 2.0.9 --components azure-cli=2.0.9 azure-cli-acr=2.0.7 azure-cli-acs=2.0.9 azure-cli-appservice=0.1.9 azure-cli-batch=3.0.2 azure-cli-billing=0.1.2 azure-cli-cdn=0.0.5 azure-cli-cloud=2.0.5 azure-cli-cognitiveservices=0.1.5 azure-cli-command_modules-nspkg=2.0.0 azure-cli-component=2.0.6 azure-cli-configure=2.0.9 azure-cli-consumption=0.1.2 azure-cli-core=2.0.10 azure-cli-cosmosdb=0.1.9 azure-cli-dla=0.0.9 azure-cli-dls=0.0.9 azure-cli-feedback=2.0.5 azure-cli-find=0.2.5 azure-cli-interactive=0.3.5 azure-cli-iot=0.1.8 azure-cli-keyvault=2.0.7 azure-cli-lab=0.0.7 azure-cli-monitor=0.0.7 azure-cli-network=2.0.9 azure-cli-nspkg=3.0.0 azure-cli-profile=2.0.7 azure-cli-rdbms=0.0.4 azure-cli-redis=0.2.6 azure-cli-resource=2.0.9 azure-cli-role=2.0.7 azure-cli-sf=1.0.4 azure-cli-sql=2.0.6 azure-cli-storage=2.0.9 azure-cli-vm=2.0.9
+python -m automation.release.packaged --version 2.0.9 --components azure-cli=2.0.9 azure-cli-acr=2.0.7 azure-cli-acs=2.0.9 azure-cli-appservice=0.1.9 azure-cli-batch=3.0.2 azure-cli-billing=0.1.2 azure-cli-cdn=0.0.5 azure-cli-cloud=2.0.5 azure-cli-cognitiveservices=0.1.5 azure-cli-command_modules-nspkg=2.0.0 azure-cli-component=2.0.6 azure-cli-configure=2.0.9 azure-cli-consumption=0.1.2 azure-cli-core=2.0.10 azure-cli-cosmosdb=0.1.9 azure-cli-dla=0.0.9 azure-cli-dls=0.0.9 azure-cli-feedback=2.0.5 azure-cli-find=0.2.5 azure-cli-interactive=0.3.5 azure-cli-iot=0.1.8 azure-cli-keyvault=2.0.7 azure-cli-lab=0.0.7 azure-cli-monitor=0.0.7 azure-cli-network=2.0.9 azure-cli-nspkg=3.0.0 azure-cli-profile=2.0.7 azure-cli-rdbms=0.0.4 azure-cli-redis=0.2.6 azure-cli-resource=2.0.9 azure-cli-role=2.0.7 azure-cli-sql=2.0.6 azure-cli-storage=2.0.9 azure-cli-vm=2.0.9
 ```
 
 2 - Upload release archive

diff --git a/packaged_releases/homebrew/README.md b/packaged_releases/homebrew/README.md
@@ -4,19 +4,26 @@ Homebrew Packaging
 
 Updating the formula
 --------------------
-1. Change the `url` in the formula to point to the new release (e.g. `azure-cli_packaged_2.0.14.tar.gz`).
-2. Modify any checksums as required.
+1. Change the `url` in the formula to point to the new release and change the `sha256` value also.
+2. Update the resources list in the formula (see below).
 3. Run the formula verification commands (see below).
 4. Submit a PR to https://github.com/Homebrew/homebrew-core.
 
 
+Updating the resources list
+---------------------------
+```
+# Create a new virtual environment first
+pip install azure-cli homebrew-pypi-poet; poet -r azure-cli
+```
+
 Verification
 ------------
 
 ```
-$ brew install --build-from-source ./azure-cli@2.0.rb
-$ brew test azure-cli@2.0
-$ brew audit --strict --online azure-cli@2.0
+brew install --build-from-source azure-cli.rb
+brew test azure-cli.rb
+brew audit --strict --online azure-cli.rb
 ```
 
 More Information

diff --git a/...mgmt_network-1.5.0a1-py2.py3-none-any.whl → ...gmt_network-1.5.0rc2-py2.py3-none-any.whl b/...mgmt_network-1.5.0a1-py2.py3-none-any.whl → ...gmt_network-1.5.0rc2-py2.py3-none-any.whl
diff --git a/scripts/batch/app/install.sh b/scripts/batch/app/install.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# install the product and test builds to a virtual environment
+
+rm -rf $AZ_BATCH_NODE_SHARED_DIR/env
+rm -rf $AZ_BATCH_NODE_SHARED_DIR/venv
+rm -rf $AZ_BATCH_NODE_SHARED_DIR/app
+
+pip install --user virtualenv
+python -m virtualenv $AZ_BATCH_NODE_SHARED_DIR/venv
+
+. $AZ_BATCH_NODE_SHARED_DIR/venv/bin/activate
+
+pip install azure-cli-fulltest -f ./build
+pip install nose
+
+cp -R ./app $AZ_BATCH_NODE_SHARED_DIR
diff --git a/scripts/batch/app/run_test.sh b/scripts/batch/app/run_test.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# execute individual test in current task
+
+# expected input: 
+#	$1 the test class
+#	$2 the test method
+
+# expected virtual environment (if run live):
+#   AUTOMATION_SP_NAME
+#   AUTOMATION_SP_PASSWORD
+#   AUTOMATION_SP_TENANT
+
+set -e
+
+# enter virtual environment:
+. $AZ_BATCH_NODE_SHARED_DIR/venv/bin/activate
+
+# execute az to create the configure folder
+az --version
+
+if [ "$AZURE_TEST_RUN_LIVE" == "True" ]; then
+    az login --service-principal -u $AUTOMATION_SP_NAME -p $AUTOMATION_SP_PASSWORD -t $AUTOMATION_SP_TENANT
+fi
+
+# execute the test
+python -m unittest -v $1.$2 2>&1
diff --git a/scripts/batch/app/schedule.py b/scripts/batch/app/schedule.py
@@ -0,0 +1,70 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import sys
+
+
+# REFERENCE:
+
+# shared virtualenv                 AZ_BATCH_NODE_SHARED_DIR/venv/bin/activate
+# batch account name                AZ_BATCH_ACCOUNT_NAME
+# batch account key                 AZURE_BATCH_KEY
+# batch account endpoint            AZURE_BATCH_ENDPOINT
+# current job id                    AZ_BATCH_JOB_ID
+# logging container url with SAS    AUTOMATION_OUTPUT_CONTAINER
+
+# https://docs.microsoft.com/en-us/azure/batch/batch-compute-node-environment-variables
+
+def create_batch_client():
+    from azure.batch import BatchServiceClient
+    from azure.batch.batch_auth import SharedKeyCredentials
+    cred = SharedKeyCredentials(os.environ['AZ_BATCH_ACCOUNT_NAME'], os.environ['AZURE_BATCH_KEY'])
+    return BatchServiceClient(cred, os.environ['AZURE_BATCH_ENDPOINT'])
+
+
+def get_command_string(*args):
+    return "/bin/bash -c 'set -e; set -o pipefail; {}; wait'".format(';'.join(args))
+
+
+def get_output_log_list(test_id):
+    from azure.batch.models import (OutputFile, OutputFileDestination, OutputFileBlobContainerDestination,
+                                    OutputFileUploadOptions, OutputFileUploadCondition)
+
+    build_container_url = os.environ['AUTOMATION_OUTPUT_CONTAINER']
+    return [OutputFile('../std*.*',
+                       OutputFileDestination(OutputFileBlobContainerDestination(build_container_url, test_id)),
+                       OutputFileUploadOptions(OutputFileUploadCondition.task_completion))]
+
+
+def main():
+    from azure.batch.models import TaskAddParameter, BatchErrorException
+    input_file = sys.argv[1]
+
+    bs = create_batch_client()
+    fail_to_schedule = []
+    for index, line in enumerate(open(input_file, 'r')):
+        try:
+            test_class, test_method = line.split()
+            commands = ['$AZ_BATCH_NODE_SHARED_DIR/app/run_test.sh {} {}'.format(test_class, test_method)]
+            test_id = '{:04}{}'.format(index, test_method)[:64]
+            bs.task.add(job_id=os.environ['AZ_BATCH_JOB_ID'],
+                        task=TaskAddParameter(id=test_id,
+                                              command_line=get_command_string(*commands),
+                                              display_name='test {} ({})'.format(test_method, test_class),
+                                              output_files=get_output_log_list(test_id)))
+        except (BatchErrorException, ValueError):
+            fail_to_schedule.append(line)
+
+    if not fail_to_schedule:
+        sys.exit(0)
+    else:
+        for line in fail_to_schedule:
+            print('FAIL: {}'.format(line))
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/batch/app/schedule.sh b/scripts/batch/app/schedule.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# The Job Management Task has its own virtualenv because it requires an azure-batch package to function.
+pip install --user virtualenv
+python -m virtualenv venv
+. ./venv/bin/activate
+
+pip install azure-batch==3.0.0
+
+# schedule test tasks
+scripts_dir=$(cd $(dirname $0); pwd)
+python $scripts_dir/schedule.py $scripts_dir/all_tests.txt
diff --git a/scripts/build.sh b/scripts/build.sh