CI Improvements (Azure#16)

- License check: Handle \r\n and \n
- Index check: Reject https://pypi.python.org urls
- Source tests: Install extension in its own dir before running test for extension

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 env/
+env27/
 build/
 develop-eggs/
 dist/

.travis.yml

@@ -25,5 +25,6 @@ jobs:
       env: PURPOSE='IndexVerify'
       script: python ./scripts/ci/test_integration.py -v
       python: 3.6
+  fast_finish: true
   allow_failures:
     - env: PURPOSE='SourceTests'

scripts/ci/test_integration.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # --------------------------------------------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License. See License.txt in the project root for license information.
@@ -151,6 +153,20 @@ def test_extension_url_filename(self):
                 self.assertEqual(os.path.basename(item['downloadUrl']), item['filename'],
                                  "Filename must match last segment of downloadUrl")
 
+    def test_extension_url_pypi(self):
+        for exts in self.index['extensions'].values():
+            for item in exts:
+                url = item['downloadUrl']
+                pypi_url_prefix = 'https://pypi.python.org/packages/'
+                pythonhosted_url_prefix = 'https://files.pythonhosted.org/packages/'
+                if url.startswith(pypi_url_prefix):
+                    new_url = url.replace(pypi_url_prefix, pythonhosted_url_prefix)
+                    hash_pos = new_url.find('#')
+                    new_url = new_url if hash_pos == -1 else new_url[:hash_pos]
+                    self.fail("Replace {} with {}\n"
+                              "See for more info https://wiki.archlinux.org/index.php/Python_package_guidelines"
+                              "#PyPI_download_URLs".format(url, new_url))
+
     def test_filename_duplicates(self):
         filenames = []
         for exts in self.index['extensions'].values():

scripts/ci/test_source.sh

@@ -1,16 +1,31 @@
 #!/usr/bin/env bash
-set -e
+set -ex
 
 # Install CLI & CLI testsdk
 echo "Installing azure-cli-testsdk and azure-cli..."
-pip install "git+https://github.com/Azure/azure-cli@dev#egg=azure-cli-testsdk&subdirectory=src/azure-cli-testsdk" -q
+# TODO Update the git commit when we need a new version of azure-cli-testsdk
+pip install "git+https://github.com/Azure/azure-cli@68460748e47f20cba462686c9fd20d2c720cf98c#egg=azure-cli-testsdk&subdirectory=src/azure-cli-testsdk" -q
 echo "Installed."
 
+
+_AZURE_EXTENSION_DIR="$AZURE_EXTENSION_DIR"
+
 for d in src/*/azext_*/tests;
     do echo "Running tests for $d";
     if [ -d $d ]; then
+        export AZURE_EXTENSION_DIR=$(mktemp -d);
+        pip install --upgrade --target $AZURE_EXTENSION_DIR/ext $d/../..;
         python -m unittest discover -v $d;
+        rm -rf $AZURE_EXTENSION_DIR;
     else
         echo "Skipped $d as not a directory."
     fi
 done;
+
+if ! [ -z "${_AZURE_EXTENSION_DIR+_}" ] ; then
+    AZURE_EXTENSION_DIR="$_AZURE_EXTENSION_DIR"
+    export AZURE_EXTENSION_DIR
+    unset _AZURE_EXTENSION_DIR
+fi
+
+echo "OK. Completed tests."

scripts/ci/verify_license.py

@@ -41,7 +41,7 @@ def main():
         file_itr = (os.path.join(current_dir, p) for p in files if p.endswith('.py'))
         for python_file in file_itr:
             with open(python_file, 'r') as f:
-                file_text = f.read()
+                file_text = f.read().replace('\r\n', '\n')
                 if file_text and (LICENSE_HEADER not in file_text and AUTOREST_LICENSE_HEADER not in file_text):
                     files_without_header.append(os.path.join(current_dir, python_file))