From 0673a65afcf57a5fbb0f8c06ef77fb2e476a256d Mon Sep 17 00:00:00 2001 From: Derek Bekoe Date: Thu, 2 Nov 2017 12:21:16 -0700 Subject: [PATCH] CI: Index verification (#10) * Index tests * Some refactoring * New flake8 settings * Metadata checks also * Better build stages --- .flake8 | 12 +- .github/PULL_REQUEST_TEMPLATE.md | 3 +- .travis.yml | 14 ++- scripts/ci/test_index.py | 187 +++++++++++++++++++++++++++++++ src/index.json | 2 +- 5 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 scripts/ci/test_index.py diff --git a/.flake8 b/.flake8 index f0e476b386d..283ad44ac5e 100644 --- a/.flake8 +++ b/.flake8 @@ -2,10 +2,8 @@ max-line-length = 120 max-complexity = 10 ignore = - E126, - E501, - E722, - E741, - F401, - F811, - C901 + E501, # line too long, it is covered by pylint + E722, # bare except, bad practice, to be removed in the future + F401, # imported but unused, too many violations, to be removed in the future + F811, # redefinition of unused, to be removed in the future + C901 # code flow is too complex, too many violations, to be removed in the future diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d6415fe5f12..40bc6de3e29 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,4 +4,5 @@ This checklist is used to make sure that common guidelines for a pull request ar ### General Guidelines -- [ ] Have you run `./scripts/ci/test_static.sh` locally? (`pip install pylint flake8` required) +- [ ] If you modified extension source code, have you run `./scripts/ci/test_static.sh` locally? (`pip install pylint flake8` required) +- [ ] If you modified the index, have you run `python scripts/ci/test_index.py -q` locally? diff --git a/.travis.yml b/.travis.yml index f7ca6fb9aaf..90eefe6232d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,14 +2,22 @@ dist: trusty sudo: off language: python install: - - pip install pylint flake8 + - pip install pylint flake8 requests jobs: include: - stage: verify + env: PURPOSE='SourceStatic' script: ./scripts/ci/test_static.sh - env: PURPOSE='VerifySource-StaticCheck' python: 3.6 - stage: verify + env: PURPOSE='SourceStatic' script: ./scripts/ci/test_static.sh - env: PURPOSE='VerifySource-StaticCheck' + python: 2.7 + - stage: verify + env: PURPOSE='IndexVerify' + script: python ./scripts/ci/test_index.py -v + python: 3.6 + - stage: verify + env: PURPOSE='IndexVerify' + script: python ./scripts/ci/test_index.py -v python: 2.7 diff --git a/scripts/ci/test_index.py b/scripts/ci/test_index.py new file mode 100644 index 00000000000..790d9be9f79 --- /dev/null +++ b/scripts/ci/test_index.py @@ -0,0 +1,187 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from __future__ import print_function + +import os +import json +import tempfile +import unittest +import zipfile +import hashlib +import shutil +from util import get_repo_root +from wheel.install import WHEEL_INFO_RE + +INDEX_PATH = os.path.join(get_repo_root(), 'src', 'index.json') + + +def catch_dup_keys(pairs): + seen = {} + for k, v in pairs: + if k in seen: + raise ValueError("duplicate key {}".format(k)) + seen[k] = v + return seen + + +def get_index_data(): + try: + with open(INDEX_PATH) as f: + return json.load(f, object_pairs_hook=catch_dup_keys) + except ValueError as err: + raise AssertionError("Invalid JSON in {}: {}".format(INDEX_PATH, err)) + + +def get_whl_from_url(url, filename, tmp_dir, whl_cache): + if url in whl_cache: + return whl_cache[url] + import requests + r = requests.get(url, stream=True) + assert r.status_code == 200, "Request to {} failed with {}".format(url, r.status_code) + ext_file = os.path.join(tmp_dir, filename) + with open(ext_file, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # ignore keep-alive new chunks + f.write(chunk) + whl_cache[url] = ext_file + return ext_file + + +def get_sha256sum(a_file): + sha256 = hashlib.sha256() + with open(a_file, 'rb') as f: + sha256.update(f.read()) + return sha256.hexdigest() + + +def get_extension_modname(ext_dir): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L153 + EXTENSIONS_MOD_PREFIX = 'azext_' + pos_mods = [n for n in os.listdir(ext_dir) + if n.startswith(EXTENSIONS_MOD_PREFIX) and os.path.isdir(os.path.join(ext_dir, n))] + return pos_mods[0] + + +def get_azext_metadata(ext_dir): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L109 + AZEXT_METADATA_FILENAME = 'azext_metadata.json' + azext_metadata = None + ext_modname = get_extension_modname(ext_dir=ext_dir) + azext_metadata_filepath = os.path.join(ext_dir, ext_modname, AZEXT_METADATA_FILENAME) + if os.path.isfile(azext_metadata_filepath): + with open(azext_metadata_filepath) as f: + azext_metadata = json.load(f) + return azext_metadata + + +def get_ext_metadata(ext_dir, ext_file, ext_name): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89 + WHL_METADATA_FILENAME = 'metadata.json' + zip_ref = zipfile.ZipFile(ext_file, 'r') + zip_ref.extractall(ext_dir) + zip_ref.close() + metadata = {} + dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')] + azext_metadata = get_azext_metadata(ext_dir) + if azext_metadata: + metadata.update(azext_metadata) + for dist_info_dirname in dist_info_dirs: + parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) + if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name: + whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME) + if os.path.isfile(whl_metadata_filepath): + with open(whl_metadata_filepath) as f: + metadata.update(json.load(f)) + return metadata + + +class TestIndex(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.longMessage = True + cls.index = get_index_data() + cls.whl_cache_dir = tempfile.mkdtemp() + cls.whl_cache = {} + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.whl_cache_dir) + + def test_format_version(self): + self.assertEqual(self.index['formatVersion'], '1') + + def test_format_extensions_key(self): + self.assertIn('extensions', self.index) + + def test_format_extensions_value(self): + self.assertIsInstance(self.index['extensions'], dict) + + def test_extension_filenames(self): + for ext_name, exts in self.index['extensions'].items(): + for item in exts: + self.assertTrue(item['filename'].endswith('.whl'), + "Filename {} must end with .whl".format(item['filename'])) + self.assertTrue(item['filename'].startswith(ext_name), + "Filename {} must start with {}".format(item['filename'], ext_name)) + parsed_filename = WHEEL_INFO_RE(item['filename']) + p = parsed_filename.groupdict() + self.assertTrue(p.get('name'), "Can't get name for {}".format(item['filename'])) + universal_wheel = p.get('pyver') == 'py2.py3' and p.get('abi') == 'none' and p.get('plat') == 'any' + self.assertTrue(universal_wheel, + "{} of {} not universal (platform independent) wheel. " + "It should end in py2.py3-none-any.whl".format(item['filename'], ext_name)) + + def test_extension_url_filename(self): + for exts in self.index['extensions'].values(): + for item in exts: + self.assertEqual(os.path.basename(item['downloadUrl']), item['filename'], + "Filename must match last segment of downloadUrl") + + def test_filename_duplicates(self): + filenames = [] + for exts in self.index['extensions'].values(): + for item in exts: + filenames.append(item['filename']) + filename_seen = set() + dups = [] + for f in filenames: + if f in filename_seen: + dups.append(f) + filename_seen.add(f) + self.assertFalse(dups, "Duplicate filenames found {}".format(dups)) + + @unittest.skipUnless(os.getenv('CI'), 'Skipped as not running on CI') + def test_checksums(self): + for exts in self.index['extensions'].values(): + for item in exts: + ext_file = get_whl_from_url(item['downloadUrl'], item['filename'], + self.whl_cache_dir, self.whl_cache) + computed_hash = get_sha256sum(ext_file) + self.assertEqual(computed_hash, item['sha256Digest'], + "Computed {} but found {} in index for {}".format(computed_hash, + item['sha256Digest'], + item['filename'])) + + @unittest.skipUnless(os.getenv('CI'), 'Skipped as not running on CI') + def test_metadata(self): + self.maxDiff = None + extensions_dir = tempfile.mkdtemp() + for ext_name, exts in self.index['extensions'].items(): + for item in exts: + ext_dir = tempfile.mkdtemp(dir=extensions_dir) + ext_file = get_whl_from_url(item['downloadUrl'], item['filename'], + self.whl_cache_dir, self.whl_cache) + metadata = get_ext_metadata(ext_dir, ext_file, ext_name) + self.assertDictEqual(metadata, item['metadata'], + "Metadata for {} in index doesn't match the expected of: \n" + "{}".format(item['filename'], json.dumps(metadata, indent=2, sort_keys=True, + separators=(',', ': ')))) + shutil.rmtree(extensions_dir) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/index.json b/src/index.json index 6b66ce4fd2f..44432f0ad5a 100644 --- a/src/index.json +++ b/src/index.json @@ -14,7 +14,7 @@ "contacts": [ { "email": "azpycli@microsoft.com", - "name": "Microsoft Corporation.", + "name": "Microsoft Corporation", "role": "author" } ],