[CI] Add check that azext_metadata.json must exist (Azure#1842)

scripts/ci/test_index.py

@@ -15,7 +15,10 @@
 import unittest
 import hashlib
 import shutil
+
+from distutils.version import LooseVersion
 from wheel.install import WHEEL_INFO_RE
+
 from util import get_ext_metadata, get_whl_from_url, get_index_data, verify_dependency
 
 
@@ -114,14 +117,56 @@ def test_checksums(self):
 
     @unittest.skipUnless(os.getenv('CI'), 'Skipped as not running on CI')
     def test_metadata(self):
-        self.maxDiff = None
+        skipable_extension_thresholds = {
+            'ip-group': '0.1.2',
+            'vm-repair': '0.3.1',
+            'mixed-reality': '0.0.2',
+            'subscription': '0.1.4',
+            'managementpartner': '0.1.3',
+            'log-analytics': '0.2.1'
+        }
+
+        historical_extensios = {
+            'keyvault-preview': '0.1.3',
+            'log-analytics': '0.2.1'
+        }
+
         extensions_dir = tempfile.mkdtemp()
         for ext_name, exts in self.index['extensions'].items():
             for item in exts:
                 ext_dir = tempfile.mkdtemp(dir=extensions_dir)
                 ext_file = get_whl_from_url(item['downloadUrl'], item['filename'],
                                             self.whl_cache_dir, self.whl_cache)
-                metadata = get_ext_metadata(ext_dir, ext_file, ext_name)
+
+                print(ext_file)
+
+                ext_version = item['metadata']['version']
+                try:
+                    metadata = get_ext_metadata(ext_dir, ext_file, ext_name)    # check file exists
+                except ValueError as ex:
+                    if ext_name in skipable_extension_thresholds:
+                        threshold_version = skipable_extension_thresholds[ext_name]
+
+                        if LooseVersion(ext_version) <= LooseVersion(threshold_version):
+                            continue
+                        else:
+                            raise ex
+                    else:
+                        raise ex
+
+                try:
+                    self.assertIn('azext.minCliCoreVersion', metadata)  # check key properties exists
+                except AssertionError as ex:
+                    if ext_name in historical_extensios:
+                        threshold_version = historical_extensios[ext_name]
+
+                        if LooseVersion(ext_version) <= LooseVersion(threshold_version):
+                            continue
+                        else:
+                            raise ex
+                    else:
+                        raise ex
+
                 # Due to https://github.com/pypa/wheel/issues/195 we prevent whls built with 0.31.0 or greater.
                 # 0.29.0, 0.30.0 are the two previous versions before that release.
                 supported_generators = ['bdist_wheel (0.29.0)', 'bdist_wheel (0.30.0)']

scripts/ci/util.py

@@ -58,9 +58,14 @@ def get_ext_metadata(ext_dir, ext_file, ext_name):
     zip_ref.close()
     metadata = {}
     dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')]
+
     azext_metadata = _get_azext_metadata(ext_dir)
-    if azext_metadata:
-        metadata.update(azext_metadata)
+
+    if not azext_metadata:
+        raise ValueError('azext_metadata.json for Extension "{}" Metadata is missing'.format(ext_name))
+
+    metadata.update(azext_metadata)
+
     for dist_info_dirname in dist_info_dirs:
         parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname)
         if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name.replace('-', '_'):