Add commands to automatically update and sync dependencies (#9811)

* Add sync command

* Begin dep sync

* Add dep sync command

* Style

* Use for comprehension

* Remove unused function and add get_check_req_file

* Add dep update

* Add ability to sync and see updates

* Update style

* Update ignore list

* Remove unneeded code

* Add option to ignore classifiers

* Simplify marker and add more os's

datadog_checks_dev/datadog_checks/dev/tooling/commands/dep.py

@@ -1,16 +1,25 @@
 # (C) Datadog, Inc. 2018-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
+import asyncio
 from collections import defaultdict
 
 import click
+import orjson
+from aiohttp import request
+from aiomultiprocess import Pool
 from packaging.markers import InvalidMarker, Marker
 from packaging.specifiers import SpecifierSet
 
 from ...fs import read_file_lines, write_file_lines
 from ..constants import get_agent_requirements
-from ..dependencies import read_check_dependencies
+from ..dependencies import read_agent_dependencies, read_check_dependencies
+from ..utils import get_check_req_file, get_valid_checks
 from .console import CONTEXT_SETTINGS, abort, echo_failure, echo_info
+from .validate.licenses import extract_classifier_value
+
+# Dependencies to ignore when update dependencies
+IGNORED_DEPS = {'psycopg2-binary'}
 
 
 @click.group(context_settings=CONTEXT_SETTINGS, short_help='Manage dependencies')
@@ -108,3 +117,173 @@ def freeze():
     lines = sorted(set(f'{d[1]}\n' for d in data))
 
     write_file_lines(static_file, lines)
+
+
+@dep.command(
+    context_settings=CONTEXT_SETTINGS,
+    short_help="Update integrations' dependencies so that they match the Agent's static environment",
+)
+def sync():
+    all_agent_dependencies, errors = read_agent_dependencies()
+
+    if errors:
+        for error in errors:
+            echo_failure(error)
+        abort()
+
+    files_updated = 0
+    checks = sorted(get_valid_checks())
+    for check_name in checks:
+        check_dependencies, check_errors = read_check_dependencies(check=check_name)
+
+        if check_errors:
+            for error in check_errors:
+                echo_failure(error)
+            abort()
+
+        deps_to_update = {
+            check_dependency_definition: agent_dep_version
+            for check_dependency_name, check_dependency_definitions in check_dependencies.items()
+            for version, dependency_definitions in check_dependency_definitions.items()
+            for check_dependency_definition in dependency_definitions
+            for agent_dep_version, agent_dependency_definitions in all_agent_dependencies[check_dependency_name].items()
+            for agent_dependency_definition in agent_dependency_definitions
+            # look for the dependency with the same marker and name since the version can be different
+            if check_dependency_definition.same_name_marker(agent_dependency_definition)
+            and version != agent_dep_version
+        }
+
+        if deps_to_update:
+            files_updated += 1
+            check_req_file = get_check_req_file(check_name)
+            old_lines = read_file_lines(check_req_file)
+            new_lines = old_lines.copy()
+
+            for dependency_definition, new_version in deps_to_update.items():
+                dependency_definition.requirement.specifier = new_version
+                new_lines[dependency_definition.line_number] = f'{dependency_definition.requirement}\n'
+
+            write_file_lines(check_req_file, new_lines)
+
+    if not files_updated:
+        echo_info('All dependencies synced.')
+
+    echo_info(f'Files updated: {files_updated}')
+
+
+async def get_version_data(url):
+    async with request('GET', url) as response:
+        try:
+            info = orjson.loads(await response.read())['info']
+        except Exception as e:
+            raise type(e)(f'Error processing URL {url}: {e}')
+        else:
+            return (
+                info['name'],
+                info['version'],
+                {
+                    extract_classifier_value(c)
+                    for c in info['classifiers']
+                    if c.startswith('Programming Language :: Python ::')
+                },
+            )
+
+
+async def scrape_version_data(urls):
+    package_data = defaultdict(lambda: {'version': '', 'classifiers': set()})
+
+    async with Pool() as pool:
+        async for package_name, package_version, version_classifiers in pool.map(get_version_data, urls):
+            data = package_data[package_name]
+            if package_version:
+                data['version'] = package_version
+
+            data['classifiers'].update(version_classifiers)
+
+    return package_data
+
+
+def is_version_compatible(marker, supported_versions):
+    """
+    Determines if any of the given versions are compatible with the given marker
+    """
+    os_markers = ['darwin', 'win32', 'linux', 'aix']
+    is_compatible = False
+
+    if marker is None:
+        # manually set marker since if there's none the dependency should work on all environments
+        marker = Marker('python_version > "2.0"')
+
+    if len(supported_versions) == 0:
+        # if there are no classifiers then assume it works on all Python versions
+        is_compatible = True
+
+    for python_version in supported_versions:
+        has_match = False
+
+        for os in os_markers:
+            env = {'python_version': python_version, 'sys_platform': os}
+            # override system python version and sys platform to evaluate whether the version is compatible
+            has_match = marker.evaluate(environment=env)
+            is_compatible = is_compatible or has_match
+
+    return is_compatible
+
+
+@dep.command(context_settings=CONTEXT_SETTINGS, short_help='Automatically check for dependency updates')
+@click.option('--sync', '-s', is_flag=True, help='Update the `agent_requirements.in` file')
+@click.option(
+    '--check-python-classifiers',
+    '-s',
+    is_flag=True,
+    help="""Only flag a dependency as needing an update if the newest version has python classifiers matching the marker.
+    NOTE: Some packages may not have proper classifiers.""",
+)
+def updates(sync, check_python_classifiers):
+
+    all_agent_dependencies, errors = read_agent_dependencies()
+
+    api_urls = []
+    for package in all_agent_dependencies.keys():
+        api_urls.append(f'https://pypi.org/pypi/{package}/json')
+
+    package_data = asyncio.run(scrape_version_data(api_urls))
+    package_data = {
+        package_name.lower().replace('_', '-'): package_info for package_name, package_info in package_data.items()
+    }
+
+    deps_to_update = {
+        agent_dependency_definition: package_data[package]['version']
+        for package, package_dependency_definitions in all_agent_dependencies.items()
+        if package not in IGNORED_DEPS
+        for agent_dep_version, agent_dependency_definitions in package_dependency_definitions.items()
+        for agent_dependency_definition in agent_dependency_definitions
+        if str(agent_dep_version)[2:] != package_data[package]['version']
+        and (
+            not check_python_classifiers
+            or is_version_compatible(
+                agent_dependency_definition.requirement.marker,
+                package_data[package]['classifiers'],
+            )
+        )
+    }
+
+    if sync:
+        static_file = get_agent_requirements()
+        if deps_to_update:
+            old_lines = read_file_lines(static_file)
+            new_lines = old_lines.copy()
+
+            for dependency_definition, new_version in deps_to_update.items():
+                dependency_definition.requirement.specifier = f'=={new_version}'
+                new_lines[dependency_definition.line_number] = f'{dependency_definition.requirement}\n'
+
+            write_file_lines(static_file, new_lines)
+            echo_info(f'Updated {len(deps_to_update.keys())} dependencies in `agent_requirements.in`')
+    else:
+        if deps_to_update:
+            echo_failure(f"{len(deps_to_update.keys())} Dependencies are out of sync:")
+            for dependency_definition, version in deps_to_update.items():
+                echo_failure(f"Dependency {dependency_definition.requirement} can be updated to version {version}")
+        else:
+            echo_info('All dependencies are up to date')

datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/licenses.py

@@ -123,7 +123,7 @@ def format_attribution_line(package_name, license_id, package_copyright):
     return f'{package_name},PyPI,{license_id},{package_copyright}\n'
 
 
-def extract_license_classifier(classifier):
+def extract_classifier_value(classifier):
     return classifier.split(' :: ')[-1]
 
 
@@ -146,7 +146,7 @@ async def get_data(url):
                 info['name'],
                 info['author'] or info['maintainer'] or info['author_email'] or info['maintainer_email'] or '',
                 info['license'],
-                {extract_license_classifier(c) for c in info['classifiers'] if c.startswith('License ::')},
+                {extract_classifier_value(c) for c in info['classifiers'] if c.startswith('License ::')},
             )
 
 
@@ -162,7 +162,7 @@ async def scrape_license_data(urls):
             data['classifiers'].update(license_classifiers)
             if package_license:
                 if ' :: ' in package_license:
-                    data['classifiers'].add(extract_license_classifier(package_license))
+                    data['classifiers'].add(extract_classifier_value(package_license))
                 else:
                     data['licenses'].append(package_license)
 

datadog_checks_dev/datadog_checks/dev/tooling/dependencies.py

@@ -21,6 +21,18 @@ def __init__(self, name, requirement, file_path, line_number, check_name=None):
     def __repr__(self):
         return f'<DependencyDefinition name={self.name} check_name={self.check_name} requirement={self.requirement}'
 
+    @property
+    def _normalized_marker(self):
+        if self.requirement.marker is None:
+            return self.requirement.marker
+
+        new_marker = str(self.requirement.marker).strip()
+        new_marker = new_marker.replace('\'', "\"")
+        return new_marker
+
+    def same_name_marker(self, other):
+        return self.name == other.name and self._normalized_marker == other._normalized_marker
+
 
 def create_dependency_data():
     return defaultdict(lambda: defaultdict(lambda: []))
@@ -38,7 +50,7 @@ def load_dependency_data(req_file, dependencies, errors, check_name=None):
             errors.append(f'File `{req_file}` has an invalid dependency: `{line}`\n{e}')
             continue
 
-        name = req.name.lower()
+        name = req.name.lower().replace('_', '-')
         dependency = dependencies[name][req.specifier]
         dependency.append(DependencyDefinition(name, req, req_file, i, check_name))
 
@@ -54,7 +66,7 @@ def load_base_check(req_file, dependencies, errors, check_name=None):
                 errors.append(f'File `{req_file}` has an invalid base check dependency: `{line}`\n{e}')
                 return
 
-            name = req.name.lower()
+            name = req.name.lower().replace('_', '-')
             dependency = dependencies[name][req.specifier]
             dependency.append(DependencyDefinition(name, req, req_file, i, check_name))
             return

datadog_checks_dev/datadog_checks/dev/tooling/utils.py

@@ -268,6 +268,10 @@ def get_config_file(check_name):
     return os.path.join(get_data_directory(check_name), 'conf.yaml.example')
 
 
+def get_check_req_file(check_name):
+    return os.path.join(get_root(), check_name, 'requirements.in')
+
+
 def get_config_spec(check_name):
     if check_name == 'agent':
         return os.path.join(get_root(), 'pkg', 'config', 'conf_spec.yaml')