Support multiple Python versions for E2E

datadog_checks_dev/datadog_checks/dev/tooling/commands/env/start.py

@@ -9,7 +9,8 @@
 
 from ....utils import dir_exists, file_exists, path_join
 from ...e2e import E2E_SUPPORTED_TYPES, derive_interface, start_environment, stop_environment
-from ...testing import get_available_tox_envs
+from ...e2e.agent import DEFAULT_PYTHON_VERSION
+from ...testing import get_available_tox_envs, get_tox_env_python_version
 from ...utils import get_tox_file
 from ..console import CONTEXT_SETTINGS, abort, echo_failure, echo_info, echo_success, echo_waiting, echo_warning
 
@@ -27,6 +28,12 @@
         'config (agent5, agent6, etc.)'
     ),
 )
+@click.option(
+    '--python',
+    '-py',
+    type=click.INT,
+    help='The version of Python to use. Defaults to {} if no tox Python is specified.'.format(DEFAULT_PYTHON_VERSION),
+)
 @click.option('--dev/--prod', help='Whether to use the latest version of a check or what is shipped')
 @click.option('--base', is_flag=True, help='Whether to use the latest version of the base check or what is shipped')
 @click.option(
@@ -39,7 +46,7 @@
     ),
 )
 @click.pass_context
-def start(ctx, check, env, agent, dev, base, env_vars):
+def start(ctx, check, env, agent, python, dev, base, env_vars):
     """Start an environment."""
     if not file_exists(get_tox_file(check)):
         abort('`{}` is not a testable check.'.format(check))
@@ -64,6 +71,16 @@ def start(ctx, check, env, agent, dev, base, env_vars):
         echo_info('See what is available via `ddev env ls {}`.'.format(check))
         abort()
 
+    env_python_version = get_tox_env_python_version(env)
+    if not python:
+        # Make the tox environment Python specifier influence the Agent
+        python = env_python_version or DEFAULT_PYTHON_VERSION
+    elif env_python_version and env_python_version != int(python):
+        echo_warning(
+            'The local environment `{}` does not match the expected Python. The Agent will use Python {}. '
+            'To influence the Agent Python version, use the `-py/--python` option.'.format(env, python)
+        )
+
     api_key = ctx.obj['dd_api_key']
     if api_key is None:
         echo_warning(
@@ -113,13 +130,11 @@ def start(ctx, check, env, agent, dev, base, env_vars):
         stop_environment(check, env, metadata=metadata)
         abort()
 
-    metadata_env_vars = metadata.get('env_vars', {})
-    if metadata_env_vars:
-        env_vars = list(env_vars)
-        for key, value in metadata_env_vars.items():
-            env_vars.append('{}={}'.format(key, value))
+    env_vars = dict(ev.split('=', 1) for ev in env_vars)
+    for key, value in metadata.get('env_vars', {}):
+        env_vars.setdefault(key, value)
 
-    environment = interface(check, env, base_package, config, env_vars, metadata, agent_build, api_key)
+    environment = interface(check, env, base_package, config, env_vars, metadata, agent_build, api_key, python)
 
     echo_waiting('Updating `{}`... '.format(agent_build), nl=False)
     environment.update_agent()

datadog_checks_dev/datadog_checks/dev/tooling/commands/env/test.py

@@ -3,6 +3,7 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import click
 
+from ...e2e.agent import DEFAULT_PYTHON_VERSION
 from ...testing import get_tox_envs
 from ..console import CONTEXT_SETTINGS, echo_info, echo_warning
 from ..test import test as test_command
@@ -22,7 +23,13 @@
         'config (agent5, agent6, etc.)'
     ),
 )
-@click.option('--dev/--prod', help='Whether to use the latest version of a check or what is shipped')
+@click.option(
+    '--python',
+    '-py',
+    type=click.INT,
+    help='The version of Python to use. Defaults to {} if no tox Python is specified.'.format(DEFAULT_PYTHON_VERSION),
+)
+@click.option('--dev/--prod', default=None, help='Whether to use the latest version of a check or what is shipped')
 @click.option('--base', is_flag=True, help='Whether to use the latest version of the base check or what is shipped')
 @click.option(
     '--env-vars',
@@ -35,7 +42,7 @@
 )
 @click.option('--new-env', '-ne', is_flag=True, help='Execute setup and tear down actions')
 @click.pass_context
-def test(ctx, checks, agent, dev, base, env_vars, new_env):
+def test(ctx, checks, agent, python, dev, base, env_vars, new_env):
     """Test an environment."""
     check_envs = get_tox_envs(checks, e2e_tests_only=True)
     tests_ran = False
@@ -46,6 +53,10 @@ def test(ctx, checks, agent, dev, base, env_vars, new_env):
     if not checks:
         new_env = True
 
+    # Default to testing the local development version.
+    if dev is None:
+        dev = True
+
     for check, envs in check_envs:
         if not envs:
             echo_warning('No end-to-end environments found for `{}`'.format(check))
@@ -57,7 +68,9 @@ def test(ctx, checks, agent, dev, base, env_vars, new_env):
 
         for env in envs:
             if new_env:
-                ctx.invoke(start, check=check, env=env, agent=agent, dev=dev, base=base, env_vars=env_vars)
+                ctx.invoke(
+                    start, check=check, env=env, agent=agent, python=python, dev=dev, base=base, env_vars=env_vars
+                )
 
             try:
                 ctx.invoke(test_command, checks=['{}:{}'.format(check, env)], e2e=True)

datadog_checks_dev/datadog_checks/dev/tooling/e2e/agent.py

@@ -6,6 +6,7 @@
 from .platform import LINUX, MAC, WINDOWS
 
 DEFAULT_AGENT_VERSION = 6
+DEFAULT_PYTHON_VERSION = 2
 
 # Must be a certain length
 FAKE_API_KEY = 'a' * 32
@@ -39,6 +40,15 @@ def get_agent_exe(agent_version, platform=LINUX):
             return '/opt/datadog-agent/agent/agent.py'
 
 
+def get_pip_exe(python_version, platform=LINUX):
+    if platform == WINDOWS:
+        return [r'C:\Program Files\Datadog\Datadog Agent\embedded{}\python.exe'.format(python_version), '-m', 'pip']
+    elif platform == MAC:
+        pass
+    else:
+        return ['/opt/datadog-agent/embedded/bin/pip{}'.format(python_version)]
+
+
 def get_agent_conf_dir(check, agent_version, platform=LINUX):
     if platform == WINDOWS:
         if agent_version >= 6:
@@ -64,13 +74,6 @@ def get_agent_version_manifest(platform):
         return '/opt/datadog-agent/version-manifest.txt'
 
 
-def get_agent_pip_install(version, platform):
-    if platform == WINDOWS:
-        return [r'C:\Program Files\Datadog\Datadog Agent\embedded\python', '-m', 'pip', 'install']
-    else:
-        return ['/opt/datadog-agent/embedded/bin/pip', 'install', '--user']
-
-
 def get_agent_service_cmd(version, platform, action):
     if platform == WINDOWS:
         arg = 'start-service' if action == 'start' else 'stopservice'

datadog_checks_dev/datadog_checks/dev/tooling/e2e/docker.py

@@ -9,11 +9,13 @@
 from ..constants import get_root
 from .agent import (
     DEFAULT_AGENT_VERSION,
+    DEFAULT_PYTHON_VERSION,
     FAKE_API_KEY,
     MANIFEST_VERSION_PATTERN,
     get_agent_conf_dir,
     get_agent_exe,
     get_agent_version_manifest,
+    get_pip_exe,
     get_rate_flag,
 )
 from .config import config_file_name, env_exists, locate_config_dir, locate_config_file, remove_env_data, write_env_data
@@ -23,23 +25,35 @@ class DockerInterface(object):
     ENV_TYPE = 'docker'
 
     def __init__(
-        self, check, env, base_package=None, config=None, env_vars=None, metadata=None, agent_build=None, api_key=None
+        self,
+        check,
+        env,
+        base_package=None,
+        config=None,
+        env_vars=None,
+        metadata=None,
+        agent_build=None,
+        api_key=None,
+        python_version=DEFAULT_PYTHON_VERSION,
     ):
         self.check = check
         self.env = env
-        self.env_vars = env_vars
+        self.env_vars = env_vars or {}
         self.base_package = base_package
         self.config = config or {}
         self.metadata = metadata or {}
         self.agent_build = agent_build
         self.api_key = api_key or FAKE_API_KEY
+        self.python_version = python_version or DEFAULT_PYTHON_VERSION
 
         self._agent_version = self.metadata.get('agent_version')
         self.container_name = 'dd_{}_{}'.format(self.check, self.env)
         self.config_dir = locate_config_dir(check, env)
         self.config_file = locate_config_file(check, env)
         self.config_file_name = config_file_name(self.check)
 
+        self.env_vars['DD_PYTHON_VERSION'] = self.python_version
+
     @property
     def agent_version(self):
         return self._agent_version or DEFAULT_AGENT_VERSION
@@ -146,11 +160,15 @@ def detect_agent_version(self):
             self.metadata['agent_version'] = self.agent_version
 
     def update_check(self):
-        command = ['docker', 'exec', self.container_name, 'pip', 'install', '-e', self.check_mount_dir]
+        command = ['docker', 'exec', self.container_name]
+        command.extend(get_pip_exe(self.python_version))
+        command.extend(('install', '-e', self.check_mount_dir))
         run_command(command, capture=True, check=True)
 
     def update_base_package(self):
-        command = ['docker', 'exec', self.container_name, 'pip', 'install', '-e', self.base_mount_dir]
+        command = ['docker', 'exec', self.container_name]
+        command.extend(get_pip_exe(self.python_version))
+        command.extend(('install', '-e', self.base_mount_dir))
         run_command(command, capture=True, check=True)
 
     def update_agent(self):
@@ -186,8 +204,8 @@ def start_agent(self):
             ]
 
             # Any environment variables passed to the start command
-            for var in self.env_vars:
-                command.extend(['-e', var])
+            for key, value in sorted(self.env_vars.items()):
+                command.extend(['-e', '{}={}'.format(key, value)])
 
             if self.base_package:
                 # Mount the check directory

datadog_checks_dev/datadog_checks/dev/tooling/e2e/local.py

@@ -6,18 +6,20 @@
 from contextlib import contextmanager
 from shutil import copyfile, move
 
+from ...structures import EnvVars
 from ...subprocess import run_command
 from ...utils import ON_LINUX, ON_MACOS, ON_WINDOWS, file_exists, path_join
 from ..constants import get_root
 from .agent import (
     DEFAULT_AGENT_VERSION,
+    DEFAULT_PYTHON_VERSION,
     FAKE_API_KEY,
     MANIFEST_VERSION_PATTERN,
     get_agent_conf_dir,
     get_agent_exe,
-    get_agent_pip_install,
     get_agent_service_cmd,
     get_agent_version_manifest,
+    get_pip_exe,
     get_rate_flag,
 )
 from .config import config_file_name, locate_config_dir, locate_config_file, remove_env_data, write_env_data
@@ -28,23 +30,35 @@ class LocalAgentInterface(object):
     ENV_TYPE = 'local'
 
     def __init__(
-        self, check, env, base_package=None, config=None, env_vars=None, metadata=None, agent_build=None, api_key=None
+        self,
+        check,
+        env,
+        base_package=None,
+        config=None,
+        env_vars=None,
+        metadata=None,
+        agent_build=None,
+        api_key=None,
+        python_version=DEFAULT_PYTHON_VERSION,
     ):
         self.check = check
         self.env = env
         self.base_package = base_package
         self.config = config or {}
         # Env vars are not currently used in local E2E
-        self.env_vars = env_vars
+        self.env_vars = env_vars or {}
         self.metadata = metadata or {}
         self.agent_build = agent_build
         self.api_key = api_key or FAKE_API_KEY
+        self.python_version = python_version or DEFAULT_PYTHON_VERSION
 
         self._agent_version = self.metadata.get('agent_version')
         self.config_dir = locate_config_dir(check, env)
         self.config_file = locate_config_file(check, env)
         self.config_file_name = config_file_name(self.check)
 
+        self.env_vars['DD_PYTHON_VERSION'] = self.python_version
+
     @property
     def platform(self):
         if ON_LINUX:
@@ -145,15 +159,14 @@ def run_check(
         return run_command(command, capture=capture)
 
     def update_check(self):
-        install_cmd = get_agent_pip_install(self.agent_version, self.platform) + [
-            '-e',
-            path_join(get_root(), self.check),
-        ]
-        return run_command(install_cmd, capture=True, check=True)
+        command = get_pip_exe(self.python_version, self.platform)
+        command.extend(('install', '-e', path_join(get_root(), self.check)))
+        return run_command(command, capture=True, check=True)
 
     def update_base_package(self):
-        install_cmd = get_agent_pip_install(self.agent_version, self.platform) + ['-e', self.base_package]
-        return run_command(install_cmd, capture=True, check=True)
+        command = get_pip_exe(self.python_version, self.platform)
+        command.extend(('install', '-e', self.base_package))
+        return run_command(command, capture=True, check=True)
 
     def update_agent(self):
         # The Local E2E assumes an Agent is already installed on the machine
@@ -170,7 +183,9 @@ def detect_agent_version(self):
             self.metadata['agent_version'] = self.agent_version
 
     def start_agent(self):
-        command = get_agent_service_cmd(self.agent_version, self.platform, 'start')
+        with EnvVars(self.env_vars):
+            command = get_agent_service_cmd(self.agent_version, self.platform, 'start')
+
         return run_command(command, capture=True)
 
     def stop_agent(self):

datadog_checks_dev/datadog_checks/dev/tooling/testing.py

@@ -1,6 +1,8 @@
 # (C) Datadog, Inc. 2018
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
+import re
+
 from ..subprocess import run_command
 from ..utils import chdir, path_join, read_file_binary, write_file_binary
 from .constants import TESTABLE_FILE_EXTENSIONS, get_root
@@ -9,6 +11,7 @@
 
 STYLE_CHECK_ENVS = {'flake8', 'style'}
 STYLE_ENVS = {'flake8', 'style', 'format_style'}
+PYTHON_MAJOR_PATTERN = r'py(\d)'
 
 
 def get_tox_envs(
@@ -215,3 +218,9 @@ def get_changed_checks():
     changed_files[:] = testable_files(changed_files)
 
     return {line.split('/')[0] for line in changed_files}
+
+
+def get_tox_env_python_version(env):
+    match = re.match(PYTHON_MAJOR_PATTERN, env)
+    if match:
+        return int(match.group(1))