[gitlab] A6+A7 builds on the same pipeline (#4210)

* [release] build nightly with python3 only

* [invoke] fix tasks issues

* [config] default python should be python3

* [gitlab] use new builders

* [invoke] support for other default potential locations libs

* [config] default python version specified at build time

* [gitlab] py2 + py3 builds

* [gitlab] init conda for tests

* [gitlab] fix jobs

* [invoke] try to address versioning disparities

* [gitlab] pull dependencies

* [gitlab] python_runtimes quoted

* [invoke] if not A7, then A6

* [invoke] fix get_version_numeric_only

* [invoke] single rpath switch supported, use colon

* [windows] fix windows py3 build

* [invoke] pass environment for numeric version hints

* [gitlab] duplicate kitchen jobs

* [gitlab+kitchen] support multi-build testing

* [gitlab+kitchen] support multi-build testing, individual resource groups

* [omnibus] chef-sugar workaround

* [kitchen] azure: cleanup using DD_PIPELINE_ID which includes A6/A7 suffix

* [Dockerfile] agent: add build arg for artifact blob

* [gitlab] more A6+A7 duplicates: image builds, artifact deploy, etc

* [gitlab+kitchen] support for A6+A7 testing

* [gitlab] pkg_size_send: fix broken stats reporting

* [gitlab] windows: fix variable interpolation (please)

* [gitlab] windows: fix variable interpolation (again)

* [gitlab] windows substition some other way

* [kitchen] increase service stop/start timeouts (30s)

* [kitchen] dd-agent-install notify immediately, not delayed

* [gitlab] kitchen: allow A7 failures while investigating

* Build python 3 by default

* [gitlab] kitchen: break off failing 2008 A7 tests

* [gitlab] kitchen: fix windows installer tests on A7

* Revert "[kitchen] dd-agent-install notify immediately, not delayed"

This reverts commit 66b9a6d.

* [kitchen] dd-agent: increase restart timeout

* [kitchen] dd-agent: increase restart timeout further (60s)

* [kitchen] windows: try larger VMs

* [gitlab] fix docker image builds + typo correction

* [gitlab] fix docker image builds + typo correction

* [gitlab] fix docker image builds + typo correction

* [dockerfiles] agent: fix docker image builds

* [gitlab] send_pkg: initialize the conda environments for scripts

* [gitlab][docker] debug docker, disable kitchen for speed

* [gitlab] fully disable - broken yaml

* [gitlab] more debug

* [gitlab][docker] should now be fixed

* [gitlab] fix windows releasing

* address PR feedback

* [gitlab] DRY

* [kitchen] bumping winstall instance sizes

* [gitlab] DRY

* [gitlab] addressing feedback

* [rtloader][test] init: python3 has different init allocations

* [gitlab] address review feedback

* [gitlab] allow failures in kitchen_windows_install-a6

Dockerfiles/agent/Dockerfile

@@ -5,6 +5,9 @@
 FROM debian:buster-slim AS extract
 ARG WITH_JMX
 ARG PYTHON_VERSION
+ARG DD_AGENT_ARTIFACT=datadog-agent*_amd64.deb
+
+# copy everything - globbing with args wont work
 COPY datadog-agent*_amd64.deb /
 WORKDIR /output
 
@@ -23,7 +26,10 @@ RUN apt-get update \
 #   - docs and manpages                  # FIXME: move upstream
 #   - static libraries                   # FIXME: move upstream
 #   - jmxfetch on nojmx build
-RUN find / -name 'datadog-agent*_amd64.deb' ! -name 'datadog-agent-dbg*_amd64.deb' -exec dpkg -X {} . \; | egrep '.*' \
+
+# DEBUGGING
+RUN find / -maxdepth 1 -type f -name 'datadog-agent*_amd64.deb' ! -name "$DD_AGENT_ARTIFACT" -exec rm {} \; \
+ && find / -maxdepth 1 -name 'datadog-agent*_amd64.deb' -exec dpkg -X {} . \; | egrep '.*' \
  && rm -rf usr etc/init lib \
     opt/datadog-agent/sources \
     opt/datadog-agent/embedded/share/doc \
@@ -34,8 +40,8 @@ RUN find / -name 'datadog-agent*_amd64.deb' ! -name 'datadog-agent-dbg*_amd64.de
     usr/lib/x86_64-linux-gnu/libsystemd.so.0.21.0 \
     # self-test certificates that are detected (false positive) as private keys
     opt/datadog-agent/embedded/lib/python2.7/site-packages/Cryptodome/SelfTest \
- && if [ "$PYTHON_VERSION" = "2" ]; then rm -rf opt/datadog-agent/embedded/lib/python3.*; fi \
- && if [ "$PYTHON_VERSION" = "3" ]; then rm -rf opt/datadog-agent/embedded/lib/python2.*; fi \
+ && if [ "$PYTHON_VERSION" = "2" ]; then rm -rf opt/datadog-agent/embedded/lib/python3.* || true; fi \
+ && if [ "$PYTHON_VERSION" = "3" ]; then rm -rf opt/datadog-agent/embedded/lib/python2.* || true; fi \
  && find opt/datadog-agent/ -iname "*.a" -delete \
  && if [ -z "$WITH_JMX" ]; then rm -rf opt/datadog-agent/bin/agent/dist/jmx; fi \
  && ln -s /opt/datadog-agent/embedded/ssl etc/ssl \

omnibus/lib/ostools.rb

@@ -43,6 +43,6 @@ def os
 end
 
 def with_python_runtime?(runtime)
-    python_runtimes = ENV['PYTHON_RUNTIMES'].nil? ? ['2'] : ENV['PYTHON_RUNTIMES'].split(',')
+    python_runtimes = ENV['PYTHON_RUNTIMES'].nil? ? ['3'] : ENV['PYTHON_RUNTIMES'].split(',')
     return python_runtimes.include? runtime
 end

omnibus/resources/agent/msi/source.wxs.erb

@@ -254,7 +254,10 @@
                 Source="$(var.BinFiles)/libdatadog-agent-three.dll"/>
             <?endif ?>
         </Component>
+
+        <?if $(var.IncludePython2) = true ?>
         <Component Id="MSVCRUNTIME" Guid="30ce4f57-47ce-47c0-8564-e510d69787a9">
+
             <Condition> <![CDATA[VersionNT <= 601]]> </Condition>
             <File Id="msvcm90.dll"
                 Name="msvcm90.dll"
@@ -277,7 +280,9 @@
                 Vital="yes"
                 DiskId="1"
                 Source="$(var.BinFiles)/Microsoft.VC90.CRT.manifest"/>
+
         </Component>
+        <?endif ?>
         <Directory Id="AGENT" Name="agent">
           <!-- Windows service declaration for APM agent -->
           <Component Id="DATADOGAPMSERVICE" Guid="a8e611fa-aedb-4c16-969c-bb827af0bd53">
@@ -452,7 +457,9 @@
       <ComponentGroupRef Id="ExtraEXAMPLECONFSLOCATION" />
       <ComponentRef Id="CleanupMainApplicationFolder" />
       <ComponentRef Id="RegisterConfVariables" />
-      <ComponentRef Id="MSVCRUNTIME" />
+      <?if $(var.IncludePython2) = true ?>
+        <ComponentRef Id="MSVCRUNTIME" />
+      <?endif ?>
     </Feature>
 
     <!-- UI Stuff: 100% Omnibus Generated -->

pkg/config/config.go

@@ -49,6 +49,11 @@ var (
 	proxies *Proxy
 )
 
+// Variables to initialize at build time
+var (
+	DefaultPython string
+)
+
 // MetadataProviders helps unmarshalling `metadata_providers` config param
 type MetadataProviders struct {
 	Name     string        `mapstructure:"name"`
@@ -133,7 +138,7 @@ func initConfig(config Config) {
 	config.BindEnvAndSetDefault("ipc_address", "localhost")
 	config.BindEnvAndSetDefault("health_port", int64(0))
 	config.BindEnvAndSetDefault("disable_py3_validation", false)
-	config.BindEnvAndSetDefault("python_version", "2")
+	config.BindEnvAndSetDefault("python_version", DefaultPython)
 	// Debugging + C-land crash feature flags
 	config.BindEnvAndSetDefault("c_stacktrace_collection", false)
 	config.BindEnvAndSetDefault("c_core_dump", false)

release.json

@@ -5,7 +5,7 @@
 		"OMNIBUS_RUBY_VERSION": "datadog-5.5.0",
 		"JMXFETCH_VERSION": "0.32.1",
 		"JMXFETCH_HASH": "01233bc18f91367476b25d5d9f64e275ed3ffe3d27efc067139ce1631e143471",
-		"PYTHON_RUNTIMES": "2,3"
+		"PYTHON_RUNTIMES": "3"
 	},
 	"6.14.0": {
 		"INTEGRATIONS_CORE_VERSION": "6.14.0",

rtloader/test/init/init_test.go

@@ -15,5 +15,5 @@ func TestInit(t *testing.T) {
 	}
 
 	// Check for expected allocations
-	helpers.AssertMemoryExpectation(t, helpers.Allocations, 1)
+	helpers.AssertMemoryExpectation(t, helpers.Allocations, initAllocations)
 }

rtloader/test/init/init_three.go

@@ -0,0 +1,7 @@
+// +build three
+
+package testinit
+
+const (
+	initAllocations = 0
+)

rtloader/test/init/init_two.go

@@ -0,0 +1,7 @@
+// +build two
+
+package testinit
+
+const (
+	initAllocations = 1
+)

tasks/agent.py

@@ -120,7 +120,7 @@ def build(ctx, rebuild=False, race=False, build_include=None, build_exclude=None
         # This generates the manifest resource. The manifest resource is necessary for
         # being able to load the ancient C-runtime that comes along with Python 2.7
         # command = "rsrc -arch amd64 -manifest cmd/agent/agent.exe.manifest -o cmd/agent/rsrc.syso"
-        ver = get_version_numeric_only(ctx)
+        ver = get_version_numeric_only(ctx, env)
         build_maj, build_min, build_patch = ver.split(".")
 
         command = "windmc --target {target_arch} -r cmd/agent cmd/agent/agentmsg.mc ".format(target_arch=windres_target)
@@ -372,7 +372,7 @@ def omnibus_build(ctx, puppy=False, log_level="info", base_dir=None, gem_path=No
             if skip_sign:
                 env['SKIP_SIGN_MAC'] = 'true'
 
-            env['PACKAGE_VERSION'] = get_version(ctx, include_git=True, url_safe=True)
+            env['PACKAGE_VERSION'] = get_version(ctx, include_git=True, url_safe=True, env=env)
 
             ctx.run(cmd.format(**args), env=env)
 

tasks/android.py

@@ -14,7 +14,7 @@
 from invoke import task
 from invoke.exceptions import Exit
 
-from .utils import bin_name, get_build_flags, get_version_numeric_only, load_release_versions, get_version
+from .utils import bin_name, get_build_flags, load_release_versions, get_version
 from .utils import REPO_PATH
 from .build_tags import get_build_tags, get_default_build_tags, LINUX_ONLY_TAGS, REDHAT_AND_DEBIAN_ONLY_TAGS, REDHAT_AND_DEBIAN_DIST
 from .go import deps

tasks/customaction.py

@@ -32,7 +32,7 @@ def build(ctx, vstudio_root=None, arch="x64"):
         print("Custom action library is only for Win32")
         raise Exit(code=1)
 
-    ver = get_version_numeric_only(ctx)
+    ver = get_version_numeric_only(ctx, env=os.environ)
     build_maj, build_min, build_patch = ver.split(".")
     verprops = " /p:MAJ_VER={build_maj} /p:MIN_VER={build_min} /p:PATCH_VER={build_patch} ".format(
             build_maj=build_maj,

tasks/process_agent.py

@@ -29,7 +29,7 @@ def build(ctx, race=False, go_version=None, incremental_build=False, puppy=False
             env["GOARCH"] = "386"
             windres_target = "pe-i386"
 
-        ver = get_version_numeric_only(ctx)
+        ver = get_version_numeric_only(ctx, env)
         maj_ver, min_ver, patch_ver = ver.split(".")
         resdir = os.path.join(".", "cmd", "process-agent", "windows_resources")
 

tasks/rtloader.py

@@ -42,7 +42,7 @@ def build(ctx, install_prefix=None, python_runtimes=None, cmake_options='', arch
 
     cmake_args = cmake_options + " -DBUILD_DEMO:BOOL=OFF -DCMAKE_INSTALL_PREFIX:PATH={}".format(install_prefix or dev_path)
 
-    python_runtimes = python_runtimes or os.environ.get("PYTHON_RUNTIMES") or "2"
+    python_runtimes = python_runtimes or os.environ.get("PYTHON_RUNTIMES") or "3"
     python_runtimes = python_runtimes.split(',')
 
     settings = {

tasks/systray.py

@@ -35,14 +35,14 @@ def build(ctx, rebuild=False, race=False, build_include=None, build_exclude=None
     # This generates the manifest resource. The manifest resource is necessary for
     # being able to load the ancient C-runtime that comes along with Python 2.7
     # command = "rsrc -arch amd64 -manifest cmd/agent/agent.exe.manifest -o cmd/agent/rsrc.syso"
-    ver = get_version_numeric_only(ctx)
+    ver = get_version_numeric_only(ctx, env=os.environ)
     build_maj, build_min, build_patch = ver.split(".")
     env = {}
     windres_target = "pe-x86-64"
     if arch == "x86":
         env["GOARCH"] = "386"
         windres_target = "pe-i386"
-    
+
     command = "windres -v  --target {target_arch} --define MAJ_VER={build_maj} --define MIN_VER={build_min} --define PATCH_VER={build_patch} ".format(
         build_maj=build_maj,
         build_min=build_min,

tasks/trace_agent.py

@@ -37,7 +37,7 @@ def build(ctx, rebuild=False, race=False, precompile_only=False, build_include=N
             env["GOARCH"] = "386"
             windres_target = "pe-i386"
 
-        ver = get_version_numeric_only(ctx)
+        ver = get_version_numeric_only(ctx, env)
         maj_ver, min_ver, patch_ver = ver.split(".")
 
         ctx.run("windmc --target {target_arch}  -r cmd/trace-agent/windows_resources cmd/trace-agent/windows_resources/trace-agent-msg.mc".format(target_arch=windres_target))

tasks/utils.py

@@ -37,9 +37,10 @@ def get_gopath(ctx):
 
 def get_multi_python_location(embedded_path=None, rtloader_root=None):
     if rtloader_root is None:
-        rtloader_lib = "{}/lib".format(embedded_path)
+        rtloader_lib = ["{}/lib".format(embedded_path),
+                        "{}/lib64".format(embedded_path)]
     else: # if rtloader_root is specified we're working in dev mode from the rtloader folder
-        rtloader_lib = "{}/rtloader".format(rtloader_root)
+        rtloader_lib = ["{}/rtloader".format(rtloader_root)]
 
     rtloader_headers = "{}/include".format(rtloader_root or embedded_path)
     rtloader_common_headers = "{}/common".format(rtloader_root or embedded_path)
@@ -58,6 +59,9 @@ def get_build_flags(ctx, static=False, prefix=None, embedded_path=None,
     ldflags = get_version_ldflags(ctx, prefix)
     env = {}
 
+    # lets pass the build runtimes around with the env
+    env['PYTHON_RUNTIMES'] = os.environ.get('PYTHON_RUNTIMES', '')
+
     if sys.platform == 'win32':
         env["CGO_LDFLAGS_ALLOW"] = "-Wl,--allow-multiple-definition"
 
@@ -74,18 +78,20 @@ def get_build_flags(ctx, static=False, prefix=None, embedded_path=None,
     if python_home_3:
         ldflags += "-X {}/pkg/collector/python.pythonHome3={} ".format(REPO_PATH, python_home_3)
 
+    ldflags += "-X {}/pkg/config.DefaultPython={} ".format(REPO_PATH, get_default_python())
+
     # adding rtloader libs and headers to the env
-    env['DYLD_LIBRARY_PATH'] = os.environ.get('DYLD_LIBRARY_PATH', '') + ":{}".format(rtloader_lib) # OSX
-    env['LD_LIBRARY_PATH'] = os.environ.get('LD_LIBRARY_PATH', '') + ":{}".format(rtloader_lib) # linux
-    env['CGO_LDFLAGS'] = os.environ.get('CGO_LDFLAGS', '') + " -L{}".format(rtloader_lib)
+    env['DYLD_LIBRARY_PATH'] = os.environ.get('DYLD_LIBRARY_PATH', '') + ":{}".format(':'.join(rtloader_lib)) # OSX
+    env['LD_LIBRARY_PATH'] = os.environ.get('LD_LIBRARY_PATH', '') + ":{}".format(':'.join(rtloader_lib)) # linux
+    env['CGO_LDFLAGS'] = os.environ.get('CGO_LDFLAGS', '') + " -L{}".format(' -L '.join(rtloader_lib))
     env['CGO_CFLAGS'] = os.environ.get('CGO_CFLAGS', '') + " -w -I{} -I{}".format(rtloader_headers,
                                                                                   rtloader_common_headers)
 
     # if `static` was passed ignore setting rpath, even if `embedded_path` was passed as well
     if static:
         ldflags += "-s -w -linkmode=external '-extldflags=-static' "
     else:
-        ldflags += "-r {}/lib ".format(embedded_path)
+        ldflags += "-r {}/lib:{}/lib64 ".format(embedded_path, embedded_path)
 
     if os.environ.get("DELVE"):
         gcflags = "-N -l"
@@ -152,6 +158,14 @@ def get_git_commit():
     """
     return check_output(['git', 'rev-parse', '--short', 'HEAD']).decode('utf-8').strip()
 
+def get_default_python():
+    """
+    Get the default python for the current build
+    """
+    py_runtimes = os.environ.get("PYTHON_RUNTIMES", "3")
+    return py_runtimes if ',' not in py_runtimes else "3"
+
+
 def get_go_version():
     """
     Get the version of Go used
@@ -172,14 +186,17 @@ def get_git_branch_name():
     return check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode('utf-8').strip()
 
 
-def query_version(ctx, git_sha_length=7, prefix=None):
+def query_version(ctx, git_sha_length=7, prefix=None, hint=None):
     # The string that's passed in will look something like this: 6.0.0-beta.0-1-g4f19118
     # if the tag is 6.0.0-beta.0, it has been one commit since the tag and that commit hash is g4f19118
     cmd = "git describe --tags --candidates=50"
     if prefix and type(prefix) == str:
         cmd += " --match \"{}-*\"".format(prefix)
     else:
-        cmd += " --match \"[0-9]*\""
+        if hint:
+            cmd += " --match \"{}\.*\"".format(hint)
+        else:
+            cmd += " --match \"[0-9]*\""
     if git_sha_length and type(git_sha_length) == int:
         cmd += " --abbrev={}".format(git_sha_length)
     described_version = ctx.run(cmd, hide=True).stdout.strip()
@@ -216,10 +233,12 @@ def query_version(ctx, git_sha_length=7, prefix=None):
     return version, pre, commit_number, git_sha
 
 
-def get_version(ctx, include_git=False, url_safe=False, git_sha_length=7, prefix=None):
+def get_version(ctx, include_git=False, url_safe=False, git_sha_length=7, prefix=None, env=os.environ):
     # we only need the git info for the non omnibus builds, omnibus includes all this information by default
+    version_hint = '7' if env.get('PYTHON_RUNTIMES', '') == '3' else '6'
+
     version = ""
-    version, pre, commits_since_version, git_sha = query_version(ctx, git_sha_length, prefix)
+    version, pre, commits_since_version, git_sha = query_version(ctx, git_sha_length, prefix, hint=version_hint)
     if pre:
         version = "{0}-{1}".format(version, pre)
     if commits_since_version and include_git:
@@ -231,14 +250,23 @@ def get_version(ctx, include_git=False, url_safe=False, git_sha_length=7, prefix
     # version could be unicode as it comes from `query_version`
     return str(version)
 
-def get_version_numeric_only(ctx):
-    version, _, _, _ = query_version(ctx)
+def get_version_numeric_only(ctx, env=os.environ):
+    # we only need the git info for the non omnibus builds, omnibus includes all this information by default
+    version_hint = '7' if env.get('PYTHON_RUNTIMES', '') == '3' else '6'
+
+    version, _, _, _ = query_version(ctx, hint=version_hint)
     return version
 
 def load_release_versions(ctx, target_version):
+    # allow overriding python runtimes with env var
+    py_runtimes = os.environ.get("PYTHON_RUNTIMES")
+
     with open("release.json", "r") as f:
         versions = json.load(f)
         if target_version in versions:
+            if py_runtimes:
+                versions[target_version]["PYTHON_RUNTIMES"] = py_runtimes
+
             # windows runners don't accepts anything else than strings in the
             # environment when running a subprocess.
             return {str(k):str(v) for k, v in versions[target_version].items()}

test/kitchen/kitchen-azure-winstall.yml

@@ -50,8 +50,8 @@ driver:
 driver_config:
   subscription_id: <%= ENV['AZURE_SUBSCRIPTION_ID'] %>
   location: <%= ENV['AZURE_LOCATION'] %>
-  <% if ENV['CI_PIPELINE_ID'] %>
-  azure_resource_group_suffix: pl<%= ENV['CI_PIPELINE_ID'] %>
+  <% if ENV['DD_PIPELINE_ID'] %>
+  azure_resource_group_suffix: pl<%= ENV['DD_PIPELINE_ID'] %>
   <% else %>
   azure_resource_group_suffix: plnone
   <% end %>
@@ -73,7 +73,7 @@ platforms:
     ]
 
     windows_sizes = [
-      "Standard_D1_v2"
+      "Standard_D2_v2"
     ]
 
     location = "North Central US"
@@ -143,22 +143,22 @@ suites:
 <%
   aptrepo = "http://apttesting.datad0g.com/"
   api_key = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
-  yumrepo = "http://yumtesting.datad0g.com/pipeline-#{ENV['CI_PIPELINE_ID']}/x86_64/"
-  yumrepo_suse = "http://yumtesting.datad0g.com/suse/pipeline-#{ENV['CI_PIPELINE_ID']}/x86_64/"
+  yumrepo = "http://yumtesting.datad0g.com/pipeline-#{ENV['DD_PIPELINE_ID']}/x86_64/"
+  yumrepo_suse = "http://yumtesting.datad0g.com/suse/pipeline-#{ENV['DD_PIPELINE_ID']}/x86_64/"
   windows_agent_url = ENV['WINDOWS_AGENT_URL'] ? ENV['WINDOWS_AGENT_URL'] : "https://s3.amazonaws.com/#{ENV['WINDOWS_TESTING_S3_BUCKET']}/"
   dd_agent_config = {
     'agent6': true,
     'api_key': api_key,
     'application_key': "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
     'url': "https://app.datad0g.com",
     'aptrepo': aptrepo,
-    'aptrepo_dist': "pipeline-#{ENV['CI_PIPELINE_ID']}",
+    'aptrepo_dist': "pipeline-#{ENV['DD_PIPELINE_ID']}",
     'yumrepo': yumrepo,
     'yumrepo_suse': yumrepo_suse,
     'agent6_aptrepo': "http://apttesting.datad0g.com/",
-    'agent6_aptrepo_dist': "pipeline-#{ENV['CI_PIPELINE_ID']}",
-    'agent6_yumrepo': "http://yumtesting.datad0g.com/pipeline-#{ENV['CI_PIPELINE_ID']}/x86_64/",
-    'agent6_yumrepo_suse': "http://yumtesting.datad0g.com/suse/pipeline-#{ENV['CI_PIPELINE_ID']}/x86_64/",
+    'agent6_aptrepo_dist': "pipeline-#{ENV['DD_PIPELINE_ID']}",
+    'agent6_yumrepo': "http://yumtesting.datad0g.com/pipeline-#{ENV['DD_PIPELINE_ID']}/x86_64/",
+    'agent6_yumrepo_suse': "http://yumtesting.datad0g.com/suse/pipeline-#{ENV['DD_PIPELINE_ID']}/x86_64/",
     'windows_agent_url': windows_agent_url,
   }
 %>
@@ -176,7 +176,6 @@ suites:
         <% end %>
       dd-agent-install:
         agent6: true
-        windows_agent_url: <%= windows_agent_url %>
         <% if ENV['AGENT_VERSION'] %>
         windows_version: "<%= ENV['AGENT_VERSION'] %>"
         <% end %>