[gitlab] Use new public Windows builders (#5479)

Uses public Windows builders for test and build jobs.
For now, the old test jobs + x64 builds are kept to compare the new jobs and the old ones.

The artifacts made with the new builders will be used when publishing.

.gitlab-ci.yml

@@ -82,7 +82,8 @@ variables:
   RELEASE_VERSION_7: nightly-a7
   DATADOG_AGENT_BUILDIMAGES: v2424505-0439a40
   DATADOG_AGENT_BUILDERS: v2448672-5304c3c
-  DATADOG_AGENT_WINBUILDIMAGES: v2348149-ba6640d
+  DATADOG_AGENT_WINBUILDIMAGES: v2468030-cbaf3fe
+  DATADOG_AGENT_WINBUILDERS: v2348149-ba6640d
   DATADOG_AGENT_ARMBUILDIMAGES: v2424505-0439a40
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v2424505-0439a40
   BCC_VERSION: v0.12.0
@@ -222,12 +223,40 @@ build_libbcc_arm64:
 #
 
 # run tests for windows-64
+.run_old_tests_windows_base:
+  stage: source_test
+  needs: ["fail_on_non_triggered_tag"]
+  tags: ["runner:windows-docker", "windowsversion:1809"]
+  script:
+    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e IS_AWS_CONTAINER=true -e SIGN_WINDOWS=true -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e NEW_BUILDER="$NEW_BUILDER" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDERS} c:\mnt\tasks\winbuildscripts\unittests.bat
+  variables:
+    NEW_BUILDER: "false"
+
+run_old_tests_windows-x64:
+  # temporarily allow failure for tests
+  extends: .run_old_tests_windows_base
+  allow_failure: true
+  variables:
+    PYTHON_RUNTIMES: 3
+    ARCH: "x64"
+
+run_old_tests_windows-x86:
+  <<: *skip_when_triggered
+  # temporarily allow failure for tests
+  extends: .run_old_tests_windows_base
+  allow_failure: true
+  variables:
+    PYTHON_RUNTIMES: 3
+    ARCH: "x86"
+
 .run_tests_windows_base:
   stage: source_test
   needs: ["fail_on_non_triggered_tag"]
   tags: ["runner:windows-docker", "windowsversion:1809"]
   script:
-    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e IS_AWS_CONTAINER=true -e SIGN_WINDOWS=true -e PY_RUNTIMES="$PYTHON_RUNTIMES" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\unittests.bat
+    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e NEW_BUILDER="$NEW_BUILDER" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\unittests.bat
+  variables:
+    NEW_BUILDER: "true"
 
 run_tests_windows-x64:
   # temporarily allow failure for tests
@@ -1137,7 +1166,7 @@ windows_zip_agent_binaries_x64-a7:
     - if (Test-Path .omnibus) { remove-item -recurse -force .omnibus }
     - if (Test-Path build-out) { remove-item -recurse -force build-out }
     - mkdir .omnibus\pkg
-    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e OMNIBUS_TARGET=${OMNIBUS_TARGET} -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e IS_AWS_CONTAINER=true -e SIGN_WINDOWS=true 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\buildwin.bat
+    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e OMNIBUS_TARGET=${OMNIBUS_TARGET} -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\buildwin.bat
     - copy build-out\*.zip .omnibus\pkg
   artifacts:
     expire_in: 2 weeks
@@ -1149,6 +1178,52 @@ windows_zip_agent_binaries_x64-a7:
 ## windows dockerized builds
 ##
 
+.windows_old_msi_base:
+  stage: package_build
+  needs: ["fail_on_non_triggered_tag", "run_dep_check_lock"]
+  tags: ["runner:windows-docker", "windowsversion:1809"]
+  # Unavailable on gitlab < 12.3
+  # timeout: 2h 00m
+  script:
+    - if (Test-Path .omnibus) { remove-item -recurse -force .omnibus }
+    - if (Test-Path build-out) { remove-item -recurse -force build-out }
+    - mkdir .omnibus\pkg
+    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e CI_JOB_ID=${CI_JOB_ID} -e OMNIBUS_TARGET=${OMNIBUS_TARGET} -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e IS_AWS_CONTAINER=true -e SIGN_WINDOWS=true -e TARGET_ARCH="$ARCH" -e NEW_BUILDER=false 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDERS} c:\mnt\tasks\winbuildscripts\oldbuildwin.bat
+    - copy build-out\${CI_JOB_ID}\*.msi .omnibus\pkg
+    - if (Test-Path build-out\${CI_JOB_ID}\*.zip) { copy build-out\${CI_JOB_ID}\*.zip .omnibus\pkg }
+    - remove-item -recurse -force build-out\${CI_JOB_ID}
+    - get-childitem build-out
+    - get-childitem .omnibus\pkg
+  artifacts:
+    expire_in: 2 weeks
+    paths:
+      - .omnibus/pkg
+
+.windows_old_main_agent_base:
+  extends: .windows_old_msi_base
+  variables:
+    OMNIBUS_TARGET: main
+
+windows_old_msi_x64-a7:
+  extends: .windows_old_main_agent_base
+  variables:
+    ARCH: "x64"
+    AGENT_MAJOR_VERSION: 7
+    PYTHON_RUNTIMES: '3'
+  before_script:
+    - set RELEASE_VERSION $RELEASE_VERSION_7
+  <<: *skip_when_unwanted_on_7
+
+windows_old_msi_x64-a6:
+  extends: .windows_old_main_agent_base
+  variables:
+    ARCH: "x64"
+    AGENT_MAJOR_VERSION: 6
+    PYTHON_RUNTIMES: '2,3'
+  before_script:
+    - set RELEASE_VERSION $RELEASE_VERSION_6
+  <<: *skip_when_unwanted_on_6
+
 .windows_msi_base:
   stage: package_build
   needs: ["fail_on_non_triggered_tag", "run_dep_check_lock"]
@@ -1159,7 +1234,7 @@ windows_zip_agent_binaries_x64-a7:
     - if (Test-Path .omnibus) { remove-item -recurse -force .omnibus }
     - if (Test-Path build-out) { remove-item -recurse -force build-out }
     - mkdir .omnibus\pkg
-    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e CI_JOB_ID=${CI_JOB_ID} -e OMNIBUS_TARGET=${OMNIBUS_TARGET} -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e IS_AWS_CONTAINER=true -e SIGN_WINDOWS=true 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\buildwin.bat
+    - docker run --rm -m 4096M -v "$(Get-Location):c:\mnt" -e CI_JOB_ID=${CI_JOB_ID} -e OMNIBUS_TARGET=${OMNIBUS_TARGET} -e WINDOWS_BUILDER=true -e RELEASE_VERSION="$RELEASE_VERSION" -e MAJOR_VERSION="$AGENT_MAJOR_VERSION" -e PY_RUNTIMES="$PYTHON_RUNTIMES" -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e TARGET_ARCH="$ARCH" -e NEW_BUILDER=true 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\buildwin.bat
     - copy build-out\${CI_JOB_ID}\*.msi .omnibus\pkg
     - if (Test-Path build-out\${CI_JOB_ID}\*.zip) { copy build-out\${CI_JOB_ID}\*.zip .omnibus\pkg }
     - remove-item -recurse -force build-out\${CI_JOB_ID}
@@ -1238,7 +1313,7 @@ windows_choco_online_7_x64:
   variables:
     ARCH: "x64"
   script:
-    - docker run --rm -v "$(Get-Location):c:\mnt" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\chocopack.bat
+    - docker run --rm -v "$(Get-Location):c:\mnt" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDERS} c:\mnt\tasks\winbuildscripts\chocopack.bat
     - copy build-out\*.nupkg .omnibus\pkg
   artifacts:
     expire_in: 2 weeks
@@ -1254,7 +1329,7 @@ windows_choco_offline_7_x64:
     ARCH: "x64"
   script:
     - copy .omnibus\pkg\*.msi chocolatey\tools\
-    - docker run --rm -v "$(Get-Location):c:\mnt" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\chocopack.bat
+    - docker run --rm -v "$(Get-Location):c:\mnt" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDERS} c:\mnt\tasks\winbuildscripts\chocopack.bat
     - copy build-out\*.nupkg .omnibus\pkg
   artifacts:
     expire_in: 2 weeks
@@ -1277,7 +1352,7 @@ publish_choco_7_x64:
     - mkdir nupkg
     - copy .omnibus\pkg\*.nupkg nupkg\
     - Get-ChildItem nupkg
-    - docker run --rm -v "$(Get-Location):c:\mnt" -e CHOCOLATEY_API_KEY=${chocolateyApiKey} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:$Env:DATADOG_AGENT_WINBUILDIMAGES c:\mnt\tasks\winbuildscripts\chocopush.bat
+    - docker run --rm -v "$(Get-Location):c:\mnt" -e CHOCOLATEY_API_KEY=${chocolateyApiKey} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1809_${ARCH}:${Env:DATADOG_AGENT_WINBUILDERS} c:\mnt\tasks\winbuildscripts\chocopush.bat
   when: manual
   <<: *skip_when_unwanted_on_7
 
@@ -2649,7 +2724,7 @@ twistlock_scan-7:
       "@ | out-file ci-scripts/docker-publish.ps1
   after_script:
     - cat ci-scripts/docker-publish.ps1
-    - docker run --rm -w C:\mnt -e IS_AWS_CONTAINER=true -e SIGN_WINDOWS=true -v "$(Get-Location):C:\mnt" -v \\.\pipe\docker_engine:\\.\pipe\docker_engine 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_${Env:VARIANT}_x64:${Env:DATADOG_AGENT_WINBUILDIMAGES} powershell -C C:\mnt\ci-scripts\docker-publish.ps1
+    - docker run --rm -w C:\mnt -e IS_AWS_CONTAINER=true -e SIGN_WINDOWS=true -v "$(Get-Location):C:\mnt" -v \\.\pipe\docker_engine:\\.\pipe\docker_engine 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_${Env:VARIANT}_x64:${Env:DATADOG_AGENT_WINBUILDERS} powershell -C C:\mnt\ci-scripts\docker-publish.ps1
 
 dev_branch_docker_hub-a6:
   <<: *skip_when_unwanted_on_6

tasks/winbuildscripts/buildwin.bat

@@ -24,18 +24,40 @@ mkdir \dev\go\src\github.com\DataDog\datadog-agent
 if not exist \dev\go\src\github.com\DataDog\datadog-agent exit /b 1
 cd \dev\go\src\github.com\DataDog\datadog-agent || exit /b 2
 xcopy /e/s/h/q c:\mnt\*.* || exit /b 3
-inv -e deps --verbose --dep-vendor-only --no-checks || exit /b 4
+
+SET PATH=%PATH%;%GOPATH%/bin
+
+@echo GOPATH %GOPATH%
+@echo PATH %PATH%
+@echo VSTUDIO_ROOT %VSTUDIO_ROOT%
+@echo TARGET_ARCH %TARGET_ARCH%
+
+REM Equivalent to the "ridk enable" command, but without the exit
+if "%TARGET_ARCH%" == "x64" (
+    @echo IN x64 BRANCH
+    @for /f "delims=" %%x in ('"ruby" --disable-gems -x '%RIDK%' enable') do set "%%x"
+)
+
+if "%TARGET_ARCH%" == "x86" (
+    @echo IN x86 BRANCH
+    REM Use 64-bit toolchain to build gems
+    Powershell -C "ridk enable; cd omnibus; bundle install"
+)
+
+pip install -r requirements.txt || exit /b 4
+
+inv -e deps --verbose --dep-vendor-only --no-checks || exit /b 5
 
 @echo "inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION%"
-inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION% || exit /b 5
+inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION% || exit /b 6
 
 dir \omnibus\pkg
 
 dir \omnibus-ruby\pkg\
 
-if not exist %PKG_OUTDIR% mkdir %PKG_OUTDIR% || exit /b 6
-if exist \omnibus-ruby\pkg\*.msi copy \omnibus-ruby\pkg\*.msi %PKG_OUTDIR% || exit /b 7
-if exist \omnibus-ruby\pkg\*.zip copy \omnibus-ruby\pkg\*.zip %PKG_OUTDIR% || exit /b 8
+if not exist %PKG_OUTDIR% mkdir %PKG_OUTDIR% || exit /b 7
+if exist \omnibus-ruby\pkg\*.msi copy \omnibus-ruby\pkg\*.msi %PKG_OUTDIR% || exit /b 8
+if exist \omnibus-ruby\pkg\*.zip copy \omnibus-ruby\pkg\*.zip %PKG_OUTDIR% || exit /b 9
 
 goto :EOF
 

tasks/winbuildscripts/oldbuildwin.bat

@@ -0,0 +1,45 @@
+if not exist c:\mnt\ goto nomntdir
+
+@echo c:\mnt found, continuing
+@echo PARAMS %*
+@echo RELEASE_VERSION %RELEASE_VERSION%
+@echo MAJOR_VERSION %MAJOR_VERSION%
+@echo PY_RUNTIMES %PY_RUNTIMES%
+
+if NOT DEFINED RELEASE_VERSION set RELEASE_VERSION=%~1
+if NOT DEFINED MAJOR_VERSION set MAJOR_VERSION=%~2
+if NOT DEFINED PY_RUNTIMES set PY_RUNTIMES=%~3
+
+REM don't use `OUTDIR` as an environment variable. It will confuse the VC build
+set PKG_OUTDIR=c:\mnt\build-out\%CI_JOB_ID%
+
+set OMNIBUS_BUILD=agent.omnibus-build
+set OMNIBUS_ARGS=--python-runtimes "%PY_RUNTIMES%"
+
+if "%OMNIBUS_TARGET%" == "iot" set OMNIBUS_ARGS=--iot
+if "%OMNIBUS_TARGET%" == "dogstatsd" set OMNIBUS_BUILD=dogstatsd.omnibus-build && set OMNIBUS_ARGS=
+if "%OMNIBUS_TARGET%" == "agent_binaries" set OMNIBUS_ARGS=%OMNIBUS_ARGS% --agent-binaries
+
+mkdir \dev\go\src\github.com\DataDog\datadog-agent
+if not exist \dev\go\src\github.com\DataDog\datadog-agent exit /b 1
+cd \dev\go\src\github.com\DataDog\datadog-agent || exit /b 2
+xcopy /e/s/h/q c:\mnt\*.* || exit /b 3
+inv -e deps --verbose --dep-vendor-only --no-checks || exit /b 4
+
+@echo "inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION%"
+inv -e %OMNIBUS_BUILD% %OMNIBUS_ARGS% --skip-deps --major-version %MAJOR_VERSION% --release-version %RELEASE_VERSION% || exit /b 5
+
+dir \omnibus\pkg
+
+dir \omnibus-ruby\pkg\
+
+if not exist %PKG_OUTDIR% mkdir %PKG_OUTDIR% || exit /b 6
+if exist \omnibus-ruby\pkg\*.msi copy \omnibus-ruby\pkg\*.msi %PKG_OUTDIR% || exit /b 7
+if exist \omnibus-ruby\pkg\*.zip copy \omnibus-ruby\pkg\*.zip %PKG_OUTDIR% || exit /b 8
+
+goto :EOF
+
+:nomntdir
+@echo directory not mounted, parameters incorrect
+exit /b 1
+goto :EOF

tasks/winbuildscripts/unittests.bat

@@ -4,8 +4,10 @@ if not exist c:\mnt\ goto nomntdir
 
 @echo PARAMS %*
 @echo PY_RUNTIMES %PY_RUNTIMES%
+@echo NEW_BUILDER %NEW_BUILDER%
 
 if NOT DEFINED PY_RUNTIMES set PY_RUNTIMES=%~1
+if NOT DEFINED NEW_BUILDER set NEW_BUILDER=%~2
 
 mkdir \dev\go\src\github.com\DataDog\datadog-agent
 cd \dev\go\src\github.com\DataDog\datadog-agent

tasks/winbuildscripts/unittests.ps1

@@ -1,14 +1,21 @@
 $Password = ConvertTo-SecureString "dummyPW_:-gch6Rejae9" -AsPlainText -Force
 New-LocalUser -Name "ddagentuser" -Description "Test user for the secrets feature on windows." -Password $Password
 
+if ($Env:NEW_BUILDER -eq "true") {
+    if ($Env:TARGET_ARCH -eq "x64") {
+        & ridk enable
+    }
+    & pip install -r requirements.txt
+}
+
 $Env:Python2_ROOT_DIR=$Env:TEST_EMBEDDED_PY2
 $Env:Python3_ROOT_DIR=$Env:TEST_EMBEDDED_PY3
 $Env:BUILD_ROOT=(Get-Location).Path
 $Env:PATH="$Env:BUILD_ROOT\dev\lib;$Env:GOPATH\bin;$Env:Python2_ROOT_DIR;$Env:Python2_ROOT_DIR\Scripts;$Env:Python3_ROOT_DIR;$Env:Python3_ROOT_DIR\Scripts;$Env:PATH"
 
-git clone --depth 1 https://github.com/datadog/integrations-core
-& $Env:Python2_ROOT_DIR\python.exe -m pip install PyYAML==5.1
-& $Env:Python3_ROOT_DIR\python.exe -m pip install PyYAML==5.1
+& git clone --depth 1 https://github.com/datadog/integrations-core
+& $Env:Python2_ROOT_DIR\python.exe -m pip install PyYAML==5.3
+& $Env:Python3_ROOT_DIR\python.exe -m pip install PyYAML==5.3
 
 $archflag = "x64"
 if ($Env:TARGET_ARCH -eq "x86") {
@@ -50,7 +57,11 @@ if($err -ne 0){
     [Environment]::Exit($err)
 }
 
-& inv -e test --race --profile --cpus 4 --arch $archflag --python-runtimes="$Env:PY_RUNTIMES" --python-home-2=$Env:Python2_ROOT_DIR --python-home-3=$Env:Python3_ROOT_DIR --rtloader-root=$Env:BUILD_ROOT\rtloader
+if ($Env:NEW_BUILDER -eq "true"){
+    & inv -e test --profile --cpus 4 --arch $archflag --python-runtimes="$Env:PY_RUNTIMES" --python-home-2=$Env:Python2_ROOT_DIR --python-home-3=$Env:Python3_ROOT_DIR --rtloader-root=$Env:BUILD_ROOT\rtloader
+} else {
+    & inv -e test --race --profile --cpus 4 --arch $archflag --python-runtimes="$Env:PY_RUNTIMES" --python-home-2=$Env:Python2_ROOT_DIR --python-home-3=$Env:Python3_ROOT_DIR --rtloader-root=$Env:BUILD_ROOT\rtloader
+}
 $err = $LASTEXITCODE
 Write-Host Test result is $err
 if($err -ne 0){

tools/windows/decompress_merge_module.ps1

@@ -4,7 +4,11 @@ param(
     )
 
 
-[Reflection.Assembly]::LoadFrom("$($Env:WIX)\Microsoft.Deployment.WindowsInstaller.dll")
+if ("$Env:NEW_BUILDER" -eq "true") {
+    [Reflection.Assembly]::LoadFrom("$($Env:WIX)\bin\Microsoft.Deployment.WindowsInstaller.dll")
+} else {
+    [Reflection.Assembly]::LoadFrom("$($Env:WIX)\Microsoft.Deployment.WindowsInstaller.dll")
+}
 
 write-host "Extracting files from merge module: "$file