Skip to content

Commit

Permalink
test new windows image
Browse files Browse the repository at this point in the history
  • Loading branch information
chouetz committed Sep 25, 2024
1 parent c758448 commit 1714e5e
Show file tree
Hide file tree
Showing 3 changed files with 152 additions and 193 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ experimental:
templates:
job_template: &job_template
docker:
- image: gcr.io/datadoghq/agent-circleci-runner:v44534774-f5cc3e24
- image: gcr.io/datadoghq/agent-circleci-runner_test_only:v45087049-5a79b9a2
environment:
USE_SYSTEM_LIBS: "1"
working_directory: /go/src/github.com/DataDog/datadog-agent
Expand Down
247 changes: 145 additions & 102 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -1,51 +1,51 @@
---
include:
- .gitlab/.pre/cancel-prev-pipelines.yml
- .gitlab/.pre/linters.yml
- .gitlab/.pre/test_gitlab_configuration.yml
- .gitlab/benchmarks/include.yml
- .gitlab/binary_build/include.yml
- .gitlab/check_deploy/check_deploy.yml
- .gitlab/check_merge/do_not_merge.yml
- .gitlab/choco_build/choco_build.yml
- .gitlab/choco_deploy/choco_deploy.yml
- .gitlab/common/shared.yml
- .gitlab/common/skip_ci_check.yml
- .gitlab/common/pr_commenter.yml
- .gitlab/common/test_infra_version.yml
- .gitlab/container_build/include.yml
- .gitlab/container_scan/container_scan.yml
- .gitlab/deploy_containers/deploy_containers.yml
- .gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml
- .gitlab/deploy_dca/deploy_dca.yml
- .gitlab/deploy_packages/include.yml
- .gitlab/deps_build/deps_build.yml
- .gitlab/deps_fetch/deps_fetch.yml
- .gitlab/dev_container_deploy/include.yml
- .gitlab/e2e/e2e.yml
- .gitlab/e2e_k8s/e2e_k8s.yml
- .gitlab/e2e_install_packages/include.yml
- .gitlab/e2e_pre_test/e2e_pre_test.yml
- .gitlab/functional_test/include.yml
- .gitlab/functional_test_cleanup/functional_test_cleanup.yml
- .gitlab/install_script_testing/install_script_testing.yml
- .gitlab/integration_test/include.yml
- .gitlab/internal_image_deploy/internal_image_deploy.yml
- .gitlab/internal_kubernetes_deploy/include.yml
- .gitlab/junit_upload/junit_upload.yml
- .gitlab/kitchen_cleanup/include.yml
- .gitlab/kitchen_deploy/kitchen_deploy.yml
- .gitlab/kitchen_testing/include.yml
- .gitlab/maintenance_jobs/include.yml
- .gitlab/notify/notify.yml
- .gitlab/package_build/include.yml
- .gitlab/packaging/include.yml
- .gitlab/package_deps_build/package_deps_build.yml
- .gitlab/pkg_metrics/pkg_metrics.yml
- .gitlab/post_rc_build/post_rc_tasks.yml
- .gitlab/setup/setup.yml
- .gitlab/source_test/include.yml
- .gitlab/trigger_release/trigger_release.yml
# include:
# - .gitlab/.pre/cancel-prev-pipelines.yml
# - .gitlab/.pre/linters.yml
# - .gitlab/.pre/test_gitlab_configuration.yml
# - .gitlab/benchmarks/include.yml
# - .gitlab/binary_build/include.yml
# - .gitlab/check_deploy/check_deploy.yml
# - .gitlab/check_merge/do_not_merge.yml
# - .gitlab/choco_build/choco_build.yml
# - .gitlab/choco_deploy/choco_deploy.yml
# - .gitlab/common/shared.yml
# - .gitlab/common/skip_ci_check.yml
# - .gitlab/common/pr_commenter.yml
# - .gitlab/common/test_infra_version.yml
# - .gitlab/container_build/include.yml
# - .gitlab/container_scan/container_scan.yml
# - .gitlab/deploy_containers/deploy_containers.yml
# - .gitlab/deploy_cws_instrumentation/deploy_cws_instrumentation.yml
# - .gitlab/deploy_dca/deploy_dca.yml
# - .gitlab/deploy_packages/include.yml
# - .gitlab/deps_build/deps_build.yml
# - .gitlab/deps_fetch/deps_fetch.yml
# - .gitlab/dev_container_deploy/include.yml
# - .gitlab/e2e/e2e.yml
# - .gitlab/e2e_k8s/e2e_k8s.yml
# - .gitlab/e2e_install_packages/include.yml
# - .gitlab/e2e_pre_test/e2e_pre_test.yml
# - .gitlab/functional_test/include.yml
# - .gitlab/functional_test_cleanup/functional_test_cleanup.yml
# - .gitlab/install_script_testing/install_script_testing.yml
# - .gitlab/integration_test/include.yml
# - .gitlab/internal_image_deploy/internal_image_deploy.yml
# - .gitlab/internal_kubernetes_deploy/include.yml
# - .gitlab/junit_upload/junit_upload.yml
# - .gitlab/kitchen_cleanup/include.yml
# - .gitlab/kitchen_deploy/kitchen_deploy.yml
# - .gitlab/kitchen_testing/include.yml
# - .gitlab/maintenance_jobs/include.yml
# - .gitlab/notify/notify.yml
# - .gitlab/package_build/include.yml
# - .gitlab/packaging/include.yml
# - .gitlab/package_deps_build/package_deps_build.yml
# - .gitlab/pkg_metrics/pkg_metrics.yml
# - .gitlab/post_rc_build/post_rc_tasks.yml
# - .gitlab/setup/setup.yml
# - .gitlab/source_test/include.yml
# - .gitlab/trigger_release/trigger_release.yml

default:
retry:
Expand All @@ -61,53 +61,96 @@ default:
- data_integrity_failure

stages:
- .pre
- setup
- maintenance_jobs
- deps_build
- deps_fetch
- source_test
- source_test_stats
- software_composition_analysis
- binary_build
- package_deps_build
- kernel_matrix_testing_prepare
- kernel_matrix_testing_system_probe
- kernel_matrix_testing_security_agent
- kernel_matrix_testing_cleanup
- integration_test
- benchmarks
- package_build
- packaging
- pkg_metrics
- kitchen_deploy
- kitchen_testing
- container_build
- container_scan
- check_deploy
- dev_container_deploy
- deploy_containers
- deploy_packages
- deploy_cws_instrumentation
- deploy_dca
- trigger_release
- choco_build
- choco_deploy
- internal_image_deploy
- install_script_testing
- e2e_pre_test
- e2e
- e2e_k8s
- e2e_install_packages
- kitchen_cleanup
- functional_test
- functional_test_cleanup
- junit_upload
- internal_kubernetes_deploy
- post_rc_build
- check_merge
# - .pre
# - setup
# - maintenance_jobs
# - deps_build
# - deps_fetch
# - source_test
# - source_test_stats
# - software_composition_analysis
# - binary_build
# - package_deps_build
# - kernel_matrix_testing_prepare
# - kernel_matrix_testing_system_probe
# - kernel_matrix_testing_security_agent
# - kernel_matrix_testing_cleanup
# - integration_test
# - benchmarks
# - package_build
# - packaging
# - pkg_metrics
# - kitchen_deploy
# - kitchen_testing
# - container_build
# - container_scan
# - check_deploy
# - dev_container_deploy
# - deploy_containers
# - deploy_packages
# - deploy_cws_instrumentation
# - deploy_dca
# - trigger_release
# - choco_build
# - choco_deploy
# - internal_image_deploy
# - install_script_testing
# - e2e_pre_test
# - e2e
# - e2e_k8s
# - e2e_install_packages
# - kitchen_cleanup
# - functional_test
# - functional_test_cleanup
# - junit_upload
# - internal_kubernetes_deploy
# - post_rc_build
# - check_merge
- notify

test_windows:
stage: notify
tags: ["runner:windows-docker", "windowsversion:1809"]
variables:
PYTHON_RUNTIMES: 3
ARCH: "x64"
script:
- $ErrorActionPreference = "Stop"
- '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"'
- >
docker run --rm
-m 16384M
--storage-opt "size=50GB"
-v "$(Get-Location):c:\mnt"
-e DD_ENV=prod
-e GITLAB_CI=true
-e CI_JOB_URL="${CI_JOB_URL}"
-e CI_JOB_NAME="${CI_JOB_NAME}"
-e CI_JOB_ID="${CI_JOB_ID}"
-e CI_PIPELINE_ID="${CI_PIPELINE_ID}"
-e CI_REPOSITORY_URL="${CI_REPOSITORY_URL}"
-e CI_COMMIT_SHA="${CI_COMMIT_SHA}"
-e CI_PROJECT_NAME="${CI_PROJECT_NAME}"
-e CI_COMMIT_REF_NAME="${CI_COMMIT_REF_NAME}"
-e AWS_NETWORKING=true
-e SIGN_WINDOWS_DD_WCS=true
-e PY_RUNTIMES="${PYTHON_RUNTIMES}"
-e GOMODCACHE="c:\modcache"
-e JUNIT_TAR="c:\mnt\junit-${CI_JOB_NAME}.tgz"
-e PIP_INDEX_URL="${PIP_INDEX_URL}"
-e TEST_OUTPUT_FILE="${TEST_OUTPUT_FILE}"
-e EXTRA_OPTS="${FAST_TESTS_FLAG}"
-e TEST_WASHER=true
-e GO_TEST_SKIP_FLAKE="${GO_TEST_SKIP_FLAKE}"
-e API_KEY_ORG2="${API_KEY_ORG2}"
-e CODECOV_TOKEN="${CODECOV_TOKEN}"
-e S3_PERMANENT_ARTIFACTS_URI="${S3_PERMANENT_ARTIFACTS_URI}"
-e COVERAGE_CACHE_FLAG="${COVERAGE_CACHE_FLAG}"
-e GITLAB_TOKEN="${GITLAB_READ_API_TOKEN}"
486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
c:\mnt\tasks\winbuildscripts\unittests.bat
- If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }

variables:
# Directory in which we execute the omnibus build.
# For an unknown reason, it does not go well with
Expand Down Expand Up @@ -169,16 +212,16 @@ variables:
# Build images versions
# To use images from datadog-agent-buildimages dev branches, set the corresponding
# SUFFIX variable to _test_only
DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
DATADOG_AGENT_BUILDIMAGES: v44534774-f5cc3e24
DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
DATADOG_AGENT_WINBUILDIMAGES: v44534774-f5cc3e24
DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
DATADOG_AGENT_ARMBUILDIMAGES: v44534774-f5cc3e24
DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v44534774-f5cc3e24
DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: ""
DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v44534774-f5cc3e24
DATADOG_AGENT_BUILDIMAGES_SUFFIX: "_test_only"
DATADOG_AGENT_BUILDIMAGES: v45087049-5a79b9a2
DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "_test_only"
DATADOG_AGENT_WINBUILDIMAGES: v45087049-5a79b9a2
DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "_test_only"
DATADOG_AGENT_ARMBUILDIMAGES: v45087049-5a79b9a2
DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "_test_only"
DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v45087049-5a79b9a2
DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: "_test_only"
DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v45087049-5a79b9a2

DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
DEB_GPG_KEY_ID: c0962c7d
Expand Down
96 changes: 6 additions & 90 deletions tasks/winbuildscripts/unittests.ps1
Original file line number Diff line number Diff line change
@@ -1,96 +1,12 @@
$ErrorActionPreference = "Stop"
$ErrorActionPreference = "Continue"
$Password = ConvertTo-SecureString "dummyPW_:-gch6Rejae9" -AsPlainText -Force
New-LocalUser -Name "ddagentuser" -Description "Test user for the secrets feature on windows." -Password $Password

$Env:Python2_ROOT_DIR=$Env:TEST_EMBEDDED_PY2
$Env:Python3_ROOT_DIR=$Env:TEST_EMBEDDED_PY3

$test_output_file = if ($Env:TEST_OUTPUT_FILE) { $Env:TEST_OUTPUT_FILE } else { "test_output.json" }

& ridk enable
& $Env:Python3_ROOT_DIR\python.exe -m pip install -r requirements.txt

$UT_BUILD_ROOT=(Get-Location).Path
$Env:PATH="$UT_BUILD_ROOT\dev\lib;$Env:GOPATH\bin;$Env:Python3_ROOT_DIR;$Env:Python3_ROOT_DIR\Scripts;$Env:PATH"

& pip install -r tasks/libs/requirements-github.txt

mkdir .\bin\agent

# Generate the datadog.yaml config file to be used in integration tests
& inv -e agent.generate-config --build-type="agent-py2py3" --output-file="./datadog.yaml"

# NG installer unit tests
if ($Env:DEBUG_CUSTOMACTION) {
& inv -e msi.test --debug
} else {
& inv -e msi.test
}
$err = $LASTEXITCODE
Write-Host Test result is $err
if($err -ne 0){
Write-Host -ForegroundColor Red "Windows installer unit test failed $err"
[Environment]::Exit($err)
}

& inv -e deps
& .\tasks\winbuildscripts\pre-go-build.ps1 -PythonRuntimes "$Env:PY_RUNTIMES"

& inv -e rtloader.test
$err = $LASTEXITCODE
Write-Host rtloader test result is $err
if($err -ne 0){
Write-Host -ForegroundColor Red "rtloader test failed $err"
[Environment]::Exit($err)
}
$TEST_WASHER_FLAG=""
if($Env:TEST_WASHER){
$TEST_WASHER_FLAG="--test-washer"
}
& inv -e install-tools
& inv -e agent.build
$err = $LASTEXITCODE
if($err -ne 0){
Write-Host -ForegroundColor Red "Agent build failed $err"
[Environment]::Exit($err)
}
& inv -e test --junit-tar="$Env:JUNIT_TAR" --race --profile --rerun-fails=2 --coverage --cpus 8 --python-runtimes="$Env:PY_RUNTIMES" --python-home-2=$Env:Python2_ROOT_DIR --python-home-3=$Env:Python3_ROOT_DIR --save-result-json C:\mnt\$test_output_file $Env:EXTRA_OPTS --build-stdlib $TEST_WASHER_FLAG

$err = $LASTEXITCODE

# Ignore upload failures
$ErrorActionPreference = "Continue"
$tmpfile = [System.IO.Path]::GetTempFileName()

# 1. Upload coverage reports to Codecov
& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" -parameterName "$Env:CODECOV_TOKEN" -tempFile "$tmpfile"
If ($LASTEXITCODE -ne "0") {
exit $LASTEXITCODE
}
$Env:CODECOV_TOKEN=$(cat "$tmpfile")
& inv -e coverage.upload-to-codecov $Env:COVERAGE_CACHE_FLAG

# 2. Upload junit files
# Copy test files to c:\mnt for further gitlab upload
Get-ChildItem -Path "$UT_BUILD_ROOT" -Filter "junit-out-*.xml" -Recurse | ForEach-Object {
Copy-Item -Path $_.FullName -Destination C:\mnt
}
& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" -parameterName "$Env:API_KEY_ORG2" -tempFile "$tmpfile"
If ($LASTEXITCODE -ne "0") {
exit $LASTEXITCODE
}
$Env:DATADOG_API_KEY=$(cat "$tmpfile")
& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" -parameterName "$Env:GITLAB_TOKEN" -tempFile "$tmpfile"
If ($LASTEXITCODE -ne "0") {
exit $LASTEXITCODE
}
$Env:GITLAB_TOKEN=$(cat "$tmpfile")
Remove-Item "$tmpfile"

& inv -e junit-upload --tgz-path $Env:JUNIT_TAR
if($err -ne 0){
Write-Host -ForegroundColor Red "test failed $err"
[Environment]::Exit($err)
}

Write-Host Test passed
py.exe -c 'import shutil; print(shutil.which("vault"))'
py.exe -c 'import shutil; print(shutil.which("vault.exe"))'
ls "c:\devtools"
ls "c:\devtools\vault"
#$result=(vault kv get -field="token" kv/k8s/gitlab-runner/datadog-agent/"api-key-org2")

0 comments on commit 1714e5e

Please sign in to comment.