diff --git a/.gitlab/choco_deploy/choco_deploy.yml b/.gitlab/choco_deploy/choco_deploy.yml index 715387a08f961..eb3a3e27f353a 100644 --- a/.gitlab/choco_deploy/choco_deploy.yml +++ b/.gitlab/choco_deploy/choco_deploy.yml @@ -11,7 +11,7 @@ publish_choco_7_x64: ARCH: "x64" before_script: - $tmpfile = [System.IO.Path]::GetTempFileName() - - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" "$Env:CHOCOLATEY_API_KEY" "$tmpfile") + - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:CHOCOLATEY_API_KEY" -tempFile "$tmpfile") - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - $chocolateyApiKey=$(cat "$tmpfile") - Remove-Item "$tmpfile" diff --git a/.gitlab/deploy_packages/winget.yml b/.gitlab/deploy_packages/winget.yml index a35239c948381..f28f946b1fb0c 100644 --- a/.gitlab/deploy_packages/winget.yml +++ b/.gitlab/deploy_packages/winget.yml @@ -11,7 +11,7 @@ publish_winget_7_x64: ARCH: "x64" before_script: - $tmpfile = [System.IO.Path]::GetTempFileName() - - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" "$Env:WINGET_PAT" "$tmpfile") + - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:WINGET_PAT" -tempFile "$tmpfile") - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - $wingetPat=$(cat "$tmpfile") - Remove-Item "$tmpfile" diff --git a/.gitlab/integration_test/windows.yml b/.gitlab/integration_test/windows.yml index 127454f00688f..2ddf04fddb997 100644 --- a/.gitlab/integration_test/windows.yml +++ b/.gitlab/integration_test/windows.yml @@ -8,7 +8,7 @@ tags: ["runner:windows-docker", "windowsversion:1809"] before_script: - $tmpfile = [System.IO.Path]::GetTempFileName() - - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" "$Env:VCPKG_BLOB_SAS_URL" "$tmpfile") + - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:VCPKG_BLOB_SAS_URL" -tempFile "$tmpfile") - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - $vcpkgBlobSaSUrl=$(cat "$tmpfile") - Remove-Item "$tmpfile" diff --git a/tasks/linter.py b/tasks/linter.py index 1d32a7e38ea43..a2a41e29cb8c0 100644 --- a/tasks/linter.py +++ b/tasks/linter.py @@ -383,7 +383,7 @@ def __repr__(self): def list_get_parameter_calls(file): aws_ssm_call = re.compile(r"^.+ssm get-parameter.+--name +(?P[^ ]+).*$") # remove the first letter of the script name because '\f' is badly interpreted for windows paths - wrapper_call = re.compile(r"^.+etch_secret.(sh|ps1)[\"]? +(?P[^ )]+).*$") + wrapper_call = re.compile(r"^.+etch_secret.(sh|ps1)[\"]? (-parameterName )?+(?P[^ )]+).*$") calls = [] with open(file) as f: try: diff --git a/tasks/unit_tests/linter_tests.py b/tasks/unit_tests/linter_tests.py index d33700ae8695f..f3e3d0c51d0ce 100644 --- a/tasks/unit_tests/linter_tests.py +++ b/tasks/unit_tests/linter_tests.py @@ -58,10 +58,10 @@ def test_with_wrapper_with_env(self): def test_multi_match_windows(self): with open(self.test_file, "w") as f: f.write( - 'DD_API_KEY=$(& "$CI_PROJECT_DIR\tools \\ci\fetch_secret.ps1" test.datadog-agent.datadog_api_key_org2 $tmpfile)\n' - 'DD_API_KEY=$(& "$CI_PROJECT_DIR\tools \\ci\fetch secret.ps1" "$Env:MISSING_UNDERSCORE" $tmpfile)\n' - '`DD_APP_KEY=$(& "$CI_PROJECT_DIR\tools\\ci\fetch_secret.ps1" "bad.name" "$tmpfile")\n' - 'DD_APP=$(& "$CI_PROJECT_DIR\tools\\ci\fetch_secret.ps1" "$Env:TEST" $tmpfile)\n' + 'DD_API_KEY=$(& "$CI_PROJECT_DIR\tools \\ci\fetch_secret.ps1" -parameterName test.datadog-agent.datadog_api_key_org2 -tempFile $tmpfile)\n' + 'DD_API_KEY=$(& "$CI_PROJECT_DIR\tools \\ci\fetch secret.ps1" -parameterName "$Env:MISSING_UNDERSCORE" -tempFile $tmpfile)\n' + '`DD_APP_KEY=$(& "$CI_PROJECT_DIR\tools\\ci\fetch_secret.ps1" -parameterName "bad.name" -tempFile "$tmpfile")\n' + 'DD_APP=$(& "$CI_PROJECT_DIR\tools\\ci\fetch_secret.ps1" -parameterName "$Env:TEST" -tempFile $tmpfile)\n' ) matched = linter.list_get_parameter_calls(self.test_file) self.assertEqual(2, len(matched)) diff --git a/tasks/winbuildscripts/unittests.ps1 b/tasks/winbuildscripts/unittests.ps1 index ad28ec0540ad6..8d09afd956ae5 100644 --- a/tasks/winbuildscripts/unittests.ps1 +++ b/tasks/winbuildscripts/unittests.ps1 @@ -63,7 +63,7 @@ $ErrorActionPreference = "Continue" $tmpfile = [System.IO.Path]::GetTempFileName() # 1. Upload coverage reports to Codecov -& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" "$Env:CODECOV_TOKEN" "$tmpfile" +& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" -parameterName "$Env:CODECOV_TOKEN" -tempFile "$tmpfile" If ($LASTEXITCODE -ne "0") { exit $LASTEXITCODE } @@ -75,12 +75,12 @@ $Env:CODECOV_TOKEN=$(cat "$tmpfile") Get-ChildItem -Path "$UT_BUILD_ROOT" -Filter "junit-out-*.xml" -Recurse | ForEach-Object { Copy-Item -Path $_.FullName -Destination C:\mnt } -& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" "$Env:API_KEY_ORG2" "$tmpfile" +& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" -parameterName "$Env:API_KEY_ORG2" -tempFile "$tmpfile" If ($LASTEXITCODE -ne "0") { exit $LASTEXITCODE } $Env:DATADOG_API_KEY=$(cat "$tmpfile") -& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" "$Env:GITLAB_TOKEN" "$tmpfile" +& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" -parameterName "$Env:GITLAB_TOKEN" -tempFile "$tmpfile" If ($LASTEXITCODE -ne "0") { exit $LASTEXITCODE } diff --git a/tools/ci/docker-login.ps1 b/tools/ci/docker-login.ps1 index e85da22733afa..840b6b786492d 100644 --- a/tools/ci/docker-login.ps1 +++ b/tools/ci/docker-login.ps1 @@ -7,12 +7,12 @@ If ($lastExitCode -ne "0") { } # DockerHub login $tmpfile = [System.IO.Path]::GetTempFileName() -& "C:\mnt\tools\ci\fetch_secret.ps1" "$Env:DOCKER_REGISTRY_LOGIN" "$tmpfile" +& "C:\mnt\tools\ci\fetch_secret.ps1" -parameterName "$Env:DOCKER_REGISTRY_LOGIN" -tempFile "$tmpfile" If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } $DOCKER_REGISTRY_LOGIN = $(cat "$tmpfile") -& "C:\mnt\tools\ci\fetch_secret.ps1" "$Env:DOCKER_REGISTRY_PWD" "$tmpfile" +& "C:\mnt\tools\ci\fetch_secret.ps1" -parameterName "$Env:DOCKER_REGISTRY_PWD" -tempFile "$tmpfile" If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } diff --git a/tools/ci/fetch_secret.ps1 b/tools/ci/fetch_secret.ps1 index 396ed6cd38a6a..10ea137405d74 100644 --- a/tools/ci/fetch_secret.ps1 +++ b/tools/ci/fetch_secret.ps1 @@ -1,5 +1,6 @@ param ( [string]$parameterName, + [string]$parameterField, [string]$tempFile ) @@ -10,15 +11,19 @@ $maxRetries = 10 $ErrorActionPreference = "Continue" while ($retryCount -lt $maxRetries) { - $result = (aws ssm get-parameter --region us-east-1 --name $parameterName --with-decryption --query "Parameter.Value" --output text 2> awsErrorFile.txt) - $error = Get-Content awsErrorFile.txt + if ($parameterField) { + $result = (vault kv get -field="$parameterField" kv/k8s/gitlab-runner/datadog-agent/"$parameterName" 2> errorFile.txt) + } else { + $result = (aws ssm get-parameter --region us-east-1 --name $parameterName --with-decryption --query "Parameter.Value" --output text 2> errorFile.txt) + } + $error = Get-Content errorFile.txt if ($result) { "$result" | Out-File -FilePath "$tempFile" -Encoding ASCII exit 0 } if ($error -match "Unable to locate credentials") { # See 5th row in https://docs.google.com/spreadsheets/d/1JvdN0N-RdNEeOJKmW_ByjBsr726E3ZocCKU8QoYchAc - Write-Error "Permanent error: unable to locate AWS credentials, not retrying" + Write-Error "Permanent error: unable to locate credentials, not retrying" exit 42 }