feat(fetch_secret): Add possibility to call vault in windows context (#…

…29467)
DataDog · Sep 23, 2024 · 34754e5 · 34754e5
1 parent 06338e9
commit 34754e5
Show file tree

Hide file tree

Showing 8 changed files with 21 additions and 16 deletions.
diff --git a/.gitlab/choco_deploy/choco_deploy.yml b/.gitlab/choco_deploy/choco_deploy.yml
@@ -11,7 +11,7 @@ publish_choco_7_x64:
     ARCH: "x64"
   before_script:
     - $tmpfile = [System.IO.Path]::GetTempFileName()
-    - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" "$Env:CHOCOLATEY_API_KEY" "$tmpfile")
+    - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:CHOCOLATEY_API_KEY" -tempFile "$tmpfile")
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
     - $chocolateyApiKey=$(cat "$tmpfile")
     - Remove-Item "$tmpfile"

diff --git a/.gitlab/deploy_packages/winget.yml b/.gitlab/deploy_packages/winget.yml
@@ -11,7 +11,7 @@ publish_winget_7_x64:
     ARCH: "x64"
   before_script:
     - $tmpfile = [System.IO.Path]::GetTempFileName()
-    - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" "$Env:WINGET_PAT" "$tmpfile")
+    - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:WINGET_PAT" -tempFile "$tmpfile")
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
     - $wingetPat=$(cat "$tmpfile")
     - Remove-Item "$tmpfile"

diff --git a/.gitlab/integration_test/windows.yml b/.gitlab/integration_test/windows.yml
@@ -8,7 +8,7 @@
   tags: ["runner:windows-docker", "windowsversion:1809"]
   before_script:
     - $tmpfile = [System.IO.Path]::GetTempFileName()
-    - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" "$Env:VCPKG_BLOB_SAS_URL" "$tmpfile")
+    - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:VCPKG_BLOB_SAS_URL" -tempFile "$tmpfile")
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
     - $vcpkgBlobSaSUrl=$(cat "$tmpfile")
     - Remove-Item "$tmpfile"

diff --git a/tasks/linter.py b/tasks/linter.py
@@ -383,7 +383,7 @@ def __repr__(self):
 def list_get_parameter_calls(file):
     aws_ssm_call = re.compile(r"^.+ssm get-parameter.+--name +(?P<param>[^ ]+).*$")
     # remove the first letter of the script name because '\f' is badly interpreted for windows paths
-    wrapper_call = re.compile(r"^.+etch_secret.(sh|ps1)[\"]? +(?P<param>[^ )]+).*$")
+    wrapper_call = re.compile(r"^.+etch_secret.(sh|ps1)[\"]? (-parameterName )?+(?P<param>[^ )]+).*$")
     calls = []
     with open(file) as f:
         try:

diff --git a/tasks/unit_tests/linter_tests.py b/tasks/unit_tests/linter_tests.py
@@ -58,10 +58,10 @@ def test_with_wrapper_with_env(self):
     def test_multi_match_windows(self):
         with open(self.test_file, "w") as f:
             f.write(
-                'DD_API_KEY=$(& "$CI_PROJECT_DIR\tools \\ci\fetch_secret.ps1" test.datadog-agent.datadog_api_key_org2 $tmpfile)\n'
-                'DD_API_KEY=$(& "$CI_PROJECT_DIR\tools \\ci\fetch secret.ps1" "$Env:MISSING_UNDERSCORE" $tmpfile)\n'
-                '`DD_APP_KEY=$(& "$CI_PROJECT_DIR\tools\\ci\fetch_secret.ps1" "bad.name" "$tmpfile")\n'
-                'DD_APP=$(& "$CI_PROJECT_DIR\tools\\ci\fetch_secret.ps1" "$Env:TEST" $tmpfile)\n'
+                'DD_API_KEY=$(& "$CI_PROJECT_DIR\tools \\ci\fetch_secret.ps1" -parameterName test.datadog-agent.datadog_api_key_org2 -tempFile $tmpfile)\n'
+                'DD_API_KEY=$(& "$CI_PROJECT_DIR\tools \\ci\fetch secret.ps1" -parameterName "$Env:MISSING_UNDERSCORE" -tempFile $tmpfile)\n'
+                '`DD_APP_KEY=$(& "$CI_PROJECT_DIR\tools\\ci\fetch_secret.ps1" -parameterName "bad.name" -tempFile "$tmpfile")\n'
+                'DD_APP=$(& "$CI_PROJECT_DIR\tools\\ci\fetch_secret.ps1" -parameterName "$Env:TEST" -tempFile $tmpfile)\n'
             )
         matched = linter.list_get_parameter_calls(self.test_file)
         self.assertEqual(2, len(matched))

diff --git a/tasks/winbuildscripts/unittests.ps1 b/tasks/winbuildscripts/unittests.ps1
@@ -63,7 +63,7 @@ $ErrorActionPreference = "Continue"
 $tmpfile = [System.IO.Path]::GetTempFileName()
 
 # 1. Upload coverage reports to Codecov
-& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" "$Env:CODECOV_TOKEN" "$tmpfile"
+& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" -parameterName "$Env:CODECOV_TOKEN" -tempFile "$tmpfile"
 If ($LASTEXITCODE -ne "0") {
     exit $LASTEXITCODE
 }
@@ -75,12 +75,12 @@ $Env:CODECOV_TOKEN=$(cat "$tmpfile")
 Get-ChildItem -Path "$UT_BUILD_ROOT" -Filter "junit-out-*.xml" -Recurse | ForEach-Object {
     Copy-Item -Path $_.FullName -Destination C:\mnt
 }
-& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" "$Env:API_KEY_ORG2" "$tmpfile"
+& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" -parameterName "$Env:API_KEY_ORG2" -tempFile "$tmpfile"
 If ($LASTEXITCODE -ne "0") {
     exit $LASTEXITCODE
 }
 $Env:DATADOG_API_KEY=$(cat "$tmpfile")
-& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" "$Env:GITLAB_TOKEN" "$tmpfile"
+& "$UT_BUILD_ROOT\tools\ci\fetch_secret.ps1" -parameterName "$Env:GITLAB_TOKEN" -tempFile "$tmpfile"
 If ($LASTEXITCODE -ne "0") {
     exit $LASTEXITCODE
 }

diff --git a/tools/ci/docker-login.ps1 b/tools/ci/docker-login.ps1
@@ -7,12 +7,12 @@ If ($lastExitCode -ne "0") {
 }
 # DockerHub login
 $tmpfile = [System.IO.Path]::GetTempFileName()
-& "C:\mnt\tools\ci\fetch_secret.ps1" "$Env:DOCKER_REGISTRY_LOGIN" "$tmpfile"
+& "C:\mnt\tools\ci\fetch_secret.ps1" -parameterName "$Env:DOCKER_REGISTRY_LOGIN" -tempFile "$tmpfile"
 If ($lastExitCode -ne "0") {
     throw "Previous command returned $lastExitCode"
 }
 $DOCKER_REGISTRY_LOGIN = $(cat "$tmpfile")
-& "C:\mnt\tools\ci\fetch_secret.ps1" "$Env:DOCKER_REGISTRY_PWD" "$tmpfile"
+& "C:\mnt\tools\ci\fetch_secret.ps1" -parameterName "$Env:DOCKER_REGISTRY_PWD" -tempFile "$tmpfile"
 If ($lastExitCode -ne "0") {
     throw "Previous command returned $lastExitCode"
 }

diff --git a/tools/ci/fetch_secret.ps1 b/tools/ci/fetch_secret.ps1
@@ -1,5 +1,6 @@
 param (
     [string]$parameterName,
+    [string]$parameterField,
     [string]$tempFile
 )
 
@@ -10,15 +11,19 @@ $maxRetries = 10
 $ErrorActionPreference = "Continue"
 
 while ($retryCount -lt $maxRetries) {
-    $result = (aws ssm get-parameter --region us-east-1 --name $parameterName --with-decryption --query "Parameter.Value" --output text 2> awsErrorFile.txt)
-    $error = Get-Content awsErrorFile.txt
+    if ($parameterField) {
+        $result = (vault kv get -field="$parameterField" kv/k8s/gitlab-runner/datadog-agent/"$parameterName" 2> errorFile.txt)
+    } else {
+        $result = (aws ssm get-parameter --region us-east-1 --name $parameterName --with-decryption --query "Parameter.Value" --output text 2> errorFile.txt)
+    } 
+    $error = Get-Content errorFile.txt
     if ($result) {
         "$result" | Out-File -FilePath "$tempFile" -Encoding ASCII
         exit 0
     }
     if ($error -match "Unable to locate credentials") {
         # See 5th row in https://docs.google.com/spreadsheets/d/1JvdN0N-RdNEeOJKmW_ByjBsr726E3ZocCKU8QoYchAc
-        Write-Error "Permanent error: unable to locate AWS credentials, not retrying"
+        Write-Error "Permanent error: unable to locate credentials, not retrying"
         exit 42
     }