Merge pull request #772 from buildkite/go-secrets-plugin

Integrate Go s3secrets-helper into elastic-stack
buildkite · Nov 23, 2020 · e71d923 · e71d923
2 parents 374bcc5 + 19eb25d
commit e71d923
Show file tree

Hide file tree

Showing 8 changed files with 65 additions and 3 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -16,6 +16,44 @@ steps:
         run: unit-tests
         config: docker-compose.unit-tests.yml
 
+  - id: "s3secrets-helper-linux-amd64"
+    name: ":golang: :linux: s3secrets-helper-linux-amd64"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    plugins:
+      docker#v3.7.0:
+        image: "golang:1.15"
+        mount-checkout: false
+        volumes:
+          - "./build:/build:rw"
+          - "./plugins/secrets/s3secrets-helper:/s3secrets-helper:ro"
+        workdir: /s3secrets-helper
+        environment:
+          - "GOOS=linux"
+          - "GOARCH=amd64"
+        command: ["go", "build", "-o", "/build/s3secrets-helper-linux-amd64"]
+    artifact_paths:
+      - build/s3secrets-helper-linux-amd64
+
+  - id: "s3secrets-helper-windows-amd64"
+    name: ":golang: :windows: s3secrets-helper-windows-amd64.exe"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    plugins:
+      docker#v3.7.0:
+        image: "golang:1.15"
+        mount-checkout: false
+        volumes:
+          - "./build:/build:rw"
+          - "./plugins/secrets/s3secrets-helper:/s3secrets-helper:ro"
+        workdir: /s3secrets-helper
+        environment:
+          - "GOOS=windows"
+          - "GOARCH=amd64"
+        command: ["go", "build", "-o", "/build/s3secrets-helper-windows-amd64.exe"]
+    artifact_paths:
+      - build/s3secrets-helper-windows-amd64.exe
+
   - id: "packer-windows"
     name: ":packer: :windows:"
     command: .buildkite/steps/packer.sh windows
@@ -26,6 +64,7 @@ steps:
     depends_on:
       - "lint"
       - "bats-tests"
+      - "s3secrets-helper-windows-amd64"
 
   - id: "windows-launch"
     name: ":cloudformation: :windows: Launch"
@@ -54,6 +93,7 @@ steps:
     depends_on:
       - "lint"
       - "bats-tests"
+      - "s3secrets-helper-linux-amd64"
 
   - id: "linux-launch"
     name: ":cloudformation: :linux: Launch"

diff --git a/.buildkite/steps/packer.sh b/.buildkite/steps/packer.sh
@@ -8,13 +8,17 @@ fi
 
 os="${1:-linux}"
 agent_binary="buildkite-agent-${os}-amd64"
+s3secrets_binary="s3secrets-helper-${os}-amd64"
 
 if [[ "$os" == "windows" ]] ; then
   agent_binary+=".exe"
+  s3secrets_binary+=".exe"
 fi
 
 mkdir -p "build/"
 
+buildkite-agent artifact download "build/$s3secrets_binary" .
+
 # Build a hash of packer files and the agent versions
 packer_files_sha=$(find Makefile "packer/${os}" plugins/ -type f -print0 | xargs -0 sha1sum | awk '{print $1}' | sort | sha1sum | awk '{print $1}')
 stable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/stable/latest/${agent_binary}.sha256")

diff --git a/Makefile b/Makefile
@@ -67,7 +67,7 @@ build/linux-ami.txt: packer-linux.output env-AWS_REGION
 	grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@
 
 # Build linux packer image
-packer-linux.output: $(PACKER_LINUX_FILES)
+packer-linux.output: $(PACKER_LINUX_FILES) build/s3secrets-helper-linux-amd64
 	docker run \
 		-e AWS_DEFAULT_REGION  \
 		-e AWS_PROFILE \
@@ -87,7 +87,7 @@ build/windows-ami.txt: packer-windows.output env-AWS_REGION
 	grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@
 
 # Build windows packer image
-packer-windows.output: $(PACKER_WINDOWS_FILES)
+packer-windows.output: $(PACKER_WINDOWS_FILES) build/s3secrets-helper-windows-amd64.exe
 	docker run \
 		-e AWS_DEFAULT_REGION  \
 		-e AWS_PROFILE \
@@ -149,3 +149,9 @@ validate: build/aws-stack.yml
 generate-toc:
 	docker run -it --rm -v "$(PWD):/app" node:slim bash \
 		-c "npm install -g markdown-toc && cd /app && markdown-toc -i README.md"
+
+build/s3secrets-helper-linux-amd64:
+	cd plugins/secrets/s3secrets-helper && GOOS=linux GOARCH=amd64 go build -o ../../../$@
+
+build/s3secrets-helper-windows-amd64.exe:
+	cd plugins/secrets/s3secrets-helper && GOOD=windows GOARCH=amd64 go build -o ../../../$@
diff --git a/packer/linux/buildkite-ami.json b/packer/linux/buildkite-ami.json
@@ -33,6 +33,11 @@
       "source": "../../plugins",
       "destination": "/tmp/plugins"
     },
+    {
+      "type": "file",
+      "source": "../../build/s3secrets-helper-linux-amd64",
+      "destination": "/tmp/s3secrets-helper"
+    },
     {
       "type": "shell",
       "script": "scripts/install-utils.sh"

diff --git a/packer/linux/scripts/install-buildkite-agent.sh b/packer/linux/scripts/install-buildkite-agent.sh
@@ -66,3 +66,4 @@ echo "Copying built-in plugins..."
 sudo mkdir -p /usr/local/buildkite-aws-stack/plugins
 sudo cp -a /tmp/plugins/* /usr/local/buildkite-aws-stack/plugins/
 sudo chown -R buildkite-agent: /usr/local/buildkite-aws-stack
+sudo install --mode=0755 /tmp/s3secrets-helper /usr/local/bin
diff --git a/packer/windows/buildkite-ami.json b/packer/windows/buildkite-ami.json
@@ -36,6 +36,11 @@
       "source": "../../plugins",
       "destination": "C:/packer-temp"
     },
+    {
+      "type": "file",
+      "source": "../../build/s3secrets-helper-windows-amd64.exe",
+      "destination": "C:/packer-temp/s3secrets-helper.exe"
+    },
     {
       "type": "powershell",
       "script": "scripts/install-utils.ps1"

diff --git a/packer/windows/scripts/install-buildkite-agent.ps1 b/packer/windows/scripts/install-buildkite-agent.ps1
@@ -45,3 +45,4 @@ Copy-Item -Path C:\packer-temp\conf\buildkite-agent\scripts\stop-agent-gracefull
 Write-Output "Copying built-in plugins..."
 New-Item -ItemType directory -Path "C:\Program Files\Git\usr\local\buildkite-aws-stack\plugins"
 Copy-Item -Recurse -Path C:\packer-temp\plugins\* -Destination "C:\Program Files\Git\usr\local\buildkite-aws-stack\plugins\"
+Copy-Item -Path C:\packer-temp\s3secrets-helper.exe -Destination C:\buildkite-agent\bin
diff --git a/plugins/secrets b/plugins/secrets
+15 −0		.buildkite/test_credentials.sh
+6 −1		README.md
+11 −98		hooks/environment
+0 −52		lib/shared.bash
+2 −0		s3secrets-helper/.gitignore
+5 −0		s3secrets-helper/go.mod
+16 −0		s3secrets-helper/go.sum
+71 −0		s3secrets-helper/main.go
+77 −0		s3secrets-helper/s3/s3.go
+254 −0		s3secrets-helper/secrets/secrets.go
+173 −0		s3secrets-helper/secrets/secrets_test.go
+13 −0		s3secrets-helper/sentinel/errors.go
+119 −0		s3secrets-helper/sshagent/sshagent.go
+31 −0		s3secrets-helper/sshagent/sshagent_test.go
+27 −0		tests/environment.bats
+0 −91		tests/run.bats