Add RPM Temporary Signing Through Shell Scripts (#2196)

* Add RPM Temporary Signing Through Shell Scripts Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Fix typo Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * More typo Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * More comments Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Add jenkins unittest Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
opensearch-project · Jun 15, 2022 · 950d55c · 950d55c
1 parent ee2d968
commit 950d55c
Show file tree

Hide file tree

Showing 17 changed files with 445 additions and 145 deletions.
diff --git a/jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile b/jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile
@@ -4,7 +4,7 @@ pipeline {
     agent {
         docker {
             label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
-            image 'opensearchstaging/ci-runner:ci-runner-centos7-v1'
+            image 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2'
             alwaysPull true
         }
     }
@@ -25,7 +25,7 @@ pipeline {
                 description: 'What platform is this distribution build for?'
         )
         choice(
-                choices: ['.sig'],
+                choices: ['.sig', '.rpm'],
                 name: 'SIGNATURE_TYPE',
                 description: 'What is signature file type?'
         )
@@ -52,8 +52,13 @@ pipeline {
                     println("Note: only supported file types will be signed")
 
                     for(filename in downloadedFiles){
-                        filenamesForUrls.add(filename)
-                        filenamesForUrls.add(filename + SIGNATURE_TYPE)
+                        if (SIGNATURE_TYPE.equals('.sig')) {
+                            filenamesForUrls.add(filename)
+                            filenamesForUrls.add(filename + SIGNATURE_TYPE)
+                        }
+                        else {
+                            filenamesForUrls.add(filename)
+                        }
                     }
 
                     finalUploadPath = ([

diff --git a/scripts/pkg/sign_templates/rpmmacros b/scripts/pkg/sign_templates/rpmmacros
@@ -0,0 +1,10 @@
+%_signature gpg
+%_gpg_path ~/.gnupg
+%_gpg_name ##key_name##
+%_gpg /usr/bin/gpg
+%__gpg_sign_cmd %{__gpg} \
+    gpg --no-verbose --no-armor --batch --yes --pinentry-mode loopback \
+    --passphrase-file ##passphrase_name## \
+    %{?_gpg_digest_algo:--digest-algo %{_gpg_digest_algo}} \
+    --no-secmem-warning \
+    -u "%{_gpg_name}" -sbo %{__signature_filename} --digest-algo sha512 %{__plaintext_filename}
diff --git a/tests/jenkins/TestAssembleManifest.groovy b/tests/jenkins/TestAssembleManifest.groovy
@@ -15,6 +15,8 @@ class TestAssembleManifest extends BuildPipelineTest {
     void testAssembleManifest_rpm() {
         this.registerLibTester(new AssembleManifestLibTester('tests/data/opensearch-build-1.3.0-rpm.yml'))
 
+        this.registerLibTester(new SignArtifactsLibTester('.rpm', 'linux', "rpm/dist/opensearch", null, null))
+
         this.registerLibTester(new BuildYumRepoTester(
             'tests/data/opensearch-build-1.3.0-rpm.yml',
             'https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64'

diff --git a/tests/jenkins/TestSignArtifacts.groovy b/tests/jenkins/TestSignArtifacts.groovy
@@ -17,6 +17,7 @@ class TestSignArtifacts extends BuildPipelineTest {
     void setUp() {
 
         this.registerLibTester(new SignArtifactsLibTester('.sig', 'linux', "${this.workspace}/artifacts", null, null))
+        this.registerLibTester(new SignArtifactsLibTester('.rpm', 'linux', "${this.workspace}/artifacts", 'null', null))
         this.registerLibTester(new SignArtifactsLibTester(null, 'linux', "${this.workspace}/file.yml", 'maven', null))
         super.setUp()
     }

diff --git a/...nkinsjob-regression-files/data-prepper/release-data-prepper-all-artifacts.jenkinsfile.txt b/...nkinsjob-regression-files/data-prepper/release-data-prepper-all-artifacts.jenkinsfile.txt
@@ -12,21 +12,22 @@
          release-data-prepper-all-artifacts.stage(Sign Archives, groovy.lang.Closure)
             release-data-prepper-all-artifacts.script(groovy.lang.Closure)
                release-data-prepper-all-artifacts.signArtifacts({artifactPath=/tmp/workspace/archive, sigtype=.sig, platform=linux})
+                  signArtifacts.echo(PGP Signature Signing)
                   signArtifacts.fileExists(/tmp/workspace/sign.sh)
                   signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
                   signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
                   signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                   signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                      signArtifacts.sh(
-            #!/bin/bash
-            set +x
-            export ROLE=dummy_signer_client_role
-            export EXTERNAL_ID=signer_client_external_id
-            export UNSIGNED_BUCKET=signer_client_unsigned_bucket
-            export SIGNED_BUCKET=signer_client_signed_bucket
+                #!/bin/bash
+                set +x
+                export ROLE=dummy_signer_client_role
+                export EXTERNAL_ID=signer_client_external_id
+                export UNSIGNED_BUCKET=signer_client_unsigned_bucket
+                export SIGNED_BUCKET=signer_client_signed_bucket
 
-            /tmp/workspace/sign.sh /tmp/workspace/archive --sigtype=.sig --platform=linux
-        )
+                /tmp/workspace/sign.sh /tmp/workspace/archive --sigtype=.sig --platform=linux
+            )
          release-data-prepper-all-artifacts.stage(Release Archives to Production Distribution Bucket, groovy.lang.Closure)
             release-data-prepper-all-artifacts.script(groovy.lang.Closure)
                release-data-prepper-all-artifacts.withAWS({role=production-role-name, roleAccount=aws-account-artifact, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
@@ -77,21 +78,22 @@
          release-data-prepper-all-artifacts.stage(Sign Maven Artifacts, groovy.lang.Closure)
             release-data-prepper-all-artifacts.script(groovy.lang.Closure)
                release-data-prepper-all-artifacts.signArtifacts({artifactPath=/tmp/workspace/maven, type=maven, platform=linux})
+                  signArtifacts.echo(PGP Signature Signing)
                   signArtifacts.fileExists(/tmp/workspace/sign.sh)
                   signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
                   signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
                   signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                   signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                      signArtifacts.sh(
-            #!/bin/bash
-            set +x
-            export ROLE=dummy_signer_client_role
-            export EXTERNAL_ID=signer_client_external_id
-            export UNSIGNED_BUCKET=signer_client_unsigned_bucket
-            export SIGNED_BUCKET=signer_client_signed_bucket
+                #!/bin/bash
+                set +x
+                export ROLE=dummy_signer_client_role
+                export EXTERNAL_ID=signer_client_external_id
+                export UNSIGNED_BUCKET=signer_client_unsigned_bucket
+                export SIGNED_BUCKET=signer_client_signed_bucket
 
-            /tmp/workspace/sign.sh /tmp/workspace/maven --type=maven --platform=linux
-        )
+                /tmp/workspace/sign.sh /tmp/workspace/maven --type=maven --platform=linux
+            )
          release-data-prepper-all-artifacts.stage(Upload Artifacts to Sonatype, groovy.lang.Closure)
             release-data-prepper-all-artifacts.script(groovy.lang.Closure)
                release-data-prepper-all-artifacts.usernamePassword({credentialsId=Sonatype, usernameVariable=SONATYPE_USERNAME, passwordVariable=SONATYPE_PASSWORD})

diff --git a/...jenkins/jenkinsjob-regression-files/maven-sign-release/maven-sign-release.jenkinsfile.txt b/...jenkins/jenkinsjob-regression-files/maven-sign-release/maven-sign-release.jenkinsfile.txt
@@ -11,21 +11,22 @@
                      downloadFromS3.s3Download({file=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=distribution-build-opensearch/1.0.0/123/linux/x64/builds/, force=true})
                maven-sign-release.echo(Signing Maven artifacts.)
                maven-sign-release.signArtifacts({artifactPath=/tmp/workspace/artifacts/distribution-build-opensearch/1.0.0/123/linux/x64/builds/opensearch/manifest.yml, type=maven, platform=linux})
+                  signArtifacts.echo(PGP Signature Signing)
                   signArtifacts.fileExists(/tmp/workspace/sign.sh)
                   signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
                   signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
                   signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                   signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                      signArtifacts.sh(
-            #!/bin/bash
-            set +x
-            export ROLE=dummy_signer_client_role
-            export EXTERNAL_ID=signer_client_external_id
-            export UNSIGNED_BUCKET=signer_client_unsigned_bucket
-            export SIGNED_BUCKET=signer_client_signed_bucket
+                #!/bin/bash
+                set +x
+                export ROLE=dummy_signer_client_role
+                export EXTERNAL_ID=signer_client_external_id
+                export UNSIGNED_BUCKET=signer_client_unsigned_bucket
+                export SIGNED_BUCKET=signer_client_signed_bucket
 
-            /tmp/workspace/sign.sh /tmp/workspace/artifacts/distribution-build-opensearch/1.0.0/123/linux/x64/builds/opensearch/manifest.yml --type=maven --platform=linux
-        )
+                /tmp/workspace/sign.sh /tmp/workspace/artifacts/distribution-build-opensearch/1.0.0/123/linux/x64/builds/opensearch/manifest.yml --type=maven --platform=linux
+            )
          maven-sign-release.stage(stage maven artifacts, groovy.lang.Closure)
             maven-sign-release.script(groovy.lang.Closure)
                maven-sign-release.usernamePassword({credentialsId=Sonatype, usernameVariable=SONATYPE_USERNAME, passwordVariable=SONATYPE_PASSWORD})

diff --git a/...sjob-regression-files/sign-standalone-artifacts/sign-standalone-artifacts.jenkinsfile.txt b/...sjob-regression-files/sign-standalone-artifacts/sign-standalone-artifacts.jenkinsfile.txt
@@ -2,28 +2,29 @@
       sign-standalone-artifacts.legacySCM(groovy.lang.Closure)
       sign-standalone-artifacts.library({identifier=jenkins@20211123, retriever=null})
       sign-standalone-artifacts.pipeline(groovy.lang.Closure)
-         sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
+         sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
          sign-standalone-artifacts.stage(sign, groovy.lang.Closure)
             sign-standalone-artifacts.script(groovy.lang.Closure)
                sign-standalone-artifacts.sh(mkdir /tmp/workspace/artifacts)
                sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o /tmp/workspace/artifacts/dummy_1_artifact.tar.gz)
                sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o /tmp/workspace/artifacts/dummy_2_artifact.tar.gz)
                sign-standalone-artifacts.signArtifacts({artifactPath=/tmp/workspace/artifacts, sigtype=.sig, platform=linux})
+                  signArtifacts.echo(PGP Signature Signing)
                   signArtifacts.fileExists(/tmp/workspace/sign.sh)
                   signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
                   signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
                   signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
                   signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
                      signArtifacts.sh(
-            #!/bin/bash
-            set +x
-            export ROLE=dummy_signer_client_role
-            export EXTERNAL_ID=signer_client_external_id
-            export UNSIGNED_BUCKET=signer_client_unsigned_bucket
-            export SIGNED_BUCKET=signer_client_signed_bucket
+                #!/bin/bash
+                set +x
+                export ROLE=dummy_signer_client_role
+                export EXTERNAL_ID=signer_client_external_id
+                export UNSIGNED_BUCKET=signer_client_unsigned_bucket
+                export SIGNED_BUCKET=signer_client_signed_bucket
 
-            /tmp/workspace/sign.sh /tmp/workspace/artifacts --sigtype=.sig --platform=linux
-        )
+                /tmp/workspace/sign.sh /tmp/workspace/artifacts --sigtype=.sig --platform=linux
+            )
                sign-standalone-artifacts.uploadToS3({sourcePath=/tmp/workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
                   uploadToS3.withAWS({role=Dummy_Upload_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                      uploadToS3.s3Upload({file=/tmp/workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})

diff --git a/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt
@@ -12,6 +12,84 @@
                   BuildManifest.asBoolean()
                   BuildManifest.getArtifactRootUrlWithoutDistribution(https://ci.opensearch.org/dbc, vars-build, 123)
                   assembleManifest.sh(./assemble.sh "tests/data/opensearch-build-1.3.0-rpm.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64)
+                  assembleManifest.signArtifacts({artifactPath=rpm/dist/opensearch, sigtype=.rpm, platform=linux})
+                     signArtifacts.echo(RPM Add Sign)
+                     signArtifacts.withAWS({role=sign_asm_role, roleAccount=sign_asm_account, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure)
+                        signArtifacts.string({credentialsId=jenkins-rpm-signing-asm-pass-id, variable=SIGNING_PASS_ID})
+                        signArtifacts.string({credentialsId=jenkins-rpm-signing-asm-secret-id, variable=SIGNING_SECRET_ID})
+                        signArtifacts.withCredentials([SIGNING_PASS_ID, SIGNING_SECRET_ID], groovy.lang.Closure)
+                           signArtifacts.sh(
+                        set -e
+                        set +x
+
+                        ARTIFACT_PATH="rpm/dist/opensearch"
+
+                        echo "------------------------------------------------------------------------"
+                        echo "Check Utility Versions"
+                        gpg_version_limit="2.2.0"
+                        rpm_version_limit="4.13.0" # https://bugzilla.redhat.com/show_bug.cgi?id=227632
+
+                        gpg_version_check=`gpg --version | head -n 1 | grep -oE '[0-9.]+'`
+                        gpg_version_check_final=`echo $gpg_version_check $gpg_version_limit | tr ' ' '
+' | sort -V | head -n 1`
+                        rpm_version_check=`rpm --version | head -n 1 | grep -oE '[0-9.]+'`
+                        rpm_version_check_final=`echo $rpm_version_check $rpm_version_limit | tr ' ' '
+' | sort -V | head -n 1`
+
+                        echo -e "gpg_version_limit gpg_version_check"
+                        echo -e "$gpg_version_limit $gpg_version_check_final"
+                        echo -e "rpm_version_limit rpm_version_check"
+                        echo -e "$rpm_version_limit $rpm_version_check_final"
+
+                        if [[ $gpg_version_limit = $gpg_version_check_final ]] && [[ $rpm_version_limit = $rpm_version_check_final ]]; then
+                            echo "Utility version is equal or greater than set limit, continue."
+                        else
+                            echo "Utility version is lower than set limit, exit 1"
+                            exit 1
+                        fi
+
+                        export GPG_TTY=`tty`
+
+                        echo "------------------------------------------------------------------------"
+                        echo "Setup RPM Macros"
+                        cp -v scripts/pkg/sign_templates/rpmmacros ~/.rpmmacros
+                        sed -i "s/##key_name##/OpenSearch project/g;s/##passphrase_name##/passphrase/g" ~/.rpmmacros
+
+                        echo "------------------------------------------------------------------------"
+                        echo "Import OpenSearch keys"
+                        aws secretsmanager get-secret-value --region "sign_asm_region" --secret-id "SIGNING_PASS_ID" | jq -r .SecretBinary | base64 --decode > passphrase
+                        aws secretsmanager get-secret-value --region "sign_asm_region" --secret-id "SIGNING_SECRET_ID" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase -
+
+                        echo "------------------------------------------------------------------------"
+                        echo "Start Signing Rpm"
+
+                        if file $ARTIFACT_PATH | grep -q directory; then
+
+                            echo "Sign directory"
+                            for rpm_file in `ls $ARTIFACT_PATH`; do
+                                if file $ARTIFACT_PATH/$rpm_file | grep -q RPM; then
+                                    rpm --addsign $ARTIFACT_PATH/$rpm_file
+                                    rpm -qip $ARTIFACT_PATH/$rpm_file | grep Signature
+                                fi
+                            done
+
+                        elif file $ARTIFACT_PATH | grep -q RPM; then
+                            echo "Sign single rpm"
+                            rpm --addsign $ARTIFACT_PATH
+                            rpm -qip $ARTIFACT_PATH | grep Signature
+
+                        else
+                            echo "This is neither a directory nor a RPM pkg, exit 1"
+                            exit 1
+                        fi
+
+                        echo "------------------------------------------------------------------------"
+                        echo "Clean up gpg"
+                        gpg --batch --yes --delete-secret-keys sign_asm_keyid
+                        gpg --batch --yes --delete-keys sign_asm_keyid
+                        rm -v passphrase
+
+                    )
                   assembleManifest.buildYumRepo({baseUrl=https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64, buildManifest=tests/data/opensearch-build-1.3.0-rpm.yml})
                      buildYumRepo.legacySCM(groovy.lang.Closure)
                      buildYumRepo.library({identifier=jenkins@20211123, retriever=null})