Support pipecfg living in a separate repo

As part of #618, we need to move to a model where the pipeline config is stored outside of this repo. Add support for this via a new `./deploy --pipecfg` option which takes a git repo containing a `config.yaml` in the root of the repo.
coreos · Sep 14, 2022 · 8a2fe58 · 8a2fe58
1 parent e66dec6
commit 8a2fe58
Show file tree

Hide file tree

Showing 17 changed files with 48 additions and 15 deletions.
diff --git a/deploy b/deploy
@@ -70,6 +70,8 @@ def parse_args():
                         help="Repo and ref to use for pipeline code")
     parser.add_argument("--config", metavar='<URL>[@REF]',
                         help="Repo and ref to use for FCOS config")
+    parser.add_argument("--pipecfg", metavar='<URL>[@REF]',
+                        help="Repo and ref to use for pipeline config")
     parser.add_argument("--cosa-img", metavar='FQIN',
                         help="Pullspec to use for COSA image")
     parser.add_argument("--oc-cmd", default='oc',
@@ -92,6 +94,8 @@ def process_template(args):
     if args.pipeline:
         params.update(params_from_git_refspec(args.pipeline, 'JENKINS_S2I'))
         params.update(params_from_git_refspec(args.pipeline, 'JENKINS_JOBS'))
+    if args.pipecfg:
+        params.update(params_from_git_refspec(args.pipecfg, 'PIPECFG'))
     if args.config:
         params.update(params_from_git_refspec(args.config, 'FCOS_CONFIG'))
     if args.cosa_img:

diff --git a/jobs/build-arch.Jenkinsfile b/jobs/build-arch.Jenkinsfile
@@ -5,7 +5,7 @@ def src_config_url, src_config_ref, s3_bucket
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
     pod = readFile(file: "manifests/pod.yaml")
 
 

diff --git a/jobs/build-cosa.Jenkinsfile b/jobs/build-cosa.Jenkinsfile
@@ -4,7 +4,7 @@ def containername = 'coreos-assembler'
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
     official = pipeutils.isOfficial()
 }
 

diff --git a/jobs/build-development.Jenkinsfile b/jobs/build-development.Jenkinsfile
@@ -2,7 +2,7 @@ def pipecfg, pipeutils
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
 }
 
 properties([

diff --git a/jobs/build-mechanical.Jenkinsfile b/jobs/build-mechanical.Jenkinsfile
@@ -2,7 +2,7 @@ def pipecfg, pipeutils
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
 }
 
 properties([

diff --git a/jobs/build.Jenkinsfile b/jobs/build.Jenkinsfile
@@ -6,7 +6,7 @@ def gcp_gs_bucket
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
     pod = readFile(file: "manifests/pod.yaml")
 
     def jenkinscfg = pipeutils.load_jenkins_config()

diff --git a/jobs/bump-lockfile.Jenkinsfile b/jobs/bump-lockfile.Jenkinsfile
@@ -2,7 +2,7 @@ def pipeutils, pipecfg, official, arches
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
     arches = pipecfg.additional_arches.plus("x86_64")
     official = pipeutils.isOfficial()
 }

diff --git a/jobs/bump-lockfiles.Jenkinsfile b/jobs/bump-lockfiles.Jenkinsfile
@@ -12,7 +12,8 @@ properties([
 
 node {
     checkout scm
-    def pipecfg = readYaml file: "config.yaml"
+    def pipeutils = load("utils.groovy")
+    def pipecfg = pipeutils.load_pipecfg()
     def development_streams = pipeutils.streams_of_type(pipecfg, 'development')
 
     parallel development_streams.collectEntries { stream -> [stream, {

diff --git a/jobs/kola-aws.Jenkinsfile b/jobs/kola-aws.Jenkinsfile
@@ -2,7 +2,7 @@ def pipeutils, pipecfg, official
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
     official = pipeutils.isOfficial()
 }
 

diff --git a/jobs/kola-azure.Jenkinsfile b/jobs/kola-azure.Jenkinsfile
@@ -5,7 +5,7 @@ def azure_testing_storage_container
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
     def jenkinscfg = pipeutils.load_jenkins_config()
     azure_testing_resource_group = pipecfg.azure?.test_resource_group
     azure_testing_storage_account = pipecfg.azure?.test_storage_account

diff --git a/jobs/kola-gcp.Jenkinsfile b/jobs/kola-gcp.Jenkinsfile
@@ -2,7 +2,7 @@ def pipeutils, pipecfg, official
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
     official = pipeutils.isOfficial()
 }
 

diff --git a/jobs/kola-kubernetes.Jenkinsfile b/jobs/kola-kubernetes.Jenkinsfile
@@ -2,7 +2,7 @@ def pipeutils, pipecfg, official
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
     official = pipeutils.isOfficial()
 }
 

diff --git a/jobs/kola-openstack.Jenkinsfile b/jobs/kola-openstack.Jenkinsfile
@@ -2,7 +2,7 @@ def pipeutils, pipecfg, official
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
     official = pipeutils.isOfficial()
 }
 

diff --git a/jobs/release.Jenkinsfile b/jobs/release.Jenkinsfile
@@ -2,7 +2,7 @@ def pipeutils, pipecfg, official, s3_bucket, jenkins_agent_image_tag
 node {
     checkout scm
     pipeutils = load("utils.groovy")
-    pipecfg = readYaml file: "config.yaml"
+    pipecfg = pipeutils.load_pipecfg()
     pod = readFile(file: "manifests/pod.yaml")
     def jenkinscfg = pipeutils.load_jenkins_config()
     s3_bucket = pipecfg.s3_bucket

diff --git a/jobs/sync-stream-metadata.Jenkinsfile b/jobs/sync-stream-metadata.Jenkinsfile
@@ -1,8 +1,8 @@
 def pipecfg, pipeutils
 node {
     checkout scm
-    pipecfg = readYaml file: "config.yaml"
     pipeutils = load("utils.groovy")
+    pipecfg = pipeutils.load_pipecfg()
 }
 
 properties([

diff --git a/manifests/pipeline.yaml b/manifests/pipeline.yaml
@@ -21,6 +21,12 @@ parameters:
   - description: Git branch/tag reference for Jenkins jobs
     name: JENKINS_JOBS_REF
     value: main
+  - description: Git source URI for pipeline configuration
+    name: PIPECFG_URL
+    value: in-tree
+  - description: Git branch/tag reference for pipeline configuration
+    name: PIPECFG_REF
+    value: main
   - description: Jenkins agent image to use
     name: JENKINS_AGENT_IMAGE_TAG
     value: base
@@ -55,7 +61,7 @@ objects:
   ### JENKINS CONFIG ###
 
   # Only keep here base Jenkins configuration knobs. Higher-level "variant"
-  # configuration should live in the `config.yaml` instead. A litmus test to identify
+  # configuration should live in the pipecfg instead. A litmus test to identify
   # such knobs is: would this knob still be needed if the pipeline were not
   # Jenkins-based?
 
@@ -67,3 +73,5 @@ objects:
       jenkins-jobs-url: ${JENKINS_JOBS_URL}
       jenkins-jobs-ref: ${JENKINS_JOBS_REF}
       jenkins-agent-image-tag: ${JENKINS_AGENT_IMAGE_TAG}
+      pipecfg-url: ${PIPECFG_URL}
+      pipecfg-ref: ${PIPECFG_REF}
diff --git a/utils.groovy b/utils.groovy
@@ -10,6 +10,26 @@ def load_jenkins_config() {
     """))
 }
 
+def load_pipecfg() {
+    def jenkinscfg = load_jenkins_config()
+    def url = jenkinscfg['pipecfg-url']
+    def ref = jenkinscfg['pipecfg-ref']
+
+    if (url == 'in-tree') {
+        return readYaml file: "config.yaml"
+    }
+
+    // this uses the `checkout` workflow step instead of just manually cloning so
+    // that changes show up in the Jenkins UI
+    checkout([
+        $class: 'GitSCM',
+        branches: [[name: "origin/${ref}"]],
+        extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'pipecfg']],
+        userRemoteConfigs: [[url: url]]
+    ])
+    return readYaml file: "pipecfg/config.yaml"
+}
+
 // Tells us if we're running if the official Jenkins for the FCOS pipeline
 boolean isOfficial() {
     return (env.JENKINS_URL in ['https://jenkins-fedora-coreos-pipeline.apps.ocp.fedoraproject.org/'])