Add support for additional root CA certs

We're working on getting this pipeline working internally. The first
thing we hit is that Red Hat has its own internal root CA.

We'll need to handle this both for cosa and the Jenkins agent which
wants to fetch `remoting.jar` from the controller as soon as it starts.

This patch handles just the latter for now. We do this by optionally
supporting building our own Jenkins agent with the certificates in it.

For cosa, I think we will be able to work around this by directly
pointing e.g. git and libdnf at the certificate.

This also fixes a mismatch issue: before we were using an unversioned
agent image from Quay. Now, we're following the in-cluster imagestream,
just like for the controller itself, so we know they're always matching.

HACKING.md

@@ -340,6 +340,8 @@ circumstances. Below are some of the options. To see more run
     - Image of coreos-assembler to use.
 - `--bucket BUCKET`
     - AWS S3 bucket in which to store builds (or blank for none).
+- `--additional-root-ca-certs-secret SECRET`
+    - Secret name to any additional root CA certs
 
 For example, to target a specific combination of pipeline, FCOS config,
 cosa image, and bucket:

deploy

@@ -83,6 +83,8 @@ def parse_args():
                         help="Pullspec to use for COSA image")
     parser.add_argument("--oc-cmd", default='oc',
                         help="The path to the oc binary")
+    parser.add_argument("--additional-root-ca-certs-secret", metavar='SECRET',
+                        help="Secret name to additional root CA certs")
     # XXX: to add as a mutually exclusive option with above
     # parser.add_argument("--cosa", metavar='<URL>[@REF]',
     #                     help="Repo and ref to use for COSA image",
@@ -100,6 +102,8 @@ def get_username():
 
 
 def process_template(args):
+    templates = ['pipeline.yaml', 'jenkins-s2i.yaml']
+
     params = {}
     if args.pipeline:
         params.update(params_from_git_refspec(args.pipeline, 'JENKINS_S2I'))
@@ -118,6 +122,10 @@ def process_template(args):
         params['AZURE_TESTING_STORAGE_ACCOUNT'] = args.azure_testing_storage_account
     if args.azure_testing_storage_container:
         params['AZURE_TESTING_STORAGE_CONTAINER'] = args.azure_testing_storage_container
+    if args.additional_root_ca_certs_secret:
+        params['ADDITIONAL_ROOT_CA_CERTS_SECRET'] = args.additional_root_ca_certs_secret
+        params['JENKINS_AGENT_IMAGE_TAG'] = "derived"
+        templates += ['jenkins-agent.yaml']
 
     print("Parameters:")
     for k, v in params.items():
@@ -129,7 +137,7 @@ def process_template(args):
         return [q for k, v in selected_params for q in ['--param', f'{k}={v}']]
 
     resources = []
-    for template in ['pipeline.yaml', 'jenkins-s2i.yaml']:
+    for template in templates:
         # we only want to pass the params which each template actually
         # supports, so filter it down for this specific template
         with open(f'manifests/{template}') as f:

jobs/build-arch.Jenkinsfile

@@ -1,6 +1,6 @@
 import org.yaml.snakeyaml.Yaml;
 
-def pipeutils, streams, official, uploading
+def pipeutils, streams, official, uploading, jenkins_agent_image_tag
 def src_config_url, src_config_ref, s3_bucket
 node {
     checkout scm
@@ -13,6 +13,7 @@ node {
     src_config_url = pipecfg['source-config-url']
     src_config_ref = pipecfg['source-config-ref']
     s3_bucket = pipecfg['s3-bucket']
+    jenkins_agent_image_tag = pipecfg['jenkins-agent-image-tag']
 
     official = pipeutils.isOfficial()
     if (official) {
@@ -95,6 +96,7 @@ pod = pod.replace("COREOS_ASSEMBLER_CPU_LIMIT", "1")
 // substitute the right COSA image and mem request into the pod definition before spawning it
 pod = pod.replace("COREOS_ASSEMBLER_MEMORY_REQUEST", "${cosa_memory_request_mb}Mi")
 pod = pod.replace("COREOS_ASSEMBLER_IMAGE", params.COREOS_ASSEMBLER_IMAGE)
+pod = pod.replace("JENKINS_AGENT_IMAGE_TAG", jenkins_agent_image_tag)
 
 def podYaml = readYaml(text: pod);
 

jobs/build.Jenkinsfile

@@ -1,6 +1,6 @@
 import org.yaml.snakeyaml.Yaml;
 
-def pipeutils, streams, official, uploading
+def pipeutils, streams, official, uploading, jenkins_agent_image_tag
 def src_config_url, src_config_ref, s3_bucket
 def gcp_gs_bucket
 node {
@@ -14,6 +14,7 @@ node {
     src_config_ref = pipecfg['source-config-ref']
     s3_bucket = pipecfg['s3-bucket']
     gcp_gs_bucket = pipecfg['gcp-gs-bucket']
+    jenkins_agent_image_tag = pipecfg['jenkins-agent-image-tag']
 
     official = pipeutils.isOfficial()
     if (official) {
@@ -95,6 +96,7 @@ pod = pod.replace("COREOS_ASSEMBLER_CPU_LIMIT", "4")
 // substitute the right COSA image and mem request into the pod definition before spawning it
 pod = pod.replace("COREOS_ASSEMBLER_MEMORY_REQUEST", "${cosa_memory_request_mb}Mi")
 pod = pod.replace("COREOS_ASSEMBLER_IMAGE", params.COREOS_ASSEMBLER_IMAGE)
+pod = pod.replace("JENKINS_AGENT_IMAGE_TAG", jenkins_agent_image_tag)
 
 def podYaml = readYaml(text: pod);
 

manifests/jenkins-agent.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: Template
+metadata:
+  name: jenkins-agent
+parameters:
+  - description: Secret name containing any additional root CA certificates
+    name: ADDITIONAL_ROOT_CA_CERTS_SECRET
+    value: additional-root-ca-certs
+objects:
+  - kind: BuildConfig
+    apiVersion: v1
+    metadata:
+      name: jenkins-agent
+    spec:
+      source:
+        dockerfile: |
+          FROM overridden
+          COPY certs/* /etc/pki/ca-trust/source/anchors/
+          RUN update-ca-trust
+        secrets:
+          - destinationDir: certs
+            secret:
+              name: ${ADDITIONAL_ROOT_CA_CERTS_SECRET}
+      strategy:
+        dockerStrategy:
+          from:
+            kind: ImageStreamTag
+            name: jenkins-agent-base:latest
+            namespace: openshift
+          forcePull: true
+      output:
+        to:
+          kind: ImageStreamTag
+          name: jenkins-agent:derived
+      successfulBuildsHistoryLimit: 2
+      failedBuildsHistoryLimit: 2

manifests/jenkins-s2i.yaml

@@ -46,3 +46,22 @@ objects:
           name: jenkins:2
       successfulBuildsHistoryLimit: 2
       failedBuildsHistoryLimit: 2
+
+  ### JENKINS AGENT ###
+
+  - apiVersion: v1
+    kind: ImageStream
+    metadata:
+      name: jenkins-agent
+    spec:
+      lookupPolicy:
+        # this allows e.g. the pipeline to directly reference the imagestream
+        local: true
+      tags:
+        - name: base
+          from:
+            kind: ImageStreamTag
+            name: jenkins-agent-base:latest
+            namespace: openshift
+          referencePolicy:
+            type: Source

manifests/pipeline.yaml

@@ -21,6 +21,9 @@ parameters:
   - description: Git branch/tag reference for Jenkins jobs
     name: JENKINS_JOBS_REF
     value: main
+  - description: Jenkins agent image to use
+    name: JENKINS_AGENT_IMAGE_TAG
+    value: base
   - description: Git source URI for FCOS config
     name: FCOS_CONFIG_URL
     value: https://github.com/coreos/fedora-coreos-config
@@ -86,3 +89,4 @@ objects:
       azure-testing-storage-container: ${AZURE_TESTING_STORAGE_CONTAINER}
       jenkins-jobs-url: ${JENKINS_JOBS_URL}
       jenkins-jobs-ref: ${JENKINS_JOBS_REF}
+      jenkins-agent-image-tag: ${JENKINS_AGENT_IMAGE_TAG}

manifests/pod.yaml

@@ -12,7 +12,7 @@ spec:
   serviceAccountName: jenkins
   containers:
    - name: jnlp
-     image: quay.io/openshift/origin-jenkins-agent-base:latest
+     image: jenkins-agent:JENKINS_AGENT_IMAGE_TAG
      args: ['$(JENKINS_SECRET)', '$(JENKINS_NAME)']
    - name: coreos-assembler
      image: COREOS_ASSEMBLER_IMAGE