Define a v0.8 release (kubeflow#572)

* Define a v0.8 release * Update applications.yaml with a v0.8 release. * The purpose of this PR is to check that just by defining the appropriate release we can begin building images from release branches and updating the release branch of kubeflow/manifests * Related to kubeflow/testing#450 - Continuous delivery of Kubeflow applications * Create a python script for opening up the PR; this script replaces the bash script rebuild-manifests.sh that was used previously * The new script doesn't assume that the base branch for PRs is master. We need this to support updating release branches. * Create a profile in skaffold.yaml for running on the release cluster. * Create an image_util package to parse image URLs. * Use the Docker image for apps-cd to run create_manifests_pr.py * Add kustomize, go, and some other tools we need * In the docker image create a symbolic link for .ssh so we can pick up ssh credentials created by Tekton. * * Define a 1.0 release now that the branches have been cut * Related to kubeflow/kubeflow#4685
Linchin · Jan 25, 2020 · f60df7a · f60df7a
1 parent ed4a09b
commit f60df7a
Show file tree

Hide file tree

Showing 14 changed files with 464 additions and 154 deletions.
diff --git a/apps-cd/Dockerfile b/apps-cd/Dockerfile
@@ -1,22 +1,46 @@
 FROM ubuntu:18.04
 
 RUN apt-get update -y && \
-    apt-get install -y curl git python3.8 python3-pip && \
+    apt-get install -y curl git python3.8 python3-pip wget && \
     ln -sf /usr/bin/python3.8 /usr/bin/python
 
+# Install go
+RUN cd /tmp && \
+    wget -O /tmp/go.tar.gz https://redirector.gvt1.com/edgedl/go/go1.12.linux-amd64.tar.gz && \
+    tar -C /usr/local -xzf go.tar.gz
+
 # Install the hub CLI for git
 RUN cd /tmp && \
     curl -LO  https://github.com/github/hub/releases/download/v2.13.0/hub-linux-amd64-2.13.0.tgz && \
     tar -xvf hub-linux-amd64-2.13.0.tgz && \
     mv hub-linux-amd64-2.13.0 /usr/local && \
     ln -sf /usr/local/hub-linux-amd64-2.13.0/bin/hub /usr/local/bin/hub
 
+RUN export KUSTOMIZE_VERSION=3.2.0 && \
+    cd /tmp && \
+    curl -LO  https://github.com/kubernetes-sigs/kustomize/releases/download/v${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64 && \
+    mv kustomize_${KUSTOMIZE_VERSION}_linux_amd64 /usr/local/bin/kustomize && \
+    chmod a+x /usr/local/bin/kustomize
+
+
 RUN python -m pip install fire \
      google-api-python-client \
      google-cloud-storage \
      kubernetes \
      watchdog
 
+
+# Create go symlinks
+RUN ln -sf /usr/local/go/bin/go /usr/local/bin && \
+    ln -sf /usr/local/go/bin/gofmt /usr/local/bin && \
+    ln -sf /usr/local/go/bin/godoc /usr/local/bin
+
 RUN mkdir -p /app
 COPY update_launcher.py /app
-COPY run_with_auto_restart.py /app
+COPY run_with_auto_restart.py /app
+
+# See(https://github.com/tektoncd/pipeline/issues/1271): Tekton will put ssh
+# credentials in /tekton/home. We can't change the home directory
+# but we can create a symbolic link for .ssh
+RUN mkdir -p /tekton/home && \
+    ln -sf /tekton/home/.ssh /root/.ssh
diff --git a/apps-cd/applications.yaml b/apps-cd/applications.yaml
@@ -96,5 +96,49 @@ versions:
           params:
             - name: revision
               value: master
+            - name: url
+              value: git@github.com:kubeflow/manifests.git
+  # Define a v1-0 release
+  - name: v1-0
+    # A tag to prefix image names with
+    tag: "v1.0.0"
+    repos:
+      - name: kubeflow
+        resourceSpec:    
+          type: git
+          params:
+            - name: revision
+              value: v1.0-branch
+            - name: url
+              value: git@github.com:kubeflow/kubeflow.git
+      - name: manifests
+        resourceSpec:    
+          type: git
+          params:
+            - name: revision
+              value: v1.0-branch
+            - name: url
+              value: git@github.com:kubeflow/manifests.git
+  # Define a v0-8 release
+  # This is primarily so we can test that the CI/CD infrastructure is workin 
+  # with release branches. We don't plan on actually releasing 0.8
+  - name: v0-8
+    # A tag to prefix image names with
+    tag: "v0.8.0"
+    repos:
+      - name: kubeflow
+        resourceSpec:    
+          type: git
+          params:
+            - name: revision
+              value: v0.8-branch
+            - name: url
+              value: git@github.com:kubeflow/kubeflow.git
+      - name: manifests
+        resourceSpec:    
+          type: git
+          params:
+            - name: revision
+              value: v0.8-branch
             - name: url
               value: git@github.com:kubeflow/manifests.git
diff --git a/apps-cd/pipelines/base/deployment.yaml b/apps-cd/pipelines/base/deployment.yaml
@@ -32,7 +32,12 @@ spec:
         - run
         # repo_dir is the directory where kubeflow testing should be checked out to
         - --repo_dir=/launcher_src/kubeflow/testing
-        - --repo=https://github.com/kubeflow/testing.git
+        # repo is the code to check out to get the configuration as well as code for
+        # launching the applications
+        #
+        # To test changes before they are checked in you can set this to the repo containing your PR
+        # e.g. - --repo=https://github.com/jlewi/testing.git?ref=cicd_0.8
+        - --repo=https://github.com/kubeflow/testing.git                
         # Extra arguments to be passed to update_kf_apps.py
         - --namespace=kf-releasing 
         - --config=/launcher_src/kubeflow/testing/apps-cd/applications.yaml 

diff --git a/apps-cd/pipelines/base/kustomization.yaml b/apps-cd/pipelines/base/kustomization.yaml
@@ -1,8 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
-- name: gcr.io/kubeflow-releasing/update_kf_apps
-  newName: gcr.io/kubeflow-releasing/update_kf_apps
+- digest: sha256:3c9360314639724f4b5e1b675673186b9bad726852b2bc1d5bd14951b6ffd26c
+  name: gcr.io/kubeflow-releasing/update_kf_apps
+  newName: gcr.io/kubeflow-releasing/update_kf_apps:043da55-dirty
 resources:
 - service-account.yaml
 - deployment.yaml

diff --git a/apps-cd/pipelines/base/task.yaml b/apps-cd/pipelines/base/task.yaml
@@ -34,7 +34,7 @@ spec:
     - --target=$(inputs.params.docker_target)
     - --destination=$(inputs.resources.image.url)
     - --context=/workspace/$(inputs.resources.kubeflow.name)/$(inputs.params.path_to_context)
-    - --digest-file=/workspace/image-digest      
+    - --digest-file=/workspace/image-digest    
     env:
     - name: GOOGLE_APPLICATION_CREDENTIALS
       value: /secret/user-gcp-sa.json     
@@ -83,8 +83,18 @@ spec:
     workingDir: /workspace/$(inputs.resources.manifests.name)/$(inputs.params.path_to_manifests_dir)
     image: $(inputs.params.container_image)    
     command:
-    - /workspace/$(inputs.resources.ci-tools.name)/py/kubeflow/testing/ci/rebuild-manifests.sh
+    # python should be python3.8
+    - python
+    - -m
+    - kubeflow.testing.cd.create_manifests_pr
+    - apply 
+    - --image_url=${IMAGE_URL}
+    - --src_image_url=${SRC_IMAGE_URL} 
+    - --manifests_dir=/workspace/$(inputs.resources.manifests.name)/$(inputs.params.path_to_manifests_dir)
+    - --manifests_base=$(inputs.resources.manifests.revision)
     env:
+    - name: PYTHONPATH
+      value: /workspace/$(inputs.resources.ci-tools.name)/py
     - name: GOOGLE_APPLICATION_CREDENTIALS
       value: /secret/gcp-credentials/user-gcp-sa.json
     - name: GITHUB_TOKEN

diff --git a/apps-cd/runs/app-pipeline.template.yaml b/apps-cd/runs/app-pipeline.template.yaml
@@ -18,7 +18,7 @@ spec:
   - name: "src_image_url"
     value: "gcr.io/kubeflow-images-public/profile-controller"
   - name: "container_image"
-    value: "gcr.io/kubeflow-releasing/test-worker@sha256:35138a42b57160a078e802b7d69aec3c3e79a3e2e55518af7798275ebcc84d25"    
+    value: "gcr.io/kubeflow-releasing/update_kf_apps:d4b8243@sha256:656a4ad689b17715a287f09772cc48444a2190c353c5b84828db4fbf4ede4acc"
   resources:
   # The git resources that will be used
   - name: kubeflow
@@ -41,16 +41,13 @@ spec:
     resourceSpec:    
       type: git
       params:
-        # TODO(jlewi): Replace with kubeflw/testing master once the PR is merged
-        #
+        # TODO(jlewi): Replace with kubeflow/testing master once the PR 
+        # https://github.com/kubeflow/testing/pull/572
+        # is merged
         - name: revision
-          value: cicd
+          value: master
         - name: url
-          value: git@github.com:jlewi/testing.git
-        #- name: revision
-        #  value: master
-        #- name: url
-        #  value: git@github.com:kubeflow/testing.git
+          value: git@github.com:kubeflow/testing.git
   # The image we want to build
   - name:  image
     resourceSpec:

diff --git a/apps-cd/skaffold.yaml b/apps-cd/skaffold.yaml
@@ -2,47 +2,50 @@
 apiVersion: skaffold/v2alpha1
 kind: Config
 metadata:
-  name: label-microservice
-build:
-  artifacts:
-  - image: gcr.io/kubeflow-releasing/update_kf_apps
-    # Set the context to the root directory. 
-    # All paths in the Dockerfile should be relative to this one.
-    context: .
-    # Automatically sync python files to the container. This should avoid
-    # the need to rebuild and redeploy when the files change.
-    # TODO(https://github.com/GoogleContainerTools/skaffold/issues/3448): We use manual sync
-    # because inferred sync doesn't work
-    #
-    # This only works if we autorestart the program on changes.
-    #
-    # Important: Make sure you current context has the namespace
-    # set to the namespace where your pods are deployed otherwise
-    # the sync doesn't appear to work.
-    sync:
-        manual:
-        - src: 'update_launcher.py'
-          dest: '/app'
-    kaniko:
-      dockerfile: Dockerfile
-      buildContext:
-        gcsBucket: kubeflow-releasing_skaffold
-      env: 
-        # TODO(GoogleContainerTools/skaffold#3468) skaffold doesn't
-        # appear to work with workload identity
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /secret/user-gcp-sa.json
-      cache: {}
-  cluster:    
-    pullSecretName: user-gcp-sa
-    # Build in a namespace with ISTIO sidecar injection disabled
-    # see  GoogleContainerTools/skaffold#3442
-    namespace: kf-releasing
-    resources:
-      requests:
-        cpu: 8
-        memory: 16Gi
+  name: kf-apps-cd
+profiles:
+  # Build on the kubeflow releasing cluster
+  - name: kf-releasing
+    build:
+      artifacts:
+      - image: gcr.io/kubeflow-releasing/update_kf_apps
+        # Set the context to the root directory. 
+        # All paths in the Dockerfile should be relative to this one.
+        context: .
+        # Automatically sync python files to the container. This should avoid
+        # the need to rebuild and redeploy when the files change.
+        # TODO(https://github.com/GoogleContainerTools/skaffold/issues/3448): We use manual sync
+        # because inferred sync doesn't work
+        #
+        # This only works if we autorestart the program on changes.
+        #
+        # Important: Make sure you current context has the namespace
+        # set to the namespace where your pods are deployed otherwise
+        # the sync doesn't appear to work.
+        sync:
+            manual:
+            - src: 'update_launcher.py'
+              dest: '/app'
+        kaniko:
+          dockerfile: Dockerfile
+          buildContext:
+            gcsBucket: kubeflow-releasing_skaffold
+          env: 
+            # TODO(GoogleContainerTools/skaffold#3468) skaffold doesn't
+            # appear to work with workload identity
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: /secret/user-gcp-sa.json
+          cache: {}
+      cluster:    
+        pullSecretName: user-gcp-sa
+        # Build in a namespace with ISTIO sidecar injection disabled
+        # see  GoogleContainerTools/skaffold#3442
+        namespace: kf-releasing
+        resources:
+          requests:
+            cpu: 8
+            memory: 16Gi
 
-deploy:
-  kustomize:
-    path: pipelines/base
+    deploy:
+      kustomize:
+        path: pipelines/base
diff --git a/images/README.md b/images/README.md
@@ -0,0 +1,13 @@
+# Worker Images for Test and Release Workers
+
+This directory contains the source for the docker images
+that we use to run a bunch of our test and release scripts.
+
+
+## To build a release image
+
+```
+skaffold build -v info -p kf-releasing 
+```
+
+* This will only work if you have access to the release project
diff --git a/images/skaffold.yaml b/images/skaffold.yaml
@@ -0,0 +1,43 @@
+# Reference: https://skaffold.dev/docs/references/yaml/
+#
+# TODO(jlewi): This is just a rudimentary skaffold config.
+# Users probably need to create their own profile based on the resources (e.g. GCR)
+# They have access too.
+apiVersion: skaffold/v2alpha1
+kind: Config
+metadata:
+  name: notebook-controller
+profiles:
+- name: kf-releasing
+  build:
+    artifacts:
+    # TODO(jlewi): We should probably use skaffold profiles to build this for the 
+    # test cluster vs release cluster
+    - image: gcr.io/kubeflow-releasing/test-worker
+      # Context should be ${KUBEFLOW_REPO}/components
+      context: .
+      kaniko:
+        dockerfile: Dockerfile
+        buildContext:
+          gcsBucket: kubeflow-releasing_skaffold
+        env: 
+          # TODO(GoogleContainerTools/skaffold#3468) skaffold doesn't
+          # appear to work with workload identity and the kubeflow-testing
+          # cluster isn't using it yet.
+          - name: GOOGLE_APPLICATION_CREDENTIALS
+            value: /secret/user-gcp-sa.json
+        cache: {}
+    cluster:    
+      pullSecretName: user-gcp-sa
+      # Build in the kaniko namespace because we need to disable ISTIO sidecar injection
+      # see  GoogleContainerTools/skaffold#3442
+      namespace: kf-releasing
+      resources:
+        requests:
+          # TODO(https://github.com/kubeflow/testing/issues/565): Right now the nodes in our cluster
+          # have a max of 8 CPU so we need to set the requests small enough so the pods get scheduled.
+          cpu: 6
+          memory: 16Gi
+# TODO(jlewi): We should add a deploy section to actually deploy the controller. Assuming 
+# kubeflow/manifests is checked out we should be able to just point to he kustomize manifest in that
+# directory