* Added xvfb to list of packages to install. (nutonomy#103)

* Re-organized Jenkins pipeline such that build and test occurs in separate stages.
* Added logic to test notebooks during Jenkins pipeline.
* Deleted old bash scripts for testing.

setup/Dockerfile_3.6

@@ -8,7 +8,8 @@ RUN apt-get update && \
         libxext6 \
         libxrender-dev \
         libgl1-mesa-glx \
-        libglib2.0-0 && \
+        libglib2.0-0 \
+        xvfb && \
     rm -rf /var/lib/apt/lists/*
 
 

setup/Dockerfile_3.7

@@ -8,7 +8,8 @@ RUN apt-get update && \
         libxext6 \
         libxrender-dev \
         libgl1-mesa-glx \
-        libglib2.0-0 && \
+        libglib2.0-0 \
+        xvfb && \
     rm -rf /var/lib/apt/lists/*
 
 

setup/Jenkinsfile

@@ -4,10 +4,12 @@ pipeline {
 
   environment {
     PROD_IMAGE = "nuscenes:production"
+    TEST_IMAGE_3_6 = "registry-local.nutonomy.team:5000/nuscenes-test:kube${UUID.nameUUIDFromBytes(new String(env.BUILD_TAG).getBytes())}"
+    TEST_IMAGE_3_7 = "registry-local.nutonomy.team:5000/nuscenes-test:kube${UUID.nameUUIDFromBytes(new String(env.BUILD_TAG).getBytes())}"
   }
 
   stages {
-    stage('Build and test'){
+    stage('Build'){
       agent {
         kubernetes {
           label 'nuscenes-builder'
@@ -37,6 +39,9 @@ pipeline {
               - name: docker
                 hostPath:
                   path: /var/run/docker.sock
+              - name: nudeep-ci
+                persistentVolumeClaim:
+                  claimName: nudeep-ci
           """
         }// kubernetes
       } // agent
@@ -46,11 +51,120 @@ pipeline {
           // Build the Docker image, and then run python -m unittest inside 
           // an activated Conda environment inside of the container.
           sh """#!/bin/bash
-          bash setup/test_multiple_docker_imgs.sh
+            set -eux
+            docker build -t $TEST_IMAGE_3_6 -f setup/Dockerfile_3.6 .
+            docker push $TEST_IMAGE_3_6
+
+            docker build -t $TEST_IMAGE_3_7 -f setup/Dockerfile_3.7 .
+            docker push $TEST_IMAGE_3_7
+          """
+        } // container
+      }
+    }
+    stage('Test 3.6'){
+      agent {
+        kubernetes {
+          label 'nuscenes-builder'
+          defaultContainer 'jnlp'
+          yaml """
+            apiVersion: v1
+            kind: Pod
+            metadata:
+              labels:
+                app: nuscenes
+            spec:
+              containers:
+              - name: jnlp
+                image: registry.nutonomy.com:5000/nu/jnlp-slave:3.19-1-lfs
+                imagePullPolicy: Always
+              - name: docker
+                image: $TEST_IMAGE_3_6
+                command:
+                - cat
+                tty: true
+                volumeMounts:
+                - mountPath: /var/run/docker.sock
+                  name: docker
+                - mountPath: /data/
+                  name: nudeep-ci
+                  subPath: data
+              imagePullSecrets:
+              - name: regcredjenkins
+              volumes:
+              - name: docker
+                hostPath:
+                  path: /var/run/docker.sock
+              - name: nudeep-ci
+                persistentVolumeClaim:
+                  claimName: nudeep-ci
+              env:
+              - name: NUSCENES
+                value: /data/sets/nuscenes
+          """
+        }// kubernetes
+      } // agent
+
+      steps {
+        container('docker') {
+          sh """#!/bin/bash
+            python -m unittest
+            bash setup/test_tutorial.sh
+          """
+        } // container
+      }
+    } // stage('Test 3.7')
+    stage('Test notebook 3.7'){
+      agent {
+        kubernetes {
+          label 'nuscenes-builder'
+          defaultContainer 'jnlp'
+          yaml """
+            apiVersion: v1
+            kind: Pod
+            metadata:
+              labels:
+                app: nuscenes
+            spec:
+              containers:
+              - name: jnlp
+                image: registry.nutonomy.com:5000/nu/jnlp-slave:3.19-1-lfs
+                imagePullPolicy: Always
+              - name: docker
+                image: $TEST_IMAGE_3_7
+                command:
+                - cat
+                tty: true
+                volumeMounts:
+                - mountPath: /var/run/docker.sock
+                  name: docker
+                - mountPath: /data/
+                  name: nudeep-ci
+                  subPath: data
+              imagePullSecrets:
+              - name: regcredjenkins
+              volumes:
+              - name: docker
+                hostPath:
+                  path: /var/run/docker.sock
+              - name: nudeep-ci
+                persistentVolumeClaim:
+                  claimName: nudeep-ci
+              env:
+              - name: NUSCENES
+                value: /data/sets/nuscenes
+          """
+        }// kubernetes
+      } // agent
+
+      steps {
+        container('docker') {
+          sh """#!/bin/bash
+            python -m unittest
+            bash setup/test_tutorial.sh
           """
         } // container
       }
-    } // stage('Build and test')
+    } // stage('Test notebook 3.7')
     stage('Deploy') {
       agent {
         kubernetes {

setup/test_tutorial.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -eux
+
+# This script is to be executed inside a Docker container
+source activate nuenv
+
+# Generate python script from Jupyter notebook and then copy into Docker image.
+jupyter nbconvert --to python python-sdk/tutorial.ipynb || { echo "Failed to convert notebook to python script"; exit 1; }
+
+# Remove extraneous matplot inline command and comment out any render* methods.
+sed -i.bak "/get_ipython.*/d; s/\(nusc\.render.*\)/#\1/" python-sdk/tutorial.py || { echo "error in sed command"; exit 1; }
+
+# Run tutorial
+xvfb-run python python-sdk/tutorial.py
+