diff --git a/setup/Dockerfile_3.6 b/setup/Dockerfile_3.6 index a0d0ba58..30effa66 100644 --- a/setup/Dockerfile_3.6 +++ b/setup/Dockerfile_3.6 @@ -8,7 +8,8 @@ RUN apt-get update && \ libxext6 \ libxrender-dev \ libgl1-mesa-glx \ - libglib2.0-0 && \ + libglib2.0-0 \ + xvfb && \ rm -rf /var/lib/apt/lists/* diff --git a/setup/Dockerfile_3.7 b/setup/Dockerfile_3.7 index d4310c7c..910487bf 100644 --- a/setup/Dockerfile_3.7 +++ b/setup/Dockerfile_3.7 @@ -8,7 +8,8 @@ RUN apt-get update && \ libxext6 \ libxrender-dev \ libgl1-mesa-glx \ - libglib2.0-0 && \ + libglib2.0-0 \ + xvfb && \ rm -rf /var/lib/apt/lists/* diff --git a/setup/Jenkinsfile b/setup/Jenkinsfile index ea780150..ad2fbfb3 100644 --- a/setup/Jenkinsfile +++ b/setup/Jenkinsfile @@ -4,10 +4,12 @@ pipeline { environment { PROD_IMAGE = "nuscenes:production" + TEST_IMAGE_3_6 = "registry-local.nutonomy.team:5000/nuscenes-test:kube${UUID.nameUUIDFromBytes(new String(env.BUILD_TAG).getBytes())}" + TEST_IMAGE_3_7 = "registry-local.nutonomy.team:5000/nuscenes-test:kube${UUID.nameUUIDFromBytes(new String(env.BUILD_TAG).getBytes())}" } stages { - stage('Build and test'){ + stage('Build'){ agent { kubernetes { label 'nuscenes-builder' @@ -37,6 +39,9 @@ pipeline { - name: docker hostPath: path: /var/run/docker.sock + - name: nudeep-ci + persistentVolumeClaim: + claimName: nudeep-ci """ }// kubernetes } // agent @@ -46,11 +51,120 @@ pipeline { // Build the Docker image, and then run python -m unittest inside // an activated Conda environment inside of the container. sh """#!/bin/bash - bash setup/test_multiple_docker_imgs.sh + set -eux + docker build -t $TEST_IMAGE_3_6 -f setup/Dockerfile_3.6 . + docker push $TEST_IMAGE_3_6 + + docker build -t $TEST_IMAGE_3_7 -f setup/Dockerfile_3.7 . + docker push $TEST_IMAGE_3_7 + """ + } // container + } + } + stage('Test 3.6'){ + agent { + kubernetes { + label 'nuscenes-builder' + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: nuscenes + spec: + containers: + - name: jnlp + image: registry.nutonomy.com:5000/nu/jnlp-slave:3.19-1-lfs + imagePullPolicy: Always + - name: docker + image: $TEST_IMAGE_3_6 + command: + - cat + tty: true + volumeMounts: + - mountPath: /var/run/docker.sock + name: docker + - mountPath: /data/ + name: nudeep-ci + subPath: data + imagePullSecrets: + - name: regcredjenkins + volumes: + - name: docker + hostPath: + path: /var/run/docker.sock + - name: nudeep-ci + persistentVolumeClaim: + claimName: nudeep-ci + env: + - name: NUSCENES + value: /data/sets/nuscenes + """ + }// kubernetes + } // agent + + steps { + container('docker') { + sh """#!/bin/bash + python -m unittest + bash setup/test_tutorial.sh + """ + } // container + } + } // stage('Test 3.7') + stage('Test notebook 3.7'){ + agent { + kubernetes { + label 'nuscenes-builder' + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: nuscenes + spec: + containers: + - name: jnlp + image: registry.nutonomy.com:5000/nu/jnlp-slave:3.19-1-lfs + imagePullPolicy: Always + - name: docker + image: $TEST_IMAGE_3_7 + command: + - cat + tty: true + volumeMounts: + - mountPath: /var/run/docker.sock + name: docker + - mountPath: /data/ + name: nudeep-ci + subPath: data + imagePullSecrets: + - name: regcredjenkins + volumes: + - name: docker + hostPath: + path: /var/run/docker.sock + - name: nudeep-ci + persistentVolumeClaim: + claimName: nudeep-ci + env: + - name: NUSCENES + value: /data/sets/nuscenes + """ + }// kubernetes + } // agent + + steps { + container('docker') { + sh """#!/bin/bash + python -m unittest + bash setup/test_tutorial.sh """ } // container } - } // stage('Build and test') + } // stage('Test notebook 3.7') stage('Deploy') { agent { kubernetes { diff --git a/setup/test_mini_split.sh b/setup/test_mini_split.sh deleted file mode 100644 index 665e5aad..00000000 --- a/setup/test_mini_split.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -dockerfile=$1 -set -eux - -data_image=registry-local.nutonomy.team:5000/nuscenes/mini_split_data -data_container=mini_split_data -data_volume=mini_split_volume - -function clean_up(){ - echo "Cleaning up docker containers and volumes if they already exist" - docker container stop ${data_container} || { echo "container does not exist"; } - docker container rm ${data_container} || { echo "container does not exist"; } - - docker rm -f test_container || { echo "test container does not exist"; } - docker volume rm ${data_volume} || { echo "volume does not already exist"; } -} - -trap clean_up EXIT - -clean_up - -echo "Pulling image containing mini split data from registry" -docker pull ${data_image} || { echo "error during docker pull;" exit 1; } - -echo "Creating Docker volume from container" -docker run -d --name=${data_container} -v ${data_volume}:/data:ro ${data_image} || { echo "container already running";} - -echo "Building image containing nuscenes-devkit" -docker build -t test_mini_split -f ${dockerfile} . || { echo "Failed to build main Docker image"; exit 1; } - -docker run --name=test_container -v ${data_volume}:/data \ - -e NUSCENES=/data/nuscenes-v1.0 test_mini_split \ - /bin/bash -c "source activate nuenv && cd python-sdk && python -m unittest" - -clean_up diff --git a/setup/test_multiple_docker_imgs.sh b/setup/test_multiple_docker_imgs.sh deleted file mode 100755 index 4de164c4..00000000 --- a/setup/test_multiple_docker_imgs.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -bash setup/test_mini_split.sh setup/Dockerfile_3.6 -exit_code_3_6="$?" - -bash setup/test_mini_split.sh setup/Dockerfile_3.7 -exit_code_3_7="$?" - -if [ ${exit_code_3_6} -ne 0 ]; then - echo "Failed with Python 3.6 environment" - exit 1 -elif [ ${exit_code_3_7} -ne 0 ]; then - echo "Failed with Python 3.7 environment" - exit 1 -else - echo "Passed all unit tests across Docker images." -fi \ No newline at end of file diff --git a/setup/test_tutorial.sh b/setup/test_tutorial.sh new file mode 100755 index 00000000..14670ee5 --- /dev/null +++ b/setup/test_tutorial.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -eux + +# This script is to be executed inside a Docker container +source activate nuenv + +# Generate python script from Jupyter notebook and then copy into Docker image. +jupyter nbconvert --to python python-sdk/tutorial.ipynb || { echo "Failed to convert notebook to python script"; exit 1; } + +# Remove extraneous matplot inline command and comment out any render* methods. +sed -i.bak "/get_ipython.*/d; s/\(nusc\.render.*\)/#\1/" python-sdk/tutorial.py || { echo "error in sed command"; exit 1; } + +# Run tutorial +xvfb-run python python-sdk/tutorial.py +