[TEST] CI infrastructure (apache#30)

Jenkinsfile

@@ -0,0 +1,128 @@
+#!groovy
+// -*- mode: groovy -*-
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// nnvm libraries
+vta_lib += "lib/libvta.so, lib/libvta.so.json"
+vta_lib += ", nnvm/tvm/lib/libtvm.so, nnvm/tvm/lib/libtopi.so, nnvm/lib/libnnvm_compiler.so"
+
+
+// command to start a docker container
+docker_run = 'tests/ci_build/ci_build.sh'
+// timeout in minutes
+max_time = 60
+
+// initialize source codes
+def init_git() {
+  checkout scm
+  retry(5) {
+    timeout(time: 2, unit: 'MINUTES') {
+      sh 'git submodule update --init --recursive'
+    }
+  }
+}
+
+def init_git_win() {
+    checkout scm
+    retry(5) {
+        timeout(time: 2, unit: 'MINUTES') {
+            bat 'git submodule update --init --recursive'
+        }
+    }
+}
+
+stage("Sanity Check") {
+  timeout(time: max_time, unit: 'MINUTES') {
+    node('linux') {
+      ws('workspace/vta/sanity') {
+        init_git()
+        sh "${docker_run} lint  ./tests/scripts/task_lint.sh"
+      }
+    }
+  }
+}
+
+// Run make. First try to do an incremental make from a previous workspace in hope to
+// accelerate the compilation. If something wrong, clean the workspace and then
+// build from scratch.
+def make(docker_type, make_flag) {
+  timeout(time: max_time, unit: 'MINUTES') {
+    sh "${docker_run} ${docker_type} cp make/sim_sample.json config.json"
+    try {
+      sh "${docker_run} ${docker_type} ./tests/scripts/task_build.sh ${make_flag}"
+    } catch (exc) {
+      echo 'Incremental compilation failed. Fall back to build from scratch'
+      sh "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh"
+      sh "${docker_run} ${docker_type} ./tests/scripts/task_build.sh ${make_flag}"
+    }
+  }
+}
+
+// pack libraries for later use
+def pack_lib(name, libs) {
+  sh """
+     echo "Packing ${libs} into ${name}"
+     echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+     """
+  stash includes: libs, name: name
+}
+
+
+// unpack libraries saved before
+def unpack_lib(name, libs) {
+  unstash name
+  sh """
+     echo "Unpacked ${libs} from ${name}"
+     echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+     """
+}
+
+stage('Build') {
+  timeout(time: max_time, unit: 'MINUTES') {
+    node('linux') {
+      ws('workspace/vta/build') {
+        init_git()
+        make('cpu', '-j2')
+        pack_lib('cpu', vta_lib)
+      }
+    }
+  }
+}
+
+stage('Tests') {
+  parallel 'python': {
+    node('linux') {
+      ws('workspace/vta/it-python') {
+        init_git()
+        unpack_lib('cpu', vta_lib)
+        timeout(time: max_time, unit: 'MINUTES') {
+          sh "${docker_run} cpu ./tests/scripts/task_python_test.sh"
+        }
+      }
+    }
+  },
+  'docs': {
+    node('linux') {
+      ws('workspace/vta/docs-python') {
+        init_git()
+        unpack_lib('cpu', vta_lib)
+        timeout(time: max_time, unit: 'MINUTES') {
+          sh "${docker_run} cpu ./tests/scripts/task_python_docs.sh"
+        }
+        pack_lib('mydocs', 'docs.tgz')
+      }
+    }
+  }
+}
+
+stage('Deploy') {
+    node('docker' && 'doc') {
+      ws('workspace/vta/deploy-docs') {
+        if (env.BRANCH_NAME == "master") {
+           unpack_lib('mydocs', 'docs.tgz')
+           sh "tar xf docs.tgz -C /var/vta-docs"
+        }
+      }
+    }
+}

docs/conf.py

@@ -16,6 +16,7 @@
 import shlex
 import recommonmark
 import sphinx_gallery
+from tvm.contrib import rpc, graph_runtime
 from recommonmark.parser import CommonMarkParser
 from recommonmark.transform import AutoStructify
 

include/vta/driver.h

@@ -1,6 +1,6 @@
 /*!
  *  Copyright (c) 2018 by Contributors
- * \file vta_driver.h
+ * \file driver.h
  * \brief Driver interface that is used by runtime.
  *
  * Driver's implementation is device specific.
@@ -46,7 +46,7 @@ void VTADeviceFree(VTADeviceHandle handle);
 
 /*!
  * \brief Launch the instructions block until done.
- * \param The device handle.
+ * \param device The device handle.
  * \param insn_phy_addr The physical address of instruction stream.
  * \param insn_count Instruction count.
  * \param wait_cycles The maximum of cycles to wait

include/vta/hw_spec.h

@@ -1,6 +1,6 @@
 /*!
  *  Copyright (c) 2018 by Contributors
- * \file vta_defines.h
+ * \file hw_spec.h
  * \brief Preprocessor definitions for VTA HLS design and runtime.
  */
 

include/vta/runtime.h

@@ -25,22 +25,19 @@ extern "C" {
 
 /*!
  * \brief Allocate data buffer.
- * \param cmd The VTA command handle.
  * \param size Buffer size.
  * \return A pointer to the allocated buffer.
  */
 void* VTABufferAlloc(size_t size);
 
 /*!
  * \brief Free data buffer.
- * \param cmd The VTA command handle.
  * \param buffer The data buffer to be freed.
  */
 void VTABufferFree(void* buffer);
 
 /*!
  * \brief Copy data buffer from one location to another.
- * \param cmd The VTA command handle.
  * \param from The source buffer base address.
  * \param from_offset The offset of the source buffer.
  * \param to The target buffer base address.
@@ -145,6 +142,7 @@ void VTALoadBuffer2D(VTACommandHandle cmd,
  * \param src_sram_index Source SRAM index.
  * \param src_memory_type Source memory type.
  * \param dst_dram_addr Destination DRAM address.
+ * \param dst_elem_offset The destination DRAM offset in number of unit elements.
  * \param x_size The lowest dimension (x axis) size in number of unit elements.
  * \param y_size The number of rows.
  * \param x_stride The x axis stride.

make/sim_sample.json

@@ -0,0 +1,14 @@
+{
+  "TARGET" : "sim",
+  "LOG_INP_WIDTH" : 3,
+  "LOG_WGT_WIDTH" : 3,
+  "LOG_ACC_WIDTH" : 5,
+  "LOG_OUT_WIDTH" : 3,
+  "LOG_BATCH" : 0,
+  "LOG_BLOCK_IN" : 4,
+  "LOG_BLOCK_OUT" : 4,
+  "LOG_UOP_BUFF_SIZE" : 15,
+  "LOG_INP_BUFF_SIZE" : 15,
+  "LOG_WGT_BUFF_SIZE" : 15,
+  "LOG_ACC_BUFF_SIZE" : 17
+}

python/vta/__init__.py

@@ -1,6 +1,8 @@
 """TVM-based VTA Compiler Toolchain"""
 from __future__ import absolute_import as _abs
 
+__version__ = "0.1.0"
+
 
 from .environment import get_env, Environment
 from .rpc_client import reconfig_runtime, program_fpga

tests/ci_build/Dockerfile.cpu

@@ -0,0 +1,26 @@
+# For CPU
+FROM ubuntu:16.04
+
+RUN apt-get update --fix-missing
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh
+RUN bash /install/ubuntu_install_llvm.sh
+
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+
+COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
+RUN bash /install/ubuntu_install_python_package.sh
+
+COPY install/ubuntu_install_sphinx.sh /install/ubuntu_install_sphinx.sh
+RUN bash /install/ubuntu_install_sphinx.sh
+
+# Fix recommonmark to latest version
+RUN git clone https://github.com/rtfd/recommonmark
+RUN cd recommonmark; python setup.py install
+
+# Enable doxygen for c++ doc build
+RUN apt-get update && apt-get install -y doxygen graphviz

tests/ci_build/Dockerfile.lint

@@ -0,0 +1,6 @@
+# For lint test
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y python-pip sudo
+RUN apt-get install -y doxygen graphviz
+RUN pip install cpplint pylint

tests/ci_build/README.md

@@ -0,0 +1,35 @@
+# CI Build Scripts
+
+This directory contains the files and setup instructions to run all tests.
+
+## Run locally
+
+To run locally, we need to first install
+[docker](https://docs.docker.com/engine/installation/)
+
+Then we can run the tasks defined in the [Jenkinsfile](../../Jenkinsfile) by
+using (`ci_build.sh`)[./ci_build.sh]. For example
+
+- lint the python codes
+
+  ```bash
+  ./ci_build.sh lint make pylint
+  ```
+
+- build codes with CUDA supports
+
+  ```bash
+  ./ci_build.sh gpu tests/scripts/task_build.sh
+  ```
+
+- do the python unittest
+
+  ```bash
+  ./ci_build.sh gpu tests/scripts/task_python_test.sh
+  ```
+
+- build the documents. The results will be available at `docs/_build/html`
+
+  ```bash
+  tests/ci_build/ci_build.sh gpu tests/scripts/task_python_docs.sh
+  ```