Skip to content

Commit 938e43e

Browse files
tqchenwweic
tqchen
authored and
wweic
committed
[DOCKER] Update docker protocol (apache#2793)
1 parent 61e6ce1 commit 938e43e

File tree

101 files changed

+207
-35
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

101 files changed

+207
-35
lines changed

‎Jenkinsfile‎

Lines changed: 42 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,28 @@
33
// Jenkins pipeline
44
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
55

6+
// Docker env used for testing
7+
// Different image may have different version tag
8+
// because some of them are more stable than anoter.
9+
//
10+
// Docker images are maintained by PMC, cached in dockerhub
11+
// and remains relatively stable over the time.
12+
// Flow for upgrading docker env(need commiter)
13+
//
14+
// - Send PR to upgrade build script in the repo
15+
// - Build the new docker image
16+
// - Tag the docker image with a new version and push to tvmai
17+
// - Update the version in the Jenkinsfile, send a PR
18+
// - Fix any issues wrt to the new image version in the PR
19+
// - Merge the PR and now we are in new version
20+
// - Tag the new version as the lates
21+
// - Periodically cleanup the old versions on local workers
22+
//
23+
ci_lint = "tvmai/ci-lint:v0.50"
24+
ci_gpu = "tvmai/ci-gpu:v0.50"
25+
ci_cpu = "tvmai/ci-cpu:v0.41"
26+
ci_i386 = "tvmai/ci-i386:v0.50"
27+
628
// tvm libraries
729
tvm_runtime = "build/libtvm_runtime.so, build/config.cmake"
830
tvm_lib = "build/libtvm.so, " + tvm_runtime
@@ -39,7 +61,7 @@ stage("Sanity Check") {
3961
node('CPU') {
4062
ws('workspace/tvm/sanity') {
4163
init_git()
42-
sh "${docker_run} tvmai/ci-lint ./tests/scripts/task_lint.sh"
64+
sh "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh"
4365
}
4466
}
4567
}
@@ -102,7 +124,7 @@ stage('Build') {
102124
echo set\\(CMAKE_CXX_COMPILER g++\\) >> config.cmake
103125
echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake
104126
"""
105-
make('tvmai/ci-gpu', 'build', '-j4')
127+
make(ci_gpu, 'build', '-j4')
106128
pack_lib('gpu', tvm_multilib)
107129
// compiler test
108130
sh """
@@ -116,7 +138,7 @@ stage('Build') {
116138
echo set\\(CMAKE_CXX_COMPILER clang-6.0\\) >> config.cmake
117139
echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake
118140
"""
119-
make('tvmai/ci-gpu', 'build2', '-j4')
141+
make(ci_gpu, 'build2', '-j4')
120142
}
121143
}
122144
},
@@ -136,14 +158,15 @@ stage('Build') {
136158
echo set\\(CMAKE_CXX_COMPILER g++\\) >> config.cmake
137159
echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake
138160
"""
139-
make('tvmai/ci-cpu', 'build', '-j4')
161+
make(ci_cpu, 'build', '-j4')
140162
pack_lib('cpu', tvm_lib)
141163
timeout(time: max_time, unit: 'MINUTES') {
142-
sh "${docker_run} tvmai/ci-cpu ./tests/scripts/task_cpp_unittest.sh"
143-
sh "${docker_run} tvmai/ci-cpu ./tests/scripts/task_python_vta.sh"
144-
sh "${docker_run} tvmai/ci-cpu ./tests/scripts/task_golang.sh"
145-
sh "${docker_run} tvmai/ci-cpu ./tests/scripts/task_python_unittest.sh"
146-
sh "${docker_run} tvmai/ci-cpu ./tests/scripts/task_python_integration.sh"
164+
sh "${docker_run} ${ci_cpu} ./tests/scripts/task_cpp_unittest.sh"
165+
sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta.sh"
166+
sh "${docker_run} ${ci_cpu} ./tests/scripts/task_rust.sh"
167+
sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh"
168+
sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_unittest.sh"
169+
sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh"
147170
}
148171
}
149172
}
@@ -163,7 +186,7 @@ stage('Build') {
163186
echo set\\(CMAKE_CXX_COMPILER g++\\) >> config.cmake
164187
echo set\\(CMAKE_CXX_FLAGS -Werror\\) >> config.cmake
165188
"""
166-
make('tvmai/ci-i386', 'build', '-j4')
189+
make(ci_i386, 'build', '-j4')
167190
pack_lib('i386', tvm_multilib)
168191
}
169192
}
@@ -177,8 +200,8 @@ stage('Unit Test') {
177200
init_git()
178201
unpack_lib('gpu', tvm_multilib)
179202
timeout(time: max_time, unit: 'MINUTES') {
180-
sh "${docker_run} tvmai/ci-gpu ./tests/scripts/task_python_unittest.sh"
181-
sh "${docker_run} tvmai/ci-gpu ./tests/scripts/task_python_integration.sh"
203+
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest.sh"
204+
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration.sh"
182205
}
183206
}
184207
}
@@ -189,9 +212,9 @@ stage('Unit Test') {
189212
init_git()
190213
unpack_lib('i386', tvm_multilib)
191214
timeout(time: max_time, unit: 'MINUTES') {
192-
sh "${docker_run} tvmai/ci-i386 ./tests/scripts/task_python_unittest.sh"
193-
sh "${docker_run} tvmai/ci-i386 ./tests/scripts/task_python_integration.sh"
194-
sh "${docker_run} tvmai/ci-i386 ./tests/scripts/task_python_vta.sh"
215+
sh "${docker_run} ${ci_i386} ./tests/scripts/task_python_unittest.sh"
216+
sh "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration.sh"
217+
sh "${docker_run} ${ci_i386} ./tests/scripts/task_python_vta.sh"
195218
}
196219
}
197220
}
@@ -202,7 +225,7 @@ stage('Unit Test') {
202225
init_git()
203226
unpack_lib('gpu', tvm_multilib)
204227
timeout(time: max_time, unit: 'MINUTES') {
205-
sh "${docker_run} tvmai/ci-gpu ./tests/scripts/task_java_unittest.sh"
228+
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh"
206229
}
207230
}
208231
}
@@ -216,7 +239,7 @@ stage('Integration Test') {
216239
init_git()
217240
unpack_lib('gpu', tvm_multilib)
218241
timeout(time: max_time, unit: 'MINUTES') {
219-
sh "${docker_run} tvmai/ci-gpu ./tests/scripts/task_python_topi.sh"
242+
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh"
220243
}
221244
}
222245
}
@@ -227,7 +250,7 @@ stage('Integration Test') {
227250
init_git()
228251
unpack_lib('gpu', tvm_multilib)
229252
timeout(time: max_time, unit: 'MINUTES') {
230-
sh "${docker_run} tvmai/ci-gpu ./tests/scripts/task_python_frontend.sh"
253+
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh"
231254
}
232255
}
233256
}
@@ -238,7 +261,7 @@ stage('Integration Test') {
238261
init_git()
239262
unpack_lib('gpu', tvm_multilib)
240263
timeout(time: max_time, unit: 'MINUTES') {
241-
sh "${docker_run} tvmai/ci-gpu ./tests/scripts/task_python_docs.sh"
264+
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh"
242265
}
243266
pack_lib('mydocs', 'docs.tgz')
244267
}

‎docker/Dockerfile.ci_cpu‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# CI docker CPU env
2+
# tag: v0.41
23
FROM ubuntu:16.04
34

45
RUN apt-get update --fix-missing

‎docker/Dockerfile.ci_gpu‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# CI docker GPU env
2-
# tag: v0.41
2+
# tag: v0.50
33
FROM nvidia/cuda:8.0-cudnn7-devel
44

55
# Base scripts

‎docker/Dockerfile.ci_i386‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# CI docker i386 env
2+
# tag: v0.50
23

34
FROM ioft/i386-ubuntu:16.04
45

‎docker/bash.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ else
3333
DOCKER_BINARY="docker"
3434
fi
3535

36+
if [[ ! -z $CUDA_VISIBLE_DEVICES ]]; then
37+
CUDA_ENV="-e CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}"
38+
else
39+
CUDA_ENV=""
40+
fi
41+
3642
# Print arguments.
3743
echo "WORKSPACE: ${WORKSPACE}"
3844
echo "DOCKER CONTAINER NAME: ${DOCKER_IMAGE_NAME}"
@@ -53,6 +59,8 @@ ${DOCKER_BINARY} run --rm --pid=host\
5359
-e "CI_BUILD_UID=$(id -u)" \
5460
-e "CI_BUILD_GROUP=$(id -g -n)" \
5561
-e "CI_BUILD_GID=$(id -g)" \
62+
-e "PYTHONPATH=python:topi/python"\
63+
${CUDA_ENV}\
5664
${CI_DOCKER_EXTRA_PARAMS[@]} \
5765
${DOCKER_IMAGE_NAME}\
5866
bash --login /docker/with_the_same_user \

‎docker/install/ubuntu_install_onnx.sh‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,7 @@ set -u
55
set -o pipefail
66

77
# fix to certain version for now
8-
pip2 install onnx>=1.1.0
98
pip3 install onnx>=1.1.0
109

11-
pip2 install http://download.pytorch.org/whl/cu75/torch-0.2.0.post3-cp27-cp27mu-manylinux1_x86_64.whl
12-
pip2 install torchvision
13-
pip3 install http://download.pytorch.org/whl/cu75/torch-0.2.0.post3-cp35-cp35m-manylinux1_x86_64.whl
10+
pip3 install https://download.pytorch.org/whl/cu80/torch-1.0.1.post2-cp36-cp36m-linux_x86_64.whl
1411
pip3 install torchvision

‎docker/with_the_same_user‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,15 @@ getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid "${CI_B
2626
usermod -a -G sudo "${CI_BUILD_USER}"
2727
echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo
2828

29+
if [[ ! -z $CUDA_VISIBLE_DEVICES ]]; then
30+
CUDA_ENV="CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}"
31+
else
32+
CUDA_ENV=""
33+
fi
34+
2935
HOME=${CI_BUILD_HOME}\
3036
sudo -u "#${CI_BUILD_UID}" --preserve-env\
37+
${CUDA_ENV}\
3138
PATH=${PATH}\
3239
JAVA_HOME=${JAVA_HOME}\
3340
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}\

‎docs/contribute/pull_request.rst‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ This is a quick guide to submit a pull request, please also refer to the detaile
1212
git rebase upstream/master
1313
1414
- Make sure code style check pass by typing the following command, and all the existing test-cases pass.
15-
- ``docker/bash.sh tvmai/ci-lint ./tests/scripts/task_lint.sh``
15+
- ``docker/bash.sh tvmai/ci-lint ./tests/scripts/task_lint.sh``
1616
(Note: You must install docker beforehand so you can run a docker image.)
1717
- Add test-cases to cover the new features or bugfix the patch introduces.
1818
- Document the code you wrote, see more at :ref:`doc_guide`
@@ -27,6 +27,28 @@ This is a quick guide to submit a pull request, please also refer to the detaile
2727

2828
- The patch can be merged after the reviewers approve the pull request.
2929

30+
31+
CI Environment
32+
--------------
33+
We use docker container to create stable CI environments
34+
that can be deployed to multiple machines.
35+
You can find the prebuilt images in `<https://hub.docker.com/r/tvmai/>`_ .
36+
Because we want a relatively stable CI environment and make use of pre-cached image,
37+
all of the CI images are built and maintained by comitters.
38+
39+
Upgrade of CI images can cause problems and need fixes to accommodate the new env.
40+
Here is the protocol to update CI image:
41+
42+
- Send PR to upgrade build script in the repo
43+
- Can be done by a contributor, the following steps need committership.
44+
- Build the new docker image
45+
- Tag the docker image with a new version and push to tvmai
46+
- Update the version(most of the time increase the minor version) in the Jenkinsfile, send a PR.
47+
- Fix any issues wrt to the new image versions in the PR.
48+
- Merge the PR and now we are in new version.
49+
- Tag the new version as the latest.
50+
- Periodically cleanup the old versions on local workers
51+
3052
Testing
3153
-------
3254
Even though we have hooks to run unit tests automatically for each pull request, It's always recommended to run unit tests

‎include/tvm/api_registry.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#define TVM_API_REGISTRY_H_
99

1010
#include <string>
11+
#include <utility>
1112
#include "base.h"
1213
#include "packed_func_ext.h"
1314
#include "runtime/registry.h"

‎include/tvm/attrs.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include <functional>
3434
#include <type_traits>
3535
#include <string>
36+
#include <utility>
3637
#include "ir.h"
3738
#include "base.h"
3839
#include "expr.h"

0 commit comments

Comments
 (0)