-
Notifications
You must be signed in to change notification settings - Fork 1.1k
162 lines (154 loc) · 6.03 KB
/
integration.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
# manually trigger integration with the latest pytorch
name: integration
on:
repository_dispatch:
type: [integration-test-command]
jobs:
integration-auto3dseg:
container:
image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38
options: --gpus "device=1" --ipc host # shm-size 4g works fine
runs-on: [self-hosted, linux, x64, command]
steps:
# checkout the pull request branch
- uses: actions/checkout@v4
with:
token: ${{ secrets.PR_MAINTAIN }}
repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
ref: ${{ github.event.client_payload.pull_request.head.ref }}
- name: cache weekly timestamp
id: pip-cache
run: echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT
- name: cache for pip
uses: actions/cache@v4
id: cache
with:
path: |
~/.cache/pip
~/.cache/torch
key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }}
- name: Install the dependencies
run: |
pwd && git log -1 && which python
python -m pip install --upgrade pip wheel
pip uninstall -y monai
pip uninstall -y monai
pip uninstall -y monai-weekly
pip uninstall -y monai-weekly
python -m pip install --upgrade torch torchvision torchaudio torchtext
python -m pip install -r requirements-dev.txt
rm -rf /github/home/.cache/torch/hub/mmars/
- name: Clean directory
run: |
python -m pip list
git config --global --add safe.directory /__w/MONAI/MONAI
git clean -ffdx && git reset --hard HEAD
nvidia-smi
export CUDA_VISIBLE_DEVICES=$(python -m tests.utils -c 1 | tail -n 1)
echo $CUDA_VISIBLE_DEVICES
python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
- name: Auto3dseg tag algo
shell: bash
env:
BUILD_MONAI: 0
run: |
pwd && git log -1 && which python
./runtests.sh -b
export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
python -m tests.test_auto3dseg_bundlegen
python -m tests.test_auto3dseg_ensemble
python -m tests.test_auto3dseg_hpo
python -m tests.test_integration_autorunner
python -m tests.test_integration_gpu_customization
- name: Integration tests
shell: bash
env:
BUILD_MONAI: 1
NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
NGC_ORG: ${{ secrets.NGC_ORG }}
NGC_TEAM: ${{ secrets.NGC_TEAM }}
run: ./runtests.sh --build --net
- name: Add reaction
uses: peter-evans/create-or-update-comment@v4
with:
token: ${{ secrets.PR_MAINTAIN }}
repository: ${{ github.event.client_payload.github.payload.repository.full_name }}
comment-id: ${{ github.event.client_payload.github.payload.comment.id }}
reactions: rocket
integration-unit:
container:
image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38
options: --gpus "device=2" --ipc host # shm-size 4g works fine
runs-on: [self-hosted, linux, x64, command1]
steps:
# checkout the pull request branch
- uses: actions/checkout@v4
with:
token: ${{ secrets.PR_MAINTAIN }}
repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
ref: ${{ github.event.client_payload.pull_request.head.ref }}
- name: cache weekly timestamp
id: pip-cache
run: echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT
- name: cache for pip
uses: actions/cache@v4
id: cache
with:
path: |
~/.cache/pip
~/.cache/torch
key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }}
- name: Install the dependencies
run: |
pwd && git log -1 && which python
python -m pip install --upgrade pip wheel
pip uninstall -y monai
pip uninstall -y monai
pip uninstall -y monai-weekly
pip uninstall -y monai-weekly
python -m pip install --upgrade torch torchvision torchaudio torchtext
python -m pip install -r requirements-dev.txt
rm -rf /github/home/.cache/torch/hub/mmars/
- name: Clean directory
run: |
python -m pip list
git config --global --add safe.directory /__w/MONAI/MONAI
git clean -ffdx && git reset --hard HEAD
nvidia-smi
export CUDA_VISIBLE_DEVICES=$(python -m tests.utils -c 1 | tail -n 1)
echo $CUDA_VISIBLE_DEVICES
python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
- name: Auto3dseg latest algo
shell: bash
env:
BUILD_MONAI: 0
run: |
pwd
cd ../
rm -rf research-contributions
rm -rf algorithm_templates
git clone --depth 1 --branch main --single-branch https://github.com/Project-MONAI/research-contributions.git
ls research-contributions/
cp -r research-contributions/auto3dseg/algorithm_templates MONAI/
cd research-contributions && git log -1 && cd ../MONAI
pwd
ls -ll
export OMP_NUM_THREADS=4
export MKL_NUM_THREADS=4
export MONAI_TESTING_ALGO_TEMPLATE=algorithm_templates
pwd && git log -1 && which python
./runtests.sh -b
export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
python -m tests.test_auto3dseg_ensemble
python -m tests.test_auto3dseg_hpo
python -m tests.test_integration_autorunner
python -m tests.test_integration_gpu_customization
- name: Add reaction
uses: peter-evans/create-or-update-comment@v4
with:
token: ${{ secrets.PR_MAINTAIN }}
repository: ${{ github.event.client_payload.github.payload.repository.full_name }}
comment-id: ${{ github.event.client_payload.github.payload.comment.id }}
reactions: +1