Skip to content

Commit 332fa02

Browse files
ydshiehydshieh
authored
Security fix for self-comment-ci.yml (#35548)
* Revert "Disable `.github/workflows/self-comment-ci.yml` for now (#35366)" This reverts commit ccc4a5a. * fix * fix * fix * least permission * add env --------- Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
1 parent 8571bb1 commit 332fa02

File tree

1 file changed

+289
-0
lines changed

1 file changed

+289
-0
lines changed

‎.github/workflows/self-comment-ci.yml‎

Lines changed: 289 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
name: PR comment GitHub CI
2+
3+
on:
4+
issue_comment:
5+
types:
6+
- created
7+
branches-ignore:
8+
- main
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.event.issue.number }}-${{ startsWith(github.event.comment.body, 'run-slow') || startsWith(github.event.comment.body, 'run slow') || startsWith(github.event.comment.body, 'run_slow') }}
11+
cancel-in-progress: true
12+
permissions: read-all
13+
14+
env:
15+
HF_HOME: /mnt/cache
16+
TRANSFORMERS_IS_CI: yes
17+
OMP_NUM_THREADS: 8
18+
MKL_NUM_THREADS: 8
19+
RUN_SLOW: yes
20+
# For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access.
21+
# This token is created under the bot `hf-transformers-bot`.
22+
HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
23+
SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
24+
TF_FORCE_GPU_ALLOW_GROWTH: true
25+
RUN_PT_TF_CROSS_TESTS: 1
26+
CUDA_VISIBLE_DEVICES: 0,1
27+
28+
jobs:
29+
get-pr-number:
30+
runs-on: ubuntu-22.04
31+
name: Get PR number
32+
# For security: only allow team members to run
33+
if: ${{ github.event.issue.state == 'open' && contains(fromJSON('["ydshieh", "ArthurZucker", "zucchini-nlp", "qubvel", "molbap", "gante", "LysandreJik", "Cyrilvallez"]'), github.actor) && (startsWith(github.event.comment.body, 'run-slow') || startsWith(github.event.comment.body, 'run slow') || startsWith(github.event.comment.body, 'run_slow')) }}
34+
outputs:
35+
PR_NUMBER: ${{ steps.set_pr_number.outputs.PR_NUMBER }}
36+
steps:
37+
- name: Get PR number
38+
shell: bash
39+
run: |
40+
if [[ "${{ github.event.issue.number }}" != "" && "${{ github.event.issue.pull_request }}" != "" ]]; then
41+
echo "PR_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
42+
else
43+
echo "PR_NUMBER=" >> $GITHUB_ENV
44+
fi
45+
46+
- name: Check PR number
47+
shell: bash
48+
run: |
49+
echo "${{ env.PR_NUMBER }}"
50+
51+
- name: Set PR number
52+
id: set_pr_number
53+
run: echo "PR_NUMBER=${{ env.PR_NUMBER }}" >> "$GITHUB_OUTPUT"
54+
55+
get-sha:
56+
runs-on: ubuntu-22.04
57+
needs: get-pr-number
58+
if: ${{ needs.get-pr-number.outputs.PR_NUMBER != ''}}
59+
outputs:
60+
PR_HEAD_SHA: ${{ steps.get_sha.outputs.PR_HEAD_SHA }}
61+
steps:
62+
- uses: actions/checkout@v4
63+
with:
64+
fetch-depth: "0"
65+
ref: "refs/pull/${{needs.get-pr-number.outputs.PR_NUMBER}}/merge"
66+
67+
- name: Get SHA (and verify timestamps against the issue comment date)
68+
id: get_sha
69+
env:
70+
PR_NUMBER: ${{ needs.get-pr-number.outputs.PR_NUMBER }}
71+
COMMENT_DATE: ${{ github.event.comment.created_at }}
72+
run: |
73+
git fetch origin refs/pull/$PR_NUMBER/head:refs/remotes/pull/$PR_NUMBER/head
74+
git checkout refs/remotes/pull/$PR_NUMBER/head
75+
echo "PR_HEAD_SHA: $(git log -1 --format=%H)"
76+
echo "PR_HEAD_SHA=$(git log -1 --format=%H)" >> "$GITHUB_OUTPUT"
77+
git fetch origin refs/pull/$PR_NUMBER/merge:refs/remotes/pull/$PR_NUMBER/merge
78+
git checkout refs/remotes/pull/$PR_NUMBER/merge
79+
PR_MERGE_COMMIT_TIMESTAMP=$(git log -1 --date=unix --format=%cd)
80+
echo "PR_MERGE_COMMIT_TIMESTAMP: $PR_MERGE_COMMIT_TIMESTAMP"
81+
COMMENT_TIMESTAMP=$(date -d "${COMMENT_DATE}" +"%s")
82+
echo "PR_HEAD_SHA: $COMMENT_DATE"
83+
echo "COMMENT_TIMESTAMP: $COMMENT_TIMESTAMP"
84+
if [ $COMMENT_TIMESTAMP -le $PR_MERGE_COMMIT_TIMESTAMP ]; then
85+
echo "Last commit on the pull request is newer than the issue comment triggering this run! Abort!";
86+
exit -1;
87+
fi
88+
89+
# use a python script to handle this complex logic
90+
# case 1: `run-slow` (auto. infer with limited number of models, but in particular, new model)
91+
# case 2: `run-slow model_1, model_2`
92+
get-tests:
93+
runs-on: ubuntu-22.04
94+
needs: get-pr-number
95+
if: ${{ needs.get-pr-number.outputs.PR_NUMBER != ''}}
96+
outputs:
97+
models: ${{ steps.models_to_run.outputs.models }}
98+
steps:
99+
- uses: actions/checkout@v4
100+
with:
101+
fetch-depth: "0"
102+
ref: "refs/pull/${{needs.get-pr-number.outputs.PR_NUMBER}}/merge"
103+
104+
- name: Get models to test
105+
env:
106+
PR_COMMENT: ${{ github.event.comment.body }}
107+
run: |
108+
python -m pip install GitPython
109+
python utils/pr_slow_ci_models.py --message "$PR_COMMENT" | tee output.txt
110+
echo "models=$(tail -n 1 output.txt)" >> $GITHUB_ENV
111+
112+
- name: Show models to test
113+
id: models_to_run
114+
run: |
115+
echo "${{ env.models }}"
116+
echo "models=${{ env.models }}" >> $GITHUB_ENV
117+
echo "models=${{ env.models }}" >> $GITHUB_OUTPUT
118+
119+
reply_to_comment:
120+
name: Reply to the comment
121+
if: ${{ needs.get-tests.outputs.models != '[]' }}
122+
needs: [get-pr-number, get-tests]
123+
permissions:
124+
pull-requests: write
125+
runs-on: ubuntu-22.04
126+
steps:
127+
- name: Reply to the comment
128+
env:
129+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
130+
MODELS: ${{ needs.get-tests.outputs.models }}
131+
run: |
132+
gh api \
133+
--method POST \
134+
-H "Accept: application/vnd.github+json" \
135+
-H "X-GitHub-Api-Version: 2022-11-28" \
136+
repos/${{ github.repository }}/issues/${{ needs.get-pr-number.outputs.PR_NUMBER }}/comments \
137+
-f "body=This comment contains run-slow, running the specified jobs: ${{ env.MODELS }} ..."
138+
139+
create_run:
140+
name: Create run
141+
if: ${{ needs.get-tests.outputs.models != '[]' }}
142+
needs: [get-sha, get-tests, reply_to_comment]
143+
permissions:
144+
statuses: write
145+
runs-on: ubuntu-22.04
146+
steps:
147+
- name: Create Run
148+
id: create_run
149+
env:
150+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
151+
# Create a commit status (pending) for a run of this workflow. The status has to be updated later in `update_run_status`.
152+
# See https://docs.github.com/en/rest/commits/statuses?apiVersion=2022-11-28#create-a-commit-status
153+
GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
154+
run: |
155+
gh api \
156+
--method POST \
157+
-H "Accept: application/vnd.github+json" \
158+
-H "X-GitHub-Api-Version: 2022-11-28" \
159+
repos/${{ github.repository }}/statuses/${{ needs.get-sha.outputs.PR_HEAD_SHA }} \
160+
-f "target_url=$GITHUB_RUN_URL" -f "state=pending" -f "description=Slow CI job" -f "context=pytest/custom-tests"
161+
162+
run_models_gpu:
163+
name: Run all tests for the model
164+
if: ${{ needs.get-tests.outputs.models != '[]' }}
165+
needs: [get-pr-number, get-tests, create_run]
166+
strategy:
167+
fail-fast: false
168+
matrix:
169+
folders: ${{ fromJson(needs.get-tests.outputs.models) }}
170+
machine_type: [aws-g4dn-2xlarge-cache, aws-g4dn-12xlarge-cache]
171+
runs-on:
172+
group: '${{ matrix.machine_type }}'
173+
container:
174+
image: huggingface/transformers-all-latest-gpu
175+
options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
176+
steps:
177+
- name: Echo input and matrix info
178+
shell: bash
179+
run: |
180+
echo "${{ matrix.folders }}"
181+
182+
- name: Echo folder ${{ matrix.folders }}
183+
shell: bash
184+
# For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
185+
# set the artifact folder names (because the character `/` is not allowed).
186+
run: |
187+
echo "${{ matrix.folders }}"
188+
matrix_folders=${{ matrix.folders }}
189+
matrix_folders=${matrix_folders/'models/'/'models_'}
190+
echo "$matrix_folders"
191+
echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV
192+
193+
- name: Checkout to PR merge commit
194+
working-directory: /transformers
195+
run: |
196+
git fetch origin refs/pull/${{ needs.get-pr-number.outputs.PR_NUMBER }}/merge:refs/remotes/pull/${{ needs.get-pr-number.outputs.PR_NUMBER }}/merge
197+
git checkout refs/remotes/pull/${{ needs.get-pr-number.outputs.PR_NUMBER }}/merge
198+
git log -1 --format=%H
199+
200+
- name: Reinstall transformers in edit mode (remove the one installed during docker image build)
201+
working-directory: /transformers
202+
run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .
203+
204+
- name: NVIDIA-SMI
205+
run: |
206+
nvidia-smi
207+
208+
- name: Set `machine_type` for report and artifact names
209+
working-directory: /transformers
210+
shell: bash
211+
run: |
212+
echo "${{ matrix.machine_type }}"
213+
if [ "${{ matrix.machine_type }}" = "aws-g4dn-2xlarge-cache" ]; then
214+
machine_type=single-gpu
215+
elif [ "${{ matrix.machine_type }}" = "aws-g4dn-12xlarge-cache" ]; then
216+
machine_type=multi-gpu
217+
else
218+
machine_type=${{ matrix.machine_type }}
219+
fi
220+
echo "$machine_type"
221+
echo "machine_type=$machine_type" >> $GITHUB_ENV
222+
223+
- name: Environment
224+
working-directory: /transformers
225+
run: |
226+
python3 utils/print_env.py
227+
228+
- name: Show installed libraries and their versions
229+
working-directory: /transformers
230+
run: pip freeze
231+
232+
- name: Run all tests on GPU
233+
working-directory: /transformers
234+
run: |
235+
export CUDA_VISIBLE_DEVICES="$(python3 utils/set_cuda_devices_for_ci.py --test_folder ${{ matrix.folders }})"
236+
echo $CUDA_VISIBLE_DEVICES
237+
python3 -m pytest -v -rsfE --make-reports=${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
238+
239+
- name: Failure short reports
240+
if: ${{ failure() }}
241+
continue-on-error: true
242+
run: cat /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
243+
244+
- name: Make sure report directory exists
245+
shell: bash
246+
run: |
247+
mkdir -p /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
248+
echo "hello" > /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
249+
echo "${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"
250+
251+
- name: "Test suite reports artifacts: ${{ env.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
252+
if: ${{ always() }}
253+
uses: actions/upload-artifact@v4
254+
with:
255+
name: ${{ env.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
256+
path: /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
257+
258+
update_run_status:
259+
name: Update Check Run Status
260+
needs: [get-sha, create_run, run_models_gpu]
261+
permissions:
262+
statuses: write
263+
if: ${{ always() && needs.create_run.result == 'success' }}
264+
runs-on: ubuntu-22.04
265+
env:
266+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
267+
GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
268+
steps:
269+
- name: Get `run_models_gpu` job status
270+
run: |
271+
echo "${{ needs.run_models_gpu.result }}"
272+
if [ "${{ needs.run_models_gpu.result }}" = "cancelled" ]; then
273+
echo "STATUS=failure" >> $GITHUB_ENV
274+
elif [ "${{ needs.run_models_gpu.result }}" = "skipped" ]; then
275+
echo "STATUS=success" >> $GITHUB_ENV
276+
else
277+
echo "STATUS=${{ needs.run_models_gpu.result }}" >> $GITHUB_ENV
278+
fi
279+
280+
- name: Update PR commit statuses
281+
run: |
282+
echo "${{ needs.run_models_gpu.result }}"
283+
echo "${{ env.STATUS }}"
284+
gh api \
285+
--method POST \
286+
-H "Accept: application/vnd.github+json" \
287+
-H "X-GitHub-Api-Version: 2022-11-28" \
288+
repos/${{ github.repository }}/statuses/${{ needs.get-sha.outputs.PR_HEAD_SHA }} \
289+
-f "target_url=$GITHUB_RUN_URL" -f "state=${{ env.STATUS }}" -f "description=Slow CI job" -f "context=pytest/custom-tests"

0 commit comments

Comments
 (0)