Skip to content

Commit 618d297

Browse files
sitaowang1998sitaowang1998
committed
Revert "Temporarily remove other GH workflows"
This reverts commit c95039c.
1 parent b4665a5 commit 618d297

File tree

6 files changed

+618
-0
lines changed

6 files changed

+618
-0
lines changed

‎.github/workflows/clp-core-build.yaml‎

Lines changed: 376 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,376 @@
1+
name: "clp-core-build"
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- ".github/actions/clp-core-build-containers/action.yaml"
7+
- ".github/actions/run-on-image/action.yaml"
8+
- ".github/workflows/clp-core-build.yaml"
9+
- ".gitmodules"
10+
- "components/core/**"
11+
- "taskfile.yaml"
12+
- "taskfiles/**"
13+
- "tools/scripts/deps-download/**"
14+
- "!components/core/tools/scripts/lib_install/macos/**"
15+
push:
16+
paths:
17+
- ".github/actions/clp-core-build-containers/action.yaml"
18+
- ".github/actions/run-on-image/action.yaml"
19+
- ".github/workflows/clp-core-build.yaml"
20+
- ".gitmodules"
21+
- "components/core/**"
22+
- "taskfile.yaml"
23+
- "taskfiles/**"
24+
- "tools/scripts/deps-download/**"
25+
- "!components/core/tools/scripts/lib_install/macos/**"
26+
schedule:
27+
# Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
28+
- cron: "15 0 * * *"
29+
workflow_dispatch:
30+
31+
env:
32+
BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-"
33+
DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-"
34+
35+
concurrency:
36+
group: "${{github.workflow}}-${{github.ref}}"
37+
38+
# Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted
39+
# publishing of container images.
40+
cancel-in-progress: "${{github.ref != 'refs/heads/main'}}"
41+
42+
jobs:
43+
filter-relevant-changes:
44+
name: "filter-relevant-changes"
45+
runs-on: "ubuntu-latest"
46+
outputs:
47+
centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}"
48+
ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}"
49+
clp_changed: "${{steps.filter.outputs.clp}}"
50+
steps:
51+
- uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
52+
with:
53+
submodules: "recursive"
54+
55+
- name: "Work around actions/runner-images/issues/6775"
56+
run: "chown $(id -u):$(id -g) -R ."
57+
shell: "bash"
58+
59+
- name: "Filter relevant changes"
60+
uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36"
61+
id: "filter"
62+
with:
63+
# Consider changes between the current commit and `main`
64+
# NOTE: If a pull request changes one of the images, then we need to (1) build the image
65+
# (based on commits in the PR) and then (2) build CLP using the changed image. If a pull
66+
# request doesn't change an image, then we don't need to rebuild the image; instead we can
67+
# use the published image which is based on `main`. So when determining what files have
68+
# changed, we need to consider the delta between the current commit and `main` (rather
69+
# than the current and previous commits) in order to detect if we need to rebuild the
70+
# image (since it would be different from the published image).
71+
base: "main"
72+
filters: |
73+
centos_stream_9_image:
74+
- ".github/actions/**"
75+
- ".github/workflows/clp-core-build.yaml"
76+
- "components/core/tools/scripts/lib_install/*.sh"
77+
- "components/core/tools/docker-images/clp-env-base-centos-stream-9/**"
78+
- "components/core/tools/scripts/lib_install/centos-stream-9/**"
79+
ubuntu_jammy_image:
80+
- ".github/actions/**"
81+
- ".github/workflows/clp-core-build.yaml"
82+
- "components/core/tools/scripts/lib_install/*.sh"
83+
- "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**"
84+
- "components/core/tools/scripts/lib_install/ubuntu-jammy/**"
85+
clp:
86+
- ".github/actions/**"
87+
- ".github/workflows/clp-core-build.yaml"
88+
- ".gitmodules"
89+
- "components/core/cmake/**"
90+
- "components/core/CMakeLists.txt"
91+
- "components/core/src/**"
92+
- "components/core/tests/**"
93+
- "components/core/tools/scripts/utils/build-and-run-unit-tests.py"
94+
- "taskfile.yaml"
95+
- "taskfiles/**"
96+
- "tools/scripts/deps-download/**"
97+
98+
centos-stream-9-deps-image:
99+
name: "centos-stream-9-deps-image"
100+
if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'"
101+
needs: "filter-relevant-changes"
102+
runs-on: "ubuntu-latest"
103+
steps:
104+
- uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
105+
with:
106+
submodules: "recursive"
107+
108+
- name: "Work around actions/runner-images/issues/6775"
109+
run: "chown $(id -u):$(id -g) -R ."
110+
shell: "bash"
111+
112+
- uses: "./.github/actions/clp-core-build-containers"
113+
env:
114+
OS_NAME: "centos-stream-9"
115+
with:
116+
image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
117+
docker_context: "components/core"
118+
docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\
119+
/Dockerfile"
120+
push_deps_image: >-
121+
${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
122+
token: "${{secrets.GITHUB_TOKEN}}"
123+
124+
ubuntu-jammy-deps-image:
125+
name: "ubuntu-jammy-deps-image"
126+
if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'"
127+
needs: "filter-relevant-changes"
128+
runs-on: "ubuntu-latest"
129+
steps:
130+
- uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
131+
with:
132+
submodules: "recursive"
133+
134+
- name: "Work around actions/runner-images/issues/6775"
135+
run: "chown $(id -u):$(id -g) -R ."
136+
shell: "bash"
137+
138+
- uses: "./.github/actions/clp-core-build-containers"
139+
env:
140+
OS_NAME: "ubuntu-jammy"
141+
with:
142+
image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
143+
docker_context: "components/core"
144+
docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\
145+
/Dockerfile"
146+
push_deps_image: >-
147+
${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
148+
token: "${{secrets.GITHUB_TOKEN}}"
149+
150+
centos-stream-9-binaries:
151+
# Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
152+
if: >-
153+
success()
154+
|| (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true')
155+
needs:
156+
- "centos-stream-9-deps-image"
157+
- "filter-relevant-changes"
158+
strategy:
159+
matrix:
160+
use_shared_libs: [true, false]
161+
name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins"
162+
continue-on-error: true
163+
runs-on: "ubuntu-latest"
164+
steps:
165+
- uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
166+
with:
167+
submodules: "recursive"
168+
169+
- name: "Work around actions/runner-images/issues/6775"
170+
run: "chown $(id -u):$(id -g) -R ."
171+
shell: "bash"
172+
173+
- uses: "./.github/actions/run-on-image"
174+
env:
175+
OS_NAME: "centos-stream-9"
176+
with:
177+
image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
178+
use_published_image: >-
179+
${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false'
180+
|| (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
181+
run_command: >-
182+
CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core
183+
&& python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py
184+
${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}}
185+
--source-dir /mnt/repo/components/core
186+
--build-dir /mnt/repo/components/core/build
187+
--num-jobs $(getconf _NPROCESSORS_ONLN)
188+
189+
ubuntu-jammy-binaries:
190+
# Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
191+
if: >-
192+
success()
193+
|| (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true')
194+
needs:
195+
- "filter-relevant-changes"
196+
- "ubuntu-jammy-deps-image"
197+
strategy:
198+
matrix:
199+
include:
200+
- use_shared_libs: true
201+
upload_binaries: false
202+
- use_shared_libs: false
203+
upload_binaries: true
204+
env:
205+
OS_NAME: "ubuntu-jammy"
206+
name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins"
207+
continue-on-error: true
208+
runs-on: "ubuntu-latest"
209+
steps:
210+
- uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
211+
with:
212+
submodules: "recursive"
213+
214+
- name: "Work around actions/runner-images/issues/6775"
215+
run: "chown $(id -u):$(id -g) -R ."
216+
shell: "bash"
217+
218+
- uses: "./.github/actions/run-on-image"
219+
with:
220+
image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
221+
use_published_image: >-
222+
${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
223+
|| (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
224+
run_command: >-
225+
CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core
226+
&& python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py
227+
${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}}
228+
--source-dir /mnt/repo/components/core
229+
--build-dir /mnt/repo/components/core/build
230+
--num-jobs $(getconf _NPROCESSORS_ONLN)
231+
232+
- if: "matrix.upload_binaries == true"
233+
id: "copy_binaries"
234+
run: |-
235+
output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
236+
echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
237+
238+
mkdir -p "${output_dir}"
239+
cd "$GITHUB_WORKSPACE/components/core/build"
240+
tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable
241+
shell: "bash"
242+
243+
- if: "matrix.upload_binaries == true"
244+
uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02"
245+
with:
246+
name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
247+
path: "${{steps.copy_binaries.outputs.output_dir}}"
248+
retention-days: 1
249+
250+
ubuntu-jammy-binaries-image:
251+
name: "ubuntu-jammy-binaries-image"
252+
# Run if the ancestor jobs were successful/skipped and building clp was successful.
253+
if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'"
254+
needs: "ubuntu-jammy-binaries"
255+
runs-on: "ubuntu-latest"
256+
env:
257+
OS_NAME: "ubuntu-jammy"
258+
TMP_OUTPUT_DIR: "/tmp"
259+
steps:
260+
- uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
261+
with:
262+
submodules: "recursive"
263+
264+
- name: "Work around actions/runner-images/issues/6775"
265+
run: "chown $(id -u):$(id -g) -R ."
266+
shell: "bash"
267+
268+
- uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e"
269+
with:
270+
name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
271+
path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
272+
273+
- name: "Untar binaries"
274+
working-directory: >-
275+
${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}
276+
run: |-
277+
tar xf clp.tar
278+
rm clp.tar
279+
280+
- uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772"
281+
with:
282+
registry: "ghcr.io"
283+
username: "${{github.actor}}"
284+
password: "${{secrets.GITHUB_TOKEN}}"
285+
286+
- name: "Sanitize the repo's name"
287+
id: "sanitize_repo_name"
288+
run: |-
289+
# Docker doesn't support repository names with uppercase characters, so we convert to
290+
# lowercase here.
291+
lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')
292+
echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT"
293+
shell: "bash"
294+
295+
- id: "core_image_meta"
296+
uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804"
297+
with:
298+
images: >-
299+
ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}}
300+
tags: "type=raw,value=${{github.ref_name}}"
301+
302+
# Only publish the image if this workflow was triggered by a push to `main`.
303+
# NOTE: We run the rest of the job to test that the binaries were uploaded correctly.
304+
- if: "github.event_name == 'push' && github.ref == 'refs/heads/main'"
305+
uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4"
306+
with:
307+
context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}"
308+
file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile"
309+
push: true
310+
tags: "${{steps.core_image_meta.outputs.tags}}"
311+
labels: "${{steps.core_image_meta.outputs.labels}}"
312+
313+
ubuntu-jammy-lint:
314+
name: "ubuntu-jammy-lint"
315+
# Run if the ancestor jobs succeeded OR they were skipped and clp was changed.
316+
if: >-
317+
success()
318+
|| (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true')
319+
needs:
320+
- "filter-relevant-changes"
321+
- "ubuntu-jammy-deps-image"
322+
runs-on: "ubuntu-latest"
323+
steps:
324+
- uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
325+
with:
326+
# Fetch history so that the `clang-tidy-diff` task can compare against the main branch.
327+
fetch-depth: 0
328+
submodules: "recursive"
329+
330+
- name: "Work around actions/runner-images/issues/6775"
331+
run: "chown $(id -u):$(id -g) -R ."
332+
shell: "bash"
333+
334+
# NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full
335+
# on all files.
336+
- if: "'schedule' != github.event_name"
337+
name: "Restore lint:check-cpp-static-full cache"
338+
id: "cache-restore-lint-check-cpp-static-full"
339+
uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684"
340+
with:
341+
path: |
342+
.task/checksum/lint-check-cpp-static-full
343+
.task/checksum/utils-cpp-lint-clang-tidy-*
344+
build/lint-clang-tidy
345+
346+
# NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy
347+
# violations.
348+
key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full"
349+
350+
- uses: "./.github/actions/run-on-image"
351+
env:
352+
OS_NAME: "ubuntu-jammy"
353+
with:
354+
image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
355+
use_published_image: >-
356+
${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
357+
|| (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
358+
# TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only.
359+
# run_command: >-
360+
# task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}}
361+
run_command: >-
362+
CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN)
363+
task lint:check-cpp-full
364+
365+
# Cache the source file checksums and the generated files (logs) for the
366+
# lint:check-cpp-static-full task, but only if it runs successfully on the main branch.
367+
# NOTE: If we don't cache the generated files, the task will re-run to generate them.
368+
- if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref"
369+
name: "Update lint:check-cpp-static-full cache"
370+
uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684"
371+
with:
372+
path: |
373+
.task/checksum/lint-check-cpp-static-full
374+
.task/checksum/utils-cpp-lint-clang-tidy-*
375+
build/lint-clang-tidy
376+
key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}"

0 commit comments

Comments
 (0)