Skip to content

Commit a805760

Browse files
andhreljaKernandhreljaKern
committed
perf: separate build from smodules merge
1 parent e78533b commit a805760

File tree

2 files changed

+135
-70
lines changed

2 files changed

+135
-70
lines changed

‎.github/workflows/pi_build.yml

Lines changed: 46 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,6 @@ jobs:
6363
fetch-depth: 0
6464
submodules: 'true'
6565

66-
- name: GitHub Configuration
67-
run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
68-
6966
- name: Set up Python
7067
uses: actions/setup-python@v5
7168
with:
@@ -74,32 +71,11 @@ jobs:
7471
- name: Install Dependencies
7572
run: python -m pip install pip-tools
7673

77-
- name: Compile Artifacts
74+
- name: Compile Requirements
7875
run: |
7976
cd ${{ github.workspace }}/submodules/parent-images
8077
git checkout ${{ github.event.pull_request.head.ref }}
8178
pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
82-
83-
cd ${{ github.workspace }}
84-
git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
85-
git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
86-
87-
git config user.email "devtools@kern.ai"
88-
git config user.name "GitHub Actions"
89-
90-
git add submodules
91-
git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
92-
git push origin ${{ github.event.pull_request.head.ref }}
93-
94-
gh pr create \
95-
--title "${{ github.event.pull_request.title }}" \
96-
--body "${{ github.event.pull_request.body }}" \
97-
--base dev \
98-
--head ${{ github.event.pull_request.head.ref }} \
99-
--repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image
100-
101-
# - name: Clone cicd-deployment-scripts
102-
# run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
10379
10480
- name: Set up Docker Buildx
10581
uses: docker/setup-buildx-action@v3
@@ -118,58 +94,58 @@ jobs:
11894
username: "${{ env.DEV_LOGIN_USERNAME }}"
11995
password: "${{ env.DEV_LOGIN_PASSWORD }}"
12096

121-
- name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:dev
97+
- name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
98+
uses: docker/build-push-action@v5
99+
with:
100+
context: .
101+
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache
102+
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
103+
platforms: linux/amd64
104+
file: Dockerfile
105+
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
106+
push: true
107+
build-args: |
108+
platform=linux/amd64
109+
label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
110+
111+
- name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
112+
uses: docker/build-push-action@v5
113+
with:
114+
context: .
115+
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache
116+
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
117+
platforms: linux/arm64
118+
file: Dockerfile
119+
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
120+
push: true
121+
build-args: |
122+
platform=linux/arm64
123+
label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
124+
125+
- name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}
122126
uses: docker/build-push-action@v5
123127
with:
124128
context: .
125-
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-cache
126-
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
129+
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
130+
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
127131
platforms: linux/amd64
128132
file: Dockerfile
129-
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}
133+
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
130134
push: true
131135
build-args: |
132136
platform=linux/amd64
133137
label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
134138
135-
# - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:dev-arm64
136-
# uses: docker/build-push-action@v5
137-
# with:
138-
# context: .
139-
# cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache
140-
# cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
141-
# platforms: linux/arm64
142-
# file: Dockerfile
143-
# tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64
144-
# push: true
145-
# build-args: |
146-
# platform=linux/arm64
147-
# label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
148-
149-
# - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha
150-
# uses: docker/build-push-action@v5
151-
# with:
152-
# context: .
153-
# cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
154-
# cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
155-
# platforms: linux/amd64
156-
# file: Dockerfile
157-
# tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
158-
# push: true
159-
# build-args: |
160-
# platform=linux/amd64
161-
# label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
162-
163-
# - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha-arm64
164-
# uses: docker/build-push-action@v5
165-
# with:
166-
# context: .
167-
# cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
168-
# cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
169-
# platforms: linux/arm64
170-
# file: Dockerfile
171-
# tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
172-
# push: true
173-
# build-args: |
174-
# platform=linux/arm64
175-
# label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
139+
- name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}-arm64
140+
uses: docker/build-push-action@v5
141+
with:
142+
context: .
143+
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
144+
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
145+
platforms: linux/arm64
146+
file: Dockerfile
147+
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
148+
push: true
149+
build-args: |
150+
platform=linux/arm64
151+
label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile

‎.github/workflows/pi_smodules_merge.yml

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
name: 'Parent Images: Submodules Merge'
2+
3+
on:
4+
workflow_call:
5+
6+
# Special permissions required for OIDC authentication
7+
permissions:
8+
id-token: write
9+
contents: read
10+
actions: read
11+
12+
env:
13+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
14+
15+
jobs:
16+
pi-diff:
17+
name: 'Parent Images: Diff Types'
18+
runs-on: ubuntu-latest
19+
environment: dev
20+
env:
21+
PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
22+
outputs:
23+
updated_parent_types: ${{ steps.get-diff.outputs.updated_parent_types }}
24+
steps:
25+
- name: Checkout Repository
26+
uses: actions/checkout@v4
27+
with:
28+
repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
29+
ref: ${{ github.event.pull_request.head.ref }}
30+
31+
- name: Clone cicd-deployment-scripts
32+
run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
33+
34+
- name: Get Diff Types
35+
id: get-diff
36+
run: |
37+
bash cicd-deployment-scripts/pi/diff.sh \
38+
-p "${{ github.event.pull_request.number }}"
39+
40+
pi-smodules-merge:
41+
name: 'Parent Images: Submodules Merge'
42+
runs-on: ubuntu-latest
43+
needs: [pi-diff]
44+
environment: dev
45+
env:
46+
PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
47+
strategy:
48+
matrix:
49+
parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
50+
steps:
51+
- name: Checkout repository
52+
uses: actions/checkout@v4
53+
with:
54+
token: ${{ secrets.GH_TOKEN }}
55+
repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
56+
fetch-depth: 0
57+
submodules: 'true'
58+
59+
- name: Set up Python
60+
uses: actions/setup-python@v5
61+
with:
62+
python-version: ${{ env.PYTHON_VERSION }}
63+
64+
- name: Install Dependencies
65+
run: python -m pip install pip-tools
66+
67+
- name: Perform Edit/Git Operations
68+
run: |
69+
cd ${{ github.workspace }}/submodules/parent-images
70+
git checkout ${{ github.event.pull_request.base.ref }}
71+
72+
cd ${{ github.workspace }}
73+
git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
74+
git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
75+
76+
git config user.email "devtools@kern.ai"
77+
git config user.name "GitHub Actions"
78+
79+
git add submodules
80+
git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
81+
git push origin ${{ github.event.pull_request.head.ref }}
82+
echo "::notice::${{ github.event.repository.name }} updated to origin/${{ github.event.pull_request.head.ref }}"
83+
84+
gh pr create \
85+
--title "${{ github.event.pull_request.title }}" \
86+
--body "${{ github.event.pull_request.body }}" \
87+
--base dev \
88+
--head ${{ github.event.pull_request.head.ref }} \
89+
--repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image

0 commit comments

Comments
 (0)