Skip to content

Commit 2e115b7

Browse files
andhreljaKernandhreljaKern
authored
Merge pull request #18 from code-kern-ai/parent-images
Parent images CI/CD
2 parents 147fa0f + 9e194ff commit 2e115b7

File tree

8 files changed

+593
-0
lines changed

8 files changed

+593
-0
lines changed

‎.github/workflows/pi_build.yml

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: 'Parent Images: Build'
2+
3+
on:
4+
workflow_call:
5+
6+
# Special permissions required for OIDC authentication
7+
permissions:
8+
id-token: write
9+
contents: read
10+
actions: read
11+
12+
env:
13+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
14+
15+
jobs:
16+
pi-matrix:
17+
uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
18+
secrets: inherit
19+
with:
20+
repository: "${{ github.repository_owner }}/refinery-submodule-parent-images"
21+
22+
pi-build:
23+
name: 'Parent Images: Docker Build'
24+
runs-on: ubuntu-latest
25+
needs: [pi-matrix]
26+
environment: dev
27+
env:
28+
PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
29+
DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
30+
DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
31+
DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
32+
IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
33+
strategy:
34+
matrix:
35+
parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.parent_image_type) }}
36+
steps:
37+
- name: Checkout repository
38+
uses: actions/checkout@v4
39+
with:
40+
token: ${{ secrets.GH_TOKEN }}
41+
repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
42+
fetch-depth: 0
43+
submodules: 'true'
44+
45+
- name: Set up Python
46+
uses: actions/setup-python@v5
47+
with:
48+
python-version: ${{ env.PYTHON_VERSION }}
49+
50+
- name: Install Dependencies
51+
run: python -m pip install pip-tools
52+
53+
- name: Compile Requirements
54+
run: |
55+
cd ${{ github.workspace }}/submodules/parent-images
56+
git checkout ${{ github.event.pull_request.head.ref }}
57+
pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
58+
59+
- name: Set up Docker Buildx
60+
uses: docker/setup-buildx-action@v3
61+
with:
62+
platforms: linux/amd64,linux/arm64
63+
64+
- name: Set up QEMU
65+
uses: docker/setup-qemu-action@v3
66+
with:
67+
platforms: arm64,arm
68+
69+
- name: Log into DEV registry
70+
uses: docker/login-action@v3
71+
with:
72+
registry: "${{ env.DEV_CONTAINER_REGISTRY }}"
73+
username: "${{ env.DEV_LOGIN_USERNAME }}"
74+
password: "${{ env.DEV_LOGIN_PASSWORD }}"
75+
76+
- name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
77+
uses: docker/build-push-action@v5
78+
with:
79+
context: .
80+
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache
81+
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
82+
platforms: linux/amd64
83+
file: Dockerfile
84+
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
85+
push: true
86+
build-args: |
87+
platform=linux/amd64
88+
label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
89+
90+
- name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
91+
uses: docker/build-push-action@v5
92+
with:
93+
context: .
94+
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache
95+
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
96+
platforms: linux/arm64
97+
file: Dockerfile
98+
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
99+
push: true
100+
build-args: |
101+
platform=linux/arm64
102+
label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
103+
104+
- name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}
105+
uses: docker/build-push-action@v5
106+
with:
107+
context: .
108+
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
109+
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
110+
platforms: linux/amd64
111+
file: Dockerfile
112+
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
113+
push: true
114+
build-args: |
115+
platform=linux/amd64
116+
label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
117+
118+
- name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}-arm64
119+
uses: docker/build-push-action@v5
120+
with:
121+
context: .
122+
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
123+
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
124+
platforms: linux/arm64
125+
file: Dockerfile
126+
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
127+
push: true
128+
build-args: |
129+
platform=linux/arm64
130+
label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile

‎.github/workflows/pi_matrix.yml

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: 'Parent Images: Matrix'
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
repository:
7+
description: 'Repository for actions/checkout'
8+
required: false
9+
type: string
10+
default: ${{ github.repository }}
11+
parent_image_type:
12+
description: 'If specified, "include" only outputs apps associated to this parent image type'
13+
required: false
14+
type: string
15+
default: ''
16+
outputs:
17+
parent_image_type:
18+
description: 'List[str] of parent image types'
19+
value: ${{ jobs.pi-matrix.outputs.parent_image_type }}
20+
include:
21+
description: 'List[Dict] of apps associated to parent image types'
22+
value: ${{ jobs.pi-matrix.outputs.include }}
23+
24+
jobs:
25+
pi-matrix:
26+
name: 'Parent Images: Generate Matrix'
27+
runs-on: ubuntu-latest
28+
environment: dev
29+
env:
30+
PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
31+
outputs:
32+
parent_image_type: ${{ steps.generate-matrix.outputs.parent_image_type }}
33+
include: ${{ steps.generate-matrix.outputs.include }}
34+
steps:
35+
- name: Checkout Repository
36+
uses: actions/checkout@v4
37+
with:
38+
repository: ${{ inputs.repository }}/refinery-submodule-parent-images
39+
ref: ${{ github.event.pull_request.head.ref || github.event.repository.default_branch }}
40+
41+
- name: Clone cicd-deployment-scripts
42+
run: git clone --branch parent-images https://oauth2:${{ secrets.GH_TOKEN }}@github.com/code-kern-ai/cicd-deployment-scripts.git
43+
44+
- name: Generate Matrix
45+
id: generate-matrix
46+
run: |
47+
bash cicd-deployment-scripts/pi/matrix.sh \
48+
-p "${{ github.event.pull_request.number || '' }}" \
49+
-s cicd-deployment-scripts/pi/settings.sh \
50+
-t "${{ inputs.parent_image_type }}"

‎.github/workflows/pi_merge.yml

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
name: 'Parent Images: Submodules Merge'
2+
3+
on:
4+
workflow_call:
5+
6+
# Special permissions required for OIDC authentication
7+
permissions:
8+
id-token: write
9+
contents: read
10+
actions: read
11+
12+
env:
13+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
14+
15+
jobs:
16+
pi-matrix:
17+
uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
18+
secrets: inherit
19+
with:
20+
repository: "${{ github.repository_owner }}/refinery-submodule-parent-images"
21+
22+
pi-update-submodule:
23+
name: 'Parent Images: Submodules'
24+
runs-on: ubuntu-latest
25+
needs: [pi-matrix]
26+
environment: dev
27+
env:
28+
PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
29+
strategy:
30+
matrix:
31+
parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.parent_image_type) }}
32+
steps:
33+
- name: Checkout repository
34+
uses: actions/checkout@v4
35+
with:
36+
token: ${{ secrets.GH_TOKEN }}
37+
repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
38+
fetch-depth: 0
39+
submodules: 'true'
40+
41+
- name: Set up Python
42+
uses: actions/setup-python@v5
43+
with:
44+
python-version: ${{ env.PYTHON_VERSION }}
45+
46+
- name: Install Dependencies
47+
run: python -m pip install pip-tools
48+
49+
- name: Perform Edit/Git Operations
50+
run: |
51+
cd ${{ github.workspace }}/submodules/parent-images
52+
git checkout ${{ github.event.pull_request.base.ref }}
53+
54+
cd ${{ github.workspace }}
55+
git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
56+
git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
57+
58+
git config user.email "devtools@kern.ai"
59+
git config user.name "GitHub Actions"
60+
61+
git add submodules
62+
git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
63+
git push origin ${{ github.event.pull_request.head.ref }}
64+
echo "::notice::${{ github.event.repository.name }} updated to origin/${{ github.event.pull_request.head.ref }}"
65+
66+
gh pr create --draft \
67+
--title "${{ github.event.pull_request.title }}" \
68+
--body "${{ github.event.pull_request.body }}" \
69+
--base dev \
70+
--head ${{ github.event.pull_request.head.ref }} \
71+
--repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image
72+
73+
pi-update-app:
74+
name: 'Parent Images: ' # suffix populated by matrix
75+
runs-on: ubuntu-latest
76+
needs: [pi-matrix]
77+
environment: dev
78+
continue-on-error: true
79+
env:
80+
PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
81+
strategy:
82+
matrix:
83+
include: ${{ fromJson(needs.pi-matrix.outputs.include) }}
84+
steps:
85+
- name: Checkout Repository
86+
uses: actions/checkout@v4
87+
with:
88+
repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
89+
submodules: 'true'
90+
91+
- name: Set up Python
92+
uses: actions/setup-python@v5
93+
with:
94+
python-version: ${{ env.PYTHON_VERSION }}
95+
96+
- name: Install Dependencies
97+
run: python -m pip install pip-tools
98+
99+
- name: Clone ${{ matrix.app }}
100+
run: git clone https://oauth2:${{ secrets.GH_TOKEN }}@github.com/${{ github.repository_owner }}/${{ matrix.app }}.git
101+
102+
- name: Compile Requirements
103+
run: |
104+
pip-compile --quiet \
105+
--output-file ${{ matrix.app }}/requirements/${{ matrix.parent_image_type }}-requirements.txt \
106+
submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
107+
108+
pip-compile --quiet \
109+
--output-file ${{ matrix.app }}/requirements.txt \
110+
${{ matrix.app }}/requirements/requirements.in
111+
112+
- name: Perform Edit/Git Operations
113+
run: |
114+
cd ${{ matrix.app }}
115+
116+
git config user.email "devtools@kern.ai"
117+
git config user.name "GitHub Actions"
118+
119+
git checkout -b ${{ github.event.pull_request.head.ref }} || git checkout ${{ github.event.pull_request.head.ref }}
120+
git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
121+
122+
git add requirements*
123+
git commit -m "ci: update ${{ matrix.parent_image_type }}-requirements.txt"
124+
git push origin ${{ github.event.pull_request.head.ref }}
125+
echo "::notice::${{ matrix.app }} updated to origin/${{ github.event.pull_request.head.ref }}"
126+
127+
gh pr create --draft \
128+
--title "${{ github.event.pull_request.title }}" \
129+
--body "${{ github.event.pull_request.body }}" \
130+
--base dev \
131+
--head ${{ github.event.pull_request.head.ref }} \
132+
--repo ${{ github.repository_owner }}/${{ matrix.app }}
133+

‎.github/workflows/pi_release.yml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: 'Parent Images: Release'
2+
3+
on:
4+
workflow_call:
5+
6+
# Special permissions required for OIDC authentication
7+
permissions:
8+
id-token: write
9+
contents: read
10+
actions: read
11+
12+
env:
13+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
14+
15+
jobs:
16+
pi-matrix:
17+
uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
18+
secrets: inherit
19+
with:
20+
repository: ${{ github.repository }}
21+
parent_image_type: ${{ vars.PARENT_IMAGE_TYPE }}
22+
23+
pi-edit:
24+
name: 'Parent Images: Dockerfile ' # suffix populated by matrix
25+
runs-on: ubuntu-latest
26+
needs: [pi-matrix]
27+
environment: dev
28+
env:
29+
PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
30+
DOCKERHUB_CONTAINER_REGISTRY: ${{ vars.DOCKERHUB_CONTAINER_REGISTRY }}
31+
DOCKERHUB_LOGIN_USERNAME: ${{ secrets.DOCKERHUB_LOGIN_USERNAME }}
32+
DOCKERHUB_LOGIN_PASSWORD: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }}
33+
PARENT_IMAGE_NAME: ${{ vars.PARENT_IMAGE_NAME }}
34+
PARENT_IMAGE_TYPE: ${{ vars.PARENT_IMAGE_TYPE }}
35+
DOCKERFILE: ${{ vars.DOCKERFILE }}
36+
strategy:
37+
matrix:
38+
include: ${{ fromJson(needs.pi-matrix.outputs.include) }}
39+
steps:
40+
- name: Checkout repository
41+
uses: actions/checkout@v4
42+
with:
43+
token: ${{ secrets.GH_TOKEN }}
44+
repository: ${{ github.repository_owner }}/${{ matrix.app }}
45+
46+
- name: Clone cicd-deployment-scripts
47+
run: git clone --branch parent-images https://oauth2:${{ secrets.GH_TOKEN }}@github.com/code-kern-ai/cicd-deployment-scripts.git
48+
49+
- name: Perform Edit/Git Operations
50+
run: |
51+
bash cicd-deployment-scripts/pi/edit_dockerfile.sh \
52+
-i ${{ env.PARENT_IMAGE_NAME }} \
53+
-t ${{ matrix.parent_image_type }} \
54+
-l ${{ github.event.release.tag_name }} \
55+
-r ${{ env.DOCKERHUB_CONTAINER_REGISTRY }} \
56+
-d ${{ env.DOCKERFILE }}
57+
58+
LATEST_IMAGE_TAG="${{ env.DOCKERHUB_CONTAINER_REGISTRY }}/${{ matrix.parent_image_type }}:${{ github.event.release.tag_name }}"
59+
BASE_REF="dev"
60+
HEAD_REF="parent-images"
61+
PR_TITLE="ci(pi): update to $LATEST_IMAGE_TAG"
62+
63+
git config user.email "devtools@kern.ai"
64+
git config user.name "GitHub Actions"
65+
66+
git checkout -b parent-images || git checkout parent-images
67+
git push origin parent-images && git pull origin parent-images
68+
69+
git add ${{ env.DOCKERFILE }}
70+
git commit -m "$PR_TITLE" || true
71+
git push origin parent-images
72+
echo "::notice::${{ matrix.app }} updated to $LATEST_IMAGE_TAG"
73+
74+
bash cicd-deployment-scripts/pi/pr_create.sh \
75+
-b "$BASE_REF" \
76+
-h "$HEAD_REF" \
77+
-t "ci(pi): update to $LATEST_IMAGE_TAG" \
78+
-o ${{ github.repository_owner }} \}}
79+
-r ${{ github.event.repository.name }} \}}
80+
-n ${{ github.event.release.tag_name }} \}}
81+
-a ${{ matrix.app }}

0 commit comments

Comments
 (0)