Skip to content

Commit 22c6797

Browse files
merlimatmerlimat
authored
Build wheel files in CI (#11)
### Motivation Added CI jobs to build wheel files for Mac and Linux. ### Modifications * Simplified `CMakeList.txt` for easier detection of Boost-Python * Added PR validation to build wheels for small number of combinations * Added job triggered on when a tag is pushed to create all the combination of wheels for Py (3.7, 3.8, 3.9, 3.10), OS (Mac, linux glibc, linux musl) and CPU (x86_64 and arm64). * The wheels binaries are tested to verify no libraries/symbols are missing * The wheels are uploaded as artifacts for the job, so that they can later be staged for official release.
1 parent d14aada commit 22c6797

19 files changed

+982
-415
lines changed

‎.github/workflows/ci-build-release-wheels.yaml‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: Build release wheels
21+
on:
22+
push:
23+
tags:
24+
- '*'
25+
26+
concurrency:
27+
group: ${{ github.workflow }}-${{ github.ref }}
28+
cancel-in-progress: true
29+
30+
jobs:
31+
32+
linux-wheel:
33+
name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} - ${{matrix.cpu.platform}}
34+
runs-on: ubuntu-22.04
35+
timeout-minutes: 300
36+
37+
strategy:
38+
fail-fast: false
39+
matrix:
40+
image:
41+
- {name: 'manylinux2014', py_suffix: ''}
42+
- {name: 'manylinux_musl', py_suffix: '-alpine'}
43+
python:
44+
- {version: '3.7', spec: 'cp37-cp37m'}
45+
- {version: '3.8', spec: 'cp38-cp38'}
46+
- {version: '3.9', spec: 'cp39-cp39'}
47+
- {version: '3.10', spec: 'cp310-cp310'}
48+
cpu:
49+
- {arch: 'x86_64', platform: 'x86_64'}
50+
- {arch: 'aarch64', platform: 'arm64'}
51+
52+
steps:
53+
- name: checkout
54+
uses: actions/checkout@v3
55+
56+
- name: Set up QEMU
57+
uses: docker/setup-qemu-action@v2
58+
59+
- uses: docker/setup-buildx-action@v2
60+
- run: build-support/copy-deps-versionfile.sh
61+
62+
- name: Build Manylinux Docker image
63+
uses: docker/build-push-action@v3
64+
with:
65+
context: ./pkg/${{matrix.image.name}}
66+
load: true
67+
tags: build:latest
68+
platforms: linux/${{matrix.cpu.arch}}
69+
build-args: |
70+
PLATFORM=${{matrix.cpu.platform}}
71+
ARCH=${{matrix.cpu.arch}}
72+
PYTHON_VERSION=${{matrix.python.version}}
73+
PYTHON_SPEC=${{matrix.python.spec}}
74+
cache-from: type=gha
75+
cache-to: type=gha,mode=max
76+
77+
- name: Build wheel file
78+
run: |
79+
docker run -i -v $PWD:/pulsar-client-python build:latest \
80+
/pulsar-client-python/pkg/build-wheel-inside-docker.sh
81+
82+
- name: Test wheel file
83+
run: |
84+
docker run -i -v $PWD:/pulsar-client-python \
85+
--platform linux/${{matrix.cpu.arch}} \
86+
python:${{matrix.python.version}}${{matrix.image.py_suffix}} \
87+
/pulsar-client-python/pkg/test-wheel.sh
88+
89+
- name: Upload artifacts
90+
uses: actions/upload-artifact@v3
91+
with:
92+
name: wheel-${{matrix.image.name}}-py${{matrix.python.version}}-${{matrix.cpu.platform}}
93+
path: wheelhouse/*.whl
94+
95+
mac-wheels:
96+
name: Wheel MacOS Universal2 - Py ${{matrix.py.version}}
97+
runs-on: macos-12
98+
timeout-minutes: 300
99+
100+
strategy:
101+
fail-fast: false
102+
matrix:
103+
py:
104+
- {version: '3.7', version_long: '3.7.14'}
105+
- {version: '3.8', version_long: '3.8.13'}
106+
- {version: '3.9', version_long: '3.9.14'}
107+
- {version: '3.10', version_long: '3.10.7'}
108+
109+
steps:
110+
- name: checkout
111+
uses: actions/checkout@v3
112+
113+
- name: Dependencies cache
114+
uses: actions/cache@v3
115+
id: cache-deps
116+
with:
117+
path: .pulsar-mac-build/deps/install
118+
key: ${{matrix.py.version_long}}-${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pkg/mac/*') }}
119+
120+
- name: Build dependencies
121+
if: steps.cache-deps.outputs.cache-hit != 'true'
122+
run: pkg/mac/build-dependencies.sh ${{matrix.py.version}} ${{matrix.py.version_long}}
123+
124+
- name: Pulsar C++ cache
125+
uses: actions/cache@v3
126+
id: cache-cpp
127+
with:
128+
path: .pulsar-mac-build/cpp/install
129+
key: ${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pulsar-client-cpp-version.txt') }}-${{ hashFiles('pkg/mac/*') }}
130+
131+
- name: Build Pulsar C++ client
132+
if: steps.cache-cpp.outputs.cache-hit != 'true'
133+
run: pkg/mac/build-pulsar-cpp.sh
134+
135+
- name: Build and test Mac wheels
136+
run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}}
137+
138+
- name: Upload artifacts
139+
uses: actions/upload-artifact@v3
140+
with:
141+
name: wheel-mac-py${{matrix.py.version}}
142+
path: dist/*.whl
143+

‎.github/workflows/ci-pr-validation.yaml‎

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030

3131
unit-tests:
3232
name: Run unit tests
33-
runs-on: ubuntu-latest
33+
runs-on: ubuntu-22.04
3434
timeout-minutes: 120
3535

3636
steps:
@@ -64,3 +64,93 @@ jobs:
6464
- name: Stop Pulsar service
6565
run: ./build-support/pulsar-test-service-stop.sh
6666

67+
68+
linux-wheel:
69+
name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} - ${{matrix.cpu.platform}}
70+
runs-on: ubuntu-22.04
71+
timeout-minutes: 300
72+
73+
strategy:
74+
fail-fast: false
75+
matrix:
76+
image:
77+
- {name: 'manylinux2014', py_suffix: ''}
78+
- {name: 'manylinux_musl', py_suffix: '-alpine'}
79+
python:
80+
- {version: '3.10', spec: 'cp310-cp310'}
81+
cpu:
82+
- {arch: 'x86_64', platform: 'x86_64'}
83+
84+
steps:
85+
- name: checkout
86+
uses: actions/checkout@v3
87+
88+
- name: Set up QEMU
89+
uses: docker/setup-qemu-action@v2
90+
91+
- uses: docker/setup-buildx-action@v2
92+
- run: build-support/copy-deps-versionfile.sh
93+
94+
- name: Build Manylinux Docker image
95+
uses: docker/build-push-action@v3
96+
with:
97+
context: ./pkg/${{matrix.image.name}}
98+
load: true
99+
tags: build:latest
100+
platforms: linux/${{matrix.cpu.arch}}
101+
build-args: |
102+
PLATFORM=${{matrix.cpu.platform}}
103+
ARCH=${{matrix.cpu.arch}}
104+
PYTHON_VERSION=${{matrix.python.version}}
105+
PYTHON_SPEC=${{matrix.python.spec}}
106+
cache-from: type=gha
107+
cache-to: type=gha,mode=max
108+
109+
- name: Build wheel file
110+
run: |
111+
docker run -i -v $PWD:/pulsar-client-python build:latest \
112+
/pulsar-client-python/pkg/build-wheel-inside-docker.sh
113+
114+
- name: Test wheel file
115+
run: |
116+
docker run -i -v $PWD:/pulsar-client-python python:${{matrix.python.version}}${{matrix.image.py_suffix}} \
117+
/pulsar-client-python/pkg/test-wheel.sh
118+
119+
mac-wheels:
120+
name: Wheel MacOS Universal2 - Py ${{matrix.py.version}}
121+
runs-on: macos-12
122+
timeout-minutes: 300
123+
124+
strategy:
125+
matrix:
126+
py:
127+
- {version: '3.10', version_long: '3.10.7'}
128+
129+
steps:
130+
- name: checkout
131+
uses: actions/checkout@v3
132+
133+
- name: Dependencies cache
134+
uses: actions/cache@v3
135+
id: cache-deps
136+
with:
137+
path: .pulsar-mac-build/deps/install
138+
key: ${{matrix.py.version_long}}-${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pkg/mac/*') }}
139+
140+
- name: Build dependencies
141+
if: steps.cache-deps.outputs.cache-hit != 'true'
142+
run: pkg/mac/build-dependencies.sh ${{matrix.py.version}} ${{matrix.py.version_long}}
143+
144+
- name: Pulsar C++ cache
145+
uses: actions/cache@v3
146+
id: cache-cpp
147+
with:
148+
path: .pulsar-mac-build/cpp/install
149+
key: ${{ hashFiles('dependencies.yaml') }}-${{ hashFiles('pulsar-client-cpp-version.txt') }}-${{ hashFiles('pkg/mac/*') }}
150+
151+
- name: Build Pulsar C++ client
152+
if: steps.cache-cpp.outputs.cache-hit != 'true'
153+
run: pkg/mac/build-pulsar-cpp.sh
154+
155+
- name: Build and test Mac wheels
156+
run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}}

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,6 @@ cmake_install.cmake
1111
__pycache__
1212
.build
1313
.pulsar-mac-wheels-cache
14-
.DS_Store
14+
.DS_Store
15+
wheelhouse
16+
.pulsar-mac-build

0 commit comments

Comments
 (0)