Skip to content

Commit 3c9679b

Browse files
jacobdrjacobdr
committed
Add support for testing arm64 variants
1 parent 2248ea2 commit 3c9679b

File tree

2 files changed

+113
-28
lines changed

2 files changed

+113
-28
lines changed

‎.github/workflows/test.yaml

Lines changed: 51 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@ jobs:
1414
runs-on: ubuntu-latest
1515
steps:
1616
- uses: actions/checkout@v3
17+
- name: Set up QEMU
18+
uses: docker/setup-qemu-action@v2
19+
with:
20+
platforms: arm64
21+
- name: Set up Docker Buildx
22+
uses: docker/setup-buildx-action@v2
23+
with:
24+
install: true
1725
- uses: actions/setup-python@v4
1826
with:
1927
python-version: "3.10"
@@ -23,15 +31,14 @@ jobs:
2331
pip install -r requirements.txt
2432
- name: Build Wheels
2533
run: |
26-
mkdir dist
2734
python make_wheels.py
2835
- name: Show built files
2936
run: |
3037
ls -l dist/*
3138
- uses: actions/upload-artifact@v3
3239
with:
3340
name: nodejs-pip-wheels
34-
path: dist/
41+
path: dist
3542
if-no-files-found: error
3643
retention-days: 1
3744

@@ -42,7 +49,7 @@ jobs:
4249
strategy:
4350
fail-fast: false
4451
matrix:
45-
os: [ubuntu-latest, windows-latest, macos-latest]
52+
os: [windows-latest, macos-latest]
4653
nodejs-version: ['14.19.3', '16.15.1', '18.4.0']
4754
python-version: ['3.7', '3.8', '3.9', '3.10']
4855

@@ -79,14 +86,16 @@ jobs:
7986
run:
8087
python -W error -m nodejs --version
8188
python -W error -m nodejs.npm --version
89+
8290
test-docker:
83-
name: "Test Docker OS:${{ matrix.os-variant }} Python:${{ matrix.python-version }} NodeJS:${{ matrix.nodejs-version }}"
91+
name: "Test Docker Architecture:${{ matrix.cpu-arch }} OS:${{ matrix.os-variant }} Python:${{ matrix.python-version }} NodeJS:${{ matrix.nodejs-version }}"
8492
runs-on: ubuntu-latest
8593
needs: [build-wheels]
8694
strategy:
8795
fail-fast: false
8896
matrix:
8997
os-variant: [alpine, slim-buster, slim-bullseye]
98+
cpu-arch: [linux/amd64, linux/arm64]
9099
python-version: ['3.7', '3.8', '3.9', '3.10']
91100
nodejs-version: ['14.19.3', '16.15.1', '18.4.0']
92101

@@ -104,16 +113,49 @@ jobs:
104113
with:
105114
name: nodejs-pip-wheels
106115
path: dist
116+
- name: List available wheels
117+
run: |
118+
ls -lah dist
107119
- name: Docker build
108120
run: |
109-
if [[ ${{ matrix.os-variant }} =~ "alpine" ]]; then
110-
WHEEL_TO_INSTALL=nodejs_bin-${{ matrix.nodejs-version }}a3-py3-none-musllinux_1_1_x86_64.whl
111-
else
112-
WHEEL_TO_INSTALL=nodejs_bin-${{ matrix.nodejs-version }}a3-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl
113-
fi
121+
wheel_prefix_except_platform=nodejs_bin-${{ matrix.nodejs-version }}a3-py3-none
122+
123+
case ${{ matrix.cpu-arch }} in
124+
linux/amd64)
125+
python_cpu_arch=x86_64
126+
python_libc_variant=2_12
127+
manylinx_variant_year=2010
128+
;;
129+
linux/arm64)
130+
python_cpu_arch=aarch64
131+
python_libc_variant=2_17
132+
manylinx_variant_year=2014
133+
;;
134+
*)
135+
echo "Could not parse the CPU architecture"
136+
exit 1
137+
;;
138+
esac
139+
140+
case ${{ matrix.os-variant }} in
141+
alpine)
142+
python_platform=musllinux_1_1_${python_cpu_arch}
143+
;;
144+
slim-buster | slim-bullseye)
145+
python_platform=manylinux_${python_libc_variant}_${python_cpu_arch}.manylinux${manylinx_variant_year}_${python_cpu_arch}
146+
;;
147+
*)
148+
echo "Could not parse the OS variant"
149+
exit 1
150+
;;
151+
esac
152+
153+
WHEEL_TO_INSTALL=${wheel_prefix_except_platform}-${python_platform}.whl
114154
echo "WHEEL_TO_INSTALL=${WHEEL_TO_INSTALL}"
155+
115156
docker build \
116157
-f Dockerfile \
158+
--platform=${{ matrix.cpu-arch }} \
117159
--build-arg PYTHON_VERSION=${{ matrix.python-version }} \
118160
--build-arg OS_VARIANT=${{ matrix.os-variant }} \
119161
--build-arg WHEEL_TO_INSTALL=${WHEEL_TO_INSTALL} \

‎make_wheels.py

Lines changed: 62 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
import os
22
import hashlib
33
import pathlib
4+
import io
5+
import subprocess
46
import urllib.request
57
import libarchive
8+
import tempfile
69
from email.message import EmailMessage
710
from wheel.wheelfile import WheelFile
8-
from zipfile import ZipInfo, ZIP_DEFLATED
11+
from zipfile import ZipInfo, ZIP_DEFLATED, ZipFile
912
from inspect import cleandoc
1013

1114

@@ -46,17 +49,52 @@
4649
'linux-x64': 'manylinux_2_12_x86_64.manylinux2010_x86_64',
4750
'linux-armv7l': 'manylinux_2_17_armv7l.manylinux2014_armv7l',
4851
'linux-arm64': 'manylinux_2_17_aarch64.manylinux2014_aarch64',
49-
'linux-x64-musl': 'musllinux_1_1_x86_64'
52+
'linux-x64-musl': 'musllinux_1_1_x86_64',
53+
'linux-arm64-musl': 'musllinux_1_1_aarch64'
5054
}
5155

5256
# https://github.com/nodejs/unofficial-builds/
5357
# Versions added here should match the keys above
5458
UNOFFICIAL_NODEJS_BUILDS = {'linux-x64-musl'}
59+
DOCKER_BASED_BUILDS = {"linux-arm64-musl"}
5560

56-
_mismatched_versions = UNOFFICIAL_NODEJS_BUILDS - set(PLATFORMS.keys())
61+
_mismatched_versions = (UNOFFICIAL_NODEJS_BUILDS|DOCKER_BASED_BUILDS) - set(PLATFORMS.keys())
5762
if _mismatched_versions:
5863
raise Exception(f"A version mismatch occurred. Check the usage of {_mismatched_versions}")
5964

65+
def _build_virtual_release_archive(docker_image: str, platform: str) -> bytes:
66+
binaries_to_copy = [x.split("bin/")[1] for x in [*NODE_BINS, *NODE_OTHER_BINS] if x.startswith("bin")]
67+
zip_bytes = io.BytesIO()
68+
with tempfile.TemporaryDirectory() as tmpdirname, ZipFile(zip_bytes, "w") as zip_file:
69+
subprocess.check_call([
70+
"docker",
71+
"run",
72+
"--rm",
73+
f"--platform={platform}",
74+
f"--volume={tmpdirname}:/external",
75+
"--entrypoint=sh",
76+
docker_image,
77+
"-c",
78+
f"""
79+
for binary in {" ".join(binaries_to_copy)}; do
80+
cp $(which $binary) /external
81+
done
82+
chmod 755 /external/*
83+
"""
84+
],
85+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
86+
87+
tmpdir_contents = list(pathlib.Path(tmpdirname).glob("*"))
88+
for binary in tmpdir_contents:
89+
with open(binary, "rb") as f:
90+
file_info = ZipInfo(
91+
filename=f"node/bin/{binary.name}",
92+
)
93+
file_info.external_attr = 0o755 << 16
94+
zip_file.writestr(file_info, f.read())
95+
96+
zip_bytes.seek(0)
97+
return zip_bytes.read()
6098

6199
class ReproducibleWheelFile(WheelFile):
62100
def writestr(self, zinfo, *args, **kwargs):
@@ -186,7 +224,7 @@ def main() -> None:
186224
elif entry_name in NODE_OTHER_BINS and NODE_OTHER_BINS[entry_name][1]:
187225
other_bin = NODE_OTHER_BINS[entry_name][0]
188226
init_imports.append(f'from . import {other_bin} as {other_bin}')
189-
script_name = '/'.join(os.path.normpath(os.path.join(os.path.dirname(entry.name), entry.linkpath)).split('/')[1:])
227+
script_name = '/'.join(os.path.normpath(os.path.join(os.path.dirname(entry.name), entry.linkpath or other_bin)).split('/')[1:])
190228
contents[f'nodejs/{NODE_OTHER_BINS[entry_name][0]}.py'] = cleandoc(f"""
191229
import os, sys
192230
from typing import TYPE_CHECKING
@@ -312,22 +350,27 @@ def make_nodejs_version(node_version, suffix=''):
312350
print('Suffix:', suffix)
313351

314352
for node_platform, python_platform in PLATFORMS.items():
315-
filetype = 'zip' if node_platform.startswith('win-') else 'tar.xz'
316-
if node_platform in UNOFFICIAL_NODEJS_BUILDS:
317-
node_url = f'https://unofficial-builds.nodejs.org/download/release/v{node_version}/node-v{node_version}-{node_platform}.{filetype}'
353+
if node_platform in DOCKER_BASED_BUILDS:
354+
docker_image = f"node:{node_version}-alpine"
355+
print(f'- Making Wheel for {node_platform} from docker image {docker_image}')
356+
node_archive = _build_virtual_release_archive(docker_image=docker_image, platform="linux/arm64")
318357
else:
319-
node_url = f'https://nodejs.org/dist/v{node_version}/node-v{node_version}-{node_platform}.{filetype}'
320-
321-
print(f'- Making Wheel for {node_platform} from {node_url}')
322-
try:
323-
with urllib.request.urlopen(node_url) as request:
324-
node_archive = request.read()
325-
print(f' {node_url}')
326-
print(f' {hashlib.sha256(node_archive).hexdigest()}')
327-
except urllib.error.HTTPError as e:
328-
print(f' {e.code} {e.reason}')
329-
print(f' Skipping {node_platform}')
330-
continue
358+
filetype = 'zip' if node_platform.startswith('win-') else 'tar.xz'
359+
if node_platform in UNOFFICIAL_NODEJS_BUILDS:
360+
node_url = f'https://unofficial-builds.nodejs.org/download/release/v{node_version}/node-v{node_version}-{node_platform}.{filetype}'
361+
else:
362+
node_url = f'https://nodejs.org/dist/v{node_version}/node-v{node_version}-{node_platform}.{filetype}'
363+
364+
print(f'- Making Wheel for {node_platform} from {node_url}')
365+
try:
366+
with urllib.request.urlopen(node_url) as request:
367+
node_archive: bytes = request.read()
368+
print(f' {node_url}')
369+
print(f' {hashlib.sha256(node_archive).hexdigest()}')
370+
except urllib.error.HTTPError as e:
371+
print(f' {e.code} {e.reason}')
372+
print(f' Skipping {node_platform}')
373+
continue
331374

332375
wheel_path = write_nodejs_wheel('dist/',
333376
node_version=node_version,

0 commit comments

Comments
 (0)