-
Notifications
You must be signed in to change notification settings - Fork 27
/
Copy pathazure-wheels-steps.yml
176 lines (161 loc) · 7.97 KB
/
azure-wheels-steps.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
parameters:
python_version: ''
steps:
# This could probably be done more elegantly. Comparison happens here so we
# can use the value of the py35 variable and assign a variable "active" that's
# available to all subsequent build steps in the pipeline
- bash: |
BUILD35=0; [ "$(py35)" == "True" ] && BUILD35=1
echo "Build Python 3.5:" $BUILD35
IS35=0; [ "${{ parameters.python_version }}" == "3.5" ] && IS35=1
echo "Is Python 3.5:" $IS35
ACTIVE=$(( IS35 && BUILD35 || ! IS35 ? 1 : 0 ))
echo "Build active:" $ACTIVE
echo "##vso[task.setvariable variable=active]$ACTIVE"
displayName: 'Set status'
- task: UsePythonVersion@0
inputs:
versionSpec: '${{ parameters.python_version }}'
allowUnstable: true
condition: eq(variables['active'], 1)
# This is messy, but the most convenient way to turn a JSON file into env
# variables without requiring extra dependencies. The "##vso" stuff makes the
# variables available to later steps.
- bash: |
eval $(python -c "
import json; import sys
with open('build-spec.json') as f: spec = json.load(f)
release_id = spec.get('upload-to', {}).get('release-id', '').replace('/', '-')
use_llvm = spec.get('options', {}).get('llvm', '')
use_rust = spec.get('options', {}).get('rust', '')
sys.stdout.write('BUILD_SPEC_CLONE_URL={}\n'.format(spec.get('clone-url')))
sys.stdout.write('BUILD_SPEC_COMMIT={}\n'.format(spec.get('commit')))
sys.stdout.write('BUILD_SPEC_PACKAGE_NAME={}\n'.format(spec.get('package-name')))
sys.stdout.write('BUILD_SPEC_RELEASE_ID={}\n'.format(release_id))
sys.stdout.write('BUILD_SPEC_LLVM={}\n'.format(use_llvm))
sys.stdout.write('BUILD_SPEC_RUST={}\n'.format(use_rust))")
echo "##vso[task.setvariable variable=clone_url]$BUILD_SPEC_CLONE_URL"
echo "##vso[task.setvariable variable=package_name]$BUILD_SPEC_PACKAGE_NAME"
echo "##vso[task.setvariable variable=commit]$BUILD_SPEC_COMMIT"
echo "##vso[task.setvariable variable=release_tag]$BUILD_SPEC_RELEASE_ID"
echo "##vso[task.setvariable variable=checkout]$REPO_DIR"
echo "##vso[task.setvariable variable=llvm]$BUILD_SPEC_LLVM"
echo "##vso[task.setvariable variable=rust]$BUILD_SPEC_RUST"
echo "##vso[build.updatebuildnumber]$BUILD_SPEC_RELEASE_ID"
condition: eq(variables['active'], 1)
displayName: 'Set variables'
# suggestion from https://github.com/pypa/cibuildwheel/issues/1586
- script: |
docker system prune -a -f
condition: and(eq(variables['os'], 'linux'), eq(variables['active'], 1))
displayName: 'Clean docker images'
- script: |
python -m pip install -U cibuildwheel pip setuptools
condition: eq(variables['active'], 1)
displayName: 'Install cibuildwheel'
- script: |
git clone $(clone_url) $(checkout)
cd $(checkout)
git checkout $(commit)
condition: eq(variables['active'], 1)
displayName: 'Clone package repo'
- script: |
cd $(checkout)
git submodule update --init
condition: eq(variables['active'], 1)
displayName: 'Update submodules'
- script: |
curl -sSf -o rustup-init.exe https://win.rustup.rs
rustup-init.exe -y --profile minimal --default-toolchain stable --default-host x86_64-pc-windows-msvc
set "PATH=%PATH%;%USERPROFILE%\.cargo\bin"
rustc --version
condition: and(eq(variables['os'], 'win'), eq(variables['rust'], 'true'), eq(variables['active'], 1))
displayName: 'Install Rust if needed (Windows)'
- script: choco install llvm
condition: and(eq(variables['os'], 'win'), eq(variables['llvm'], 'true'), eq(variables['active'], 1))
displayName: 'Install LLVM if needed (Windows)'
# The env variables are needed to compile things like Blis
- script: |
set "PATH=C:\Program Files\LLVM\bin;%PATH%"
set "INCLUDE=%$VC_INCLUDEPATH%"
condition: and(eq(variables['os'], 'win'), eq(variables['active'], 1))
displayName: 'Set up env (Windows)'
- bash: |
cd $(checkout)
if [ "$(rust)" == "True" ]; then
export CIBW_BEFORE_ALL_LINUX="curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable"
export CIBW_BEFORE_ALL_MACOS="$CIBW_BEFORE_ALL_LINUX && rustup target add aarch64-apple-darwin"
fi
export CIBW_ENVIRONMENT='PATH="$PATH:$HOME/.cargo/bin" SPACY_NUM_BUILD_JOBS=2'
if [ -e "build-constraints.txt" ]; then export CIBW_ENVIRONMENT="$CIBW_ENVIRONMENT PIP_CONSTRAINT=build-constraints.txt"; fi
export CIBW_BUILD_VERBOSITY=1
# build constraints through PIP_CONSTRAINT only work with pip frontend,
# with the drawback that the pip builds aren't isolated
export CIBW_BUILD_FRONTEND=pip
export CIBW_ARCHS_MACOS="x86_64 arm64"
export CIBW_TEST_SKIP="*-macosx_arm64"
export CIBW_SKIP="pp* *-musllinux* *i686* *-win32 *-win_arm64 cp36-* cp37-* cp38-*"
if [ "$(package_name)" == "spacy" ]; then
export CIBW_BEFORE_BUILD="pip install -r requirements.txt && python setup.py clean"
fi
if [ "$(python_version)" == "3.9" ]; then
export CIBW_BUILD="cp39-*"
fi
if [ "$(python_version)" == "3.10" ]; then
export CIBW_BUILD="cp310-*"
fi
if [ "$(python_version)" == "3.11" ]; then
export CIBW_BUILD="cp311-*"
fi
if [ "$(python_version)" == "3.12" ]; then
export CIBW_BUILD="cp312-*"
fi
# for spacy: clean cython-generated files between builds to handle
# profiling settings, since the builds aren't isolated; a cleaner/nicer
# version would install the build requirements from pyproject.toml
# instead, but it's a hassle to parse
if [ "$(package_name)" == "spacy" ]; then
export CIBW_BEFORE_BUILD="pip install -r requirements.txt && python setup.py clean"
fi
export CIBW_BEFORE_TEST="unset PIP_CONSTRAINT && pip install -U -r requirements.txt"
export CIBW_BEFORE_TEST_WINDOWS="set PIP_CONSTRAINT= && pip install -U -r requirements.txt"
export CIBW_TEST_COMMAND="pytest --tb=native --pyargs $(package_name)"
export CIBW_TEST_COMMAND_LINUX="unset PIP_CONSTRAINT && pip install 'urllib3<2' && $CIBW_TEST_COMMAND"
if [ "$(package_name)" == "spacy_transformers" ]; then
export CIBW_TEST_COMMAND_WINDOWS="set PIP_CONSTRAINT= && pip install -U numpy tokenizers transformers {wheel} && $CIBW_TEST_COMMAND"
export CIBW_TEST_COMMAND_MACOS="unset PIP_CONSTRAINT && pip install -U numpy tokenizers transformers {wheel} && $CIBW_TEST_COMMAND"
export CIBW_TEST_COMMAND_LINUX="unset PIP_CONSTRAINT && pip install -U numpy tokenizers transformers {wheel} && $CIBW_TEST_COMMAND_LINUX"
fi
export CIBW_REPAIR_WHEEL_COMMAND_LINUX="auditwheel repair --strip -w {dest_dir} {wheel}"
# CIBW_ENVIRONMENT needs to remain unset for blis in order to use the
# overrides from pyproject.toml, which are needed for cross-compiling
# the macos arm64 wheels
if [ "$(package_name)" == "blis" ]; then unset CIBW_ENVIRONMENT; fi
cibuildwheel --output-dir wheelhouse .
condition: eq(variables['active'], 1)
displayName: 'Build wheels'
- task: CopyFiles@2
inputs:
contents: '$(checkout)/wheelhouse/**'
targetFolder: $(Build.ArtifactStagingDirectory)
flattenFolders: true
condition: eq(variables['active'], 1)
displayName: 'Copy wheels'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: $(Build.ArtifactStagingDirectory)
artifactName: '$(ARTIFACT_NAME)'
condition: eq(variables['active'], 1)
displayName: 'Publish artifact'
- task: GitHubRelease@0
inputs:
gitHubConnection: '$(GH_CONNECTION)'
repositoryName: '$(Build.Repository.Name)'
action: 'edit'
tagSource: manual
tag: '$(release_tag)'
addChangeLog: false
assetUploadMode: replace
assets: '$(Build.ArtifactStagingDirectory)/*'
condition: and(not(eq(variables['GH_CONNECTION'], 'false')), eq(variables['active'], 1))
displayName: 'Upload to GitHub release'