Skip to content

Commit 0fd450a

Browse files
luhenryluhenry
authored
[sdks] Add Groovy script to build the SDKs archives for the products (mono#11137)
1 parent 0beb087 commit 0fd450a

File tree

3 files changed

+180
-61
lines changed

3 files changed

+180
-61
lines changed

‎scripts/ci/pipeline/sdks-archive.groovy

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
properties([compressBuildLog()])
2+
3+
parallel (
4+
"Archive-android-debug-Darwin": {
5+
node ("osx-devices") {
6+
archive ("android", "debug", "Darwin")
7+
}
8+
},
9+
"Archive-android-release-Darwin": {
10+
node ("osx-devices") {
11+
archive ("android", "release", "Darwin")
12+
}
13+
},
14+
// "Archive-android-debug-Linux": {
15+
// node ("debian-9-amd64multiarchi386-preview") {
16+
// archive ("android", "debug", "Linux",)
17+
// }
18+
// },
19+
// "Archive-android-release-Linux": {
20+
// node ("debian-9-amd64multiarchi386-preview") {
21+
// archive ("android", "release", "Linux",)
22+
// }
23+
// },
24+
"Archive-ios-release-Darwin": {
25+
node ("osx-devices") {
26+
archive ("ios", "release", "Darwin")
27+
}
28+
},
29+
"Archive-wasm-release-Linux": {
30+
node ("ubuntu-1804-amd64") {
31+
archive ("wasm", "release", "Linux", "ubuntu-1804-amd64-preview", "npm dotnet-sdk-2.1 nuget")
32+
}
33+
}
34+
)
35+
36+
def archive (product, configuration, platform, chrootname = "", chrootadditionalpackages = "") {
37+
def isPr = (env.ghprbPullId && !env.ghprbPullId.empty ? true : false)
38+
def monoBranch = (isPr ? "pr" : env.BRANCH_NAME)
39+
def jobName = (isPr ? "archive-mono-pullrequest" : "archive-mono")
40+
def packageFileName = null
41+
def commitHash = null
42+
def utils = null
43+
44+
ws ("workspace/${jobName}/${monoBranch}/${product}/${configuration}") {
45+
timestamps {
46+
stage('Checkout') {
47+
// clone and checkout repo
48+
checkout scm
49+
50+
utils = load "scripts/ci/pipeline/utils.groovy"
51+
52+
// remove old stuff
53+
sh 'git reset --hard HEAD'
54+
sh 'git submodule foreach --recursive git reset --hard HEAD'
55+
sh 'git clean -xdff'
56+
sh 'git submodule foreach --recursive git clean -xdff'
57+
58+
// get current commit sha
59+
commitHash = sh (script: 'git rev-parse HEAD', returnStdout: true).trim()
60+
currentBuild.displayName = "${commitHash.substring(0,7)}"
61+
}
62+
try {
63+
stage('Build') {
64+
utils.reportGitHubStatus (isPr ? env.ghprbActualCommit : commitHash, "Archive-${product}-${configuration}-${platform}", env.BUILD_URL, 'PENDING', 'Building...')
65+
66+
// build the Archive
67+
timeout (time: 300, unit: 'MINUTES') {
68+
if (platform == "Darwin") {
69+
def brewpackages = "autoconf automake ccache cmake coreutils gdk-pixbuf gettext glib gnu-sed gnu-tar intltool ios-deploy jpeg libffi libidn2 libpng libtiff libtool libunistring ninja openssl p7zip pcre pkg-config scons wget xz"
70+
sh "brew install ${brewpackages} || brew upgrade ${brewpackages}"
71+
72+
sh "CI_TAGS=sdks-${product},no-tests,${configuration} scripts/ci/run-jenkins.sh"
73+
} else if (platform == "Linux") {
74+
chroot chrootName: chrootname,
75+
command: "CI_TAGS=sdks-${product},no-tests,${configuration} scripts/ci/run-jenkins.sh",
76+
additionalPackages: "xvfb xauth mono-devel git python wget bc build-essential libtool autoconf automake gettext iputils-ping cmake lsof libkrb5-dev curl p7zip-full ninja-build zip unzip gcc-multilib g++-multilib mingw-w64 binutils-mingw-w64 openjdk-8-jre ${chrootadditionalpackages}"
77+
} else {
78+
throw new Exception("Unknown platform \"${platform}\"")
79+
}
80+
}
81+
// move Archive to the workspace root
82+
packageFileName = findFiles (glob: "${product}-${configuration}-${platform}-${commitHash}.zip")[0].name
83+
}
84+
stage('Upload Archive to Azure') {
85+
step([
86+
$class: 'WAStoragePublisher',
87+
allowAnonymousAccess: true,
88+
cleanUpContainer: false,
89+
cntPubAccess: true,
90+
containerName: "mono-sdks",
91+
doNotFailIfArchivingReturnsNothing: false,
92+
doNotUploadIndividualFiles: false,
93+
doNotWaitForPreviousBuild: true,
94+
excludeFilesPath: '',
95+
filesPath: "${packageFileName}",
96+
storageAccName: 'credential for xamjenkinsartifact',
97+
storageCredentialId: 'fbd29020e8166fbede5518e038544343',
98+
uploadArtifactsOnlyIfSuccessful: true,
99+
uploadZips: false,
100+
virtualPath: ""
101+
])
102+
}
103+
104+
utils.reportGitHubStatus (isPr ? env.ghprbActualCommit : commitHash, "Archive-${product}-${configuration}-${platform}", "https://xamjenkinsartifact.azureedge.net/mono-sdks/${packageFileName}", 'SUCCESS', packageFileName)
105+
}
106+
catch (Exception e) {
107+
utils.reportGitHubStatus (isPr ? env.ghprbActualCommit : commitHash, "Archive-${product}-${configuration}-${platform}", env.BUILD_URL, 'FAILURE', "Build failed.")
108+
throw e
109+
}
110+
}
111+
}
112+
}

‎scripts/ci/run-jenkins.sh

Lines changed: 66 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -114,24 +114,26 @@ if [[ ${CI_TAGS} == *'sdks-ios'* ]];
114114

115115
${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-ios NINJA=
116116

117-
${TESTCMD} --label=build-tests --timeout=10m --fatal make -C sdks/ios compile-tests
118-
${TESTCMD} --label=run-sim --timeout=20m make -C sdks/ios run-ios-sim-all
119-
${TESTCMD} --label=build-ios-dev --timeout=60m make -C sdks/ios build-ios-dev-all
120-
if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
121-
for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
122-
fi
123-
${TESTCMD} --label=build-ios-dev-llvm --timeout=60m make -C sdks/ios build-ios-dev-llvm-all
124-
if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
125-
for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-llvm-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
126-
fi
127-
${TESTCMD} --label=build-ios-dev-interp-only --timeout=60m make -C sdks/ios build-ios-dev-interp-only-all
128-
if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
129-
for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-only-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
130-
fi
131-
${TESTCMD} --label=build-ios-dev-interp-mixed --timeout=60m make -C sdks/ios build-ios-dev-interp-mixed-all
132-
if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
133-
for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-mixed-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
134-
fi
117+
if [[ ${CI_TAGS} != *'no-tests'* ]]; then
118+
${TESTCMD} --label=build-tests --timeout=10m --fatal make -C sdks/ios compile-tests
119+
${TESTCMD} --label=run-sim --timeout=20m make -C sdks/ios run-ios-sim-all
120+
${TESTCMD} --label=build-ios-dev --timeout=60m make -C sdks/ios build-ios-dev-all
121+
if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
122+
for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
123+
fi
124+
${TESTCMD} --label=build-ios-dev-llvm --timeout=60m make -C sdks/ios build-ios-dev-llvm-all
125+
if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
126+
for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-llvm-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
127+
fi
128+
${TESTCMD} --label=build-ios-dev-interp-only --timeout=60m make -C sdks/ios build-ios-dev-interp-only-all
129+
if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
130+
for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-only-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
131+
fi
132+
${TESTCMD} --label=build-ios-dev-interp-mixed --timeout=60m make -C sdks/ios build-ios-dev-interp-mixed-all
133+
if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
134+
for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-mixed-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
135+
fi
136+
fi
135137
exit 0
136138
fi
137139

@@ -149,34 +151,37 @@ if [[ ${CI_TAGS} == *'sdks-android'* ]];
149151
# but doesn't get stuck when called via the shell, so let's just call it here now.
150152
${TESTCMD} --label=provision-android --timeout=120m --fatal make -j ${CI_CPU_COUNT} -C sdks/builds provision-android && make -C sdks/android accept-android-license
151153
${TESTCMD} --label=provision-mxe --timeout=240m --fatal make -j ${CI_CPU_COUNT} -C sdks/builds provision-mxe
152-
${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
153-
if [[ ${CI_TAGS} != *'pull-request'* ]]; then
154-
${TESTCMD} --label=archive-debug --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1 CONFIGURATION=debug
154+
if [[ ${CI_TAGS} != *'debug'* ]]; then
155+
${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
156+
else
157+
${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1 CONFIGURATION=debug
155158
fi
156159

157-
${TESTCMD} --label=mini --timeout=60m make -C sdks/android check-mini
158-
${TESTCMD} --label=corlib --timeout=60m make -C sdks/android check-corlib
159-
${TESTCMD} --label=System --timeout=60m make -C sdks/android check-System
160-
${TESTCMD} --label=System.Core --timeout=60m make -C sdks/android check-System.Core
161-
${TESTCMD} --label=System.Data --timeout=60m make -C sdks/android check-System.Data
162-
${TESTCMD} --label=System.IO.Compression.FileSystem --timeout=60m make -C sdks/android check-System.IO.Compression.FileSystem
163-
${TESTCMD} --label=System.IO.Compression --timeout=60m make -C sdks/android check-System.IO.Compression
164-
${TESTCMD} --label=System.Json --timeout=60m make -C sdks/android check-System.Json
165-
${TESTCMD} --label=System.Net.Http --timeout=60m make -C sdks/android check-System.Net.Http
166-
${TESTCMD} --label=System.Numerics --timeout=60m make -C sdks/android check-System.Numerics
167-
${TESTCMD} --label=System.Runtime.Serialization --timeout=60m make -C sdks/android check-System.Runtime.Serialization
168-
${TESTCMD} --label=System.ServiceModel.Web --timeout=60m make -C sdks/android check-System.ServiceModel.Web
169-
${TESTCMD} --label=System.Transactions --timeout=60m make -C sdks/android check-System.Transactions
170-
${TESTCMD} --label=System.Xml --timeout=60m make -C sdks/android check-System.Xml
171-
${TESTCMD} --label=System.Xml.Linq --timeout=60m make -C sdks/android check-System.Xml.Linq
172-
${TESTCMD} --label=Mono.CSharp --timeout=60m make -C sdks/android check-Mono.CSharp
173-
${TESTCMD} --label=Mono.Data.Sqlite --timeout=60m make -C sdks/android check-Mono.Data.Sqlite
174-
${TESTCMD} --label=Mono.Data.Tds --timeout=60m make -C sdks/android check-Mono.Data.Tds
175-
${TESTCMD} --label=Mono.Security --timeout=60m make -C sdks/android check-Mono.Security
160+
if [[ ${CI_TAGS} != *'no-tests'* ]]; then
161+
${TESTCMD} --label=mini --timeout=60m make -C sdks/android check-mini
162+
${TESTCMD} --label=corlib --timeout=60m make -C sdks/android check-corlib
163+
${TESTCMD} --label=System --timeout=60m make -C sdks/android check-System
164+
${TESTCMD} --label=System.Core --timeout=60m make -C sdks/android check-System.Core
165+
${TESTCMD} --label=System.Data --timeout=60m make -C sdks/android check-System.Data
166+
${TESTCMD} --label=System.IO.Compression.FileSystem --timeout=60m make -C sdks/android check-System.IO.Compression.FileSystem
167+
${TESTCMD} --label=System.IO.Compression --timeout=60m make -C sdks/android check-System.IO.Compression
168+
${TESTCMD} --label=System.Json --timeout=60m make -C sdks/android check-System.Json
169+
${TESTCMD} --label=System.Net.Http --timeout=60m make -C sdks/android check-System.Net.Http
170+
${TESTCMD} --label=System.Numerics --timeout=60m make -C sdks/android check-System.Numerics
171+
${TESTCMD} --label=System.Runtime.Serialization --timeout=60m make -C sdks/android check-System.Runtime.Serialization
172+
${TESTCMD} --label=System.ServiceModel.Web --timeout=60m make -C sdks/android check-System.ServiceModel.Web
173+
${TESTCMD} --label=System.Transactions --timeout=60m make -C sdks/android check-System.Transactions
174+
${TESTCMD} --label=System.Xml --timeout=60m make -C sdks/android check-System.Xml
175+
${TESTCMD} --label=System.Xml.Linq --timeout=60m make -C sdks/android check-System.Xml.Linq
176+
${TESTCMD} --label=Mono.CSharp --timeout=60m make -C sdks/android check-Mono.CSharp
177+
${TESTCMD} --label=Mono.Data.Sqlite --timeout=60m make -C sdks/android check-Mono.Data.Sqlite
178+
${TESTCMD} --label=Mono.Data.Tds --timeout=60m make -C sdks/android check-Mono.Data.Tds
179+
${TESTCMD} --label=Mono.Security --timeout=60m make -C sdks/android check-Mono.Security
180+
fi
176181
exit 0
177182
fi
178183

179-
if [[ ${CI_TAGS} == *'webassembly'* ]];
184+
if [[ ${CI_TAGS} == *'webassembly'* ]] || [[ ${CI_TAGS} == *'wasm'* ]];
180185
then
181186
echo "DISABLE_ANDROID=1" > sdks/Make.config
182187
echo "DISABLE_IOS=1" >> sdks/Make.config
@@ -190,24 +195,26 @@ if [[ ${CI_TAGS} == *'webassembly'* ]];
190195

191196
${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-wasm NINJA=
192197

193-
${TESTCMD} --label=wasm-build --timeout=60m --fatal make -j ${CI_CPU_COUNT} -C sdks/wasm build
194-
${TESTCMD} --label=ch-mini-test --timeout=60m make -C sdks/wasm run-ch-mini
195-
${TESTCMD} --label=v8-mini-test --timeout=60m make -C sdks/wasm run-v8-mini
196-
${TESTCMD} --label=sm-mini-test --timeout=60m make -C sdks/wasm run-sm-mini
197-
${TESTCMD} --label=jsc-mini-test --timeout=60m make -C sdks/wasm run-jsc-mini
198-
#The following tests are not passing yet, so enabling them would make us perma-red
199-
#${TESTCMD} --label=mini-corlib --timeout=60m make -C sdks/wasm run-all-corlib
200-
#${TESTCMD} --label=mini-system --timeout=60m make -C sdks/wasm run-all-system
201-
# Chakra crashes with System.Core. See https://github.com/mono/mono/issues/8345
202-
${TESTCMD} --label=ch-system-core --timeout=60m make -C sdks/wasm run-ch-system-core
203-
${TESTCMD} --label=v8-system-core --timeout=60m make -C sdks/wasm run-v8-system-core
204-
${TESTCMD} --label=sm-system-core --timeout=60m make -C sdks/wasm run-sm-system-core
205-
${TESTCMD} --label=jsc-system-core --timeout=60m make -C sdks/wasm run-jsc-system-core
206-
${TESTCMD} --label=aot-mini --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm run-aot-mini
207-
${TESTCMD} --label=build-aot-all --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm build-aot-all
208-
for suite in ${aot_test_suites}; do ${TESTCMD} --label=run-aot-${suite} --timeout=10m make -C sdks/wasm run-aot-${suite}; done
209-
#${TESTCMD} --label=check-aot --timeout=60m make -C sdks/wasm check-aot
210-
${TESTCMD} --label=package --timeout=60m make -C sdks/wasm package
198+
if [[ ${CI_TAGS} != *'no-tests'* ]]; then
199+
${TESTCMD} --label=wasm-build --timeout=60m --fatal make -j ${CI_CPU_COUNT} -C sdks/wasm build
200+
${TESTCMD} --label=ch-mini-test --timeout=60m make -C sdks/wasm run-ch-mini
201+
${TESTCMD} --label=v8-mini-test --timeout=60m make -C sdks/wasm run-v8-mini
202+
${TESTCMD} --label=sm-mini-test --timeout=60m make -C sdks/wasm run-sm-mini
203+
${TESTCMD} --label=jsc-mini-test --timeout=60m make -C sdks/wasm run-jsc-mini
204+
#The following tests are not passing yet, so enabling them would make us perma-red
205+
#${TESTCMD} --label=mini-corlib --timeout=60m make -C sdks/wasm run-all-corlib
206+
#${TESTCMD} --label=mini-system --timeout=60m make -C sdks/wasm run-all-system
207+
# Chakra crashes with System.Core. See https://github.com/mono/mono/issues/8345
208+
${TESTCMD} --label=ch-system-core --timeout=60m make -C sdks/wasm run-ch-system-core
209+
${TESTCMD} --label=v8-system-core --timeout=60m make -C sdks/wasm run-v8-system-core
210+
${TESTCMD} --label=sm-system-core --timeout=60m make -C sdks/wasm run-sm-system-core
211+
${TESTCMD} --label=jsc-system-core --timeout=60m make -C sdks/wasm run-jsc-system-core
212+
${TESTCMD} --label=aot-mini --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm run-aot-mini
213+
${TESTCMD} --label=build-aot-all --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm build-aot-all
214+
for suite in ${aot_test_suites}; do ${TESTCMD} --label=run-aot-${suite} --timeout=10m make -C sdks/wasm run-aot-${suite}; done
215+
#${TESTCMD} --label=check-aot --timeout=60m make -C sdks/wasm check-aot
216+
${TESTCMD} --label=package --timeout=60m make -C sdks/wasm package
217+
fi
211218
exit 0
212219
fi
213220

‎sdks/builds/Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,11 +90,11 @@ endif
9090
# $(1): target (android, ios, wasm)
9191
define ArchiveTemplate
9292
_$(1)_HASH = $$(shell git -C $$(TOP) rev-parse HEAD)
93-
_$(1)_PACKAGE = $(1)-$$(CONFIGURATION)-$$(_$(1)_HASH)-$$(UNAME).tar.gz
93+
_$(1)_PACKAGE = $(1)-$$(CONFIGURATION)-$$(UNAME)-$$(_$(1)_HASH).zip
9494

9595
.PHONY: archive-$(1)
9696
archive-$(1): $$(foreach target,$$(filter $(1)-%,$$(patsubst %-$$(CONFIGURATION),%,$$($(1)_TARGETS))),package-$$(target))
97-
tar -cvzf $$(TOP)/$$(_$(1)_PACKAGE) -C $$(TOP)/sdks/out $$(sort $$(foreach target,$$($(1)_TARGETS),$$(or $$(wildcard $$(target)-$$(CONFIGURATION)),$$(target))))
97+
cd $$(TOP)/sdks/out && 7z a $$(TOP)/$$(_$(1)_PACKAGE) $$(sort $$(foreach target,$$($(1)_TARGETS),$$(or $$(wildcard $$(target)-$$(CONFIGURATION)),$$(target))))
9898
endef
9999

100100
ifndef DISABLE_ANDROID

0 commit comments

Comments
 (0)