Skip to content

Commit 6a3d4e3

Browse files
grendellogrendello
committed
Xamarin.Android build preparation utility
Testing: * Default mode: run `make prepare` * Verbose debug: run `make V=1 prepare` * CI mode: run `make PREPARE_CI=1 prepare` * Help: run `make prepare-help` * Logs: `bin/BuildDebug` Supported operating systems: * macOS * Linux Windows support is planned, present but not fully implemented or tested. This commit implements a `make prepare` replacement that does not rely on shell scripts or MSBuild. The idea is that a standalone C# program can perform all the steps in a more streamlined, more clear way than the current solution. One of the main goals is to make the process more approachable by external contributors, but also by the core developers who may want to change some aspects of XA build preparation (e.g. Android platforms, Mono version etc) but are not familiar with the build system. Towards that goal, the codebase is designed so that the minimum amount of searching around it is necessary to figure out where to make the desired change. Main location serving this purpose is the `build-tools/xaprepare/xaprepare/ConfigAndData` directory. It should be the *only* location where one needs to look in order to change all and any aspects of XA preparation. [TBC]
1 parent 5f705f2 commit 6a3d4e3

File tree

260 files changed

+17515
-3485
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

260 files changed

+17515
-3485
lines changed

‎CODEOWNERS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
/build-tools/manifest-attribute-codegen @jonpryor
2121
/build-tools/scripts/PrepareWindows.targets @jonathanpeppers
2222
/build-tools/timing @jonathanpeppers @radekdoulik
23+
/build-tools/xaprepare @grendello
2324

2425
/src/OpenTK-1.0 @radekdoulik @jonpryor
2526
/src/Mono.Android.Export @jonpryor

‎Configuration.props‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,17 @@
3838
<AndroidUseLatestPlatformSdk Condition=" '$(AndroidFrameworkVersion)' == '' And '$(_IsRunningNuGetRestore)' == 'True' ">True</AndroidUseLatestPlatformSdk>
3939
<DebugType Condition=" '$(DebugType)' == '' ">portable</DebugType>
4040
</PropertyGroup>
41+
<PropertyGroup Condition=" '$(HostOS)' == '' ">
42+
<HostOS Condition="$([MSBuild]::IsOSPlatform('windows'))">Windows</HostOS>
43+
<HostOS Condition="$([MSBuild]::IsOSPlatform('linux'))">Linux</HostOS>
44+
<HostOS Condition="$([MSBuild]::IsOSPlatform('osx'))">Darwin</HostOS>
45+
</PropertyGroup>
4146
<PropertyGroup>
4247
<AutoProvision Condition=" '$(AutoProvision)' == '' ">False</AutoProvision>
4348
<AutoProvisionUsesSudo Condition=" '$(AutoProvisionUsesSudo)' == '' ">False</AutoProvisionUsesSudo>
44-
<XAInstallPrefix Condition=" '$(XAInstallPrefix)' == '' ">$(MSBuildThisFileDirectory)bin\$(Configuration)\lib\xamarin.android\</XAInstallPrefix>
49+
<_XABinRelativeInstallPrefix>lib\xamarin.android</_XABinRelativeInstallPrefix>
50+
<XAInstallPrefix Condition=" '$(XAInstallPrefix)' == '' ">$(MSBuildThisFileDirectory)bin\$(Configuration)\$(_XABinRelativeInstallPrefix)\</XAInstallPrefix>
4551
<MingwDependenciesRootDirectory Condition=" '$(MingwDependenciesRootDirectory)' == '' ">$(MSBuildThisFileDirectory)\bin\Build$(Configuration)\mingw-deps</MingwDependenciesRootDirectory>
46-
<HostOS Condition=" '$(HostOS)' == '' And '$(OS)' == 'Windows_NT' ">Windows</HostOS>
4752
<HostCc Condition=" '$(HostCc)' == '' ">$(HostCc64)</HostCc>
4853
<HostCxx Condition=" '$(HostCxx)' == '' ">$(HostCxx64)</HostCxx>
4954
<HostCc Condition=" '$(HostCc)' == '' ">$(HostCc32)</HostCc>
@@ -171,7 +176,8 @@
171176
<AndroidSupportedTargetAotAbisSplit>$(AndroidSupportedTargetAotAbis.Split(':'))</AndroidSupportedTargetAotAbisSplit>
172177
</PropertyGroup>
173178
<PropertyGroup>
174-
<RemapAssemblyRefTool>$(ManagedRuntime) $(ManagedRuntimeArgs) &quot;$(MSBuildThisFileDirectory)bin\Build$(Configuration)\remap-assembly-ref.exe&quot;</RemapAssemblyRefTool>
179+
<RemapAssemblyRefToolExecutable>$(MSBuildThisFileDirectory)bin\Build$(Configuration)\remap-assembly-ref.exe</RemapAssemblyRefToolExecutable>
180+
<RemapAssemblyRefTool>$(ManagedRuntime) $(ManagedRuntimeArgs) &quot;$(RemapAssemblyRefToolExecutable)&quot;</RemapAssemblyRefTool>
175181
</PropertyGroup>
176182

177183
<!-- Unit Test Properties -->

‎Makefile‎

Lines changed: 120 additions & 143 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,73 @@
1-
export OS_NAME := $(shell uname)
2-
export OS_ARCH := $(shell uname -m)
3-
export NO_SUDO ?= false
41
V ?= 0
52
prefix = /usr/local
63
CONFIGURATION = Debug
74
RUNTIME := $(shell which mono64 2> /dev/null && echo mono64 || echo mono) --debug=casts
85
SOLUTION = Xamarin.Android.sln
96
TEST_TARGETS = build-tools/scripts/RunTests.targets
107
API_LEVEL ?=
8+
PREPARE_ARGS =
9+
PREPARE_BUILD_LOG = bin/Build$(CONFIGURATION)/bootstrap-build.binlog
10+
PREPARE_RESTORE_LOG = bin/Build$(CONFIGURATION)/bootstrap-restore.binlog
11+
PREPARE_SOURCE_DIR = build-tools/xaprepare
12+
PREPARE_SOLUTION = $(PREPARE_SOURCE_DIR)/xaprepare.sln
13+
PREPARE_EXE = $(PREPARE_SOURCE_DIR)/xaprepare/bin/$(CONFIGURATION)/xaprepare.exe
14+
PREPARE_COMMON_MSBUILD_FLAGS = /p:Configuration=$(CONFIGURATION) $(PREPARE_MSBUILD_ARGS) $(MSBUILD_ARGS)
15+
PREPARE_MSBUILD_FLAGS = /binaryLogger:"$(PREPARE_BUILD_LOG)" $(PREPARE_COMMON_MSBUILD_FLAGS)
16+
PREPARE_RESTORE_FLAGS = /binaryLogger:"$(PREPARE_RESTORE_LOG)" $(PREPARE_COMMON_MSBUILD_FLAGS)
17+
PREPARE_SCENARIO =
18+
PREPARE_CI ?= 0
19+
PREPARE_AUTOPROVISION ?= 0
20+
PREPARE_IGNORE_MONO_VERSION ?= 1
21+
22+
_PREPARE_CI_MODE_ARGS = --no-emoji --run-mode=CI -a
23+
24+
all:
25+
$(call MSBUILD_BINLOG,all,$(_SLN_BUILD)) $(MSBUILD_FLAGS) $(SOLUTION)
26+
27+
-include bin/Build$(CONFIGURATION)/rules.mk
28+
29+
ifeq ($(OS_NAME),)
30+
export OS_NAME := $(shell uname)
31+
endif
32+
33+
ifeq ($(OS_ARCH),)
34+
export OS_ARCH := $(shell uname -m)
35+
endif
36+
37+
export NO_SUDO ?= false
38+
39+
ifneq ($(NO_SUDO),false)
40+
PREPARE_ARGS += --auto-provisioning-uses-sudo=false
41+
endif
42+
43+
ifneq ($(V),0)
44+
MONO_OPTIONS += --debug
45+
NUGET_VERBOSITY = -Verbosity Detailed
46+
PREPARE_ARGS += -v:d
47+
endif
48+
49+
ifneq ($(PREPARE_CI),0)
50+
PREPARE_ARGS += $(_PREPARE_CI_MODE_ARGS)
51+
endif
52+
53+
ifneq ($(PREPARE_AUTOPROVISION),0)
54+
PREPARE_ARGS += --auto-provision=yes --auto-provision-uses-sudo=yes
55+
endif
1156

1257
ifeq ($(OS_NAME),Darwin)
13-
export MACOSX_DEPLOYMENT_TARGET := 10.11
58+
ifeq ($(HOMEBREW_PREFIX),)
1459
HOMEBREW_PREFIX ?= $(shell brew --prefix)
60+
endif
1561
else
1662
HOMEBREW_PREFIX := $prefix
1763
endif
1864

19-
ifneq ($(V),0)
20-
MONO_OPTIONS += --debug
21-
NUGET_VERBOSITY = -Verbosity Detailed
65+
ifeq ($(wildcard Configuration.OperatingSystem.props),)
66+
PREPARE_MSBUILD_FLAGS += "/p:HostHomebrewPrefix=$(HOMEBREW_PREFIX)"
2267
endif
2368

24-
ifneq ($(MONO_OPTIONS),)
25-
export MONO_OPTIONS
69+
ifneq ($(PREPARE_SCENARIO),)
70+
PREPARE_ARGS += -s:"$(PREPARE_SCENARIO)"
2671
endif
2772

2873
include build-tools/scripts/msbuild.mk
@@ -34,12 +79,9 @@ _SLN_BUILD = MSBUILD="$(MSBUILD)" tools/scripts/xabuild
3479
endif # $(USE_MSBUILD) == 1
3580

3681
ifneq ($(API_LEVEL),)
37-
MSBUILD_FLAGS += /p:AndroidApiLevel=$(API_LEVEL) /p:AndroidFrameworkVersion=$(word $(API_LEVEL), $(ALL_FRAMEWORKS))
82+
MSBUILD_FLAGS += /p:AndroidApiLevel=$(API_LEVEL) /p:AndroidFrameworkVersion=$(word $(API_LEVEL), $(ALL_FRAMEWORKS)) /p:AndroidPlatformId=$(word $(a), $(ALL_PLATFORM_IDS))
3883
endif
3984

40-
all::
41-
$(call MSBUILD_BINLOG,all,$(_SLN_BUILD)) $(MSBUILD_FLAGS) $(SOLUTION)
42-
4385
all-tests::
4486
MSBUILD="$(MSBUILD)" $(call MSBUILD_BINLOG,all-tests,tools/scripts/xabuild) $(MSBUILD_FLAGS) Xamarin.Android-Tests.sln
4587

@@ -70,87 +112,6 @@ uninstall::
70112
rm -rf "$(prefix)/lib/mono/xbuild/Xamarin/Android"
71113
rm -rf "$(prefix)/lib/mono/xbuild-frameworks/MonoAndroid"
72114

73-
ifeq ($(OS_NAME),Linux)
74-
export LINUX_DISTRO := $(shell lsb_release -i -s || true)
75-
export LINUX_DISTRO_RELEASE := $(shell lsb_release -r -s || true)
76-
prepare:: linux-prepare
77-
endif # $(OS_NAME)=Linux
78-
79-
prepare:: prepare-paths prepare-msbuild
80-
81-
linux-prepare::
82-
BINFMT_MISC_TROUBLE="cli win" \
83-
BINFMT_WARN=no ; \
84-
for m in ${BINFMT_MISC_TROUBLE}; do \
85-
if [ -f "/proc/sys/fs/binfmt_misc/$$m" ]; then \
86-
BINFMT_WARN=yes ; \
87-
fi ; \
88-
done ; \
89-
if [ "x${BINFMT_WARN}" = "xyes" ]; then \
90-
cat Documentation/binfmt_misc-warning-Linux.txt ; \
91-
fi; \
92-
if [ -f build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO)-$(LINUX_DISTRO_RELEASE).sh ]; then \
93-
sh build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO)-$(LINUX_DISTRO_RELEASE).sh $(LINUX_DISTRO_RELEASE); \
94-
elif [ -f build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO).sh ]; then \
95-
sh build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO).sh $(LINUX_DISTRO_RELEASE); \
96-
fi
97-
98-
# $(call GetPath,path)
99-
GetPath = $(shell $(MSBUILD) $(MSBUILD_FLAGS) /p:DoNotLoadOSProperties=True /nologo /v:minimal /t:Get$(1)FullPath build-tools/scripts/Paths.targets | tr -d '[[:space:]]' )
100-
101-
MSBUILD_PREPARE_PROJS = \
102-
src/mono-runtimes/mono-runtimes.csproj \
103-
src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj
104-
105-
prepare-deps: prepare-cmake-mingw-toolchain
106-
git submodule update --init --recursive
107-
./build-tools/scripts/generate-os-info Configuration.OperatingSystem.props
108-
mkdir -p bin/Build$(CONFIGURATION)
109-
$(call MSBUILD_BINLOG,prepare-deps) build-tools/dependencies/dependencies.csproj
110-
111-
#
112-
# $(1): output file name
113-
#
114-
define create_cmake_toolchain
115-
prepare-cmake-mingw-toolchain:: bin/Build$(CONFIGURATION)/$(1)
116-
117-
bin/Build$(CONFIGURATION)/$(1): build-tools/scripts/$(1).in
118-
mkdir -p $$(dir $$@)
119-
sed -e 's;@HOMEBREW_PREFIX@;$$(HOMEBREW_PREFIX);g' < $$< > $$@
120-
endef
121-
122-
$(eval $(call create_cmake_toolchain,mingw-32.cmake))
123-
$(eval $(call create_cmake_toolchain,mingw-64.cmake))
124-
125-
prepare-external: prepare-deps
126-
nuget restore $(NUGET_VERBOSITY) $(SOLUTION)
127-
nuget restore $(NUGET_VERBOSITY) Xamarin.Android-Tests.sln
128-
(cd external/xamarin-android-tools && make prepare CONFIGURATION=$(CONFIGURATION))
129-
(cd $(call GetPath,JavaInterop) && make prepare CONFIGURATION=$(CONFIGURATION) JI_MAX_JDK=8)
130-
(cd $(call GetPath,JavaInterop) && make bin/Build$(CONFIGURATION)/JdkInfo.props CONFIGURATION=$(CONFIGURATION) JI_MAX_JDK=8)
131-
132-
prepare-bundle: prepare-external
133-
$(call MSBUILD_BINLOG,prepare-bundle) build-tools/download-bundle/download-bundle.csproj
134-
$(call MSBUILD_BINLOG,prepare-restore) $(MSBUILD_FLAGS) tests/Xamarin.Forms-Performance-Integration/Xamarin.Forms.Performance.Integration.csproj /t:Restore
135-
136-
prepare-props: prepare-bundle
137-
cp $(call GetPath,JavaInterop)/external/Mono.Cecil* "$(call GetPath,MonoSource)/external"
138-
cp "$(call GetPath,JavaInterop)/product.snk" "$(call GetPath,MonoSource)"
139-
cp build-tools/scripts/Configuration.Java.Interop.Override.props external/Java.Interop/Configuration.Override.props
140-
cp $(call GetPath,MonoSource)/mcs/class/msfinal.pub .
141-
142-
prepare-msbuild: prepare-props
143-
ifeq ($(USE_MSBUILD),1)
144-
for proj in $(MSBUILD_PREPARE_PROJS); do \
145-
$(call MSBUILD_BINLOG,prepare-msbuild) "$$proj" || exit 1; \
146-
done
147-
endif # msbuild
148-
149-
prepare-image-dependencies:
150-
$(call MSBUILD_BINLOG,prepare-image-deps) build-tools/scripts/PrepareImageDependencies.targets /t:PrepareImageDependencies \
151-
/p:AndroidSupportedHostJitAbis=mxe-Win32:mxe-Win64
152-
cat bin/Build$(CONFIGURATION)/prepare-image-dependencies.sh | tr -d '\r' > prepare-image-dependencies.sh
153-
154115
include build-tools/scripts/BuildEverything.mk
155116

156117
# Must be after BuildEverything.mk - it uses variables defined there
@@ -159,59 +120,10 @@ include tests/api-compatibility/api-compatibility.mk
159120

160121
topdir := $(shell pwd)
161122

162-
163-
XA_BUILD_PATHS_OUT = bin/Test$(CONFIGURATION)/XABuildPaths.cs
164-
165-
prepare-paths: $(XA_BUILD_PATHS_OUT)
166-
167-
$(XA_BUILD_PATHS_OUT): bin/Test%/XABuildPaths.cs: build-tools/scripts/XABuildPaths.cs.in
168-
mkdir -p $(shell dirname $@)
169-
sed -e 's;@CONFIGURATION@;$*;g' \
170-
-e 's;@TOP_DIRECTORY@;$(topdir);g' < $< > $@
171-
cat $@
172-
173-
174-
# Usage: $(call CALL_CREATE_THIRD_PARTY_NOTICES,path,licenseType,includeExternalDeps,includeBuildDeps)
175-
define CREATE_THIRD_PARTY_NOTICES
176-
$(call MSBUILD_BINLOG,create-tpn,$(MSBUILD)) $(_MSBUILD_ARGS) \
177-
$(topdir)/build-tools/ThirdPartyNotices/ThirdPartyNotices.csproj \
178-
/p:Configuration=$(CONFIGURATION) \
179-
/p:ThirdPartyNoticeFile=$(topdir)/$(1) \
180-
/p:ThirdPartyNoticeLicenseType=$(2) \
181-
/p:TpnIncludeExternalDependencies=$(3) \
182-
/p:TpnIncludeBuildDependencies=$(4)
183-
endef # CREATE_THIRD_PARTY_NOTICES
184-
185-
prepare:: prepare-tpn
186-
187-
TPN_LICENSE_FILES = $(shell grep -h '<LicenseFile>' external/*.tpnitems src/*.tpnitems \
188-
| sed -E 's,<LicenseFile>(.*)</LicenseFile>,\1,g;s,.\(MSBuildThisFileDirectory\),$(topdir)/external/,g' \
189-
| tr \\ / )
190-
191-
# Usage: $(call CREATE_THIRD_PARTY_NOTICES_RULE,path,licenseType,includeExternalDeps,includeBuildDeps)
192-
define CREATE_THIRD_PARTY_NOTICES_RULE
193-
prepare-tpn:: $(1)
194-
195-
$(1) $(topdir)/$(1): build-tools/ThirdPartyNotices/ThirdPartyNotices.csproj \
196-
$(wildcard external/*.tpnitems src/*.tpnitems build-tools/*.tpnitems) \
197-
$(TPN_LICENSE_FILES)
198-
$(call CREATE_THIRD_PARTY_NOTICES,$(1),$(2),$(3),$(4))
199-
endef # CREATE_THIRD_PARTY_NOTICES_RULE
200-
201-
THIRD_PARTY_NOTICE_LICENSE_TYPE = microsoft-oss
202-
203-
$(eval $(call CREATE_THIRD_PARTY_NOTICES_RULE,ThirdPartyNotices.txt,foundation,False,False))
204-
$(eval $(call CREATE_THIRD_PARTY_NOTICES_RULE,bin/$(CONFIGURATION)/lib/xamarin.android/ThirdPartyNotices.txt,$(THIRD_PARTY_NOTICE_LICENSE_TYPE),True,False))
205-
206123
# Used by External XA Build
207124
EXTERNAL_XA_PATH=$(topdir)
208125
EXTERNAL_GIT_PATH=$(topdir)/external
209126

210-
prepare-external-git-dependencies:
211-
msbuild build-tools/xa-prep-tasks/xa-prep-tasks.csproj /p:Configuration=$(CONFIGURATION)
212-
msbuild build-tools/xa-prep-tasks/xa-prep-tasks.csproj /p:Configuration=$(CONFIGURATION) \
213-
/t:CheckoutExternalGitSources /p:ExternalSourceDependencyDirectory='$(EXTERNAL_GIT_PATH)'
214-
215127
-include $(EXTERNAL_GIT_PATH)/monodroid/xa-integration.mk
216128

217129
run-all-tests:
@@ -257,3 +169,68 @@ list-nunit-tests:
257169
$(MSBUILD) $(MSBUILD_FLAGS) $(TEST_TARGETS) /t:ListNUnitTests
258170

259171
include build-tools/scripts/runtime-helpers.mk
172+
173+
.PHONY: prepare-build-init
174+
prepare-build-init:
175+
mkdir -p $(dir $(PREPARE_BUILD_LOG))
176+
msbuild $(PREPARE_RESTORE_FLAGS) $(PREPARE_SOLUTION) /t:Restore
177+
178+
.PHONY: prepare-build
179+
prepare-build: prepare-build-init
180+
msbuild $(PREPARE_MSBUILD_FLAGS) $(PREPARE_SOLUTION)
181+
182+
.PHONY: prepare-build-ci
183+
prepare-build-ci: prepare-build-init
184+
msbuild $(PREPARE_MSBUILD_FLAGS) $(PREPARE_SOLUTION) $(_MSBUILD_ARGS)
185+
186+
.PHONY: prepare
187+
prepare:: prepare-build
188+
mono --debug $(PREPARE_EXE) $(PREPARE_ARGS)
189+
190+
.PHONY: prepare-help
191+
prepare-help: prepare-build
192+
mono --debug $(PREPARE_EXE) -h
193+
194+
# Hack: The current commercial pipeline doesn't pass all the required arguments when preparing the build, in particular it doesn't override the
195+
# ABI targets to build and so the prepare step configures only for the default set (armeabi-v7a, arm64-v8a, x86, $HOST_OS) which is not enough.
196+
# The `jenkins` rule in `BuildEverything.mk`, invoked by the commercial pipeline, now calls the rule below in which we rebuild the bootstrapper
197+
# with all the required properties set to include all the ABIs - it should fix the build. After the PR is merged, the commercial pipeline should
198+
# be modified to do the right thing instead.
199+
#
200+
# Commercial pipeline should also set PREPARE_CI=1 when calling targets. Since this is currently not done, we have to pass $(_PREPARE_CI_MODE_ARGS)
201+
# directly below
202+
#
203+
.PHONY: prepare-jenkins
204+
prepare-jenkins: prepare-build-ci prepare-commercial
205+
@echo preparing jenkins build
206+
mono --debug $(PREPARE_EXE) $(PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS)
207+
208+
# This should go away once we can modify the commercial pipeline for the bootstrapper
209+
.PHONY: prepare-commercial
210+
ifeq ($(USE_COMMERCIAL_INSTALLER_NAME),true)
211+
prepare-commercial:
212+
cd $(TOP) && ./configure --with-xamarin-android='$(XAMARIN_ANDROID_PATH)'
213+
mkdir -p $(XA_MSBUILD_DIR)
214+
215+
else
216+
prepare-commercial:
217+
endif
218+
219+
.PHONY: prepare-update-mono
220+
221+
prepare-update-mono: prepare-build-ci
222+
mono --debug $(PREPARE_EXE) $(PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS) /s:UpdateMono
223+
224+
# These targets exist only temporarily to satisfy requirements of the commercial build (since we can't modify the pipeline script in this PR)
225+
.PHONY: prepare-deps
226+
227+
# Commercial pipeline installs an older version of Mono and in effect we fail. `prepare-deps` is called after provisionator is ran and so we
228+
# can, temporarily, re-update Mono here. After the PR is merged and commercial pipeline updated, this step should be removed.
229+
prepare-deps: prepare-update-mono
230+
@echo prepare-deps is no-op, prepare-jenkins or prepare do the work instead
231+
232+
prepare-image-dependencies: prepare-build-ci
233+
mono --debug $(PREPARE_EXE) $(PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS) -s:PrepareImageDependencies
234+
235+
prepare-external-git-dependencies: prepare-build-ci prepare-update-mono
236+
mono --debug $(PREPARE_EXE) $(PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS) -s:PrepareExternalGitDependencies

0 commit comments

Comments
 (0)