[build] Xamarin.Android build preparation utility (#3051)

Fixes: #2562

The `make prepare` **make** target and `msbuild /t:Prepare` MSBuild
target are used to "prepare" a `xamarin-android` checkout so that
the subsequent `make all` or `msbuild /t:Build` invocation will work.

The preparation step involves ensuring that all known required
dependencies exist or are provisioned (54a7a02), such as the build
utilities (`autoconf`/etc.), cross-compilers (MinGW), the Android SDK
and NDK, and building "enough" to allow `Xamarin.Android.sln` to
build (see also 7343965).

Unfortunately, the Glorious Idea™ of using MSBuild for significant
portions of this process is "baroque", making it difficult to
understand and maintain, and the requirement on `msbuild` means
that things break in really bizarre and obscure ways on Linux and
macOS if the installed `mono` version is "too old"...and the
determination of "too old" was being done via `mono` & `msbuild`!

Furthermore, there was duplication and repetition of various bits of
build configuration on Linux and macOS between Makefile and the
MSBuild project system, which made it easier to make mistakes.
(Then there's the occasional bizarro failures that would result from
attempting to evaluate `$(PRODUCT_VERSION)` within `make`; it would
fail when `MONO_LOG_LEVEL=info` was set.  Then there's...)

Instead of having `make prepare` be implemented as an unholy mess of
`make` and `msbuild`, introduce the new `build-tools/xaprepare` app.
`xaprepare` will be built and executed as part of `make prepare`,
and takes over responsibility for repo build preparation.

See also: build-tools/xaprepare/README.md

TODO: Windows support

Author: grendello

CODEOWNERS

@@ -20,6 +20,7 @@
 /build-tools/manifest-attribute-codegen @jonpryor
 /build-tools/scripts/PrepareWindows.targets @jonathanpeppers
 /build-tools/timing @jonathanpeppers @radekdoulik
+/build-tools/xaprepare @grendello
 
 /src/OpenTK-1.0 @radekdoulik @jonpryor
 /src/Mono.Android.Export @jonpryor

Configuration.props

@@ -38,12 +38,17 @@
     <AndroidUseLatestPlatformSdk Condition=" '$(AndroidFrameworkVersion)' == '' And '$(_IsRunningNuGetRestore)' == 'True' ">True</AndroidUseLatestPlatformSdk>
     <DebugType Condition=" '$(DebugType)' == '' ">portable</DebugType>
   </PropertyGroup>
+  <PropertyGroup Condition=" '$(HostOS)' == '' ">
+    <HostOS Condition="$([MSBuild]::IsOSPlatform('windows'))">Windows</HostOS>
+    <HostOS Condition="$([MSBuild]::IsOSPlatform('linux'))">Linux</HostOS>
+    <HostOS Condition="$([MSBuild]::IsOSPlatform('osx'))">Darwin</HostOS>
+  </PropertyGroup>
   <PropertyGroup>
     <AutoProvision Condition=" '$(AutoProvision)' == '' ">False</AutoProvision>
     <AutoProvisionUsesSudo Condition=" '$(AutoProvisionUsesSudo)' == '' ">False</AutoProvisionUsesSudo>
-    <XAInstallPrefix Condition=" '$(XAInstallPrefix)' == '' ">$(MSBuildThisFileDirectory)bin\$(Configuration)\lib\xamarin.android\</XAInstallPrefix>
+    <_XABinRelativeInstallPrefix>lib\xamarin.android</_XABinRelativeInstallPrefix>
+    <XAInstallPrefix Condition=" '$(XAInstallPrefix)' == '' ">$(MSBuildThisFileDirectory)bin\$(Configuration)\$(_XABinRelativeInstallPrefix)\</XAInstallPrefix>
     <MingwDependenciesRootDirectory Condition=" '$(MingwDependenciesRootDirectory)' == '' ">$(MSBuildThisFileDirectory)\bin\Build$(Configuration)\mingw-deps</MingwDependenciesRootDirectory>
-    <HostOS Condition=" '$(HostOS)' == '' And '$(OS)' == 'Windows_NT' ">Windows</HostOS>
     <HostCc Condition=" '$(HostCc)' == '' ">$(HostCc64)</HostCc>
     <HostCxx Condition=" '$(HostCxx)' == '' ">$(HostCxx64)</HostCxx>
     <HostCc Condition=" '$(HostCc)' == '' ">$(HostCc32)</HostCc>
@@ -171,7 +176,8 @@
     <AndroidSupportedTargetAotAbisSplit>$(AndroidSupportedTargetAotAbis.Split(':'))</AndroidSupportedTargetAotAbisSplit>
   </PropertyGroup>
   <PropertyGroup>
-    <RemapAssemblyRefTool>$(ManagedRuntime) $(ManagedRuntimeArgs) &quot;$(MSBuildThisFileDirectory)bin\Build$(Configuration)\remap-assembly-ref.exe&quot;</RemapAssemblyRefTool>
+    <RemapAssemblyRefToolExecutable>$(MSBuildThisFileDirectory)bin\Build$(Configuration)\remap-assembly-ref.exe</RemapAssemblyRefToolExecutable>
+    <RemapAssemblyRefTool>$(ManagedRuntime) $(ManagedRuntimeArgs) &quot;$(RemapAssemblyRefToolExecutable)&quot;</RemapAssemblyRefTool>
   </PropertyGroup>
 
   <!-- Unit Test Properties -->

Makefile

@@ -1,30 +1,78 @@
-export OS_NAME       := $(shell uname)
-export OS_ARCH       := $(shell uname -m)
-export NO_SUDO       ?= false
 V             ?= 0
 prefix         = /usr/local
 CONFIGURATION  = Debug
 RUNTIME       := $(shell which mono64 2> /dev/null && echo mono64 || echo mono) --debug=casts
 SOLUTION       = Xamarin.Android.sln
 TEST_TARGETS   = build-tools/scripts/RunTests.targets
 API_LEVEL     ?=
+PREPARE_ARGS =
+PREPARE_BUILD_LOG = bin/Build$(CONFIGURATION)/bootstrap-build.binlog
+PREPARE_RESTORE_LOG = bin/Build$(CONFIGURATION)/bootstrap-restore.binlog
+PREPARE_SOURCE_DIR = build-tools/xaprepare
+PREPARE_SOLUTION = $(PREPARE_SOURCE_DIR)/xaprepare.sln
+PREPARE_EXE = $(PREPARE_SOURCE_DIR)/xaprepare/bin/$(CONFIGURATION)/xaprepare.exe
+PREPARE_COMMON_MSBUILD_FLAGS = /p:Configuration=$(CONFIGURATION) $(PREPARE_MSBUILD_ARGS) $(MSBUILD_ARGS)
+PREPARE_MSBUILD_FLAGS = /binaryLogger:"$(PREPARE_BUILD_LOG)" $(PREPARE_COMMON_MSBUILD_FLAGS)
+PREPARE_RESTORE_FLAGS = /binaryLogger:"$(PREPARE_RESTORE_LOG)" $(PREPARE_COMMON_MSBUILD_FLAGS)
+PREPARE_SCENARIO =
+PREPARE_CI ?= 0
+PREPARE_AUTOPROVISION ?= 0
+PREPARE_IGNORE_MONO_VERSION ?= 1
+
+_PREPARE_CI_MODE_ARGS = --no-emoji --run-mode=CI -a
+_PREPARE_ARGS =
+
+all:
+	$(call MSBUILD_BINLOG,all,$(_SLN_BUILD)) $(MSBUILD_FLAGS) $(SOLUTION)
+
+-include bin/Build$(CONFIGURATION)/rules.mk
+
+ifeq ($(OS_NAME),)
+export OS_NAME       := $(shell uname)
+endif
+
+ifeq ($(OS_ARCH),)
+export OS_ARCH       := $(shell uname -m)
+endif
+
+export NO_SUDO       ?= false
+
+ifneq ($(NO_SUDO),false)
+_PREPARE_ARGS += --auto-provisioning-uses-sudo=false
+endif
+
+ifneq ($(V),0)
+MONO_OPTIONS   += --debug
+NUGET_VERBOSITY = -Verbosity Detailed
+_PREPARE_ARGS += -v:d
+endif
+
+ifneq ($(PREPARE_CI),0)
+_PREPARE_ARGS += $(_PREPARE_CI_MODE_ARGS)
+endif
+
+ifneq ($(PREPARE_AUTOPROVISION),0)
+_PREPARE_ARGS += --auto-provision=yes --auto-provision-uses-sudo=yes
+endif
 
 ifeq ($(OS_NAME),Darwin)
-export MACOSX_DEPLOYMENT_TARGET := 10.11
+ifeq ($(HOMEBREW_PREFIX),)
 HOMEBREW_PREFIX ?= $(shell brew --prefix)
+endif
 else
 HOMEBREW_PREFIX := $prefix
 endif
 
-ifneq ($(V),0)
-MONO_OPTIONS   += --debug
-NUGET_VERBOSITY = -Verbosity Detailed
+ifeq ($(wildcard Configuration.OperatingSystem.props),)
+PREPARE_MSBUILD_FLAGS += "/p:HostHomebrewPrefix=$(HOMEBREW_PREFIX)"
 endif
 
-ifneq ($(MONO_OPTIONS),)
-export MONO_OPTIONS
+ifneq ($(PREPARE_SCENARIO),)
+_PREPARE_ARGS += -s:"$(PREPARE_SCENARIO)"
 endif
 
+_PREPARE_ARGS += $(PREPARE_ARGS)
+
 include build-tools/scripts/msbuild.mk
 
 ifeq ($(USE_MSBUILD),1)
@@ -34,12 +82,9 @@ _SLN_BUILD  = MSBUILD="$(MSBUILD)" tools/scripts/xabuild
 endif   # $(USE_MSBUILD) == 1
 
 ifneq ($(API_LEVEL),)
-MSBUILD_FLAGS += /p:AndroidApiLevel=$(API_LEVEL) /p:AndroidFrameworkVersion=$(word $(API_LEVEL), $(ALL_FRAMEWORKS))
+MSBUILD_FLAGS += /p:AndroidApiLevel=$(API_LEVEL) /p:AndroidFrameworkVersion=$(word $(API_LEVEL), $(ALL_FRAMEWORKS)) /p:AndroidPlatformId=$(word $(a), $(ALL_PLATFORM_IDS))
 endif
 
-all::
-	$(call MSBUILD_BINLOG,all,$(_SLN_BUILD)) $(MSBUILD_FLAGS) $(SOLUTION)
-
 all-tests::
 	MSBUILD="$(MSBUILD)" $(call MSBUILD_BINLOG,all-tests,tools/scripts/xabuild) $(MSBUILD_FLAGS) Xamarin.Android-Tests.sln
 
@@ -70,87 +115,6 @@ uninstall::
 	rm -rf "$(prefix)/lib/mono/xbuild/Xamarin/Android"
 	rm -rf "$(prefix)/lib/mono/xbuild-frameworks/MonoAndroid"
 
-ifeq ($(OS_NAME),Linux)
-export LINUX_DISTRO         := $(shell lsb_release -i -s || true)
-export LINUX_DISTRO_RELEASE := $(shell lsb_release -r -s || true)
-prepare:: linux-prepare
-endif # $(OS_NAME)=Linux
-
-prepare:: prepare-paths prepare-msbuild
-
-linux-prepare::
-	BINFMT_MISC_TROUBLE="cli win" \
-	BINFMT_WARN=no ; \
-	for m in ${BINFMT_MISC_TROUBLE}; do \
-		if [ -f "/proc/sys/fs/binfmt_misc/$$m" ]; then \
-			BINFMT_WARN=yes ; \
-		fi ; \
-	done ; \
-	if [ "x${BINFMT_WARN}" = "xyes" ]; then \
-		cat Documentation/binfmt_misc-warning-Linux.txt ; \
-	fi; \
-	if [ -f build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO)-$(LINUX_DISTRO_RELEASE).sh ]; then \
-		sh build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO)-$(LINUX_DISTRO_RELEASE).sh $(LINUX_DISTRO_RELEASE); \
-	elif [ -f build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO).sh ]; then \
-		sh build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO).sh $(LINUX_DISTRO_RELEASE); \
-	fi
-
-# $(call GetPath,path)
-GetPath   = $(shell $(MSBUILD) $(MSBUILD_FLAGS) /p:DoNotLoadOSProperties=True /nologo /v:minimal /t:Get$(1)FullPath build-tools/scripts/Paths.targets | tr -d '[[:space:]]' )
-
-MSBUILD_PREPARE_PROJS = \
-	src/mono-runtimes/mono-runtimes.csproj \
-	src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj
-
-prepare-deps: prepare-cmake-mingw-toolchain
-	git submodule update --init --recursive
-	./build-tools/scripts/generate-os-info Configuration.OperatingSystem.props
-	mkdir -p bin/Build$(CONFIGURATION)
-	$(call MSBUILD_BINLOG,prepare-deps) build-tools/dependencies/dependencies.csproj
-
-#
-# $(1): output file name
-#
-define create_cmake_toolchain
-prepare-cmake-mingw-toolchain:: bin/Build$(CONFIGURATION)/$(1)
-
-bin/Build$(CONFIGURATION)/$(1): build-tools/scripts/$(1).in
-	mkdir -p $$(dir $$@)
-	sed -e 's;@HOMEBREW_PREFIX@;$$(HOMEBREW_PREFIX);g' < $$< > $$@
-endef
-
-$(eval $(call create_cmake_toolchain,mingw-32.cmake))
-$(eval $(call create_cmake_toolchain,mingw-64.cmake))
-
-prepare-external: prepare-deps
-	nuget restore $(NUGET_VERBOSITY) $(SOLUTION)
-	nuget restore $(NUGET_VERBOSITY) Xamarin.Android-Tests.sln
-	(cd external/xamarin-android-tools && make prepare CONFIGURATION=$(CONFIGURATION))
-	(cd $(call GetPath,JavaInterop) && make prepare CONFIGURATION=$(CONFIGURATION) JI_MAX_JDK=8)
-	(cd $(call GetPath,JavaInterop) && make bin/Build$(CONFIGURATION)/JdkInfo.props CONFIGURATION=$(CONFIGURATION) JI_MAX_JDK=8)
-
-prepare-bundle: prepare-external
-	$(call MSBUILD_BINLOG,prepare-bundle) build-tools/download-bundle/download-bundle.csproj
-	$(call MSBUILD_BINLOG,prepare-restore) $(MSBUILD_FLAGS) tests/Xamarin.Forms-Performance-Integration/Xamarin.Forms.Performance.Integration.csproj /t:Restore
-
-prepare-props: prepare-bundle
-	cp $(call GetPath,JavaInterop)/external/Mono.Cecil* "$(call GetPath,MonoSource)/external"
-	cp "$(call GetPath,JavaInterop)/product.snk" "$(call GetPath,MonoSource)"
-	cp build-tools/scripts/Configuration.Java.Interop.Override.props external/Java.Interop/Configuration.Override.props
-	cp $(call GetPath,MonoSource)/mcs/class/msfinal.pub .
-
-prepare-msbuild: prepare-props
-ifeq ($(USE_MSBUILD),1)
-	for proj in $(MSBUILD_PREPARE_PROJS); do \
-		$(call MSBUILD_BINLOG,prepare-msbuild) "$$proj" || exit 1; \
-	done
-endif	# msbuild
-
-prepare-image-dependencies:
-	$(call MSBUILD_BINLOG,prepare-image-deps) build-tools/scripts/PrepareImageDependencies.targets /t:PrepareImageDependencies \
-		/p:AndroidSupportedHostJitAbis=mxe-Win32:mxe-Win64
-	cat bin/Build$(CONFIGURATION)/prepare-image-dependencies.sh | tr -d '\r' > prepare-image-dependencies.sh
-
 include build-tools/scripts/BuildEverything.mk
 
 # Must be after BuildEverything.mk - it uses variables defined there
@@ -159,59 +123,10 @@ include tests/api-compatibility/api-compatibility.mk
 
 topdir  := $(shell pwd)
 
-
-XA_BUILD_PATHS_OUT = bin/Test$(CONFIGURATION)/XABuildPaths.cs
-
-prepare-paths: $(XA_BUILD_PATHS_OUT)
-
-$(XA_BUILD_PATHS_OUT): bin/Test%/XABuildPaths.cs: build-tools/scripts/XABuildPaths.cs.in
-	mkdir -p $(shell dirname $@)
-	sed -e 's;@CONFIGURATION@;$*;g' \
-	    -e 's;@TOP_DIRECTORY@;$(topdir);g' < $< > $@
-	cat $@
-
-
-# Usage: $(call CALL_CREATE_THIRD_PARTY_NOTICES,path,licenseType,includeExternalDeps,includeBuildDeps)
-define CREATE_THIRD_PARTY_NOTICES
-	$(call MSBUILD_BINLOG,create-tpn,$(MSBUILD)) $(_MSBUILD_ARGS) \
-		$(topdir)/build-tools/ThirdPartyNotices/ThirdPartyNotices.csproj \
-		/p:Configuration=$(CONFIGURATION) \
-		/p:ThirdPartyNoticeFile=$(topdir)/$(1) \
-		/p:ThirdPartyNoticeLicenseType=$(2) \
-		/p:TpnIncludeExternalDependencies=$(3) \
-		/p:TpnIncludeBuildDependencies=$(4)
-endef # CREATE_THIRD_PARTY_NOTICES
-
-prepare:: prepare-tpn
-
-TPN_LICENSE_FILES = $(shell grep -h '<LicenseFile>' external/*.tpnitems src/*.tpnitems \
-	| sed -E 's,<LicenseFile>(.*)</LicenseFile>,\1,g;s,.\(MSBuildThisFileDirectory\),$(topdir)/external/,g' \
-	| tr \\ / )
-
-# Usage: $(call CREATE_THIRD_PARTY_NOTICES_RULE,path,licenseType,includeExternalDeps,includeBuildDeps)
-define CREATE_THIRD_PARTY_NOTICES_RULE
-prepare-tpn:: $(1)
-
-$(1) $(topdir)/$(1): build-tools/ThirdPartyNotices/ThirdPartyNotices.csproj \
-		$(wildcard external/*.tpnitems src/*.tpnitems build-tools/*.tpnitems) \
-		$(TPN_LICENSE_FILES)
-	$(call CREATE_THIRD_PARTY_NOTICES,$(1),$(2),$(3),$(4))
-endef # CREATE_THIRD_PARTY_NOTICES_RULE
-
-THIRD_PARTY_NOTICE_LICENSE_TYPE = microsoft-oss
-
-$(eval $(call CREATE_THIRD_PARTY_NOTICES_RULE,ThirdPartyNotices.txt,foundation,False,False))
-$(eval $(call CREATE_THIRD_PARTY_NOTICES_RULE,bin/$(CONFIGURATION)/lib/xamarin.android/ThirdPartyNotices.txt,$(THIRD_PARTY_NOTICE_LICENSE_TYPE),True,False))
-
 # Used by External XA Build
 EXTERNAL_XA_PATH=$(topdir)
 EXTERNAL_GIT_PATH=$(topdir)/external
 
-prepare-external-git-dependencies:
-	msbuild build-tools/xa-prep-tasks/xa-prep-tasks.csproj /p:Configuration=$(CONFIGURATION)
-	msbuild build-tools/xa-prep-tasks/xa-prep-tasks.csproj /p:Configuration=$(CONFIGURATION) \
-		/t:CheckoutExternalGitSources /p:ExternalSourceDependencyDirectory='$(EXTERNAL_GIT_PATH)'
-
 -include $(EXTERNAL_GIT_PATH)/monodroid/xa-integration.mk
 
 run-all-tests:
@@ -257,3 +172,68 @@ list-nunit-tests:
 	$(MSBUILD) $(MSBUILD_FLAGS) $(TEST_TARGETS) /t:ListNUnitTests
 
 include build-tools/scripts/runtime-helpers.mk
+
+.PHONY: prepare-build-init
+prepare-build-init:
+	mkdir -p $(dir $(PREPARE_BUILD_LOG))
+	msbuild $(PREPARE_RESTORE_FLAGS) $(PREPARE_SOLUTION) /t:Restore
+
+.PHONY: prepare-build
+prepare-build: prepare-build-init
+	msbuild $(PREPARE_MSBUILD_FLAGS) $(PREPARE_SOLUTION)
+
+.PHONY: prepare-build-ci
+prepare-build-ci: prepare-build-init
+	msbuild $(PREPARE_MSBUILD_FLAGS) $(PREPARE_SOLUTION) $(_MSBUILD_ARGS)
+
+.PHONY: prepare
+prepare:: prepare-build
+	mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS)
+
+.PHONY: prepare-help
+prepare-help: prepare-build
+	mono --debug $(PREPARE_EXE) -h
+
+# Hack: The current commercial pipeline doesn't pass all the required arguments when preparing the build, in particular it doesn't override the
+# ABI targets to build and so the prepare step configures only for the default set (armeabi-v7a, arm64-v8a, x86, $HOST_OS) which is not enough.
+# The `jenkins` rule in `BuildEverything.mk`, invoked by the commercial pipeline, now calls the rule below in which we rebuild the bootstrapper
+# with all the required properties set to include all the ABIs - it should fix the build. After the PR is merged, the commercial pipeline should
+# be modified to do the right thing instead.
+#
+# Commercial pipeline should also set PREPARE_CI=1 when calling targets. Since this is currently not done, we have to pass $(_PREPARE_CI_MODE_ARGS)
+# directly below
+#
+.PHONY: prepare-jenkins
+prepare-jenkins: prepare-build-ci prepare-commercial
+	@echo preparing jenkins build
+	mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS)
+
+# This should go away once we can modify the commercial pipeline for the bootstrapper
+.PHONY: prepare-commercial
+ifeq ($(USE_COMMERCIAL_INSTALLER_NAME),true)
+prepare-commercial:
+	cd $(TOP) && ./configure --with-xamarin-android='$(XAMARIN_ANDROID_PATH)'
+	mkdir -p $(XA_MSBUILD_DIR)
+
+else
+prepare-commercial:
+endif
+
+.PHONY: prepare-update-mono
+
+prepare-update-mono: prepare-build-ci
+	mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS) /s:UpdateMono
+
+# These targets exist only temporarily to satisfy requirements of the commercial build (since we can't modify the pipeline script in this PR)
+.PHONY: prepare-deps
+
+# Commercial pipeline installs an older version of Mono and in effect we fail. `prepare-deps` is called after provisionator is ran and so we
+# can, temporarily, re-update Mono here. After the PR is merged and commercial pipeline updated, this step should be removed.
+prepare-deps: prepare-update-mono
+	@echo prepare-deps is no-op, prepare-jenkins or prepare do the work instead
+
+prepare-image-dependencies: prepare-build-ci
+	mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS) -s:PrepareImageDependencies
+
+prepare-external-git-dependencies: prepare-build-ci prepare-update-mono
+	mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS) -s:PrepareExternalGitDependencies