Skip to content

Xamarin.Android build preparation utility #3051

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 7, 2019
Merged

Xamarin.Android build preparation utility #3051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bce78fe
Xamarin.Android build preparation utility
grendello Mar 8, 2019
5217736
[xaprepare] Add AndroidToolchain scenario
pjcollins Jun 4, 2019
0f65bbd
[xaprepare] Update ExternalGitDependency usage (#2)
pjcollins Jun 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CODEOWNERS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
/build-tools/manifest-attribute-codegen @jonpryor
/build-tools/scripts/PrepareWindows.targets @jonathanpeppers
/build-tools/timing @jonathanpeppers @radekdoulik
/build-tools/xaprepare @grendello

/src/OpenTK-1.0 @radekdoulik @jonpryor
/src/Mono.Android.Export @jonpryor
Expand Down
12 changes: 9 additions & 3 deletions Configuration.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,17 @@
<AndroidUseLatestPlatformSdk Condition=" '$(AndroidFrameworkVersion)' == '' And '$(_IsRunningNuGetRestore)' == 'True' ">True</AndroidUseLatestPlatformSdk>
<DebugType Condition=" '$(DebugType)' == '' ">portable</DebugType>
</PropertyGroup>
<PropertyGroup Condition=" '$(HostOS)' == '' ">
<HostOS Condition="$([MSBuild]::IsOSPlatform('windows'))">Windows</HostOS>
<HostOS Condition="$([MSBuild]::IsOSPlatform('linux'))">Linux</HostOS>
<HostOS Condition="$([MSBuild]::IsOSPlatform('osx'))">Darwin</HostOS>
</PropertyGroup>
<PropertyGroup>
<AutoProvision Condition=" '$(AutoProvision)' == '' ">False</AutoProvision>
<AutoProvisionUsesSudo Condition=" '$(AutoProvisionUsesSudo)' == '' ">False</AutoProvisionUsesSudo>
<XAInstallPrefix Condition=" '$(XAInstallPrefix)' == '' ">$(MSBuildThisFileDirectory)bin\$(Configuration)\lib\xamarin.android\</XAInstallPrefix>
<_XABinRelativeInstallPrefix>lib\xamarin.android</_XABinRelativeInstallPrefix>
<XAInstallPrefix Condition=" '$(XAInstallPrefix)' == '' ">$(MSBuildThisFileDirectory)bin\$(Configuration)\$(_XABinRelativeInstallPrefix)\</XAInstallPrefix>
<MingwDependenciesRootDirectory Condition=" '$(MingwDependenciesRootDirectory)' == '' ">$(MSBuildThisFileDirectory)\bin\Build$(Configuration)\mingw-deps</MingwDependenciesRootDirectory>
<HostOS Condition=" '$(HostOS)' == '' And '$(OS)' == 'Windows_NT' ">Windows</HostOS>
<HostCc Condition=" '$(HostCc)' == '' ">$(HostCc64)</HostCc>
<HostCxx Condition=" '$(HostCxx)' == '' ">$(HostCxx64)</HostCxx>
<HostCc Condition=" '$(HostCc)' == '' ">$(HostCc32)</HostCc>
Expand Down Expand Up @@ -171,7 +176,8 @@
<AndroidSupportedTargetAotAbisSplit>$(AndroidSupportedTargetAotAbis.Split(':'))</AndroidSupportedTargetAotAbisSplit>
</PropertyGroup>
<PropertyGroup>
<RemapAssemblyRefTool>$(ManagedRuntime) $(ManagedRuntimeArgs) &quot;$(MSBuildThisFileDirectory)bin\Build$(Configuration)\remap-assembly-ref.exe&quot;</RemapAssemblyRefTool>
<RemapAssemblyRefToolExecutable>$(MSBuildThisFileDirectory)bin\Build$(Configuration)\remap-assembly-ref.exe</RemapAssemblyRefToolExecutable>
<RemapAssemblyRefTool>$(ManagedRuntime) $(ManagedRuntimeArgs) &quot;$(RemapAssemblyRefToolExecutable)&quot;</RemapAssemblyRefTool>
</PropertyGroup>

<!-- Unit Test Properties -->
Expand Down
266 changes: 123 additions & 143 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,78 @@
export OS_NAME := $(shell uname)
export OS_ARCH := $(shell uname -m)
export NO_SUDO ?= false
V ?= 0
prefix = /usr/local
CONFIGURATION = Debug
RUNTIME := $(shell which mono64 2> /dev/null && echo mono64 || echo mono) --debug=casts
SOLUTION = Xamarin.Android.sln
TEST_TARGETS = build-tools/scripts/RunTests.targets
API_LEVEL ?=
PREPARE_ARGS =
PREPARE_BUILD_LOG = bin/Build$(CONFIGURATION)/bootstrap-build.binlog
PREPARE_RESTORE_LOG = bin/Build$(CONFIGURATION)/bootstrap-restore.binlog
PREPARE_SOURCE_DIR = build-tools/xaprepare
PREPARE_SOLUTION = $(PREPARE_SOURCE_DIR)/xaprepare.sln
PREPARE_EXE = $(PREPARE_SOURCE_DIR)/xaprepare/bin/$(CONFIGURATION)/xaprepare.exe
PREPARE_COMMON_MSBUILD_FLAGS = /p:Configuration=$(CONFIGURATION) $(PREPARE_MSBUILD_ARGS) $(MSBUILD_ARGS)
PREPARE_MSBUILD_FLAGS = /binaryLogger:"$(PREPARE_BUILD_LOG)" $(PREPARE_COMMON_MSBUILD_FLAGS)
PREPARE_RESTORE_FLAGS = /binaryLogger:"$(PREPARE_RESTORE_LOG)" $(PREPARE_COMMON_MSBUILD_FLAGS)
PREPARE_SCENARIO =
PREPARE_CI ?= 0
PREPARE_AUTOPROVISION ?= 0
PREPARE_IGNORE_MONO_VERSION ?= 1

_PREPARE_CI_MODE_ARGS = --no-emoji --run-mode=CI -a
_PREPARE_ARGS =

all:
$(call MSBUILD_BINLOG,all,$(_SLN_BUILD)) $(MSBUILD_FLAGS) $(SOLUTION)

-include bin/Build$(CONFIGURATION)/rules.mk

ifeq ($(OS_NAME),)
export OS_NAME := $(shell uname)
endif

ifeq ($(OS_ARCH),)
export OS_ARCH := $(shell uname -m)
endif

export NO_SUDO ?= false

ifneq ($(NO_SUDO),false)
_PREPARE_ARGS += --auto-provisioning-uses-sudo=false
endif

ifneq ($(V),0)
MONO_OPTIONS += --debug
NUGET_VERBOSITY = -Verbosity Detailed
_PREPARE_ARGS += -v:d
endif

ifneq ($(PREPARE_CI),0)
_PREPARE_ARGS += $(_PREPARE_CI_MODE_ARGS)
endif

ifneq ($(PREPARE_AUTOPROVISION),0)
_PREPARE_ARGS += --auto-provision=yes --auto-provision-uses-sudo=yes
endif

ifeq ($(OS_NAME),Darwin)
export MACOSX_DEPLOYMENT_TARGET := 10.11
ifeq ($(HOMEBREW_PREFIX),)
HOMEBREW_PREFIX ?= $(shell brew --prefix)
endif
else
HOMEBREW_PREFIX := $prefix
endif

ifneq ($(V),0)
MONO_OPTIONS += --debug
NUGET_VERBOSITY = -Verbosity Detailed
ifeq ($(wildcard Configuration.OperatingSystem.props),)
PREPARE_MSBUILD_FLAGS += "/p:HostHomebrewPrefix=$(HOMEBREW_PREFIX)"
endif

ifneq ($(MONO_OPTIONS),)
export MONO_OPTIONS
ifneq ($(PREPARE_SCENARIO),)
_PREPARE_ARGS += -s:"$(PREPARE_SCENARIO)"
endif

_PREPARE_ARGS += $(PREPARE_ARGS)

include build-tools/scripts/msbuild.mk

ifeq ($(USE_MSBUILD),1)
Expand All @@ -34,12 +82,9 @@ _SLN_BUILD = MSBUILD="$(MSBUILD)" tools/scripts/xabuild
endif # $(USE_MSBUILD) == 1

ifneq ($(API_LEVEL),)
MSBUILD_FLAGS += /p:AndroidApiLevel=$(API_LEVEL) /p:AndroidFrameworkVersion=$(word $(API_LEVEL), $(ALL_FRAMEWORKS))
MSBUILD_FLAGS += /p:AndroidApiLevel=$(API_LEVEL) /p:AndroidFrameworkVersion=$(word $(API_LEVEL), $(ALL_FRAMEWORKS)) /p:AndroidPlatformId=$(word $(a), $(ALL_PLATFORM_IDS))
endif

all::
$(call MSBUILD_BINLOG,all,$(_SLN_BUILD)) $(MSBUILD_FLAGS) $(SOLUTION)

all-tests::
MSBUILD="$(MSBUILD)" $(call MSBUILD_BINLOG,all-tests,tools/scripts/xabuild) $(MSBUILD_FLAGS) Xamarin.Android-Tests.sln

Expand Down Expand Up @@ -70,87 +115,6 @@ uninstall::
rm -rf "$(prefix)/lib/mono/xbuild/Xamarin/Android"
rm -rf "$(prefix)/lib/mono/xbuild-frameworks/MonoAndroid"

ifeq ($(OS_NAME),Linux)
export LINUX_DISTRO := $(shell lsb_release -i -s || true)
export LINUX_DISTRO_RELEASE := $(shell lsb_release -r -s || true)
prepare:: linux-prepare
endif # $(OS_NAME)=Linux

prepare:: prepare-paths prepare-msbuild

linux-prepare::
BINFMT_MISC_TROUBLE="cli win" \
BINFMT_WARN=no ; \
for m in ${BINFMT_MISC_TROUBLE}; do \
if [ -f "/proc/sys/fs/binfmt_misc/$$m" ]; then \
BINFMT_WARN=yes ; \
fi ; \
done ; \
if [ "x${BINFMT_WARN}" = "xyes" ]; then \
cat Documentation/binfmt_misc-warning-Linux.txt ; \
fi; \
if [ -f build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO)-$(LINUX_DISTRO_RELEASE).sh ]; then \
sh build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO)-$(LINUX_DISTRO_RELEASE).sh $(LINUX_DISTRO_RELEASE); \
elif [ -f build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO).sh ]; then \
sh build-tools/scripts/dependencies/linux-prepare-$(LINUX_DISTRO).sh $(LINUX_DISTRO_RELEASE); \
fi

# $(call GetPath,path)
GetPath = $(shell $(MSBUILD) $(MSBUILD_FLAGS) /p:DoNotLoadOSProperties=True /nologo /v:minimal /t:Get$(1)FullPath build-tools/scripts/Paths.targets | tr -d '[[:space:]]' )

MSBUILD_PREPARE_PROJS = \
src/mono-runtimes/mono-runtimes.csproj \
src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj

prepare-deps: prepare-cmake-mingw-toolchain
git submodule update --init --recursive
./build-tools/scripts/generate-os-info Configuration.OperatingSystem.props
mkdir -p bin/Build$(CONFIGURATION)
$(call MSBUILD_BINLOG,prepare-deps) build-tools/dependencies/dependencies.csproj

#
# $(1): output file name
#
define create_cmake_toolchain
prepare-cmake-mingw-toolchain:: bin/Build$(CONFIGURATION)/$(1)

bin/Build$(CONFIGURATION)/$(1): build-tools/scripts/$(1).in
mkdir -p $$(dir $$@)
sed -e 's;@HOMEBREW_PREFIX@;$$(HOMEBREW_PREFIX);g' < $$< > $$@
endef

$(eval $(call create_cmake_toolchain,mingw-32.cmake))
$(eval $(call create_cmake_toolchain,mingw-64.cmake))

prepare-external: prepare-deps
nuget restore $(NUGET_VERBOSITY) $(SOLUTION)
nuget restore $(NUGET_VERBOSITY) Xamarin.Android-Tests.sln
(cd external/xamarin-android-tools && make prepare CONFIGURATION=$(CONFIGURATION))
(cd $(call GetPath,JavaInterop) && make prepare CONFIGURATION=$(CONFIGURATION) JI_MAX_JDK=8)
(cd $(call GetPath,JavaInterop) && make bin/Build$(CONFIGURATION)/JdkInfo.props CONFIGURATION=$(CONFIGURATION) JI_MAX_JDK=8)

prepare-bundle: prepare-external
$(call MSBUILD_BINLOG,prepare-bundle) build-tools/download-bundle/download-bundle.csproj
$(call MSBUILD_BINLOG,prepare-restore) $(MSBUILD_FLAGS) tests/Xamarin.Forms-Performance-Integration/Xamarin.Forms.Performance.Integration.csproj /t:Restore

prepare-props: prepare-bundle
cp $(call GetPath,JavaInterop)/external/Mono.Cecil* "$(call GetPath,MonoSource)/external"
cp "$(call GetPath,JavaInterop)/product.snk" "$(call GetPath,MonoSource)"
cp build-tools/scripts/Configuration.Java.Interop.Override.props external/Java.Interop/Configuration.Override.props
cp $(call GetPath,MonoSource)/mcs/class/msfinal.pub .

prepare-msbuild: prepare-props
ifeq ($(USE_MSBUILD),1)
for proj in $(MSBUILD_PREPARE_PROJS); do \
$(call MSBUILD_BINLOG,prepare-msbuild) "$$proj" || exit 1; \
done
endif # msbuild

prepare-image-dependencies:
$(call MSBUILD_BINLOG,prepare-image-deps) build-tools/scripts/PrepareImageDependencies.targets /t:PrepareImageDependencies \
/p:AndroidSupportedHostJitAbis=mxe-Win32:mxe-Win64
cat bin/Build$(CONFIGURATION)/prepare-image-dependencies.sh | tr -d '\r' > prepare-image-dependencies.sh

include build-tools/scripts/BuildEverything.mk

# Must be after BuildEverything.mk - it uses variables defined there
Expand All @@ -159,59 +123,10 @@ include tests/api-compatibility/api-compatibility.mk

topdir := $(shell pwd)


XA_BUILD_PATHS_OUT = bin/Test$(CONFIGURATION)/XABuildPaths.cs

prepare-paths: $(XA_BUILD_PATHS_OUT)

$(XA_BUILD_PATHS_OUT): bin/Test%/XABuildPaths.cs: build-tools/scripts/XABuildPaths.cs.in
mkdir -p $(shell dirname $@)
sed -e 's;@CONFIGURATION@;$*;g' \
-e 's;@TOP_DIRECTORY@;$(topdir);g' < $< > $@
cat $@


# Usage: $(call CALL_CREATE_THIRD_PARTY_NOTICES,path,licenseType,includeExternalDeps,includeBuildDeps)
define CREATE_THIRD_PARTY_NOTICES
$(call MSBUILD_BINLOG,create-tpn,$(MSBUILD)) $(_MSBUILD_ARGS) \
$(topdir)/build-tools/ThirdPartyNotices/ThirdPartyNotices.csproj \
/p:Configuration=$(CONFIGURATION) \
/p:ThirdPartyNoticeFile=$(topdir)/$(1) \
/p:ThirdPartyNoticeLicenseType=$(2) \
/p:TpnIncludeExternalDependencies=$(3) \
/p:TpnIncludeBuildDependencies=$(4)
endef # CREATE_THIRD_PARTY_NOTICES

prepare:: prepare-tpn

TPN_LICENSE_FILES = $(shell grep -h '<LicenseFile>' external/*.tpnitems src/*.tpnitems \
| sed -E 's,<LicenseFile>(.*)</LicenseFile>,\1,g;s,.\(MSBuildThisFileDirectory\),$(topdir)/external/,g' \
| tr \\ / )

# Usage: $(call CREATE_THIRD_PARTY_NOTICES_RULE,path,licenseType,includeExternalDeps,includeBuildDeps)
define CREATE_THIRD_PARTY_NOTICES_RULE
prepare-tpn:: $(1)

$(1) $(topdir)/$(1): build-tools/ThirdPartyNotices/ThirdPartyNotices.csproj \
$(wildcard external/*.tpnitems src/*.tpnitems build-tools/*.tpnitems) \
$(TPN_LICENSE_FILES)
$(call CREATE_THIRD_PARTY_NOTICES,$(1),$(2),$(3),$(4))
endef # CREATE_THIRD_PARTY_NOTICES_RULE

THIRD_PARTY_NOTICE_LICENSE_TYPE = microsoft-oss

$(eval $(call CREATE_THIRD_PARTY_NOTICES_RULE,ThirdPartyNotices.txt,foundation,False,False))
$(eval $(call CREATE_THIRD_PARTY_NOTICES_RULE,bin/$(CONFIGURATION)/lib/xamarin.android/ThirdPartyNotices.txt,$(THIRD_PARTY_NOTICE_LICENSE_TYPE),True,False))

# Used by External XA Build
EXTERNAL_XA_PATH=$(topdir)
EXTERNAL_GIT_PATH=$(topdir)/external

prepare-external-git-dependencies:
msbuild build-tools/xa-prep-tasks/xa-prep-tasks.csproj /p:Configuration=$(CONFIGURATION)
msbuild build-tools/xa-prep-tasks/xa-prep-tasks.csproj /p:Configuration=$(CONFIGURATION) \
/t:CheckoutExternalGitSources /p:ExternalSourceDependencyDirectory='$(EXTERNAL_GIT_PATH)'

-include $(EXTERNAL_GIT_PATH)/monodroid/xa-integration.mk

run-all-tests:
Expand Down Expand Up @@ -257,3 +172,68 @@ list-nunit-tests:
$(MSBUILD) $(MSBUILD_FLAGS) $(TEST_TARGETS) /t:ListNUnitTests

include build-tools/scripts/runtime-helpers.mk

.PHONY: prepare-build-init
prepare-build-init:
mkdir -p $(dir $(PREPARE_BUILD_LOG))
msbuild $(PREPARE_RESTORE_FLAGS) $(PREPARE_SOLUTION) /t:Restore

.PHONY: prepare-build
prepare-build: prepare-build-init
msbuild $(PREPARE_MSBUILD_FLAGS) $(PREPARE_SOLUTION)

.PHONY: prepare-build-ci
prepare-build-ci: prepare-build-init
msbuild $(PREPARE_MSBUILD_FLAGS) $(PREPARE_SOLUTION) $(_MSBUILD_ARGS)

.PHONY: prepare
prepare:: prepare-build
mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS)

.PHONY: prepare-help
prepare-help: prepare-build
mono --debug $(PREPARE_EXE) -h

# Hack: The current commercial pipeline doesn't pass all the required arguments when preparing the build, in particular it doesn't override the
# ABI targets to build and so the prepare step configures only for the default set (armeabi-v7a, arm64-v8a, x86, $HOST_OS) which is not enough.
# The `jenkins` rule in `BuildEverything.mk`, invoked by the commercial pipeline, now calls the rule below in which we rebuild the bootstrapper
# with all the required properties set to include all the ABIs - it should fix the build. After the PR is merged, the commercial pipeline should
# be modified to do the right thing instead.
#
# Commercial pipeline should also set PREPARE_CI=1 when calling targets. Since this is currently not done, we have to pass $(_PREPARE_CI_MODE_ARGS)
# directly below
#
.PHONY: prepare-jenkins
prepare-jenkins: prepare-build-ci prepare-commercial
@echo preparing jenkins build
mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS)

# This should go away once we can modify the commercial pipeline for the bootstrapper
.PHONY: prepare-commercial
ifeq ($(USE_COMMERCIAL_INSTALLER_NAME),true)
prepare-commercial:
cd $(TOP) && ./configure --with-xamarin-android='$(XAMARIN_ANDROID_PATH)'
mkdir -p $(XA_MSBUILD_DIR)

else
prepare-commercial:
endif

.PHONY: prepare-update-mono

prepare-update-mono: prepare-build-ci
mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS) /s:UpdateMono

# These targets exist only temporarily to satisfy requirements of the commercial build (since we can't modify the pipeline script in this PR)
.PHONY: prepare-deps

# Commercial pipeline installs an older version of Mono and in effect we fail. `prepare-deps` is called after provisionator is ran and so we
# can, temporarily, re-update Mono here. After the PR is merged and commercial pipeline updated, this step should be removed.
prepare-deps: prepare-update-mono
@echo prepare-deps is no-op, prepare-jenkins or prepare do the work instead

prepare-image-dependencies: prepare-build-ci
mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS) -s:PrepareImageDependencies

prepare-external-git-dependencies: prepare-build-ci prepare-update-mono
mono --debug $(PREPARE_EXE) $(_PREPARE_ARGS) $(_PREPARE_CI_MODE_ARGS) -s:PrepareExternalGitDependencies
Loading