Skip to content

Commit 5c31fd1

Browse files
grendellogrendello
committed
New unit test runner
This is a rewrite of our (very old) NUnit test runner to support tests using other test suites. The reason for this is that Mono BCL tests are starting to use xUnit tests and we want to be able to run all of them. At the same time this code drops support for the "GUI" we previously had to run tests (not maintained, makes the actual runner code messy) and restructures the runner infrastructure so that it's possible to add any number of runners in the future without rewriting any base code. We now use only Android instrumentations to execute the code. A few changes to the test running code were necessary since we now support more than one instrumentation per apk, with different result outputs, options etc. Support for granting Android permissions on apk install time was added as well (might be necessary when running tests on devices with Oreo)
1 parent 08c6fd0 commit 5c31fd1

File tree

69 files changed

+3069
-332
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+3069
-332
lines changed

‎Xamarin.Android-Tests.sln

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Core", "Core", "{EFBC4DC0-D
1111
EndProject
1212
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Locales", "Locales", "{D6BFEDF6-2F48-44B2-9553-F2F6F92531BD}"
1313
EndProject
14+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "BCL-Tests", "BCL-Tests", "{6C86878D-9EBF-45B3-9368-C0EA4026F706}"
15+
EndProject
1416
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibraryResources", "tests\locales\LibraryResources\LibraryResources.csproj", "{05768F39-7BAF-43E6-971E-712F5771E88E}"
1517
EndProject
1618
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Android.Locale-Tests", "tests\locales\Xamarin.Android.Locale-Tests\Xamarin.Android.Locale-Tests.csproj", "{9D5C83B5-70D5-4CC2-9DB7-78B23DC8F255}"
@@ -49,7 +51,15 @@ Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "Xamarin.Forms.Performance.I
4951
EndProject
5052
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Forms.Performance.Integration.Droid", "tests\Xamarin.Forms-Performance-Integration\Droid\Xamarin.Forms.Performance.Integration.Droid.csproj", "{576312CC-83FF-48B1-A473-488CDC7121AD}"
5153
EndProject
52-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Android.Bcl-Tests", "tests\Xamarin.Android.Bcl-Tests\Xamarin.Android.Bcl-Tests.csproj", "{E865C28E-32AF-4210-A41D-5791C39A9D85}"
54+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xamarin.Android.Bcl-Tests", "tests\BCL-Tests\Xamarin.Android.Bcl-Tests\Xamarin.Android.Bcl-Tests.csproj", "{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC}"
55+
EndProject
56+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestRunner.Core", "tests\TestRunner.Core\TestRunner.Core.csproj", "{3CC4E384-4985-4D93-A34C-73F69A379FA7}"
57+
EndProject
58+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestRunner.NUnit", "tests\TestRunner.NUnit\TestRunner.NUnit.csproj", "{CB2335CB-0050-4020-8A05-E9614EDAA05E}"
59+
EndProject
60+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestRunner.xUnit", "tests\TestRunner.xUnit\TestRunner.xUnit.csproj", "{57DC8529-2628-40C4-B27E-BAC1AE44A706}"
61+
EndProject
62+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalTests.NUnit", "tests\BCL-Tests\LocalTests.NUnit\LocalTests.NUnit.csproj", "{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}"
5363
EndProject
5464
Global
5565
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -125,10 +135,26 @@ Global
125135
{576312CC-83FF-48B1-A473-488CDC7121AD}.Debug|Any CPU.Build.0 = Debug|Any CPU
126136
{576312CC-83FF-48B1-A473-488CDC7121AD}.Release|Any CPU.ActiveCfg = Release|Any CPU
127137
{576312CC-83FF-48B1-A473-488CDC7121AD}.Release|Any CPU.Build.0 = Release|Any CPU
128-
{E865C28E-32AF-4210-A41D-5791C39A9D85}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
129-
{E865C28E-32AF-4210-A41D-5791C39A9D85}.Debug|Any CPU.Build.0 = Debug|Any CPU
130-
{E865C28E-32AF-4210-A41D-5791C39A9D85}.Release|Any CPU.ActiveCfg = Release|Any CPU
131-
{E865C28E-32AF-4210-A41D-5791C39A9D85}.Release|Any CPU.Build.0 = Release|Any CPU
138+
{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
139+
{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
140+
{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
141+
{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC}.Release|Any CPU.Build.0 = Release|Any CPU
142+
{3CC4E384-4985-4D93-A34C-73F69A379FA7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
143+
{3CC4E384-4985-4D93-A34C-73F69A379FA7}.Debug|Any CPU.Build.0 = Debug|Any CPU
144+
{3CC4E384-4985-4D93-A34C-73F69A379FA7}.Release|Any CPU.ActiveCfg = Release|Any CPU
145+
{3CC4E384-4985-4D93-A34C-73F69A379FA7}.Release|Any CPU.Build.0 = Release|Any CPU
146+
{CB2335CB-0050-4020-8A05-E9614EDAA05E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
147+
{CB2335CB-0050-4020-8A05-E9614EDAA05E}.Debug|Any CPU.Build.0 = Debug|Any CPU
148+
{CB2335CB-0050-4020-8A05-E9614EDAA05E}.Release|Any CPU.ActiveCfg = Release|Any CPU
149+
{CB2335CB-0050-4020-8A05-E9614EDAA05E}.Release|Any CPU.Build.0 = Release|Any CPU
150+
{57DC8529-2628-40C4-B27E-BAC1AE44A706}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
151+
{57DC8529-2628-40C4-B27E-BAC1AE44A706}.Debug|Any CPU.Build.0 = Debug|Any CPU
152+
{57DC8529-2628-40C4-B27E-BAC1AE44A706}.Release|Any CPU.ActiveCfg = Release|Any CPU
153+
{57DC8529-2628-40C4-B27E-BAC1AE44A706}.Release|Any CPU.Build.0 = Release|Any CPU
154+
{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
155+
{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
156+
{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
157+
{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}.Release|Any CPU.Build.0 = Release|Any CPU
132158
EndGlobalSection
133159
GlobalSection(NestedProjects) = preSolution
134160
{2305B00D-DE81-4744-B0DA-357835CAFE5A} = {43A4FB09-279A-4138-8027-EC1E1CED2E8A}
@@ -147,5 +173,10 @@ Global
147173
{9802CB35-0BC0-4EE1-9A81-BB40BD97945A} = {E49089E4-4CE0-475E-BE9C-0AB4E4D56EE9}
148174
{5BDCEF07-E3D7-4E73-B025-6E43A5A7D7F1} = {E49089E4-4CE0-475E-BE9C-0AB4E4D56EE9}
149175
{B297008B-C313-455E-B230-E119589D2D79} = {E49089E4-4CE0-475E-BE9C-0AB4E4D56EE9}
176+
{04D4E45E-0EBE-40F7-B170-0A8F4D74DCBC} = {6C86878D-9EBF-45B3-9368-C0EA4026F706}
177+
{3CC4E384-4985-4D93-A34C-73F69A379FA7} = {6C86878D-9EBF-45B3-9368-C0EA4026F706}
178+
{CB2335CB-0050-4020-8A05-E9614EDAA05E} = {6C86878D-9EBF-45B3-9368-C0EA4026F706}
179+
{57DC8529-2628-40C4-B27E-BAC1AE44A706} = {6C86878D-9EBF-45B3-9368-C0EA4026F706}
180+
{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF} = {6C86878D-9EBF-45B3-9368-C0EA4026F706}
150181
EndGlobalSection
151182
EndGlobal

‎build-tools/mono-runtimes/ProfileAssemblies.projitems

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,10 @@
192192
<MonoTestAssembly Include="monodroid_corlib_test.dll">
193193
<SourcePath>corlib</SourcePath>
194194
</MonoTestAssembly>
195+
<MonoTestAssembly Include="monodroid_corlib_xunit-test.dll">
196+
<SourcePath>corlib</SourcePath>
197+
<TestType>xunit</TestType>
198+
</MonoTestAssembly>
195199
<MonoTestAssembly Include="monodroid_I18N.CJK_test.dll">
196200
<SourcePath>I18N/CJK</SourcePath>
197201
</MonoTestAssembly>
@@ -237,6 +241,10 @@
237241
<MonoTestAssembly Include="monodroid_System.Numerics_test.dll">
238242
<SourcePath>System.Numerics</SourcePath>
239243
</MonoTestAssembly>
244+
<MonoTestAssembly Include="monodroid_System.Numerics_xunit-test.dll">
245+
<SourcePath>System.Numerics</SourcePath>
246+
<TestType>xunit</TestType>
247+
</MonoTestAssembly>
240248
<MonoTestAssembly Include="monodroid_System.Runtime.Serialization_test.dll">
241249
<SourcePath>System.Runtime.Serialization</SourcePath>
242250
</MonoTestAssembly>

‎build-tools/mono-runtimes/mono-runtimes.targets

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,11 @@
310310
IgnoreStandardErrorWarningFormat="True"
311311
WorkingDirectory="$(IntermediateOutputPath)\%(_MonoRuntime.Identity)\runtime"
312312
/>
313+
<Exec
314+
Condition=" '%(MonoTestAssembly.TestType)' == 'xunit' "
315+
Command="make -C $(MonoSourceFullPath)\mcs\class\%(MonoTestAssembly.SourcePath) xunit-test-local"
316+
WorkingDirectory="$(MonoSourceFullPath)"
317+
/>
313318
<Copy
314319
SourceFiles="@(_BclTestAssemblySource)"
315320
DestinationFiles="@(_BclTestAssemblyDestination)"

‎build-tools/scripts/RunTests.targets

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
<_ApkTestProject Include="$(_TopDir)\tests\CodeGen-Binding\Xamarin.Android.JcwGen-Tests\Xamarin.Android.JcwGen-Tests.csproj" />
2020
<_ApkTestProject Include="$(_TopDir)\tests\CodeGen-MkBundle\Xamarin.Android.MakeBundle-Tests\Xamarin.Android.MakeBundle-Tests.csproj" />
2121
<_ApkTestProject Include="$(_TopDir)\tests\locales\Xamarin.Android.Locale-Tests\Xamarin.Android.Locale-Tests.csproj" />
22-
<_ApkTestProject Include="$(_TopDir)\tests\Xamarin.Android.Bcl-Tests\Xamarin.Android.Bcl-Tests.csproj" />
22+
<_ApkTestProject Include="$(_TopDir)\tests\BCL-Tests\Xamarin.Android.Bcl-Tests\Xamarin.Android.Bcl-Tests.csproj" />
2323
<_ApkTestProject Include="$(_TopDir)\tests\Xamarin.Forms-Performance-Integration\Droid\Xamarin.Forms.Performance.Integration.Droid.csproj" />
2424
<_ApkTestProjectAot Include="$(_TopDir)\src\Mono.Android\Test\Mono.Android-Tests.csproj" />
2525
</ItemGroup>

‎build-tools/scripts/TestApks.targets

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -153,16 +153,22 @@
153153

154154
<Target Name="RunTestApks"
155155
Condition=" '@(TestApk)' != '' ">
156+
<Xamarin.Android.Tools.BootstrapTasks.Adb
157+
Condition=" '@(TestApkPermission)' != '' "
158+
Arguments="$(_AdbTarget) $(AdbOptions) shell pm grant %(TestApkPermission.Package) android.permission.%(TestApkPermission.Identity)"
159+
ToolExe="$(AdbToolExe)"
160+
ToolPath="$(AdbToolPath)"
161+
/>
156162
<PropertyGroup>
157163
<_IncludeCategories Condition=" '$(IncludeCategories)' != '' ">include=$(IncludeCategories)</_IncludeCategories>
158164
<_ExcludeCategories Condition=" '$(ExcludeCategories)' != '' ">exclude=$(ExcludeCategories)</_ExcludeCategories>
159165
</PropertyGroup>
160166
<RunInstrumentationTests
161-
Condition=" '%(TestApk.InstrumentationType)' != ''"
167+
Condition=" '%(TestApkInstrumentation.Identity)' != ''"
162168
AdbTarget="$(_AdbTarget)"
163169
AdbOptions="$(AdbOptions)"
164-
Component="%(TestApk.Package)/%(TestApk.InstrumentationType)"
165-
NUnit2TestResultsFile="%(TestApk.ResultsPath)"
170+
Component="%(TestApkInstrumentation.Package)/%(TestApkInstrumentation.Identity)"
171+
NUnit2TestResultsFile="%(TestApkInstrumentation.ResultsPath)"
166172
InstrumentationArguments="$(_IncludeCategories);$(_ExcludeCategories)"
167173
TestFixture="$(TestFixture)"
168174
ToolExe="$(AdbToolExe)"
@@ -223,11 +229,11 @@
223229
<Target Name="RenameApkTestCases"
224230
Condition=" '@(TestApk)' != '' ">
225231
<RenameTestCases
226-
Condition=" '%(TestApk.ResultsPath)' != '' "
232+
Condition=" '%(TestApkInstrumentation.ResultsPath)' != '' "
227233
Configuration="$(Configuration)$(TestsAotName)"
228234
DeleteSourceFiles="True"
229235
DestinationFolder="$(MSBuildThisFileDirectory)..\.."
230-
SourceFile="%(TestApk.ResultsPath)"
236+
SourceFile="%(TestApkInstrumentation.ResultsPath)"
231237
/>
232238
</Target>
233239
<Target Name="RecordApkSizes"

‎src/Mono.Android/Test/Mono.Android-Tests.projitems

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
33
<PropertyGroup>
44
<_MonoAndroidTestPackage>Mono.Android_Tests</_MonoAndroidTestPackage>
5+
<_PackageName>Mono.Android_Tests</_PackageName>
56
</PropertyGroup>
67
<ItemGroup>
78
<TestApk Include="$(OutputPath)Mono.Android_Tests-Signed.apk">
@@ -15,4 +16,11 @@
1516
<ApkSizesResultsFilename>$(MSBuildThisFileDirectory)..\..\..\TestResult-Mono.Android_Tests-values.csv</ApkSizesResultsFilename>
1617
</TestApk>
1718
</ItemGroup>
19+
20+
<ItemGroup>
21+
<TestApkInstrumentation Include="xamarin.android.runtimetests.TestInstrumentation">
22+
<Package>Mono.Android_Tests</Package>
23+
<ResultsPath>$(OutputPath)TestResult-Mono.Android_Tests.xml</ResultsPath>
24+
</TestApkInstrumentation>
25+
</ItemGroup>
1826
</Project>

‎tests/BCL-Tests/LocalTests.NUnit/LocalTests.NUnit.csproj

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<PropertyGroup>
4+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
5+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
6+
<ProjectGuid>{910DCD88-D50A-4AAD-BA7A-CD51AB8532BF}</ProjectGuid>
7+
<ProjectTypeGuids>{EFBA0AD7-5A72-4C68-AF49-83D382785DCF};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
8+
<OutputType>Library</OutputType>
9+
<RootNamespace>LocalTests.NUnit</RootNamespace>
10+
<AssemblyName>LocalTests.NUnit</AssemblyName>
11+
<TargetFrameworkVersion>v8.1</TargetFrameworkVersion>
12+
<AndroidResgenFile>Resources\Resource.designer.cs</AndroidResgenFile>
13+
<AndroidResgenClass>Resource</AndroidResgenClass>
14+
<MonoAndroidResourcePrefix>Resources</MonoAndroidResourcePrefix>
15+
<MonoAndroidAssetsPrefix>Assets</MonoAndroidAssetsPrefix>
16+
<AndroidUseLatestPlatformSdk>true</AndroidUseLatestPlatformSdk>
17+
</PropertyGroup>
18+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
19+
<DebugSymbols>true</DebugSymbols>
20+
<DebugType>full</DebugType>
21+
<Optimize>false</Optimize>
22+
<OutputPath>bin\Debug</OutputPath>
23+
<DefineConstants>DEBUG;</DefineConstants>
24+
<ErrorReport>prompt</ErrorReport>
25+
<WarningLevel>4</WarningLevel>
26+
<AndroidLinkMode>None</AndroidLinkMode>
27+
</PropertyGroup>
28+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
29+
<DebugSymbols>true</DebugSymbols>
30+
<DebugType>pdbonly</DebugType>
31+
<Optimize>true</Optimize>
32+
<OutputPath>bin\Release</OutputPath>
33+
<ErrorReport>prompt</ErrorReport>
34+
<WarningLevel>4</WarningLevel>
35+
<AndroidManagedSymbols>true</AndroidManagedSymbols>
36+
<AndroidUseSharedRuntime>false</AndroidUseSharedRuntime>
37+
</PropertyGroup>
38+
<ItemGroup>
39+
<Reference Include="System" />
40+
<Reference Include="System.Xml" />
41+
<Reference Include="System.Core" />
42+
<Reference Include="Mono.Android" />
43+
<Reference Include="Xamarin.Android.NUnitLite" />
44+
<Reference Include="System.Net" />
45+
<Reference Include="System.Net.Http" />
46+
</ItemGroup>
47+
<ItemGroup>
48+
<Compile Include="Resources\Resource.designer.cs" />
49+
<Compile Include="Properties\AssemblyInfo.cs" />
50+
<Compile Include="System.Net\NetworkChangeTest.cs" />
51+
<Compile Include="System.Net.Http\HttpClientTest.cs" />
52+
</ItemGroup>
53+
<ItemGroup>
54+
<None Include="Resources\AboutResources.txt" />
55+
</ItemGroup>
56+
<ItemGroup>
57+
<AndroidResource Include="Resources\values\Strings.xml" />
58+
</ItemGroup>
59+
<Import Project="$(MSBuildExtensionsPath)\Xamarin\Android\Xamarin.Android.CSharp.targets" />
60+
</Project>

‎tests/Xamarin.Android.Bcl-Tests/Properties/AssemblyInfo.cs renamed to ‎tests/BCL-Tests/LocalTests.NUnit/Properties/AssemblyInfo.cs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
11
using System.Reflection;
22
using System.Runtime.CompilerServices;
3-
using Android.App;
43

54
// Information about this assembly is defined by the following attributes.
65
// Change them to the values specific to your project.
76

8-
[assembly: AssemblyTitle ("BCL")]
7+
[assembly: AssemblyTitle ("LocalTests.NUnit")]
98
[assembly: AssemblyDescription ("")]
109
[assembly: AssemblyConfiguration ("")]
11-
[assembly: AssemblyCompany ("Microsoft Corporation")]
10+
[assembly: AssemblyCompany ("")]
1211
[assembly: AssemblyProduct ("")]
13-
[assembly: AssemblyCopyright ("")]
12+
[assembly: AssemblyCopyright ("Microsoft, Inc")]
1413
[assembly: AssemblyTrademark ("")]
1514
[assembly: AssemblyCulture ("")]
1615

‎tests/Xamarin.Android.Bcl-Tests/Resources/AboutResources.txt renamed to ‎tests/BCL-Tests/LocalTests.NUnit/Resources/AboutResources.txt

File renamed without changes.

‎tests/BCL-Tests/LocalTests.NUnit/Resources/values/Strings.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<resources>
3+
<string name="library_name">LocalTests.NUnit</string>
4+
</resources>

0 commit comments

Comments
 (0)