Skip to content

Commit

Permalink
Add visual tree verification baseline (microsoft#1130)
Browse files Browse the repository at this point in the history
* Add visual tree verification baseline

* cr feedbacks

* Add min/max width/height

* Upload master files to Helix

* Update master files

* Set width/height on ColorPicker

* Renames

* Add script to update master files

* Update script to merge duplicate masters

* Copy master files to artifact drop folder

* Update masters

* Create LinksToHelixTestFiles only when test fails
  • Loading branch information
kaiguo authored Oct 2, 2019
1 parent 18ef85a commit 44575e6
Show file tree
Hide file tree
Showing 91 changed files with 31,502 additions and 119 deletions.
6 changes: 6 additions & 0 deletions MUXControls.sln
Original file line number Diff line number Diff line change
Expand Up @@ -605,6 +605,8 @@ Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "AutoSuggestBox_APITests", "
EndProject
Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "AutoSuggestBox_InteractionTests", "dev\AutoSuggestBox\InteractionTests\AutoSuggestBox_InteractionTests.shproj", "{ED3E2182-D259-4DC0-B7BB-9B4219762F4C}"
EndProject
Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "CalendarView_APITests", "dev\CalendarView\APITests\CalendarView_APITests.shproj", "{AA73FBEB-6B3B-490C-B582-3993F0040061}"
EndProject
Global
GlobalSection(SharedMSBuildProjectFiles) = preSolution
dev\ComboBox\ComboBox.vcxitems*{00523caf-422a-4185-9392-d374b72a019a}*SharedItemsImports = 9
Expand Down Expand Up @@ -733,6 +735,7 @@ Global
dev\ColorPicker\InteractionTests\ColorPicker_InteractionTests.projitems*{a4d2fae8-e7fc-4100-91ff-2202fac91c70}*SharedItemsImports = 13
dev\CommonStyles\TestUI\CommonStyles_TestUI.projitems*{a7f6d6c4-a5a9-43eb-930c-b766417a5e5c}*SharedItemsImports = 13
dev\Materials\Acrylic\TestUI\AcrylicBrush_TestUI.projitems*{a800e818-7212-4fd7-ae3a-1dcab539db87}*SharedItemsImports = 13
dev\CalendarView\APITests\CalendarView_APITests.projitems*{aa73fbeb-6b3b-490c-b582-3993f0040061}*SharedItemsImports = 13
dev\AnimatedVisualPlayer\AnimatedVisualPlayer.vcxitems*{ad0c90b0-4845-4d4b-88f1-86f653f8171b}*SharedItemsImports = 4
dev\AutoSuggestBox\AutoSuggestBox.vcxitems*{ad0c90b0-4845-4d4b-88f1-86f653f8171b}*SharedItemsImports = 4
dev\CalendarDatePicker\CalendarDatePicker.vcxitems*{ad0c90b0-4845-4d4b-88f1-86f653f8171b}*SharedItemsImports = 4
Expand Down Expand Up @@ -825,6 +828,7 @@ Global
dev\AutoSuggestBox\APITests\AutoSuggestBox_APITests.projitems*{dedc1e4f-cfa5-4443-83eb-e79d425df7e7}*SharedItemsImports = 4
dev\AutoSuggestBox\TestUI\AutoSuggestBox_TestUI.projitems*{dedc1e4f-cfa5-4443-83eb-e79d425df7e7}*SharedItemsImports = 4
dev\CalendarDatePicker\TestUI\CalendarDatePicker_TestUI.projitems*{dedc1e4f-cfa5-4443-83eb-e79d425df7e7}*SharedItemsImports = 4
dev\CalendarView\APITests\CalendarView_APITests.projitems*{dedc1e4f-cfa5-4443-83eb-e79d425df7e7}*SharedItemsImports = 4
dev\CalendarView\TestUI\CalendarView_TestUI.projitems*{dedc1e4f-cfa5-4443-83eb-e79d425df7e7}*SharedItemsImports = 4
dev\CheckBox\TestUI\CheckBox_TestUI.projitems*{dedc1e4f-cfa5-4443-83eb-e79d425df7e7}*SharedItemsImports = 4
dev\ColorPicker\APITests\ColorPicker_APITests.projitems*{dedc1e4f-cfa5-4443-83eb-e79d425df7e7}*SharedItemsImports = 4
Expand Down Expand Up @@ -909,6 +913,7 @@ Global
dev\AutoSuggestBox\APITests\AutoSuggestBox_APITests.projitems*{fbc396f5-26dd-4ca3-981e-c7bc9fea4546}*SharedItemsImports = 4
dev\AutoSuggestBox\TestUI\AutoSuggestBox_TestUI.projitems*{fbc396f5-26dd-4ca3-981e-c7bc9fea4546}*SharedItemsImports = 4
dev\CalendarDatePicker\TestUI\CalendarDatePicker_TestUI.projitems*{fbc396f5-26dd-4ca3-981e-c7bc9fea4546}*SharedItemsImports = 4
dev\CalendarView\APITests\CalendarView_APITests.projitems*{fbc396f5-26dd-4ca3-981e-c7bc9fea4546}*SharedItemsImports = 4
dev\CalendarView\TestUI\CalendarView_TestUI.projitems*{fbc396f5-26dd-4ca3-981e-c7bc9fea4546}*SharedItemsImports = 4
dev\CheckBox\TestUI\CheckBox_TestUI.projitems*{fbc396f5-26dd-4ca3-981e-c7bc9fea4546}*SharedItemsImports = 4
dev\ColorPicker\APITests\ColorPicker_APITests.projitems*{fbc396f5-26dd-4ca3-981e-c7bc9fea4546}*SharedItemsImports = 4
Expand Down Expand Up @@ -1591,6 +1596,7 @@ Global
{CA704A8C-2624-4630-89A0-D86CB1EE409A} = {4C9CEC92-77C8-4B7B-8595-6C736A660F93}
{84673739-208C-427E-A19E-E090EC3733CE} = {CFAFD6A7-FC7B-4C96-B292-A440C7C65D89}
{ED3E2182-D259-4DC0-B7BB-9B4219762F4C} = {CFAFD6A7-FC7B-4C96-B292-A440C7C65D89}
{AA73FBEB-6B3B-490C-B582-3993F0040061} = {8599B569-6AF6-47C2-A490-D28C511C8DE4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {D93836AB-52D3-4DE2-AE25-23F26F55ECED}
Expand Down
14 changes: 7 additions & 7 deletions build/AzurePipelinesTemplates/MUX-ProcessTestResults-Job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
vmImage: 'windows-2019'
timeoutInMinutes: 120
variables:
crashDumpsListFilePath: $(Build.SourcesDirectory)\crashdumps.html
helixOutputFolder: $(Build.SourcesDirectory)\HelixOutput

steps:
- task: powershell@2
Expand All @@ -37,18 +37,18 @@ jobs:
arguments: -MinimumExpectedTestsExecutedCount '${{ parameters.minimumExpectedTestsExecutedCount }}' -CheckJobAttempt $${{ parameters.checkJobAttempt }}

- task: powershell@2
displayName: 'CreateListOfCrashDumpsFromTestPass.ps1'
displayName: 'ProcessHelixFiles.ps1'
condition: succeededOrFailed()
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
inputs:
targetType: filePath
filePath: build\Helix\CreateListOfCrashDumpsFromTestPass.ps1
arguments: -OutputFilePath '$(crashDumpsListFilePath)'
filePath: build\Helix\ProcessHelixFiles.ps1
arguments: -OutputFilePath '$(helixOutputFolder)'

- task: PublishBuildArtifacts@1
displayName: 'Publish Crash Dump List'
displayName: 'Publish Helix files'
condition: succeededOrFailed()
inputs:
PathtoPublish: $(crashDumpsListFilePath)
artifactName: drop
PathtoPublish: $(helixOutputFolder)
artifactName: drop
2 changes: 2 additions & 0 deletions build/Helix/CopyVisualTreeMasters.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
$picturesFolder = [Environment]::GetFolderPath('MyPictures')
Copy-Item $picturesFolder\*.xml $Env:HELIX_WORKITEM_UPLOAD_ROOT
63 changes: 0 additions & 63 deletions build/Helix/CreateListOfCrashDumpsFromTestPass.ps1

This file was deleted.

1 change: 1 addition & 0 deletions build/Helix/PrepareHelixPayload.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -69,3 +69,4 @@ Copy-Item "build\helix\runtests.cmd" $payloadDir
Copy-Item "build\helix\InstallTestAppDependencies.ps1" "$payloadDir"
Copy-Item "build\Helix\EnsureMachineState.ps1" "$payloadDir"
Copy-Item "version.props" "$payloadDir"
Copy-Item "build\Helix\CopyVisualTreeMasters.ps1" "$payloadDir"
134 changes: 134 additions & 0 deletions build/Helix/ProcessHelixFiles.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
Param(
[string]$AccessToken = $env:SYSTEM_ACCESSTOKEN,
[string]$CollectionUri = $env:SYSTEM_COLLECTIONURI,
[string]$TeamProject = $env:SYSTEM_TEAMPROJECT,
[string]$BuildUri = $env:BUILD_BUILDURI,
[string]$OutPutFolder = "HelixOutput"
)

$helixLinkFile = "$OutPutFolder\LinksToHelixTestFiles.html"
$visualTreeMasterFolder = "$OutPutFolder\VisualTreeMasters"

function Generate-File-Links
{
Param ([Array[]]$files,[string]$sectionName)
if($files.Count -gt 0)
{
Out-File -FilePath $helixLinkFile -Append -InputObject "<div class=$sectionName>"
Out-File -FilePath $helixLinkFile -Append -InputObject "<h4>$sectionName</h4>"
Out-File -FilePath $helixLinkFile -Append -InputObject "<ul>"
foreach($file in $files)
{
Out-File -FilePath $helixLinkFile -Append -InputObject "<li><a href=$($file.Link)>$($file.Name)</a></li>"
}
Out-File -FilePath $helixLinkFile -Append -InputObject "</ul>"
Out-File -FilePath $helixLinkFile -Append -InputObject "</div>"
}
}

#Create output directory
New-Item $OutPutFolder -ItemType Directory

$azureDevOpsRestApiHeaders = @{
"Accept"="application/json"
"Authorization"="Basic $([System.Convert]::ToBase64String([System.Text.ASCIIEncoding]::ASCII.GetBytes(":$AccessToken")))"
}

. "$PSScriptRoot/AzurePipelinesHelperScripts.ps1"

$queryUri = GetQueryTestRunsUri -CollectionUri $CollectionUri -TeamProject $TeamProject -BuildUri $BuildUri -IncludeRunDetails
Write-Host "queryUri = $queryUri"

$testRuns = Invoke-RestMethod -Uri $queryUri -Method Get -Headers $azureDevOpsRestApiHeaders
$webClient = New-Object System.Net.WebClient
[System.Collections.Generic.List[string]]$workItems = @()

foreach ($testRun in $testRuns.value)
{
$testRunResultsUri = "$($testRun.url)/results?api-version=5.0"
$testResults = Invoke-RestMethod -Uri "$($testRun.url)/results?api-version=5.0" -Method Get -Headers $azureDevOpsRestApiHeaders
$isTestRunNameShown = $false

foreach ($testResult in $testResults.value)
{
$info = ConvertFrom-Json $testResult.comment
$helixJobId = $info.HelixJobId
$helixWorkItemName = $info.HelixWorkItemName

$workItem = "$helixJobId-$helixWorkItemName"

if (-not $workItems.Contains($workItem))
{
$workItems.Add($workItem)
$filesQueryUri = "https://helix.dot.net/api/2019-06-17/jobs/$helixJobId/workitems/$helixWorkItemName/files"
$files = Invoke-RestMethod -Uri $filesQueryUri -Method Get

$screenShots = $files | where { $_.Name.EndsWith(".jpg") }
$dumps = $files | where { $_.Name.EndsWith(".dmp") }
$visualTreeMasters = $files | where { $_.Name.EndsWith(".xml") -And (-Not $_.Name.Contains('testResults')) }
if($screenShots.Count+$dumps.Count+$visualTreeMasters.Count -gt 0)
{
if(-Not $isTestRunNameShown)
{
Out-File -FilePath $helixLinkFile -Append -InputObject "<h2>$($testRun.name)</h2>"
$isTestRunNameShown = $true
}
Out-File -FilePath $helixLinkFile -Append -InputObject "<h3>$helixWorkItemName</h3>"
Generate-File-Links $screenShots "Screenshots"
Generate-File-Links $dumps "CrashDumps"
Generate-File-Links $visualTreeMasters "VisualTreeMasters"
$misc = $files | where { ($screenShots -NotContains $_) -And ($dumps -NotContains $_) -And ($visualTreeMasters -NotContains $_) }
Generate-File-Links $misc "Misc"

if( -Not (Test-Path $visualTreeMasterFolder) )
{
New-Item $visualTreeMasterFolder -ItemType Directory
}
foreach($masterFile in $visualTreeMasters)
{
$destination = "$visualTreeMasterFolder\$($masterFile.Name)"
Write-Host "Copying $($masterFile.Name) to $destination"
$webClient.DownloadFile($masterFile.Link, $destination)
}
}
}
}
}

if(Test-Path $visualTreeMasterFolder)
{
Write-Host "Merge duplicated master files..."
$masterFiles = Get-ChildItem $visualTreeMasterFolder
$prefixList = @()
foreach($file in $masterFiles)
{
$prefix = $file.BaseName.Split('-')[0]
if($prefixList -NotContains $prefix)
{
$prefixList += $prefix
}
}

foreach($prefix in $prefixList)
{
$filesToDelete = @()
$versionedMasters = $masterFiles | Where { $_.BaseName.StartsWith($prefix) } | Sort-Object -Property Name -Descending
for ($i=0; $i -lt $versionedMasters.Length-1; $i++)
{
$v1 = Get-Content $versionedMasters[$i].FullName
$v2 = Get-Content $versionedMasters[$i+1].FullName
$diff = Compare-Object $v1 $v2
if($diff.Length -eq 0)
{
$filesToDelete += $versionedMasters[$i]
}
}
$filesToDelete | ForEach-Object {
Write-Host "Deleting $($_.Name)"
Remove-Item $_.FullName
}

Write-Host "Renaming $($versionedMasters[-1].Name) to $prefix.xml"
Move-Item $versionedMasters[-1].FullName "$visualTreeMasterFolder\$prefix.xml" -Force
}
}
1 change: 1 addition & 0 deletions build/Helix/runtests.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ move te.wtl te_rerun_multiple.wtl

copy /y te_rerun_multiple.wtl %HELIX_WORKITEM_UPLOAD_ROOT%
copy /y WexLogFileOutput\*.jpg %HELIX_WORKITEM_UPLOAD_ROOT%
powershell -ExecutionPolicy Bypass .\CopyVisualTreeMasters.ps1

:SkipReruns

Expand Down
39 changes: 26 additions & 13 deletions dev/AutoSuggestBox/APITests/AutoSuggestBoxTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,19 +38,7 @@ public void VerifyAutoSuggestBoxCornerRadius()
return;
}

AutoSuggestBox autoSuggestBox = null;
RunOnUIThread.Execute(() =>
{
autoSuggestBox = new AutoSuggestBox();
List<string> suggestions = new List<string>
{
"Item 1", "Item 2", "Item 3"
};
autoSuggestBox.ItemsSource = suggestions;
});
IdleSynchronizer.Wait();
Verify.IsNotNull(autoSuggestBox);
TestUtilities.SetAsVisualTreeRoot(autoSuggestBox);
var autoSuggestBox = SetupAutoSuggestBox();

RunOnUIThread.Execute(() =>
{
Expand Down Expand Up @@ -78,5 +66,30 @@ public void VerifyAutoSuggestBoxCornerRadius()
});
}

[TestMethod]
public void VerifyVisualTree()
{
var autoSuggestBox = SetupAutoSuggestBox();
VisualTreeTestHelper.VerifyVisualTree(root: autoSuggestBox, masterFilePrefix: "AutoSuggestBox");
}

private AutoSuggestBox SetupAutoSuggestBox()
{
AutoSuggestBox autoSuggestBox = null;
RunOnUIThread.Execute(() =>
{
autoSuggestBox = new AutoSuggestBox();
List<string> suggestions = new List<string>
{
"Item 1", "Item 2", "Item 3"
};
autoSuggestBox.ItemsSource = suggestions;
autoSuggestBox.Width = 400;
});
TestUtilities.SetAsVisualTreeRoot(autoSuggestBox);
Verify.IsNotNull(autoSuggestBox);
return autoSuggestBox;
}

}
}
48 changes: 48 additions & 0 deletions dev/CalendarView/APITests/CalendarViewTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in the project root for license information.

using Common;
using MUXControlsTestApp.Utilities;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Media;
using System.Linq;
using System.Collections.Generic;
using System;

#if USING_TAEF
using WEX.TestExecution;
using WEX.TestExecution.Markup;
using WEX.Logging.Interop;
#else
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Microsoft.VisualStudio.TestTools.UnitTesting.Logging;
#endif

namespace Windows.UI.Xaml.Tests.MUXControls.ApiTests
{
[TestClass]
public class CalendarViewTests
{
[TestCleanup]
public void TestCleanup()
{
TestUtilities.ClearVisualTreeRoot();
}

[TestMethod]
public void VerifyVisualTree()
{
CalendarView calendarView = null;
RunOnUIThread.Execute(() =>
{
calendarView = new CalendarView();
calendarView.Width = 400;
calendarView.Height = 400;
calendarView.SetDisplayDate(new DateTime(2000, 1, 1));
});
TestUtilities.SetAsVisualTreeRoot(calendarView);

VisualTreeTestHelper.VerifyVisualTree(root: calendarView, masterFilePrefix: "CalendarView");
}
}
}
Loading

0 comments on commit 44575e6

Please sign in to comment.