diff --git a/eng/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/scripts/job-matrix/Create-JobMatrix.ps1 index c98e08ce5e91..32956f058beb 100644 --- a/eng/scripts/job-matrix/Create-JobMatrix.ps1 +++ b/eng/scripts/job-matrix/Create-JobMatrix.ps1 @@ -12,14 +12,23 @@ param ( [Parameter(Mandatory=$True)][string] $ConfigPath, [Parameter(Mandatory=$True)][string] $Selection, [Parameter(Mandatory=$False)][string] $DisplayNameFilter, - [Parameter(Mandatory=$False)][array] $Filters + [Parameter(Mandatory=$False)][array] $Filters, + [Parameter(Mandatory=$False)][array] $NonSparseParameters ) . $PSScriptRoot/job-matrix-functions.ps1 $config = GetMatrixConfigFromJson (Get-Content $ConfigPath) +# Strip empty string filters in order to be able to use azure pipelines yaml join() +$Filters = $Filters | Where-Object { $_ } + +[array]$matrix = GenerateMatrix ` + -config $config ` + -selectFromMatrixType $Selection ` + -displayNameFilter $DisplayNameFilter ` + -filters $Filters ` + -nonSparseParameters $NonSparseParameters -[array]$matrix = GenerateMatrix $config $Selection $DisplayNameFilter $Filters $serialized = SerializePipelineMatrix $matrix Write-Output $serialized.pretty diff --git a/eng/scripts/job-matrix/README.md b/eng/scripts/job-matrix/README.md index 61df7a2435b5..8b370c0f09d8 100644 --- a/eng/scripts/job-matrix/README.md +++ b/eng/scripts/job-matrix/README.md @@ -2,18 +2,20 @@ * [Usage in a pipeline](#usage-in-a-pipeline) * [Matrix config file syntax](#matrix-config-file-syntax) - * [Fields](#fields) - * [matrix](#matrix) - * [include](#include) - * [exclude](#exclude) - * [displayNames](#displaynames) + * [Fields](#fields) + * [matrix](#matrix) + * [include](#include) + * [exclude](#exclude) + * [displayNames](#displaynames) + * [$IMPORT](#import) * [Matrix Generation behavior](#matrix-generation-behavior) - * [all](#all) - * [sparse](#sparse) - * [include/exclude](#includeexclude) - * [displayNames](#displaynames-1) - * [Filters](#filters) - * [Under the hood](#under-the-hood) + * [all](#all) + * [sparse](#sparse) + * [include/exclude](#includeexclude) + * [displayNames](#displaynames-1) + * [Filters](#filters) + * [NonSparseParameters](#nonsparseparameters) + * [Under the hood](#under-the-hood) * [Testing](#testing) @@ -38,13 +40,14 @@ jobs: - Name: base_product_matrix Path: /eng/pipelines/matrix.json Selection: sparse - GenerateVMJobs: true + NonSparseParameters: + GenerateVMJobs: true - Name: sdk_specific_matrix Path: /sdk/foobar/matrix.json Selection: all - GenerateContainerJobs: true - steps: - - pwsh: + GenerateContainerJobs: true + steps: + - pwsh: ... ``` @@ -58,14 +61,14 @@ type is useful for when 2 or more parameters need to be grouped together, but wi "": [ ], "": [ ], "": { - "": { - "", - }, - "": { - "", - } + "": { + "", + }, + "": { + "", + } } } "include": [ , , ... ], @@ -144,6 +147,150 @@ readable value here. For example: } ``` +#### $IMPORT + +Matrix configs can also import another matrix config. The effect of this is the imported matrix will be generated, +and then the importing config will be combined with that matrix (as if each entry of the imported matrix was a parameter). +To import a matrix, add a parameter with the key `$IMPORT`: + +``` +"matrix": { + "$IMPORT": "path/to/matrix.json", + "JavaVersion": [ "1.8", "1.11" ] +} +``` + +Importing can be useful, for example, in cases where there is a shared base matrix, but there is a need to run it +once for each instance of a language version. + +The processing order is as follows: + +Given a matrix and import matrix like below: +``` +{ + "matrix": { + "$IMPORT": "example-matrix.json", + "endpointType": [ "storage", "cosmos" ], + "JavaVersion": [ "1.8", "1.11" ] + }, + "include": [ + { + "operatingSystem": "windows", + "mode": "TestFromSource", + "JavaVersion": "1.8" + } + ] +} + +### example-matrix.json to import +{ + "matrix": { + "operatingSystem": [ "windows", "linux" ], + "client": [ "netty", "okhttp" ] + }, + "include": [ + { + "operatingSystem": "mac", + "client": "netty" + } + ] +} +``` + +1. The base matrix is generated (sparse in this example): + ``` + { + "storage_18": { + "endpointType": "storage", + "JavaVersion": "1.8" + }, + "cosmos_111": { + "endpointType": "cosmos", + "JavaVersion": "1.11" + } + } + ``` +1. The imported base matrix is generated (sparse in this example): + ``` + { + "windows_netty": { + "operatingSystem": "windows", + "client": "netty" + }, + "linux_okhttp": { + "operatingSystem": "linux", + "client": "okhttp" + } + } + ``` +1. Includes/excludes from the imported matrix get applied to the imported matrix + ``` + { + "windows_netty": { + "operatingSystem": "windows", + "client": "netty" + }, + "linux_okhttp": { + "operatingSystem": "linux", + "client": "okhttp" + }, + "mac_netty": { + "operatingSystem": "mac", + "client": "netty" + } + } + ``` +1. The base matrix is multipled by the imported matrix (in this case, the base matrix has 2 elements, and the imported + matrix has 3 elements, so the product is a matrix with 6 elements: + ``` + "storage_18_windows_netty": { + "endpointType": "storage", + "JavaVersion": "1.8", + "operatingSystem": "windows", + "client": "netty" + }, + "storage_18_linux_okhttp": { + "endpointType": "storage", + "JavaVersion": "1.8", + "operatingSystem": "linux", + "client": "okhttp" + }, + "storage_18_mac_netty": { + "endpointType": "storage", + "JavaVersion": "1.8", + "operatingSystem": "mac", + "client": "netty" + }, + "cosmos_111_windows_netty": { + "endpointType": "cosmos", + "JavaVersion": "1.11", + "operatingSystem": "windows", + "client": "netty" + }, + "cosmos_111_linux_okhttp": { + "endpointType": "cosmos", + "JavaVersion": "1.11", + "operatingSystem": "linux", + "client": "okhttp" + }, + "cosmos_111_mac_netty": { + "endpointType": "cosmos", + "JavaVersion": "1.11", + "operatingSystem": "mac", + "client": "netty" + } + } + ``` +1. Includes/excludes from the top-level matrix get applied to the multiplied matrix, so the below element will be added + to the above matrix, for an output matrix with 7 elements: + ``` + "windows_TestFromSource_18": { + "operatingSystem": "windows", + "mode": "TestFromSource", + "JavaVersion": "1.8" + } + ``` + ## Matrix Generation behavior #### all @@ -220,7 +367,7 @@ The logic for generating display names works like this: - Join parameter values by "_" a. If the parameter value exists as a key in `displayNames` in the matrix config, replace it with that value. b. For each name value, strip all non-alphanumeric characters (excluding "_"). - c. If the name is greater than 100 characters, truncate it. + c. If the name is greater than 100 characters, truncate it. #### Filters @@ -242,6 +389,38 @@ named "ExcludedKey", a framework variable containing either "461" or "5.0", and -Filters @("ExcludedKey=^$", "framework=(461|5\.0)", "SupportedClouds=^$|.*Public.*") ``` +#### NonSparseParameters + +Sometimes it may be necessary to generate a sparse matrix, but keep the full combination of a few parameters. The +NonSparseParameters argument allows for more fine-grained control of matrix generation. For example: + +``` +./Create-JobMatrix.ps1 ` + -ConfigPath /path/to/matrix.json ` + -Selection sparse ` + -NonSparseParameters @("JavaTestVersion") +``` + +Given a matrix like below with `JavaTestVersion` marked as a non-sparse parameter: + +``` +{ + "matrix": { + "Agent": { + "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }, + "ubuntu-1804": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" }, + "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } + }, + "JavaTestVersion": [ "1.8", "1.11" ], + "AZURE_TEST_HTTP_CLIENTS": "netty", + "ArmTemplateParameters": [ "@{endpointType='storage'}", "@{endpointType='cosmos'}" ] + } +} +``` + +A matrix with 6 entries will be generated: A sparse matrix of Agent, AZURE_TEST_HTTP_CLIENTS and ArmTemplateParameters +(3 total entries) will be multipled by the two `JavaTestVersion` parameters `1.8` and `1.11`. + #### Under the hood The script generates an N-dimensional matrix with dimensions equal to the parameter array lengths. For example, diff --git a/eng/scripts/job-matrix/job-matrix-functions.modification.tests.ps1 b/eng/scripts/job-matrix/job-matrix-functions.modification.tests.ps1 new file mode 100644 index 000000000000..9a861cb0ba7c --- /dev/null +++ b/eng/scripts/job-matrix/job-matrix-functions.modification.tests.ps1 @@ -0,0 +1,219 @@ +Import-Module Pester + + +BeforeAll { + . ./job-matrix-functions.ps1 + + function CompareMatrices([Array]$matrix, [Array]$expected) { + $matrix.Length | Should -Be $expected.Length + + for ($i = 0; $i -lt $matrix.Length; $i++) { + foreach ($entry in $matrix[$i]) { + $expected[$i].name | Should -Be $entry.name + foreach ($param in $entry.parameters.GetEnumerator()) { + $expected[$i].parameters[$param.Name] | Should -Be $param.Value + } + } + } + } +} + +Describe "Platform Matrix nonSparse" -Tag "nonsparse" { + BeforeEach { + $matrixJson = @' +{ + "matrix": { + "testField1": [ 1, 2 ], + "testField2": [ 1, 2, 3 ], + "testField3": [ 1, 2, 3, 4 ], + } +} +'@ + $config = GetMatrixConfigFromJson $matrixJson + } + + It "Should process nonSparse parameters" { + $parameters, $nonSparse = ProcessNonSparseParameters $config.orderedMatrix "testField1","testField3" + $parameters.Count | Should -Be 1 + $parameters["testField2"] | Should -Be 1,2,3 + $nonSparse.Count | Should -Be 2 + $nonSparse["testField1"] | Should -Be 1,2 + $nonSparse["testField3"] | Should -Be 1,2,3,4 + + $parameters, $nonSparse = ProcessNonSparseParameters $config.orderedMatrix "testField3" + $parameters.Count | Should -Be 2 + $parameters.Contains("testField3") | Should -Be $false + $nonSparse.Count | Should -Be 1 + $nonSparse["testField3"] | Should -Be 1,2,3,4 + } + + It "Should ignore nonSparse with all selection" { + $matrix = GenerateMatrix $config "all" -nonSparseParameters "testField3" + $matrix.Length | Should -Be 24 + } + + It "Should combine sparse matrix with nonSparse parameters" { + $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3" + $matrix.Length | Should -Be 12 + } + + It "Should combine with multiple nonSparse fields" { + $matrixJson = @' +{ + "matrix": { + "testField1": [ 1, 2 ], + "testField2": [ 1, 2 ], + "testField3": [ 31, 32 ], + "testField4": [ 41, 42 ] + } +} +'@ + $config = GetMatrixConfigFromJson $matrixJson + + $matrix = GenerateMatrix $config "all" -nonSparseParameters "testField3","testField4" + $matrix.Length | Should -Be 16 + + $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3","testField4" + $matrix.Length | Should -Be 8 + } +} + +Describe "Platform Matrix Import" -Tag "import" { + It "Should generate a matrix with nonSparseParameters and an imported sparse matrix" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField": [ "test1", "test2" ] + } +} +'@ + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" -nonSparseParameters "testField" + + $matrix.Length | Should -Be 6 + + $matrix[0].name | Should -Be test1_foo1_bar1 + $matrix[0].parameters.testField | Should -Be "test1" + $matrix[0].parameters.Foo | Should -Be "foo1" + $matrix[2].name | Should -Be test1_importedBaz + $matrix[2].parameters.testField | Should -Be "test1" + $matrix[2].parameters.Baz | Should -Be "importedBaz" + $matrix[4].name | Should -Be test2_foo2_bar2 + $matrix[4].parameters.testField | Should -Be "test2" + $matrix[4].parameters.Foo | Should -Be "foo2" + } + + It "Should generate a sparse matrix with an imported a sparse matrix" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField1": [ "test11", "test12" ], + "testField2": [ "test21", "test22" ] + } +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "testField1": "test11", "testField2": "test21", "Foo": "foo1", "Bar": "bar1" }, + "name": "test11_test21_foo1_bar1" + }, + { + "parameters": { "testField1": "test11", "testField2": "test21", "Foo": "foo2", "Bar": "bar2" }, + "name": "test11_test21_foo2_bar2" + }, + { + "parameters": { "testField1": "test11", "testField2": "test21", "Baz": "importedBaz" }, + "name": "test11_test21_importedBaz" + }, + { + "parameters": { "testField1": "test12", "testField2": "test22", "Foo": "foo1", "Bar": "bar1" }, + "name": "test12_test22_foo1_bar1" + }, + { + "parameters": { "testField1": "test12", "testField2": "test22", "Foo": "foo2", "Bar": "bar2" }, + "name": "test12_test22_foo2_bar2" + }, + { + "parameters": { "testField1": "test12", "testField2": "test22", "Baz": "importedBaz" }, + "name": "test12_test22_importedBaz" + } +] +'@ + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 6 + CompareMatrices $matrix $expected + } + + It "Should import a sparse matrix with import, include, and exclude" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField": [ "test1", "test2", "test3" ], + }, + "include": [ + { + "testImportIncludeName": [ "testInclude1", "testInclude2" ] + } + ], + "exclude": [ + { + "testField": "test1" + }, + { + "testField": "test3", + "Baz": "importedBaz" + } + ] +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "testField": "test2", "Foo": "foo1", "Bar": "bar1" }, + "name": "test2_foo1_bar1" + }, + { + "parameters": { "testField": "test2", "Foo": "foo2", "Bar": "bar2" }, + "name": "test2_foo2_bar2" + }, + { + "parameters": { "testField": "test2", "Baz": "importedBaz" }, + "name": "test2_importedBaz" + }, + { + "parameters": { "testField": "test3", "Foo": "foo1", "Bar": "bar1" }, + "name": "test3_foo1_bar1" + }, + { + "parameters": { "testField": "test3", "Foo": "foo2", "Bar": "bar2" }, + "name": "test3_foo2_bar2" + }, + { + "parameters": { "testImportIncludeName": "testInclude1" }, + "name": "testInclude1" + }, + { + "parameters": { "testImportIncludeName": "testInclude2" }, + "name": "testInclude2" + } +] +'@ + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 7 + CompareMatrices $matrix $expected + } +} diff --git a/eng/scripts/job-matrix/job-matrix-functions.ps1 b/eng/scripts/job-matrix/job-matrix-functions.ps1 index 2fbd8958acf7..07b2363765cb 100644 --- a/eng/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/scripts/job-matrix/job-matrix-functions.ps1 @@ -9,6 +9,8 @@ class MatrixConfig { [Array]$exclude } +$IMPORT_KEYWORD = '$IMPORT' + function CreateDisplayName([string]$parameter, [Hashtable]$displayNamesLookup) { $name = $parameter.ToString() @@ -25,23 +27,31 @@ function CreateDisplayName([string]$parameter, [Hashtable]$displayNamesLookup) function GenerateMatrix( [MatrixConfig]$config, - [string]$selectFromMatrixType, - [string]$displayNameFilter = ".*", - [array]$filters = @() + [String]$selectFromMatrixType, + [String]$displayNameFilter = ".*", + [Array]$filters = @(), + [Array]$nonSparseParameters = @() ) { + $orderedMatrix, $importedMatrix = ProcessImport $config.orderedMatrix $selectFromMatrixType if ($selectFromMatrixType -eq "sparse") { - [Array]$matrix = GenerateSparseMatrix $config.orderedMatrix $config.displayNamesLookup + [Array]$matrix = GenerateSparseMatrix $orderedMatrix $config.displayNamesLookup $nonSparseParameters } elseif ($selectFromMatrixType -eq "all") { - [Array]$matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup + [Array]$matrix = GenerateFullMatrix $orderedMatrix $config.displayNamesLookup } else { throw "Matrix generator not implemented for selectFromMatrixType: $($platform.selectFromMatrixType)" } + # Combine with imported after matrix generation, since a sparse selection should result in a full combination of the + # top level and imported sparse matrices (as opposed to a sparse selection of both matrices). + if ($importedMatrix) { + [Array]$matrix = CombineMatrices $matrix $importedMatrix + } + if ($config.exclude) { [Array]$matrix = ProcessExcludes $matrix $config.exclude } if ($config.include) { - [Array]$matrix = ProcessIncludes $matrix $config.include $config.displayNamesLookup + [Array]$matrix = ProcessIncludes $config $matrix $selectFromMatrixType } [Array]$matrix = FilterMatrixDisplayName $matrix $displayNameFilter @@ -49,6 +59,28 @@ function GenerateMatrix( return $matrix } +function ProcessNonSparseParameters( + [System.Collections.Specialized.OrderedDictionary]$parameters, + [Array]$nonSparseParameters +) { + if (!$nonSparseParameters) { + return $parameters, $null + } + + $sparse = [ordered]@{} + $nonSparse = [ordered]@{} + + foreach ($param in $parameters.GetEnumerator()) { + if ($param.Name -in $nonSparseParameters) { + $nonSparse[$param.Name] = $param.Value + } else { + $sparse[$param.Name] = $param.Value + } + } + + return $sparse, $nonSparse +} + function FilterMatrixDisplayName([array]$matrix, [string]$filter) { return $matrix | ForEach-Object { if ($_.Name -match $filter) { @@ -97,7 +129,7 @@ function ParseFilter([string]$filter) { # Importing the JSON as PSCustomObject preserves key ordering, # whereas ConvertFrom-Json -AsHashtable does not -function GetMatrixConfigFromJson($jsonConfig) +function GetMatrixConfigFromJson([String]$jsonConfig) { [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json $config.orderedMatrix = [ordered]@{} @@ -137,8 +169,7 @@ function ProcessExcludes([Array]$matrix, [Array]$excludes) $exclusionMatrix = @() foreach ($exclusion in $excludes) { - $converted = ConvertToMatrixArrayFormat $exclusion - $full = GenerateFullMatrix $converted + $full = GenerateFullMatrix $exclusion $exclusionMatrix += $full } @@ -154,45 +185,83 @@ function ProcessExcludes([Array]$matrix, [Array]$excludes) return $matrix | Where-Object { !$_.parameters.Contains($deleteKey) } } -function ProcessIncludes([Array]$matrix, [Array]$includes, [Hashtable]$displayNamesLookup) +function ProcessIncludes([MatrixConfig]$config, [Array]$matrix) { - foreach ($inclusion in $includes) { - $converted = ConvertToMatrixArrayFormat $inclusion - $full = GenerateFullMatrix $converted $displayNamesLookup - $matrix += $full + $inclusionMatrix = @() + foreach ($inclusion in $config.include) { + $full = GenerateFullMatrix $inclusion $config.displayNamesLookup + $inclusionMatrix += $full } - return $matrix + return $matrix + $inclusionMatrix } -function MatrixElementMatch([System.Collections.Specialized.OrderedDictionary]$source, [System.Collections.Specialized.OrderedDictionary]$target) +function ProcessImport([System.Collections.Specialized.OrderedDictionary]$matrix, [String]$selection) { - if ($target.Count -eq 0) { - return $false + if (!$matrix -or !$matrix.Contains($IMPORT_KEYWORD)) { + return $matrix } - foreach ($key in $target.Keys) { - if (-not $source.Contains($key) -or $source[$key] -ne $target[$key]) { - return $false + $importPath = $matrix[$IMPORT_KEYWORD] + $matrix.Remove($IMPORT_KEYWORD) + + $matrixConfig = GetMatrixConfigFromJson (Get-Content $importPath) + $importedMatrix = GenerateMatrix $matrixConfig $selection + + return $matrix, $importedMatrix +} + +function CombineMatrices([Array]$matrix1, [Array]$matrix2) +{ + $combined = @() + if (!$matrix1) { + return $matrix2 + } + if (!$matrix2) { + return $matrix1 + } + + foreach ($entry1 in $matrix1) { + foreach ($entry2 in $matrix2) { + $newEntry = @{ + name = $entry1.name + parameters = CloneOrderedDictionary $entry1.parameters + } + foreach($param in $entry2.parameters.GetEnumerator()) { + if (!$newEntry.Contains($param.Name)) { + $newEntry.parameters[$param.Name] = $param.Value + } else { + Write-Warning "Skipping duplicate parameter `"$($param.Name)`" when combining matrix." + } + } + + # The maximum allowed matrix name length is 100 characters + $entry2.name = $entry2.name.TrimStart("job_") + $newEntry.name = $newEntry.name, $entry2.name -join "_" + if ($newEntry.name.Length -gt 100) { + $newEntry.name = $newEntry.name[0..99] -join "" + } + + $combined += $newEntry } } - return $true + return $combined } -function ConvertToMatrixArrayFormat([System.Collections.Specialized.OrderedDictionary]$matrix) +function MatrixElementMatch([System.Collections.Specialized.OrderedDictionary]$source, [System.Collections.Specialized.OrderedDictionary]$target) { - $converted = [Ordered]@{} + if ($target.Count -eq 0) { + return $false + } - foreach ($key in $matrix.Keys) { - if ($matrix[$key] -isnot [Array]) { - $converted[$key] = ,$matrix[$key] - } else { - $converted[$key] = $matrix[$key] + foreach ($key in $target.Keys) { + if (!$source.Contains($key) -or $source[$key] -ne $target[$key]) { + return $false } } - return $converted + return $true } function CloneOrderedDictionary([System.Collections.Specialized.OrderedDictionary]$dictionary) { @@ -219,13 +288,33 @@ function SerializePipelineMatrix([Array]$matrix) } } -function GenerateSparseMatrix([System.Collections.Specialized.OrderedDictionary]$parameters, [Hashtable]$displayNamesLookup) -{ +function GenerateSparseMatrix( + [System.Collections.Specialized.OrderedDictionary]$parameters, + [Hashtable]$displayNamesLookup, + [Array]$nonSparseParameters = @() +) { + $parameters, $nonSparse = ProcessNonSparseParameters $parameters $nonSparseParameters [Array]$dimensions = GetMatrixDimensions $parameters - $size = ($dimensions | Measure-Object -Maximum).Maximum - [Array]$matrix = GenerateFullMatrix $parameters $displayNamesLookup + $sparseMatrix = @() + $indexes = GetSparseMatrixIndexes $dimensions + foreach ($idx in $indexes) { + $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions + } + + if ($nonSparse) { + [Array]$allOfMatrix = GenerateFullMatrix $nonSparse $displayNamesLookup + return CombineMatrices $allOfMatrix $sparseMatrix + } + + return $sparseMatrix +} + +function GetSparseMatrixIndexes([Array]$dimensions) +{ + $size = ($dimensions | Measure-Object -Maximum).Maximum + $indexes = @() # With full matrix, retrieve items by doing diagonal lookups across the matrix N times. # For example, given a matrix with dimensions 3, 2, 2: @@ -238,14 +327,16 @@ function GenerateSparseMatrix([System.Collections.Specialized.OrderedDictionary] for ($j = 0; $j -lt $dimensions.Length; $j++) { $idx += $i % $dimensions[$j] } - $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions + $indexes += ,$idx } - return $sparseMatrix + return $indexes } -function GenerateFullMatrix([System.Collections.Specialized.OrderedDictionary] $parameters, [Hashtable]$displayNamesLookup = @{}) -{ +function GenerateFullMatrix( + [System.Collections.Specialized.OrderedDictionary] $parameters, + [Hashtable]$displayNamesLookup = @{} +) { # Handle when the config does not have a matrix specified (e.g. only the include field is specified) if ($parameters.Count -eq 0) { return @() @@ -256,7 +347,7 @@ function GenerateFullMatrix([System.Collections.Specialized.OrderedDictionary] $ $matrix = [System.Collections.ArrayList]::new() InitializeMatrix $parameterArray $displayNamesLookup $matrix - return $matrix.ToArray() + return $matrix } function CreateMatrixEntry([System.Collections.Specialized.OrderedDictionary]$permutation, [Hashtable]$displayNamesLookup = @{}) @@ -308,19 +399,20 @@ function InitializeMatrix [System.Collections.ArrayList]$permutations, $permutation = [Ordered]@{} ) + $head, $tail = $parameters - if (-not $parameters) { + if (!$head) { $entry = CreateMatrixEntry $permutation $displayNamesLookup $permutations.Add($entry) | Out-Null return } - $head, $tail = $parameters - foreach ($value in $head.value) { - $newPermutation = CloneOrderedDictionary($permutation) + # This behavior implicitly treats non-array values as single elements + foreach ($value in $head.Value) { + $newPermutation = CloneOrderedDictionary $permutation if ($value -is [PSCustomObject]) { foreach ($nestedParameter in $value.PSObject.Properties) { - $nestedPermutation = CloneOrderedDictionary($newPermutation) + $nestedPermutation = CloneOrderedDictionary $newPermutation $nestedPermutation[$nestedParameter.Name] = $nestedParameter.Value InitializeMatrix $tail $displayNamesLookup $permutations $nestedPermutation } @@ -334,11 +426,11 @@ function InitializeMatrix function GetMatrixDimensions([System.Collections.Specialized.OrderedDictionary]$parameters) { $dimensions = @() - foreach ($val in $parameters.Values) { - if ($val -is [PSCustomObject]) { - $dimensions += ($val.PSObject.Properties | Measure-Object).Count - } elseif ($val -is [Array]) { - $dimensions += $val.Length + foreach ($param in $parameters.GetEnumerator()) { + if ($param.Value -is [PSCustomObject]) { + $dimensions += ($param.Value.PSObject.Properties | Measure-Object).Count + } elseif ($param.Value -is [Array]) { + $dimensions += $param.Value.Length } else { $dimensions += 1 } diff --git a/eng/scripts/job-matrix/job-matrix-functions.tests.ps1 b/eng/scripts/job-matrix/job-matrix-functions.tests.ps1 index 5e50596e6cf6..cc36fcd61680 100644 --- a/eng/scripts/job-matrix/job-matrix-functions.tests.ps1 +++ b/eng/scripts/job-matrix/job-matrix-functions.tests.ps1 @@ -46,7 +46,6 @@ BeforeAll { ] } "@ - $config = GetMatrixConfigFromJson $matrixConfig } Describe "Matrix-Lookup" -Tag "lookup" { @@ -394,25 +393,31 @@ Describe "Platform Matrix Generation" -Tag "generate" { } Describe "Config File Object Conversion" -Tag "convert" { - It "Should convert a matrix config" { - $converted = GetMatrixConfigFromJson $matrixConfig + BeforeEach { + $config = GetMatrixConfigFromJson $matrixConfig + } - $converted.orderedMatrix | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] - $converted.orderedMatrix.operatingSystem[0] | Should -Be "windows-2019" + It "Should convert a matrix config" { + $config.orderedMatrix | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + $config.orderedMatrix.operatingSystem[0] | Should -Be "windows-2019" - $converted.displayNamesLookup | Should -BeOfType [Hashtable] - $converted.displayNamesLookup["--enableFoo"] | Should -Be "withFoo" + $config.displayNamesLookup | Should -BeOfType [Hashtable] + $config.displayNamesLookup["--enableFoo"] | Should -Be "withFoo" - $converted.include | ForEach-Object { + $config.include | ForEach-Object { $_ | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] } - $converted.exclude | ForEach-Object { + $config.exclude | ForEach-Object { $_ | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] } } } Describe "Platform Matrix Post Transformation" -Tag "transform" { + BeforeEach { + $config = GetMatrixConfigFromJson $matrixConfig + } + It "Should match partial matrix elements" -TestCases @( @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1 }; expected = $true } @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1; b = 2 }; expected = $true } @@ -423,29 +428,6 @@ Describe "Platform Matrix Post Transformation" -Tag "transform" { MatrixElementMatch $source $target | Should -Be $expected } - It "Should convert singular elements" { - $ordered = [Ordered]@{} - $ordered.Add("a", 1) - $ordered.Add("b", 2) - $matrix = ConvertToMatrixArrayFormat $ordered - $matrix.a.Length | Should -Be 1 - $matrix.b.Length | Should -Be 1 - - $ordered = [Ordered]@{} - $ordered.Add("a", 1) - $ordered.Add("b", @(1, 2)) - $matrix = ConvertToMatrixArrayFormat $ordered - $matrix.a.Length | Should -Be 1 - $matrix.b.Length | Should -Be 2 - - $ordered = [Ordered]@{} - $ordered.Add("a", @(1, 2)) - $ordered.Add("b", @()) - $matrix = ConvertToMatrixArrayFormat $ordered - $matrix.a.Length | Should -Be 2 - $matrix.b.Length | Should -Be 0 - } - It "Should remove matrix elements based on exclude filters" { $matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup $withExclusion = ProcessExcludes $matrix $config.exclude @@ -458,7 +440,7 @@ Describe "Platform Matrix Post Transformation" -Tag "transform" { It "Should add matrix elements based on include elements" { $matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup - $withInclusion = ProcessIncludes $matrix $config.include $config.displayNamesLookup + $withInclusion = ProcessIncludes $config $matrix "all" $withInclusion.Length | Should -Be 15 } diff --git a/eng/scripts/job-matrix/test-import-matrix.json b/eng/scripts/job-matrix/test-import-matrix.json new file mode 100644 index 000000000000..2048bd390c5f --- /dev/null +++ b/eng/scripts/job-matrix/test-import-matrix.json @@ -0,0 +1,11 @@ +{ + "matrix": { + "Foo": [ "foo1", "foo2" ], + "Bar": [ "bar1", "bar2" ] + }, + "include": [ + { + "Baz": "importedBaz" + } + ] +}