Skip to content

Commit 7a89243

Browse files
Pan-QiPan-Qi
committed
filter
1 parent b1fc7ba commit 7a89243

File tree

3 files changed

+231
-20
lines changed

3 files changed

+231
-20
lines changed

‎.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
[CmdletBinding(DefaultParameterSetName="AllSet")]
2+
param (
3+
[int]$MaxParallelJobs = 3,
4+
[string[]]$ChangedFiles
5+
)
6+
7+
$autorestFolders = @{}
8+
9+
foreach ($file in $changedFiles) {
10+
if ($file -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
11+
$parent = $Matches[2]
12+
$child = $Matches[3]
13+
$key = "$parent/$child"
14+
15+
$autorestFolders[$key] = $true
16+
}
17+
}
18+
19+
$subModules = $autorestFolders.Keys
20+
Write-Host "Outer Group ${subModules}:"
21+
22+
function Split-List {
23+
param (
24+
[array]$subModules,
25+
[int]$maxParallelJobs
26+
)
27+
28+
$count = $subModules.Count
29+
$n = [Math]::Min($count, $maxParallelJobs)
30+
if ($n -eq 0) {
31+
return @()
32+
}
33+
34+
$result = @()
35+
$sizePerGroup = [Math]::Ceiling($count / $n)
36+
37+
for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
38+
$group = $subModules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
39+
$result += ,$group
40+
}
41+
42+
return $result
43+
}
44+
45+
$devidedSubModules = Split-List -subModules $subModules -maxParallelJobs $MaxParallelJobs
46+
47+
$index = 0
48+
foreach ($subModules in $devidedSubModules) {
49+
Write-Host "Outer Group ${index}:"
50+
$subIndex = 0
51+
foreach ($subModule in $subModules) {
52+
Write-Host "Inner Group ${subIndex}: $($subModule -join ',')"
53+
$subIndex++
54+
}
55+
56+
$moduleNamesStr = $subModules -join ','
57+
$key = ($index + 1).ToString() + "-" + $subModules.Count
58+
$MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
59+
$index++
60+
}
61+
62+
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
63+
$MatrixStr = $MatrixStr.Substring(1)
64+
}
65+
Write-Host "##vso[task.setVariable variable=analyzeTargets;isOutput=true]{$MatrixStr}"

‎.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1

+6-3
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,12 @@ foreach ($subModules in $devidedSubModules) {
7272

7373
$moduleNames = $subModules | ForEach-Object { $_[0] }
7474
$moduleNamesStr = $moduleNames -join ','
75-
$MatrixStr="$MatrixStr,'" + ($index + 1) + "-" + $subModules.Count + "':{'Target':'$moduleNamesStr'}"
75+
$key = ($index + 1).ToString() + "-" + $subModules.Count
76+
$MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
7677
$index++
7778
}
7879

79-
$MatrixStr=$MatrixStr.Substring(1)
80-
Write-Host "##vso[task.setVariable variable=Targets;isOutput=true]{$MatrixStr}"
80+
if ($MatrixStr -and $MatrixStr.Length -gt 1) {
81+
$MatrixStr = $MatrixStr.Substring(1)
82+
}
83+
Write-Host "##vso[task.setVariable variable=buildTargets;isOutput=true]{$MatrixStr}"

‎.azure-pipelines/batch-generation.yml

+160-17
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,23 @@
11
variables:
22
BuildTimeoutInMinutes: 120
33
AnalysisTimeoutInMinutes: 120
4-
MaxParallelJobs: 3
4+
MaxParalleBuildJobs: 3
55
GenerationBranch: batch-generation/branch-$(Build.BuildId)
6+
MaxParalleAnalyzeJobs: 3
7+
MaxParalleWindowsTestJobs: 3
8+
MaxParalleLinuxTestJobs: 3
9+
MaxParalleMacTestJobs: 3
610

711
trigger: none
812

913
jobs:
1014
- job: prepare
11-
displayName: Generate Matrix
15+
displayName: Generate Build Targets Matrix
1216
pool: pool-windows-2019
1317
steps:
14-
- checkout: none
18+
19+
# TODO: (Bernard) Uncomment the no checkout step after automatically install repo into agnets
20+
# - checkout: none
1521

1622
- template: util/get-github-pat-steps.yml
1723

@@ -29,13 +35,13 @@ jobs:
2935
3036
- task: PowerShell@2
3137
name: mtrx
32-
displayName: 'Generate Matrix'
38+
displayName: 'Generate Build Targets Matrix'
3339
inputs:
3440
targetType: inline
3541
pwsh: true
3642
script: |
3743
$prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
38-
& $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelJobs }}"
44+
& $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParalleBuildJobs }}"
3945
4046

4147
- job: build
@@ -44,8 +50,8 @@ jobs:
4450
timeoutInMinutes: 120
4551
pool: pool-windows-2019
4652
strategy:
47-
matrix: $[ dependencies.prepare.outputs['mtrx.Targets'] ]
48-
maxParallel: 15
53+
matrix: $[ dependencies.prepare.outputs['mtrx.buildTargets'] ]
54+
maxParallel: ${{ variables.MaxParalleBuildJobs }}
4955

5056
steps:
5157
- checkout: self
@@ -55,7 +61,7 @@ jobs:
5561
# - checkout: azure-powershell-devops
5662
# path: "azure-powershell-devops"
5763

58-
- template: util/get-github-pat-steps.yml
64+
# - template: util/get-github-pat-steps.yml
5965

6066
- task: PowerShell@2
6167
name: build
@@ -70,20 +76,157 @@ jobs:
7076
$target = "$(Target)"
7177
Write-Host "Building targets: $target"
7278
79+
$patchDir = "$(Build.ArtifactStagingDirectory)/$(MatrixKey)"
80+
New-Item -ItemType Directory -Force -Path $patchDir | Out-Null
81+
7382
$modules = $target -split ','
7483
7584
foreach ($module in $modules) {
76-
$folderPath = Join-Path $(Build.SourcesDirectory) 'src' $module
77-
$filePath = Join-Path $folderPath "newfile.txt"
78-
"This is a new file in $folder" | Out-File -FilePath $filePath -Encoding UTF8
79-
}
85+
$modulePath = Join-Path $(Build.SourcesDirectory) 'src' $module
86+
$subModulePath = Get-ChildItem -Path $modulePath -Directory |
87+
Where-Object { $_.Name -like '*.autorest' }
88+
foreach ($subModule in $subModulePath) {
89+
$filePath = Join-Path $subModule "newfile.txt"
90+
"This is a new file in $folder" | Out-File -FilePath $filePath -Encoding UTF8
91+
}
92+
}
8093
81-
git config user.email "65331932+azure-powershell-bot@users.noreply.github.com";
82-
git config user.name "azure-powershell-bot";
83-
git remote set-url origin 'https://azure-powershell-bot:$(GithubToken)@github.com/Azure/azure-powershell.git';
94+
# git config user.email "65331932+azure-powershell-bot@users.noreply.github.com";
95+
# git config user.name "azure-powershell-bot";
96+
# git remote set-url origin 'https://azure-powershell-bot:$(GithubToken)@github.com/Azure/azure-powershell.git';
97+
98+
$random = Get-Random -Minimum 10 -Maximum 121
99+
Write-Host "Sleeping for $random seconds..."
100+
Start-Sleep -Seconds $random
84101
85102
git add .
86-
git commit -m "Build $target"
87-
git push origin "${{ variables.GenerationBranch }}"
103+
# git commit -m "Build $target"
104+
# git push origin "${{ variables.GenerationBranch }}"
105+
106+
107+
$patchPath = "$patchDir/changed.patch"
108+
git diff --cached > $patchPath
109+
110+
Get-Content -Path $patchPath
88111
89112
workingDirectory: $(Build.SourcesDirectory)
113+
114+
- task: PublishPipelineArtifact@1
115+
displayName: 'Save patch'
116+
inputs:
117+
targetPath: '$(Build.ArtifactStagingDirectory)/$(MatrixKey)'
118+
artifact: 'patch-$(MatrixKey)'
119+
condition: always()
120+
121+
- job: filter
122+
displayName: 'Filter Changed Modules'
123+
dependsOn: build
124+
steps:
125+
- checkout: self
126+
persistCredentials: true
127+
fetchTags: false
128+
129+
- task: PowerShell@2
130+
name: checkout
131+
displayName: 'Checkout Generation Branch'
132+
inputs:
133+
targetType: inline
134+
pwsh: true
135+
script: |
136+
git fetch origin "${{ variables.GenerationBranch }}"
137+
git checkout "${{ variables.GenerationBranch }}"
138+
139+
- download: current
140+
patterns: '**/*.patch'
141+
displayName: 'Download all .patch artifacts'
142+
143+
- task: PowerShell@2
144+
name: apply
145+
displayName: 'Apply all patches'
146+
inputs:
147+
targetType: inline
148+
pwsh: true
149+
script: |
150+
git config user.email "65331932+azure-powershell-bot@users.noreply.github.com"
151+
git config user.name "azure-powershell-bot"
152+
153+
Write-Host "Applying all .patch files to ${{ variables.GenerationBranch }}..."
154+
$patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch
155+
156+
# $patchFiles = Get-ChildItem -Path "$env:PIPELINE_WORKSPACE" -Recurse -Filter *.patch |
157+
# Where-Object { $_.Directory.Parent.FullName -eq $env:PIPELINE_WORKSPACE }
158+
159+
foreach ($patch in $patchFiles) {
160+
Write-Host "Applying patch: $($patch.FullName)"
161+
git apply --whitespace=fix "$($patch.FullName)"
162+
}
163+
164+
git add .
165+
git commit -m "Apply matrix job patches to ${{ variables.GenerationBranch }}"
166+
git push origin "${{ variables.GenerationBranch }}"
167+
168+
- task: PowerShell@2
169+
name: mtrx
170+
displayName: 'Generate Test Targets Matrix'
171+
inputs:
172+
targetType: inline
173+
pwsh: true
174+
script: |
175+
$base = git merge-base HEAD origin/main
176+
$changedFiles = git diff --name-only $base HEAD
177+
178+
$sourceBranchName = "$(Build.SourceBranch)".Replace('refs/heads/', '')
179+
git checkout $sourceBranchName
180+
181+
$filterModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'filter.ps1'
182+
& $filterModulesPath -MaxParallelJobs "${{ variables.MaxParalleAnalyzeJobs }}" -ChangedFiles $changedFiles
183+
184+
# - checkout: none
185+
186+
# - script: |
187+
# git init
188+
# git remote add origin https://github.com/Azure/azure-powershell.git
189+
# git fetch origin main --depth=1
190+
# git checkout FETCH_HEAD -- .azure-pipelines/
191+
192+
# displayName: 'Checkout Pipeline Templates'
193+
194+
# - template: util/get-github-pat-steps.yml
195+
196+
# - script: |
197+
# $token = "$(GitHubToken)"
198+
# changedModules=$(curl -s -H "Authorization: Bearer $token" \
199+
# https://api.github.com/repos/Azure/azure-powershell/compare/main...${{ variables.GenerationBranch }} \
200+
# | jq -r '.files[].filename' \
201+
# | grep -E '^(src|generated)/[^/]+/[^/]+\.autorest/' \
202+
# | sed -E 's#^(src|generated)/##' \
203+
# | awk -F/ '{print $1 "/" $2}' \
204+
# | sort -u)
205+
206+
# IFS=$'\n' read -r -d '' -a changedModulePath <<< "$changedModules"
207+
208+
# for path in "${changedModulePath[@]}"; do
209+
# echo "$path"
210+
# done
211+
212+
- job: analyze
213+
displayName: "Analyze:"
214+
dependsOn: filter
215+
timeoutInMinutes: 120
216+
pool: pool-windows-2019
217+
strategy:
218+
matrix: $[ dependencies.filter.outputs['mtrx.analyzeTargets'] ]
219+
maxParallel: ${{ variables.MaxParalleAnalyzeJobs }}
220+
221+
steps:
222+
- checkout: none
223+
224+
- task: PowerShell@2
225+
name: analyze
226+
displayName: 'Analyze modules'
227+
inputs:
228+
targetType: inline
229+
pwsh: true
230+
script: |
231+
Write-Host "Matrix Key: $(MatrixKey)"
232+
Write-Host "Analyze Targets: $(Target)"

0 commit comments

Comments
 (0)