Skip to content

Commit

Permalink
Sync eng/common directory with azure-sdk-tools for PR 8830 (#45542)
Browse files Browse the repository at this point in the history 
* Add RestAPISpecsDocsRepos to branch cleanup.

* Remove commented out yml, add pre-message to limit output and output core limit at the start of processing

* updates for feedback

---------

Co-authored-by: James Suplizio <jasupliz@microsoft.com>
  • Loading branch information
@azure-sdk @JimSuplizio
azure-sdk and JimSuplizio authored Aug 15, 2024
1 parent be99b33 commit b132865
Show file tree
Hide file tree
Showing 2 changed files with 244 additions and 50 deletions.
152 changes: 104 additions & 48 deletions eng/common/scripts/Delete-RemoteBranches.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ param(
$AuthToken
)
Set-StrictMode -version 3

. (Join-Path $PSScriptRoot common.ps1)

function Get-AllBranchesAndPullRequestInfo($owner, $repo) {
Expand Down Expand Up @@ -66,67 +65,124 @@ if ($AuthToken) {
}

$owner, $repo = $RepoId -split "/"
$branches = Get-AllBranchesAndPullRequestInfo $owner $repo

foreach ($branch in $branches)
{
$branchName = $branch.Name
if ($branchName -notmatch $BranchRegex) {
continue
# These will always be output at the end of the script. Their only purpose is for information gathering
# Total number returned from query
$totalBranchesFromQuery = 0
# reasons why a branch was skipped
$skippedBranchNotMatchRegex = 0
$skippedForCommitDate = 0
$skippedForOpenPRs = 0
$skippedForPRNotInBranch = 0
$skippedForPRNotInRepo = 0
# gh call counters
$ghPRViewCalls = 0
$ghBranchDeleteCalls = 0

try {
# Output the core rate limit at the start of processing. There's no real need
# to output this at the end because the GH call counts are being output
$coreRateLimit = Get-RateLimit core
Write-RateLimit $coreRateLimit
# Output the GraphQL rate limit before and after the call
$graphqlRateLimit = Get-RateLimit graphql
Write-RateLimit $graphqlRateLimit "Before GraphQL Call"
$branches = Get-AllBranchesAndPullRequestInfo $owner $repo
$graphqlRateLimit = Get-RateLimit graphql
Write-RateLimit $graphqlRateLimit "After GraphQL Call"

if ($branches) {
$totalBranchesFromQuery = $branches.Count
}
$openPullRequests = @($branch.pullRequests | Where-Object { !$_.Closed })

# If we have a central PR that created this branch still open don't delete the branch
if ($CentralRepoId)
foreach ($branch in $branches)
{
$pullRequestNumber = $matches["PrNumber"]
# If central PR number is not found, then skip
if (!$pullRequestNumber) {
LogError "No PR number found in the branch name. Please check the branch name '$branchName'. Skipping..."
$branchName = $branch.Name
if ($branchName -notmatch $BranchRegex) {
$skippedBranchNotMatchRegex++
continue
}
$openPullRequests = @($branch.pullRequests | Where-Object { !$_.Closed })

$centralPR = gh pr view --json 'url,closed' --repo $CentralRepoId $pullRequestNumber | ConvertFrom-Json
if ($LASTEXITCODE) {
LogError "PR '$pullRequestNumber' not found in repo '$CentralRepoId'. Skipping..."
continue;
} else {
LogDebug "Found central PR $($centralPR.url) and Closed=$($centralPR.closed)"
if (!$centralPR.Closed) {
# Skipping if there is an open central PR open for the branch.
LogDebug "Central PR is still open so skipping the deletion of branch '$branchName'. Skipping..."
continue;
# If we have a central PR that created this branch still open don't delete the branch
if ($CentralRepoId)
{
$pullRequestNumber = $matches["PrNumber"]
# If central PR number is not found, then skip
if (!$pullRequestNumber) {
LogError "No PR number found in the branch name. Please check the branch name '$branchName'. Skipping..."
$skippedForPRNotInBranch++
continue
}

$ghPRViewCalls++
$centralPR = gh pr view --json 'url,closed' --repo $CentralRepoId $pullRequestNumber | ConvertFrom-Json
if ($LASTEXITCODE) {
LogError "PR '$pullRequestNumber' not found in repo '$CentralRepoId'. Skipping..."
$skippedForPRNotInRepo++
continue
} else {
LogDebug "Found central PR $($centralPR.url) and Closed=$($centralPR.closed)"
if (!$centralPR.Closed) {
$skippedForOpenPRs++
# Skipping if there is an open central PR open for the branch.
LogDebug "Central PR is still open so skipping the deletion of branch '$branchName'. Skipping..."
continue
}
}
}
}
else {
# Not CentralRepoId - not associated with a central repo PR
if ($openPullRequests.Count -gt 0 -and !$DeleteBranchesEvenIfThereIsOpenPR) {
LogDebug "Found open PRs associate with branch '$branchName'. Skipping..."
continue
else {
# Not CentralRepoId - not associated with a central repo PR
if ($openPullRequests.Count -gt 0 -and !$DeleteBranchesEvenIfThereIsOpenPR) {
$skippedForOpenPRs++
LogDebug "Found open PRs associate with branch '$branchName'. Skipping..."
continue
}
}
}

# If there is date filter, then check if branch last commit is older than the date.
if ($LastCommitOlderThan)
{
$commitDate = $branch.committedDate
if ($commitDate -gt $LastCommitOlderThan) {
LogDebug "The branch $branch last commit date '$commitDate' is newer than the date '$LastCommitOlderThan'. Skipping..."
continue
# If there is date filter, then check if branch last commit is older than the date.
if ($LastCommitOlderThan)
{
$commitDate = $branch.committedDate
if ($commitDate -gt $LastCommitOlderThan) {
$skippedForCommitDate++
LogDebug "The branch $branch last commit date '$commitDate' is newer than the date '$LastCommitOlderThan'. Skipping..."
continue
}
}
}

foreach ($openPullRequest in $openPullRequests) {
Write-Host "Note: Open pull Request '$($openPullRequest.url)' will be closed after branch deletion, given the central PR is closed."
}
foreach ($openPullRequest in $openPullRequests) {
LogDebug "Note: Open pull Request '$($openPullRequest.url)' will be closed after branch deletion, given the central PR is closed."
}

$commitUrl = $branch.commitUrl
if ($PSCmdlet.ShouldProcess("'$branchName' in '$RepoId'", "Deleting branch on cleanup script")) {
gh api "repos/${RepoId}/git/refs/heads/${branchName}" -X DELETE
if ($LASTEXITCODE) {
LogError "Deletion of branch '$branchName` failed"
$commitUrl = $branch.commitUrl
if ($PSCmdlet.ShouldProcess("'$branchName' in '$RepoId'", "Deleting branch on cleanup script")) {
$ghBranchDeleteCalls++
gh api "repos/${RepoId}/git/refs/heads/${branchName}" -X DELETE
if ($LASTEXITCODE) {
LogError "Deletion of branch '$branchName` failed, see command output above"
exit $LASTEXITCODE
}
LogDebug "The branch '$branchName' at commit '$commitUrl' in '$RepoId' has been deleted."
}
Write-Host "The branch '$branchName' at commit '$commitUrl' in '$RepoId' has been deleted."
}
}
finally {


Write-Host "Number of branches returned from graphql query: $totalBranchesFromQuery"
# The $BranchRegex seems to be always set
if ($BranchRegex) {
Write-Host "Number of branches that didn't match the BranchRegex: $skippedBranchNotMatchRegex"
}
Write-Host "Number of branches skipped for newer last commit date: $skippedForCommitDate"
Write-Host "Number of branches skipped for open PRs: $skippedForOpenPRs"
Write-Host "Number of gh api calls to delete branches: $ghBranchDeleteCalls"
# The following are only applicable when $CentralRepoId is passed in
if ($CentralRepoId) {
Write-Host "The following are applicable because CentralRepoId was passed in:"
Write-Host " Number of gh pr view calls: $ghPRViewCalls"
Write-Host " Number of branches skipped due to PR not in the repository: $skippedForPRNotInRepo "
Write-Host " Number of branches skipped due to PR not in the branch name: $skippedForPRNotInBranch"
}
}
142 changes: 140 additions & 2 deletions eng/common/scripts/Helpers/git-helpers.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ function Get-ChangedFiles {
return ""
}

# Add config to disable the quote and encoding on file name.
# Add config to disable the quote and encoding on file name.
# Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-quoted-file-names
# Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-commit-message-transcoding
# Git PR diff: https://docs.github.com/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-comparing-branches-in-pull-requests#three-dot-and-two-dot-git-diff-comparisons
Expand Down Expand Up @@ -91,7 +91,7 @@ class ConflictedFile {
$lines = $IncomingContent -split "`r?`n"
$l = @()
$r = @()

foreach($line in $lines) {
if ($line -match "^<<<<<<<\s*(.+)") {
$this.IsConflicted = $true
Expand All @@ -110,3 +110,141 @@ class ConflictedFile {
}
}
}

# The rate limit comes back in the following format:
# The top level "rate" object is deprecated and the resources->core object should be used
# in its place.
# {
# "resources": {
# "core": {
# "limit": 5000,
# "used": 1087,
# "remaining": 3913,
# "reset": 1722876411
# },
# "search": {
# "limit": 30,
# "used": 0,
# "remaining": 30,
# "reset": 1722875519
# },
# "graphql": {
# "limit": 5000,
# "used": 0,
# "remaining": 5000,
# "reset": 1722879059
# },
# "integration_manifest": {
# "limit": 5000,
# "used": 0,
# "remaining": 5000,
# "reset": 1722879059
# },
# "source_import": {
# "limit": 100,
# "used": 0,
# "remaining": 100,
# "reset": 1722875519
# },
# "code_scanning_upload": {
# "limit": 1000,
# "used": 0,
# "remaining": 1000,
# "reset": 1722879059
# },
# "actions_runner_registration": {
# "limit": 10000,
# "used": 0,
# "remaining": 10000,
# "reset": 1722879059
# },
# "scim": {
# "limit": 15000,
# "used": 0,
# "remaining": 15000,
# "reset": 1722879059
# },
# "dependency_snapshots": {
# "limit": 100,
# "used": 0,
# "remaining": 100,
# "reset": 1722875519
# },
# "audit_log": {
# "limit": 1750,
# "used": 0,
# "remaining": 1750,
# "reset": 1722879059
# },
# "audit_log_streaming": {
# "limit": 15,
# "used": 0,
# "remaining": 15,
# "reset": 1722879059
# },
# "code_search": {
# "limit": 10,
# "used": 0,
# "remaining": 10,
# "reset": 1722875519
# }
# },
# "rate": {
# "limit": 5000,
# "used": 1087,
# "remaining": 3913,
# "reset": 1722876411
# }
# }

# These are the rate limit types we care about. If others needed in the future they
# can be defined here. The reason these need to be defined is because Get-RateLimit
# call needs to select the particular property to return the right limit. This ensures
# that rate limit type being passed to the function will exist.
enum RateLimitTypes {
core
search
graphql
}

# Fetch the rate limit for the given RateLimitType
function Get-RateLimit([RateLimitTypes]$RateLimitType) {
$returnValue = gh api rate_limit
if ($LASTEXITCODE) {
LogError "Get-RateLimit::unable to get rate limit"
exit $LASTEXITCODE
}
# All rate limits have the following fields: limit, used, remaning, reset.
# Returning -AsHashtable allows easier access, eg. $rate_limit.remaining
$rate_limit = $returnValue | ConvertFrom-Json -AsHashtable | Select-Object -ExpandProperty resources | Select-Object -ExpandProperty $RateLimitType
# Add the limit type for convenance
$rate_limit["type"] = $RateLimitType
return $rate_limit
}

# Get the number of minutes until RateLimit reset rounded up to the nearest minute
# for the passed in RateLimit. This is more applicable to the core and graphql rate
# limits than search because the search rate limit resets every minute
function Get-MinutesUntilRateLimitReset($RateLimit) {
$TimeSpan = [System.DateTimeOffset]::FromUnixTimeSeconds($rate.reset).UtcDateTime.Subtract([System.DateTime]::UtcNow)
$MinutesRoundedUp = [Math]::Ceiling($TimeSpan.TotalMinutes)
return $MinutesRoundedUp
}

# Output the rate limit
function Write-RateLimit {
param (
$RateLimit,
[string]$PreMsg = $null
)

if ($PreMsg) {
Write-Host $PreMsg
}
Write-Host "Limit Type=$($RateLimit.type)"
Write-Host " limit=$($RateLimit.limit)"
Write-Host " used=$($RateLimit.used)"
Write-Host " remaining=$($RateLimit.remaining)"
Write-Host " reset=$($RateLimit.reset)"
Write-Host ""
}

0 comments on commit b132865

Please sign in to comment.