Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(scoop-virustotal): Migrate to VirusTotal API v3 #4983

Merged
merged 30 commits into from
Jun 15, 2022
Merged

feat(scoop-virustotal): Migrate to VirusTotal API v3 #4983

Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
37f448b
feat(scoop-virustotal): migrate to virustotal api v3, improve flows, …
Jun 10, 2022
94bacbc
remove unused codes
Jun 11, 2022
bc6fa8c
fix bug
Jun 11, 2022
8a497fa
adjust output
Jun 11, 2022
5f5eb32
fix
Jun 11, 2022
81e759e
fix output
Jun 11, 2022
1a40167
Adjust output
Jun 12, 2022
8737f9e
show vendors directly if malicious/suspicious != 0
Jun 12, 2022
e4207f4
adjust output
Jun 13, 2022
e3249ed
fix for some rare cases
Jun 13, 2022
301c7e9
add '--passthru' parameter
Jun 13, 2022
a925252
feat(scoop-virustotal): migrate to virustotal api v3, improve flows, …
Jun 10, 2022
aa830bd
remove unused codes
Jun 11, 2022
331f134
fix bug
Jun 11, 2022
20d5f47
adjust output
Jun 11, 2022
5a20a6c
fix
Jun 11, 2022
1fc1cbb
fix output
Jun 11, 2022
9d9fc9c
Adjust output
Jun 12, 2022
36d5ce3
show vendors directly if malicious/suspicious != 0
Jun 12, 2022
152f21a
adjust output
Jun 13, 2022
dd797d6
fix for some rare cases
Jun 13, 2022
08e81c7
add '--passthru' parameter
Jun 13, 2022
3b293df
Merge branch 'virustotal' of https://github.com/ClassicDarkChocolate/…
Jun 13, 2022
3bdd4f5
add '-p' parameter
Jun 13, 2022
c60558d
fix special cases
Jun 14, 2022
92e481c
fix help message
ClassicDarkChocolate Jun 14, 2022
1d05ea3
add API Key hint
ClassicDarkChocolate Jun 14, 2022
593a968
Merge branch 'develop' into virustotal
ClassicDarkChocolate Jun 14, 2022
0d88679
update changelog
Jun 14, 2022
6ced2c3
fix case
Jun 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
adjust output
  • Loading branch information
ClassicDarkChocolate committed Jun 13, 2022
commit e4207f489b33b8b9791387a7e7c4b1941c1eaee5
119 changes: 57 additions & 62 deletions libexec/scoop-virustotal.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ $architecture = ensure_architecture ($opt.a + $opt.arch)

if (is_scoop_outdated) {
if ($opt.u -or $opt.'no-update-scoop') {
Write-Warning 'Scoop is out of date.'
warn 'Scoop is out of date.'
} else {
scoop update
}
Expand All @@ -67,9 +67,8 @@ $exit_code = 0
# Global API key:
$api_key = get_config virustotal_api_key
if (!$api_key) {
Write-Warning ("VirusTotal API key is not configured`n`n" +
"`tscoop config virustotal_api_key <API key>")
exit $_ERR_NO_API_KEY
abort ("VirusTotal API key is not configured`n`n" +
"`tscoop config virustotal_api_key <API key>") $_ERR_NO_API_KEY
ClassicDarkChocolate marked this conversation as resolved.
Show resolved Hide resolved
}

# Global flag to explain only once about sleep between requests
Expand All @@ -89,9 +88,7 @@ Function ConvertTo-VirusTotalUrlId ($url) {

Function Get-RemoteFileSize ($url) {
$response = Invoke-WebRequest -Uri $url -Method HEAD -UseBasicParsing
$response.Headers.'Content-Length' | ForEach-Object {
([System.Convert]::ToDouble($_) / 1048576)
}
$response.Headers.'Content-Length' | ForEach-Object { [System.Convert]::ToInt32($_) }
}

Function Get-VirusTotalResultByHash ($hash, $app) {
Expand All @@ -109,31 +106,31 @@ Function Get-VirusTotalResultByHash ($hash, $app) {
[int]$undetected = json_path $stats '$.undetected'
[int]$unsafe = $malicious + $suspicious
[int]$total = $unsafe + $undetected
$fileSize = ([System.Convert]::ToDouble((json_path $result '$.data.attributes.size')) / 1048576)
[int]$fileSize = json_path $result '$.data.attributes.size'
$report_url = "https://www.virustotal.com/gui/file/$hash"
if ($total -eq 0) {
Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue'
info "$app`: Analysis in progress."
[PSCustomObject] @{
'App.Name' = $app
'App.Hash' = $hash
'App.Size (MB)' = $fileSize.ToString('0.00')
FileReport = $report_url
UrlReport = $null
'App.Name' = $app
'App.Hash' = $hash
'App.Size' = filesize $fileSize
'FileReport.Url' = $report_url
'UrlReport.Url' = $null
}
} else {
$vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value
switch ($unsafe) {
0 {
Write-Information "$($PSStyle.Foreground.BrightGreen)INFO : $app`: $unsafe/$total$($PSStyle.Reset)" -InformationAction 'Continue'
success "$app`: $unsafe/$total"
}
1 {
Write-Warning "$app`: $unsafe/$total"
warn "$app`: $unsafe/$total"
}
2 {
Write-Warning "$app`: $unsafe/$total"
warn "$app`: $unsafe/$total"
}
Default {
Write-Warning "$app`: $($PSStyle.Formatting.Error)$unsafe$($PSStyle.Formatting.Warning)/$total"
warn "$app`: $unsafe/$total"
}
}
$maliciousResults = $vendorResults |
Expand All @@ -143,15 +140,15 @@ Function Get-VirusTotalResultByHash ($hash, $app) {
Where-Object -Property category -EQ 'suspicious' |
Select-Object -ExpandProperty engine_name
[PSCustomObject] @{
'App.Name' = $app
'App.Hash' = $hash
'App.Size (MB)' = $fileSize.ToString('0.00')
FileReport = $report_url
Malicious = if ($maliciousResults) { $maliciousResults } else { 0 }
Suspicious = if ($suspiciousResults) { $suspiciousResults } else { 0 }
Timeout = $timeout
Undetected = $undetected
UrlReport = $null
'App.Name' = $app
'App.Hash' = $hash
'App.Size' = filesize $fileSize
'FileReport.Url' = $report_url
'FileReport.Malicious' = if ($maliciousResults) { $maliciousResults } else { 0 }
'FileReport.Suspicious' = if ($suspiciousResults) { $suspiciousResults } else { 0 }
'FileReport.Timeout' = $timeout
'FileReport.Undetected' = $undetected
'UrlReport.Url' = $null
}
}
if ($unsafe -gt 0) {
Expand All @@ -170,23 +167,23 @@ Function Get-VirusTotalResultByUrl ($url, $app) {
$id = json_path $result '$.data.id'
$hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null
$url_report_url = "https://www.virustotal.com/gui/url/$id"
Write-Information "INFO : $app`: Url report found."
info "$app`: Url report found."
if (!$hash) {
Write-Information "INFO : $app`: Related file report not found."
Write-Warning "$app`: Manual file upload is required (instead of url submission)."
info "$app`: Related file report not found."
warn "$app`: Manual file upload is required (instead of url submission)."
[PSCustomObject] @{
'App.Name' = $app
'App.Hash' = $null
FileReport = $null
UrlReport = $url_report_url
'App.Name' = $app
'App.Hash' = $null
'FileReport.Url' = $null
'UrlReport.Url' = $url_report_url
}
} else {
Write-Information "INFO : $app`: Related file report found."
info "$app`: Related file report found."
[PSCustomObject] @{
'App.Name' = $app
'App.Hash' = $hash
FileReport = $null
UrlReport = $url_report_url
'App.Name' = $app
'App.Hash' = $hash
'FileReport.Url' = $null
'UrlReport.Url' = $url_report_url
}
}
}
Expand All @@ -203,7 +200,7 @@ Function Get-VirusTotalResultByUrl ($url, $app) {
# overflow) if the submission keeps failing.
Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) {
if (!$do_scan) {
Write-Warning "$app`: not found`: you can manually submit $url"
warn "$app`: not found`: you can manually submit $url"
return
}

Expand All @@ -223,15 +220,15 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) {
$id = ((json_path $result '$.data.id') -split '-')[1]
$url_report_url = "https://www.virustotal.com/gui/url/$id"
$fileSize = Get-RemoteFileSize $url
if ($fileSize -gt 70) {
Write-Information "INFO : $app`: Remote file size: $($fileSize.ToString('0.00 MB')). Large files might require manual file upload instead of url submission." -InformationAction 'Continue'
if ($fileSize -gt 80000000) {
info "$app`: Remote file size: $(filesize $fileSize). Large files might require manual file upload instead of url submission."
}
Write-Information "INFO : $app`: Analysis in progress." -InformationAction 'Continue'
info "$app`: Analysis in progress."
[PSCustomObject] @{
'App.Name' = $app
'App.Size (MB)' = $fileSize.ToString('0.00')
FileReport = $null
UrlReport = $url_report_url
'App.Name' = $app
'App.Size' = filesize $fileSize
'FileReport.Url' = $null
'UrlReport.Url' = $url_report_url
}
return
}
Expand All @@ -240,16 +237,16 @@ Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) {
if (!$retrying) {
if (!$explained_rate_limit_sleeping) {
$explained_rate_limit_sleeping = $True
Write-Information "INFO : Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" -InformationAction 'Continue'
info "Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit"
}
Start-Sleep -s (60 + $requests)
Submit-ToVirusTotal $url $app $do_scan $True
} else {
Write-Warning "$app`: VirusTotal submission of $url failed`:`n" +
warn "$app`: VirusTotal submission of $url failed`:`n" +
"`tAPI returned $($result.StatusCode) after retrying"
}
} catch [Exception] {
Write-Warning "$app`: VirusTotal submission failed`: $($_.Exception.Message)"
warn "$app`: VirusTotal submission failed`: $($_.Exception.Message)"
return
}
}
Expand All @@ -259,7 +256,7 @@ $reports = $apps | ForEach-Object {
$null, $manifest, $bucket, $null = Get-Manifest $app
if (!$manifest) {
$exit_code = $exit_code -bor $_ERR_NO_INFO
Write-Warning "$app`: manifest not found"
warn "$app`: manifest not found"
return
}

Expand All @@ -275,26 +272,25 @@ $reports = $apps | ForEach-Object {
if ($matches.algo -notmatch '(md5|sha1|sha256)') {
$hash = $null
$isHashUnsupported = $true
Write-Warning "$app`: Unsupported hash $($matches.algo). Will search by url instead."
warn "$app`: Unsupported hash $($matches.algo). Will search by url instead."
}
}
if ($hash) {
$file_report = Get-VirusTotalResultByHash $hash $app
$file_report
return
} elseif (!$isHashUnsupported) {
Write-Warning "$app`: Hash not found. Will search by url instead."
warn "$app`: Hash not found. Will search by url instead."
}
} catch [Exception] {
$exit_code = $exit_code -bor $_ERR_EXCEPTION
if ($_.Exception.Response.StatusCode -eq 404) {
Write-Warning "$app`: File report not found. Will search by url instead."
warn "$app`: File report not found. Will search by url instead."
} else {
if ($_.Exception.Response.StatusCode -in 204, 429) {
Write-Error "$app`: VirusTotal request failed`: $($_.Exception.Message)"
exit $exit_code
abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code
}
Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)"
warn "$app`: VirusTotal request failed`: $($_.Exception.Message)"
return
}
}
Expand All @@ -304,22 +300,21 @@ $reports = $apps | ForEach-Object {
$hash = $url_report.'App.Hash'
if ($hash) {
$file_report = Get-VirusTotalResultByHash $hash $app
$file_report.'UrlReport' = $url_report.'UrlReport'
$file_report.'UrlReport.Url' = $url_report.'UrlReport.Url'
$file_report
} else {
$url_report
}
} catch [Exception] {
$exit_code = $exit_code -bor $_ERR_EXCEPTION
if ($_.Exception.Response.StatusCode -eq 404) {
Write-Warning "$app`: Url report not found. Will submit $url"
warn "$app`: Url report not found. Will submit $url"
Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s)
} else {
if ($_.Exception.Response.StatusCode -in 204, 429) {
Write-Error "$app`: VirusTotal request failed`: $($_.Exception.Message)"
exit $exit_code
abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code
}
Write-Warning "$app`: VirusTotal request failed`: $($_.Exception.Message)"
warn "$app`: VirusTotal request failed`: $($_.Exception.Message)"
return
}
}
Expand Down