Skip to content

Commit

Permalink
test: add tests for --ignore-errors flag
Browse files Browse the repository at this point in the history 
Signed-off-by: Anubhav Gupta <mail.anubhav06@gmail.com>
  • Loading branch information
@anubhav06
anubhav06 committed Aug 15, 2023
1 parent bf15f30 commit b3a86ec
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 47 deletions.
44 changes: 32 additions & 12 deletions integration/fixtures/test-images.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,83 +4,103 @@
"tag": "8.5.0",
"digest": "sha256:42d3e6bc186572245aded5a0be381012adba6d89355fa9486dd81b0c634695b5",
"distro": "Alpine",
"description": "Valid apk/db, apk present"
"description": "Valid apk/db, apk present",
"ignoreErrors": false
},
{
"image": "docker.io/library/nginx",
"tag": "1.21.6",
"digest": "sha256:2bcabc23b45489fb0885d69a06ba1d648aeda973fae7bb981bafbb884165e514",
"distro": "Debian",
"description": "Valid dpkg/status, apt present"
"description": "Valid dpkg/status, apt present",
"ignoreErrors": false
},
{
"image": "registry.k8s.io/kube-proxy",
"tag": "v1.23.4",
"digest": "sha256:30116c7218264d95623d3918a50da703675755cae866cd4c324586611fcd50ea",
"distro": "Debian",
"description": "Valid dpkg/status, apt present, custom network config"
"description": "Valid dpkg/status, apt present, custom network config",
"ignoreErrors": false
},
{
"image": "registry.k8s.io/kube-proxy",
"tag": "v1.27.2",
"digest": "sha256:1e4f13f5f5c215813fb9c9c6f56da1c0354363f2a69bd12732658f79d585864f",
"distro": "Custom Google Distroless",
"description": "Custom dpkg/status.d with text names, no apt, libssl1.1"
"description": "Custom dpkg/status.d with text names, no apt, libssl1.1",
"ignoreErrors": false
},
{
"image": "docker.io/fluent/fluent-bit",
"tag": "1.8.4",
"digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d",
"distro": "Google Distroless",
"description": "Custom dpkg/status.d with base64 names, no apt"
"description": "Custom dpkg/status.d with base64 names, no apt",
"ignoreErrors": false
},
{
"image": "docker.io/openpolicyagent/opa",
"tag": "0.46.0",
"digest": "sha256:c4b11c9b86eaba41276ae682bb6875332316242010b7523efe30f365ad0c3cb8",
"distro": "Google Distroless",
"description": "Custom dpkg/status.d with text names, no apt, libssl1"
"description": "Custom dpkg/status.d with text names, no apt, libssl1",
"ignoreErrors": false
},
{
"image": "quay.io/calico/cni",
"tag": "v3.15.1",
"digest": "sha256:a925b445c2688fc9c149b20ea04faabd40610d3304a6efda68e5dada7a41b813",
"distro": "Redhat",
"description": "Valid rpm DB, microdnf & rpm present"
"description": "Valid rpm DB, microdnf & rpm present",
"ignoreErrors": false
},
{
"image": "mcr.microsoft.com/cbl-mariner/base/core",
"tag": "1.0.20220218",
"digest": "sha256:830120b2cbfb7489c6f3270e1c74f3db0de84a4d33fecfffd427890b94d2f236",
"distro": "Mariner",
"description": "Valid rpm DB, no dnf, yum & rpm present"
"description": "Valid rpm DB, no dnf, yum & rpm present",
"ignoreErrors": false
},
{
"image": "mcr.microsoft.com/cbl-mariner/base/core",
"tag": "1.0.20220218-arm64",
"digest": "sha256:f97ccb4565f8985c28c6dbc7f13cfea0877652b70545f844eb0b83e4475954d1",
"distro": "Mariner",
"description": "Valid rpm DB, no dnf, yum & rpm present, arm64 cross-arch"
"description": "Valid rpm DB, no dnf, yum & rpm present, arm64 cross-arch",
"ignoreErrors": false
},
{
"image": "mcr.microsoft.com/cbl-mariner/distroless/base",
"tag": "2.0.20220527",
"digest": "sha256:f550c5428df17b145851ad75983aca6d613ad4b51ca7983b2a83e67d0ac91a5d",
"distro": "Mariner Distroless",
"description": "Custom rpmmanifest files, no yum/dnf/microdnf/rpm"
"description": "Custom rpmmanifest files, no yum/dnf/microdnf/rpm",
"ignoreErrors": false
},
{
"image": "docker.io/library/centos",
"tag": "7.6.1810",
"digest": "sha256:62d9e1c2daa91166139b51577fe4f4f6b4cc41a3a2c7fc36bd895e2a17a3e4e6",
"distro": "CentOS",
"description": "Valid rpm DB, yum present"
"description": "Valid rpm DB, yum present",
"ignoreErrors": false
},
{
"image": "docker.io/library/amazonlinux",
"tag": "2.0.20210326.0",
"digest": "sha256:06380711d6a8ac0b6989f7e2a4419e560796791d9c7c843753a719c73552dc30",
"distro": "Amazon Linux",
"description": "Valid rpm DB, yum present"
"description": "Valid rpm DB, yum present",
"ignoreErrors": false
},
{
"image": "mcr.microsoft.com/oss/nginx/nginx",
"tag" : "1.21.6",
"digest": "sha256:bcdc041d100a3cc9fec472a8133ad566166e366241587ff92e8cc8fcf955229d",
"distro": "Debian",
"description": "Valid dpkg/status, apt present",
"ignoreErrors": true
}
]
2 changes: 2 additions & 0 deletions integration/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ import (
var (
buildkitAddr string
copaPath string
ignoreErrors bool
)

func TestMain(m *testing.M) {
flag.StringVar(&buildkitAddr, "addr", "", "buildkit address to pass through to copa binary")
flag.StringVar(&copaPath, "copa", "./copa", "path to copa binary")
flag.BoolVar(&ignoreErrors, "ignore-errors", false, "Ignore errors and continue patching")
flag.Parse()

if copaPath == "" {
Expand Down
25 changes: 14 additions & 11 deletions integration/patch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,12 @@ var (
)

type testImage struct {
Image string `json:"image"`
Tag string `json:"tag"`
Distro string `json:"distro"`
Digest digest.Digest `json:"digest"`
Description string `json:"description"`
Image string `json:"image"`
Tag string `json:"tag"`
Distro string `json:"distro"`
Digest digest.Digest `json:"digest"`
Description string `json:"description"`
IgnoreErrors bool `json:"ignoreErrors"`
}

func TestPatch(t *testing.T) {
Expand Down Expand Up @@ -57,23 +58,23 @@ func TestPatch(t *testing.T) {
withIgnoreFile(ignoreFile).
withOutput(output).
// Do not set a non-zero exit code because we are expecting vulnerabilities.
scan(t, ref)
scan(t, ref, img.IgnoreErrors)

t.Log("patching image")
patch(t, ref, tagPatched, output)
patch(t, ref, tagPatched, output, img.IgnoreErrors)

t.Log("scanning patched image")
scanner().
withIgnoreFile(ignoreFile).
withSkipDBUpdate().
// here we want a non-zero exit code because we are expecting no vulnerabilities.
withExitCode(1).
scan(t, patchedRef)
scan(t, patchedRef, img.IgnoreErrors)
})
}
}

func patch(t *testing.T, ref, patchedTag, scan string) {
func patch(t *testing.T, ref, patchedTag, scan string, ignoreErrors bool) {
var addrFl string
if buildkitAddr != "" {
addrFl = "-a=" + buildkitAddr
Expand All @@ -88,6 +89,7 @@ func patch(t *testing.T, ref, patchedTag, scan string) {
"-r="+scan,
"--timeout=20m",
addrFl,
"--ignore-errors="+strconv.FormatBool(ignoreErrors),
)
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
Expand All @@ -104,7 +106,7 @@ type scannerCmd struct {
exitCode int
}

func (s *scannerCmd) scan(t *testing.T, ref string) {
func (s *scannerCmd) scan(t *testing.T, ref string, ignoreErrors bool) {
args := []string{
"trivy",
"image",
Expand All @@ -121,7 +123,8 @@ func (s *scannerCmd) scan(t *testing.T, ref string) {
if s.ignoreFile != "" {
args = append(args, "--ignore-policy="+s.ignoreFile)
}
if s.exitCode != 0 {
// If ignoreErrors is false, we expect a non-zero exit code.
if s.exitCode != 0 && !ignoreErrors {
args = append(args, "--exit-code="+strconv.Itoa(s.exitCode))
}

Expand Down
5 changes: 5 additions & 0 deletions pkg/patch/cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ func TestNewPatchCmd(t *testing.T) {
args: []string{"-i", "images/python:3.7-alpine", "-t", "3.7-alpine-patched"},
expected: "required flag(s) \"report\" not set",
},
{
name: "Missing report flag with ignore-errors flag",
args: []string{"-i", "images/python:3.7-alpine", "-t", "3.7-alpine-patched", "--ignore-errors"},
expected: "required flag(s) \"report\" not set",
},
}

// Run test cases
Expand Down
54 changes: 33 additions & 21 deletions pkg/pkgmgr/apk_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,40 +103,52 @@ func TestValidateAPKPackageVersions(t *testing.T) {

// Define some test cases with inputs and expected outputs
testCases := []struct {
name string
updates types.UpdatePackages
cmp VersionComparer
resultsPath string
expectedErr error
name string
updates types.UpdatePackages
cmp VersionComparer
resultsPath string
ignoreErrors bool
expectedErr error
}{
{
name: "valid updates",
updates: []types.UpdatePackage{{Name: "apk-tools", Version: "2.12.7-r0"}, {Name: "busybox", Version: "1.33.1-r8"}},
cmp: apkComparer,
resultsPath: "testdata/apk_valid.txt",
expectedErr: nil,
name: "valid updates",
updates: []types.UpdatePackage{{Name: "apk-tools", Version: "2.12.7-r0"}, {Name: "busybox", Version: "1.33.1-r8"}},
cmp: apkComparer,
resultsPath: "testdata/apk_valid.txt",
ignoreErrors: false,
expectedErr: nil,
},
{
name: "invalid version",
updates: []types.UpdatePackage{{Name: "apk-tools", Version: "1.0"}, {Name: "busybox", Version: "2.0"}},
cmp: apkComparer,
resultsPath: "testdata/apk_invalid.txt",
expectedErr: fmt.Errorf("2 errors occurred:\n\t* invalid version x.y found for package apk-tools\n\t* invalid version a.b.c found for package busybox"),
name: "invalid version",
updates: []types.UpdatePackage{{Name: "apk-tools", Version: "1.0"}, {Name: "busybox", Version: "2.0"}},
cmp: apkComparer,
resultsPath: "testdata/apk_invalid.txt",
ignoreErrors: false,
expectedErr: fmt.Errorf("2 errors occurred:\n\t* invalid version x.y found for package apk-tools\n\t* invalid version a.b.c found for package busybox"),
},
{
name: "expected 2 updates, installed 1",
updates: []types.UpdatePackage{{Name: "apk-tools", Version: "2.12.7-r0"}},
cmp: apkComparer,
resultsPath: "testdata/apk_valid.txt",
expectedErr: fmt.Errorf("expected 2 updates, installed 1"),
name: "invalid version with ignore errors",
updates: []types.UpdatePackage{{Name: "apk-tools", Version: "1.0"}, {Name: "busybox", Version: "2.0"}},
cmp: apkComparer,
resultsPath: "testdata/apk_valid.txt",
ignoreErrors: true,
expectedErr: nil,
},
{
name: "expected 2 updates, installed 1",
updates: []types.UpdatePackage{{Name: "apk-tools", Version: "2.12.7-r0"}},
cmp: apkComparer,
resultsPath: "testdata/apk_valid.txt",
ignoreErrors: false,
expectedErr: fmt.Errorf("expected 2 updates, installed 1"),
},
}

for _, tc := range testCases {
// Use t.Run to run each test case as a subtest
t.Run(tc.name, func(t *testing.T) {
// Run the function to be tested
err := validateAPKPackageVersions(tc.updates, tc.cmp, tc.resultsPath, false)
err := validateAPKPackageVersions(tc.updates, tc.cmp, tc.resultsPath, tc.ignoreErrors)
if tc.expectedErr != nil {
if err == nil || errors.Is(err, tc.expectedErr) {
t.Errorf("expected error %v, got %v", tc.expectedErr, err)
Expand Down
20 changes: 17 additions & 3 deletions pkg/pkgmgr/rpm_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@ func TestValidateRPMPackageVersions(t *testing.T) {
updates types.UpdatePackages
cmp VersionComparer
resultsPath string
ignoreErrors bool
expectedError error
}{
{
Expand All @@ -268,8 +269,9 @@ func TestValidateRPMPackageVersions(t *testing.T) {
{Name: "openssl", Version: "1.1.1k-21.cm2"},
{Name: "openssl-libs", Version: "1.1.1k-21.cm2"},
},
cmp: rpmComparer,
resultsPath: "testdata/rpm_valid.txt",
cmp: rpmComparer,
resultsPath: "testdata/rpm_valid.txt",
ignoreErrors: false,
},
{
name: "downloaded package version lower than required",
Expand All @@ -279,8 +281,20 @@ func TestValidateRPMPackageVersions(t *testing.T) {
},
cmp: rpmComparer,
resultsPath: "testdata/rpm_valid.txt",
ignoreErrors: false,
expectedError: fmt.Errorf("2 errors occurred:\n\t* downloaded package openssl version 2.1.1k-21.cm2 lower than required 3.1.1k-21.cm2 for update\n\t* downloaded package openssl-libs version 2.1.1k-21.cm2 lower than required 3.1.1k-21.cm2 for update"), // nolint:lll
},
{
name: "downloaded package version lower than required with ignore errors",
updates: types.UpdatePackages{
{Name: "openssl", Version: "3.1.1k-21.cm2"},
{Name: "openssl-libs", Version: "3.1.1k-21.cm2"},
},
cmp: rpmComparer,
resultsPath: "testdata/rpm_valid.txt",
ignoreErrors: true,
expectedError: nil,
},
{
name: "unexpected number of installed packages",
updates: types.UpdatePackages{
Expand All @@ -294,7 +308,7 @@ func TestValidateRPMPackageVersions(t *testing.T) {

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
err := validateRPMPackageVersions(tc.updates, tc.cmp, tc.resultsPath, false)
err := validateRPMPackageVersions(tc.updates, tc.cmp, tc.resultsPath, tc.ignoreErrors)
if tc.expectedError != nil {
if err == nil || errors.Is(err, tc.expectedError) {
t.Errorf("expected error %v, got %v", tc.expectedError, err)
Expand Down

0 comments on commit b3a86ec

Please sign in to comment.