go-containerregistry dep update, cleanup and adding missing docs for …

…a numbeer of flags

Signed-off-by: Kyle Quest <kcq.public@gmail.com>

README.md

@@ -333,6 +333,7 @@ Global options:
 - `--verbose` - enable info logs
 - `--log-level` - set the logging level ('debug', 'info', 'warn' (default), 'error', 'fatal', 'panic')
 - `--log-format` - set the format used by logs ('text' (default), or 'json')
+- `--crt-api-version` - Container runtime API version
 - `--quiet` - less verbose CLI execution mode
 - `--output-format` - set the output format to use ('text' (default), or 'json')
 - `--log` - log file to store logs
@@ -464,10 +465,12 @@ In the interactive CLI prompt mode you must specify the target image using the `
 - `--entrypoint` - Override ENTRYPOINT analyzing image at runtime
 - `--cmd` - Override CMD analyzing image at runtime
 - `--mount` - Mount volume analyzing image (the mount parameter format is identical to the `-v` mount command in Docker) [can use this flag multiple times]
-- `--include-path` - Include directory or file from image [can use this flag multiple times] (optionally overwriting the artifact's permissions, user and group information; format: `target:octalPermFlags#uid#gid` ; see the non-default USER FAQ section for more details)
-- `--include-path-file` - Load directory or file includes from a file (optionally overwriting the artifact's permissions, user and group information; format: `target:octalPermFlags#uid#gid` ; see the non-default USER FAQ section for more details)
+- `--include-path` - Include directory (and what's in it) or file from image [can use this flag multiple times] (optionally overwriting the artifact's permissions, user and group information; full format: `targetPath:octalPermFlags#uid#gid`, mimimal format: `targetPath` ; see the non-default USER FAQ section for more details)
+- `--include-path-file` - Load directory or file includes from a file (optionally overwriting the artifact's permissions, user and group information; full format: `targetPath:octalPermFlags#uid#gid`, mimimal format: `targetPath` ; see the non-default USER FAQ section for more details)
+- `--include-paths-creport-file` - Keep files from the referenced creport
 - `--include-bin value` - Include binary from image (executable or shared object using its absolute path)
 - `--include-bin-file` - Load shared binary file includes from a file (similar to `--include-path-file`)
+- `--include-dir-bins value` - Include binaries in the target directory and include their dependencies, which could be in other locations (executables or shared objects using its absolute path)
 - `--include-exe value` - Include executable from image (by executable name)
 - `--include-exe-file` - Load executable file includes from a file (similar to `--include-path-file`)
 - `--include-shell` - Include basic shell functionality (default value: false)
@@ -479,6 +482,7 @@ In the interactive CLI prompt mode you must specify the target image using the `
 - `--include-cert-pk-dirs` - Keep known cert private key directories and all files in them
 - `--include-new` - Keep new files created by target during dynamic analysis (default value: true)
 - `--include-oslibs-net` - Keep the common networking OS libraries (default value: true)
+- `--include-ssh-client` - Keep the common SSH client components and configs
 - `--include-zoneinfo` - Keep the OS/libc zoneinfo data (default value: false)
 - `--include-app-nuxt-dir` - Keep the root Nuxt.js app directory (default value: false)
 - `--include-app-nuxt-build-dir` - Keep the build Nuxt.js app directory (default value: false)
@@ -495,7 +499,7 @@ In the interactive CLI prompt mode you must specify the target image using the `
 - `--preserve-path-file` - File with paths to keep from original image in their original state (changes to the selected container image files when it runs will be discarded).
 - `--path-perms` - Set path permissions/user/group in optimized image (format: `target:octalPermFlags#uid#gid` ; see the non-default USER FAQ section for more details)
 - `--path-perms-file` - File with path permissions to set (format: `target:octalPermFlags#uid#gid` ; see the non-default USER FAQ section for more details)
-- `--exclude-pattern` - Exclude path pattern ([Glob/Match in Go](https://golang.org/pkg/path/filepath/#Match) and `**`) from image
+- `--exclude-pattern` - Exclude path pattern ([Glob/Match in Go](https://golang.org/pkg/path/filepath/#Match) and `**`) from image (useful when `--include-path` keeps a directory and you need to exclude / filter out some of the files in that directory)
 - `--exclude-varlock-files` - Exclude the files in the var and run lock directory (default value: true)
 - `--exclude-mounts` - Exclude mounted volumes from image (default value: true)
 - `--label` - Override or add LABEL analyzing image at runtime [can use this flag multiple times]

go.mod

@@ -18,17 +18,20 @@ require (
 	github.com/getkin/kin-openapi v0.76.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/gocolly/colly/v2 v2.1.0
-	github.com/google/go-containerregistry v0.16.1
+	github.com/google/go-containerregistry v0.19.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/gorilla/websocket v1.4.2
+	github.com/jedib0t/go-pretty/v6 v6.4.9
 	github.com/moby/term v0.5.0
+	github.com/opencontainers/image-spec v1.1.0-rc5
 	github.com/pkg/errors v0.9.1
 	github.com/segmentio/ksuid v1.0.4
 	github.com/sirupsen/logrus v1.9.3
 	github.com/slimtoolkit/go-update v0.0.0-20231119011834-99945ebd76f7
 	github.com/slimtoolkit/uiprogress v0.0.0-20231119012247-4a052fb12f37
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
 	github.com/urfave/cli/v2 v2.3.0
+	golang.org/x/crypto v0.17.0
 	golang.org/x/net v0.18.0
 	golang.org/x/sys v0.15.0
 	k8s.io/api v0.27.3
@@ -68,7 +71,6 @@ require (
 	github.com/hooklift/assert v0.1.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/jedib0t/go-pretty/v6 v6.4.9 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/klauspost/compress v1.17.3 // indirect
@@ -84,7 +86,6 @@ require (
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/opencontainers/runc v1.1.10 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -99,7 +100,6 @@ require (
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/oauth2 v0.10.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect

go.sum

@@ -179,8 +179,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ=
-github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
+github.com/google/go-containerregistry v0.19.0 h1:uIsMRBV7m/HDkDxE/nXMnv1q+lOOSPlQ/ywc5JbB8Ic=
+github.com/google/go-containerregistry v0.19.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -256,8 +256,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
-github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
@@ -496,17 +494,13 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
-golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
-golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
 golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/app/master/cli.go

@@ -12,21 +12,21 @@ import (
 	"github.com/slimtoolkit/slim/pkg/app/master/command"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/appbom"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/build"
-	"github.com/slimtoolkit/slim/pkg/app/master/command/containerize"
-	"github.com/slimtoolkit/slim/pkg/app/master/command/convert"
+	//"github.com/slimtoolkit/slim/pkg/app/master/command/containerize"
+	//"github.com/slimtoolkit/slim/pkg/app/master/command/convert"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/debug"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/dockerclipm"
-	"github.com/slimtoolkit/slim/pkg/app/master/command/edit"
+	//"github.com/slimtoolkit/slim/pkg/app/master/command/edit"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/help"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/images"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/install"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/lint"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/merge"
-	"github.com/slimtoolkit/slim/pkg/app/master/command/probe"
+	//"github.com/slimtoolkit/slim/pkg/app/master/command/probe"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/profile"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/registry"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/run"
-	"github.com/slimtoolkit/slim/pkg/app/master/command/server"
+	//"github.com/slimtoolkit/slim/pkg/app/master/command/server"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/update"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/version"
 	"github.com/slimtoolkit/slim/pkg/app/master/command/vulnerability"
@@ -60,13 +60,13 @@ func registerCommands() {
 	help.RegisterCommand()
 	update.RegisterCommand()
 	install.RegisterCommand()
-	edit.RegisterCommand()
-	probe.RegisterCommand()
-	convert.RegisterCommand()
+	//edit.RegisterCommand() - not doing anything yet
+	//probe.RegisterCommand() - not doing anything yet
+	//convert.RegisterCommand() - not doing anything yet
 	run.RegisterCommand()
-	server.RegisterCommand()
+	//server.RegisterCommand() - not doing anything yet
 	debug.RegisterCommand()
-	containerize.RegisterCommand()
+	//containerize.RegisterCommand() - not doing anything yet
 	dockerclipm.RegisterCommand()
 }
 

pkg/app/master/command/build/cli.go

@@ -661,6 +661,9 @@ var CLI = &cli.Command{
 			}
 		}
 
+		//note: if path perms, ID change are provided they are applied to all matching binaries
+		includeDirBinsList := command.ParsePaths(ctx.StringSlice(FlagIncludeDirBins))
+
 		includeExes := command.ParsePaths(ctx.StringSlice(FlagIncludeExe))
 		moreIncludeExes, err := command.ParsePathsFile(ctx.String(FlagIncludeExeFile))
 		if err != nil {
@@ -806,6 +809,7 @@ var CLI = &cli.Command{
 			preservePaths,
 			includePaths,
 			includeBins,
+			includeDirBinsList,
 			includeExes,
 			doIncludeShell,
 			doIncludeWorkdir,

pkg/app/master/command/build/flags.go

@@ -34,21 +34,40 @@ const (
 	FlagIncludeExeFile   = "include-exe-file"
 	FlagIncludeShell     = "include-shell"
 
+	FlagIncludeDirBins = "include-dir-bins"
+	FlagIncludeDirBinsUsage = "Keep binaries in the target directory (executables or shared objects) and their dependencies, which could be in other locations"
+
 	FlagIncludeWorkdir            = "include-workdir"
-	FlagWorkdirExclude            = "workdir-exclude"
-	FlagIncludeAppImageAddCopyAll = "include-app-image-addcopy-all"
-	FlagIncludeAppImageRun        = "include-app-image-run"
+	FlagIncludeWorkdirUsage = "Keep files in working directory"
+
+	//TBD
+	FlagWorkdirExclude      = "workdir-exclude"
+	FlagWorkdirExcludeUsage = "Exclude filter for artifacts when working directory is included"
+
+	FlagIncludeAppImageAddCopyAll = "include-app-image-addcopy-all" //TBD
+	FlagIncludeAppImageRun        = "include-app-image-run" //TBD
+
 	FlagIncludeAppImageAll        = "include-app-image-all"
+	FlagIncludeAppImageAllUsage   = "Keep everything in the app part of the container image"
 
 	FlagAppImageStartInst       = "app-image-start-instruction"
-	FlagAppImageStartLayerCount = "app-image-start-layer-count"
+	FlagAppImageStartInstUsage  = "Instruction (prefix) that indicates where the app starts in the container image"
+
+	FlagAppImageStartLayerCount = "app-image-start-layer-count" //TBD
+
 	FlagAppImageStartInstGroup  = "app-image-start-instruction-group"
-	FlagAppImageStartDetect     = "app-image-start-detect"
+	FlagAppImageStartInstGroupUsage = "Instruction group (reverse) index that indicates where the app starts in the container image"
+
+	FlagAppImageStartDetect     = "app-image-start-detect" //TBD
+
 	FlagAppImageDockerfile      = "app-image-dockerfile" //TODO: make it work with FlagBuildFromDockerfile too
+	FlagAppImageDockerfileUsage     = "Path to app image Dockerfile (used to determine where the application part of the image starts)"
 
 	FlagIncludePathsCreportFile = "include-paths-creport-file"
+	FlagIncludePathsCreportFileUsage = "Keep files from the referenced creport"
 
 	FlagIncludeOSLibsNet = "include-oslibs-net"
+	FlagIncludeOSLibsNetUsage = "Keep the common networking OS libraries"
 
 	FlagIncludeSSHClient           = "include-ssh-client"
 	FlagIncludeSSHClientUsage      = "Keep the common SSH client components and configs"
@@ -152,17 +171,6 @@ const (
 	FlagIncludeExeUsage       = "Keep executable from original image (by executable name)"
 	FlagIncludeShellUsage     = "Keep basic shell functionality"
 
-	FlagIncludeWorkdirUsage = "Keep files in working directory"
-
-	FlagIncludeAppImageAllUsage     = "Keep everything in the app part of the container image"
-	FlagAppImageStartInstGroupUsage = "Instruction group (reverse) index that indicates where the app starts in the container image"
-	FlagAppImageStartInstUsage      = "Instruction (prefix) that indicates where the app starts in the container image"
-	FlagAppImageDockerfileUsage     = "Path to app image Dockerfile (used to determine where the application part of the image starts)"
-
-	FlagIncludePathsCreportFileUsage = "Keep files from the referenced creport"
-
-	FlagIncludeOSLibsNetUsage = "Keep the common networking OS libraries"
-
 	FlagIncludeZoneInfoUsage = "Keep the OS/libc zoneinfo data"
 
 	FlagIncludeCertAllUsage     = "Keep all discovered cert files"
@@ -271,6 +279,12 @@ var Flags = map[string]cli.Flag{
 		Usage:   FlagIncludeBinUsage,
 		EnvVars: []string{"DSLIM_INCLUDE_BIN"},
 	},
+	FlagIncludeDirBins: &cli.StringSliceFlag{
+		Name:    FlagIncludeDirBins,
+		Value:   cli.NewStringSlice(),
+		Usage:   FlagIncludeDirBinsUsage,
+		EnvVars: []string{"DSLIM_INCLUDE_DIR_BINS"},
+	},
 	FlagIncludeExe: &cli.StringSliceFlag{
 		Name:    FlagIncludeExe,
 		Value:   cli.NewStringSlice(),

pkg/app/master/command/build/handler.go

@@ -119,6 +119,7 @@ func OnCommand(
 	preservePaths map[string]*fsutil.AccessInfo,
 	includePaths map[string]*fsutil.AccessInfo,
 	includeBins map[string]*fsutil.AccessInfo,
+	includeDirBinsList map[string]*fsutil.AccessInfo,
 	includeExes map[string]*fsutil.AccessInfo,
 	doIncludeShell bool,
 	doIncludeWorkdir bool,
@@ -1086,6 +1087,7 @@ func OnCommand(
 		preservePaths,
 		includePaths,
 		includeBins,
+		includeDirBinsList,
 		includeExes,
 		doIncludeShell,
 		doIncludeWorkdir,

pkg/app/master/command/profile/handler.go

@@ -253,6 +253,7 @@ func OnCommand(
 		nil,   //preservePaths,
 		nil,   //includePaths,
 		nil,   //includeBins,
+		nil,   //includeDirBinsList,
 		nil,   //includeExes,
 		false, //doIncludeShell,
 		false, //doIncludeWorkdir,

pkg/app/master/inspectors/container/container_inspector.go

@@ -116,6 +116,7 @@ type Inspector struct {
 	PreservePaths         map[string]*fsutil.AccessInfo
 	IncludePaths          map[string]*fsutil.AccessInfo
 	IncludeBins           map[string]*fsutil.AccessInfo
+	IncludeDirBinsList    map[string]*fsutil.AccessInfo
 	IncludeExes           map[string]*fsutil.AccessInfo
 	DoIncludeShell        bool
 	DoIncludeWorkdir      bool
@@ -196,6 +197,7 @@ func NewInspector(
 	preservePaths map[string]*fsutil.AccessInfo,
 	includePaths map[string]*fsutil.AccessInfo,
 	includeBins map[string]*fsutil.AccessInfo,
+	includeDirBinsList map[string]*fsutil.AccessInfo,
 	includeExes map[string]*fsutil.AccessInfo,
 	doIncludeShell bool,
 	doIncludeWorkdir bool,
@@ -253,6 +255,7 @@ func NewInspector(
 		PreservePaths:         preservePaths,
 		IncludePaths:          includePaths,
 		IncludeBins:           includeBins,
+		IncludeDirBinsList:    includeDirBinsList,
 		IncludeExes:           includeExes,
 		DoIncludeShell:        doIncludeShell,
 		DoIncludeWorkdir:      doIncludeWorkdir,
@@ -797,6 +800,10 @@ func (i *Inspector) RunContainer() error {
 		cmd.IncludeBins = pathMapKeys(i.IncludeBins)
 	}
 
+	if len(i.IncludeDirBinsList) > 0 {
+		cmd.IncludeDirBinsList = pathMapKeys(i.IncludeDirBinsList)
+	}
+
 	if len(i.IncludeExes) > 0 {
 		cmd.IncludeExes = pathMapKeys(i.IncludeExes)
 	}