Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(trivy): go mod update trivy v0.17.2 #1235

Merged
merged 9 commits into from
May 12, 2021
Merged

feat(trivy): go mod update trivy v0.17.2 #1235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5a7cb32
feat(trivy): go mod update trivy v0.17.2
kotakanbe May 10, 2021
c697a3a
wg.Wait
kotakanbe May 11, 2021
4ea76ef
fix reporting
kotakanbe May 11, 2021
06118cc
fix test case
kotakanbe May 11, 2021
67c070a
add gemfile.lock of redmine to integration test
kotakanbe May 12, 2021
0a24738
fix(test): add Pipfile.lock
kotakanbe May 12, 2021
133c766
add poetry.lock to integration test
kotakanbe May 12, 2021
25aa1ee
add composer.lock to integration test
kotakanbe May 12, 2021
09110a3
add integration test case
kotakanbe May 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 80 additions & 40 deletions GNUmakefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,12 @@ build-future-vuls: pretest fmt

# integration-test
BASE_DIR := '${PWD}/integration/results'
# $(shell mkdir -p ${BASE_DIR})
NOW=$(shell date --iso-8601=seconds)
NOW_JSON_DIR := '${BASE_DIR}/$(NOW)'
ONE_SEC_AFTER=$(shell date -d '+1 second' --iso-8601=seconds)
ONE_SEC_AFTER_JSON_DIR := '${BASE_DIR}/$(ONE_SEC_AFTER)'
LIBS := 'gemfile' 'pipfile' 'poetry' 'composer' 'packagelock' 'yarn' 'cargo'

diff:
# git clone git@github.com:vulsio/vulsctl.git
Expand All @@ -100,18 +102,26 @@ diff:
# make int
# (ex. test 10 times: for i in `seq 10`; do make int ARGS=-quiet ; done)
ifneq ($(shell ls -U1 ${BASE_DIR} | wc -l), 0)
mv ${BASE_DIR}/* /tmp
mv ${BASE_DIR} /tmp/${NOW}
endif
mkdir -p ${NOW_JSON_DIR}
cp integration/data/*.json ${NOW_JSON_DIR}
./vuls.old report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-config.toml $(ARGS)
sleep 1
./vuls.old scan -config=./integration/int-config.toml --results-dir=${BASE_DIR} ${LIBS}
cp ${BASE_DIR}/current/*.json ${NOW_JSON_DIR}
cp integration/data/results/*.json ${NOW_JSON_DIR}
./vuls.old report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-config.toml ${NOW}

mkdir -p ${ONE_SEC_AFTER_JSON_DIR}
cp integration/data/*.json ${ONE_SEC_AFTER_JSON_DIR}
./vuls.new report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-config.toml $(ARGS)
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/reportedAt/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/reportedAt/d' {} \;
diff -c ${NOW_JSON_DIR} ${ONE_SEC_AFTER_JSON_DIR}
sleep 1
./vuls.new scan -config=./integration/int-config.toml --results-dir=${BASE_DIR} ${LIBS}
cp ${BASE_DIR}/current/*.json ${ONE_SEC_AFTER_JSON_DIR}
cp integration/data/results/*.json ${ONE_SEC_AFTER_JSON_DIR}
./vuls.new report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-config.toml ${ONE_SEC_AFTER}

$(call sed-d)
- diff -c ${NOW_JSON_DIR} ${ONE_SEC_AFTER_JSON_DIR}
echo "old: ${NOW_JSON_DIR} , new: ${ONE_SEC_AFTER_JSON_DIR}"
$(call count-cve)

diff-redis:
# docker network create redis-nw
Expand All @@ -125,47 +135,50 @@ diff-redis:
# ln -s oldvuls vuls.old
# make int-redis
ifneq ($(shell ls -U1 ${BASE_DIR} | wc -l), 0)
mv ${BASE_DIR}/* /tmp
mv ${BASE_DIR} /tmp/${NOW}
endif
mkdir -p ${NOW_JSON_DIR}
cp integration/data/*.json ${NOW_JSON_DIR}
./vuls.old report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-redis-config.toml
sleep 1
./vuls.old scan -config=./integration/int-config.toml --results-dir=${BASE_DIR} ${LIBS}
cp -f ${BASE_DIR}/current/*.json ${NOW_JSON_DIR}
cp integration/data/results/*.json ${NOW_JSON_DIR}
./vuls.old report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-redis-config.toml ${NOW}

mkdir -p ${ONE_SEC_AFTER_JSON_DIR}
cp integration/data/*.json ${ONE_SEC_AFTER_JSON_DIR}
./vuls.new report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-redis-config.toml
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/reportedAt/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/reportedAt/d' {} \;
diff -c ${NOW_JSON_DIR} ${ONE_SEC_AFTER_JSON_DIR}
sleep 1
./vuls.new scan -config=./integration/int-config.toml --results-dir=${BASE_DIR} ${LIBS}
cp -f ${BASE_DIR}/current/*.json ${ONE_SEC_AFTER_JSON_DIR}
cp integration/data/results/*.json ${ONE_SEC_AFTER_JSON_DIR}
./vuls.new report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-redis-config.toml ${ONE_SEC_AFTER}

$(call sed-d)
- diff -c ${NOW_JSON_DIR} ${ONE_SEC_AFTER_JSON_DIR}
echo "old: ${NOW_JSON_DIR} , new: ${ONE_SEC_AFTER_JSON_DIR}"
$(call count-cve)

diff-rdb-redis:
ifneq ($(shell ls -U1 ${BASE_DIR} | wc -l), 0)
mv ${BASE_DIR}/* /tmp
mv ${BASE_DIR} /tmp/${NOW}
endif
mkdir -p ${NOW_JSON_DIR}
cp integration/data/*.json ${NOW_JSON_DIR}
./vuls.new report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-config.toml
sleep 1
# new vs new
./vuls.new scan -config=./integration/int-config.toml --results-dir=${BASE_DIR} ${LIBS}
cp -f ${BASE_DIR}/current/*.json ${NOW_JSON_DIR}
cp integration/data/results/*.json ${NOW_JSON_DIR}
./vuls.new report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-config.toml ${NOW}

mkdir -p ${ONE_SEC_AFTER_JSON_DIR}
cp integration/data/*.json ${ONE_SEC_AFTER_JSON_DIR}
./vuls.new report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-redis-config.toml
# remove reportedAt line
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/reportedAt/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/reportedAt/d' {} \;
# remove "Type": line
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/"Type":/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/"Type":/d' {} \;
# remove "SQLite3Path": line
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/"SQLite3Path":/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/"SQLite3Path":/d' {} \;
diff -c ${NOW_JSON_DIR} ${ONE_SEC_AFTER_JSON_DIR}
sleep 1
./vuls.new scan -config=./integration/int-config.toml --results-dir=${BASE_DIR} ${LIBS}
cp -f ${BASE_DIR}/current/*.json ${ONE_SEC_AFTER_JSON_DIR}
cp integration/data/results/*.json ${ONE_SEC_AFTER_JSON_DIR}
./vuls.new report --format-json --refresh-cve --results-dir=${BASE_DIR} -config=./integration/int-redis-config.toml ${ONE_SEC_AFTER}

$(call sed-d)
- diff -c ${NOW_JSON_DIR} ${ONE_SEC_AFTER_JSON_DIR}
echo "old: ${NOW_JSON_DIR} , new: ${ONE_SEC_AFTER_JSON_DIR}"
for jsonfile in ${NOW_JSON_DIR}/*.json ; do \
echo $$jsonfile; cat $$jsonfile | jq ".scannedCves | length" ; \
done
for jsonfile in ${ONE_SEC_AFTER_JSON_DIR}/*.json ; do \
echo $$jsonfile; cat $$jsonfile | jq ".scannedCves | length" ; \
done

$(call count-cve)

head= $(shell git rev-parse HEAD)
prev= $(shell git rev-parse HEAD^)
Expand Down Expand Up @@ -199,5 +212,32 @@ build-integration:
# $ ln -s ./vuls.${head} ./vuls.old
# or
# $ ln -s ./vuls.${prev} ./vuls.old
# $ make int
# $ make int-redis
# then
# $ make diff
# $ make diff-redis
# $ make diff-rdb-redis


define sed-d
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/scannedAt/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/scannedAt/d' {} \;
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/reportedAt/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/reportedAt/d' {} \;
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/"Type":/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/"Type":/d' {} \;
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/"SQLite3Path":/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/"SQLite3Path":/d' {} \;
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/reportedRevision/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/reportedRevision/d' {} \;
find ${NOW_JSON_DIR} -type f -exec sed -i -e '/scannedRevision/d' {} \;
find ${ONE_SEC_AFTER_JSON_DIR} -type f -exec sed -i -e '/scannedRevision/d' {} \;
endef

define count-cve
for jsonfile in ${NOW_JSON_DIR}/*.json ; do \
echo $$jsonfile; cat $$jsonfile | jq ".scannedCves | length" ; \
done
for jsonfile in ${ONE_SEC_AFTER_JSON_DIR}/*.json ; do \
echo $$jsonfile; cat $$jsonfile | jq ".scannedCves | length" ; \
done
endef
8 changes: 3 additions & 5 deletions detector/detector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,10 @@ func Detect(rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
return nil, xerrors.Errorf("Failed to detect WordPress Cves: %w", err)
}

logging.Log.Infof("Fill CVE detailed with gost")
if err := gost.FillCVEsWithRedHat(&r, config.Conf.Gost); err != nil {
return nil, xerrors.Errorf("Failed to fill with gost: %w", err)
}

logging.Log.Infof("Fill CVE detailed with go-cve-dictionary")
if err := FillCvesWithNvdJvn(&r, config.Conf.CveDict, config.Conf.LogOpts); err != nil {
return nil, xerrors.Errorf("Failed to fill with CVE: %w", err)
}
Expand All @@ -92,13 +90,13 @@ func Detect(rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
if err != nil {
return nil, xerrors.Errorf("Failed to fill with exploit: %w", err)
}
logging.Log.Infof("%s: %d exploits are detected", r.FormatServerName(), nExploitCve)
logging.Log.Infof("%s: %d PoC are detected", r.FormatServerName(), nExploitCve)

nMetasploitCve, err := FillWithMetasploit(&r, config.Conf.Metasploit)
if err != nil {
return nil, xerrors.Errorf("Failed to fill with metasploit: %w", err)
}
logging.Log.Infof("%s: %d modules are detected", r.FormatServerName(), nMetasploitCve)
logging.Log.Infof("%s: %d exploits are detected", r.FormatServerName(), nMetasploitCve)

FillCweDict(&r)

Expand Down Expand Up @@ -244,7 +242,7 @@ func DetectWordPressCves(r *models.ScanResult, wpCnf config.WpScanConf) error {
if len(r.WordPressPackages) == 0 {
return nil
}
logging.Log.Infof("Detect WordPress CVE. pkgs: %d ", len(r.WordPressPackages))
logging.Log.Infof("%s: Detect WordPress CVE. pkgs: %d ", r.ServerInfo(), (r.WordPressPackages))
n, err := detectWordPressCves(r, wpCnf)
if err != nil {
return xerrors.Errorf("Failed to detect WordPress CVE: %w", err)
Expand Down
11 changes: 5 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ go 1.16
require (
github.com/Azure/azure-sdk-for-go v50.2.0+incompatible
github.com/BurntSushi/toml v0.3.1
github.com/aquasecurity/fanal v0.0.0-20210119051230-28c249da7cfd
github.com/aquasecurity/trivy v0.16.0
github.com/aquasecurity/trivy-db v0.0.0-20210121143430-2a5c54036a86
github.com/aquasecurity/fanal v0.0.0-20210501093021-8aaac3e8dea7
github.com/aquasecurity/trivy v0.17.2
github.com/aquasecurity/trivy-db v0.0.0-20210429114658-ae22941a55d0
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
github.com/aws/aws-sdk-go v1.36.31
github.com/boltdb/bolt v1.3.1
Expand Down Expand Up @@ -44,17 +44,16 @@ require (
github.com/parnurzeal/gorequest v0.2.16
github.com/pelletier/go-toml v1.8.1 // indirect
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
github.com/sirupsen/logrus v1.7.0
github.com/sirupsen/logrus v1.8.0
github.com/spf13/afero v1.6.0
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/cobra v1.1.3
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/takuzoo3868/go-msfdb v0.1.5
github.com/vulsio/go-exploitdb v0.1.7
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
golang.org/x/oauth2 v0.0.0-20210125201302-af13f521f196
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect
golang.org/x/tools v0.1.0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
Expand Down
Loading