From 5433e2ad15b01bae3adf928474fb4a629fb743bb Mon Sep 17 00:00:00 2001 From: Max Fisher <112151114+maxfisher-g@users.noreply.github.com> Date: Thu, 2 Nov 2023 18:54:42 +1100 Subject: [PATCH] add separate result bucket for execution log (#950) * add separate result bucket for execution log Signed-off-by: Max Fisher * update e2e test and production configs with the new bucket name Signed-off-by: Max Fisher --------- Signed-off-by: Max Fisher --- Makefile | 7 ++++- cmd/analyze/main.go | 57 ++++++++++++++++++---------------- cmd/worker/main.go | 22 ++++++++----- configs/e2e/docker-compose.yml | 11 ++++--- infra/worker/workers-set.yaml | 6 ++-- internal/worker/save_data.go | 9 +++--- scripts/run_analysis.sh | 2 +- 7 files changed, 66 insertions(+), 48 deletions(-) diff --git a/Makefile b/Makefile index 1df4aff3..c8f28990 100644 --- a/Makefile +++ b/Makefile @@ -129,11 +129,16 @@ run: E2E_TEST_COMPOSE_ARGS := -p pa-e2e-testing -f ./configs/e2e/docker-compose.yml -f ./test/e2e/docker-compose.test.yml +.PHONY: e2e_test_build +e2e_test_build: build_e2e_test_images + .PHONY: e2e_test_start -e2e_test_start: build_e2e_test_images +e2e_test_start: docker-compose $(E2E_TEST_COMPOSE_ARGS) up -d @echo @echo "To see analysis results, go to http://localhost:9000/minio/package-analysis" + @echo "Username: minio" + @echo "Password: minio123" @echo @echo "Remember to run 'make e2e_test_stop' when done!" @sleep 5 diff --git a/cmd/analyze/main.go b/cmd/analyze/main.go index 55fedb41..90cd3f2d 100644 --- a/cmd/analyze/main.go +++ b/cmd/analyze/main.go @@ -24,42 +24,45 @@ import ( ) var ( - pkgName = flag.String("package", "", "package name") - localPkg = flag.String("local", "", "local package path") - ecosystem pkgecosystem.Ecosystem - version = flag.String("version", "", "version") - noPull = flag.Bool("nopull", false, "disables pulling down sandbox images") - imageTag = flag.String("image-tag", "", "set image tag for analysis sandboxes") - dynamicUpload = flag.String("upload", "", "bucket path for uploading dynamic analysis results") - staticUpload = flag.String("upload-static", "", "bucket path for uploading static analysis results") - uploadFileWriteInfo = flag.String("upload-file-write-info", "", "bucket path for uploading information from file writes") - uploadAnalyzedPkg = flag.String("upload-analyzed-pkg", "", "bucket path for uploading analyzed packages") - offline = flag.Bool("offline", false, "disables sandbox network access") - customSandbox = flag.String("sandbox-image", "", "override default dynamic analysis sandbox with custom image") - customAnalysisCmd = flag.String("analysis-command", "", "override default dynamic analysis script path (use with custom sandbox image)") - listModes = flag.Bool("list-modes", false, "prints out a list of available analysis modes") - features = flag.String("features", "", "override features that are enabled/disabled by default") - listFeatures = flag.Bool("list-features", false, "list available features that can be toggled") - help = flag.Bool("help", false, "print help on available options") - analysisMode = utils.CommaSeparatedFlags("mode", []string{"static", "dynamic"}, + pkgName = flag.String("package", "", "package name") + localPkg = flag.String("local", "", "local package path") + ecosystem pkgecosystem.Ecosystem + version = flag.String("version", "", "version") + noPull = flag.Bool("nopull", false, "disables pulling down sandbox images") + imageTag = flag.String("image-tag", "", "set image tag for analysis sandboxes") + dynamicBucket = flag.String("dynamic-bucket", "", "bucket path for uploading dynamic analysis results") + staticBucket = flag.String("static-bucket", "", "bucket path for uploading static analysis results") + executionLogBucket = flag.String("execution-log-bucket", "", "bucket path for uploading execution log (dynamic analysis)") + fileWritesBucket = flag.String("file-writes-bucket", "", "bucket path for uploading file writes data (dynamic analysis)") + analyzedPkgBucket = flag.String("analyzed-pkg-bucket", "", "bucket path for uploading analyzed packages") + offline = flag.Bool("offline", false, "disables sandbox network access") + customSandbox = flag.String("sandbox-image", "", "override default dynamic analysis sandbox with custom image") + customAnalysisCmd = flag.String("analysis-command", "", "override default dynamic analysis script path (use with custom sandbox image)") + listModes = flag.Bool("list-modes", false, "prints out a list of available analysis modes") + features = flag.String("features", "", "override features that are enabled/disabled by default") + listFeatures = flag.Bool("list-features", false, "list available features that can be toggled") + help = flag.Bool("help", false, "print help on available options") + analysisMode = utils.CommaSeparatedFlags("mode", []string{"static", "dynamic"}, "list of analysis modes to run, separated by commas. Use -list-modes to see available options") ) func makeResultStores() worker.ResultStores { rs := worker.ResultStores{} - if *dynamicUpload != "" { - rs.DynamicAnalysis = resultstore.New(*dynamicUpload) + if *analyzedPkgBucket != "" { + rs.AnalyzedPackage = resultstore.New(*analyzedPkgBucket) } - if *staticUpload != "" { - rs.StaticAnalysis = resultstore.New(*staticUpload) + if *dynamicBucket != "" { + rs.DynamicAnalysis = resultstore.New(*dynamicBucket) } - if *uploadFileWriteInfo != "" { - rs.FileWrites = resultstore.New(*uploadFileWriteInfo) + if *executionLogBucket != "" { + rs.ExecutionLog = resultstore.New(*executionLogBucket) } - - if *uploadAnalyzedPkg != "" { - rs.AnalyzedPackage = resultstore.New(*uploadAnalyzedPkg) + if *fileWritesBucket != "" { + rs.FileWrites = resultstore.New(*fileWritesBucket) + } + if *staticBucket != "" { + rs.StaticAnalysis = resultstore.New(*staticBucket) } return rs diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 57c1c3fb..9d5069a3 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -37,10 +37,11 @@ const ( // resultBucketPaths holds bucket paths for the different types of results. type resultBucketPaths struct { + analyzedPkg string dynamicAnalysis string - staticAnalysis string + executionLog string fileWrites string - analyzedPkg string + staticAnalysis string } type sandboxImageSpec struct { @@ -79,17 +80,20 @@ func copyPackageToLocalFile(ctx context.Context, packagesBucket *blob.Bucket, bu func makeResultStores(dest resultBucketPaths) worker.ResultStores { resultStores := worker.ResultStores{} + if dest.analyzedPkg != "" { + resultStores.AnalyzedPackage = resultstore.New(dest.analyzedPkg, resultstore.ConstructPath()) + } if dest.dynamicAnalysis != "" { resultStores.DynamicAnalysis = resultstore.New(dest.dynamicAnalysis, resultstore.ConstructPath()) } - if dest.staticAnalysis != "" { - resultStores.StaticAnalysis = resultstore.New(dest.staticAnalysis, resultstore.ConstructPath()) + if dest.executionLog != "" { + resultStores.ExecutionLog = resultstore.New(dest.executionLog, resultstore.ConstructPath()) } if dest.fileWrites != "" { resultStores.FileWrites = resultstore.New(dest.fileWrites, resultstore.ConstructPath()) } - if dest.analyzedPkg != "" { - resultStores.AnalyzedPackage = resultstore.New(dest.analyzedPkg, resultstore.ConstructPath()) + if dest.staticAnalysis != "" { + resultStores.StaticAnalysis = resultstore.New(dest.staticAnalysis, resultstore.ConstructPath()) } return resultStores @@ -274,10 +278,11 @@ func main() { } resultsBuckets := resultBucketPaths{ + analyzedPkg: os.Getenv("OSSF_MALWARE_ANALYZED_PACKAGES"), dynamicAnalysis: os.Getenv("OSSF_MALWARE_ANALYSIS_RESULTS"), - staticAnalysis: os.Getenv("OSSF_MALWARE_STATIC_ANALYSIS_RESULTS"), + executionLog: os.Getenv("OSSF_MALWARE_ANALYSIS_EXECUTION_LOGS"), fileWrites: os.Getenv("OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS"), - analyzedPkg: os.Getenv("OSSF_MALWARE_ANALYZED_PACKAGES"), + staticAnalysis: os.Getenv("OSSF_MALWARE_STATIC_ANALYSIS_RESULTS"), } resultStores := makeResultStores(resultsBuckets) @@ -305,6 +310,7 @@ func main() { "static_results_bucket", resultsBuckets.staticAnalysis, "file_write_results_bucket", resultsBuckets.fileWrites, "analyzed_packages_bucket", resultsBuckets.analyzedPkg, + "execution_log_bucket", resultsBuckets.executionLog, "image_tag", imageSpec.tag, "image_nopull", imageSpec.noPull, "topic_notification", notificationTopicURL, diff --git a/configs/e2e/docker-compose.yml b/configs/e2e/docker-compose.yml index 34c1c643..1aac5c8e 100644 --- a/configs/e2e/docker-compose.yml +++ b/configs/e2e/docker-compose.yml @@ -47,7 +47,7 @@ services: MINIO_ROOT_PASSWORD: minio123 MINIO_REGION_NAME: dummy_region entrypoint: sh - command: -c 'mkdir -p /data/package-analysis{,-static,-file-writes,-analyzed-packages} && /usr/bin/minio server /data' + command: -c 'mkdir -p /data/package-analysis/{analyzed-packages,dynamic,execution-logs,file-writes,static} && /usr/bin/minio server /data' healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 30s @@ -64,10 +64,11 @@ services: environment: OSSMALWARE_WORKER_SUBSCRIPTION: kafka://worker?topic=workers OSSF_MALWARE_NOTIFICATION_TOPIC: kafka://notifications - OSSF_MALWARE_ANALYSIS_RESULTS: s3://package-analysis?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true - OSSF_MALWARE_STATIC_ANALYSIS_RESULTS: s3://package-analysis-static?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true - OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS: s3://package-analysis-file-writes?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true - OSSF_MALWARE_ANALYZED_PACKAGES: s3://package-analysis-analyzed-packages?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true + OSSF_MALWARE_ANALYZED_PACKAGES: s3://package-analysis/analyzed-packages?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true + OSSF_MALWARE_ANALYSIS_RESULTS: s3://package-analysis/dynamic?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true + OSSF_MALWARE_ANALYSIS_EXECUTION_LOGS: s3://package-analysis/execution-logs?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true + OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS: s3://package-analysis/file-writes?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true + OSSF_MALWARE_STATIC_ANALYSIS_RESULTS: s3://package-analysis/static?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true OSSF_MALWARE_ANALYSIS_ENABLE_PROFILER: "true" OSSF_MALWARE_FEATURE_FLAGS: "" KAFKA_BROKERS: kafka:9092 diff --git a/infra/worker/workers-set.yaml b/infra/worker/workers-set.yaml index a3edce0c..151fd0d9 100644 --- a/infra/worker/workers-set.yaml +++ b/infra/worker/workers-set.yaml @@ -25,10 +25,12 @@ spec: value: gcppubsub://projects/ossf-malware-analysis/subscriptions/workers?nacklazy=1 - name: OSSF_MALWARE_ANALYSIS_RESULTS value: gs://ossf-malware-analysis-results - - name: OSSF_MALWARE_STATIC_ANALYSIS_RESULTS - value: gs://ossf-malware-static-analysis-results-v1 + - name: OSSF_MALWARE_ANALYSIS_EXECUTION_LOGS + value: gs://ossf-malware-analysis-execution-logs - name: OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS value: gs://ossf-malware-analysis-file-write-results + - name: OSSF_MALWARE_STATIC_ANALYSIS_RESULTS + value: gs://ossf-malware-static-analysis-results-v1 - name: OSSF_MALWARE_ANALYZED_PACKAGES value: gs://ossf-malware-analysis-analyzed-packages - name: LOGGER_ENV diff --git a/internal/worker/save_data.go b/internal/worker/save_data.go index 31216325..2a5ffef0 100644 --- a/internal/worker/save_data.go +++ b/internal/worker/save_data.go @@ -18,10 +18,11 @@ import ( // ResultStores holds ResultStore instances for saving each kind of analysis data. // They can be nil, in which case calling the associated Upload function here is a no-op type ResultStores struct { + AnalyzedPackage *resultstore.ResultStore DynamicAnalysis *resultstore.ResultStore - StaticAnalysis *resultstore.ResultStore + ExecutionLog *resultstore.ResultStore FileWrites *resultstore.ResultStore - AnalyzedPackage *resultstore.ResultStore + StaticAnalysis *resultstore.ResultStore AnalyzedPackageSaved bool } @@ -64,7 +65,7 @@ func SaveDynamicAnalysisData(ctx context.Context, pkg *pkgmanager.Pkg, dest *Res // saveExecutionLog saves the execution log to the dynamic analysis resultstore, only if it is nonempty func saveExecutionLog(ctx context.Context, pkg *pkgmanager.Pkg, dest *ResultStores, data analysisrun.DynamicAnalysisResults) error { - if dest.DynamicAnalysis == nil || len(data.ExecutionLog) == 0 { + if dest.ExecutionLog == nil || len(data.ExecutionLog) == 0 { // nothing to do return nil } @@ -74,7 +75,7 @@ func saveExecutionLog(ctx context.Context, pkg *pkgmanager.Pkg, dest *ResultStor execLogFilename = fmt.Sprintf("execution-log-%s.json", pkg.Version()) } - if err := dest.DynamicAnalysis.SaveDynamicAnalysis(ctx, pkg, data.ExecutionLog, execLogFilename); err != nil { + if err := dest.ExecutionLog.SaveDynamicAnalysis(ctx, pkg, data.ExecutionLog, execLogFilename); err != nil { return fmt.Errorf("failed to save execution log to %s: %w", dest.DynamicAnalysis, err) } diff --git a/scripts/run_analysis.sh b/scripts/run_analysis.sh index 23c8021c..7d67e8f5 100755 --- a/scripts/run_analysis.sh +++ b/scripts/run_analysis.sh @@ -157,7 +157,7 @@ DOCKER_MOUNTS=("-v" "$CONTAINER_MOUNT_DIR:/var/lib/containers" "-v" "$RESULTS_DI ANALYSIS_IMAGE=gcr.io/ossf-malware-analysis/analysis -ANALYSIS_ARGS=("analyze" "-upload" "file:///results/" "-upload-file-write-info" "file:///writeResults/" "-upload-static" "file:///staticResults/" "-upload-analyzed-pkg" "file:///analyzedPackages/") +ANALYSIS_ARGS=("analyze" "-dynamic-bucket" "file:///results/" "-file-writes-bucket" "file:///writeResults/" "-static-bucket" "file:///staticResults/" "-analyzed-pkg-bucket" "file:///analyzedPackages/" "-execution-log-bucket" "file:///results") # Add the remaining command line arguments ANALYSIS_ARGS=("${ANALYSIS_ARGS[@]}" "${args[@]}")