add separate result bucket for execution log (ossf#950)

* add separate result bucket for execution log Signed-off-by: Max Fisher <maxfisher@google.com> * update e2e test and production configs with the new bucket name Signed-off-by: Max Fisher <maxfisher@google.com> --------- Signed-off-by: Max Fisher <maxfisher@google.com>
another-rex · Nov 2, 2023 · 5433e2a · 5433e2a
1 parent ee301c1
commit 5433e2a
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 48 deletions.
diff --git a/Makefile b/Makefile
@@ -129,11 +129,16 @@ run:
 
 E2E_TEST_COMPOSE_ARGS := -p pa-e2e-testing -f ./configs/e2e/docker-compose.yml -f ./test/e2e/docker-compose.test.yml
 
+.PHONY: e2e_test_build
+e2e_test_build: build_e2e_test_images
+
 .PHONY: e2e_test_start
-e2e_test_start: build_e2e_test_images
+e2e_test_start:
 	docker-compose $(E2E_TEST_COMPOSE_ARGS) up -d
 	@echo
 	@echo "To see analysis results, go to http://localhost:9000/minio/package-analysis"
+	@echo "Username: minio"
+	@echo "Password: minio123"
 	@echo
 	@echo "Remember to run 'make e2e_test_stop' when done!"
 	@sleep 5

diff --git a/cmd/analyze/main.go b/cmd/analyze/main.go
@@ -24,42 +24,45 @@ import (
 )
 
 var (
-	pkgName             = flag.String("package", "", "package name")
-	localPkg            = flag.String("local", "", "local package path")
-	ecosystem           pkgecosystem.Ecosystem
-	version             = flag.String("version", "", "version")
-	noPull              = flag.Bool("nopull", false, "disables pulling down sandbox images")
-	imageTag            = flag.String("image-tag", "", "set image tag for analysis sandboxes")
-	dynamicUpload       = flag.String("upload", "", "bucket path for uploading dynamic analysis results")
-	staticUpload        = flag.String("upload-static", "", "bucket path for uploading static analysis results")
-	uploadFileWriteInfo = flag.String("upload-file-write-info", "", "bucket path for uploading information from file writes")
-	uploadAnalyzedPkg   = flag.String("upload-analyzed-pkg", "", "bucket path for uploading analyzed packages")
-	offline             = flag.Bool("offline", false, "disables sandbox network access")
-	customSandbox       = flag.String("sandbox-image", "", "override default dynamic analysis sandbox with custom image")
-	customAnalysisCmd   = flag.String("analysis-command", "", "override default dynamic analysis script path (use with custom sandbox image)")
-	listModes           = flag.Bool("list-modes", false, "prints out a list of available analysis modes")
-	features            = flag.String("features", "", "override features that are enabled/disabled by default")
-	listFeatures        = flag.Bool("list-features", false, "list available features that can be toggled")
-	help                = flag.Bool("help", false, "print help on available options")
-	analysisMode        = utils.CommaSeparatedFlags("mode", []string{"static", "dynamic"},
+	pkgName            = flag.String("package", "", "package name")
+	localPkg           = flag.String("local", "", "local package path")
+	ecosystem          pkgecosystem.Ecosystem
+	version            = flag.String("version", "", "version")
+	noPull             = flag.Bool("nopull", false, "disables pulling down sandbox images")
+	imageTag           = flag.String("image-tag", "", "set image tag for analysis sandboxes")
+	dynamicBucket      = flag.String("dynamic-bucket", "", "bucket path for uploading dynamic analysis results")
+	staticBucket       = flag.String("static-bucket", "", "bucket path for uploading static analysis results")
+	executionLogBucket = flag.String("execution-log-bucket", "", "bucket path for uploading execution log (dynamic analysis)")
+	fileWritesBucket   = flag.String("file-writes-bucket", "", "bucket path for uploading file writes data (dynamic analysis)")
+	analyzedPkgBucket  = flag.String("analyzed-pkg-bucket", "", "bucket path for uploading analyzed packages")
+	offline            = flag.Bool("offline", false, "disables sandbox network access")
+	customSandbox      = flag.String("sandbox-image", "", "override default dynamic analysis sandbox with custom image")
+	customAnalysisCmd  = flag.String("analysis-command", "", "override default dynamic analysis script path (use with custom sandbox image)")
+	listModes          = flag.Bool("list-modes", false, "prints out a list of available analysis modes")
+	features           = flag.String("features", "", "override features that are enabled/disabled by default")
+	listFeatures       = flag.Bool("list-features", false, "list available features that can be toggled")
+	help               = flag.Bool("help", false, "print help on available options")
+	analysisMode       = utils.CommaSeparatedFlags("mode", []string{"static", "dynamic"},
 		"list of analysis modes to run, separated by commas. Use -list-modes to see available options")
 )
 
 func makeResultStores() worker.ResultStores {
 	rs := worker.ResultStores{}
 
-	if *dynamicUpload != "" {
-		rs.DynamicAnalysis = resultstore.New(*dynamicUpload)
+	if *analyzedPkgBucket != "" {
+		rs.AnalyzedPackage = resultstore.New(*analyzedPkgBucket)
 	}
-	if *staticUpload != "" {
-		rs.StaticAnalysis = resultstore.New(*staticUpload)
+	if *dynamicBucket != "" {
+		rs.DynamicAnalysis = resultstore.New(*dynamicBucket)
 	}
-	if *uploadFileWriteInfo != "" {
-		rs.FileWrites = resultstore.New(*uploadFileWriteInfo)
+	if *executionLogBucket != "" {
+		rs.ExecutionLog = resultstore.New(*executionLogBucket)
 	}
-
-	if *uploadAnalyzedPkg != "" {
-		rs.AnalyzedPackage = resultstore.New(*uploadAnalyzedPkg)
+	if *fileWritesBucket != "" {
+		rs.FileWrites = resultstore.New(*fileWritesBucket)
+	}
+	if *staticBucket != "" {
+		rs.StaticAnalysis = resultstore.New(*staticBucket)
 	}
 
 	return rs

diff --git a/cmd/worker/main.go b/cmd/worker/main.go
@@ -37,10 +37,11 @@ const (
 
 // resultBucketPaths holds bucket paths for the different types of results.
 type resultBucketPaths struct {
+	analyzedPkg     string
 	dynamicAnalysis string
-	staticAnalysis  string
+	executionLog    string
 	fileWrites      string
-	analyzedPkg     string
+	staticAnalysis  string
 }
 
 type sandboxImageSpec struct {
@@ -79,17 +80,20 @@ func copyPackageToLocalFile(ctx context.Context, packagesBucket *blob.Bucket, bu
 func makeResultStores(dest resultBucketPaths) worker.ResultStores {
 	resultStores := worker.ResultStores{}
 
+	if dest.analyzedPkg != "" {
+		resultStores.AnalyzedPackage = resultstore.New(dest.analyzedPkg, resultstore.ConstructPath())
+	}
 	if dest.dynamicAnalysis != "" {
 		resultStores.DynamicAnalysis = resultstore.New(dest.dynamicAnalysis, resultstore.ConstructPath())
 	}
-	if dest.staticAnalysis != "" {
-		resultStores.StaticAnalysis = resultstore.New(dest.staticAnalysis, resultstore.ConstructPath())
+	if dest.executionLog != "" {
+		resultStores.ExecutionLog = resultstore.New(dest.executionLog, resultstore.ConstructPath())
 	}
 	if dest.fileWrites != "" {
 		resultStores.FileWrites = resultstore.New(dest.fileWrites, resultstore.ConstructPath())
 	}
-	if dest.analyzedPkg != "" {
-		resultStores.AnalyzedPackage = resultstore.New(dest.analyzedPkg, resultstore.ConstructPath())
+	if dest.staticAnalysis != "" {
+		resultStores.StaticAnalysis = resultstore.New(dest.staticAnalysis, resultstore.ConstructPath())
 	}
 
 	return resultStores
@@ -274,10 +278,11 @@ func main() {
 	}
 
 	resultsBuckets := resultBucketPaths{
+		analyzedPkg:     os.Getenv("OSSF_MALWARE_ANALYZED_PACKAGES"),
 		dynamicAnalysis: os.Getenv("OSSF_MALWARE_ANALYSIS_RESULTS"),
-		staticAnalysis:  os.Getenv("OSSF_MALWARE_STATIC_ANALYSIS_RESULTS"),
+		executionLog:    os.Getenv("OSSF_MALWARE_ANALYSIS_EXECUTION_LOGS"),
 		fileWrites:      os.Getenv("OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS"),
-		analyzedPkg:     os.Getenv("OSSF_MALWARE_ANALYZED_PACKAGES"),
+		staticAnalysis:  os.Getenv("OSSF_MALWARE_STATIC_ANALYSIS_RESULTS"),
 	}
 	resultStores := makeResultStores(resultsBuckets)
 
@@ -305,6 +310,7 @@ func main() {
 		"static_results_bucket", resultsBuckets.staticAnalysis,
 		"file_write_results_bucket", resultsBuckets.fileWrites,
 		"analyzed_packages_bucket", resultsBuckets.analyzedPkg,
+		"execution_log_bucket", resultsBuckets.executionLog,
 		"image_tag", imageSpec.tag,
 		"image_nopull", imageSpec.noPull,
 		"topic_notification", notificationTopicURL,

diff --git a/configs/e2e/docker-compose.yml b/configs/e2e/docker-compose.yml
@@ -47,7 +47,7 @@ services:
       MINIO_ROOT_PASSWORD: minio123
       MINIO_REGION_NAME: dummy_region
     entrypoint: sh
-    command: -c 'mkdir -p /data/package-analysis{,-static,-file-writes,-analyzed-packages} && /usr/bin/minio server /data'
+    command: -c 'mkdir -p /data/package-analysis/{analyzed-packages,dynamic,execution-logs,file-writes,static} && /usr/bin/minio server /data'
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -64,10 +64,11 @@ services:
     environment:
       OSSMALWARE_WORKER_SUBSCRIPTION: kafka://worker?topic=workers
       OSSF_MALWARE_NOTIFICATION_TOPIC: kafka://notifications
-      OSSF_MALWARE_ANALYSIS_RESULTS: s3://package-analysis?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
-      OSSF_MALWARE_STATIC_ANALYSIS_RESULTS: s3://package-analysis-static?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
-      OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS: s3://package-analysis-file-writes?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
-      OSSF_MALWARE_ANALYZED_PACKAGES: s3://package-analysis-analyzed-packages?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_ANALYZED_PACKAGES: s3://package-analysis/analyzed-packages?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_ANALYSIS_RESULTS: s3://package-analysis/dynamic?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_ANALYSIS_EXECUTION_LOGS: s3://package-analysis/execution-logs?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS: s3://package-analysis/file-writes?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
+      OSSF_MALWARE_STATIC_ANALYSIS_RESULTS: s3://package-analysis/static?endpoint=minio:9000&disableSSL=true&s3ForcePathStyle=true
       OSSF_MALWARE_ANALYSIS_ENABLE_PROFILER: "true"
       OSSF_MALWARE_FEATURE_FLAGS: ""
       KAFKA_BROKERS: kafka:9092

diff --git a/infra/worker/workers-set.yaml b/infra/worker/workers-set.yaml
@@ -25,10 +25,12 @@ spec:
           value: gcppubsub://projects/ossf-malware-analysis/subscriptions/workers?nacklazy=1
         - name: OSSF_MALWARE_ANALYSIS_RESULTS
           value: gs://ossf-malware-analysis-results
-        - name: OSSF_MALWARE_STATIC_ANALYSIS_RESULTS
-          value: gs://ossf-malware-static-analysis-results-v1
+        - name: OSSF_MALWARE_ANALYSIS_EXECUTION_LOGS
+          value: gs://ossf-malware-analysis-execution-logs
         - name: OSSF_MALWARE_ANALYSIS_FILE_WRITE_RESULTS
           value: gs://ossf-malware-analysis-file-write-results
+        - name: OSSF_MALWARE_STATIC_ANALYSIS_RESULTS
+          value: gs://ossf-malware-static-analysis-results-v1
         - name: OSSF_MALWARE_ANALYZED_PACKAGES
           value: gs://ossf-malware-analysis-analyzed-packages
         - name: LOGGER_ENV

diff --git a/internal/worker/save_data.go b/internal/worker/save_data.go
@@ -18,10 +18,11 @@ import (
 // ResultStores holds ResultStore instances for saving each kind of analysis data.
 // They can be nil, in which case calling the associated Upload function here is a no-op
 type ResultStores struct {
+	AnalyzedPackage      *resultstore.ResultStore
 	DynamicAnalysis      *resultstore.ResultStore
-	StaticAnalysis       *resultstore.ResultStore
+	ExecutionLog         *resultstore.ResultStore
 	FileWrites           *resultstore.ResultStore
-	AnalyzedPackage      *resultstore.ResultStore
+	StaticAnalysis       *resultstore.ResultStore
 	AnalyzedPackageSaved bool
 }
 
@@ -64,7 +65,7 @@ func SaveDynamicAnalysisData(ctx context.Context, pkg *pkgmanager.Pkg, dest *Res
 
 // saveExecutionLog saves the execution log to the dynamic analysis resultstore, only if it is nonempty
 func saveExecutionLog(ctx context.Context, pkg *pkgmanager.Pkg, dest *ResultStores, data analysisrun.DynamicAnalysisResults) error {
-	if dest.DynamicAnalysis == nil || len(data.ExecutionLog) == 0 {
+	if dest.ExecutionLog == nil || len(data.ExecutionLog) == 0 {
 		// nothing to do
 		return nil
 	}
@@ -74,7 +75,7 @@ func saveExecutionLog(ctx context.Context, pkg *pkgmanager.Pkg, dest *ResultStor
 		execLogFilename = fmt.Sprintf("execution-log-%s.json", pkg.Version())
 	}
 
-	if err := dest.DynamicAnalysis.SaveDynamicAnalysis(ctx, pkg, data.ExecutionLog, execLogFilename); err != nil {
+	if err := dest.ExecutionLog.SaveDynamicAnalysis(ctx, pkg, data.ExecutionLog, execLogFilename); err != nil {
 		return fmt.Errorf("failed to save execution log to %s: %w", dest.DynamicAnalysis, err)
 	}
 

diff --git a/scripts/run_analysis.sh b/scripts/run_analysis.sh
@@ -157,7 +157,7 @@ DOCKER_MOUNTS=("-v" "$CONTAINER_MOUNT_DIR:/var/lib/containers" "-v" "$RESULTS_DI
 
 ANALYSIS_IMAGE=gcr.io/ossf-malware-analysis/analysis
 
-ANALYSIS_ARGS=("analyze" "-upload" "file:///results/" "-upload-file-write-info" "file:///writeResults/" "-upload-static" "file:///staticResults/" "-upload-analyzed-pkg" "file:///analyzedPackages/")
+ANALYSIS_ARGS=("analyze" "-dynamic-bucket" "file:///results/" "-file-writes-bucket" "file:///writeResults/" "-static-bucket" "file:///staticResults/" "-analyzed-pkg-bucket" "file:///analyzedPackages/" "-execution-log-bucket" "file:///results")
 
 # Add the remaining command line arguments
 ANALYSIS_ARGS=("${ANALYSIS_ARGS[@]}" "${args[@]}")