Improve Technology SCA to group issues by files

commands/curation/curationaudit.go

@@ -39,6 +39,7 @@ import (
 	"github.com/jfrog/jfrog-cli-security/commands/audit"
 	"github.com/jfrog/jfrog-cli-security/commands/audit/sca/python"
 	"github.com/jfrog/jfrog-cli-security/formats"
+	"github.com/jfrog/jfrog-cli-security/technologies"
 	"github.com/jfrog/jfrog-cli-security/utils"
 	"github.com/jfrog/jfrog-cli-security/utils/techutils"
 	"github.com/jfrog/jfrog-cli-security/utils/xray"
@@ -373,7 +374,7 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map
 	if err != nil {
 		return err
 	}
-	depTreeResult, err := audit.GetTechDependencyTree(params, serverDetails, tech)
+	depTreeResult, err := technologies.GetTechDependencyTree(params, serverDetails, tech)
 	if err != nil {
 		return err
 	}

resources/deptreemanager.go

@@ -0,0 +1,11 @@
+package resources
+
+import (
+	_ "embed"
+)
+
+//go:embed java/settings.xml
+var SettingsXmlTemplate string
+
+//go:embed java/maven-dep-tree.jar
+var MavenDepTreeJar []byte

resources/java/settings.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.2.0 http://maven.apache.org/xsd/settings-1.2.0.xsd"
+          xmlns="http://maven.apache.org/SETTINGS/1.2.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <servers>
+        <server>
+            <id>artifactory</id>
+            <username>{{.Username}}</username>
+            <password>{{.Password}}</password>
+        </server>
+    </servers>
+    <mirrors>
+        <mirror>
+            <id>artifactory</id>
+            <url>{{.RemoteRepositoryFullPath}}</url>
+            <mirrorOf>*</mirrorOf>
+        </mirror>
+    </mirrors>
+    <profiles>
+        <profile>
+            <repositories>
+                <repository>
+                    <snapshots>
+                        <enabled>true</enabled>
+                    </snapshots>
+                    <id>artifactory</id>
+                    <name>mavenRepo</name>
+                    <url>{{.RemoteRepositoryFullPath}}</url>
+                </repository>
+            </repositories>
+            <id>artifactory</id>
+        </profile>
+    </profiles>
+    <activeProfiles>
+        <activeProfile>artifactory</activeProfile>
+    </activeProfiles>
+</settings>

resources/resources.go

@@ -1,3 +1,7 @@
 package resources
 
+import (
+	_ "embed"
+)
+
 const BaseResourcesUrl = "https://raw.githubusercontent.com/jfrog/jfrog-cli-security/main/resources"

technologies/java/mvn.go

@@ -0,0 +1,31 @@
+package java
+
+import (
+	"fmt"
+
+	"github.com/owenrumney/go-sarif/v2/sarif"
+
+	"github.com/jfrog/jfrog-cli-security/utils/techutils"
+)
+
+const (
+	mavenDepTreeJarFile    = "maven-dep-tree.jar"
+	mavenDepTreeOutputFile = "mavendeptree.out"
+	// Changing this version also requires a change in MAVEN_DEP_TREE_VERSION within buildscripts/download_jars.sh
+	mavenDepTreeVersion = "1.1.1"
+	settingsXmlFile     = "settings.xml"
+)
+
+type MavenTechnologyHandler struct {}
+
+func (handler *MavenTechnologyHandler) GetTechDependencyTree(params techutils.DetectDependencyTreeParams) (techutils.TechnologyDependencyTrees, error) {
+	return techutils.TechnologyDependencyTrees{}, fmt.Errorf("Not implemented")
+}
+
+func (handler *MavenTechnologyHandler) GetTechDependencyLocations(directDependencyName, directDependencyVersion string, descriptorPaths ...string) ([]*sarif.Location, error) {
+	return nil, fmt.Errorf("Not implemented")
+}
+
+func (handler *MavenTechnologyHandler) ChangeTechDependencyVersion(directDependencyName, directDependencyVersion, fixVersion string, descriptorPaths ...string) error {
+	return fmt.Errorf("Not implemented")
+}

technologies/technologies.go

@@ -0,0 +1,249 @@
+package technologies
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/jfrog/build-info-go/utils/pythonutils"
+	"github.com/owenrumney/go-sarif/v2/sarif"
+
+	clientUtils "github.com/jfrog/jfrog-client-go/utils"
+	"github.com/jfrog/jfrog-client-go/utils/errorutils"
+	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
+	"github.com/jfrog/jfrog-client-go/utils/log"
+	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
+
+	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
+
+	"github.com/jfrog/jfrog-cli-security/commands/audit/sca/conan"
+	_go "github.com/jfrog/jfrog-cli-security/commands/audit/sca/go"
+	"github.com/jfrog/jfrog-cli-security/commands/audit/sca/java"
+	"github.com/jfrog/jfrog-cli-security/commands/audit/sca/npm"
+	"github.com/jfrog/jfrog-cli-security/commands/audit/sca/nuget"
+	"github.com/jfrog/jfrog-cli-security/commands/audit/sca/pnpm"
+	"github.com/jfrog/jfrog-cli-security/commands/audit/sca/python"
+	"github.com/jfrog/jfrog-cli-security/commands/audit/sca/yarn"
+
+	javaHandlers "github.com/jfrog/jfrog-cli-security/technologies/java"
+
+	"github.com/jfrog/jfrog-cli-security/utils"
+	"github.com/jfrog/jfrog-cli-security/utils/techutils"
+	"github.com/jfrog/jfrog-cli-security/utils/xray"
+)
+
+var TechnologyHandlers = map[techutils.Technology]techutils.TechnologyHandler{
+	// Java technologies
+	techutils.Maven:  &javaHandlers.MavenTechnologyHandler{},
+}
+
+func GetTechHandler(tech techutils.Technology) (techutils.TechnologyHandler, error) {
+	if handler, ok := TechnologyHandlers[tech]; ok {
+		return handler, nil
+	}
+	return nil, fmt.Errorf("%s is currently not supported", tech.ToFormal())
+}
+
+func GetTechDependencyLocations(tech techutils.Technology, directDependencyName, directDependencyVersion string, descriptorPaths ...string) ([]*sarif.Location, error) {
+	handler, err := GetTechHandler(tech)
+	if err != nil {
+		return nil, err
+	}
+	return handler.GetTechDependencyLocations(directDependencyName, directDependencyVersion, descriptorPaths...)
+}
+
+func ChangeTechDependencyVersion(tech techutils.Technology, directDependencyName, directDependencyVersion, fixVersion string, descriptorPaths ...string) error {
+	handler, err := GetTechHandler(tech)
+	if err != nil {
+		return err
+	}
+	return handler.ChangeTechDependencyVersion(directDependencyName, directDependencyVersion, fixVersion, descriptorPaths...)
+}
+
+func GetDependencyTree(params techutils.DetectDependencyTreeParams) (techutils.TechnologyDependencyTrees, error) {
+	msg := fmt.Sprintf("Calculating %s dependencies", params.Technology.ToFormal())
+	if params.IncludeCuration {
+		if logExtraMsg, cacheDir, err := getCurationCacheFolderAndLogMsg(params.Technology); err != nil {
+			return techutils.TechnologyDependencyTrees{}, err
+		} else {
+			params.CurationCacheFolder = cacheDir
+			msg += logExtraMsg
+		}
+	}
+	log.Info(msg)
+	if handler, err := GetTechHandler(params.Technology); err == nil {
+		if tree, err := handler.GetTechDependencyTree(params); err == nil {
+			return tree, nil
+		}
+		return techutils.TechnologyDependencyTrees{}, fmt.Errorf("Handler failed to calculate %s dependencies", params.Technology.ToFormal())
+	}
+	return runFallback(params)
+}
+
+func runFallback(params techutils.DetectDependencyTreeParams) (techutils.TechnologyDependencyTrees, error) {
+	log.Info(fmt.Sprintf("Calculating %s dependencies...", params.Technology.ToFormal()))
+	auditParams := toAuditParams(params)
+	oldTreeStruct, err := GetTechDependencyTree(auditParams, params.ServerDetails, params.Technology)
+	if err != nil {
+		return techutils.TechnologyDependencyTrees{}, err
+	}
+	return toResultNewStruct(oldTreeStruct), nil
+}
+
+func toAuditParams(params techutils.DetectDependencyTreeParams) utils.AuditParams {
+	auditParams := &utils.AuditBasicParams{}
+	auditParams.SetServerDetails(params.ServerDetails)
+	return auditParams
+}
+
+func toResultNewStruct(oldTreeStruct DependencyTreeResult) techutils.TechnologyDependencyTrees {
+	uniqueDeps := make([]string, 0, len(oldTreeStruct.FlatTree.Nodes))
+	for _, node := range oldTreeStruct.FlatTree.Nodes {
+		uniqueDeps = append(uniqueDeps, node.Id)
+	}
+	tree := map[string]*xrayUtils.GraphNode{}
+	for _, node := range oldTreeStruct.FullDepTrees {
+		tree["root"] = node
+	}
+	return techutils.TechnologyDependencyTrees{DownloadUrls: oldTreeStruct.DownloadUrls, UniqueDependencies: uniqueDeps, DependencyTrees: tree}
+}
+
+type DependencyTreeResult struct {
+	FlatTree     *xrayUtils.GraphNode
+	FullDepTrees []*xrayUtils.GraphNode
+	DownloadUrls map[string]string
+}
+
+func GetTechDependencyTree(params utils.AuditParams, artifactoryServerDetails *config.ServerDetails, tech techutils.Technology) (depTreeResult DependencyTreeResult, err error) {
+	logMessage := fmt.Sprintf("Calculating %s dependencies", tech.ToFormal())
+	curationLogMsg, curationCacheFolder, err := getCurationCacheFolderAndLogMsg(tech)
+	if err != nil {
+		return
+	}
+	// In case it's not curation command these 'curationLogMsg' be empty
+	logMessage += curationLogMsg
+	log.Info(logMessage + "...")
+	if params.Progress() != nil {
+		params.Progress().SetHeadlineMsg(logMessage)
+	}
+
+	var uniqueDeps []string
+	var uniqDepsWithTypes map[string]*xray.DepTreeNode
+	startTime := time.Now()
+
+	switch tech {
+	case techutils.Maven, techutils.Gradle:
+		depTreeResult.FullDepTrees, uniqDepsWithTypes, err = java.BuildDependencyTree(java.DepTreeParams{
+			Server:                  artifactoryServerDetails,
+			DepsRepo:                params.DepsRepo(),
+			IsMavenDepTreeInstalled: params.IsMavenDepTreeInstalled(),
+			UseWrapper:              params.UseWrapper(),
+			IsCurationCmd:           params.IsCurationCmd(),
+			CurationCacheFolder:     curationCacheFolder,
+		}, tech)
+	case techutils.Npm:
+		depTreeResult.FullDepTrees, uniqueDeps, err = npm.BuildDependencyTree(params)
+	case techutils.Pnpm:
+		depTreeResult.FullDepTrees, uniqueDeps, err = pnpm.BuildDependencyTree(params)
+	case techutils.Conan:
+		depTreeResult.FullDepTrees, uniqueDeps, err = conan.BuildDependencyTree(params)
+	case techutils.Yarn:
+		depTreeResult.FullDepTrees, uniqueDeps, err = yarn.BuildDependencyTree(params)
+	case techutils.Go:
+		depTreeResult.FullDepTrees, uniqueDeps, err = _go.BuildDependencyTree(params)
+	case techutils.Pipenv, techutils.Pip, techutils.Poetry:
+		depTreeResult.FullDepTrees, uniqueDeps,
+			depTreeResult.DownloadUrls, err = python.BuildDependencyTree(&python.AuditPython{
+			Server:              artifactoryServerDetails,
+			Tool:                pythonutils.PythonTool(tech),
+			RemotePypiRepo:      params.DepsRepo(),
+			PipRequirementsFile: params.PipRequirementsFile(),
+			InstallCommandArgs:  params.InstallCommandArgs(),
+			IsCurationCmd:       params.IsCurationCmd(),
+		})
+	case techutils.Nuget:
+		depTreeResult.FullDepTrees, uniqueDeps, err = nuget.BuildDependencyTree(params)
+	default:
+		err = errorutils.CheckErrorf("%s is currently not supported", string(tech))
+	}
+	if err != nil || (len(uniqueDeps) == 0 && len(uniqDepsWithTypes) == 0) {
+		return
+	}
+	log.Debug(fmt.Sprintf("Created '%s' dependency tree with %d nodes. Elapsed time: %.1f seconds.", tech.ToFormal(), len(uniqueDeps), time.Since(startTime).Seconds()))
+	if len(uniqDepsWithTypes) > 0 {
+		depTreeResult.FlatTree, err = createFlatTreeWithTypes(uniqDepsWithTypes)
+		return
+	}
+	depTreeResult.FlatTree, err = createFlatTree(uniqueDeps)
+	return
+}
+
+func createFlatTreeWithTypes(uniqueDeps map[string]*xray.DepTreeNode) (*xrayUtils.GraphNode, error) {
+	if err := logDeps(uniqueDeps); err != nil {
+		return nil, err
+	}
+	var uniqueNodes []*xrayUtils.GraphNode
+	for uniqueDep, nodeAttr := range uniqueDeps {
+		node := &xrayUtils.GraphNode{Id: uniqueDep}
+		if nodeAttr != nil {
+			node.Types = nodeAttr.Types
+			node.Classifier = nodeAttr.Classifier
+		}
+		uniqueNodes = append(uniqueNodes, node)
+	}
+	return &xrayUtils.GraphNode{Id: "root", Nodes: uniqueNodes}, nil
+}
+
+func createFlatTree(uniqueDeps []string) (*xrayUtils.GraphNode, error) {
+	if err := logDeps(uniqueDeps); err != nil {
+		return nil, err
+	}
+	uniqueNodes := []*xrayUtils.GraphNode{}
+	for _, uniqueDep := range uniqueDeps {
+		uniqueNodes = append(uniqueNodes, &xrayUtils.GraphNode{Id: uniqueDep})
+	}
+	return &xrayUtils.GraphNode{Id: "root", Nodes: uniqueNodes}, nil
+}
+
+func logDeps(uniqueDeps any) (err error) {
+	if log.GetLogger().GetLogLevel() != log.DEBUG {
+		// Avoid printing and marshaling if not on DEBUG mode.
+		return
+	}
+	jsonList, err := json.Marshal(uniqueDeps)
+	if errorutils.CheckError(err) != nil {
+		return err
+	}
+	log.Debug("Unique dependencies list:\n" + clientUtils.IndentJsonArray(jsonList))
+
+	return
+}
+
+func getCurationCacheFolderAndLogMsg(tech techutils.Technology) (logMessage string, curationCacheFolder string, err error) {
+	if curationCacheFolder, err = getCurationCacheByTech(tech); err != nil || curationCacheFolder == "" {
+		return
+	}
+
+	dirExist, err := fileutils.IsDirExists(curationCacheFolder, false)
+	if err != nil {
+		return
+	}
+
+	if dirExist {
+		if dirIsEmpty, scopErr := fileutils.IsDirEmpty(curationCacheFolder); scopErr != nil || !dirIsEmpty {
+			err = scopErr
+			return
+		}
+	}
+
+	logMessage = ". Quick note: we're running our first scan on the project with curation-audit. Expect this one to take a bit longer. Subsequent scans will be faster. Thanks for your patience"
+
+	return logMessage, curationCacheFolder, err
+}
+
+func getCurationCacheByTech(tech techutils.Technology) (string, error) {
+	if tech == techutils.Maven || tech == techutils.Go {
+		return utils.GetCurationCacheFolderByTech(tech)
+	}
+	return "", nil
+}

technologies/technologies_test.go

@@ -0,0 +1,7 @@
+package technologies
+
+import "testing"
+
+func TestGetDependencyTree(t *testing.T) {
+	t.Log("TestGetDependencyTree")
+}