CM-40909 - Implement Go restore support for SCA

cycode/cli/files_collector/sca/base_restore_dependencies.py

@@ -13,9 +13,10 @@
     return join_paths(get_file_dir(path), generated_file_name)
 
 
-def execute_command(command: List[str], file_name: str, command_timeout: int) -> Optional[str]:
+def execute_command(command: List[str], file_name: str, command_timeout: int, dependencies_file_name: str = None) -> \
+        Optional[str]:
     try:
-        dependencies = shell(command, command_timeout)
+        dependencies = shell(command=command, timeout=command_timeout, output_file_path=dependencies_file_name)
     except Exception as e:
         logger.debug('Failed to restore dependencies via shell command, %s', {'filename': file_name}, exc_info=e)
         return None
@@ -24,10 +25,12 @@
 
 
 class BaseRestoreDependencies(ABC):
-    def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None:
+    def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int,
+                 create_output_file_manually: bool) -> None:
         self.context = context
         self.is_git_diff = is_git_diff
         self.command_timeout = command_timeout
+        self.create_output_file_manually = create_output_file_manually
 
     def restore(self, document: Document) -> Optional[Document]:
         return self.try_restore_dependencies(document)
@@ -46,9 +49,11 @@
         if self.verify_restore_file_already_exist(restore_file_path):
             restore_file_content = get_file_content(restore_file_path)
         else:
-            restore_file_content = execute_command(
-                self.get_command(manifest_file_path), manifest_file_path, self.command_timeout
+            output_file_path = restore_file_path if self.create_output_file_manually else None
+            execute_command(
+                self.get_command(manifest_file_path), manifest_file_path, self.command_timeout, output_file_path
             )
+            restore_file_content = get_file_content(restore_file_path)
 
         return Document(restore_file_path, restore_file_content, self.is_git_diff)
 

cycode/cli/files_collector/sca/go/restore_go_dependencies.py

@@ -0,0 +1,31 @@
+import os
+from typing import List
+
+import click
+
+from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies
+from cycode.cli.models import Document
+
+GO_PROJECT_FILE_EXTENSIONS = ['.mod']
+GO_RESTORE_FILE_NAME = 'go.sum'
+BUILD_GO_FILE_NAME = 'go.mod'
+
+
+class RestoreGoDependencies(BaseRestoreDependencies):
+    def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None:
+        super().__init__(context, is_git_diff, command_timeout, True)
+
+    def is_project(self, document: Document) -> bool:
+        return any(document.path.endswith(ext) for ext in GO_PROJECT_FILE_EXTENSIONS)
+
+    def get_command(self, manifest_file_path: str) -> List[str]:
+        return ['cd', self.prepare_tree_file_path_for_command(manifest_file_path), '&&', 'go', 'list', '-m', '-json']
+
+    def get_lock_file_name(self) -> str:
+        return GO_RESTORE_FILE_NAME
+
+    def verify_restore_file_already_exist(self, restore_file_path: str) -> bool:
+        return os.path.isfile(restore_file_path)
+
+    def prepare_tree_file_path_for_command(self, manifest_file_path: str) -> str:
+        return manifest_file_path.replace('/' + BUILD_GO_FILE_NAME, '')

cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py

@@ -9,11 +9,12 @@
 BUILD_GRADLE_FILE_NAME = 'build.gradle'
 BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts'
 BUILD_GRADLE_DEP_TREE_FILE_NAME = 'gradle-dependencies-generated.txt'
+OUTPUT_FILE_MANUALLY = True
 
 
 class RestoreGradleDependencies(BaseRestoreDependencies):
     def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None:
-        super().__init__(context, is_git_diff, command_timeout)
+        super().__init__(context, is_git_diff, command_timeout, OUTPUT_FILE_MANUALLY)
 
     def is_project(self, document: Document) -> bool:
         return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME)
@@ -26,3 +27,6 @@ def get_lock_file_name(self) -> str:
 
     def verify_restore_file_already_exist(self, restore_file_path: str) -> bool:
         return os.path.isfile(restore_file_path)
+
+    def prepare_tree_file_path_for_command(self, manifest_file_path: str) -> str:
+        return '/' + manifest_file_path.strip('/' + BUILD_GRADLE_FILE_NAME) + '/' + BUILD_GRADLE_DEP_TREE_FILE_NAME

cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py

@@ -1,3 +1,4 @@
+import os
 from os import path
 from typing import List, Optional
 
@@ -14,11 +15,12 @@
 BUILD_MAVEN_FILE_NAME = 'pom.xml'
 MAVEN_CYCLONE_DEP_TREE_FILE_NAME = 'bom.json'
 MAVEN_DEP_TREE_FILE_NAME = 'bcde.mvndeps'
+OUTPUT_FILE_MANUALLY = False
 
 
 class RestoreMavenDependencies(BaseRestoreDependencies):
     def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None:
-        super().__init__(context, is_git_diff, command_timeout)
+        super().__init__(context, is_git_diff, command_timeout, False)
 
     def is_project(self, document: Document) -> bool:
         return path.basename(document.path).split('/')[-1] == BUILD_MAVEN_FILE_NAME
@@ -30,7 +32,7 @@ def get_lock_file_name(self) -> str:
         return join_paths('target', MAVEN_CYCLONE_DEP_TREE_FILE_NAME)
 
     def verify_restore_file_already_exist(self, restore_file_path: str) -> bool:
-        return False
+        return os.path.isfile(restore_file_path)
 
     def try_restore_dependencies(self, document: Document) -> Optional[Document]:
         restore_dependencies_document = super().try_restore_dependencies(document)
@@ -47,7 +49,7 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]:
         return restore_dependencies_document
 
     def restore_from_secondary_command(
-        self, document: Document, manifest_file_path: str, restore_dependencies_document: Optional[Document]
+            self, document: Document, manifest_file_path: str, restore_dependencies_document: Optional[Document]
     ) -> Optional[Document]:
         # TODO(MarshalX): does it even work? Ignored restore_dependencies_document arg
         secondary_restore_command = create_secondary_restore_command(manifest_file_path)

cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py

@@ -0,0 +1,33 @@
+import os
+from typing import List
+
+import click
+
+from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies
+from cycode.cli.models import Document
+
+NPM_PROJECT_FILE_EXTENSIONS = ['.json']
+NPM_LOCK_FILE_NAME = 'package-lock.json'
+NPM_MANIFEST_FILE_NAME = 'package.json'
+OUTPUT_FILE_MANUALLY = False
+
+
+class RestoreNpmDependencies(BaseRestoreDependencies):
+    def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None:
+        super().__init__(context, is_git_diff, command_timeout, OUTPUT_FILE_MANUALLY)
+
+    def is_project(self, document: Document) -> bool:
+        return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS)
+
+    def get_command(self, manifest_file_path: str) -> List[str]:
+        return ['npm', 'install', '--prefix', self.prepare_manifest_file_path_for_command(manifest_file_path),
+                '--package-lock-only', '--ignore-scripts', '--no-audit']
+
+    def get_lock_file_name(self) -> str:
+        return NPM_LOCK_FILE_NAME
+
+    def verify_restore_file_already_exist(self, restore_file_path: str) -> bool:
+        return os.path.isfile(restore_file_path)
+
+    def prepare_manifest_file_path_for_command(self, manifest_file_path: str) -> str:
+        return '/' + manifest_file_path.strip('/' + NPM_MANIFEST_FILE_NAME)

cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py

@@ -8,11 +8,12 @@
 
 NUGET_PROJECT_FILE_EXTENSIONS = ['.csproj', '.vbproj']
 NUGET_LOCK_FILE_NAME = 'packages.lock.json'
+OUTPUT_FILE_MANUALLY = False
 
 
 class RestoreNugetDependencies(BaseRestoreDependencies):
     def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None:
-        super().__init__(context, is_git_diff, command_timeout)
+        super().__init__(context, is_git_diff, command_timeout, OUTPUT_FILE_MANUALLY)
 
     def is_project(self, document: Document) -> bool:
         return any(document.path.endswith(ext) for ext in NUGET_PROJECT_FILE_EXTENSIONS)

cycode/cli/files_collector/sca/sca_code_scanner.py

@@ -5,8 +5,10 @@
 
 from cycode.cli import consts
 from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies
+from cycode.cli.files_collector.sca.go.restore_go_dependencies import RestoreGoDependencies
 from cycode.cli.files_collector.sca.maven.restore_gradle_dependencies import RestoreGradleDependencies
 from cycode.cli.files_collector.sca.maven.restore_maven_dependencies import RestoreMavenDependencies
+from cycode.cli.files_collector.sca.npm.restore_npm_dependencies import RestoreNpmDependencies
 from cycode.cli.files_collector.sca.nuget.restore_nuget_dependencies import RestoreNugetDependencies
 from cycode.cli.models import Document
 from cycode.cli.utils.git_proxy import git_proxy
@@ -18,30 +20,31 @@
 
 BUILD_GRADLE_DEP_TREE_TIMEOUT = 180
 BUILD_NUGET_DEP_TREE_TIMEOUT = 180
-
+BUILD_NPM_DEP_TREE_TIMEOUT = 180
+BUILD_GO_DEP_TREE_TIMEOUT = 180
 
 def perform_pre_commit_range_scan_actions(
-    path: str,
-    from_commit_documents: List[Document],
-    from_commit_rev: str,
-    to_commit_documents: List[Document],
-    to_commit_rev: str,
+        path: str,
+        from_commit_documents: List[Document],
+        from_commit_rev: str,
+        to_commit_documents: List[Document],
+        to_commit_rev: str,
 ) -> None:
     repo = git_proxy.get_repo(path)
     add_ecosystem_related_files_if_exists(from_commit_documents, repo, from_commit_rev)
     add_ecosystem_related_files_if_exists(to_commit_documents, repo, to_commit_rev)
 
 
 def perform_pre_hook_range_scan_actions(
-    git_head_documents: List[Document], pre_committed_documents: List[Document]
+        git_head_documents: List[Document], pre_committed_documents: List[Document]
 ) -> None:
     repo = git_proxy.get_repo(os.getcwd())
     add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV)
     add_ecosystem_related_files_if_exists(pre_committed_documents)
 
 
 def add_ecosystem_related_files_if_exists(
-    documents: List[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None
+        documents: List[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None
 ) -> None:
     documents_to_add: List[Document] = []
     for doc in documents:
@@ -56,7 +59,7 @@ def add_ecosystem_related_files_if_exists(
 
 
 def get_doc_ecosystem_related_project_files(
-    doc: Document, documents: List[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo']
+        doc: Document, documents: List[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo']
 ) -> List[Document]:
     documents_to_add: List[Document] = []
     for ecosystem_project_file in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem):
@@ -86,10 +89,10 @@ def get_project_file_ecosystem(document: Document) -> Optional[str]:
 
 
 def try_restore_dependencies(
-    context: click.Context,
-    documents_to_add: Dict[str, Document],
-    restore_dependencies: 'BaseRestoreDependencies',
-    document: Document,
+        context: click.Context,
+        documents_to_add: Dict[str, Document],
+        restore_dependencies: 'BaseRestoreDependencies',
+        document: Document,
 ) -> None:
     if restore_dependencies.is_project(document):
         restore_dependencies_document = restore_dependencies.restore(document)
@@ -115,7 +118,7 @@ def try_restore_dependencies(
 
 
 def add_dependencies_tree_document(
-    context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False
+        context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False
 ) -> None:
     documents_to_add: Dict[str, Document] = {}
     restore_dependencies_list = restore_handlers(context, is_git_diff)
@@ -132,6 +135,8 @@ def restore_handlers(context: click.Context, is_git_diff: bool) -> List[BaseRest
         RestoreGradleDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT),
         RestoreMavenDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT),
         RestoreNugetDependencies(context, is_git_diff, BUILD_NUGET_DEP_TREE_TIMEOUT),
+        RestoreNpmDependencies(context, is_git_diff, BUILD_NPM_DEP_TREE_TIMEOUT),
+        RestoreGoDependencies(context, is_git_diff, BUILD_GO_DEP_TREE_TIMEOUT)
     ]
 
 
@@ -147,7 +152,7 @@ def get_file_content_from_commit(repo: 'Repo', commit: str, file_path: str) -> O
 
 
 def perform_pre_scan_documents_actions(
-    context: click.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False
+        context: click.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False
 ) -> None:
     if scan_type == consts.SCA_SCAN_TYPE and not context.obj.get(consts.SCA_SKIP_RESTORE_DEPENDENCIES_FLAG):
         logger.debug('Perform pre-scan document add_dependencies_tree_document action')

cycode/cli/utils/shell_executor.py

@@ -9,19 +9,26 @@
 
 
 def shell(
-    command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, execute_in_shell: bool = False
+        command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC,
+        output_file_path: Optional[str] = None
 ) -> Optional[str]:
     logger.debug('Executing shell command: %s', command)
 
     try:
         result = subprocess.run(  # noqa: S603
             command,
             timeout=timeout,
-            shell=execute_in_shell,
+            shell=False,
             check=True,
             capture_output=True,
+            text=True,
         )
 
+        # Write stdout output to the file if output_file_path is provided
+        if output_file_path:
+            with open(output_file_path, 'w') as output_file:
+                output_file.write(result.stdout)
+
         return result.stdout.decode('UTF-8').strip()
     except subprocess.CalledProcessError as e:
         logger.debug('Error occurred while running shell command', exc_info=e)