CM-22718 - Support scan in batches with progress bar (#129)

Author: MarshalX

cycode/cli/auth/auth_command.py

@@ -48,14 +48,14 @@ def authorization_check(context: click.Context):
             return printer.print_result(passed_auth_check_res)
     except (NetworkError, HttpUnauthorizedError):
         if context.obj['verbose']:
-            click.secho(f'Error: {traceback.format_exc()}', fg='red', nl=False)
+            click.secho(f'Error: {traceback.format_exc()}', fg='red')
 
         return printer.print_result(failed_auth_check_res)
 
 
 def _handle_exception(context: click.Context, e: Exception):
     if context.obj['verbose']:
-        click.secho(f'Error: {traceback.format_exc()}', fg='red', nl=False)
+        click.secho(f'Error: {traceback.format_exc()}', fg='red')
 
     errors: CliErrors = {
         AuthProcessError: CliError(

cycode/cli/code_scanner.py

@@ -84,6 +84,13 @@
 # 200MB in bytes (in binary)
 SCA_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 209715200
 
+# scan in batches
+SCAN_BATCH_MAX_SIZE_IN_BYTES = 9 * 1024 * 1024
+SCAN_BATCH_MAX_FILES_COUNT = 1000
+# if we increase this values, the server doesn't allow connecting (ConnectionError)
+SCAN_BATCH_MAX_PARALLEL_SCANS = 5
+SCAN_BATCH_SCANS_PER_CPU = 1
+
 # scan with polling
 SCAN_POLLING_WAIT_INTERVAL_IN_SECONDS = 5
 DEFAULT_SCAN_POLLING_TIMEOUT_IN_SECONDS = 3600
@@ -93,8 +100,6 @@
 DEFAULT_SCA_PRE_COMMIT_TIMEOUT_IN_SECONDS = 600
 SCA_PRE_COMMIT_TIMEOUT_IN_SECONDS_ENV_VAR_NAME = 'SCA_PRE_COMMIT_TIMEOUT_IN_SECONDS'
 
-PROGRESS_UPDATE_COMMITS_INTERVAL = 100
-
 # pre receive scan
 PRE_RECEIVE_MAX_COMMITS_TO_SCAN_COUNT_ENV_VAR_NAME = 'PRE_RECEIVE_MAX_COMMITS_TO_SCAN_COUNT'
 DEFAULT_PRE_RECEIVE_MAX_COMMITS_TO_SCAN_COUNT = 50
@@ -124,6 +129,9 @@
 SKIP_SCAN_FLAG = 'skip-cycode-scan'
 VERBOSE_SCAN_FLAG = 'verbose'
 
+ISSUE_DETECTED_STATUS_CODE = 1
+NO_ISSUES_STATUS_CODE = 0
+
 LICENSE_COMPLIANCE_POLICY_ID = '8f681450-49e1-4f7e-85b7-0c8fe84b3a35'
 PACKAGE_VULNERABILITY_POLICY_ID = '9369d10a-9ac0-48d3-9921-5de7fe9a37a7'
 

cycode/cli/consts.py

@@ -3,9 +3,10 @@
 import click
 import sys
 
-from typing import List, Optional
+from typing import List, Optional, TYPE_CHECKING
 
 from cycode import __version__
+from cycode.cli.consts import NO_ISSUES_STATUS_CODE, ISSUE_DETECTED_STATUS_CODE
 from cycode.cli.models import Severity
 from cycode.cli.config import config
 from cycode.cli import code_scanner
@@ -14,14 +15,17 @@
 from cycode.cli.user_settings.user_settings_commands import set_credentials, add_exclusions
 from cycode.cli.auth.auth_command import authenticate
 from cycode.cli.utils import scan_utils
+from cycode.cli.utils.progress_bar import get_progress_bar
 from cycode.cyclient import logger
+from cycode.cli.utils.progress_bar import logger as progress_bar_logger
 from cycode.cyclient.cycode_client_base import CycodeClientBase
 from cycode.cyclient.models import UserAgentOptionScheme
 from cycode.cyclient.scan_config.scan_config_creator import create_scan_client
 
+if TYPE_CHECKING:
+    from cycode.cyclient.scan_client import ScanClient
+
 CONTEXT = dict()
-ISSUE_DETECTED_STATUS_CODE = 1
-NO_ISSUES_STATUS_CODE = 0
 
 
 @click.group(
@@ -106,25 +110,41 @@ def code_scan(context: click.Context, scan_type, client_id, secret, show_secret,
         context.obj["soft_fail"] = config["soft_fail"]
 
     context.obj["scan_type"] = scan_type
+
+    # save backward compatability with old style command
     if output is not None:
-        # save backward compatability with old style command
         context.obj["output"] = output
+        if output == "json":
+            context.obj["no_progress_meter"] = True
+
     context.obj["client"] = get_cycode_client(client_id, secret)
     context.obj["severity_threshold"] = severity_threshold
     context.obj["monitor"] = monitor
     context.obj["report"] = report
+
     _sca_scan_to_context(context, sca_scan)
 
+    context.obj["progress_bar"] = get_progress_bar(hidden=context.obj["no_progress_meter"])
+    context.obj["progress_bar"].start()
+
     return 1
 
 
 @code_scan.result_callback()
 @click.pass_context
-def finalize(context: click.Context, *args, **kwargs):
-    if context.obj["soft_fail"]:
+def finalize(context: click.Context, *_, **__):
+    progress_bar = context.obj.get('progress_bar')
+    if progress_bar:
+        progress_bar.stop()
+
+    if context.obj['soft_fail']:
         sys.exit(0)
 
-    sys.exit(ISSUE_DETECTED_STATUS_CODE if _should_fail_scan(context) else NO_ISSUES_STATUS_CODE)
+    exit_code = NO_ISSUES_STATUS_CODE
+    if _should_fail_scan(context):
+        exit_code = ISSUE_DETECTED_STATUS_CODE
+
+    sys.exit(exit_code)
 
 
 @click.group(
@@ -139,6 +159,9 @@ def finalize(context: click.Context, *args, **kwargs):
 @click.option(
     "--verbose", "-v", is_flag=True, default=False, help="Show detailed logs",
 )
+@click.option(
+    '--no-progress-meter', is_flag=True, default=False, help='Do not show the progress meter',
+)
 @click.option(
     '--output',
     default='text',
@@ -153,23 +176,29 @@ def finalize(context: click.Context, *args, **kwargs):
 )
 @click.version_option(__version__, prog_name="cycode")
 @click.pass_context
-def main_cli(context: click.Context, verbose: bool, output: str, user_agent: Optional[str]):
+def main_cli(context: click.Context, verbose: bool, no_progress_meter: bool, output: str, user_agent: Optional[str]):
     context.ensure_object(dict)
     configuration_manager = ConfigurationManager()
 
     verbose = verbose or configuration_manager.get_verbose_flag()
     context.obj['verbose'] = verbose
+    # TODO(MarshalX): rework setting the log level for loggers
     log_level = logging.DEBUG if verbose else logging.INFO
     logger.setLevel(log_level)
+    progress_bar_logger.setLevel(log_level)
 
     context.obj['output'] = output
+    if output == 'json':
+        no_progress_meter = True
+
+    context.obj['no_progress_meter'] = no_progress_meter
 
     if user_agent:
         user_agent_option = UserAgentOptionScheme().loads(user_agent)
         CycodeClientBase.enrich_user_agent(user_agent_option.user_agent_suffix)
 
 
-def get_cycode_client(client_id, client_secret):
+def get_cycode_client(client_id: str, client_secret: str) -> 'ScanClient':
     if not client_id or not client_secret:
         client_id, client_secret = _get_configured_credentials()
         if not client_id:

cycode/cli/main.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import List, NamedTuple, Dict, Type
+from typing import List, NamedTuple, Dict, Type, Optional
 
 from cycode.cyclient.models import Detection
 
@@ -58,3 +58,12 @@ class CliError(NamedTuple):
 class CliResult(NamedTuple):
     success: bool
     message: str
+
+
+class LocalScanResult(NamedTuple):
+    scan_id: str
+    report_url: Optional[str]
+    document_detections: List[DocumentDetections]
+    issue_detected: bool
+    detections_count: int
+    relevant_detections_count: int

cycode/cli/models.py

@@ -1,9 +1,12 @@
 from abc import ABC, abstractmethod
-from typing import List
+from typing import List, TYPE_CHECKING
 
 import click
 
-from cycode.cli.models import DocumentDetections, CliResult, CliError
+from cycode.cli.models import CliResult, CliError
+
+if TYPE_CHECKING:
+    from cycode.cli.models import LocalScanResult
 
 
 class BasePrinter(ABC):
@@ -15,7 +18,7 @@ def __init__(self, context: click.Context):
         self.context = context
 
     @abstractmethod
-    def print_scan_results(self, results: List[DocumentDetections]) -> None:
+    def print_scan_results(self, local_scan_results: List['LocalScanResult']) -> None:
         pass
 
     @abstractmethod

cycode/cli/printers/base_printer.py

@@ -1,18 +1,20 @@
 import abc
-from typing import List
+from typing import List, TYPE_CHECKING
 
 import click
 
 from cycode.cli.printers.text_printer import TextPrinter
-from cycode.cli.models import DocumentDetections, CliError, CliResult
+from cycode.cli.models import CliError, CliResult
 from cycode.cli.printers.base_printer import BasePrinter
 
+if TYPE_CHECKING:
+    from cycode.cli.models import LocalScanResult
+
 
 class BaseTablePrinter(BasePrinter, abc.ABC):
     def __init__(self, context: click.Context):
         super().__init__(context)
         self.context = context
-        self.scan_id: str = context.obj.get('scan_id')
         self.scan_type: str = context.obj.get('scan_type')
         self.show_secret: bool = context.obj.get('show_secret', False)
 
@@ -22,22 +24,16 @@ def print_result(self, result: CliResult) -> None:
     def print_error(self, error: CliError) -> None:
         TextPrinter(self.context).print_error(error)
 
-    def print_scan_results(self, results: List[DocumentDetections]):
-        click.secho(f'Scan Results: (scan_id: {self.scan_id})')
-
-        if not results:
+    def print_scan_results(self, local_scan_results: List['LocalScanResult']):
+        if all(result.issue_detected == 0 for result in local_scan_results):
             click.secho('Good job! No issues were found!!! 👏👏👏', fg=self.GREEN_COLOR_NAME)
             return
 
-        self._print_results(results)
-
-        report_url = self.context.obj.get('report_url')
-        if report_url:
-            click.secho(f'Report URL: {report_url}')
+        self._print_results(local_scan_results)
 
     def _is_git_repository(self) -> bool:
         return self.context.obj.get('remote_url') is not None
 
     @abc.abstractmethod
-    def _print_results(self, results: List[DocumentDetections]) -> None:
+    def _print_results(self, local_scan_results: List['LocalScanResult']) -> None:
         raise NotImplementedError

cycode/cli/printers/base_table_printer.py

@@ -2,13 +2,14 @@
 from typing import List, TYPE_CHECKING
 
 from cycode.cli.exceptions.custom_exceptions import CycodeError
-from cycode.cli.models import DocumentDetections, CliResult, CliError
+from cycode.cli.models import CliResult, CliError
 from cycode.cli.printers.table_printer import TablePrinter
 from cycode.cli.printers.sca_table_printer import SCATablePrinter
 from cycode.cli.printers.json_printer import JsonPrinter
 from cycode.cli.printers.text_printer import TextPrinter
 
 if TYPE_CHECKING:
+    from cycode.cli.models import LocalScanResult
     from cycode.cli.printers.base_printer import BasePrinter
 
 
@@ -31,9 +32,9 @@ def __init__(self, context: click.Context):
         if self._printer_class is None:
             raise CycodeError(f'"{self.output_type}" output type is not supported.')
 
-    def print_scan_results(self, detections_results_list: List[DocumentDetections]) -> None:
+    def print_scan_results(self, local_scan_results: List['LocalScanResult']) -> None:
         printer = self._get_scan_printer()
-        printer.print_scan_results(detections_results_list)
+        printer.print_scan_results(local_scan_results)
 
     def _get_scan_printer(self) -> 'BasePrinter':
         printer_class = self._printer_class

cycode/cli/printers/console_printer.py

@@ -1,18 +1,17 @@
 import json
-from typing import List
+from typing import List, TYPE_CHECKING
 
 import click
 
-from cycode.cli.models import DocumentDetections, CliResult, CliError
+from cycode.cli.models import CliResult, CliError
 from cycode.cli.printers.base_printer import BasePrinter
 from cycode.cyclient.models import DetectionSchema
 
+if TYPE_CHECKING:
+    from cycode.cli.models import LocalScanResult
 
-class JsonPrinter(BasePrinter):
-    def __init__(self, context: click.Context):
-        super().__init__(context)
-        self.scan_id = context.obj.get('scan_id')
 
+class JsonPrinter(BasePrinter):
     def print_result(self, result: CliResult) -> None:
         result = {
             'result': result.success,
@@ -29,18 +28,19 @@ def print_error(self, error: CliError) -> None:
 
         click.secho(self.get_data_json(result))
 
-    def print_scan_results(self, results: List[DocumentDetections]) -> None:
+    def print_scan_results(self, local_scan_results: List['LocalScanResult']) -> None:
         detections = []
-        for result in results:
-            detections.extend(result.detections)
+        for local_scan_result in local_scan_results:
+            for document_detections in local_scan_result.document_detections:
+                detections.extend(document_detections.detections)
 
         detections_dict = DetectionSchema(many=True).dump(detections)
 
         click.secho(self._get_json_scan_result(detections_dict))
 
     def _get_json_scan_result(self, detections: dict) -> str:
         result = {
-            'scan_id': str(self.scan_id),
+            'scan_id': 'DEPRECATED',    # FIXME(MarshalX): we need change JSON struct to support multiple scan results
             'detections': detections
         }