CM-30564 - Add support for report command in secret scanning

cycode/cli/commands/scan/code_scanner.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import os
 import sys
@@ -99,6 +98,10 @@ def set_issue_detected_by_scan_results(context: click.Context, scan_results: Lis
     set_issue_detected(context, any(scan_result.issue_detected for scan_result in scan_results))
 
 
+def _should_use_scan_service(scan_type: str, scan_parameters: Optional[dict] = None) -> bool:
+    return scan_type == consts.SECRET_SCAN_TYPE and scan_parameters['report'] is True
+
+
 def _enrich_scan_result_with_data_from_detection_rules(
     cycode_client: 'ScanClient', scan_type: str, scan_result: ZippedFileScanResult
 ) -> None:
@@ -148,14 +151,21 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local
 
         scan_id = str(_generate_unique_id())
         scan_completed = False
+        should_use_scan_service = _should_use_scan_service(scan_type, scan_parameters)
 
         try:
             logger.debug('Preparing local files, %s', {'batch_size': len(batch)})
             zipped_documents = zip_documents(scan_type, batch)
             zip_file_size = zipped_documents.size
-
             scan_result = perform_scan(
-                cycode_client, zipped_documents, scan_type, scan_id, is_git_diff, is_commit_range, scan_parameters
+                cycode_client,
+                zipped_documents,
+                scan_type,
+                scan_id,
+                is_git_diff,
+                is_commit_range,
+                scan_parameters,
+                should_use_scan_service,
             )
 
             _enrich_scan_result_with_data_from_detection_rules(cycode_client, scan_type, scan_result)
@@ -194,6 +204,7 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local
             zip_file_size,
             command_scan_type,
             error_message,
+            should_use_scan_service,
         )
 
         return scan_id, error, local_scan_result
@@ -315,14 +326,14 @@ def scan_commit_range_documents(
     local_scan_result = error_message = None
     scan_completed = False
     scan_id = str(_generate_unique_id())
-
+    should_use_scan_service = _should_use_scan_service(scan_type, scan_parameters)
     from_commit_zipped_documents = InMemoryZip()
     to_commit_zipped_documents = InMemoryZip()
 
     try:
         progress_bar.set_section_length(ScanProgressBarSection.SCAN, 1)
 
-        scan_result = init_default_scan_result(scan_id)
+        scan_result = init_default_scan_result(cycode_client, scan_id, scan_type, should_use_scan_service)
         if should_scan_documents(from_documents_to_scan, to_documents_to_scan):
             logger.debug('Preparing from-commit zip')
             from_commit_zipped_documents = zip_documents(scan_type, from_documents_to_scan)
@@ -337,6 +348,7 @@ def scan_commit_range_documents(
                 scan_type,
                 scan_parameters,
                 timeout,
+                should_use_scan_service,
             )
 
         progress_bar.update(ScanProgressBarSection.SCAN)
@@ -428,9 +440,10 @@ def perform_scan(
     is_git_diff: bool,
     is_commit_range: bool,
     scan_parameters: dict,
+    should_use_scan_service: bool = False,
 ) -> ZippedFileScanResult:
-    if scan_type in (consts.SCA_SCAN_TYPE, consts.SAST_SCAN_TYPE):
-        return perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters)
+    if scan_type in (consts.SCA_SCAN_TYPE, consts.SAST_SCAN_TYPE) or should_use_scan_service:
+        return perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters, should_use_scan_service)
 
     if is_commit_range:
         return cycode_client.commit_range_zipped_file_scan(scan_type, zipped_documents, scan_id)
@@ -439,12 +452,24 @@ def perform_scan(
 
 
 def perform_scan_async(
-    cycode_client: 'ScanClient', zipped_documents: 'InMemoryZip', scan_type: str, scan_parameters: dict
+    cycode_client: 'ScanClient',
+    zipped_documents: 'InMemoryZip',
+    scan_type: str,
+    scan_parameters: dict,
+    should_use_scan_service: bool = False,
 ) -> ZippedFileScanResult:
-    scan_async_result = cycode_client.zipped_file_scan_async(zipped_documents, scan_type, scan_parameters)
+    scan_async_result = cycode_client.zipped_file_scan_async(
+        zipped_documents, scan_type, scan_parameters, should_use_scan_service=should_use_scan_service
+    )
     logger.debug('scan request has been triggered successfully, scan id: %s', scan_async_result.scan_id)
 
-    return poll_scan_results(cycode_client, scan_async_result.scan_id, scan_type)
+    return poll_scan_results(
+        cycode_client,
+        scan_async_result.scan_id,
+        scan_type,
+        scan_parameters.get('report'),
+        should_use_scan_service=should_use_scan_service,
+    )
 
 
 def perform_commit_range_scan_async(
@@ -454,20 +479,30 @@ def perform_commit_range_scan_async(
     scan_type: str,
     scan_parameters: dict,
     timeout: Optional[int] = None,
+    should_use_scan_service: bool = False,
 ) -> ZippedFileScanResult:
     scan_async_result = cycode_client.multiple_zipped_file_scan_async(
         from_commit_zipped_documents, to_commit_zipped_documents, scan_type, scan_parameters
     )
 
     logger.debug('scan request has been triggered successfully, scan id: %s', scan_async_result.scan_id)
-    return poll_scan_results(cycode_client, scan_async_result.scan_id, scan_type, timeout)
+    return poll_scan_results(
+        cycode_client,
+        scan_async_result.scan_id,
+        scan_type,
+        scan_parameters.get('report'),
+        timeout,
+        should_use_scan_service,
+    )
 
 
 def poll_scan_results(
     cycode_client: 'ScanClient',
     scan_id: str,
     scan_type: str,
+    should_get_report: bool = False,
     polling_timeout: Optional[int] = None,
+    should_use_scan_service: bool = False,
 ) -> ZippedFileScanResult:
     if polling_timeout is None:
         polling_timeout = configuration_manager.get_scan_polling_timeout_in_seconds()
@@ -476,14 +511,14 @@ def poll_scan_results(
     end_polling_time = time.time() + polling_timeout
 
     while time.time() < end_polling_time:
-        scan_details = cycode_client.get_scan_details(scan_type, scan_id)
+        scan_details = cycode_client.get_scan_details(scan_type, scan_id, should_use_scan_service)
 
         if scan_details.scan_update_at is not None and scan_details.scan_update_at != last_scan_update_at:
             last_scan_update_at = scan_details.scan_update_at
             print_debug_scan_details(scan_details)
 
         if scan_details.scan_status == consts.SCAN_STATUS_COMPLETED:
-            return _get_scan_result(cycode_client, scan_type, scan_id, scan_details)
+            return _get_scan_result(cycode_client, scan_type, scan_id, scan_details, should_get_report)
 
         if scan_details.scan_status == consts.SCAN_STATUS_ERROR:
             raise custom_exceptions.ScanAsyncError(
@@ -575,7 +610,6 @@ def get_default_scan_parameters(context: click.Context) -> dict:
         'report': context.obj.get('report'),
         'package_vulnerabilities': context.obj.get('package-vulnerabilities'),
         'license_compliance': context.obj.get('license-compliance'),
-        'command_type': context.info_name,
     }
 
 
@@ -735,6 +769,7 @@ def _report_scan_status(
     zip_size: int,
     command_scan_type: str,
     error_message: Optional[str],
+    should_use_scan_service: bool = False,
 ) -> None:
     try:
         end_scan_time = time.time()
@@ -751,7 +786,7 @@ def _report_scan_status(
             'scan_type': scan_type,
         }
 
-        cycode_client.report_scan_status(scan_type, scan_id, scan_status)
+        cycode_client.report_scan_status(scan_type, scan_id, scan_status, should_use_scan_service)
     except Exception as e:
         logger.debug('Failed to report scan status, %s', {'exception_message': str(e)})
 
@@ -769,37 +804,49 @@ def _does_severity_match_severity_threshold(severity: str, severity_threshold: s
 
 
 def _get_scan_result(
-    cycode_client: 'ScanClient', scan_type: str, scan_id: str, scan_details: 'ScanDetailsResponse'
+    cycode_client: 'ScanClient',
+    scan_type: str,
+    scan_id: str,
+    scan_details: 'ScanDetailsResponse',
+    should_get_report: bool = False,
 ) -> ZippedFileScanResult:
     if not scan_details.detections_count:
-        return init_default_scan_result(scan_id, scan_details.metadata)
+        return init_default_scan_result(cycode_client, scan_id, scan_type, should_get_report)
 
     wait_for_detections_creation(cycode_client, scan_type, scan_id, scan_details.detections_count)
 
     scan_detections = cycode_client.get_scan_detections(scan_type, scan_id)
+
     return ZippedFileScanResult(
         did_detect=True,
         detections_per_file=_map_detections_per_file(scan_detections),
         scan_id=scan_id,
-        report_url=_try_get_report_url(scan_details.metadata),
+        report_url=_try_get_report_url_if_needed(cycode_client, should_get_report, scan_id, scan_type),
     )
 
 
-def init_default_scan_result(scan_id: str, scan_metadata: Optional[str] = None) -> ZippedFileScanResult:
+def init_default_scan_result(
+    cycode_client: 'ScanClient', scan_id: str, scan_type: str, should_get_report: bool = False
+) -> ZippedFileScanResult:
     return ZippedFileScanResult(
-        did_detect=False, detections_per_file=[], scan_id=scan_id, report_url=_try_get_report_url(scan_metadata)
+        did_detect=False,
+        detections_per_file=[],
+        scan_id=scan_id,
+        report_url=_try_get_report_url_if_needed(cycode_client, should_get_report, scan_id, scan_type),
     )
 
 
-def _try_get_report_url(metadata_json: Optional[str]) -> Optional[str]:
-    if metadata_json is None:
+def _try_get_report_url_if_needed(
+    cycode_client: 'ScanClient', should_get_report: bool, scan_id: str, scan_type: str
+) -> Optional[str]:
+    if not should_get_report:
         return None
 
     try:
-        metadata_json = json.loads(metadata_json)
-        return metadata_json.get('report_url')
-    except json.JSONDecodeError:
-        return None
+        report_url_response = cycode_client.get_scan_report_url(scan_id, scan_type)
+        return report_url_response.report_url
+    except Exception as e:
+        logger.debug('Failed to get report url: %s', str(e))
 
 
 def wait_for_detections_creation(
@@ -856,9 +903,18 @@ def _get_file_name_from_detection(detection: dict) -> str:
     if detection['category'] == 'SAST':
         return detection['detection_details']['file_path']
 
+    if detection['category'] == 'SecretDetection':
+        return _get_secret_file_name_from_detection(detection)
+
     return detection['detection_details']['file_name']
 
 
+def _get_secret_file_name_from_detection(detection: dict) -> str:
+    file_path: str = detection['detection_details']['file_path']
+    file_name: str = detection['detection_details']['file_name']
+    return os.path.join(file_path, file_name)
+
+
 def _does_reach_to_max_commits_to_scan_limit(commit_ids: List[str], max_commits_count: Optional[int]) -> bool:
     if max_commits_count is None:
         return False

cycode/cyclient/models.py

@@ -171,6 +171,23 @@ def __init__(
         self.err = err
 
 
+class ScanReportUrlResponse(Schema):
+    def __init__(
+        self,
+        report_url: str,
+    ) -> None:
+        super().__init__()
+        self.report_url = report_url
+
+
+class ScanReportUrlResponseSchema(Schema):
+    report_url = fields
+
+    @post_load
+    def build_dto(self, data: Dict[str, Any], **_) -> 'ScanReportUrlResponse':
+        return ScanReportUrlResponse(**data)
+
+
 class ScanDetailsResponseSchema(Schema):
     class Meta:
         unknown = EXCLUDE

cycode/cyclient/scan_client.py

@@ -42,8 +42,8 @@ def get_detections_service_controller_path(self, scan_type: str) -> str:
 
         return self._DETECTIONS_SERVICE_CONTROLLER_PATH
 
-    def get_scan_service_url_path(self, scan_type: str) -> str:
-        service_path = self.scan_config.get_service_name(scan_type)
+    def get_scan_service_url_path(self, scan_type: str, should_use_scan_service: bool = False) -> str:
+        service_path = self.scan_config.get_service_name(scan_type, should_use_scan_service)
         controller_path = self.get_scan_controller_path(scan_type)
         return f'{service_path}/{controller_path}'
 
@@ -72,17 +72,27 @@ def zipped_file_scan(
 
         return self.parse_zipped_file_scan_response(response)
 
-    def get_zipped_file_scan_async_url_path(self, scan_type: str) -> str:
+    def get_scan_report_url(self, scan_id: str, scan_type: str) -> models.ScanReportUrlResponse:
+        response = self.scan_cycode_client.get(url_path=self.get_scan_report_url_path(scan_id, scan_type))
+        return models.ScanReportUrlResponseSchema().build_dto(response.json())
+
+    def get_zipped_file_scan_async_url_path(self, scan_type: str, should_use_scan_service: bool = False) -> str:
         async_scan_type = self.scan_config.get_async_scan_type(scan_type)
         async_entity_type = self.scan_config.get_async_entity_type(scan_type)
-        return f'{self.get_scan_service_url_path(scan_type)}/{async_scan_type}/{async_entity_type}'
+        scan_service_url_path = self.get_scan_service_url_path(scan_type, should_use_scan_service)
+        return f'{scan_service_url_path}/{async_scan_type}/{async_entity_type}'
 
     def zipped_file_scan_async(
-        self, zip_file: InMemoryZip, scan_type: str, scan_parameters: dict, is_git_diff: bool = False
+        self,
+        zip_file: InMemoryZip,
+        scan_type: str,
+        scan_parameters: dict,
+        is_git_diff: bool = False,
+        should_use_scan_service: bool = False,
     ) -> models.ScanInitializationResponse:
         files = {'file': ('multiple_files_scan.zip', zip_file.read())}
         response = self.scan_cycode_client.post(
-            url_path=self.get_zipped_file_scan_async_url_path(scan_type),
+            url_path=self.get_zipped_file_scan_async_url_path(scan_type, should_use_scan_service),
             data={'is_git_diff': is_git_diff, 'scan_parameters': json.dumps(scan_parameters)},
             files=files,
         )
@@ -108,11 +118,16 @@ def multiple_zipped_file_scan_async(
         )
         return models.ScanInitializationResponseSchema().load(response.json())
 
-    def get_scan_details_path(self, scan_type: str, scan_id: str) -> str:
-        return f'{self.get_scan_service_url_path(scan_type)}/{scan_id}'
+    def get_scan_details_path(self, scan_type: str, scan_id: str, should_use_scan_service: bool = False) -> str:
+        return f'{self.get_scan_service_url_path(scan_type, should_use_scan_service)}/{scan_id}'
+
+    def get_scan_report_url_path(self, scan_id: str, scan_type: str) -> str:
+        return f'{self.get_scan_service_url_path(scan_type, True)}/reportUrl/{scan_id}'
 
-    def get_scan_details(self, scan_type: str, scan_id: str) -> models.ScanDetailsResponse:
-        path = self.get_scan_details_path(scan_type, scan_id)
+    def get_scan_details(
+        self, scan_type: str, scan_id: str, should_use_scan_service: bool = False
+    ) -> models.ScanDetailsResponse:
+        path = self.get_scan_details_path(scan_type, scan_id, should_use_scan_service)
         response = self.scan_cycode_client.get(url_path=path)
         return models.ScanDetailsResponseSchema().load(response.json())
 
@@ -222,11 +237,18 @@ def commit_range_zipped_file_scan(
         )
         return self.parse_zipped_file_scan_response(response)
 
-    def get_report_scan_status_path(self, scan_type: str, scan_id: str) -> str:
-        return f'{self.get_scan_service_url_path(scan_type)}/{scan_id}/status'
+    def get_report_scan_status_path(self, scan_type: str, scan_id: str, should_use_scan_service: bool = False) -> str:
+        return f'{self.get_scan_service_url_path(scan_type, should_use_scan_service)}/{scan_id}/status'
 
-    def report_scan_status(self, scan_type: str, scan_id: str, scan_status: dict) -> None:
-        self.scan_cycode_client.post(url_path=self.get_report_scan_status_path(scan_type, scan_id), body=scan_status)
+    def report_scan_status(
+        self, scan_type: str, scan_id: str, scan_status: dict, should_use_scan_service: bool = False
+    ) -> None:
+        self.scan_cycode_client.post(
+            url_path=self.get_report_scan_status_path(
+                scan_type, scan_id, should_use_scan_service=should_use_scan_service
+            ),
+            body=scan_status,
+        )
 
     @staticmethod
     def parse_scan_response(response: Response) -> models.ScanResult: