CM-22829 - Migrate to Poetry; fix package structure; use dynamic versioning from Git Tags

.github/workflows/tests.yml

@@ -7,18 +7,22 @@ permissions:
 
 jobs:
   deploy:
-
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.7'
+
+      - name: Setup Poetry
+        run: curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python
+
       - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+        run: poetry install
+
       - name: Run Tests
-        run: pytest
+        run: poetry run pytest

Dockerfile

@@ -1,13 +1,26 @@
-FROM python:3.8.16-alpine3.17 as builder
+FROM python:3.8.16-alpine3.17 as base
 WORKDIR /usr/cycode/app
-COPY . ./
-RUN python -m pip install --upgrade --no-cache-dir pip==22.0.4 setuptools==65.5.1 wheel==0.37.1
-RUN python3 setup.py sdist bdist_wheel
+RUN apk add git=2.38.5-r0
 
-FROM python:3.8.16-alpine3.17
-RUN apk add git=2.38.3-r1
-WORKDIR /usr/cycode/app
-COPY --from=builder usr/cycode/app/dist ./
+FROM base as builder
+ENV POETRY_VERSION=1.4.2
+
+# deps are required to build cffi
+RUN apk add --no-cache --virtual .build-deps gcc libffi-dev musl-dev &&  \
+    pip install "poetry==$POETRY_VERSION" "poetry-dynamic-versioning[plugin]" &&  \
+    apk del .build-deps gcc libffi-dev musl-dev
+
+COPY pyproject.toml poetry.lock README.md ./
+# to be able to automatically detect version from Git Tag
+COPY .git ./.git
+# src
+COPY cycode ./cycode
+RUN poetry config virtualenvs.in-project true && \
+    poetry install --only=main --no-root && \
+    poetry build
+
+FROM base as final
+COPY --from=builder /usr/cycode/app/dist ./
 RUN pip install --no-cache-dir cycode*.whl
 
 # Add cycode group and user, alpine way

cycode/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.0.0'   # placeholder. Will be filled automatically on poetry build from Git Tag

cli/auth/auth_command.py → cycode/cli/auth/auth_command.py

@@ -3,11 +3,11 @@
 import click
 import traceback
 
-from cli.auth.auth_manager import AuthManager
-from cli.user_settings.credentials_manager import CredentialsManager
-from cli.exceptions.custom_exceptions import AuthProcessError, NetworkError, HttpUnauthorizedError
-from cyclient import logger
-from cyclient.cycode_token_based_client import CycodeTokenBasedClient
+from cycode.cli.auth.auth_manager import AuthManager
+from cycode.cli.user_settings.credentials_manager import CredentialsManager
+from cycode.cli.exceptions.custom_exceptions import AuthProcessError, NetworkError, HttpUnauthorizedError
+from cycode.cyclient import logger
+from cycode.cyclient.cycode_token_based_client import CycodeTokenBasedClient
 
 
 @click.group(invoke_without_command=True)

cli/auth/auth_manager.py → cycode/cli/auth/auth_manager.py

@@ -2,13 +2,14 @@
 import webbrowser
 from requests import Request
 from typing import Optional
-from cli.exceptions.custom_exceptions import AuthProcessError
-from cli.utils.string_utils import generate_random_string, hash_string_to_sha256
-from cli.user_settings.configuration_manager import ConfigurationManager
-from cli.user_settings.credentials_manager import CredentialsManager
-from cyclient.auth_client import AuthClient
-from cyclient.models import ApiToken, ApiTokenGenerationPollingResponse
-from cyclient import logger
+
+from cycode.cli.exceptions.custom_exceptions import AuthProcessError
+from cycode.cli.utils.string_utils import generate_random_string, hash_string_to_sha256
+from cycode.cli.user_settings.configuration_manager import ConfigurationManager
+from cycode.cli.user_settings.credentials_manager import CredentialsManager
+from cycode.cyclient.auth_client import AuthClient
+from cycode.cyclient.models import ApiToken, ApiTokenGenerationPollingResponse
+from cycode.cyclient import logger
 
 
 class AuthManager:

cli/code_scanner.py → cycode/cli/code_scanner.py

@@ -14,22 +14,22 @@
 
 from halo import Halo
 
-from cli.printers import ResultsPrinter
-from cli.models import Document, DocumentDetections, Severity
-from cli.ci_integrations import get_commit_range
-from cli.consts import *
-from cli.config import configuration_manager
-from cli.utils.path_utils import is_sub_path, is_binary_file, get_file_size, get_relevant_files_in_path, \
+from cycode.cli.printers import ResultsPrinter
+from cycode.cli.models import Document, DocumentDetections, Severity
+from cycode.cli.ci_integrations import get_commit_range
+from cycode.cli.consts import *
+from cycode.cli.config import configuration_manager
+from cycode.cli.utils.path_utils import is_sub_path, is_binary_file, get_file_size, get_relevant_files_in_path, \
     get_path_by_os, get_file_content
-from cli.utils.string_utils import get_content_size, is_binary_content
-from cli.utils.task_timer import TimeoutAfter
-from cli.utils import scan_utils
-from cli.user_settings.config_file_manager import ConfigFileManager
-from cli.zip_file import InMemoryZip
-from cli.exceptions.custom_exceptions import *
-from cyclient import logger
-from cyclient.models import *
-from cli.helpers import sca_code_scanner
+from cycode.cli.utils.string_utils import get_content_size, is_binary_content
+from cycode.cli.utils.task_timer import TimeoutAfter
+from cycode.cli.utils import scan_utils
+from cycode.cli.user_settings.config_file_manager import ConfigFileManager
+from cycode.cli.zip_file import InMemoryZip
+from cycode.cli.exceptions.custom_exceptions import *
+from cycode.cli.helpers import sca_code_scanner
+from cycode.cyclient import logger
+from cycode.cyclient.models import *
 
 start_scan_time = time.time()
 

cli/config.py → cycode/cli/config.py

@@ -1,6 +1,8 @@
 import os
-from cli.utils.yaml_utils import read_file
-from cli.user_settings.configuration_manager import ConfigurationManager
+
+from cycode.cli.utils.yaml_utils import read_file
+from cycode.cli.user_settings.configuration_manager import ConfigurationManager
+
 
 relative_path = os.path.dirname(__file__)
 config_file_path = os.path.join(relative_path, 'config.yaml')

cli/helpers/maven/base_restore_maven_dependencies.py → cycode/cli/helpers/maven/base_restore_maven_dependencies.py

@@ -3,10 +3,10 @@
 
 import click
 
-from cli.models import Document
-from cli.utils.path_utils import join_paths, get_file_dir
-from cli.utils.shell_executor import shell
-from cyclient import logger
+from cycode.cli.models import Document
+from cycode.cli.utils.path_utils import join_paths, get_file_dir
+from cycode.cli.utils.shell_executor import shell
+from cycode.cyclient import logger
 
 
 def build_dep_tree_path(path: str, generated_file_name: str) -> str:

cli/helpers/maven/restore_gradle_dependencies.py → cycode/cli/helpers/maven/restore_gradle_dependencies.py

@@ -2,8 +2,8 @@
 
 import click
 
-from cli.helpers.maven.base_restore_maven_dependencies import BaseRestoreMavenDependencies
-from cli.models import Document
+from cycode.cli.helpers.maven.base_restore_maven_dependencies import BaseRestoreMavenDependencies
+from cycode.cli.models import Document
 
 BUILD_GRADLE_FILE_NAME = 'build.gradle'
 BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts'

cli/helpers/maven/restore_maven_dependencies.py → cycode/cli/helpers/maven/restore_maven_dependencies.py

@@ -3,10 +3,10 @@
 
 import click
 
-from cli.helpers.maven.base_restore_maven_dependencies import BaseRestoreMavenDependencies, build_dep_tree_path, \
+from cycode.cli.helpers.maven.base_restore_maven_dependencies import BaseRestoreMavenDependencies, build_dep_tree_path, \
     execute_command
-from cli.models import Document
-from cli.utils.path_utils import get_file_dir, get_file_content, join_paths
+from cycode.cli.models import Document
+from cycode.cli.utils.path_utils import get_file_dir, get_file_content, join_paths
 
 BUILD_MAVEN_FILE_NAME = 'pom.xml'
 MAVEN_CYCLONE_DEP_TREE_FILE_NAME = 'bom.json'

cli/helpers/sca_code_scanner.py → cycode/cli/helpers/sca_code_scanner.py

@@ -4,12 +4,12 @@
 import click
 from git import Repo, GitCommandError
 
-from cli.consts import *
-from cli.helpers.maven.restore_gradle_dependencies import RestoreGradleDependencies
-from cli.helpers.maven.restore_maven_dependencies import RestoreMavenDependencies
-from cli.models import Document
-from cli.utils.path_utils import get_file_dir, join_paths, get_file_content
-from cyclient import logger
+from cycode.cli.consts import *
+from cycode.cli.helpers.maven.restore_gradle_dependencies import RestoreGradleDependencies
+from cycode.cli.helpers.maven.restore_maven_dependencies import RestoreMavenDependencies
+from cycode.cli.models import Document
+from cycode.cli.utils.path_utils import get_file_dir, join_paths, get_file_content
+from cycode.cyclient import logger
 
 BUILD_GRADLE_FILE_NAME = 'build.gradle'
 BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts'

cli/cycode.py → cycode/cli/main.py

@@ -5,17 +5,17 @@
 
 from typing import List
 
-from cli.models import Severity
-from cli.config import config
-from cli import code_scanner, __version__
-from cyclient import logger
-from cyclient.scan_client import ScanClient
-from cli.user_settings.credentials_manager import CredentialsManager
-from cli.user_settings.configuration_manager import ConfigurationManager
-from cli.user_settings.user_settings_commands import set_credentials, add_exclusions
-from cli.auth.auth_command import authenticate
-from cli.utils import scan_utils
-from cyclient.scan_config.scan_config_creator import create_scan_client
+from cycode import __version__
+from cycode.cli.models import Severity
+from cycode.cli.config import config
+from cycode.cli import code_scanner
+from cycode.cli.user_settings.credentials_manager import CredentialsManager
+from cycode.cli.user_settings.configuration_manager import ConfigurationManager
+from cycode.cli.user_settings.user_settings_commands import set_credentials, add_exclusions
+from cycode.cli.auth.auth_command import authenticate
+from cycode.cli.utils import scan_utils
+from cycode.cyclient import logger
+from cycode.cyclient.scan_config.scan_config_creator import create_scan_client
 
 CONTEXT = dict()
 ISSUE_DETECTED_STATUS_CODE = 1

cli/models.py → cycode/cli/models.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from typing import List
-from cyclient.models import Detection
+from cycode.cyclient.models import Detection
 
 
 class Document:

cycode/cli/printers/__init__.py

@@ -0,0 +1,10 @@
+from .json_printer import JsonPrinter
+from .text_printer import TextPrinter
+from .results_printer import ResultsPrinter
+
+
+__all__ = [
+    'JsonPrinter',
+    'TextPrinter',
+    'ResultsPrinter'
+]

cli/printers/base_printer.py → cycode/cli/printers/base_printer.py

@@ -1,7 +1,9 @@
-import click
 from abc import ABC, abstractmethod
 from typing import List
-from cli.models import DocumentDetections
+
+import click
+
+from cycode.cli.models import DocumentDetections
 
 
 class BasePrinter(ABC):

cli/printers/json_printer.py → cycode/cli/printers/json_printer.py

@@ -3,9 +3,9 @@
 
 import click
 
-from cli.models import DocumentDetections
-from cli.printers.base_printer import BasePrinter
-from cyclient.models import DetectionSchema, Detection
+from cycode.cli.models import DocumentDetections
+from cycode.cli.printers.base_printer import BasePrinter
+from cycode.cyclient.models import DetectionSchema
 
 
 class JsonPrinter(BasePrinter):

cli/printers/results_printer.py → cycode/cli/printers/results_printer.py

@@ -1,10 +1,10 @@
 import click
 from typing import List
 
-from cli.consts import SCA_SCAN_TYPE
-from cli.printers import JsonPrinter, TextPrinter
-from cli.models import DocumentDetections
-from cli.printers.table_printer import TablePrinter
+from cycode.cli.consts import SCA_SCAN_TYPE
+from cycode.cli.printers import JsonPrinter, TextPrinter
+from cycode.cli.models import DocumentDetections
+from cycode.cli.printers.table_printer import TablePrinter
 
 
 class ResultsPrinter:

cli/printers/table_printer.py → cycode/cli/printers/table_printer.py

@@ -3,10 +3,10 @@
 import click
 from texttable import Texttable
 
-from cli.consts import LICENSE_COMPLIANCE_POLICY_ID, \
+from cycode.cli.consts import LICENSE_COMPLIANCE_POLICY_ID, \
     PACKAGE_VULNERABILITY_POLICY_ID
-from cli.models import DocumentDetections, Detection
-from cli.printers.base_printer import BasePrinter
+from cycode.cli.models import DocumentDetections, Detection
+from cycode.cli.printers.base_printer import BasePrinter
 
 SEVERITY_COLUMN = 'Severity'
 LICENSE_COLUMN = 'License'

cli/printers/text_printer.py → cycode/cli/printers/text_printer.py

@@ -1,11 +1,13 @@
-import click
 import math
 from typing import List, Optional
-from cli.printers.base_printer import BasePrinter
-from cli.models import DocumentDetections, Detection, Document
-from cli.config import config
-from cli.consts import SECRET_SCAN_TYPE, COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES
-from cli.utils.string_utils import obfuscate_text
+
+import click
+
+from cycode.cli.printers.base_printer import BasePrinter
+from cycode.cli.models import DocumentDetections, Detection, Document
+from cycode.cli.config import config
+from cycode.cli.consts import SECRET_SCAN_TYPE, COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES
+from cycode.cli.utils.string_utils import obfuscate_text
 
 
 class TextPrinter(BasePrinter):

cli/user_settings/base_file_manager.py → cycode/cli/user_settings/base_file_manager.py

@@ -1,6 +1,6 @@
 import os
 from abc import ABC, abstractmethod
-from cli.utils.yaml_utils import update_file, read_file
+from cycode.cli.utils.yaml_utils import update_file, read_file
 
 
 class BaseFileManager(ABC):

cli/user_settings/config_file_manager.py → cycode/cli/user_settings/config_file_manager.py

@@ -1,7 +1,8 @@
 import os
 from typing import Optional, List, Dict
-from cli.user_settings.base_file_manager import BaseFileManager
-from cli.consts import CYCODE_CONFIGURATION_DIRECTORY
+
+from cycode.cli.user_settings.base_file_manager import BaseFileManager
+from cycode.cli.consts import CYCODE_CONFIGURATION_DIRECTORY
 
 
 class ConfigFileManager(BaseFileManager):

cli/user_settings/configuration_manager.py → cycode/cli/user_settings/configuration_manager.py

@@ -1,8 +1,9 @@
 import os
 from pathlib import Path
 from typing import Optional, Dict
-from cli.user_settings.config_file_manager import ConfigFileManager
-from cli.consts import *
+
+from cycode.cli.user_settings.config_file_manager import ConfigFileManager
+from cycode.cli.consts import *
 
 
 class ConfigurationManager:

cli/user_settings/credentials_manager.py → cycode/cli/user_settings/credentials_manager.py

@@ -1,8 +1,9 @@
 import os
 from pathlib import Path
-from cli.utils.yaml_utils import read_file
-from cli.config import CYCODE_CLIENT_ID_ENV_VAR_NAME, CYCODE_CLIENT_SECRET_ENV_VAR_NAME
-from cli.user_settings.base_file_manager import BaseFileManager
+
+from cycode.cli.utils.yaml_utils import read_file
+from cycode.cli.config import CYCODE_CLIENT_ID_ENV_VAR_NAME, CYCODE_CLIENT_SECRET_ENV_VAR_NAME
+from cycode.cli.user_settings.base_file_manager import BaseFileManager
 
 
 class CredentialsManager(BaseFileManager):