Feature/277 add option to export slc to uncompressed tar

.github/workflows/checks.yml

@@ -47,6 +47,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
           poetry-version: 2.1.2
 
+      - name: Run Unit tests
+        run: poetry run nox -s test:unit
+
       - name: Run lint
         run: poetry run nox -s lint:code
 

exasol/slc/api/deploy.py

@@ -23,6 +23,10 @@
 
 from exasol.slc.internal.tasks.upload.deploy_containers import DeployContainers
 from exasol.slc.internal.tasks.upload.deploy_info import toDeployResult
+from exasol.slc.models.compression_strategy import (
+    CompressionStrategy,
+    defaultCompressionStrategy,
+)
 from exasol.slc.models.deploy_result import DeployResult
 
 
@@ -61,6 +65,7 @@ def deploy(
     task_dependencies_dot_file: Optional[str] = None,
     log_level: Optional[str] = None,
     use_job_specific_log_file: bool = True,
+    compression_strategy: CompressionStrategy = defaultCompressionStrategy(),
 ) -> Dict[str, Dict[str, DeployResult]]:
     """
     This command uploads the whole script-language-container package of the flavor to the database.
@@ -118,6 +123,7 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             bucketfs_name=bucketfs_name,
             ssl_cert_path=ssl_cert_path,
             use_ssl_cert_validation=use_ssl_cert_validation,
+            compression_strategy=compression_strategy,
         )
 
     deploy_infos = run_task(

exasol/slc/api/export.py

@@ -21,6 +21,10 @@
 )
 
 from exasol.slc.internal.tasks.export.export_containers import ExportContainers
+from exasol.slc.models.compression_strategy import (
+    CompressionStrategy,
+    defaultCompressionStrategy,
+)
 from exasol.slc.models.export_container_result import ExportContainerResult
 
 
@@ -51,6 +55,7 @@ def export(
     log_level: Optional[str] = None,
     use_job_specific_log_file: bool = True,
     cleanup_docker_images: bool = False,
+    compression_strategy: CompressionStrategy = defaultCompressionStrategy(),
 ) -> ExportContainerResult:
     """
     This command exports the whole script-language-container package of the flavor,
@@ -93,6 +98,7 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             export_path=export_path,
             release_name=release_name,
             cleanup_docker_images=cleanup_docker_images,
+            compression_strategy=compression_strategy,
         )
 
     return run_task(

exasol/slc/api/run_db_tests.py

@@ -35,6 +35,10 @@
 from exasol.slc.internal.tasks.test.test_container_content import (
     build_test_container_content,
 )
+from exasol.slc.models.compression_strategy import (
+    CompressionStrategy,
+    defaultCompressionStrategy,
+)
 from exasol.slc.models.test_result import AllTestsResult
 
 
@@ -97,6 +101,7 @@ def run_db_test(
     task_dependencies_dot_file: Optional[str] = None,
     log_level: Optional[str] = None,
     use_job_specific_log_file: bool = True,
+    compression_strategy: CompressionStrategy = defaultCompressionStrategy(),
 ) -> AllTestsResult:
     """
     This command runs the integration tests in local docker-db.
@@ -146,6 +151,8 @@ def run_db_test(
             raise api_errors.MissingArgumentError("external_exasol_db_port")
         if external_exasol_bucketfs_port is None:
             raise api_errors.MissingArgumentError("external_exasol_bucketfs_port")
+        if external_exasol_ssh_port is None:
+            raise api_errors.MissingArgumentError("external_exasol_ssh_port")
 
     def root_task_generator() -> DependencyLoggerBaseTask:
         return generate_root_task(
@@ -190,6 +197,7 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             create_certificates=create_certificates,
             additional_db_parameter=additional_db_parameter,
             test_container_content=build_test_container_content(test_container_folder),
+            compression_strategy=compression_strategy,
         )
 
     return run_task(

exasol/slc/api/upload.py

@@ -24,6 +24,10 @@
 )
 
 from exasol.slc.internal.tasks.upload.upload_containers import UploadContainers
+from exasol.slc.models.compression_strategy import (
+    CompressionStrategy,
+    defaultCompressionStrategy,
+)
 
 
 @cli_function
@@ -61,6 +65,7 @@ def upload(
     use_job_specific_log_file: bool = True,
     ssl_cert_path: str = "",
     use_ssl_cert_validation: bool = True,
+    compression_strategy: CompressionStrategy = defaultCompressionStrategy(),
 ) -> luigi.LocalTarget:
     warnings.warn(
         "The 'upload' function is deprecated, use 'deploy' instead", DeprecationWarning
@@ -113,6 +118,7 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             bucketfs_name=bucketfs_name,
             ssl_cert_path=ssl_cert_path,
             use_ssl_cert_validation=use_ssl_cert_validation,
+            compression_strategy=compression_strategy,
         )
 
     return run_task(

exasol/slc/internal/tasks/export/export_container_base_task.py

@@ -26,6 +26,7 @@
 from exasol.slc.internal.tasks.export.remove_cached_export_file_task import (
     RemoveCachedExportTask,
 )
+from exasol.slc.models.compression_strategy import CompressionStrategy
 from exasol.slc.models.export_info import ExportInfo
 
 
@@ -112,6 +113,16 @@ def _create_export_info(
         )
         return export_info
 
+    def _get_export_file_extension(self) -> str:
+        if self.compression_strategy == CompressionStrategy.GZIP:
+            return ".tar.gz"
+        elif self.compression_strategy == CompressionStrategy.NONE:
+            return ".tar"
+        else:
+            raise ValueError(
+                f"Unsupported compression_strategy: {self.compression_strategy}"
+            )
+
     def _get_cache_file_path(
         self,
         image_info_of_release_image: ImageInfo,
@@ -120,5 +131,7 @@ def _get_cache_file_path(
         release_image_name = image_info_of_release_image.get_target_complete_name()
         export_path = Path(export_directory_future.get_output()).absolute()
         release_complete_name = f"""{image_info_of_release_image.target_tag}-{image_info_of_release_image.hash}"""
-        cache_file = Path(export_path, release_complete_name + ".tar.gz").absolute()
+        cache_file = Path(
+            export_path, release_complete_name + self._get_export_file_extension()
+        ).absolute()
         return str(cache_file), release_complete_name, release_image_name

exasol/slc/internal/tasks/export/export_container_parameters.py

@@ -1,12 +1,20 @@
 from typing import Optional, Tuple
 
 import luigi
-from luigi import Config
+
+from exasol.slc.models.compression_strategy import (
+    CompressionStrategy,
+    defaultCompressionStrategy,
+)
 
 CHECKSUM_ALGORITHM = "sha512sum"
 
 
-class ExportContainerParameterBase(Config):
+class ExportContainerOptionsParameter:
+    compression_strategy: CompressionStrategy = luigi.EnumParameter(enum=CompressionStrategy, default=defaultCompressionStrategy())  # type: ignore
+
+
+class ExportContainerParameterBase(ExportContainerOptionsParameter):
     export_path: Optional[str] = luigi.OptionalParameter(None)  # type: ignore
     release_name: Optional[str] = luigi.OptionalParameter(None)  # type: ignore
     cleanup_docker_images: bool = luigi.BoolParameter(False)  # type: ignore

exasol/slc/internal/tasks/export/export_container_to_cache_task.py

@@ -36,6 +36,7 @@
     CHECKSUM_ALGORITHM,
     ExportContainerParameter,
 )
+from exasol.slc.models.compression_strategy import CompressionStrategy
 
 
 class CheckCacheFileTask(DependencyLoggerBaseTask):
@@ -170,11 +171,17 @@ def _pack_release_file(
     ) -> None:
         self.logger.info("Pack container file %s", release_file)
         extract_content = " ".join(f"'{file}'" for file in os.listdir(extract_dir))
-        if not str(release_file).endswith("tar.gz"):
+        if str(release_file).endswith("tar.gz"):
+            tmp_release_file = release_file.with_suffix(
+                ""
+            )  # cut off ".gz" from ".tar.gz"
+        elif str(release_file).endswith("tar"):
+            tmp_release_file = release_file
+        else:
             raise ValueError(
-                f"Unexpected release file: '{release_file}'. Expected suffix 'tar.gz'."
+                f"Unexpected release file: '{release_file}'. Expected suffix: 'tar.gz' or 'tar'."
             )
-        tmp_release_file = release_file.with_suffix("")  # cut off ".gz" from ".tar.gz"
+
         command = (
             f"""tar -C '{extract_dir}' -vcf '{tmp_release_file}' {extract_content}"""
         )
@@ -195,12 +202,19 @@ def _pack_release_file(
             log_path.joinpath("pack_release_file.log"),
         )
         shutil.rmtree(extract_dir)
-        command = f"""gzip {tmp_release_file}"""
-        self.run_command(
-            command,
-            f"Creating '{release_file}'",
-            log_path.joinpath("pack_release_file.log"),
-        )
+        if self.compression_strategy == CompressionStrategy.GZIP:
+            command = f"""gzip {tmp_release_file}"""
+            self.run_command(
+                command,
+                f"Creating '{release_file}'",
+                log_path.joinpath("pack_release_file.log"),
+            )
+        elif self.compression_strategy == CompressionStrategy.NONE:
+            pass
+        else:
+            raise ValueError(
+                f"Unexpected compression_strategy: {self.compression_strategy}"
+            )
 
     @staticmethod
     def _modify_extracted_container(extract_dir: str) -> None:

exasol/slc/internal/tasks/export/export_container_to_file_task.py

@@ -20,6 +20,7 @@
 from exasol.slc.internal.tasks.export.export_container_to_cache_task import (
     ExportContainerToCacheTask,
 )
+from exasol.slc.internal.utils.file_utilities import detect_container_file_extension
 
 
 class ExportContainerToFileInfo(Info):
@@ -66,9 +67,8 @@ def _copy_cache_file_to_output_path(
                 suffix = f"""_{self.release_name}"""
             else:
                 suffix = ""
-            file_name = (
-                f"""{self.get_flavor_name()}_{self.release_goal}{suffix}.tar.gz"""
-            )
+            file_extension = detect_container_file_extension(cache_file.name)
+            file_name = f"""{self.get_flavor_name()}_{self.release_goal}{suffix}{file_extension}"""
             output_file = Path(str(self.export_path), file_name)
             output_checksum_file = Path(
                 str(self.export_path), file_name + "." + CHECKSUM_ALGORITHM

exasol/slc/internal/tasks/test/test_container.py

@@ -13,6 +13,9 @@
     SpawnTestEnvironmentParameter,
 )
 
+from exasol.slc.internal.tasks.export.export_container_parameters import (
+    ExportContainerOptionsParameter,
+)
 from exasol.slc.internal.tasks.test.run_db_tests_parameter import (
     GeneralRunDBTestParameter,
     RunDBTestsInTestConfigParameter,
@@ -30,7 +33,9 @@
 
 
 class TestContainerParameter(
-    RunDBTestsInTestConfigParameter, GeneralRunDBTestParameter
+    RunDBTestsInTestConfigParameter,
+    GeneralRunDBTestParameter,
+    ExportContainerOptionsParameter,
 ):
     release_goals: Tuple[str, ...] = luigi.ListParameter(["release"])  # type: ignore
     languages: Tuple[Optional[str], ...] = luigi.ListParameter([None])  # type: ignore

exasol/slc/internal/tasks/test/test_runner_db_test_with_export_task.py

@@ -1,5 +1,8 @@
 from typing import Dict
 
+from exasol.slc.internal.tasks.export.export_container_parameters import (
+    ExportContainerOptionsParameter,
+)
 from exasol.slc.internal.tasks.export.export_containers import ExportFlavorContainer
 from exasol.slc.internal.tasks.test.container_file_under_test_info import (
     ContainerFileUnderTestInfo,
@@ -10,7 +13,9 @@
 from exasol.slc.models.export_info import ExportInfo
 
 
-class TestRunnerDBTestWithExportTask(TestRunnerDBTestBaseTask):
+class TestRunnerDBTestWithExportTask(
+    TestRunnerDBTestBaseTask, ExportContainerOptionsParameter
+):
 
     def register_required(self) -> None:
         self.register_export_container()
@@ -21,6 +26,7 @@ def register_export_container(self) -> None:
             ExportFlavorContainer,
             release_goals=[self.release_goal],
             flavor_path=self.flavor_path,
+            compression_strategy=self.compression_strategy,
         )
         self._export_infos_future = self.register_dependency(export_container_task)
 

exasol/slc/internal/tasks/test/upload_exported_container.py

@@ -1,12 +1,7 @@
-from pathlib import Path
-
 import luigi
-from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import (
-    JsonPickleParameter,
-)
 
 from exasol.slc.internal.tasks.test.upload_file_to_bucket_fs import UploadFileToBucketFS
-from exasol.slc.models.export_info import ExportInfo
+from exasol.slc.internal.utils.file_utilities import detect_container_file_extension
 
 
 class UploadExportedContainer(UploadFileToBucketFS):
@@ -24,7 +19,11 @@ def get_file_to_upload(self) -> str:
 
     def get_upload_target(self) -> str:
         return (
-            "myudfs/" + self.target_name + ".tar.gz"  # pylint: disable=no-member
+            "myudfs/"
+            + self.target_name
+            + detect_container_file_extension(
+                self.file_to_upload
+            )  # pylint: disable=no-member
         )  # pylint: disable=no-member
 
     def get_sync_time_estimation(self) -> int:

exasol/slc/internal/tasks/upload/deploy_container_base_task.py

@@ -24,6 +24,7 @@
 from exasol.slc.internal.tasks.upload.upload_container_parameter import (
     UploadContainerParameter,
 )
+from exasol.slc.internal.utils.file_utilities import detect_container_file_extension
 from exasol.slc.models.export_info import ExportInfo
 from exasol.slc.models.language_definition_components import (
     LanguageDefinitionComponents,
@@ -80,6 +81,7 @@ def run_task(self) -> None:
             complete_release_name=self._get_complete_release_name(export_info),
             human_readable_location=self._complete_url(export_info),
             language_definition_builder=lang_def_builder,
+            file_extension=detect_container_file_extension(path_in_bucket.name),
         )
         self.return_object(result)
 
@@ -98,7 +100,10 @@ def build_file_path_in_bucket(self, release_info: ExportInfo) -> bfs.path.PathLi
             verify=verify,
             path=self.path_in_bucket or "",
         )
-        return path_in_bucket_to_upload_path / f"{complete_release_name}.tar.gz"
+        return (
+            path_in_bucket_to_upload_path
+            / f"{complete_release_name}{detect_container_file_extension(release_info.cache_file)}"
+        )
 
     @property
     def _url(self) -> str:
@@ -108,7 +113,7 @@ def _complete_url(self, export_info: ExportInfo):
         path_in_bucket = (
             f"{self.path_in_bucket}/" if self.path_in_bucket not in [None, ""] else ""
         )
-        return f"{self._url}/{self.bucket_name}/{path_in_bucket}{self._get_complete_release_name(export_info)}.tar.gz"
+        return f"{self._url}/{self.bucket_name}/{path_in_bucket}{self._get_complete_release_name(export_info)}{detect_container_file_extension(export_info.cache_file)}"
 
     def _upload_container(self, release_info: ExportInfo) -> bfs.path.PathLike:
         bucket_path = self.build_file_path_in_bucket(release_info)

exasol/slc/internal/tasks/upload/deploy_info.py

@@ -2,6 +2,7 @@
 
 import exasol.bucketfs as bfs  # type: ignore
 
+from exasol.slc.internal.utils.file_utilities import detect_container_file_extension
 from exasol.slc.models.deploy_result import DeployResult
 from exasol.slc.models.language_definitions_builder import LanguageDefinitionsBuilder
 
@@ -12,6 +13,7 @@ class DeployInfo:
     complete_release_name: str
     human_readable_location: str
     language_definition_builder: LanguageDefinitionsBuilder
+    file_extension: str
 
 
 def toDeployResult(
@@ -45,7 +47,7 @@ def toDeployResult(
             verify=verify,
             path=path_in_bucket or "",
         )
-        / f"{complete_release_name}.tar.gz"
+        / f"{complete_release_name}{deploy_info.file_extension}"
     )
 
     return DeployResult(