Add a new engine to manage the local file system

.github/workflows/ci.yml

@@ -1,6 +1,6 @@
 name: Test and build
 
-on: [push, pull_request, release]
+on: [push, pull_request]
 
 jobs:
   qa:

.semversioner/next-release/minor-20220824162419350511.json

@@ -0,0 +1,4 @@
+{
+  "type": "minor",
+  "description": "Add a new engine to manage the local file system"
+}

.semversioner/next-release/patch-20220824203839091496.json

@@ -0,0 +1,4 @@
+{
+  "type": "patch",
+  "description": "remove release option from the triggerings to build and test the package"
+}

.semversioner/next-release/patch-20220824203853764987.json

@@ -0,0 +1,4 @@
+{
+  "type": "patch",
+  "description": "remove obsolete test file"
+}

file_storehouse/engine/__init__.py

@@ -1,9 +1,11 @@
 """Engines."""
 
 from .base import EngineABC
+from .local_file_system import EngineLocal
 from .s3 import EngineS3
 
 __all__ = [
     "EngineABC",
+    "EngineLocal",
     "EngineS3",
 ]

file_storehouse/engine/base.py

@@ -3,24 +3,24 @@
 from abc import ABC, abstractmethod
 from typing import Any, Iterator
 
-from ..type import FileLike, PathLike
+from ..type import PathLike
 
 
 class EngineABC(ABC):
     """Base for engine."""
 
     @abstractmethod
-    def get_item(self, key: PathLike) -> FileLike:
+    def get_item(self, key: Any) -> bytes:
         """Get the item related to the key."""
         pass
 
     @abstractmethod
-    def set_item(self, key: PathLike, file_content: FileLike) -> None:
+    def set_item(self, key: Any, file_content: bytes) -> None:
         """Set the item related to the key."""
         pass
 
     @abstractmethod
-    def delete_item(self, key: PathLike) -> None:
+    def delete_item(self, key: Any) -> None:
         """Delete the item related to the key."""
         pass
 

file_storehouse/engine/local_file_system.py

@@ -0,0 +1,58 @@
+"""Engine for the local file system."""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator
+
+from file_storehouse.engine.base import EngineABC
+from file_storehouse.type import PathLike
+
+
+@dataclass
+class EngineLocal(EngineABC):
+    """Engine for the local file system."""
+
+    base_path: Path
+
+    def get_item(self, key: Path) -> bytes:
+        """Get the item related to the key."""
+        if key.is_file():
+            return key.read_bytes()
+        raise KeyError(f"No such key {key=}")
+
+    def set_item(self, key: Path, file_content: bytes) -> None:
+        """Set the item related to the key."""
+        key.parent.mkdir(parents=True, exist_ok=True)
+        key.write_bytes(file_content)
+
+    def delete_item(self, key: Path) -> None:
+        """Delete the item related to the key."""
+        if key.is_file():
+            key.unlink()
+        else:
+            raise KeyError(f"No such key {key=}")
+
+        self._remove_empty_folders(key)
+
+    def list_keys(self) -> Iterator[PathLike]:
+        """List the keys related to the engine."""
+        return (path for path in self.base_path.rglob("*") if path.is_file())
+
+    def convert_to_absolute_path(self, relative_path: PathLike) -> Path:
+        """Convert to absolute path."""
+        return Path(self.base_path, relative_path).resolve()
+
+    def convert_to_relative_path(self, absolute_path: Path) -> Path:
+        """Convert to relative path."""
+        return Path(absolute_path).relative_to(self.base_path)
+
+    def _remove_empty_folders(self, key: Path) -> None:
+        """Remove folders in the path that become empty."""
+        rel_path = Path(key)
+        while rel_path != self.base_path:
+            print()
+            try:
+                rel_path.rmdir()
+            except OSError:
+                break
+            rel_path = rel_path.parent

file_storehouse/engine/s3.py

@@ -7,7 +7,7 @@
 from botocore.client import BaseClient
 
 from file_storehouse.engine.base import EngineABC
-from file_storehouse.type import FileLike, PathLike
+from file_storehouse.type import PathLike
 
 
 @dataclass
@@ -22,7 +22,7 @@ class EngineS3Data:
 class EngineS3(EngineS3Data, EngineABC):
     """Engine for S3 buckets."""
 
-    def get_item(self, key: PathLike) -> FileLike:
+    def get_item(self, key: str) -> bytes:
         """Get the item related to the key."""
         try:
             response = self.s3_client.get_object(Bucket=self.bucket_name, Key=key)
@@ -31,7 +31,7 @@ def get_item(self, key: PathLike) -> FileLike:
 
         return response["Body"].read()
 
-    def set_item(self, key: PathLike, file_content: FileLike) -> None:
+    def set_item(self, key: str, file_content: bytes) -> None:
         """Set the item related to the key."""
         try:
             self.s3_client.put_object(
@@ -40,7 +40,7 @@ def set_item(self, key: PathLike, file_content: FileLike) -> None:
         except self.s3_client.exceptions.NoSuchKey:
             raise KeyError(f"No such {key=}")
 
-    def delete_item(self, key: PathLike) -> None:
+    def delete_item(self, key: str) -> None:
         """Delete the item related to the key."""
         try:
             self.s3_client.delete_object(Bucket=self.bucket_name, Key=key)

tests/engine/test_local_file_system.py

@@ -0,0 +1,113 @@
+"""Test the local file system."""
+
+from pathlib import Path
+
+import pytest
+
+from file_storehouse.engine.local_file_system import EngineLocal
+
+
+@pytest.fixture
+def local_engine(tmp_path):
+    """Fixture to provide the engine."""
+    yield EngineLocal(base_path=tmp_path)
+
+
+@pytest.fixture
+def dummy_nested_folders(local_engine):
+    """Fixture to create a nested folder structure."""
+    test_path = Path(local_engine.base_path)
+    for i in range(4):
+        test_path /= str(i)
+    test_path.mkdir(parents=True)
+    return test_path
+
+
+@pytest.fixture(scope="module")
+def filename():
+    """Fixture with a dummy filename."""
+    return "test_file.txt"
+
+
+@pytest.fixture(scope="module")
+def file_content():
+    """Fixture with a dummy file content."""
+    return b"testing file"
+
+
+@pytest.fixture
+def file_path(local_engine, file_content, filename):
+    """Fixture that creates the dummy file and returns its path."""
+    file_path = local_engine.base_path / filename
+    file_path.write_bytes(file_content)
+    return file_path
+
+
+class TestEngineLocal:
+    """Test the engine to manage the local file system."""
+
+    def test_get_item_success(self, local_engine, file_content, file_path):
+        """Test get item."""
+        assert local_engine.get_item(file_path) == file_content
+
+    def test_get_item_fail_no_such_key(self, local_engine):
+        """Test that KeyError is raised when the file is not found."""
+        filename = "test_file.txt"
+        file_path = local_engine.base_path / filename
+
+        with pytest.raises(KeyError):
+            local_engine.get_item(file_path)
+
+    def test_set_item(self, local_engine, file_path, file_content):
+        """Test set item."""
+        local_engine.set_item(file_path, file_content)
+
+        assert file_path.read_bytes() == file_content
+
+    def test_remove_empty_folders_initial_length(
+        self, local_engine, dummy_nested_folders
+    ):
+        """Test the initial length of the nested folders fixture."""
+        assert len(list(local_engine.base_path.rglob("*"))) == 4
+
+    def test_remove_empty_folders_do_not_work_if_folder_is_not_empty(
+        self,
+        local_engine,
+        dummy_nested_folders,
+    ):
+        """Test that the engine can not remove a folder that when that are files."""
+        filename = "test_file.txt"
+        file_content = b"testing file"
+        file_path = dummy_nested_folders / filename
+
+        file_path.write_bytes(file_content)
+
+        local_engine._remove_empty_folders(dummy_nested_folders)
+
+        assert len(list(local_engine.base_path.rglob("*"))) == 5
+
+    def test_remove_empty_folders_empty(
+        self,
+        local_engine,
+        dummy_nested_folders,
+    ):
+        """Test that the engine delete empty nested folders when they are all empty."""
+        local_engine._remove_empty_folders(dummy_nested_folders)
+
+        assert len(list(local_engine.base_path.rglob("*"))) == 0
+
+    def test_delete_item(self, local_engine, file_path):
+        """Test delete item."""
+        assert file_path.is_file()
+        local_engine.delete_item(file_path)
+        assert not file_path.is_file()
+
+    def test_convert_to_absolute_path(self, local_engine, filename, file_path):
+        """Test convert the path from relative to absolute."""
+        absolute_path = local_engine.convert_to_absolute_path(filename)
+        assert absolute_path == file_path
+
+    def test_convert_to_relative_path(self, local_engine, filename, file_path):
+        """Test convert the path from absolute to relative."""
+        relative_path = local_engine.convert_to_relative_path(file_path)
+        assert relative_path == Path(filename)

tests/test_user_story.py

@@ -1,45 +1,52 @@
 """User story for an end-to-end test."""
 
+import tempfile
+from pathlib import Path
 from sys import platform
-from typing import Any
 
 from pytest import mark
 
 import file_storehouse as s3
 from file_storehouse.engine import EngineS3
+from file_storehouse.engine.base import EngineABC
+from file_storehouse.engine.local_file_system import EngineLocal
 from file_storehouse.key_mapping import KeyMappingNumeratedFile
 from file_storehouse.transformation import TransformationCodecs, TransformationJson
 
 
 @mark.skipif(platform != "linux", reason="just runs on linux for now")
-def test_acceptance(docker_services: Any) -> None:
-    """
-    User story for an end-to-end test.
-
-    Parameters
-    ----------
-    docker_services : Any
-        Test fixture used to start all services from a docker compose file.
-        After test are finished, shutdown all services.
-
-    """
-    s3_client = s3.client(
+def test_s3_engine(docker_services):
+    """S3 engine used to acceptance test."""
+    client = s3.client(
         "s3",
         endpoint_url="http://localhost:9000",
         aws_access_key_id="compose-s3-key",
         aws_secret_access_key="compose-s3-secret",
     )
 
-    s3_engine = EngineS3(
-        s3_client=s3_client,
+    engine = EngineS3(
+        s3_client=client,
         bucket_name="file-storehouse-s3",
         prefix="products",
     )
 
-    s3_engine.ensure_bucket()
+    engine.ensure_bucket()
 
+    run_acceptance(engine)
+
+
+def test_local_engine():
+    """Local engine used to acceptance test."""
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        tmp_path = Path(tmpdirname)
+        engine = EngineLocal(tmp_path)
+        run_acceptance(engine)
+
+
+def run_acceptance(engine: EngineABC) -> None:
+    """User story for an end-to-end test."""
     s3_manager = s3.FileManager(
-        engine=s3_engine,
+        engine=engine,
         transformation_list=[
             TransformationCodecs(),
             TransformationJson(),