feat(integration-tests): Add infrastructure for running integration tests; Add basic integration tests for clp & clp-s compression.

.gitignore

@@ -1,3 +1,4 @@
+__pycache__/
 .clang-format
 .clang-tidy
 .lint-venv/

docs/src/dev-docs/building-package.md

@@ -15,6 +15,7 @@ prebuilt version instead, check out the [releases](https://github.com/y-scope/cl
 * python3-venv (for the version of Python installed)
 * [Task] 3.44.0
   * We pin the version to 3.44.0 due to [y-scope/clp-ffi-js#110].
+* [uv] >= 0.8
 
 ## Setup
 
@@ -80,4 +81,5 @@ task docker-images:package
 This will create a Docker image named `clp-package:dev`.
 
 [Task]: https://taskfile.dev/
+[uv]: https://docs.astral.sh/uv/
 [y-scope/clp-ffi-js#110]: https://github.com/y-scope/clp-ffi-js/issues/110

docs/src/dev-docs/components-core/index.md

@@ -14,6 +14,7 @@ CLP core is the low-level component that performs compression, decompression, an
   * We constrain the version to < 4.0.0 due to [y-scope/clp#795].
 * [Task] 3.44.0
   * We pin the version to 3.44.0 due to [y-scope/clp-ffi-js#110].
+* [uv] >= 0.8
 
 To build, we require some source dependencies, packages from package managers, and libraries built
 from source.
@@ -113,6 +114,12 @@ the relevant paths on your machine.
   make -j
   ```
 
+## Test
+
+```shell
+task tests:integration:core
+```
+
 :::{toctree}
 :hidden:
 
@@ -125,6 +132,7 @@ regex-utils
 [CMake]: https://cmake.org/
 [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&template=feature-request.yml
 [Task]: https://taskfile.dev/
+[uv]: https://docs.astral.sh/uv/
 [y-scope/clp-ffi-js#110]: https://github.com/y-scope/clp-ffi-js/issues/110
 [y-scope/clp#795]: https://github.com/y-scope/clp/issues/795
 [yscope-dev-utils]: https://github.com/y-scope/yscope-dev-utils

docs/src/dev-docs/index.md

@@ -15,6 +15,13 @@ Building
 Docs about building CLP.
 :::
 
+:::{grid-item-card}
+:link: testing/index
+Testing
+^^^
+Docs about testing CLP.
+:::
+
 :::{grid-item-card}
 :link: contributing-getting-started
 Contributing
@@ -50,6 +57,15 @@ Any design docs describing parts of this project.
 building-package
 :::
 
+:::{toctree}
+:caption: Testing
+:hidden:
+
+testing/index
+testing/unit-tests
+testing/integration-tests
+:::
+
 :::{toctree}
 :caption: Contributing
 :hidden:

docs/src/dev-docs/testing/index.md

@@ -0,0 +1,19 @@
+# Overview
+
+::::{grid} 1 1 1 1
+:gutter: 2
+
+:::{grid-item-card}
+:link: unit-tests
+Unit tests
+^^^
+Docs about running unit tests for each component.
+:::
+
+:::{grid-item-card}
+:link: integration-tests
+Integration tests
+^^^
+Docs about running CLP's integration tests.
+:::
+::::

docs/src/dev-docs/testing/integration-tests.md

@@ -0,0 +1,48 @@
+# Integration tests
+
+The `integration-tests` directory contains a Python project that provides end-to-end tests for
+CLP via the `pytest` framework.
+
+## Running tests
+
+To run all integration tests:
+
+```shell
+task tests:integration
+```
+
+To test the core CLP binaries:
+
+```shell
+task tests:integration:core
+```
+
+### Using `pytest` markers
+
+To run more specific sets of tests, you can use `pytest` directly with `pytest` markers.
+
+:::{note}
+Before running tests using `pytest`, ensure that the CLP package and/or core binaries have been
+built.
+:::
+
+Ensure all commands below are run from inside the `integration-tests` directory.
+
+To list all available markers:
+
+```shell
+uv run python -m pytest --markers
+```
+
+To run tests related to a specific marker (e.g., `clp_s`):
+
+```shell
+uv run python -m pytest -m clp_s
+```
+
+### Specifying custom CLP binary paths
+
+You can override the default binary paths by setting the following environment variables:
+
+* **`CLP_CORE_BINS_DIR`**: Directory containing the CLP core binaries to test.
+* **`CLP_PACKAGE_DIR`**: Directory of the CLP package to test.

docs/src/dev-docs/testing/unit-tests.md

@@ -0,0 +1,5 @@
+# Unit tests
+
+:::{warning}
+🚧 This section is under construction.
+:::

integration-tests/.pytest.ini

@@ -0,0 +1,20 @@
+[pytest]
+addopts =
+    --capture=no
+    --code-highlight=yes
+    --color=yes
+    --strict-config
+    --strict-markers
+    --verbose
+env =
+    D:CLP_BUILD_DIR=../build
+    D:CLP_CORE_BINS_DIR=../build/core
+    D:CLP_PACKAGE_DIR=../build/clp-package
+log_cli = True
+log_cli_date_format = %Y-%m-%d %H:%M:%S,%f
+log_cli_format = %(name)s %(asctime)s [%(levelname)s] %(message)s
+log_cli_level = INFO
+markers =
+    clp: mark tests that use the CLP storage engine
+    clp_s: mark tests that use the CLP-S storage engine
+    core: mark tests that test the CLP core binaries

integration-tests/.python-version

@@ -0,0 +1 @@
+3.9

integration-tests/README.md

@@ -0,0 +1,7 @@
+# CLP integration tests
+
+This Python project provides end-to-end tests for CLP via the `pytest` framework.
+
+For more information, see the [integration test docs][integration-test-docs].
+
+[integration-test-docs]: https://docs.yscope.com/clp/main/dev-docs/testing/integration-tests

integration-tests/pyproject.toml

@@ -0,0 +1,71 @@
+[project]
+name = "integration-tests"
+version = "0.1.0"
+description = "Integration tests for CLP."
+readme = "README.md"
+authors = [
+    { name = "YScope Inc.", email = "dev@yscope.com" }
+]
+requires-python = ">=3.9"
+
+[project.scripts]
+integration-tests = "integration_tests:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "mypy>=1.16.0",
+    "ruff>=0.11.12",
+    "pytest>=8.4.1",
+    "pytest-env>=1.1.5",
+]
+
+[tool.mypy]
+strict = true
+
+# Additional output
+pretty = true
+show_error_code_links = true
+show_error_context = true
+show_error_end = true
+
+[tool.ruff]
+line-length = 100
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "ANN401",  # Allow using `Any` type for function signatures
+    "COM812",  # Redundant and conflicts with ruff format
+    "D203",  # No blank line before docstrings (D211)
+    "D205",  # Breaks if summary is larger than one line due to wrapping or if no summary exists
+    "D212",  # Enforce docstring summary line on the next line after quotes (D213)
+    "D400",  # First line of docstrings may not end in period
+    "D401",  # Docstrings should be written in present tense (not imperative)
+    "D415",  # First line of docstrings may not end in a period, question mark, or exclamation point
+    "FBT",  # Allow bool positional parameters since other value positions are allowed
+    "FIX002",  # Allow todo statements
+    "PERF401",  # Allow for loops when creating lists
+    "PERF403",  # Allow for loops when creating dicts
+    "S311",  # Allow usage of `random` package
+    "S603",  # Automatically trust inputs of subprocess execution
+    "SIM102",  # Allow collapsible if statements for readability
+    "SIM300",  # Skip Yoda-condition format fixes
+    "TD002",  # Author unnecessary for todo statement
+    "TD003",  # Issue link unnecessary for todo statement
+    "UP015",  # Explicit open modes are helpful
+]
+isort.order-by-type = false
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = [
+    "S101",  # Allow usage of pytest `assert`
+    "TC003",  # Ignore performance overhead of imports only used for type checking
+]
+
+[tool.ruff.format]
+docstring-code-format = true
+docstring-code-line-length = 100

integration-tests/src/integration_tests/__init__.py

@@ -0,0 +1 @@
+"""Integration test package for CLP."""

integration-tests/tests/__init__.py

@@ -0,0 +1 @@
+"""Top-level package for CLP integration tests."""

integration-tests/tests/conftest.py

@@ -0,0 +1,6 @@
+"""Make the fixtures defined in `tests/fixtures/` globally available without imports."""
+
+pytest_plugins = [
+    "tests.fixtures.integration_test_config",
+    "tests.fixtures.integration_test_logs",
+]

integration-tests/tests/fixtures/__init__.py

@@ -0,0 +1 @@
+"""Fixtures for CLP integration tests."""

integration-tests/tests/fixtures/integration_test_config.py

@@ -0,0 +1,29 @@
+"""Define the integration test configuration fixture."""
+
+from pathlib import Path
+
+import pytest
+
+from tests.utils.config import (
+    CoreConfig,
+    IntegrationTestConfig,
+    PackageConfig,
+)
+from tests.utils.utils import get_env_var
+
+
+@pytest.fixture(scope="session")
+def integration_test_config() -> IntegrationTestConfig:
+    """Fixture that provides an IntegrationTestConfig shared across tests."""
+    core_config = CoreConfig(
+        clp_core_bins_dir=Path(get_env_var("CLP_CORE_BINS_DIR")).expanduser().resolve()
+    )
+    package_config = PackageConfig(
+        clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).expanduser().resolve()
+    )
+    test_root_dir = Path(get_env_var("CLP_BUILD_DIR")).expanduser().resolve() / "integration-tests"
+    return IntegrationTestConfig(
+        core_config=core_config,
+        package_config=package_config,
+        test_root_dir=test_root_dir,
+    )

integration-tests/tests/fixtures/integration_test_logs.py

@@ -0,0 +1,96 @@
+"""Define test logs fixtures."""
+
+import logging
+import shutil
+import subprocess
+
+import pytest
+
+from tests.utils.config import (
+    IntegrationTestConfig,
+    IntegrationTestLogs,
+)
+from tests.utils.utils import unlink
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture(scope="session")
+def hive_24hr(
+    request: pytest.FixtureRequest,
+    integration_test_config: IntegrationTestConfig,
+) -> IntegrationTestLogs:
+    """Fixture that provides `hive_24hr` test logs shared across tests."""
+    return _download_and_extract_dataset(
+        request=request,
+        integration_test_config=integration_test_config,
+        name="hive-24hr",
+        tarball_url="https://zenodo.org/records/7094921/files/hive-24hr.tar.gz?download=1",
+    )
+
+
+@pytest.fixture(scope="session")
+def postgresql(
+    request: pytest.FixtureRequest,
+    integration_test_config: IntegrationTestConfig,
+) -> IntegrationTestLogs:
+    """Fixture that provides `postgresql` test logs shared across tests."""
+    return _download_and_extract_dataset(
+        request=request,
+        integration_test_config=integration_test_config,
+        name="postgresql",
+        tarball_url="https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1",
+    )
+
+
+def _download_and_extract_dataset(
+    request: pytest.FixtureRequest,
+    integration_test_config: IntegrationTestConfig,
+    name: str,
+    tarball_url: str,
+) -> IntegrationTestLogs:
+    integration_test_logs = IntegrationTestLogs(
+        name=name,
+        tarball_url=tarball_url,
+        integration_test_config=integration_test_config,
+    )
+    if request.config.cache.get(name, False):
+        logger.info("Test logs `%s` are up-to-date. Skipping download.", name)
+        return integration_test_logs
+
+    curl_bin = shutil.which("curl")
+    if curl_bin is None:
+        err_msg = "curl executable not found"
+        raise RuntimeError(err_msg)
+
+    try:
+        # fmt: off
+        curl_cmds = [
+            curl_bin,
+            "--fail",
+            "--location",
+            "--output", str(integration_test_logs.tarball_path),
+            "--show-error",
+            tarball_url,
+        ]
+        # fmt: on
+        subprocess.run(curl_cmds, check=True)
+
+        unlink(integration_test_logs.extraction_dir)
+        shutil.unpack_archive(
+            integration_test_logs.tarball_path, integration_test_logs.extraction_dir
+        )
+    except Exception as e:
+        err_msg = f"Failed to download and extract dataset `{name}`."
+        raise RuntimeError(err_msg) from e
+
+    # Allow the extracted content to be deletable or overwritable
+    chmod_bin = shutil.which("chmod")
+    if chmod_bin is None:
+        err_msg = "chmod executable not found"
+        raise RuntimeError(err_msg)
+    subprocess.run([chmod_bin, "-R", "gu+w", integration_test_logs.extraction_dir], check=True)
+
+    logger.info("Downloaded and extracted uncompressed logs for dataset `%s`.", name)
+    request.config.cache.set(name, True)
+    return integration_test_logs