test: Set up minio-based testing, replace moto (#553)

* Add docker and minio dev deps

* Set up minio conftest

* set up conftest

* Fix minio conftest

* fix tests in test_s3

* passing tests!

* remove moto

* Remove old conftest

Author: kylebarron

pyproject.toml

@@ -13,6 +13,7 @@ dev-dependencies = [
     "arro3-core>=0.4.2",
     "azure-identity>=1.21.0",
     "boto3>=1.38.21",
+    "docker>=7.1.0",
     "fastapi>=0.115.12",                  # used in example but added here for pyright CI
     "fsspec>=2024.10.0",
     "google-auth>=2.38.0",
@@ -22,12 +23,12 @@ dev-dependencies = [
     "maturin-import-hook>=0.2.0",
     "maturin>=1.7.4",
     "mike>=2.1.3",
+    "minio>=7.2.16",
     "mkdocs-material[imaging]>=9.6.3",
     "mkdocs-redirects>=1.2.2",
     "mkdocs>=1.6.1",
     "mkdocstrings-python>=1.13.0",
     "mkdocstrings>=0.27.0",
-    "moto[s3,server]>=5.1.1",
     "mypy>=1.15.0",
     "obspec>=0.1.0",
     "pip>=24.2",
@@ -93,14 +94,10 @@ ignore = [
 ]
 
 [tool.pyright]
-exclude = [
-    "**/__pycache__",
-    "examples",
-    ".venv",
-]
+exclude = ["**/__pycache__", "examples", ".venv"]
 executionEnvironments = [
-  { root = "./", extraPaths = ["./obstore/python"] },  # Tests.
-  { root = "./obstore/python" }
+    { root = "./", extraPaths = ["./obstore/python"] }, # Tests.
+    { root = "./obstore/python" },
 ]
 
 [tool.pytest.ini_options]

tests/conftest.py

@@ -1,75 +1,147 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+import socket
+import time
+import warnings
+from typing import TYPE_CHECKING, Any
 
-import boto3
+import docker
 import pytest
 import requests
-from botocore import UNSIGNED
-from botocore.client import Config
-from moto.moto_server.threaded_moto_server import ThreadedMotoServer
+from minio import Minio
+from requests.exceptions import RequestException
 
 from obstore.store import S3Store
 
 if TYPE_CHECKING:
-    from obstore.store import S3Config
+    from collections.abc import Generator
 
-TEST_BUCKET_NAME = "test"
+    from obstore.store import ClientConfig, S3Config
+
+TEST_BUCKET_NAME = "test-bucket"
+
+
+def find_available_port() -> int:
+    """Find a free port on localhost.
+
+    Note that this is susceptible to race conditions.
+    """
+    # https://stackoverflow.com/a/36331860
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        # Bind to a free port provided by the host.
+        s.bind(("", 0))
+
+        # Return the port number assigned.
+        return s.getsockname()[1]
 
 
-# See docs here: https://docs.getmoto.org/en/latest/docs/server_mode.html
 @pytest.fixture(scope="session")
-def moto_server_uri():
-    """Fixture to run a mocked AWS server for testing."""
-    # Note: pass `port=0` to get a random free port.
-    server = ThreadedMotoServer(ip_address="localhost", port=0)
-    server.start()
-    if hasattr(server, "get_host_and_port"):
-        host, port = server.get_host_and_port()
-    else:
-        s = server._server
-        assert s is not None
-        # An AF_INET6 socket address has 4 components.
-        host, port = s.server_address[:2]
-    uri = f"http://{host}:{port}"
-    yield uri
-    server.stop()
+def minio_config() -> Generator[tuple[S3Config, ClientConfig], Any, None]:
+    warnings.warn(
+        "Creating Docker client...",
+        UserWarning,
+        stacklevel=1,
+    )
+    docker_client = docker.from_env()
+    warnings.warn(
+        "Finished creating Docker client...",
+        UserWarning,
+        stacklevel=1,
+    )
 
+    username = "minioadmin"
+    password = "minioadmin"  # noqa: S105
+    port = find_available_port()
+    console_port = find_available_port()
 
-@pytest.fixture
-def s3(moto_server_uri: str):
-    client = boto3.client(
-        "s3",
-        config=Config(signature_version=UNSIGNED),
-        region_name="us-east-1",
-        endpoint_url=moto_server_uri,
+    print(f"Using ports: {port=}, {console_port=}")  # noqa: T201
+    print(  # noqa: T201
+        f"Log on to MinIO console at http://localhost:{console_port} with "
+        f"{username=} and {password=}",
     )
-    client.create_bucket(Bucket=TEST_BUCKET_NAME, ACL="public-read")
-    client.put_object(Bucket=TEST_BUCKET_NAME, Key="afile", Body=b"hello world")
-    yield moto_server_uri
-    objects = client.list_objects_v2(Bucket=TEST_BUCKET_NAME)
-    for name in objects.get("Contents", []):
-        key = name.get("Key")
-        assert key is not None
-        client.delete_object(Bucket=TEST_BUCKET_NAME, Key=key)
-    requests.post(f"{moto_server_uri}/moto-api/reset", timeout=30)
 
+    warnings.warn(
+        "Starting MinIO container...",
+        UserWarning,
+        stacklevel=1,
+    )
+    minio_container = docker_client.containers.run(
+        "quay.io/minio/minio",
+        "server /data --console-address :9001",
+        detach=True,
+        ports={
+            "9000/tcp": port,
+            "9001/tcp": console_port,
+        },
+        environment={
+            "MINIO_ROOT_USER": username,
+            "MINIO_ROOT_PASSWORD": password,
+        },
+    )
+    warnings.warn(
+        "Finished starting MinIO container...",
+        UserWarning,
+        stacklevel=1,
+    )
 
-@pytest.fixture
-def s3_store(s3: str):
-    return S3Store.from_url(
-        f"s3://{TEST_BUCKET_NAME}/",
-        endpoint=s3,
-        region="us-east-1",
-        skip_signature=True,
-        client_options={"allow_http": True},
+    # Wait for MinIO to be ready
+    endpoint = f"http://localhost:{port}"
+    wait_for_minio(endpoint, timeout=30)
+
+    minio_client = Minio(
+        f"localhost:{port}",
+        access_key=username,
+        secret_key=password,
+        secure=False,
     )
+    minio_client.make_bucket(TEST_BUCKET_NAME)
+
+    s3_config: S3Config = {
+        "bucket": TEST_BUCKET_NAME,
+        "endpoint": endpoint,
+        "access_key_id": username,
+        "secret_access_key": password,
+        "virtual_hosted_style_request": False,
+    }
+    client_options: ClientConfig = {"allow_http": True}
+
+    yield (s3_config, client_options)
+
+    minio_container.stop()
+    minio_container.remove()
 
 
 @pytest.fixture
-def s3_store_config(s3: str) -> S3Config:
-    return {
-        "endpoint": s3,
-        "region": "us-east-1",
-        "skip_signature": True,
-    }
+def minio_bucket(
+    minio_config: tuple[S3Config, ClientConfig],
+) -> Generator[tuple[S3Config, ClientConfig], Any, None]:
+    yield minio_config
+
+    # Remove all files from bucket
+    store = S3Store(config=minio_config[0], client_options=minio_config[1])
+    objects = store.list().collect()
+    paths = [obj["path"] for obj in objects]
+    store.delete(paths)
+
+
+@pytest.fixture
+def minio_store(minio_bucket: tuple[S3Config, ClientConfig]) -> S3Store:
+    """Create an S3Store configured for MinIO integration testing."""
+    return S3Store(config=minio_bucket[0], client_options=minio_bucket[1])
+
+
+def wait_for_minio(endpoint: str, timeout: int):
+    start_time = time.time()
+    while time.time() - start_time < timeout:
+        try:
+            # MinIO health check endpoint
+            response = requests.get(f"{endpoint}/minio/health/live", timeout=2)
+            if response.status_code == 200:
+                return
+        except RequestException:
+            pass
+        time.sleep(0.5)
+
+    exc_str = f"MinIO failed to start within {timeout} seconds"
+    raise TimeoutError(exc_str)

tests/store/test_s3.py

@@ -6,7 +6,6 @@
 
 import pytest
 
-import obstore as obs
 from obstore.exceptions import BaseError, UnauthenticatedError
 from obstore.store import S3Store, from_url
 
@@ -16,8 +15,10 @@
     reason="Moto doesn't seem to support Python 3.9",
 )
 @pytest.mark.asyncio
-async def test_list_async(s3_store: S3Store):
-    list_result = await obs.list(s3_store).collect_async()
+async def test_list_async(minio_store: S3Store):
+    await minio_store.put_async("afile", b"hello world")
+
+    list_result = await minio_store.list().collect_async()
     assert any("afile" in x["path"] for x in list_result)
 
 
@@ -26,8 +27,10 @@ async def test_list_async(s3_store: S3Store):
     reason="Moto doesn't seem to support Python 3.9",
 )
 @pytest.mark.asyncio
-async def test_get_async(s3_store: S3Store):
-    resp = await obs.get_async(s3_store, "afile")
+async def test_get_async(minio_store: S3Store):
+    await minio_store.put_async("afile", b"hello world")
+
+    resp = await minio_store.get_async("afile")
     buf = await resp.bytes_async()
     assert buf == b"hello world"
 
@@ -78,7 +81,7 @@ async def test_from_url():
         region="us-west-2",
         skip_signature=True,
     )
-    _meta = await obs.head_async(store, "2024-01-01_performance_fixed_tiles.parquet")
+    _meta = await store.head_async("2024-01-01_performance_fixed_tiles.parquet")
 
 
 def test_pickle():
@@ -88,7 +91,7 @@ def test_pickle():
         skip_signature=True,
     )
     restored = pickle.loads(pickle.dumps(store))
-    _objects = next(obs.list(restored))
+    _objects = next(restored.list())
 
 
 def test_config_round_trip():
@@ -120,7 +123,7 @@ def credential_provider():
 
     store = S3Store("bucket", credential_provider=credential_provider)  # type: ignore
     with pytest.raises(UnauthenticatedError):
-        obs.list(store).collect()
+        store.list().collect()
 
 
 @pytest.mark.asyncio
@@ -130,7 +133,7 @@ async def credential_provider():
 
     store = S3Store("bucket", credential_provider=credential_provider)  # type: ignore
     with pytest.raises(UnauthenticatedError):
-        await obs.list(store).collect_async()
+        await store.list().collect_async()
 
 
 def test_eq():