feat: python 3.11 migration

Author: Integer-Ctrl

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 WORKDIR /data
 

README.md

@@ -20,8 +20,9 @@ Command-line and Python client for downloading and deploying datasets on DBpedia
   - [Delete](#cli-delete)
 - [Module Usage](#module-usage)
   - [Deploy](#module-deploy)
-- [Contributing](#contributing)
+- [Development & Contributing](#development--contributing)
   - [Linting](#linting)
+  - [Testing](#testing)
 
 
 ## Quickstart
@@ -32,7 +33,7 @@ You can use either **Python** or **Docker**. Both methods support all client fea
 
 ### Python
 
-Requirements: [Python](https://www.python.org/downloads/) and [pip](https://pip.pypa.io/en/stable/installation/)
+Requirements: [Python 3.11+](https://www.python.org/downloads/) and [pip](https://pip.pypa.io/en/stable/installation/)
 
 Before using the client, install it via pip:
 
@@ -186,8 +187,8 @@ Options:
                       e.g. https://databus.dbpedia.org/sparql)
   --vault-token TEXT  Path to Vault refresh token file
   --databus-key TEXT  Databus API key to download from protected databus
-  --latest-only       When downloading artifacts, only download the latest
-                      version
+  --all-versions      When downloading artifacts, download all versions
+                      instead of only the latest
   --authurl TEXT      Keycloak token endpoint URL  [default:
                       https://auth.dbpedia.org/realms/dbpedia/protocol/openid-
                       connect/token]
@@ -557,7 +558,7 @@ from databusclient import deploy
 deploy(dataset, "mysterious API key")
 ```
 
-## Development
+## Development & Contributing
 
 Install development dependencies yourself or via [Poetry](https://python-poetry.org/):
 
@@ -569,14 +570,32 @@ poetry install --with dev
 
 The used linter is [Ruff](https://ruff.rs/). Ruff is configured in `pyproject.toml` and is enforced in CI (`.github/workflows/ruff.yml`).
 
-For development, you can run linting locally with `ruff check . ` and optionally auto-format with `ruff format .`.
+For development, you can run linting locally with `ruff check .` and optionally auto-format with `ruff format .`.
 
-To ensuere compatibility with the `pyproject.toml` configured dependencies, run Ruff via Poetry:
+To ensure compatibility with the `pyproject.toml` configured dependencies, run Ruff via Poetry:
 
 ```bash
 # To check for linting issues:
 poetry run ruff check .
 
 # To auto-format code:
 poetry run ruff format .
+```
+
+### Testing
+
+When developing new features please make sure to add appropriate tests and ensure that all tests pass. Tests are under `tests/` and use [pytest](https://docs.pytest.org/en/7.4.x/) as test framework.
+
+When fixing bugs or refactoring existing code, please make sure to add tests that cover the affected functionality. The current test coverage is very low, so any additional tests are highly appreciated.
+
+To run tests locally, use:
+
+```bash
+pytest tests/
+```
+
+Or to ensure compatibility with the `pyproject.toml` configured dependencies, run pytest via Poetry:
+
+```bash
+poetry run pytest tests/
 ```

databusclient/api/delete.py

@@ -3,7 +3,10 @@
 
 import requests
 
-from databusclient.api.utils import fetch_databus_jsonld, get_databus_id_parts_from_uri
+from databusclient.api.utils import (
+    fetch_databus_jsonld,
+    get_databus_id_parts_from_file_url,
+)
 
 
 def _confirm_delete(databusURI: str) -> str:
@@ -161,7 +164,7 @@ def _delete_group(
         uri = item.get("@id")
         if not uri:
             continue
-        _, _, _, _, version, _ = get_databus_id_parts_from_uri(uri)
+        _, _, _, _, version, _ = get_databus_id_parts_from_file_url(uri)
         if version is None:
             artifact_uris.append(uri)
 
@@ -188,8 +191,8 @@ def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool)
     """
 
     for databusURI in databusURIs:
-        _host, _account, group, artifact, version, file = get_databus_id_parts_from_uri(
-            databusURI
+        _host, _account, group, artifact, version, file = (
+            get_databus_id_parts_from_file_url(databusURI)
         )
 
         if group == "collections" and artifact is not None:

databusclient/api/deploy.py

@@ -5,7 +5,7 @@
 
 import requests
 
-__debug = False
+_debug = False
 
 
 class DeployError(Exception):
@@ -36,6 +36,11 @@ def _get_content_variants(distribution_str: str) -> Optional[Dict[str, str]]:
 
     cvs = {}
     for kv in cv_str.split("_"):
+        if "=" not in kv:
+            raise BadArgumentException(
+                f"Invalid content variant format: '{kv}'. Expected 'key=value' format."
+            )
+
         key, value = kv.split("=")
         cvs[key] = value
 
@@ -141,8 +146,8 @@ def _get_file_stats(distribution_str: str) -> Tuple[Optional[str], Optional[int]
 
 
 def _load_file_stats(url: str) -> Tuple[str, int]:
-    resp = requests.get(url)
-    if resp.status_code > 400:
+    resp = requests.get(url, timeout=30)
+    if resp.status_code >= 400:
         raise requests.exceptions.RequestException(response=resp)
 
     sha256sum = hashlib.sha256(bytes(resp.content)).hexdigest()
@@ -156,7 +161,7 @@ def get_file_info(distribution_str: str) -> Tuple[Dict[str, str], str, str, str,
 
     # content_variant_part = "_".join([f"{key}={value}" for key, value in cvs.items()])
 
-    if __debug:
+    if _debug:
         print("DEBUG", distribution_str, extension_part)
 
     sha256sum, content_length = _get_file_stats(distribution_str)
@@ -306,7 +311,13 @@ def create_dataset(
     """
 
     _versionId = str(version_id).strip("/")
-    _, account_name, group_name, artifact_name, version = _versionId.rsplit("/", 4)
+    parts = _versionId.rsplit("/", 4)
+    if len(parts) < 5:
+        raise BadArgumentException(
+            f"Invalid version_id format: '{version_id}'. "
+            f"Expected format: <BASE>/<ACCOUNT>/<GROUP>/<ARTIFACT>/<VERSION>"
+        )
+    _, _account_name, _group_name, _artifact_name, version = parts
 
     # could be build from stuff above,
     # was not sure if there are edge cases BASE=http://databus.example.org/"base"/...
@@ -428,22 +439,30 @@ def deploy(
 
     headers = {"X-API-KEY": f"{api_key}", "Content-Type": "application/json"}
     data = json.dumps(dataid)
-    base = "/".join(dataid["@graph"][0]["@id"].split("/")[0:3])
+
+    try:
+        base = "/".join(dataid["@graph"][0]["@id"].split("/")[0:3])
+    except (KeyError, IndexError, TypeError) as e:
+        raise DeployError(f"Invalid dataid structure: {e}")
+
     api_uri = (
         base
         + f"/api/publish?verify-parts={str(verify_parts).lower()}&log-level={log_level.name}"
     )
-    resp = requests.post(api_uri, data=data, headers=headers)
+    resp = requests.post(api_uri, data=data, headers=headers, timeout=30)
 
-    if debug or __debug:
-        dataset_uri = dataid["@graph"][0]["@id"]
+    if debug or _debug:
+        try:
+            dataset_uri = dataid["@graph"][0]["@id"]
+        except (KeyError, IndexError, TypeError) as e:
+            raise DeployError(f"Invalid dataid structure: {e}")
         print(f"Trying submitting data to {dataset_uri}:")
         print(data)
 
     if resp.status_code != 200:
         raise DeployError(f"Could not deploy dataset to databus. Reason: '{resp.text}'")
 
-    if debug or __debug:
+    if debug or _debug:
         print("---------")
         print(resp.text)