feat: add curl command to logging

closes: #42 

Co-authored-by: Christina Ludwig <christina.ludwig@uni-heidelberg.de>

Author: SlowMo24

CHANGELOG.md

@@ -5,6 +5,7 @@
  - support for python 3.12
  - custom [retry](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry) configuration
  - start and end timestamp meta information of the client are now datetime objects
+ - if a request fails a bash script containing the respective `curl` command is logged (if possible). This allows for easier debugging and sharing of failed requests.
 
 ### Removed
 

ohsome/exceptions.py

@@ -7,6 +7,8 @@
 import json
 from pathlib import Path
 
+from curlify2 import Curlify
+
 
 class OhsomeException(Exception):
     """Exception to handle ohsome API errors"""
@@ -33,8 +35,17 @@ def log(self, log_dir: Path):
         self.log_bpolys(log_dir, log_file_name)
         self.log_parameter(log_dir, log_file_name)
         if self.response is not None:
+            self.log_curl(log_dir, log_file_name)
             self.log_response(log_dir, log_file_name)
 
+    def log_curl(self, log_dir: Path, log_file_name: str) -> None:
+        """Log the respective curl command for the request for easy debugging and sharing."""
+        log_file = log_dir / f"{log_file_name}_curl.sh"
+        curl = Curlify(self.response.request)
+        curl_command = curl.to_curl()
+        with log_file.open(mode="w") as dst:
+            dst.write(curl_command)
+
     def log_response(self, log_dir: Path, log_file_name: str):
         """
         Log raw response. This may duplicate much data but is helpful for debugging to know the exact raw answer by the

ohsome/test/cassettes/test_exceptions/test_log_curl.yaml

@@ -0,0 +1,57 @@
+interactions:
+- request:
+    body: bboxes=8.67555%2C49.39885%2C8.69637%2C49.41122&timeout=0.001
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '60'
+      Content-Type:
+      - application/x-www-form-urlencoded
+      user-agent:
+      - ohsome-py/0.2.0
+    method: POST
+    uri: https://api.ohsome.org/v1/elements/count
+  response:
+    body:
+      string: "{\n  \"timestamp\" : \"2023-11-17T12:28:28.06766606\",\n  \"status\"
+        : 413,\n  \"message\" : \"The given query is too large in respect to the given
+        timeout. Please use a smaller region and/or coarser time period.\",\n  \"requestUrl\"
+        : \"https://api.ohsome.org/v1/elements/count\"\n}"
+    headers:
+      Access-Control-Allow-Credentials:
+      - 'true'
+      Access-Control-Allow-Headers:
+      - Origin,Accept,X-Requested-With,Content-Type,Access-Control-Request-Method,Access-Control-Request-Headers,Authorization
+      Access-Control-Allow-Methods:
+      - POST, GET
+      Access-Control-Allow-Origin:
+      - '*'
+      Access-Control-Max-Age:
+      - '3600'
+      Cache-Control:
+      - no-cache, no-store, must-revalidate
+      Connection:
+      - close
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 17 Nov 2023 12:28:27 GMT
+      Server:
+      - Apache
+      Strict-Transport-Security:
+      - max-age=63072000; includeSubdomains;
+      Transfer-Encoding:
+      - chunked
+      vary:
+      - accept-encoding
+    status:
+      code: 413
+      message: ''
+version: 1

ohsome/test/test_exceptions.py

@@ -118,14 +118,46 @@ def test_log_bpolys(base_client_without_log, tmpdir):
         base_client_without_log.elements.count.post(
             bpolys=bpolys, time=time, filter=fltr, timeout=timeout
         )
-    log_file_patterns = ["ohsome_*_bpolys.geojson", "ohsome_*.json", "ohsome_*raw.txt"]
+    log_file_patterns = [
+        "ohsome_*_bpolys.geojson",
+        "ohsome_*_curl.sh",
+        "ohsome_*.json",
+        "ohsome_*raw.txt",
+    ]
     for p in log_file_patterns:
         log_file = list(Path(base_client_without_log.log_dir).glob(p))
         assert len(log_file) == 1, f"Log file {p} not found"
         logger.info(f"Found log file: {log_file[0].name}")
         log_file[0].unlink()
 
 
+@pytest.mark.vcr
+def test_log_curl(base_client_without_log, tmpdir):
+    """
+    Test whether log file containing curl command is created when request fails
+    :return:
+    """
+
+    base_client_without_log.log = True
+    base_client_without_log.log_dir = tmpdir.mkdir("logs").strpath
+
+    bboxes = [8.67555, 49.39885, 8.69637, 49.41122]
+    timeout = 0.001
+
+    with pytest.raises(ohsome.OhsomeException):
+        base_client_without_log.elements.count.post(bboxes=bboxes, timeout=timeout)
+
+    log_file = list(Path(base_client_without_log.log_dir).glob("ohsome_*_curl.sh"))
+    with open(log_file[0]) as file:
+        assert file.read() == (
+            'curl -X POST -H "user-agent: ohsome-py/0.2.0" -H "Accept-Encoding: gzip, '
+            'deflate" -H "Accept: */*" -H "Connection: keep-alive" -H "Content-Length: 60" '
+            '-H "Content-Type: application/x-www-form-urlencoded" '
+            "-d 'bboxes=8.67555%2C49.39885%2C8.69637%2C49.41122&timeout=0.001' "
+            "https://api.ohsome.org/v1/elements/count"
+        )
+
+
 def test_metadata_invalid_baseurl(custom_client_with_wrong_url):
     """
     Throw exception if the ohsome API is not available

poetry.lock

@@ -14,6 +14,7 @@ pandas = "^2.1.3"
 numpy = "^1.20.0"
 geopandas = "^0.14.1"
 urllib3 = "^2.1.0"
+curlify2 = "^2.0.0"
 
 [tool.poetry.group.test.dependencies]
 pytest = "^7.4.3"