Feat-filterable-aretry (#30)

Nacho Maiz · web-flow · commit e5c672706b1a · 2024-01-04T16:40:41.000Z
* added retry functionality to handshake

* simplified query ensure method to avoid a copy

* fix on_errors docstring

* fix import order

* changed back parameters title to attributes

* added test to check exception type handling

* changed page names and added automatic color scheme

* changed new flag to use minor versions

* added links to dependencies

* changed title

* moved batched behavior section to the end

* removed ssl warning and reordered sections

* summarized changes for 1.0

* version bump and stable tag

* added note about reaching stable

* added links to endpoint doc pages

* updated release date formats
diff --git a/bavapi/__init__.py b/bavapi/__init__.py
@@ -42,8 +42,8 @@
     collections,
     companies,
     countries,
-    regions,
     raw_query,
+    regions,
     sectors,
     studies,
     years,
diff --git a/bavapi/_fetcher.py b/bavapi/_fetcher.py
@@ -19,6 +19,7 @@
 import functools
 import warnings
 from typing import (
+    Any,
     Callable,
     Coroutine,
     Generic,
@@ -28,6 +29,7 @@
     NamedTuple,
     Optional,
     Protocol,
+    Type,
     TypeVar,
 )
 
@@ -37,7 +39,7 @@
 Q = TypeVar("Q", bound="_Query")
 P = ParamSpec("P")
 
-AsyncCallable = Callable[P, Coroutine[None, None, T]]
+AsyncCallable = Callable[P, Coroutine[Any, Any, T]]
 
 
 class _Query(Protocol):
@@ -196,7 +198,10 @@ def warn_if_errors(self) -> None:
 
 
 def aretry(
-    func: AsyncCallable[P, T], retries: int = 3, delay: float = 0
+    func: AsyncCallable[P, T],
+    retries: int = 3,
+    delay: float = 0,
+    exc_types: Iterable[Type[Exception]] = (Exception,),
 ) -> AsyncCallable[P, T]:
     """Retry an asynchronous function upon failure a number of times.
 
@@ -212,6 +217,9 @@ def aretry(
         If the value is 0 or negative, the function will only be tried once
     delay : float, optional
         Time to wait between retries, default 0
+    exc_types: Iterable[type[Exception]], optional
+        Exception types to retry on, default (Exception,)
+
     Returns
     -------
     AsyncCallable[P, T]
@@ -229,7 +237,7 @@ async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
         for retry_count in range(retries + 1):
             try:
                 return await func(*args, **kwargs)
-            except Exception as exc:
+            except exc_types as exc:
                 if retry_count == retries:
                     raise exc
                 await asyncio.sleep(delay)
diff --git a/bavapi/client.py b/bavapi/client.py
@@ -270,7 +270,7 @@ def retries(self, value: int) -> None:
 
     @property
     def on_errors(self) -> Literal["warn", "raise"]:
-        """Number of times to retry a request."""
+        """Raise immediately on errors or warn at the end of the request."""
         return self._client.on_errors
 
     @on_errors.setter
diff --git a/bavapi/filters.py b/bavapi/filters.py
@@ -61,7 +61,7 @@ class FountFilters(BaseModel):
 
     Can be used with `raw_query` endpoints.
 
-    Parameters
+    Attributes
     ----------
     updated_since : str, date or datetime, optional
         Request items that have been updated since the specified date, default None
@@ -130,7 +130,7 @@ class AudiencesFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/audiences>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     active : Literal[0, 1], optional
         Return active audiences only if set to `1`, default 0
@@ -165,7 +165,7 @@ class BrandsFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/brands>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     country_codes : str or list[str], optional
         Two-letter ISO-3166 country code or list of country codes, default None
@@ -208,7 +208,7 @@ class BrandMetricsFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/brand-metrics>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     active : Literal[0, 1], optional
         Return active brand metrics when set to `1`, default 0
@@ -252,7 +252,7 @@ class BrandMetricGroupsFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/brand-metric-groups>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     active : Literal[0, 1], optional
         Return active brand metrics when set to `1`, default 0
@@ -288,7 +288,7 @@ class BrandscapeFilters(FountFilters):
 
     The `Audiences` class is provided to make it easier to filter audiences.
 
-    Parameters
+    Attributes
     ----------
     country_code : str or list[str], optional
         Two-letter ISO-3166 country code or list of country codes, default None
@@ -370,7 +370,7 @@ class CategoriesFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/categories>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     sector : int or list[int], optional
         Fount sector ID or list of sector IDs, default None
@@ -390,7 +390,7 @@ class CitiesFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/cities>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     capitals: Literal[0, 1], optional
         Return capitals only, default 0
@@ -421,7 +421,7 @@ class CompaniesFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/companies>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     public: Literal[0, 1], optional
         Return public (listed) companies only, default 0
@@ -449,7 +449,7 @@ class CountriesFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/countries>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     active: Literal[0, 1], optional
         Return active countries only, default 0
@@ -480,7 +480,7 @@ class CollectionsFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/collections>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     public : Literal[0, 1], optional
         Return public collections only, default 0
@@ -521,7 +521,7 @@ class SectorsFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/sectors>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     in_most_influential : Literal[0, 1], optional
         Sectors that are part of the Most Influential lists, default 0
@@ -546,7 +546,7 @@ class StudiesFilters(FountFilters):
     See <https://developer.wppbav.com/docs/2.x/core-resources/studies>
     for more info.
 
-    Parameters
+    Attributes
     ----------
     country_codes : str or list[str], optional
         Two-letter ISO-3166 country code or list of country codes, default None
diff --git a/bavapi/http.py b/bavapi/http.py
@@ -4,6 +4,7 @@
 
 import asyncio
 import math
+import ssl
 from json import JSONDecodeError
 from typing import (
     TYPE_CHECKING,
@@ -296,7 +297,8 @@ async def query(
         """
         per_page = query.per_page or self.per_page
         init_per_page = per_page if query.is_single_page() else 1
-        resp = await self.get(endpoint, query.with_page(per_page=init_per_page))
+        handshake_func = aretry(self.get, self.retries, 0.25, (ssl.SSLError,))
+        resp = await handshake_func(endpoint, query.with_page(per_page=init_per_page))
 
         payload: Dict[str, JSONData] = resp.json()
         data: JSONData = payload["data"]
diff --git a/bavapi/query.py b/bavapi/query.py
@@ -242,13 +242,14 @@ def ensure(
         Query
             `Query` class with additional parameters added if any
         """
-        params = {k: v for k, v in kwargs.items() if v}
+        params: MutableMapping[str, QueryParamValues[F]] = {
+            k: v for k, v in kwargs.items() if v
+        }
 
         if query is None:
             return cls(**params)  # type: ignore[arg-type]
 
-        new_params = cast(MutableMapping[str, QueryParamValues[F]], params.copy())
-        new_params.update(query.model_dump(exclude={"filters"}, exclude_defaults=True))
-        new_params.update({"filters": query.filters})  # type: ignore[arg-type]
+        params.update(query.model_dump(exclude={"filters"}, exclude_defaults=True))
+        params.update({"filters": query.filters})
 
-        return cls(**new_params)  # type: ignore[arg-type]
+        return cls(**params)  # type: ignore[arg-type]
diff --git a/docs/endpoints/cities.md b/docs/endpoints/cities.md
@@ -1,6 +1,6 @@
 # Cities
 
-!!! abstract "New in `v1.0.0`"
+!!! abstract "New in `v1.0`"
 
 The `cities` endpoint has full support, including query validation.
 
diff --git a/docs/endpoints/companies.md b/docs/endpoints/companies.md
@@ -1,6 +1,6 @@
 # Companies
 
-!!! abstract "New in `v1.0.0`"
+!!! abstract "New in `v1.0`"
 
 The `companies` endpoint has full support, including query validation.
 
diff --git a/docs/endpoints/countries.md b/docs/endpoints/countries.md
@@ -1,6 +1,6 @@
 # Countries
 
-!!! abstract "New in `v1.0.0`"
+!!! abstract "New in `v1.0`"
 
 The `countries` endpoint has full support, including query validation.
 
diff --git a/docs/endpoints/regions.md b/docs/endpoints/regions.md
@@ -1,6 +1,6 @@
 # Regions
 
-!!! abstract "New in `v1.0.0`"
+!!! abstract "New in `v1.0`"
 
 The `regions` endpoint has full support, including query validation.
 
diff --git a/docs/endpoints/years.md b/docs/endpoints/years.md
@@ -1,6 +1,6 @@
 # Years
 
-!!! abstract "New in `v1.0.0`"
+!!! abstract "New in `v1.0`"
 
 The `years` endpoint has full support, including query validation.
 
diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md
@@ -14,18 +14,18 @@ Once you have installed Python and have acquired your Fount API token, return to
 
 `bavapi` depends on the following excellent libraries:
 
-- `httpx` for communication with the Fount API.
-- `pandas` for processing retrieved data into tables.
-- `pydantic` to validate query and filter parameters.
-- `nest-asyncio` to support Jupyter notebooks.
-- `tqdm` to show helpful progress bars.
-- `typing-extensions` for type-checking compatibility in Python < 3.12.
+- [`httpx`](https://www.python-httpx.org/) for communication with the Fount API.
+- [`pandas`](https://pandas.pydata.org/docs/index.html) for processing retrieved data into tables.
+- [`pydantic`](https://docs.pydantic.dev/latest/) to validate query and filter parameters.
+- [`nest-asyncio`](https://github.com/erdewit/nest_asyncio) to support Jupyter notebooks.
+- [`tqdm`](https://tqdm.github.io/) to show helpful progress bars.
+- [`typing-extensions`](https://typing-extensions.readthedocs.io/en/latest/) for type-checking compatibility in Python < 3.12.
 
 These libraries will be installed automatically when you install `bavapi`.
 
 ## Installing `bavapi`
 
-Once you have your virtual environment activated, you can install `bavapi` with the following command:
+Once you have your virtual (or conda) environment activated, you can install `bavapi` with the following command:
 
 ```prompt
 pip install wpp-bavapi
@@ -34,7 +34,7 @@ pip install wpp-bavapi
 !!! tip "Installing with `conda`"
     `bavapi` is not currently available from `conda` directly, though it should be possible to install and use it within a `conda` environment.
 
-    Use the following commands to maximize compatibility between `conda` and `pip`:
+    You can use the following commands to maximize compatibility between `conda` and `pip`:
 
     ```prompt
     conda install httpx, pandas, pydantic, nest-asyncio, tqdm, typing_extensions
diff --git a/docs/getting-started/reference-classes.md b/docs/getting-started/reference-classes.md
@@ -1,4 +1,4 @@
-# Installing `bavapi` *Reference* classes
+# Reference classes
 
 `bavapi` can generate some convenience reference classes which map Fount resource IDs with a more readable name, both for ease of use and for autocompletion in IDEs.
 
@@ -7,7 +7,7 @@ These classes are automatically generated by a console command that becomes avai
 !!! info "Protected Access"
     A Fount API token is required to generate reference files. See the [Authentication](authentication.md) section for more information and instructions for using `.env` files.
 
-As of `v0.13` the following reference classes will be generated in a folder named `bavapi_refs` (by default, see [below](#specify-destination-folder)):
+As of `v1.0` the following reference classes will be generated in a folder named `bavapi_refs` (by default, see [below](#specify-destination-folder)):
 
 - `Audiences`: encodes audience IDs
 - `Countries`: encodes country IDs
diff --git a/docs/index.md b/docs/index.md
@@ -41,7 +41,7 @@ See [Installation](getting-started/installation.md) for more detailed instructio
 >>> result
 ```
 
-1. :lock: Replace "TOKEN" with your token.
+1. :lock: Replace `"TOKEN"` with your Fount API token.
 
 |     | sector_id | sector_name           | id   | name   | ... |
 | --: | :-------- | :-------------------- | :--- | :----- | :-- |
@@ -52,20 +52,20 @@ See [Installation](getting-started/installation.md) for more detailed instructio
 
 - Support for all endpoints in the Fount API.
     - Extended support for the following endpoints:
-        - `audiences`
-        - `brand-metrics`
-        - `brand-metric-groups`
-        - `brands`
-        - `brandscape-data`
-        - `categories`
-        - `cities`
-        - `collections`
-        - `companies`
-        - `countries`
-        - `sectors`
-        - `studies`
-        - `years`
-    - Other endpoints are available via the `raw_query` functions and methods.
+        - [`audiences`](endpoints/audiences.md)
+        - [`brand-metrics`](endpoints/brand-metrics.md)
+        - [`brand-metric-groups`](endpoints/brand-metric-groups.md)
+        - [`brands`](endpoints/brands.md)
+        - [`brandscape-data`](endpoints/brandscape-data.md)
+        - [`categories`](endpoints/categories.md)
+        - [`cities`](endpoints/cities.md)
+        - [`collections`](endpoints/collections.md)
+        - [`companies`](endpoints/companies.md)
+        - [`countries`](endpoints/countries.md)
+        - [`sectors`](endpoints/sectors.md)
+        - [`studies`](endpoints/studies.md)
+        - [`years`](endpoints/years.md)
+    - Other endpoints are available via the [`raw_query`](endpoints/index.md#other-endpoints) functions and methods.
 - Validates query parameters are of the correct types and provides type hints for better IDE support.
 - Retrieve multiple pages of data simultaneously, monitoring and preventing exceeding API rate limit.
 - Both synchronous and asynchronous APIs for accessing BAV data.
diff --git a/docs/release-notes.md b/docs/release-notes.md
diff --git a/docs/roadmap.md b/docs/roadmap.md
diff --git a/docs/usage/advanced.md b/docs/usage/advanced.md
diff --git a/docs/usage/basic.md b/docs/usage/basic.md
diff --git a/mkdocs.yml b/mkdocs.yml
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py
