Skip to content

Update arrow notes, small doc changes. #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 20, 2023
Merged

Update arrow notes, small doc changes. #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 62 additions & 7 deletions btrdb/conn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,10 @@ def __init__(self, addrportstr, apikey=None):

Parameters
----------
addrportstr: str
addrportstr: str, required
The address of the cluster to connect to, e.g 123.123.123:4411
apikey: str
The option API key to authenticate requests
apikey: str, optional
The optional API key to authenticate requests

"""
addrport = addrportstr.split(":", 2)
Expand Down Expand Up @@ -224,6 +224,17 @@ def query(
a list of parameter values to be sanitized and interpolated into the
SQL statement. Using parameters forces value/type checking and is
considered a best practice at the very least.
auto_retry: bool, default: False
Whether to retry this request in the event of an error
retries: int, default: 5
Number of times to retry this request if there is an error. Will
be ignored if auto_retry is False
retry_delay: int, default: 3
initial time to wait before retrying function call if there is an error.
Will be ignored if auto_retry is False
retry_backoff: int, default: 4
Exponential factor by which the backoff increases between retries.
Will be ignored if auto_retry is False

Returns
-------
Expand Down Expand Up @@ -348,8 +359,25 @@ def create(

Parameters
----------
uuid: UUID
uuid: UUID, required
The uuid of the requested stream.
collection: str, required
The collection string prefix that the stream will belong to.
tags: dict, required
The tags-level immutable metadata key:value pairs.
annotations: dict, optional
The mutable metadata of the stream, key:value pairs
auto_retry: bool, default: False
Whether to retry this request in the event of an error
retries: int, default: 5
Number of times to retry this request if there is an error. Will
be ignored if auto_retry is False
retry_delay: int, default: 3
initial time to wait before retrying function call if there is an error.
Will be ignored if auto_retry is False
retry_backoff: int, default: 4
Exponential factor by which the backoff increases between retries.
Will be ignored if auto_retry is False

Returns
-------
Expand All @@ -376,7 +404,7 @@ def create(

def info(self):
"""
Returns information about the connected BTrDB srerver.
Returns information about the connected BTrDB server.

Returns
-------
Expand All @@ -397,9 +425,14 @@ def list_collections(self, starts_with=""):
Returns a list of collection paths using the `starts_with` argument for
filtering.

Parameters
----------
starts_with: str, optional, default = ''
Filter collections that start with the string provided, if none passed, will list all collections.

Returns
-------
collection paths: list[str]
collections: List[str]

"""
return [c for some in self.ep.listCollections(starts_with) for c in some]
Expand Down Expand Up @@ -492,6 +525,17 @@ def streams_in_collection(
The tags to identify the stream.
annotations: Dict[str, str]
The annotations to identify the stream.
auto_retry: bool, default: False
Whether to retry this request in the event of an error
retries: int, default: 5
Number of times to retry this request if there is an error. Will
be ignored if auto_retry is False
retry_delay: int, default: 3
initial time to wait before retrying function call if there is an error.
Will be ignored if auto_retry is False
retry_backoff: int, default: 4
Exponential factor by which the backoff increases between retries.
Will be ignored if auto_retry is False

Returns
------
Expand Down Expand Up @@ -546,8 +590,19 @@ def collection_metadata(

Parameters
----------
prefix: str
prefix: str, required
A prefix of the collection names to look at
auto_retry: bool, default: False
Whether to retry this request in the event of an error
retries: int, default: 5
Number of times to retry this request if there is an error. Will
be ignored if auto_retry is False
retry_delay: int, default: 3
initial time to wait before retrying function call if there is an error.
Will be ignored if auto_retry is False
retry_backoff: int, default: 4
Exponential factor by which the backoff increases between retries.
Will be ignored if auto_retry is False

Returns
-------
Expand Down
24 changes: 11 additions & 13 deletions btrdb/stream.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ def __init__(self, btrdb, uuid, **db_values):

def refresh_metadata(self):
"""
Refreshes the locally cached meta data for a stream
Refreshes the locally cached metadata for a stream from the server.

Queries the BTrDB server for all stream metadata including collection,
annotation, and tags. This method requires a round trip to the server.
Expand Down Expand Up @@ -576,7 +576,6 @@ def insert(self, data, merge="never"):
-------
int
The version of the stream after inserting new points.

"""
version = 0
i = 0
Expand Down Expand Up @@ -615,7 +614,7 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int:

Notes
-----
ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
if not self._btrdb._ARROW_ENABLED:
raise NotImplementedError(_arrow_not_impl_str.format("arrow_insert"))
Expand Down Expand Up @@ -758,7 +757,6 @@ def update(
int
The version of the metadata (separate from the version of the data)
also known as the "property version".

"""
if tags is None and annotations is None and collection is None:
raise BTRDBValueError(
Expand Down Expand Up @@ -826,7 +824,6 @@ def delete(
-------
int
The version of the new stream created

"""
return self._btrdb.ep.deleteRange(
self._uuid, to_nanoseconds(start), to_nanoseconds(end)
Expand Down Expand Up @@ -948,7 +945,7 @@ def arrow_values(
granularity. In the tree data structure of BTrDB, this data is stored in
the vector nodes.

ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
if not self._btrdb._ARROW_ENABLED:
raise NotImplementedError(_arrow_not_impl_str.format("arrow_values"))
Expand Down Expand Up @@ -1108,7 +1105,7 @@ def arrow_aligned_windows(
As the window-width is a power-of-two, it aligns with BTrDB internal
tree data structure and is faster to execute than `windows()`.

ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
if not self._btrdb._ARROW_ENABLED:
raise NotImplementedError(
Expand Down Expand Up @@ -1202,7 +1199,7 @@ def windows(
parameter previously available has been deprecated. The only valid value
for depth is now 0.

ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
materialized = []
start = to_nanoseconds(start)
Expand Down Expand Up @@ -1270,7 +1267,7 @@ def arrow_windows(
end = start + width * floordiv(end - start, width). The `depth`
parameter previously available has been deprecated. The only valid value
for depth is now 0.
ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
if not self._btrdb._ARROW_ENABLED:
raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows"))
Expand Down Expand Up @@ -1827,12 +1824,12 @@ def windows(self, width, depth=0):
Windows returns arbitrary precision windows from BTrDB. It is slower
than aligned_windows, but still significantly faster than values. Each
returned window will be width nanoseconds long. start is inclusive, but
end is exclusive (e.g if end < start+width you will get no results).
end is exclusive (e.g. if end < start+width you will get no results).
That is, results will be returned for all windows that start at a time
less than the end timestamp. If (end - start) is not a multiple of
width, then end will be decreased to the greatest value less than end
such that (end - start) is a multiple of width (i.e., we set end = start
+ width * floordiv(end - start, width). The `depth` parameter previously
+ width * floordiv(end - start, width)). The `depth` parameter previously
available has been deprecated. The only valid value for depth is now 0.

"""
Expand Down Expand Up @@ -2046,7 +2043,8 @@ def arrow_insert(self, data_map: dict, merge: str = "never") -> dict:
Notes
-----
BTrDB expects datetimes to be in UTC+0.
ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL

This method is available for commercial customers with arrow-enabled servers.

Returns
-------
Expand Down Expand Up @@ -2098,7 +2096,7 @@ def arrow_values(

Notes
-----
ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
params = self._params_from_filters()
versions = self._pinned_versions
Expand Down
12 changes: 6 additions & 6 deletions btrdb/transformers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ def arrow_to_series(streamset, agg="mean", name_callable=None):

Notes
-----
ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
if not streamset._btrdb._ARROW_ENABLED:
raise NotImplementedError(
Expand Down Expand Up @@ -161,7 +161,7 @@ def arrow_to_dataframe(

Notes
-----
ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
if not streamset._btrdb._ARROW_ENABLED:
raise NotImplementedError(
Expand Down Expand Up @@ -314,7 +314,7 @@ def arrow_to_polars(streamset, agg=None, name_callable=None):

Notes
-----
ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
if not streamset._btrdb._ARROW_ENABLED:
raise NotImplementedError(
Expand All @@ -341,7 +341,7 @@ def arrow_to_arrow_table(streamset):

Notes
-----
ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
if not streamset._btrdb._ARROW_ENABLED:
raise NotImplementedError(
Expand Down Expand Up @@ -457,7 +457,7 @@ def arrow_to_numpy(streamset, agg=None):
-----
This method first converts to a pandas data frame then to a numpy array.

ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
if not streamset._btrdb._ARROW_ENABLED:
raise NotImplementedError(
Expand Down Expand Up @@ -529,7 +529,7 @@ def arrow_to_dict(streamset, agg=None, name_callable=None):

Notes
-----
ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL
This method is available for commercial customers with arrow-enabled servers.
"""
if not streamset._btrdb._ARROW_ENABLED:
raise NotImplementedError(
Expand Down