Update staging branch with latest master changes

.github/workflows/release.yaml

@@ -67,12 +67,12 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel
+        pip install setuptools wheel build
     - name: Build and publish
       run: |
-        python setup.py sdist bdist_wheel
+          python -m build --sdist --wheel --outdir dist/ .
     - name: Publish package
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1
       with:
         user: __token__
         password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }}

.readthedocs.yaml

@@ -0,0 +1,34 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+    # You can also specify other tool versions:
+    # nodejs: "19"
+    # rust: "1.64"
+    # golang: "1.19"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+   configuration: docs/source/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+   - pdf
+#    - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+   install:
+   - requirements: docs/requirements.txt
+   - method: pip
+     path: .

CHANGELOG.md

@@ -0,0 +1,40 @@
+# Changelog
+## 5.30.2
+### What's Changed
+* Update readthedocs to new yaml for testing. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/40
+* Converting pandas index takes very long, add in arrow_table. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/41
+
+
+**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.30.1...v5.30.2
+
+## 5.30.1
+### What's Changed
+* Small version bump for pypi release
+
+
+**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.30.0...v5.30.1
+
+
+## 5.30.0
+### What's Changed
+* Merge Arrow support into Main for Release by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/37
+  * This PR contains many changes that support the commercial Arrow data fetches and inserts
+  * `arrow_` prefixed methods for `Stream` Objects:
+    * `insert, aligned_windows, windows, values`
+  * `arrow_` prefixed methods for StreamSet` objects:
+    * `insert, values, to_dataframe, to_polars, to_arrow_table, to_numpy, to_dict, to_series`
+* Justin gilmer patch 1 by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/39
+
+
+**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.28.1...v5.30.0
+
+
+## 5.28.1
+### What's Changed
+* Upgrade ray versions by @jleifnf in https://github.com/PingThingsIO/btrdb-python/pull/15
+* Release v5.28.1 and Update Python by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/17
+
+### New Contributors
+* @jleifnf made their first contribution in https://github.com/PingThingsIO/btrdb-python/pull/15
+
+**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.15.1...v5.28.1

btrdb/conn.py

@@ -61,7 +61,10 @@ def __init__(self, addrportstr, apikey=None):
 
         """
         addrport = addrportstr.split(":", 2)
-        chan_ops = []
+        # 100MB size limit ~ 2500 streams for 5000 points with each point being 64bit
+        # 500MB size limit ~ 13K streams for 5000 points
+        # -1 size limit = no limit of size to send
+        chan_ops = [("grpc.max_receive_message_length", -1)]
 
         if len(addrport) != 2:
             raise ValueError("expecting address:port")
@@ -313,6 +316,7 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False):
             raise ValueError(
                 f"Could not identify stream based on `{ident}`.  Identifier must be UUID or collection/name."
             )
+
         obj = StreamSet(streams)
 
         if versions:

btrdb/stream.py

@@ -918,7 +918,7 @@ def arrow_values(
         end : int or datetime like object
             The end time in nanoseconds for the range to be queried. (see
             :func:`btrdb.utils.timez.to_nanoseconds` for valid input types)
-        version: int
+        version: int, default: 0
             The version of the stream to be queried
         auto_retry: bool, default: False
             Whether to retry this request in the event of an error
@@ -932,6 +932,7 @@ def arrow_values(
             Exponential factor by which the backoff increases between retries.
             Will be ignored if auto_retry is False
 
+
         Returns
         ------
         pyarrow.Table
@@ -1050,6 +1051,7 @@ def arrow_aligned_windows(
         start: int,
         end: int,
         pointwidth: int,
+        sort_time: bool = False,
         version: int = 0,
         auto_retry=False,
         retries=5,
@@ -1082,7 +1084,9 @@ def arrow_aligned_windows(
             :func:`btrdb.utils.timez.to_nanoseconds` for valid input types)
         pointwidth : int, required
             Specify the number of ns between data points (2**pointwidth)
-        version : int
+        sort_time : bool, default: False
+            Should the table be sorted on the 'time' column?
+        version : int, default: 0
             Version of the stream to query
         auto_retry: bool, default: False
             Whether to retry this request in the event of an error
@@ -1124,7 +1128,10 @@ def arrow_aligned_windows(
         )
         if len(tables) > 0:
             tabs, ver = zip(*tables)
-            return pa.concat_tables(tabs)
+            if sort_time:
+                return pa.concat_tables(tabs).sort_by("time")
+            else:
+                return pa.concat_tables(tabs)
         else:
             schema = pa.schema(
                 [
@@ -1218,6 +1225,7 @@ def arrow_windows(
         start: int,
         end: int,
         width: int,
+        sort_time: bool = False,
         version: int = 0,
         auto_retry=False,
         retries=5,
@@ -1236,6 +1244,8 @@ def arrow_windows(
             :func:`btrdb.utils.timez.to_nanoseconds` for valid input types)
         width : int, required
             The number of nanoseconds in each window.
+        sort_time : bool, default: False
+            Should the table be sorted on the 'time' column.
         version : int, default=0, optional
             The version of the stream to query.
         auto_retry: bool, default: False
@@ -1286,7 +1296,10 @@ def arrow_windows(
         )
         if len(tables) > 0:
             tabs, ver = zip(*tables)
-            return pa.concat_tables(tabs)
+            if sort_time:
+                return pa.concat_tables(tabs).sort_by("time")
+            else:
+                return pa.concat_tables(tabs)
         else:
             schema = pa.schema(
                 [
@@ -2101,6 +2114,8 @@ def arrow_values(
     ):
         """Return a pyarrow table of stream values based on the streamset parameters.
 
+        This data will be sorted by the 'time' column.
+
         Notes
         -----
         This method is available for commercial customers with arrow-enabled servers.
@@ -2145,6 +2160,7 @@ def arrow_values(
                 data = tablex
             else:
                 data = tablex
+            data = data.sort_by("time")
 
         elif self.width is not None and self.depth is not None:
             # create list of stream.windows data (the windows method should
@@ -2175,6 +2191,7 @@ def arrow_values(
                 data = tablex
             else:
                 data = tablex
+            data = data.sort_by("time")
         else:
             sampling_freq = params.pop("sampling_frequency", 0)
             period_ns = 0

btrdb/transformers.py

@@ -224,10 +224,7 @@ def arrow_to_dataframe(
         tmp = tmp_table.select(["time", *usable_cols])
     else:
         tmp = tmp_table
-    df = tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype)
-    df = df.set_index("time")
-    df.index = pd.DatetimeIndex(df.index, tz="UTC")
-    return df
+    return tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype)
 
 
 def to_dataframe(streamset, columns=None, agg="mean", name_callable=None):
@@ -668,5 +665,7 @@ class StreamSetTransformer(object):
     to_polars = to_polars
     arrow_to_polars = arrow_to_polars
 
+    arrow_to_arrow_table = arrow_to_arrow_table
+
     to_csv = to_csv
     to_table = to_table

btrdb/version.py

@@ -15,7 +15,7 @@
 ## Module Info
 ##########################################################################
 
-__version_info__ = {"major": 5, "minor": 28, "micro": 1, "releaselevel": "final"}
+__version_info__ = {"major": 5, "minor": 30, "micro": 2, "releaselevel": "final"}
 
 ##########################################################################
 ## Helper Functions

docs/requirements.txt

@@ -1,3 +1,5 @@
 alabaster>=0.7.12
 Sphinx>=1.7
 sphinx-rtd-theme
+numpydoc
+pydata-sphinx-theme

docs/source/api-index.rst

@@ -0,0 +1,16 @@
+
+.. _API REF:
+
+API Reference
+-------------
+
+.. toctree::
+   :maxdepth: 2
+
+   api/package
+   api/conn
+   api/streams
+   api/points
+   api/exceptions
+   api/transformers
+   api/utils-timez

docs/source/api/conn.rst

@@ -1,6 +1,7 @@
 btrdb.conn
 =============
 
+.. _Conn info:
 .. automodule:: btrdb.conn
 
 .. autoclass:: BTrDB

docs/source/api/streams.rst

@@ -1,10 +1,12 @@
 btrdb.stream
 ==============
 
+.. _StreamGeneralDocs:
 .. automodule:: btrdb.stream
 
 .. autoclass:: Stream
     :members:
 
+.. _StreamSet API:
 .. autoclass:: StreamSetBase
     :members:

docs/source/api/transformers.rst

@@ -1,3 +1,4 @@
+.. _TransformersDocs:
 btrdb.transformers
 =========================
 

docs/source/changelog.rst

@@ -0,0 +1,80 @@
+.. _changelog:
+Changelog
+=========
+
+5.30.2
+------
+What’s Changed
+^^^^^^^^^^^^^^
+-  Update readthedocs to new yaml for testing. by @justinGilmer in
+   https://github.com/PingThingsIO/btrdb-python/pull/40
+-  Converting pandas index takes very long, fix this in arrow_table. by
+   @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/41
+
+**Full Changelog**:
+`5.30.2 <https://github.com/PingThingsIO/btrdb-python/compare/v5.30.1…v5.30.2>`_
+
+.. _section-1:
+
+5.30.1
+------
+.. _whats-changed-1:
+
+What’s Changed
+^^^^^^^^^^^^^^
+
+-  Small version bump for pypi release
+
+**Full Changelog**:
+`5.30.1 <https://github.com/PingThingsIO/btrdb-python/compare/v5.30.0…v5.30.1>`_
+
+.. _section-2:
+
+5.30.0
+------
+.. _whats-changed-2:
+
+What’s Changed
+^^^^^^^^^^^^^^
+
+-  Merge Arrow support into Main for Release by @youngale-pingthings in
+   https://github.com/PingThingsIO/btrdb-python/pull/37
+
+   -  This PR contains many changes that support the commercial Arrow
+      data fetches and inserts
+   -  ``arrow_`` prefixed methods for ``Stream`` Objects:
+
+      -  ``insert, aligned_windows, windows, values``
+
+   -  ``arrow_`` prefixed methods for StreamSet\` objects:
+
+      -  ``insert, values, to_dataframe, to_polars, to_arrow_table, to_numpy, to_dict, to_series``
+
+-  Justin gilmer patch 1 by @justinGilmer in
+   https://github.com/PingThingsIO/btrdb-python/pull/39
+
+**Full Changelog**:
+`5.30.0 <https://github.com/PingThingsIO/btrdb-python/compare/v5.28.1…v5.30.0>`_
+
+.. _section-3:
+
+5.28.1
+------
+.. _whats-changed-3:
+
+What’s Changed
+^^^^^^^^^^^^^^
+
+-  Upgrade ray versions by @jleifnf in
+   https://github.com/PingThingsIO/btrdb-python/pull/15
+-  Release v5.28.1 and Update Python by @youngale-pingthings in
+   https://github.com/PingThingsIO/btrdb-python/pull/17
+
+New Contributors
+^^^^^^^^^^^^^^^^
+
+-  @jleifnf made their first contribution in
+   https://github.com/PingThingsIO/btrdb-python/pull/15
+
+**Full Changelog**:
+`5.28.1 <https://github.com/PingThingsIO/btrdb-python/compare/v5.15.1…v5.28.1>`_