frames and session iterators

README.md

@@ -15,6 +15,9 @@ GridMQTTClient - Client for GRID MQTT API. It useful for realtime connection (re
 # Examples how to use
 ## GridAuthClient
 ```
+from keycloak import KeycloakOpenID
+from gridgs.sdk.auth import Client as GridAuthClient
+
 keycloak_openid = KeycloakOpenID(server_url="https://login.gridgs.com", client_id="grid-api", realm_name="grid")
 grid_auth_client = GridAuthClient(open_id_client=keycloak_openid, username="user@gridgs.com", password="userpass", company_id=1, logger=logging.getLogger('grid_auth_client'))
 ```
@@ -29,18 +32,35 @@ grid_api_client = GridApiClient(base_url="https://api.gridgs.com" auth_client=gr
 
 ### Get sessions
 ```
-from gridgs.sdk.api import SortOrder, SessionQueryParams, SessionSortParam, SessionSortField
+from gridgs.sdk.api import SortOrder, SessionQueryParams, SessionSortField
 
 params = SessionQueryParams(
         satellite=1,
         ground_station=13,
         status=Session.STATUS_SUCCESS,
-        offset=0, limit=3, sort_by=SessionSortField.END_DATE, sort_order=SortOrder.ASC)
+        offset=0, limit=3, 
+        sort_by=SessionSortField.END_DATE, sort_order=SortOrder.ASC)
 sessions_result = grid_api_client.find_sessions(params)
 
 print(f'Total: {sessions_result.total}')
 ```
 
+### Get and Iterate ALL sessions
+it iterates by chunks all sessions which can be found on api based on SessionQueryParams. Default chunk size is 500. 
+```
+from gridgs.sdk.api import SortOrder, SessionQueryParams, SessionSortField
+
+params = SessionQueryParams(
+        offset=0, limit=1000000,
+        satellite=1,
+        ground_station=13,
+        status=Session.STATUS_SUCCESS,
+        sort_by=SessionSortField.END_DATE, sort_order=SortOrder.ASC)
+for session in grid_api_client.iterate_sessions(params):
+    print(session)
+```
+
+
 ### Get session by Id
 ```
 session = grid_api_client.find_session(session_uuid)
@@ -74,26 +94,46 @@ grid_api_client.delete_session(session_uuid)
 
 ### Get frames
 ```
-from gridgs.sdk.api import SortOrder, FrameSortField, FrameSortParam, FrameQueryParams
+from gridgs.sdk.api import SortOrder, FrameSortField, FrameQueryParams
 
 params = FrameQueryParams(
     satellite=2, 
     ground_station=13, 
     date_from=datetime.fromisoformat("2025-02-07 00:00:00"), 
     date_to=datetime.fromisoformat("2025-02-07 00:48:00"), 
-    offset=0, limit=5, sort_by=FrameSortField.TYPE, sort_order=SortOrder.ASC)
+    offset=0, limit=5, 
+    sort_by=FrameSortField.CREATED_AT, sort_order=SortOrder.ASC)
     )
 
 frames_result = grid_api_client.find_frames(params) 
 
 print(f'Total: {frames_result.total}')
 ```
 
+### Get and Iterate ALL frames
+it iterates by chunks all frames which can be found on api based on FrameQueryParams. Default chunk size is 500
+```
+from gridgs.sdk.api import SortOrder, FrameSortField, FrameQueryParams
+
+params = FrameQueryParams(
+        offset=0, limit=1000000, 
+        satellite=1,
+        ground_station=13,
+        date_from=datetime.fromisoformat("2025-02-07 00:00:00"), 
+        date_to=datetime.fromisoformat("2025-02-07 00:48:00"), 
+        sort_by=FrameSortField.CREATED_AT, sort_order=SortOrder.ASC)
+for frame in grid_api_client.iterate_frames(params):
+    print(frame)
+```
+
 ## GridEventSubscriber
 
 Receive statuses of sessions
 
 ```
+from gridgs.sdk.entity import SessionEvent
+from gridgs.sdk.event import Subscriber as GridEventSubscriber
+
 grid_event_subscriber = GridEventSubscriber(host="api.gridgs.com", port=1883, auth_client=grid_auth_client, logger=logging.getLogger('grid_event_subscriber'))
 
 def on_event(event: SessionEvent):
@@ -108,6 +148,9 @@ grid_event_subscriber.run()
 ## GridMQTTClient
 
 ```
+from gridgs.sdk.entity import Frame
+from gridgs.sdk.mqtt import Client as GridMQTTClient
+
 grid_mqtt_client = GridMQTTClient(host="api.gridgs.com", port=1883, auth_client=grid_auth_client, logger=logging.getLogger('grid_event_subscriber'))
 
 def on_downlink_frame(frame: Frame):

src/gridgs/sdk/api/base_client.py

@@ -1,13 +1,18 @@
 import json
 import logging
+from dataclasses import replace
+from typing import Callable, Iterator
 from urllib.parse import urljoin
 
 import requests
 
 from gridgs.sdk.auth import Client as AuthClient
+from .params import PaginatedQueryParams
 
 
 class BaseClient:
+    __iterator_chunk_size = 500
+
     def __init__(self, base_url: str, auth_client: AuthClient, logger: logging.Logger, verify=True):
         self.__base_url = base_url
         self.__auth_client = auth_client
@@ -27,3 +32,26 @@ def request(self, method: str, path: str, params: dict | None = None, data: dict
     def __build_auth_header(self) -> dict:
         token = self.__auth_client.token()
         return {'Authorization': 'Bearer ' + token.access_token}
+
+    def _iterate_items(self, params: PaginatedQueryParams, items_fetcher: Callable[[PaginatedQueryParams], requests.Response], item_builder: Callable[[dict], object]) -> Iterator:
+        iterated = 0
+        total_limit = params.limit if isinstance(params.limit, int) else 0
+        chunk_size = min(total_limit, self.__iterator_chunk_size) if total_limit > 0 else self.__iterator_chunk_size
+        total = params.offset + 1  # to run while loop it should be more than offset
+        params = replace(params, limit=chunk_size)  # limit plays role of chunk size here
+
+        while params.offset < total:
+            if 0 < total_limit <= iterated:
+                return
+
+            response = items_fetcher(params)
+
+            for row in response.json():
+                if 0 < total_limit <= iterated:
+                    return
+
+                yield item_builder(row)
+                iterated += 1
+
+            total = int(response.headers.get('Pagination-Count'))
+            params = replace(params, offset=params.offset + params.limit)

src/gridgs/sdk/api/frame_client.py

@@ -2,11 +2,13 @@
 from datetime import datetime
 from enum import Enum
 from http.client import HTTPException
-from typing import List
+from typing import List, Iterator
+
+import requests
 
 from gridgs.sdk.entity import frame_from_dict, Frame, FrameType
 from .base_client import BaseClient
-from .params import PaginatedQueryParams, SortQueryParam
+from .params import PaginatedQueryParams, SortQueryParam, QueryParams
 
 
 class FrameSortField(Enum):
@@ -51,14 +53,23 @@ class FramesResult:
 
 
 class FrameClient(BaseClient):
-    def find_frames(self, params: FrameQueryParams) -> FramesResult:
-        response = self.request('get', 'frames', params=params.to_dict())
 
-        if response.status_code != 200:
-            raise HTTPException('Cannot get frames', response.reason, response.json())
+    def find_frames(self, params: FrameQueryParams) -> FramesResult:
+        response = self.__find_frames_request(params)
 
         frames = []
         for row in response.json():
             frames.append(frame_from_dict(row))
 
         return FramesResult(frames=frames, total=int(response.headers.get('Pagination-Count')))
+
+    def iterate_frames(self, params: FrameQueryParams) -> Iterator[Frame]:
+        return self._iterate_items(params, self.__find_frames_request, frame_from_dict)
+
+    def __find_frames_request(self, params: QueryParams) -> requests.Response:
+        response = self.request('get', 'frames', params=params.to_dict())
+
+        if response.status_code != 200:
+            raise HTTPException('Cannot get frames', response.reason, response.json())
+
+        return response

src/gridgs/sdk/api/params.py

@@ -1,11 +1,16 @@
+from abc import abstractmethod, ABC
 from dataclasses import dataclass
 from enum import Enum
 
+class QueryParams(ABC):
+    @abstractmethod
+    def to_dict(self) -> dict:
+        pass
 
 @dataclass(frozen=True)
-class PaginatedQueryParams:
+class PaginatedQueryParams(QueryParams):
+    offset: int  = 0
     limit: int | None = None
-    offset: int | None = None
 
     def to_dict(self) -> dict:
         return {'offset': self.offset, 'limit': self.limit}
@@ -17,7 +22,7 @@ class SortOrder(Enum):
 
 
 @dataclass(frozen=True)
-class SortQueryParam:
+class SortQueryParam(QueryParams):
     sort_by: Enum | None = None
     sort_order: SortOrder = SortOrder.ASC
 

src/gridgs/sdk/api/session_client.py

@@ -3,11 +3,13 @@
 from datetime import datetime
 from enum import Enum
 from http.client import HTTPException
-from typing import List
+from typing import List, Iterator
+
+import requests
 
 from gridgs.sdk.entity import Session, session_from_dict
 from .base_client import BaseClient
-from .params import PaginatedQueryParams, SortQueryParam
+from .params import PaginatedQueryParams, SortQueryParam, QueryParams
 
 
 class SessionSortField(Enum):
@@ -60,20 +62,26 @@ class SessionsResult:
 
 
 class SessionClient(BaseClient):
-    def find_sessions(self, params: SessionQueryParams) -> SessionsResult:
-        response = self.request('get', 'sessions', params=params.to_dict())
 
-        if response.status_code != 200:
-            raise HTTPException('Cannot find sessions', response.reason, response.json())
+    def find_sessions(self, params: SessionQueryParams) -> SessionsResult:
+        response = self.__find_sessions_request(params)
 
         sessions = []
         for row in response.json():
             sessions.append(session_from_dict(row))
 
-        return SessionsResult(
-            sessions=sessions,
-            total=int(response.headers.get('Pagination-Count'))
-        )
+        return SessionsResult(sessions=sessions, total=int(response.headers.get('Pagination-Count')))
+
+    def iterate_sessions(self, params: SessionQueryParams) -> Iterator[Session]:
+        return self._iterate_items(params, self.__find_sessions_request, session_from_dict)
+
+    def __find_sessions_request(self, params: QueryParams) -> requests.Response:
+        response = self.request('get', 'sessions', params=params.to_dict())
+
+        if response.status_code != 200:
+            raise HTTPException('Cannot find sessions', response.reason, response.json())
+
+        return response
 
     def find_session(self, id: uuid.UUID) -> Session | None:
         response = self.request('get', f'sessions/{str(id)}')