feat: STAC style pagination (#70)

* fix: add err msg when next & prev url does not generate + update makefile for local

* feat: update to STAC specific schema ref from CMR

* feat: make pagination class

Author: wphyojpl

ci.cd/Makefile

@@ -4,7 +4,7 @@ export VERSION ?= latest
 
 
 all: build_lambda upload_lambda update_lambda_function build_docker
-
+local: build_lambda upload_lambda update_lambda_function_1 update_lambda_function_2 update_lambda_function_3
 build_docker:
 	docker build -t "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" -f docker/Dockerfile .
 
@@ -19,6 +19,9 @@ build_lambda_public:
 
 upload_lambda:
 	aws --profile saml-pub s3 cp cumulus_lambda_functions_deployment.zip s3://am-uds-dev-cumulus-tf-state/unity_cumulus_lambda/
-
-update_lambda_function:
-	aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:Test1 --publish
+update_lambda_function_1:
+	aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_dapa --publish &>/dev/null
+update_lambda_function_2:
+	aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_granules_dapa --publish &>/dev/null
+update_lambda_function_3:
+	aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_ingest_cnm_dapa --publish &>/dev/null

cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py

@@ -46,6 +46,14 @@ def __get_size(self):
             cumulus_size = {'total_size': -1}
         return cumulus_size
 
+    def __get_pagination_urls(self):
+        try:
+            pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links()
+        except Exception as e:
+            LOGGER.exception(f'error while generating pagination links')
+            return [{'message': f'error while generating pagination links: {str(e)}'}]
+        return pagination_links
+
     def start(self):
         try:
             cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix)
@@ -63,11 +71,11 @@ def start(self):
             return {
                 'statusCode': 200,
                 'body': json.dumps({
-                    'size': cumulus_size['total_size'],
-                    'rel': {
-                        'next': LambdaApiGatewayUtils.generate_next_url(self.__event, self.__limit),
-                        'prev': LambdaApiGatewayUtils.generate_prev_url(self.__event, self.__limit),
-                    },
+                    'numberMatched': cumulus_size['total_size'],
+                    'numberReturned': len(cumulus_result['results']),
+                    'stac_version': '1.0.0',
+                    'type': 'FeatureCollection',
+                    'links': self.__get_pagination_urls(),
                     'features': cumulus_result['results'],
                 })
             }

cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py

@@ -96,6 +96,14 @@ def __get_size(self):
             cumulus_size = {'total_size': -1}
         return cumulus_size
 
+    def __get_pagination_urls(self):
+        try:
+            pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links()
+        except Exception as e:
+            LOGGER.exception(f'error while generating pagination links')
+            return [{'message': f'error while generating pagination links: {str(e)}'}]
+        return pagination_links
+
     def start(self):
         try:
             cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix)
@@ -113,11 +121,11 @@ def start(self):
             return {
                 'statusCode': 200,
                 'body': json.dumps({
-                    'size': cumulus_size['total_size'],
-                    'rel': {
-                        'next': LambdaApiGatewayUtils.generate_next_url(self.__event, self.__limit),
-                        'prev': LambdaApiGatewayUtils.generate_prev_url(self.__event, self.__limit),
-                    },
+                    'numberMatched': cumulus_size['total_size'],
+                    'numberReturned': len(cumulus_result['results']),
+                    'stac_version': '1.0.0',
+                    'type': 'FeatureCollection',  # TODO correct name?
+                    'links': self.__get_pagination_urls(),
                     'features': cumulus_result['results']
                 })
             }

cumulus_lambda_functions/lib/utils/lambda_api_gateway_utils.py

@@ -1,7 +1,7 @@
 from copy import deepcopy
 
 from cumulus_lambda_functions.lib.json_validator import JsonValidator
-
+from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
 
 API_GATEWAY_EVENT_SCHEMA = {
     'type': 'object',
@@ -34,43 +34,74 @@
         }
     }
 }
+LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
 
 
 class LambdaApiGatewayUtils:
-    @staticmethod
-    def generate_requesting_url(event: dict):
+    def __init__(self, event: dict, default_limit: int = 10):
+        self.__event = event
+        self.__default_limit = default_limit
         api_gateway_event_validator_result = JsonValidator(API_GATEWAY_EVENT_SCHEMA).validate(event)
         if api_gateway_event_validator_result is not None:
             raise ValueError(f'invalid event: {api_gateway_event_validator_result}. event: {event}')
-        requesting_url = f"https://{event['headers']['Host']}{event['requestContext']['path']}"
+
+    def __get_current_page(self):
+        try:
+            requesting_base_url = f"https://{self.__event['headers']['Host']}{self.__event['requestContext']['path']}"
+            new_queries = deepcopy(self.__event['queryStringParameters']) if 'queryStringParameters' in self.__event and self.__event[
+                'queryStringParameters'] is not None else {}
+            limit = int(new_queries['limit'] if 'limit' in new_queries else self.__default_limit)
+            offset = int(new_queries['offset'] if 'offset' in new_queries else 0)
+            new_queries['limit'] = limit
+            new_queries['offset'] = offset
+            requesting_url = f"{requesting_base_url}?{'&'.join([f'{k}={v}' for k, v in new_queries.items()])}"
+        except Exception as e:
+            LOGGER.exception(f'error while getting current page URL')
+            return f'unable to get current page URL, {str(e)}'
         return requesting_url
 
-    @staticmethod
-    def generate_next_url(event: dict, default_limit: int = 10):
-        requesting_base_url = LambdaApiGatewayUtils.generate_requesting_url(event)
-        new_queries = deepcopy(event['queryStringParameters']) if 'queryStringParameters' in event and event['queryStringParameters'] is not None else {}
-        limit = int(new_queries['limit'] if 'limit' in new_queries else default_limit)
-        if limit == 0:
-            return ''
-        offset = int(new_queries['offset'] if 'offset' in new_queries else 0)
-        offset += limit
-        new_queries['limit'] = limit
-        new_queries['offset'] = offset
-        requesting_url = f"{requesting_base_url}?{'&'.join([f'{k}={v}' for k, v in new_queries.items()])}"
+    def __get_next_page(self):
+        try:
+            requesting_base_url = f"https://{self.__event['headers']['Host']}{self.__event['requestContext']['path']}"
+            new_queries = deepcopy(self.__event['queryStringParameters']) if 'queryStringParameters' in self.__event and self.__event[
+                'queryStringParameters'] is not None else {}
+            limit = int(new_queries['limit'] if 'limit' in new_queries else self.__default_limit)
+            if limit == 0:
+                return ''
+            offset = int(new_queries['offset'] if 'offset' in new_queries else 0)
+            offset += limit
+            new_queries['limit'] = limit
+            new_queries['offset'] = offset
+            requesting_url = f"{requesting_base_url}?{'&'.join([f'{k}={v}' for k, v in new_queries.items()])}"
+        except Exception as e:
+            LOGGER.exception(f'error while getting next page URL')
+            return f'unable to get next page URL, {str(e)}'
         return requesting_url
 
-    @staticmethod
-    def generate_prev_url(event: dict, default_limit: int = 10):
-        requesting_base_url = LambdaApiGatewayUtils.generate_requesting_url(event)
-        new_queries = deepcopy(event['queryStringParameters']) if 'queryStringParameters' in event and event['queryStringParameters'] is not None else {}
-        limit = int(new_queries['limit'] if 'limit' in new_queries else default_limit)
-        if limit == 0:
-            return ''
-        offset = int(new_queries['offset'] if 'offset' in new_queries else 0)
-        offset -= limit
-        if offset < 0:
-            offset = 0
-        new_queries['limit'] = limit
-        new_queries['offset'] = offset
-        requesting_url = f"{requesting_base_url}?{'&'.join([f'{k}={v}' for k, v in new_queries.items()])}"
+    def __get_prev_page(self):
+        try:
+            requesting_base_url = f"https://{self.__event['headers']['Host']}{self.__event['requestContext']['path']}"
+            new_queries = deepcopy(self.__event['queryStringParameters']) if 'queryStringParameters' in self.__event and self.__event[
+                'queryStringParameters'] is not None else {}
+            limit = int(new_queries['limit'] if 'limit' in new_queries else self.__default_limit)
+            if limit == 0:
+                return ''
+            offset = int(new_queries['offset'] if 'offset' in new_queries else 0)
+            offset -= limit
+            if offset < 0:
+                offset = 0
+            new_queries['limit'] = limit
+            new_queries['offset'] = offset
+            requesting_url = f"{requesting_base_url}?{'&'.join([f'{k}={v}' for k, v in new_queries.items()])}"
+        except Exception as e:
+            LOGGER.exception(f'error while getting previous page URL')
+            return f'unable to get previous page URL, {str(e)}'
         return requesting_url
+
+    def generate_pagination_links(self):
+        return [
+            {'rel': 'self', 'href': self.__get_current_page()},
+            {'rel': 'root', 'href': f"https://{self.__event['headers']['Host']}"},
+            {'rel': 'next', 'href': self.__get_next_page()},
+            {'rel': 'prev', 'href': self.__get_prev_page()},
+        ]

tests/cumulus_lambda_functions/lib/utils/test_lambda_api_gateway_utils.py

@@ -52,15 +52,36 @@ def test_01(self):
             'body': None,
             'isBase64Encoded': False
         }
+        lambda_pagination = LambdaApiGatewayUtils(sample_event, 10)
+        links = lambda_pagination.generate_pagination_links()
+        links_dict = {k['rel']: k['href'] for k in links}
+        self.assertTrue('next' in links_dict, f'next missing in {links}')
+        self.assertTrue('prev' in links_dict, f'prev missing in {links}')
+        self.assertTrue('self' in links_dict, f'self missing in {links}')
+        self.assertTrue('root' in links_dict, f'root missing in {links}')
         next_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=10&offset=10'
+        self.assertEqual(sorted(next_url), sorted(links_dict['next']), f'wrong next url. {next_url} vs {links_dict["next"]}')
         prev_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=10&offset=0'
-        self.assertEqual(sorted(next_url), sorted(LambdaApiGatewayUtils.generate_next_url(sample_event, 10)), f'wrong next url')
-        self.assertEqual(sorted(prev_url), sorted(LambdaApiGatewayUtils.generate_prev_url(sample_event, 10)), f'wrong prev url')
-        self.assertEqual('', LambdaApiGatewayUtils.generate_next_url(sample_event, 0), f'wrong next empty url')
-        self.assertEqual('', LambdaApiGatewayUtils.generate_prev_url(sample_event, 0), f'wrong next empty url')
+        self.assertEqual(sorted(prev_url), sorted(links_dict['prev']), f'wrong next url. {prev_url} vs {links_dict["prev"]}')
+        current_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=10&offset=0'
+        self.assertEqual(sorted(current_url), sorted(links_dict['self']), f'wrong self url. {current_url} vs {links_dict["self"]}')
+
+        lambda_pagination = LambdaApiGatewayUtils(sample_event, 0)
+        links = lambda_pagination.generate_pagination_links()
+        links_dict = {k['rel']: k['href'] for k in links}
+        self.assertEqual('', links_dict['next'], f'wrong next empty url. {links_dict["next"]}')
+        self.assertEqual('', links_dict['prev'], f'wrong next empty url. {links_dict["prev"]}')
+        current_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=0&offset=0'
+        self.assertEqual(sorted(current_url), sorted(links_dict['self']), f'wrong self url. {current_url} vs {links_dict["self"]}')
+
         sample_event['queryStringParameters']['offset'] = 10
+        lambda_pagination = LambdaApiGatewayUtils(sample_event, 5)
+        links = lambda_pagination.generate_pagination_links()
+        links_dict = {k['rel']: k['href'] for k in links}
         next_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=5&offset=15'
+        self.assertEqual(sorted(next_url), sorted(links_dict['next']), f'wrong next url. {next_url} vs {links_dict["next"]}')
         prev_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=5&offset=5'
-        self.assertEqual(sorted(next_url), sorted(LambdaApiGatewayUtils.generate_next_url(sample_event, 5)), f'wrong next url 2')
-        self.assertEqual(sorted(prev_url), sorted(LambdaApiGatewayUtils.generate_prev_url(sample_event, 5)), f'wrong prev url 2')
+        self.assertEqual(sorted(prev_url), sorted(links_dict['prev']), f'wrong next url. {prev_url} vs {links_dict["prev"]}')
+        current_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=5&offset=10'
+        self.assertEqual(sorted(current_url), sorted(links_dict['self']), f'wrong self url. {current_url} vs {links_dict["self"]}')
         return