fix: need to fall back to semi-brute force to avoid duplication

cumulus_lambda_functions/lib/uds_db/granules_db_index.py

@@ -218,6 +218,26 @@ def add_entry(self, tenant: str, tenant_venue: str, json_body: dict, doc_id: str
 
     def dsl_search(self, tenant: str, tenant_venue: str, search_dsl: dict):
         read_alias_name = f'{DBConstants.granules_read_alias_prefix}_{tenant}_{tenant_venue}'.lower().strip()
-        search_result = self.__es.query(search_dsl, querying_index=read_alias_name) if 'sort' in search_dsl else self.__es.query(search_dsl, querying_index=read_alias_name)
-        LOGGER.debug(f'search_finished: {len(search_result["hits"]["hits"])}')
-        return search_result
+        original_size = search_dsl['size']
+        result = []
+        duplicates = set([])
+        while len(result) < original_size:
+            search_dsl['size'] = (original_size - len(result)) * 2
+            search_result = self.__es.query_pages(search_dsl, querying_index=read_alias_name) if 'sort' in search_dsl else self.__es.query(search_dsl, querying_index=read_alias_name)
+            if len(search_result['hits']['hits']) < 1:
+                break
+            for each in search_result['hits']['hits']:
+                if each['_id'] not in duplicates:
+                    duplicates.add(each['_id'])
+                    result.append(each)
+            search_dsl['search_after'] = search_result['hits']['hits'][-1]['sort']
+
+        LOGGER.debug(f'search_finished: {len(result)}')
+        return {
+            'hits': {
+                "total": {
+                    "value": len(result)
+                },
+                'hits': result
+            }
+        }

cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py

@@ -36,7 +36,7 @@ def __generate_es_dsl(self):
         query_dsl = {
             'track_total_hits': True,
             'size': self.__limit,
-            "collapse": {"field": "id"},
+            # "collapse": {"field": "id"},
             'sort': [
                 {'properties.datetime': {'order': 'desc'}},
                 {'id': {'order': 'asc'}}

tests/cumulus_lambda_functions/lib/uds_db/test_granules_db_index.py

@@ -0,0 +1,53 @@
+import json
+import os
+from unittest import TestCase
+
+from cumulus_lambda_functions.lib.uds_db.granules_db_index import GranulesDbIndex
+
+
+class TestGranulesDbIndex(TestCase):
+    def test_01(self):
+        os.environ['ES_URL'] = 'dummy'
+        granules_index = GranulesDbIndex()
+        expected = custom_metadata_body = {
+            'tag': {'type': 'keyword'},
+            'c_data1': {'type': 'long'},
+            'c_data2': {'type': 'boolean'},
+            'c_data3': {'type': 'keyword'},
+            'soil10': {
+                "properties": {
+                    "0_0": {"type": "integer"},
+                    "0_1": {"type": "integer"},
+                    "0_2": {"type": "integer"},
+                }
+            }
+        }
+        granule_index_mapping = {'unity_granule_uds_black_dev__v01': {'mappings': {'dynamic': 'strict', 'properties': {'archive_error_code': {'type': 'keyword'}, 'archive_error_message': {'type': 'text'}, 'archive_status': {'type': 'keyword'}, 'assets': {'type': 'object', 'dynamic': 'false'}, 'bbox': {'type': 'geo_shape'}, 'collection': {'type': 'keyword'}, 'event_time': {'type': 'long'}, 'geometry': {'type': 'geo_shape'}, 'id': {'type': 'keyword'}, 'links': {'properties': {'href': {'type': 'keyword'}, 'rel': {'type': 'keyword'}, 'title': {'type': 'text'}, 'type': {'type': 'keyword'}}}, 'properties': {'dynamic': 'false', 'properties': {'c_data1': {'type': 'long'}, 'c_data2': {'type': 'boolean'}, 'c_data3': {'type': 'keyword'}, 'created': {'type': 'date', 'format': "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"}, 'datetime': {'type': 'date', 'format': "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"}, 'end_datetime': {'type': 'date', 'format': "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"}, 'provider': {'type': 'keyword'}, 'soil10': {'properties': {'0_0': {'type': 'integer'}, '0_1': {'type': 'integer'}, '0_2': {'type': 'integer'}}}, 'start_datetime': {'type': 'date', 'format': "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"}, 'status': {'type': 'keyword'}, 'tag': {'type': 'keyword'}, 'updated': {'type': 'date', 'format': "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ss'Z'||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'||yyyy-MM-dd||epoch_millis"}}}, 'stac_extensions': {'type': 'keyword'}, 'stac_version': {'type': 'keyword'}, 'type': {'type': 'keyword'}}}}}
+        custom_metadata = granules_index.get_custom_metadata_fields(granule_index_mapping)
+        print(custom_metadata)
+        self.assertEqual(custom_metadata, expected)
+        return
+
+    def test_02(self):
+        os.environ['ES_URL'] = 'https://vpc-uds-sbx-cumulus-es-qk73x5h47jwmela5nbwjte4yzq.us-west-2.es.amazonaws.com'
+        os.environ['ES_PORT'] = '9200'
+        self.tenant = 'UDS_MY_LOCAL_ARCHIVE_TEST'  # 'uds_local_test'  # 'uds_sandbox'
+        self.tenant_venue = 'DEV'  # 'DEV1'  # 'dev'
+        search_dsl = {
+            'track_total_hits': True,
+            'size': 20,
+            'sort': [
+                {'properties.datetime': {'order': 'desc'}},
+                {'id': {'order': 'asc'}}
+            ],
+            'query': {
+                'bool': {
+                    'must': {'term': {'collection': {'value': f'URN:NASA:UNITY:{self.tenant}:{self.tenant_venue}:UDS_UNIT_COLLECTION___2408290522'}}}
+                }
+            },
+        }
+        granules_index = GranulesDbIndex()
+        query_result = granules_index.dsl_search(self.tenant, self.tenant_venue, search_dsl)
+        print(json.dumps(query_result, indent=4))
+        # self.assertEqual(custom_metadata, expected)
+        return

tests/integration_tests/test_uds_api.py

@@ -185,7 +185,7 @@ def test_collections_get_single_granule(self):
         return
 
     def test_granules_get(self):
-        post_url = f'{self.uds_url}collections/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2312041030/items/'  # MCP Dev
+        post_url = f'{self.uds_url}collections/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2408290522/items/'  # MCP Dev
         headers = {
             'Authorization': f'Bearer {self.bearer_token}',
         }
@@ -195,7 +195,7 @@ def test_granules_get(self):
                                     )
         self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}')
         response_json = json.loads(query_result.text)
-        print(json.dumps(response_json))
+        print(json.dumps(response_json, indent=4))
         links = {k['rel']: k['href'] for k in response_json['links'] if k['rel'] != 'root'}
         for k, v in links.items():
             self.assertTrue(v.startswith(self.uds_url), f'missing stage: {self.stage} in {v} for {k}')