make scene list func + tests

modispds/main.py

@@ -5,22 +5,28 @@
 import logging
 import argparse
 from modispds.cmr import query, download_granule
-from modispds.pds import push_to_s3, s3_list, make_index
+from modispds.pds import push_to_s3, s3_list, make_index, make_scene_list
 import gippy
 from modispds.version import __version__
 from modispds.products import products
 
 logger = logging.getLogger('modispds')
 
+# default product
+_PRODUCT = 'MCD43A4.006'
+
+# environment variables
 bucket = os.getenv('BUCKET', 'modis-pds')
 
 
-def ingest(date1, date2, outdir=''):
+def ingest(date1, date2, product=_PRODUCT, outdir=''):
     """ Ingest all granules between two dates """
-    granules = query(date1, date2)
+    granules = query(date1, date2, product=product)
 
+    metadata = []
     for gran in granules:
-        ingest_granule(gran, outdir=outdir)
+        metadata.append(ingest_granule(gran, outdir=outdir))
+    fname = make_scene_list(metadata)
 
 
 def ingest_granule(gran, outdir='', prefix=''):
@@ -57,7 +63,7 @@ def ingest_granule(gran, outdir='', prefix=''):
 
     logger.info('Completed processing granule %s in : %ss' % (gid, time.time() - start_time))
     return {
-        'granuleId': gid,
+        'gid': gid,
         'date': get_date(gid),
         'download_url': os.path.join('https://%s.s3.amazonaws.com', path, 'index.html')
     }
@@ -122,13 +128,14 @@ def parse_args(args):
 
     parser.add_argument('start_date', help='First date')
     parser.add_argument('end_date', help='End date')
+    parser.add_argument('-p', '--product', default=_PRODUCT)
 
     return parser.parse_args(args)
 
 
 def cli():
     args = parse_args(sys.argv[1:])
-    ingest(args.start_date, args.end_date)
+    ingest(args.start_date, args.end_date, product=args.product)
 
 
 if __name__ == "__main__":

modispds/pds.py

@@ -34,6 +34,18 @@ def make_index(thumb, product, files):
     return index_fname
 
 
+def make_scene_list(metadata, fout='scene_list.txt'):
+    """ Create a scene list from metadata """
+    # assume keys are same in all the metadata
+    keys = sorted(metadata[0].keys())
+    with open(fout, 'w') as f:
+        f.write(','.join(keys) + '\n')
+        for md in metadata:
+            items = [md[k] for k in keys]
+            f.write(','.join(items) + '\n')
+    return fout
+
+
 def push_to_s3(filename, bucket, prefix=''):
     """ Copy file to S3 """
     s3 = boto3.client(

test/test_main.py

@@ -63,10 +63,10 @@ def test_ingest_granule(self):
         result = ingest_granule(self.q[0], prefix='testing')
         # granuleid
         self.assertEqual(result['gid'], 'MCD43A4.A2016001.h11v12.006.2016174075640')
-        self.assertEqual(result['date'], datetime.date(2016, 1, 1))
+        self.assertEqual(result['date'], datetime.datetime(2016, 1, 1))
         # path
         path = get_s3_path(result['gid'], prefix='testing')
-        self.assertEqual(path, 'MCD43A4.006/12/07/2015266')
+        self.assertEqual(path, 'testing/MCD43A4.006/12/07/2015266')
         # download url
         url = os.path.join('https://modis-pds.s3.amazonaws.com', path, 'index.html')
         self.assertEqual(result['download_url'], url)

test/test_pds.py

@@ -1,16 +1,28 @@
 import os
 import unittest
-from modispds.pds import push_to_s3, exists, s3_list, del_from_s3, make_index
+from modispds.pds import push_to_s3, exists, s3_list, del_from_s3, make_index, make_scene_list
 
 
 class TestPDS(unittest.TestCase):
     """ Test utiltiies for publishing data on AWS PDS """
 
+    metadata = [
+        {'key1': 'val1', 'key2': 'val2', 'key3': 'val3'},
+        {'key1': 'val4', 'key2': 'val5', 'key3': 'val6'},
+    ]
+
     def test_make_index(self):
         """ Create HTML index of some files """
         fname = make_index('thumbnail.jpg', 'product', ['file1.tif', 'file2.tif'])
         self.assertTrue(os.path.exists(fname))
 
+    def test_make_scene_list(self):
+        """ Create a scene list """
+        fout = make_scene_list(self.metadata)
+        self.assertTrue(os.path.exists(fout))
+        os.remove(fout)
+        self.assertFalse(os.path.exists(fout))
+
     def test_exists(self):
         """ Check for existence of fake object """
         self.assertFalse(exists('s3://modis-pds/nothinghere'))