catching errors with earthdata connection

AstroDigital · Feb 7, 2017 · 1ded7a7 · 1ded7a7
1 parent ee8a7c6
commit 1ded7a7
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 4 deletions.
diff --git a/modispds/cmr.py b/modispds/cmr.py
@@ -95,7 +95,7 @@ def download_file(url, noauth=False, outdir=''):
             for chunk in stream.iter_content(chunk_size):
                 f.write(chunk)
     except:
-        raise Exception("Problem fetching %s" % stream)
+        raise RuntimeError("Problem fetching %s" % stream)
 
     return fout
 
@@ -120,7 +120,7 @@ def get_stream(session, url, auth, previous_tries):
         link.feed(stream.text)
         return get_stream(session, link.download_link, auth, previous_tries)
     else:
-        raise Exception("Earthdata Authentication Error")
+        raise RuntimeError("Earthdata Authentication Error")
 
 
 class LinkFinder(HTMLParser):

diff --git a/modispds/main.py b/modispds/main.py
@@ -42,8 +42,13 @@ def ingest(start_date, end_date, product=_PRODUCT, outdir=''):
         granules = query(day, day, product=product)
 
         metadata = []
-        for gran in granules:
-            metadata.append(ingest_granule(gran, outdir=outdir))
+        try:
+            for gran in granules:
+                metadata.append(ingest_granule(gran, outdir=outdir))
+        except RuntimeError as e:
+            logger.error('Error processing %s: %s' % (day.date(), str(e)))
+            # skip this entire date for now
+            continue
         # upload index file
         if len(granules) > 0:
             fname = make_scene_list(metadata, fout=index_fname)