better exit code handling in migrations code (Netflix#303)

Author: oavdeev

services/migration_service/api/utils.py

@@ -3,10 +3,10 @@
 from . import version_dict, latest, \
     make_goose_migration_template, make_goose_template
 from services.migration_service.migration_config import db_conf
+import sys
 
 
 class ApiUtils(object):
-
     @staticmethod
     def list_migrations():
         migrations_list = list((version_dict.keys()))
@@ -54,10 +54,12 @@ async def get_latest_compatible_version():
             version = await ApiUtils.get_goose_version()
             return version_dict[version]
         else:
+            print("Running initial migration..", file=sys.stderr)
             goose_version_cmd = make_goose_migration_template(db_conf.connection_string_url, 'up')
             p = Popen(goose_version_cmd, shell=True,
                       close_fds=True)
-            p.wait()
+            if p.wait() != 0:
+                raise Exception("Failed to run initial migration")
             return latest
 
     @staticmethod

services/migration_service/get_virtual_env.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import requests
 import socket
 import time
@@ -27,10 +28,16 @@
             retry_count = retry_count - 1
     # continue
     s.close()
+    if retry_count == 0:
+        print("ran out of retries to get migration version, exiting")
+        sys.exit(1)
 except Exception as e:
     print(e)
+    sys.exit(1)
 
 r = requests.get('http://localhost:{0}/version'.format(port))
+r.raise_for_status()
+
 conf_file = open('/root/services/migration_service/config', 'w')
 print(r.text, file=conf_file)
 conf_file.close()

services/migration_service/run_script.py

@@ -1,5 +1,6 @@
 from subprocess import Popen
 import os
+import sys
 
 
 def setup_env(version_value: str):
@@ -26,7 +27,9 @@ def setup_env(version_value: str):
             close_fds=True
         )
 
-        get_env_version.wait()
+        if get_env_version.wait() != 0:
+            print("Failed to get env version", file=sys.stderr)
+            sys.exit(1)
 
         # read in version of metadata service to load
         version_value_file = open('/root/services/migration_service/config', 'r')
@@ -40,10 +43,14 @@ def setup_env(version_value: str):
             env=setup_env(version_value)
         )
 
-        metadata_server_process.wait()
-        migration_server_process.wait()
+        rc = metadata_server_process.wait()
+        if rc != 0:
+            print("Metadata server exited with non zero status")
+            sys.exit(rc)
+        rc = migration_server_process.wait()
+        if rc != 0:
+            print("Migration server exited with non zero status")
+            sys.exit(rc)
     except Exception as e:
         print(e)
-    finally:
-        # should never be reached
-        exit(1)
+        sys.exit(1)