Merge pull request #3 from SciLifeLab/master

Version sync

backup_github.py

@@ -52,16 +52,17 @@ def credentials():
     return conf
 
 
-def backup(user, password, organizations, dest):
+def backup(user, password, access_token, organizations, dest):
     """Performs a backup of all the accessible repos in given organizations
     """
     if password is None or user is None:
         logger.error("No valid github credentials provided. Exiting!")
         sys.exit(-1)
     if password is not None:
-        github_instance = Github(user, password)
+        github_instance = Github(access_token)
         repositories = []  # list of repository *iterators*
         for organization in organizations:
+            logger.info("Github API rate limit: {}".format(github_instance.get_rate_limit()))
             github_organization = github_instance.get_organization(organization)
 
             repositories.append(github_organization.get_repos(type='all'))
@@ -72,6 +73,7 @@ def backup(user, password, organizations, dest):
                 os.mkdir(organization_destination_path)
 
     for repository in chain(*repositories):
+        logger.info("Github API rate limit: {}".format(github_instance.get_rate_limit()))
         if password is not None and repository.private is True:
             source = repository.clone_url.replace(
                                 "https://",
@@ -150,9 +152,6 @@ def compress_and_move(source, final_dest):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--user", nargs='?', type=str, help="GitHub username")
-    parser.add_argument("--password", nargs='?', type=str,
-                        help="GitHub password")
     parser.add_argument("--dest", type=str,
                         help="Destination of the uncompressed copy")
     parser.add_argument('--final_dest', type=str, help="Final destination of "
@@ -164,14 +163,6 @@ def compress_and_move(source, final_dest):
                         help="File to append the log to.")
     args = parser.parse_args()
 
-    # Command line flags take priority. Otherwise use config
-    user = args.user
-    password = args.password
-    config = credentials()
-    if user is None or password is None:
-        user = config.get("github_username")
-        password = config.get("github_password")
-
     dest = os.getcwd() if not args.dest else args.dest
 
     # Need to check if the directory exists for the given log file
@@ -196,5 +187,21 @@ def compress_and_move(source, final_dest):
                                             dest,
                                             ", ".join(args.organizations)
                                             ))
-    backup(user, password, args.organizations, dest)
+
+    config = credentials()
+    user = config.get("github_username")
+    password = config.get("github_password")
+    access_token = config.get("access_token")
+
+    if user is None:
+        logger.error("Missing user from the .githubcredentials file. Exiting!")
+        sys.exit(-1)
+    if password is None:
+        logger.error("Missing password from the .githubcredentials file. Exiting!")
+        sys.exit(-1)
+    if access_token is None:
+        logger.error("Missing access_token from the .githubcredentials file. Exiting!")
+        sys.exit(-1)
+
+    backup(user, password, access_token, args.organizations, dest)
     compress_and_move(dest, args.final_dest)

snic_util.py

@@ -11,7 +11,7 @@
 import sys
 import yaml
 from collections import OrderedDict
-from taca_ngi_pipeline.utils.database import statusdb_session as sdb
+from taca.utils.statusdb import StatusdbSession as sdb
 from ngi_pipeline.database.classes import CharonSession, CharonError
 
 # set logger object
@@ -211,7 +211,7 @@ def _execute_search(self, exec_func, filter_keys=[], all_info=False):
 class DbConnections():
     def __init__(self):
         with open(os.getenv('STATUS_DB_CONFIG'), 'r') as db_cred_file:
-            db_conf = yaml.load(db_cred_file)['statusdb']
+            db_conf = yaml.load(db_cred_file, Loader=yaml.SafeLoader)['statusdb']
         self.statusdbSess = sdb(db_conf, db="projects")
         self.CharonSess = CharonSession()
 
@@ -287,7 +287,7 @@ def add_delivery_proj_in_statusdb(self, delivery_proj, projectid):
     params = vars(parser.parse_args())
     # try loading config file
     try:
-        snic_config = yaml.load(params["config"])["snic"]
+        snic_config = yaml.load(params["config"], Loader=yaml.SafeLoader)["snic"]
     except:
         logger.error("Error loading config file, make sure config is in following format\n\n{}".format(config_format))
         raise