GHOST-601: Handle previous generation AWS EC2 data

Author: Shr3ps

aws_data.py

@@ -146,16 +146,17 @@ def load_instance_data(instance_types, regions):
     for region in regions:
         if region not in instance_types:
             instance_types[region] = []
-            region_data = get_aws_per_region_data(region)
-            if not region_data:
+            aws_db_data = get_aws_per_region_data(region)
+            if not aws_db_data:
                 print 'Cannot get region {r}'.format(r=region)
                 continue
-            for p in region_data['prices']:
-                size = p['attributes']
-                instance_types[region].append(InstanceType(name=size['aws:ec2:instanceType'],
-                                                           cores=size['aws:ec2:vcpu'],
-                                                           memory=size['aws:ec2:memory'],
-                                                           disk=size['aws:ec2:storage']))
+            for region_data in [aws_db_data.get('data_latest', {}), aws_db_data.get('data_previous', {})]:
+                for p in region_data.get('prices', []):
+                    size = p['attributes']
+                    instance_types[region].append(InstanceType(name=size['aws:ec2:instanceType'],
+                                                               cores=size['aws:ec2:vcpu'],
+                                                               memory=size['aws:ec2:memory'],
+                                                               disk=size['aws:ec2:storage']))
 
 
 load_instance_data(instance_types, regions_locations)

aws_data_updater.py

@@ -13,10 +13,11 @@
 from aws_data import regions_locations
 
 AWS_INSTANCES_DATA_URL = 'https://d2xn1uj035lhvj.cloudfront.net/pricing/1.0/ec2/region/{region}/ondemand/linux/index.json'
+AWS_OLD_INSTANCES_DATA_URL = 'https://d2xn1uj035lhvj.cloudfront.net/pricing/1.0/ec2/region/{region}/previous-generation/ondemand/linux/index.json'
 
 
-def get_region_data(aws_region):
-    with requests.get(AWS_INSTANCES_DATA_URL.format(region=aws_region)) as resp:
+def get_region_data(aws_region, url):
+    with requests.get(url.format(region=aws_region)) as resp:
         if resp.status_code != 200:
             print 'Cannot get region {r}'.format(r=aws_region)
             return None
@@ -36,7 +37,9 @@ def main():
     args = parse_args()
     for aws_region in [args.region] if args.region else regions_locations:
         print('Updating AWS Data for region "{r}"'.format(r=aws_region))
-        update_aws_data_region(aws_region, get_region_data(aws_region))
+        old_region_data = get_region_data(aws_region, AWS_OLD_INSTANCES_DATA_URL) or {}
+        latest_region_data = get_region_data(aws_region, AWS_INSTANCES_DATA_URL) or {}
+        update_aws_data_region(aws_region, old_region_data, latest_region_data)
 
 
 if __name__ == '__main__':

data/aws_data_dal.py

@@ -25,12 +25,12 @@ def get_aws_per_region_data(aws_region):
     if not aws_region:
         return None
     region_data = db.aws_data.find_one({'region': aws_region})
-    return region_data['data'] if region_data else None
+    return region_data
 
 
-def update_aws_data_region(aws_region, json_data):
+def update_aws_data_region(aws_region, json_latest_data, json_previous_data):
     return db.aws_data.replace_one(
         {'region': aws_region},
-        {'region': aws_region, 'data': json_data},
+        {'region': aws_region, 'data_latest': json_latest_data, 'data_previous': json_previous_data},
         upsert=True,
     )