Major commit supporting the installation/setup of Atlas.ai models and the downloading of their results

Author: brandomr

Atlas-Integration/README.md

@@ -0,0 +1,5 @@
+# Setting up Atlas.ai 
+
+You should ensure that the `DATA_PATH` in `config.ini` matches whatever you set as the `output_directory` when you run `atlas_data.py`.
+
+Also, ensure that `URL` in `config.ini` matches your development or production server (e.g. `http://localhost:8080` or `https://model-service.worldmodelers.com`).

Atlas-Integration/atlas_data.py

@@ -1,9 +1,32 @@
 import boto3
 import os
 import argparse
+import redis
+import configparser
+from hashlib import sha256
+import json
+
+config = configparser.ConfigParser()
+config.read('../REST-Server/config.ini')
+
+r = redis.Redis(host=config['REDIS']['HOST'],
+                port=config['REDIS']['PORT'],
+                db=config['REDIS']['DB'])
 
 profile = "wmuser"
 bucket_name = "world-modelers"
+models = ['consumption_model','asset_wealth_model']
+formats = ['tif','geojson']
+atlas_lookup = {
+               'asset_wealth_model':{
+                  'geojson':'november_tests_asset_wealth.geojson',
+                  'tif':'november_tests_atlasai_assetwealth_allyears_2km.tif'
+               },
+               'consumption_model':{
+                  'geojson':'november_tests_consumption.geojson',
+                  'tif':'november_tests_atlasai_consumption_allyears_2km.tif'
+               }
+            }
 
 session = boto3.Session(profile_name=profile)
 
@@ -14,9 +37,9 @@
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Atlas.a Data Downloader')
-    parser.add_argument('output_directory', type=str,
+    parser.add_argument('--output_directory', type=str,
                         help='Where should Atlas.ai data be downloaded?')
-    parser.add_argument('key_file', type=str,
+    parser.add_argument('--key_file', type=str,
                         help='A text file containing the S3 keys for the Atlas files.')
 
     args = parser.parse_args()
@@ -28,7 +51,40 @@
     # make Atlas.ai data directory
     if not os.path.exists(output_directory):
         os.mkdir(output_directory)
+        print(f"Created {output_directory} directory.\n")
 
+    # download files
+    print("Downloading data files...")
     for k in keys:
         file_name = k.split('/')[-1]
         bucket.download_file(k, f"{output_directory}/{file_name}")
+    print("...download completed!\n")
+
+    # update redis with model run
+    print("Updating Redis...")
+    for model_name in models:
+        for f in formats:
+            model_config = {
+              "config": {"format": f},
+              "name": model_name
+            }
+            
+            run_id = sha256(json.dumps(model_config).encode('utf-8')).hexdigest()
+            
+            # Add to model set in Redis
+            r.sadd(model_name, run_id)
+            
+            run_obj = {'status': 'SUCCESS',
+             'name': model_name,
+             'config': model_config["config"],
+            }
+
+            run_obj['config']['run_id'] = run_id
+            run_obj['config'] = json.dumps(run_obj['config'])
+            
+            r.hmset(run_id, run_obj)
+            
+            # rename files to correspond with model run_id
+            file_name = atlas_lookup[model_name][f]
+            os.rename(f"{output_directory}/{file_name}", f"{output_directory}/{run_id}.{f}")
+    print("...Redis update completed!")

Atlas-Integration/maas_install.sh

@@ -0,0 +1,2 @@
+# run Atlas setup script from this directory
+python atlas_data.py --output_directory=/home/ubuntu/data --key_file=atlas_s3_keys.txt

REST-Server/config.ini

@@ -1,3 +1,7 @@
+[APP]
+DATA_PATH = /home/ubuntu/data
+URL = https://model-service.worldmodelers.com
+
 [S3]
 BUCKET = world-modelers
 
@@ -20,4 +24,4 @@ OUTPUT_PATH = /home/ubuntu/dssat
 
 [MALNUTRITION]
 INSTALL_PATH = /home/ubuntu/ModelService/Kimetrica-Integration/darpa
-S3_CRED_PATH = /home/ubuntu/.aws
+S3_CRED_PATH = /home/ubuntu/.aws

REST-Server/openapi_server/controllers/execution_controller.py

@@ -1,5 +1,6 @@
 import connexion
 import six
+import flask
 
 from openapi_server.models.model_config import ModelConfig  # noqa: E501
 from openapi_server.models.run_results import RunResults  # noqa: E501
@@ -17,8 +18,12 @@
 import boto3
 import botocore
 import logging
+import os
 logging.basicConfig(level=logging.INFO)
 
+data_file_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')
+print(data_file_dir)
+
 config = configparser.ConfigParser()
 config.read('config.ini')
 
@@ -29,7 +34,15 @@
 client = docker.from_env()
 containers = client.containers
 
-available_models = ['population_model', 'malnutrition_model', 'fsc', 'dssat']
+data_path = config['APP']['DATA_PATH']
+site_url = config['APP']['URL']
+
+available_models = ['population_model', 
+                    'malnutrition_model', 
+                    'fsc', 
+                    'dssat',
+                    'asset_wealth_model',
+                    'consumption_model']
 
 def list_runs_model_name_get(ModelName):  # noqa: E501
     """Obtain a list of runs for a given model
@@ -131,16 +144,27 @@ def run_results_run_idget(RunID):  # noqa: E501
     if not r.exists(RunID):
         return 'Run Not Found', 404, {'x-error': 'not found'}
 
-    update_run_status(RunID)
     run = r.hgetall(RunID)
     status = run[b'status'].decode('utf-8')
+
+    # Only update the run status if the status is still PENDING
+    if status == 'PENDING':
+        update_run_status(RunID)
+        run = r.hgetall(RunID)
+        status = run[b'status'].decode('utf-8')
+
     config = json.loads(run[b'config'].decode('utf-8'))
     model_name = run[b'name'].decode('utf-8')
     output = ''
     output_config = {'config': config, 'name': model_name}
     results = {'status': status, 'config': output_config, 'output': output}
 
-    if status == 'SUCCESS':
+    if model_name in ['consumption_model', 'asset_wealth_model']:
+        # special handler for Atlas.ai models
+        URI = f"{url}/result_file/{RunID}.{config['format']}"
+        results['output'] = URI
+        return results 
+    elif status == 'SUCCESS':
         bucket = run[b'bucket'].decode('utf-8')
         key = run[b'key'].decode('utf-8')
         URI = f"https://s3.amazonaws.com/{bucket}/{key}"
@@ -185,6 +209,28 @@ def available_results_get():  # noqa: E501
     return results
 
 
+def result_file_result_file_name_get(ResultFileName):  # noqa: E501
+    """Obtain the result file for a given model run.
+
+    Submit a `ResultFileName` and receive model run result file. # noqa: E501
+
+    :param result_file_name: A file name of a result file.
+    :type result_file_name: str
+
+    :rtype: None
+    """
+
+    # If the file does not exist:
+    if not os.path.exists(f"{data_path}/{ResultFileName}"):
+        return 'Result File Not Found', 404, {'x-error': 'not found'}
+
+    # Otherwise, serve the file:
+    else:
+        response = flask.send_from_directory(data_path, ResultFileName)  
+        response.direct_passthrough = False
+        return response    
+
+
 def update_run_status(RunID):
     if not r.exists(RunID):
         return 'Run Not Found', 404, {'x-error': 'not found'}

REST-Server/openapi_server/openapi/openapi.yaml

@@ -268,6 +268,26 @@ paths:
       tags:
       - execution
       x-openapi-router-controller: openapi_server.controllers.execution_controller
+  /result_file/{ResultFileName}:
+    get:
+      description: Submit a `ResultFileName` and receive model run result file.
+      operationId: result_file_result_file_name_get
+      parameters:
+      - description: A file name of a result file.
+        explode: false
+        in: path
+        name: ResultFileName
+        required: true
+        schema:
+          $ref: '#/components/schemas/ResultFileName'
+        style: simple
+      responses:
+        200:
+          description: A result file
+      summary: Obtain the result file for a given model run.
+      tags:
+      - execution
+      x-openapi-router-controller: openapi_server.controllers.execution_controller
 components:
   schemas:
     ModelName:
@@ -604,6 +624,10 @@ components:
       description: ID associated with a model run. This is the SHA256 hash of the ModelConfig sent to the /run_model endpoint
       example: 3A3B3E0AE57AD4A7EF658C1F7832774F55E403F01FDF44B68B355EC4587D7A04
       type: string
+    ResultFileName:
+      description: The name of a model run result file.
+      example: 3A3B3E0AE57AD4A7EF658C1F7832774F55E403F01FDF44B68B355EC4587D7A04.csv
+      type: string
     RunStatus:
       description: Status information about a model run.
       enum:

REST-Server/openapi_server/test/test_execution_controller.py

@@ -14,6 +14,17 @@
 class TestExecutionController(BaseTestCase):
     """ExecutionController integration test stubs"""
 
+    def test_available_results_get(self):
+        """Test case for available_results_get
+
+        Obtain a list of run results
+        """
+        response = self.client.open(
+            '/available_results',
+            method='GET')
+        self.assert200(response,
+                       'Response body is : ' + response.data.decode('utf-8'))
+
     def test_list_runs_model_name_get(self):
         """Test case for list_runs_model_name_get
 
@@ -25,6 +36,17 @@ def test_list_runs_model_name_get(self):
         self.assert200(response,
                        'Response body is : ' + response.data.decode('utf-8'))
 
+    def test_result_file_run_idget(self):
+        """Test case for result_file_run_idget
+
+        Obtain the result file for a given model run.
+        """
+        response = self.client.open(
+            '/result_file/{RunID}'.format(run_id='run_id_example'),
+            method='GET')
+        self.assert200(response,
+                       'Response body is : ' + response.data.decode('utf-8'))
+
     def test_run_model_post(self):
         """Test case for run_model_post
 

REST-Server/openapi_server/test/test_exploration_controller.py

@@ -5,12 +5,12 @@
 from flask import json
 from six import BytesIO
 
+from openapi_server.models.error import Error  # noqa: E501
 from openapi_server.models.io_file import IOFile  # noqa: E501
 from openapi_server.models.io_request import IORequest  # noqa: E501
 from openapi_server.models.model import Model  # noqa: E501
 from openapi_server.models.model_config import ModelConfig  # noqa: E501
 from openapi_server.models.parameter import Parameter  # noqa: E501
-from openapi_server.models.search_result import SearchResult  # noqa: E501
 from openapi_server.models.unknownbasetype import UNKNOWN_BASE_TYPE  # noqa: E501
 from openapi_server.test import BaseTestCase
 

model_service_api.yaml

@@ -236,6 +236,22 @@ paths:
                 type: "array"
                 items:
                   $ref: '#/components/schemas/RunResults'                   
+  /result_file/{ResultFileName}:
+    get:
+      tags:
+      - "execution"
+      summary: "Obtain the result file for a given model run."
+      description: "Submit a `ResultFileName` and receive model run result file."
+      parameters:
+      - in: path
+        name: ResultFileName
+        description: "A file name of a result file."
+        required: true
+        schema:
+          $ref: "#/components/schemas/ResultFileName"
+      responses:
+        '200':
+          description: A result file
 
 components:
   securitySchemes:
@@ -492,6 +508,10 @@ components:
       type: "string"
       description: "ID associated with a model run. This is the SHA256 hash of the ModelConfig sent to the /run_model endpoint"
       example: "3A3B3E0AE57AD4A7EF658C1F7832774F55E403F01FDF44B68B355EC4587D7A04"
+    ResultFileName:
+      type: "string"
+      description: "The name of a model run result file."
+      example: "3A3B3E0AE57AD4A7EF658C1F7832774F55E403F01FDF44B68B355EC4587D7A04.csv"      
     RunStatus:
       type: "string"
       description: "Status information about a model run."