HaaS lives! (hail-is#4589)

* added ServerBackend

* wip

* fixed rebase

hail/python/hail-apiserver/hail-apiserver.py

@@ -0,0 +1,32 @@
+import hail as hl
+
+from hail.utils.java import Env, info
+
+import logging
+import flask
+
+hl.init()
+
+app = flask.Flask('hail-apiserver')
+
+@app.route('/execute', methods=['POST'])
+def execute():
+    code = flask.request.json
+
+    info(f'execute: {code}')
+
+    jir = Env.hail().expr.Parser.parse_value_ir(code, {}, {})
+
+    typ = hl.HailType._from_java(jir.typ())
+    value = Env.hail().expr.ir.Interpret.interpretPyIR(code, {}, {})
+
+    result = {
+        'type': str(typ),
+        'value': value
+    }
+
+    info(f'result: {result}')
+
+    return flask.jsonify(result)
+
+app.run(threaded=False, host='0.0.0.0')

hail/python/hail/backend/__init__.py

@@ -0,0 +1,7 @@
+from .backend import *
+
+__all__ = [
+    'Backend',
+    'SparkBackend',
+    'ServiceBackend'
+]

hail/python/hail/utils/backend.py → hail/python/hail/backend/backend.py

@@ -25,3 +25,27 @@ def interpret(self, ir):
         result = Env.hail().expr.ir.Interpret.interpretPyIR(code, {}, ir_map)
 
         return typ._from_json(result)
+
+
+class ServiceBackend(Backend):
+    def __init__(self, host, port=80, scheme='http'):
+        self.scheme = scheme
+        self.host = host
+        self.port = port
+
+    def interpret(self, ir):
+        assert isinstance(ir, hail.ir.IR)
+
+        r = Renderer(stop_at_jir=True)
+        code = r(ir)
+        assert len(r.jirs) == 0
+
+        resp = requests.post(f'http://hail-apiserver:5000/execute', json=code)
+        resp.raise_for_status()
+
+        resp_json = resp.json()
+
+        typ = dtype(resp_json['type'])
+        result = resp_json['value']
+
+        return typ._from_json(result)

hail/python/hail/context.py

@@ -7,7 +7,7 @@
 from hail.typecheck import nullable, typecheck, typecheck_method, enumeration
 from hail.utils import wrap_to_list, get_env_or_default
 from hail.utils.java import Env, joption, FatalError, connect_logger, install_exception_handler, uninstall_exception_handler
-from hail.utils.backend import SparkBackend
+from hail.backend import Backend, ServiceBackend, SparkBackend
 
 import sys
 import os
@@ -26,12 +26,13 @@ class HailContext(object):
                       tmp_dir=nullable(str),
                       default_reference=str,
                       idempotent=bool,
-                      global_seed=nullable(int))
+                      global_seed=nullable(int),
+                      _backend=nullable(Backend))
     def __init__(self, sc=None, app_name="Hail", master=None, local='local[*]',
                  log=None, quiet=False, append=False,
                  min_block_size=1, branching_factor=50, tmp_dir=None,
                  default_reference="GRCh37", idempotent=False,
-                 global_seed=6348563392232659379):
+                 global_seed=6348563392232659379, _backend=None):
 
         if Env._hc:
             if idempotent:
@@ -65,7 +66,9 @@ def __init__(self, sc=None, app_name="Hail", master=None, local='local[*]',
 
         jsc = sc._jsc.sc() if sc else None
 
-        self._backend = SparkBackend()
+        if _backend is None:
+            _backend = SparkBackend()
+        self._backend = _backend
 
         tmp_dir = get_env_or_default(tmp_dir, 'TMPDIR', '/tmp')
 
@@ -168,12 +171,13 @@ def upload_log(self):
            tmp_dir=str,
            default_reference=enumeration('GRCh37', 'GRCh38'),
            idempotent=bool,
-           global_seed=nullable(int))
+           global_seed=nullable(int),
+           _backend=nullable(Backend))
 def init(sc=None, app_name='Hail', master=None, local='local[*]',
          log=None, quiet=False, append=False,
          min_block_size=1, branching_factor=50, tmp_dir='/tmp',
          default_reference='GRCh37', idempotent=False,
-         global_seed=6348563392232659379):
+         global_seed=6348563392232659379, _backend=None):
     """Initialize Hail and Spark.
 
     Parameters
@@ -209,7 +213,7 @@ def init(sc=None, app_name='Hail', master=None, local='local[*]',
     """
     HailContext(sc, app_name, master, local, log, quiet, append,
                 min_block_size, branching_factor, tmp_dir,
-                default_reference, idempotent, global_seed)
+                default_reference, idempotent, global_seed, _backend)
 
 def stop():
     """Stop the currently running Hail session."""