diff --git a/hail/python/hail-apiserver/hail-apiserver.py b/hail/python/hail-apiserver/hail-apiserver.py new file mode 100644 index 00000000000..5866c6bbb95 --- /dev/null +++ b/hail/python/hail-apiserver/hail-apiserver.py @@ -0,0 +1,32 @@ +import hail as hl + +from hail.utils.java import Env, info + +import logging +import flask + +hl.init() + +app = flask.Flask('hail-apiserver') + +@app.route('/execute', methods=['POST']) +def execute(): + code = flask.request.json + + info(f'execute: {code}') + + jir = Env.hail().expr.Parser.parse_value_ir(code, {}, {}) + + typ = hl.HailType._from_java(jir.typ()) + value = Env.hail().expr.ir.Interpret.interpretPyIR(code, {}, {}) + + result = { + 'type': str(typ), + 'value': value + } + + info(f'result: {result}') + + return flask.jsonify(result) + +app.run(threaded=False, host='0.0.0.0') diff --git a/hail/python/hail/backend/__init__.py b/hail/python/hail/backend/__init__.py new file mode 100644 index 00000000000..c6a6df1449e --- /dev/null +++ b/hail/python/hail/backend/__init__.py @@ -0,0 +1,7 @@ +from .backend import * + +__all__ = [ + 'Backend', + 'SparkBackend', + 'ServiceBackend' +] diff --git a/hail/python/hail/utils/backend.py b/hail/python/hail/backend/backend.py similarity index 50% rename from hail/python/hail/utils/backend.py rename to hail/python/hail/backend/backend.py index 539b1df70c9..49333c9cab7 100644 --- a/hail/python/hail/utils/backend.py +++ b/hail/python/hail/backend/backend.py @@ -25,3 +25,27 @@ def interpret(self, ir): result = Env.hail().expr.ir.Interpret.interpretPyIR(code, {}, ir_map) return typ._from_json(result) + + +class ServiceBackend(Backend): + def __init__(self, host, port=80, scheme='http'): + self.scheme = scheme + self.host = host + self.port = port + + def interpret(self, ir): + assert isinstance(ir, hail.ir.IR) + + r = Renderer(stop_at_jir=True) + code = r(ir) + assert len(r.jirs) == 0 + + resp = requests.post(f'http://hail-apiserver:5000/execute', json=code) + resp.raise_for_status() + + resp_json = resp.json() + + typ = dtype(resp_json['type']) + result = resp_json['value'] + + return typ._from_json(result) diff --git a/hail/python/hail/context.py b/hail/python/hail/context.py index ef249e7585d..9b985c9ccd6 100644 --- a/hail/python/hail/context.py +++ b/hail/python/hail/context.py @@ -7,7 +7,7 @@ from hail.typecheck import nullable, typecheck, typecheck_method, enumeration from hail.utils import wrap_to_list, get_env_or_default from hail.utils.java import Env, joption, FatalError, connect_logger, install_exception_handler, uninstall_exception_handler -from hail.utils.backend import SparkBackend +from hail.backend import Backend, ServiceBackend, SparkBackend import sys import os @@ -26,12 +26,13 @@ class HailContext(object): tmp_dir=nullable(str), default_reference=str, idempotent=bool, - global_seed=nullable(int)) + global_seed=nullable(int), + _backend=nullable(Backend)) def __init__(self, sc=None, app_name="Hail", master=None, local='local[*]', log=None, quiet=False, append=False, min_block_size=1, branching_factor=50, tmp_dir=None, default_reference="GRCh37", idempotent=False, - global_seed=6348563392232659379): + global_seed=6348563392232659379, _backend=None): if Env._hc: if idempotent: @@ -65,7 +66,9 @@ def __init__(self, sc=None, app_name="Hail", master=None, local='local[*]', jsc = sc._jsc.sc() if sc else None - self._backend = SparkBackend() + if _backend is None: + _backend = SparkBackend() + self._backend = _backend tmp_dir = get_env_or_default(tmp_dir, 'TMPDIR', '/tmp') @@ -168,12 +171,13 @@ def upload_log(self): tmp_dir=str, default_reference=enumeration('GRCh37', 'GRCh38'), idempotent=bool, - global_seed=nullable(int)) + global_seed=nullable(int), + _backend=nullable(Backend)) def init(sc=None, app_name='Hail', master=None, local='local[*]', log=None, quiet=False, append=False, min_block_size=1, branching_factor=50, tmp_dir='/tmp', default_reference='GRCh37', idempotent=False, - global_seed=6348563392232659379): + global_seed=6348563392232659379, _backend=None): """Initialize Hail and Spark. Parameters @@ -209,7 +213,7 @@ def init(sc=None, app_name='Hail', master=None, local='local[*]', """ HailContext(sc, app_name, master, local, log, quiet, append, min_block_size, branching_factor, tmp_dir, - default_reference, idempotent, global_seed) + default_reference, idempotent, global_seed, _backend) def stop(): """Stop the currently running Hail session."""