feat: added renku service with cache and datasets

Author: jsam

Dockerfile renamed to Dockerfile.cli

@@ -1,4 +1,4 @@
-FROM python:3.6-alpine as base
+FROM python:3.7-alpine as base
 
 RUN apk add --no-cache git && \
     pip install --no-cache --upgrade pip

Dockerfile.svc

@@ -0,0 +1,18 @@
+FROM python:3.7-alpine
+
+RUN apk add --update --no-cache alpine-sdk g++ gcc linux-headers libxslt-dev python3-dev build-base openssl-dev libffi-dev git && \
+    pip install --no-cache --upgrade pip setuptools pipenv requirements-builder
+
+RUN apk add --no-cache --allow-untrusted \
+    --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/community \
+    --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/main \
+    --repository http://nl.alpinelinux.org/alpine/edge/community \
+    git-lfs && \
+    git lfs install
+
+COPY . /code/renku
+WORKDIR /code/renku
+RUN requirements-builder -e all --level=pypi setup.py > requirements.txt && pip install -r requirements.txt && pip install -e . && pip install gunicorn
+
+
+ENTRYPOINT ["gunicorn", "renku.service.entrypoint:app", "-b", "0.0.0.0:8080"]

MANIFEST.in

@@ -39,6 +39,7 @@ include babel.ini
 include brew.py
 include pytest.ini
 include snap/snapcraft.yaml
+recursive-include renku *.json
 recursive-include .github CODEOWNERS
 recursive-include .travis *.sh
 recursive-include docs *.bat

Pipfile.lock

@@ -30,6 +30,8 @@
 import responses
 from click.testing import CliRunner
 
+from renku.service.entrypoint import create_app
+
 
 @pytest.fixture(scope='module')
 def renku_path(tmpdir_factory):
@@ -474,3 +476,40 @@ def sleep_after():
     import time
     yield
     time.sleep(0.5)
+
+
+@pytest.fixture(scope='module')
+def svc_client():
+    """Renku service client."""
+    flask_app = create_app()
+
+    testing_client = flask_app.test_client()
+    testing_client.testing = True
+
+    ctx = flask_app.app_context()
+    ctx.push()
+
+    yield testing_client
+
+    ctx.pop()
+
+
+@pytest.fixture(scope='function')
+def svc_client_with_repo(svc_client):
+    """Renku service remote repository."""
+    access_token = 'contact:EcfPJvEqjJepyu6XyqKZ'
+    remote_url = 'https://{0}@renkulab.io/gitlab/contact/integration-tests.git'
+    headers = {'Authorization': 'Bearer b4b4de0eda0f471ab82702bd5c367fa7'}
+
+    params = {'git_url': remote_url.format(access_token), 'force': 1}
+
+    response = svc_client.get(
+        '/cache/project-clone', query_string=params, headers=headers
+    )
+
+    assert response
+    assert 'result' in response.json
+    assert 'error' not in response.json
+    assert 'integration-tests' == response.json['result']['project_id']
+
+    yield svc_client, headers

conftest.py

@@ -25,6 +25,8 @@
 import yaml
 
 from renku.core.management import LocalClient
+from renku.core.management.config import RENKU_HOME
+from renku.core.management.repository import default_path
 
 from .git import get_git_isolation
 
@@ -63,8 +65,17 @@ def pass_local_client(
         )
 
     def new_func(*args, **kwargs):
-        ctx = click.get_current_context()
-        client = ctx.ensure_object(LocalClient)
+        ctx = click.get_current_context(silent=True)
+        if not ctx:
+            client = LocalClient(
+                path=default_path(),
+                renku_home=RENKU_HOME,
+                use_external_storage=True,
+            )
+            ctx = click.Context(click.Command(method))
+        else:
+            client = ctx.ensure_object(LocalClient)
+
         stack = contextlib.ExitStack()
 
         # Handle --isolation option:
@@ -85,8 +96,11 @@ def new_func(*args, **kwargs):
         if lock or (lock is None and commit):
             stack.enter_context(client.lock)
 
-        with stack:
-            result = ctx.invoke(method, client, *args, **kwargs)
+        result = None
+        if ctx:
+            with stack:
+                result = ctx.invoke(method, client, *args, **kwargs)
+
         return result
 
     return functools.update_wrapper(new_func, method)

renku/core/commands/client.py

@@ -106,7 +106,7 @@ def create_dataset(client, name, handle_duplicate_fn=None):
     :raises: ``renku.core.errors.ParameterError``
     """
     existing = client.load_dataset(name=name)
-    if (not existing or handle_duplicate_fn and handle_duplicate_fn(existing)):
+    if not existing or handle_duplicate_fn and handle_duplicate_fn(existing):
         with client.with_dataset(name=name) as dataset:
             creator = Creator.from_git(client.repo)
             if creator not in dataset.creator:
@@ -143,9 +143,12 @@ def add_file(
     sources=(),
     destination='',
     with_metadata=None,
-    urlscontext=contextlib.nullcontext
+    urlscontext=contextlib.nullcontext,
+    use_external_storage=True
 ):
     """Add data file to a dataset."""
+    client.use_external_storage = use_external_storage
+
     add_to_dataset(
         client, urls, name, link, force, sources, destination, with_metadata,
         urlscontext

renku/core/commands/dataset.py

@@ -296,7 +296,8 @@ def _add_from_url(self, dataset, dataset_path, url, link, destination):
         mode = dst.stat().st_mode & 0o777
         dst.chmod(mode & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
 
-        self.track_paths_in_storage(str(dst.relative_to(self.path)))
+        if self.has_external_storage:
+            self.track_paths_in_storage(str(dst.relative_to(self.path)))
 
         return [{
             'path': dst.relative_to(self.path),

renku/core/management/datasets.py

@@ -47,13 +47,18 @@ def default_path():
         return '.'
 
 
+def path_converter(path):
+    """Converter for path in PathMixin."""
+    return Path(path).resolve()
+
+
 @attr.s
 class PathMixin:
     """Define a default path attribute."""
 
     path = attr.ib(
         default=default_path,
-        converter=lambda arg: Path(arg).resolve().absolute(),
+        converter=path_converter,
     )
 
     @path.validator

renku/core/management/repository.py

@@ -0,0 +1,6 @@
+FROM python:3.6
+ADD . /app
+WORKDIR /app
+RUN pip install -e .
+EXPOSE 8000
+CMD ["gunicorn", "-b", "0.0.0.0:8000", "app"]

renku/service/Dockerfile

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku service."""

renku/service/__init__.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku service config."""
+import os
+import tempfile
+from pathlib import Path
+
+INVALID_PARAMS_ERROR_CODE = -32602
+INTERNAL_FAILURE_ERROR_CODE = -32603
+
+API_VERSION = 'v1'
+
+SWAGGER_URL = '/api/docs'
+API_URL = os.getenv(
+    'RENKU_SVC_SWAGGER_URL', '/api/{0}/spec'.format(API_VERSION)
+)
+
+UPLOAD_FOLDER = tempfile.TemporaryDirectory()
+
+CACHE_UPLOADS_PATH = Path(UPLOAD_FOLDER.name) / Path('uploads')
+CACHE_PROJECTS_PATH = Path(UPLOAD_FOLDER.name) / Path('projects')
+
+ALLOWED_EXTENSIONS = {'txt', 'pdf', 'csv'}
+
+JWT_ALGORITHM = 'HS256'
+JWT_KEY = 'renku-svc-secret-key'

renku/service/config.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku service entry point."""
+import os
+import uuid
+
+from flask import Flask
+from flask_swagger_ui import get_swaggerui_blueprint
+
+from renku.service.config import API_URL, API_VERSION, CACHE_PROJECTS_PATH, \
+    CACHE_UPLOADS_PATH, SWAGGER_URL, UPLOAD_FOLDER
+from renku.service.views.cache import clone_repository_view, \
+    list_projects_view, list_uploaded_files_view, upload_files_view
+from renku.service.views.datasets import add_file_to_dataset_view, \
+    create_dataset_view, list_dataset_files_view, list_datasets_view
+from renku.service.views.docs import api_docs_view
+
+
+def make_cache():
+    """Create cache structure."""
+    sub_dirs = [CACHE_UPLOADS_PATH, CACHE_PROJECTS_PATH]
+
+    for subdir in sub_dirs:
+        if not subdir.exists():
+            subdir.mkdir()
+
+
+def create_app():
+    """Creates a Flask app with necessary configuration."""
+    app = Flask(__name__)
+    app.secret_key = os.getenv('RENKU_SVC_SERVICE_KEY', uuid.uuid4().hex)
+
+    app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER.name
+    app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
+
+    make_cache()
+    build_routes(app)
+
+    return app
+
+
+def build_routes(app):
+    """Register routes to given app instance."""
+    swaggerui_blueprint = get_swaggerui_blueprint(
+        SWAGGER_URL, API_URL, config={'app_name': 'RenkuSvc'}
+    )
+
+    app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL)
+
+    app.add_url_rule(
+        '/api/{0}/spec'.format(API_VERSION),
+        'docs',
+        api_docs_view,
+        methods=['GET']
+    )
+
+    app.add_url_rule(
+        '/cache/files-list',
+        'list_files',
+        list_uploaded_files_view,
+        methods=['GET']
+    )
+
+    app.add_url_rule(
+        '/cache/files-upload',
+        'upload_files',
+        upload_files_view,
+        methods=['POST']
+    )
+
+    app.add_url_rule(
+        '/cache/project-clone',
+        'clone_project',
+        clone_repository_view,
+        methods=['GET']
+    )
+
+    app.add_url_rule(
+        '/cache/project-list',
+        'list_projects',
+        list_projects_view,
+        methods=['GET']
+    )
+
+    app.add_url_rule(
+        '/datasets/list', 'list_datasets', list_datasets_view, methods=['GET']
+    )
+
+    app.add_url_rule(
+        '/datasets/files',
+        'list_dataset_files',
+        list_dataset_files_view,
+        methods=['GET']
+    )
+
+    app.add_url_rule(
+        '/datasets/add',
+        'add_dataset_file',
+        add_file_to_dataset_view,
+        methods=['GET']
+    )
+
+    app.add_url_rule(
+        '/datasets/create',
+        'create_data',
+        create_dataset_view,
+        methods=['GET']
+    )
+
+
+app = create_app()
+
+if __name__ == '__main__':
+    app.run()

renku/service/entrypoint.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku service management commands."""