From 442c80449d4253a66fb247ea345f5f82422c6632 Mon Sep 17 00:00:00 2001 From: Yaron Haviv Date: Wed, 8 May 2019 11:58:33 +0300 Subject: [PATCH] add nuclio components (to build/deploy, delete, invoke functions) (#1295) * add support for flexible config (via env var) for the pipline service and UI, fix broken links (pointed to API vs UI service) * add namespace and change address to endpoint --- components/nuclio/README.md | 44 ++++++++++++++++++++++++ components/nuclio/delete/component.yaml | 14 ++++++++ components/nuclio/deploy/component.yaml | 28 +++++++++++++++ components/nuclio/invoker/component.yaml | 21 +++++++++++ 4 files changed, 107 insertions(+) create mode 100644 components/nuclio/README.md create mode 100644 components/nuclio/delete/component.yaml create mode 100644 components/nuclio/deploy/component.yaml create mode 100644 components/nuclio/invoker/component.yaml diff --git a/components/nuclio/README.md b/components/nuclio/README.md new file mode 100644 index 00000000000..81a3141e89f --- /dev/null +++ b/components/nuclio/README.md @@ -0,0 +1,44 @@ +# Nuclio (Serverless) Components + +[Nuclio](https://nuclio.io/) is a native and high performance serverless platform over Kubernetes +which automate the process of build, deployment, monitoring, and auto-scaling of micro-services. +Nuclio support variety of data and data-science related features (e.g. stream processing, +GPUs, volume/DB mounts, high concurrency, etc.) + +To install Nuclio over Kubernetes follow the [instruction in Github](https://github.com/nuclio/nuclio), +or this [interactive tutorial](https://www.katacoda.com/javajon/courses/kubernetes-serverless/nuclio). + +Nuclio functions can be used in the following ML pipline tasks: +* Data collectors, ETL, stream processing +* Data preparation and analysis +* Hyper parameter model training +* Real-time model serving +* Feature vector assembly (real-time data preparation) + +Read more on the use of Nuclio in [data-science here](https://towardsdatascience.com/serverless-can-it-simplify-data-science-projects-8821369bf3bd). +Nuclio functions can be generated automatically from 8 code languages, from Jupyter Notebooks, Zip, Git, Docker, etc. +The [nuclio-jupyter repo](https://github.com/nuclio/nuclio-jupyter) provide guidance and many examples. + +## Components + +There are currently 3 components in this package: +* [deploy](deploy/component.yaml) - Automatically build and deploy/re-deploy functions + from code/zip/notebooks/git/.. and/or override various deployment configurations such as + setting cpu/mem/gpu resources, scaling, environment variables, triggers, etc. +* [delete](delete/component.yaml) - Delete a function +* [invoker](invoker/component.yaml) - invoke a function and return the results/logs + +Additional components and examples will be added soon for parallel batch/stream processing + +## Examples + +**Deploy a function (from Github)** + +```python +nuclio_dep = kfp.components.load_component_from_file('deploy/component.yaml') + +def my_pipeline(): + new_func = nuclio_dep(url='git://github.com/nuclio/nuclio#master:/hack/examples/python/helloworld', name='myfunc', project='myproj', tag='0.11') + + ... +``` diff --git a/components/nuclio/delete/component.yaml b/components/nuclio/delete/component.yaml new file mode 100644 index 00000000000..4f900e3035a --- /dev/null +++ b/components/nuclio/delete/component.yaml @@ -0,0 +1,14 @@ +name: nuclio delete +description: delete a nuclio function. +inputs: + - {name: Name, type: String, description: 'function name'} + - {name: Namespace, type: String, description: 'Kubernetes namespace', default: ''} + - {name: Dashboard, type: String, description: 'nuclio dashboard service url', default: 'http://nuclio-dashboard.nuclio:8070'} +implementation: + container: + image: nuclio/pydeploy + command: [ + python, -m, nuclio, del, {inputValue: Name}, + --dashboard-url, {inputValue: Dashboard}, + --namespace, {inputValue: Namespace}, + ] \ No newline at end of file diff --git a/components/nuclio/deploy/component.yaml b/components/nuclio/deploy/component.yaml new file mode 100644 index 00000000000..b442b43dd47 --- /dev/null +++ b/components/nuclio/deploy/component.yaml @@ -0,0 +1,28 @@ +name: nuclio deploy +description: auto build and deploy nuclio function. +inputs: + - {name: Url, type: String, description: 'url/path to source code, zip archive, git path, or notebook'} + - {name: Name, type: String, description: 'function name'} + - {name: Project, type: String, description: 'project name', default: 'default'} + - {name: Tag, type: String, description: 'function version tag', default: ''} + - {name: Dashboard, type: String, description: 'nuclio dashboard service url', default: 'http://nuclio-dashboard.nuclio:8070'} + - {name: Spec, type: String, description: 'override function spec, Json {key: value, ..}', default: ''} + - {name: Env, type: String, description: 'override function env var, Json {key: value, ..}', default: ''} + - {name: Mount, type: String, description: 'volume mount, [vol-type:]:', default: ''} +outputs: + - {name: Endpoint, type: String, description: 'function endpoint url'} +implementation: + container: + image: nuclio/pydeploy + command: [ + python, -m, nuclio, deploy, {inputValue: Url}, + --dashboard-url, {inputValue: Dashboard}, + --name, {inputValue: Name}, + --project, {inputValue: Project}, + --tag, {inputValue: Tag}, + --env-json, {inputValue: Env}, + --spec-json, {inputValue: Spec}, + --mount, {inputValue: Mount}, + ] + fileOutputs: + Endpoint: /tmp/output \ No newline at end of file diff --git a/components/nuclio/invoker/component.yaml b/components/nuclio/invoker/component.yaml new file mode 100644 index 00000000000..3f0df3e42f1 --- /dev/null +++ b/components/nuclio/invoker/component.yaml @@ -0,0 +1,21 @@ +name: nuclio invoker +description: invoke nuclio function. +inputs: + - {name: Url, type: String, description: 'function URL/endpoint'} + - {name: Body, type: String, description: 'request body', default: ''} + - {name: Log level, type: String, description: 'log level', default: 'info'} + - {name: Method, type: String, description: 'HTTP method GET|POST|..', default: ''} +outputs: + - {name: output, type: String, description: 'function output'} +implementation: + container: + image: nuclio/invoker + command: [ + invoke, + -a, {inputValue: Url}, + -b, {inputValue: Body}, + -m, {inputValue: Method}, + -l, {inputValue: Log level}, + ] + fileOutputs: + output: /tmp/output \ No newline at end of file