Training and Serving Pipeline leveraging WML (#800)

* initial watson pipeline

* updates

* updates

* Update creds

* updated dockerhub base files

* update watson pipeline (#2)

Removing references to wml-base image, and recreating it with python-slim as base image

* address review comment (#3)

* change s3 to cos (#4)

components/ibm-components/OWNERS

@@ -1,4 +1,6 @@
 approvers:
   - animeshsingh
 reviewers:
-  - animeshsingh
+  - animeshsingh
+  - tomcli
+  - adrian555

components/ibm-components/commons/config/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.6-slim
+
+# Directories for model codes and secrets
+RUN mkdir /app
+
+# Install curl and kubectl
+RUN apt-get update
+RUN apt-get install -y curl gnupg
+RUN apt-get install -y apt-transport-https
+RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+RUN echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list
+RUN apt-get update
+RUN apt-get install -y kubectl
+
+# Directory for secrets
+COPY src/config.py /app

components/ibm-components/commons/config/src/config.py

@@ -0,0 +1,58 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--token', type=str, required=True)
+    parser.add_argument('--url', type=str, required=True)
+    parser.add_argument('--name', type=str)
+    args = parser.parse_args()
+
+    access_token = args.token
+    config_file_path = args.url
+
+    # download config file
+    # the default creds.ini is in the public accesible github repo
+    import subprocess
+    import os
+    config_file = os.path.basename(config_file_path)
+    config_local_path = os.path.join('/tmp', config_file)
+    command = ['curl', '-H', 'Authorization: token %s' % access_token, '-L', '-o', config_local_path, config_file_path]
+    subprocess.run(command)
+
+    secret_name = args.name
+    if (not secret_name):
+        secret_name = 'ai-pipeline-' + os.path.splitext(config_file)[0]
+    command = ['kubectl', 'delete', 'secret', secret_name]
+    subprocess.run(command)
+
+    # gather all secrets
+    command = ['kubectl', 'create', 'secret', 'generic', secret_name]
+
+    import configparser
+    config = configparser.ConfigParser()
+    config.read(config_local_path)
+    for section in config.sections():
+        for key in config[section]:
+            command.append('--from-literal=%s=\'%s\'' % (key, config[section][key]))
+
+    # create the secret
+    subprocess.run(command)
+
+    # verify secret is created
+    subprocess.run(['kubectl', 'describe', 'secret', secret_name])
+
+    # indicate that secret is created
+    with open("/tmp/" + secret_name, "w") as f:
+        f.write('created')
+    f.close()

components/ibm-components/watson/deploy/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.6-slim
+
+# Directories for model codes and secrets
+RUN mkdir /app
+RUN mkdir /app/secrets
+
+# Watson studio and machine learning python client
+RUN pip install watson_machine_learning_client minio
+
+# Python functions with endpoints to Watson Machine Learning
+COPY src/wml-deploy.py /app

components/ibm-components/watson/deploy/src/wml-deploy.py

@@ -0,0 +1,88 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# define the function to deploy the model
+
+def getSecret(secret):
+    with open(secret, 'r') as f:
+        res = f.readline().strip('\'')
+    f.close()
+    return res
+
+def deploy(args):
+    from watson_machine_learning_client import WatsonMachineLearningAPIClient
+    from minio import Minio
+    import os
+
+    wml_model_name = args.model_name
+    wml_scoring_payload = args.scoring_payload
+    model_uid = args.model_uid
+
+    # retrieve credentials
+    wml_url = getSecret("/app/secrets/wml_url")
+    wml_username = getSecret("/app/secrets/wml_username")
+    wml_password = getSecret("/app/secrets/wml_password")
+    wml_instance_id = getSecret("/app/secrets/wml_instance_id")
+
+    cos_endpoint = getSecret("/app/secrets/cos_endpoint")
+    cos_access_key = getSecret("/app/secrets/cos_access_key")
+    cos_secret_key = getSecret("/app/secrets/cos_secret_key")
+
+    cos_input_bucket = getSecret("/app/secrets/cos_input_bucket")
+
+    # set up the WML client
+    wml_credentials = {
+                       "url": wml_url,
+                       "username": wml_username,
+                       "password": wml_password,
+                       "instance_id": wml_instance_id
+                      }
+    client = WatsonMachineLearningAPIClient( wml_credentials )
+
+    # deploy the model
+    deployment_name    = wml_model_name
+    deployment_desc    = "deployment of %s" %wml_model_name
+    deployment       = client.deployments.create( model_uid, deployment_name, deployment_desc )
+    scoring_endpoint = client.deployments.get_scoring_url( deployment )
+    print( "scoring_endpoint: ", scoring_endpoint )
+
+    # download scoring payload
+    payload_file = os.path.join('/app', wml_scoring_payload)
+
+    cos = Minio(cos_endpoint,
+               access_key = cos_access_key,
+               secret_key = cos_secret_key)
+    cos.fget_object(cos_input_bucket, wml_scoring_payload, payload_file)
+
+    # scoring the deployment
+    import json
+    with open( payload_file ) as data_file: 
+        test_data = json.load( data_file )
+    payload = test_data[ 'payload' ]
+    data_file.close()
+
+    print("Scoring result: ")
+    result = client.deployments.score( scoring_endpoint, payload )
+    print(result)
+
+    with open("/tmp/output", "w") as f:
+        print(result, file=f)
+    f.close()
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--model-name', type=str, required=True)
+    parser.add_argument('--scoring-payload', type=str, required=True)
+    parser.add_argument('--model-uid', type=str, required=True)
+    args = parser.parse_args()
+    deploy(args)

components/ibm-components/watson/store/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.6-slim
+
+# Directories for model codes and secrets
+RUN mkdir /app
+RUN mkdir /app/secrets
+
+# Watson studio and machine learning python client
+RUN pip install watson_machine_learning_client minio
+
+# Python functions with endpoints to Watson Machine Learning
+COPY src/wml-store.py /app

components/ibm-components/watson/store/src/wml-store.py

@@ -0,0 +1,58 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+# define the function to store the model
+
+def getSecret(secret):
+    with open(secret, 'r') as f:
+        res = f.readline().strip('\'')
+    f.close()
+    return res
+
+def store(wml_model_name, run_uid):
+    from watson_machine_learning_client import WatsonMachineLearningAPIClient
+
+    # retrieve credentials
+    wml_url = getSecret("/app/secrets/wml_url")
+    wml_username = getSecret("/app/secrets/wml_username")
+    wml_password = getSecret("/app/secrets/wml_password")
+    wml_instance_id = getSecret("/app/secrets/wml_instance_id")
+
+    # set up the WML client
+    wml_credentials = {
+                       "url": wml_url,
+                       "username": wml_username,
+                       "password": wml_password,
+                       "instance_id": wml_instance_id
+                      }
+    client = WatsonMachineLearningAPIClient( wml_credentials )
+
+    # store the model
+    stored_model_name    = wml_model_name
+    stored_model_details = client.repository.store_model( run_uid, stored_model_name )
+    model_uid            = client.repository.get_model_uid( stored_model_details )
+    print( "model_uid: ", model_uid )
+
+    with open("/tmp/model_uid", "w") as f:
+        f.write(model_uid)
+    f.close()
+
+    import time
+    time.sleep(120)
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--model-name', type=str, required=True)
+    parser.add_argument('--run-uid', type=str, required=True)
+    args = parser.parse_args()
+    store(args.model_name, args.run_uid)

components/ibm-components/watson/train/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.6-slim
+
+# Directories for model codes and secrets
+RUN mkdir /app
+RUN mkdir /app/secrets
+
+# Watson studio and machine learning python client
+RUN pip install watson_machine_learning_client minio
+
+# Python functions with endpoints to Watson Machine Learning
+COPY src/wml-train.py /app