First commit with all basics of how to setup Metaflow for Kubernetes.

valayDave · valayDave · commit 91c0b810443f · 2020-03-01T00:12:38.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+scripts/
diff --git a/Metaflow_service/Readme.md b/Metaflow_service/Readme.md
@@ -0,0 +1,6 @@
+# Metaflow Service
+
+- Creates the Pods and Services for the Metaflow metadata service. 
+- Setups: 
+    - Postgres : Service, Deployment, Persistant Volume and Persistant Volume
+    - Metaflow_Service : Deployment,service 
diff --git a/Metaflow_service/database/postgres-db-deployment.yaml b/Metaflow_service/database/postgres-db-deployment.yaml
@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment                                # Type of the kubernetes resource
+metadata:
+  name: metaflow-database                       # Name of the deployment
+  labels:                                        # Labels applied to this deployment 
+    app: metaflow-database
+spec:
+  selector:
+    matchLabels:                                  # This deployment applies to the Pods matching the specified labels
+      app: metaflow-database
+      tier: postgres
+  replicas: 1
+  template:                                       # Template for the Pods in this deployment
+    metadata:
+      labels:                                     # Labels to be applied to the Pods in this deployment
+        app: metaflow-database
+        tier: postgres
+    spec:                                         # The spec for the containers that will be run inside the Pods in this deployment
+      containers:
+        - image: postgres:latest                  # The container image
+          name: postgres
+          imagePullPolicy: "IfNotPresent"
+          envFrom:                                # Get the environmental variables from a secret file whose name is "postgres-secret"
+            - secretRef:
+                name: postgres-secret
+            
+          ports:
+            - containerPort: 5432                   # The port that the container exposes 
+              name: postgres
+          volumeMounts:
+            - mountPath: /var/lib/postgresql/data
+              name: postgres-persistent-storage     # This name should match the name specified in `volumes.name`
+      volumes:                                      # A PersistentVolume is mounted as a volume to the Pod 
+        - name: postgres-persistent-storage
+          persistentVolumeClaim:
+            claimName: postgres-pv-claim
+
+
+
diff --git a/Metaflow_service/database/postgres-db-pv.yaml b/Metaflow_service/database/postgres-db-pv.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: PersistentVolume                        # Create a PV
+metadata:
+  name: postgres-pv-volume
+  labels:
+    type: local
+    app: metaflow-database
+spec:
+  storageClassName: manual                    # Storage class. A PV Claim requesting the same storageClass can be bound to this volume.
+  capacity:
+    storage: 5Gi
+  accessModes:
+    - ReadWriteMany
+  hostPath:                                   # hostPath PersistentVolume is used for development and testing. It uses a file/directory on the Node to emulate network-attached storage
+    path: "/mnt/data"
+  persistentVolumeReclaimPolicy: Retain       # Retain the PV even after PVC is deleted. 
diff --git a/Metaflow_service/database/postgres-db-pvc.yaml b/Metaflow_service/database/postgres-db-pvc.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim                   # Create a PersistentVolumeClaim to request a PersistentVolume storage
+metadata:                                     # Claim name and lab-pv-claim
+  name: postgres-pv-claim
+  labels:
+    app: metaflow-database
+spec:                                         # Access mode and resource limits
+  storageClassName: manual                    # Request a certain storage class
+  accessModes:
+    - ReadWriteMany                           # ReadWriteMany means the volume can be mounted as read-write by many Nodes
+  resources:
+    requests:
+      storage: 5Gi
diff --git a/Metaflow_service/database/postgres-db-service.yaml b/Metaflow_service/database/postgres-db-service.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1                                      # API version
+kind: Service                                       # Type of kubernetes resource 
+metadata:
+  name: metaflow-database                          # Name of the resource
+  labels:                                           # Labels that will be applied to the resource
+    app: metaflow-database
+spec:
+  type: NodePort
+  ports:
+    - port: 5432
+  selector:                                         # Selects any Pod with labels `app=fullstack-postgres,tier=postgres`
+    app: metaflow-database
+    tier: postgres
diff --git a/Metaflow_service/deleter.sh b/Metaflow_service/deleter.sh
@@ -0,0 +1,11 @@
+kubectl delete -f postgres-secret.yaml
+
+# $ Create the PG Db in the Kube Cluster. 
+kubectl delete -f database/postgres-db-pv.yaml
+kubectl delete -f database/postgres-db-pvc.yaml
+kubectl delete -f database/postgres-db-deployment.yaml
+kubectl delete -f database/postgres-db-service.yaml
+
+# $ Create the service once the app is created. 
+kubectl delete -f service_app/metaflow-metadata-service-deployment.yaml
+kubectl delete -f service_app/metaflow-metadata-service.yaml
diff --git a/Metaflow_service/postgres-secret.yaml b/Metaflow_service/postgres-secret.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres-secret
+type: Opaque
+stringData:  # We dont need to worry about converting to base64
+  POSTGRES_USER: 'metaflow_service'
+  POSTGRES_PASSWORD: 'r4nd0mp455w07d'
+  POSTGRES_DB: 'metaflow'
+  MF_METADATA_DB_NAME: 'metaflow'
+  MF_METADATA_DB_HOST: 'metaflow-database'
+  MF_METADATA_DB_PORT: '5432' 
+  API_SECRET: '98hbun98h'
+  MF_METADATA_DB_USER: 'metaflow_service'
+  MF_METADATA_DB_PSWD: 'r4nd0mp455w07d'
+  DB_NAME: 'metaflow'
+  DB_PORT: '5432'
+  
diff --git a/Metaflow_service/runner.sh b/Metaflow_service/runner.sh
@@ -0,0 +1,11 @@
+kubectl create -f postgres-secret.yaml
+
+# $ Create the PG Db in the Kube Cluster. 
+kubectl apply -f database/postgres-db-pv.yaml
+kubectl apply -f database/postgres-db-pvc.yaml
+kubectl apply -f database/postgres-db-deployment.yaml
+kubectl apply -f database/postgres-db-service.yaml
+
+# $ Create the service once the app is created. 
+kubectl apply -f service_app/metaflow-metadata-service-deployment.yaml
+kubectl apply -f service_app/metaflow-metadata-service.yaml
diff --git a/Metaflow_service/service_app/metaflow-metadata-service-deployment.yaml b/Metaflow_service/service_app/metaflow-metadata-service-deployment.yaml
@@ -0,0 +1,28 @@
+apiVersion: apps/v1 
+kind: Deployment                                  # Type of the kubernetes resource
+metadata:
+  name: metaflow-metadata-service                    # Name of the Kubernetes resource
+  labels:
+    app: metaflow-metadata-service
+spec:
+  replicas: 1                                     # Number of pods to run at any given time
+  selector:
+    matchLabels: 
+      app: metaflow-metadata-service                 # This deployment applies to any Pods matching the specified label
+  template:                                       # This deployment will create a set of pods using the configurations in this template
+    metadata:
+      labels:                                     # The labels that will be applied to all of the pods in this deployment
+        app: metaflow-metadata-service
+    spec:                                         # Spec for the container which will run in the Pod
+      containers:
+      - name: metaflow-metadata-service 
+        image: netflixoss/metaflow_metadata_service       # The image we are getting from dockerhub
+        imagePullPolicy: IfNotPresent             # If we have not pulled it before, get it from dockerhub
+        ports:
+          - name: http
+            containerPort: 8080                   # Should match the port number that the Go application listens on 
+        envFrom:
+          - secretRef:
+              name: postgres-secret               # Name of the secret environmental variable file to load
+        
+
diff --git a/Metaflow_service/service_app/metaflow-metadata-service.yaml b/Metaflow_service/service_app/metaflow-metadata-service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1  
+kind: Service                                 # Type of the kubernetes resource
+metadata:
+  name: metaflow-metadata-service                # Name of the Kubernetes resource
+  labels:                                     # Labels that will be applied to the resource
+    app: metaflow-metadata-service
+spec:
+  type: NodePort                              # Gives each node an external IP that’s accessible from outside the cluster and also opens a Port.
+  selector:
+    app: metaflow-metadata-service               # Selects any Pod with labels `app=fullstack-app-postgres`
+  ports: 
+  - name: http
+    port: 80
+    targetPort: 8080
diff --git a/Readme.md b/Readme.md
@@ -0,0 +1,148 @@
+# Metaflow on Kubernetes
+
+This Repository will contain the basic scripts and files needed to setup Kubernetes cluster for running and working with Metaflow. 
+It contains Kops setup and Kubernetes templates to deploy necessary services on kubernetes. 
+
+# Kops Guide. 
+
+## Admin Guide 
+This Involves the steps the admin needs to take to Setup cluster and some useful commands that will help maintain things around the cluster. 
+
+- Setup Cluster 
+    ```sh
+    # $ Setup Kubecluster
+    export AWS_PROFILE=default
+    # Keeping k8s.local as domain root ensure private DNS and No need for Public DNS. 
+    export DOMAIN_ROOT=k8s.local
+    export CLUSTER_NAME=dev.$DOMAIN_ROOT
+    export KOPS_BUCKET=$CLUSTER_NAME-test-store
+    export KOPS_STATE_STORE=s3://$KOPS_BUCKET
+
+    aws s3api create-bucket \
+        --bucket $KOPS_BUCKET \
+        --region us-east-1
+
+    aws s3api put-bucket-versioning --bucket $KOPS_BUCKET  --versioning-configuration Status=Enabled
+    # Ensure private DNS so that this can be done quickly with too much route53 setup. 
+    # This will only setup cluster spec. To actually Launch it the command needs to run with ``--yes``
+    kops create cluster --zones=us-east-1c --dns private --master-size t2.micro --master-count 3 --node-size c4.xlarge --node-count 3 $CLUSTER_NAME
+    ```
+- Wait for cluster initialisation to finish. Check via ``kops validate cluster $CLUSTER_NAME``
+
+- To update the Number of instance in the worker nodes run : ``kops edit ig nodes``. This will show the configuration for Instance Group named `nodes`. We can create different instance groups that will be essential for the different purposes. This is how an instance group configuration looks like.
+    ```yml
+    apiVersion: kops.k8s.io/v1alpha2
+    kind: InstanceGroup
+    metadata:
+    creationTimestamp: "2020-02-28T09:47:43Z"
+    generation: 1
+    labels:
+        kops.k8s.io/cluster: dev.k8s.local
+    name: nodes
+    spec:
+    image: kope.io/k8s-1.15-debian-stretch-amd64-hvm-ebs-2020-01-17
+    machineType: c4.xlarge
+    maxSize: 4 # change here to deploy more instances
+    minSize: 4 # change here to deploy more instances
+    nodeLabels:
+        kops.k8s.io/instancegroup: nodes
+    role: Node
+    subnets:
+    - us-east-1c
+    ```
+- run : ``kops update cluster $CLUSTER_NAME --yes``
+
+## User Guide
+
+This involves using AWS Creds to set environment variables that give access to a bucket from which the `kubecfg` can be retrieved.
+
+- Steps:
+    1. install kops, kubectl on your CI server.
+    2. config the AWS access credential on your CI server (either via IAM Role or simply env vars), make sure it has access to your s3 state store path.
+    3. set env var for kops to access your cluster:
+    ```sh
+    export NAME=${YOUR_CLUSTER_NAME}
+    export KOPS_STATE_STORE=s3://${YOUR_CLUSTER_KOPS_STATE_STORE}
+    ```
+    4. Use kops export command to get the kubecfg needed for running kubectl
+    ``kops export kubecfg ${YOUR_CLUSTER_NAME}``
+        - see https://github.com/kubernetes/kops/blob/master/docs/cli/kops_export.md
+
+    5. Now the ~/.kube/config file on your CI server should contain all the information kubectl needs to access your cluster.
+
+
+# `Metaflow_service` 
+
+- This contains the neccessary kube configurations to setup the metaflow environment. 
+- `runner.sh` will create the neccesary Pods/Services on kubernetes for Metaflow Service and its Postgres DB
+- To change secrets, Change the `postgres-secret.yml`
+
+# Deploying Metaflow Job into Kubernetes
+
+- Requirements:
+    - Create a Dockerfile which will build your metaflow Flow into an image
+        ```dockerfile
+        # this is an example Docker file of how to create and image of the Metaflow Run and Put it on Kubernetes. 
+        FROM python:3.7
+
+        RUN pip install https://github.com/valayDave/metaflow/archive/kube_cpu_stable.zip
+        RUN pip install numpy
+
+        WORKDIR /app
+
+        COPY ./data /app/data
+        COPY ./multi_step_mnist.py /app/multi_step_mnist.py
+
+        ENTRYPOINT [ "python",'/app/multi_step_mnist.py']
+        ``` 
+    - Build image of the repo : `docker build -t "<dockerhubid>/<image_name>:<tag>" .`
+    - Push to docker Hub : `docker push <dockerhubid>/<image_name>:<tag>`     
+
+- ``kubectl create -f metaflow-native-cluster-role.yml`` : this will allocate the a cluster role to allow deployments from within a cluster. 
+- Sample Deployment file : 
+    ```yml
+    apiVersion: batch/v1
+    kind: Job
+    metadata:
+    name: metaflow-native-runtime-job-test
+    spec:
+    template:
+        spec:
+        containers:
+            - name: metaflow-native-runtime
+            image: valaygaurang/metaflow:multi_step_mnist_example-in-cluster # Image of the Metaflow Run. 
+            args:
+                [
+                "--with",
+                "kube:cpu=2,memory=2000,image=tensorflow/tensorflow:latest-py3",
+                "run",
+                "--num_training_examples",
+                "40000",
+                ]
+            env:
+                - name: AWS_ACCESS_KEY_ID
+                value: <ACCESS_TOKEN_COMES_HERE>
+                - name: AWS_SECRET_ACCESS_KEY
+                value: <SECRET_VALUE_COMES_HERE>
+                - name: AWS_DEFAULT_REGION
+                value: <AWS_REGION_COMES_HERE>
+                - name: METAFLOW_SERVICE_URL
+                value: http://metaflow-metadata-service.default.svc.cluster.local/ # $ This is what is set in the Metaflow_service/service_app/metaflow-metadata-service-deployment.yaml
+                - name: METAFLOW_DEFAULT_METADATA
+                value: service
+                - name: METAFLOW_DATASTORE_SYSROOT_S3
+                value: s3://<S3_ROOT>
+                - name: METAFLOW_DATATOOLS_SYSROOT_S3
+                value: s3://<S3_ROOT>data
+                - name: METAFLOW_DEFAULT_DATASTORE
+                value: s3
+                - name: USERNAME
+                value: <USERNAME> 
+                - name: METAFLOW_RUNTIME_IN_CLUSTER # Important Environment variable to make it run with a Kube Cluster. 
+                value: 'yes'
+        restartPolicy: Never
+    backoffLimit: 4
+    ```
+- Deploy the job using : ``kubectl create -f metaflow-job-runner.yml``
+- Once Done Executing : 
+    - Once the``kubectl port-forward deployment/metaflow-metadata-service 8080:8080`` to port forward metatdata service for accesss on localmachine. Please note that because this is directly port forwarding to the pod were are taking the 8080 port for the service. 
diff --git a/kops.sh b/kops.sh
@@ -0,0 +1,41 @@
+# $ Setup Kubecluster
+export AWS_PROFILE=default
+# Keeping k8s.local as domain root ensure private DNS and No need for Public DNS. 
+export DOMAIN_ROOT=k8s.local
+export CLUSTER_NAME=dev.$DOMAIN_ROOT
+export KOPS_BUCKET=$CLUSTER_NAME-test-store
+export KOPS_STATE_STORE=s3://$KOPS_BUCKET
+
+aws s3api create-bucket \
+    --bucket $KOPS_BUCKET \
+    --region us-east-1
+
+aws s3api put-bucket-versioning --bucket $KOPS_BUCKET  --versioning-configuration Status=Enabled
+
+kops create cluster --zones=us-east-1c --dns private --master-size t2.micro --master-count 3 --node-size c4.xlarge --node-count 3 $CLUSTER_NAME
+
+kops validate cluster $CLUSTER_NAME # Runs and CHecks the state of the Cluster. 
+
+# $ DELETE CLUSTER COMPLETELY 
+# kops delete cluster dev.k8s.local --yes
+
+# $ Kubernetes Dashboard
+kubectl create -f https://raw.githubusercontent.com/kubernetes/kops/master/addons/kubernetes-dashboard/v1.8.3.yaml 
+
+kubectl create serviceaccount dashboard -n default
+
+kubectl create clusterrolebinding dashboard-admin -n default \
+--clusterrole=cluster-admin \
+--serviceaccount=default:dashboard
+
+# Gets u passworkl 
+kops get secrets kube --type secret -oplaintext
+
+# Get Dashboard URL
+kubectl cluster-info | grep master # Provides info on where the Dashboard is . 
+
+# Get the Loging token here which will be useful in the dashboard
+kubectl get secret $(kubectl get serviceaccount dashboard -o jsonpath="{.secrets[0].name}") -o jsonpath="{.data.token}" | base64 --decode
+
+# http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/#/node?namespace=default
+
diff --git a/metaflow-job-runner.yml b/metaflow-job-runner.yml
diff --git a/metaflow-native-cluster-role.yml b/metaflow-native-cluster-role.yml
