kuberay/spark operator

.devcontainer/vm/devcontainer.json

@@ -27,7 +27,7 @@
     }
   },
   "runArgs": [
-    "--network=host",
+    "--network=host", // Allows accessing k3d from within container using docker outside of docker
   ],
   "features": {
     "ghcr.io/meaningful-ooo/devcontainer-features/homebrew:2": {

examples/k3s/README.md

@@ -75,7 +75,17 @@ terraform apply -var kubeconfig=$KUBECONFIG
 
 ### 3. Verify the installation
 
-Once terraform has finished, connect to the central dashboard and login as the default user.
+Watch k9s pods until all of them are in a ready state.
+
+You can also check argocd with:
+
+```
+kubectl port-forward  --namespace "argocd" svc/argo-cd-argocd-server 8081:80
+```
+
+then go the localhost:8081 and enter username 'admin', password (decoded `argocd/argocd-initial-admin-secret`)
+
+Once all is finished you can port forward into the ingress gateway:
 
 ```sh
 kubectl port-forward  --namespace "istio-system" svc/istio-ingressgateway 8080:http2

helm/kubeflow-core/templates/300-distributed/300-kuberay.yaml

@@ -0,0 +1,12 @@
+{{ if .Values.kuberay.enabled }}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: 300-kuberay
+  annotations:
+    argocd.argoproj.io/sync-wave: "300"
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+{{ .Values.kuberay.spec | toYaml | indent 2}}
+{{- end -}}

helm/kubeflow-core/templates/300-distributed/300-spark-operator.yaml

@@ -0,0 +1,12 @@
+{{ if .Values.sparkOperator.enabled }}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: 300-spark-operator
+  annotations:
+    argocd.argoproj.io/sync-wave: "300"
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+{{ .Values.sparkOperator.spec | toYaml | indent 2}}
+{{- end -}}

helm/kubeflow-core/values.yaml

@@ -612,3 +612,48 @@ tensorboardsWebApp:
         prune: false
       syncOptions:
         - ServerSideApply=true
+
+kuberay:
+  enabled: false
+  spec:
+    project: default
+    sources:
+      - chart: 'kuberay-operator'
+        repoURL: 'https://ray-project.github.io/kuberay-helm/'
+        targetRevision: "1.1.0"
+        helm:
+          releaseName: "kuberay"
+          ignoreMissingValueFiles: true
+          valuesObject:
+            resources: {}
+    destination:
+      namespace: 'kubeflow'
+      name: 'in-cluster'
+    syncPolicy:
+      automated:
+        prune: false
+      syncOptions:
+        - CreateNamespace=true
+        - ServerSideApply=true
+
+sparkOperator:
+  enabled: false
+  spec:
+    project: default
+    sources:
+      - chart: 'spark-operator'
+        repoURL: 'https://kubeflow.github.io/spark-operator'
+        targetRevision: "1.1.27"
+        helm:
+          releaseName: "spark-operator"
+          ignoreMissingValueFiles: true
+          valuesObject: {}
+    destination:
+      namespace: 'kubeflow'
+      name: 'in-cluster'
+    syncPolicy:
+      automated:
+        prune: false
+      syncOptions:
+        - CreateNamespace=true
+        - ServerSideApply=true