chore: add optional prometheus deployment (#4323)

* enable pagination when expanding experiment in both the home page and the archive page

* Revert "enable pagination when expanding experiment in both the home page and the archive page"

This reverts commit 5b67273.

* Add prometheus deployment as an optional deployment

* move prom dir to under third-party

* comments

* third party folder to the kustomize folder

manifests/kustomize/sample/kustomization.yaml

@@ -4,6 +4,11 @@ kind: Kustomization
 bases:
   # Or github.com/kubeflow/pipelines/manifests/kustomize/env/gcp?ref=1.0.0
   - ../env/gcp
+  # Kubeflow Pipelines servers are capable of collecting Prometheus metrics.
+  # If you want to monitor your Kubeflow Pipelines servers with those metrics, you'll need a Prometheus server in your Kubeflow Pipelines cluster.
+  # If you don't already have a Prometheus server up, you can use the following configuration files for Prometheus.
+  # If you have your own Prometheus server up already or you don't want a Prometheus server for monitoring, you can comment the following line out.
+  - ../third-party/prometheus
 
 # Identifier for application manager to apply ownerReference.
 # The ownerReference ensures the resources get garbage collected

manifests/kustomize/third-party/prometheus/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- prometheus-configmap.yaml
+- prometheus-sa.yaml
+- prometheus-role.yaml
+- prometheus-rolebinding.yaml
+- prometheus-service.yaml
+- prometheus-deployment.yaml
+images:
+  - name: prom/prometheus

manifests/kustomize/third-party/prometheus/prometheus-configmap.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval:     15s # By default, scrape targets every 15 seconds.
+
+      # Attach these labels to any time series or alerts when communicating with
+      # external systems (federation, remote storage, Alertmanager).
+      external_labels:
+        monitor: 'kubeflow-pipelines-monitor'
+
+    # A scrape configuration containing exactly one endpoint to scrape:
+    # Here it's Prometheus itself.
+    scrape_configs:
+      # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+      - job_name: 'prometheus'
+
+        # Override the global default and scrape targets from this job every 5 seconds.
+        scrape_interval: 5s
+
+        static_configs:
+          - targets: ['localhost:9090']
+
+      # Monitoring ml-pipeline (aka Kubeflow Pipelines API server)
+      - job_name: 'ml-pipeline'
+        scrape_interval: 60s
+        static_configs:
+          - targets: ['ml-pipeline:8888']
+kind: ConfigMap
+metadata:
+  name: prometheus-configmap

manifests/kustomize/third-party/prometheus/prometheus-deployment.yaml

@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  labels:
+    app: prometheus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+    spec:
+      containers:
+      - name: prometheus
+        image: prom/prometheus
+        volumeMounts:
+          - name: config-volume
+            mountPath: /etc/prometheus/prometheus.yml
+            subPath: prometheus.yml
+        args: ["--storage.tsdb.retention.time=7d",  # Adjust retention policy if necessary
+               "--storage.tsdb.retention.size=1GB",
+               "--config.file=/etc/prometheus/prometheus.yml",
+               ]
+        ports:
+        - containerPort: 9090
+      volumes:
+        - name: config-volume
+          configMap:
+           name: prometheus-configmap
+      serviceAccountName: prometheus

manifests/kustomize/third-party/prometheus/prometheus-role.yaml

@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: prometheus
+  name: prometheus
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - services
+  - endpoints
+  - pods
+  verbs: ["get", "list", "watch"]
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses
+  verbs: ["get", "list", "watch"]

manifests/kustomize/third-party/prometheus/prometheus-rolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: prometheus
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus

manifests/kustomize/third-party/prometheus/prometheus-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus
+  # namespace: default

manifests/kustomize/third-party/prometheus/prometheus-service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: prometheus
+  name: prometheus
+spec:
+  selector:
+    app: prometheus
+  ports:
+  - name: promui
+    protocol: TCP
+    port: 9090
+    targetPort: 9090