Merge pull request #29 from vshn/add-dbaas-cronjob

add dbaas cronjob
vshn · Dec 9, 2022 · 0bff9a6 · 0bff9a6
2 parents a9e67f0 + 90f880c
commit 0bff9a6
Show file tree

Hide file tree

Showing 8 changed files with 162 additions and 72 deletions.
diff --git a/component/class/defaults.yml b/component/class/defaults.yml
@@ -15,5 +15,14 @@ parameters:
         repository: 'vshn/exoscale-metrics-collector'
         tag: 'v0.3.0'
     # Times in UTC! Don't run job around midnight as exoscale API may return incomplete data
-    # default: Every day at minute 10 past hour 10, 16 and 20.
-    schedule: '10 10,16,20 * * *'
+
+    objectStorage:
+      # schedule for objectstorage cronjob
+      # default: Every day at minute 10 past hour 10, 16 and 20.
+      schedule: '10 10,16,20 * * *'
+
+    dbaas:
+      # enable DBaaS cronjob in addition to objectstorage cronjob.
+      enabled: false
+      # schedule for DBaaS cronjob every 15min
+      schedule: '*/15 * * * *'
diff --git a/component/component/main.jsonnet b/component/component/main.jsonnet
@@ -28,6 +28,82 @@ local secrets = [
   for s in std.objectFields(params.secrets)
 ];
 
+local cronjob(name, subcommand, schedule) = {
+  kind: 'CronJob',
+  apiVersion: 'batch/v1',
+  metadata: {
+    name: name,
+    namespace: paramsACR.namespace,
+    labels+: labels,
+  },
+  spec: {
+    concurrencyPolicy: 'Forbid',
+    failedJobsHistoryLimit: 5,
+    jobTemplate: {
+      spec: {
+        template: {
+          spec: {
+            restartPolicy: 'OnFailure',
+            containers: [
+              {
+                name: 'exoscale-metrics-collector-backfill',
+                image: collectorImage,
+                args: [
+                  subcommand,
+                ],
+                envFrom: [
+                  {
+                    secretRef: {
+                      name: credentials_secret_name,
+                    },
+                  },
+                ],
+                env: [
+                  {
+                    name: 'password',
+                    valueFrom: {
+                      secretKeyRef: {
+                        key: 'password',
+                        name: 'reporting-db',
+                      },
+                    },
+                  },
+                  {
+                    name: 'username',
+                    valueFrom: {
+                      secretKeyRef: {
+                        key: 'username',
+                        name: 'reporting-db',
+                      },
+                    },
+                  },
+                  {
+                    name: 'ACR_DB_URL',
+                    value: 'postgres://$(username):$(password)@%(host)s:%(port)s/%(name)s?%(parameters)s' % paramsACR.database,
+                  },
+                ],
+                resources: {},
+              },
+            ],
+          },
+        },
+      },
+    },
+    schedule: schedule,
+    successfulJobsHistoryLimit: 3,
+  },
+};
+
+local objectStorageSchedule =
+  if std.objectHas(params, 'schedule') then
+    std.trace(
+      'Parameter `schedule` is deprecated,'
+      + ' please use parameter `objectStorage.schedule` instead',
+      params.schedule
+    )
+  else
+    params.objectStorage.schedule;
+
 {
   assert params.secrets != null : 'secrets must be set.',
   assert params.secrets.credentials != null : 'secrets.credentials must be set.',
@@ -39,69 +115,6 @@ local secrets = [
 
   secrets: std.filter(function(it) it != null, secrets),
 
-  cronjob: {
-    kind: 'CronJob',
-    apiVersion: 'batch/v1',
-    metadata: {
-      name: alias,
-      namespace: paramsACR.namespace,
-      labels+: labels,
-    },
-    spec: {
-      concurrencyPolicy: 'Forbid',
-      failedJobsHistoryLimit: 5,
-      jobTemplate: {
-        spec: {
-          template: {
-            spec: {
-              restartPolicy: 'OnFailure',
-              containers: [
-                {
-                  name: 'exoscale-metrics-collector-backfill',
-                  image: collectorImage,
-                  args: [
-                    'objectstorage',
-                  ],
-                  envFrom: [
-                    {
-                      secretRef: {
-                        name: credentials_secret_name
-                      }
-                    }
-                  ],
-                  env: [
-                    {
-                      name: 'password',
-                      valueFrom: {
-                        secretKeyRef: {
-                          key: 'password',
-                          name: 'reporting-db',
-                        },
-                      },
-                    },
-                    {
-                      name: 'username',
-                      valueFrom: {
-                        secretKeyRef: {
-                          key: 'username',
-                          name: 'reporting-db',
-                        },
-                      },
-                    },
-                    {
-                      name: 'ACR_DB_URL',
-                      value: 'postgres://$(username):$(password)@%(host)s:%(port)s/%(name)s?%(parameters)s' % paramsACR.database,
-                    },
-                  ],
-                  resources: {},
-                },
-              ],
-            },
-          },
-        },
-      },
-      schedule: params.schedule,
-      successfulJobsHistoryLimit: 3,
-    },
-  },
+  objectStorageCronjob: cronjob(alias + '-objectstorage', 'objectstorage', objectStorageSchedule),
+  [if params.dbaas.enabled then 'dbaasCronjob']: cronjob(alias + '-dbaas', 'dbaas', params.dbaas.schedule),
 }
diff --git a/component/tests/exoscale-metrics-collector.yml b/component/tests/exoscale-metrics-collector.yml
@@ -7,3 +7,6 @@ parameters:
       parameters: 'sslmode=disable'
       password: 'passw0rd'
       port: 5432
+  exoscale_metrics_collector:
+    dbaas:
+      enabled: true
diff --git a/...2/collector-cloudscale-lpg-2/cronjob.yaml → ...loudscale-lpg-2/objectStorageCronjob.yaml b/...2/collector-cloudscale-lpg-2/cronjob.yaml → ...loudscale-lpg-2/objectStorageCronjob.yaml
@@ -6,7 +6,7 @@ metadata:
     app.kubernetes.io/managed-by: commodore
     app.kubernetes.io/name: exoscale-metrics-collector
     app.kubernetes.io/part-of: appuio-cloud-reporting
-  name: collector-cloudscale-lpg-2
+  name: collector-cloudscale-lpg-2-objectstorage
   namespace: appuio-cloud-reporting
 spec:
   concurrencyPolicy: Forbid

diff --git a/...metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/dbaasCronjob.yaml b/...metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/dbaasCronjob.yaml
@@ -0,0 +1,42 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  labels:
+    app.kubernetes.io/component: exoscale-metrics-collector
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: exoscale-metrics-collector
+    app.kubernetes.io/part-of: appuio-cloud-reporting
+  name: exoscale-metrics-collector-dbaas
+  namespace: appuio-cloud-reporting
+spec:
+  concurrencyPolicy: Forbid
+  failedJobsHistoryLimit: 5
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - args:
+                - dbaas
+              env:
+                - name: password
+                  valueFrom:
+                    secretKeyRef:
+                      key: password
+                      name: reporting-db
+                - name: username
+                  valueFrom:
+                    secretKeyRef:
+                      key: username
+                      name: reporting-db
+                - name: ACR_DB_URL
+                  value: postgres://$(username):$(password)@reporting-db.appuio-reporting.svc:5432/reporting?sslmode=disable
+              envFrom:
+                - secretRef:
+                    name: credentials-exoscale-metrics-collector
+              image: ghcr.io/vshn/exoscale-metrics-collector:v0.3.0
+              name: exoscale-metrics-collector-backfill
+              resources: {}
+          restartPolicy: OnFailure
+  schedule: '*/15 * * * *'
+  successfulJobsHistoryLimit: 3
diff --git a/...r/exoscale-metrics-collector/cronjob.yaml → ...trics-collector/objectStorageCronjob.yaml b/...r/exoscale-metrics-collector/cronjob.yaml → ...trics-collector/objectStorageCronjob.yaml
@@ -6,7 +6,7 @@ metadata:
     app.kubernetes.io/managed-by: commodore
     app.kubernetes.io/name: exoscale-metrics-collector
     app.kubernetes.io/part-of: appuio-cloud-reporting
-  name: exoscale-metrics-collector
+  name: exoscale-metrics-collector-objectstorage
   namespace: appuio-cloud-reporting
 spec:
   concurrencyPolicy: Forbid

diff --git a/docs/modules/ROOT/pages/how-tos/installation.adoc b/docs/modules/ROOT/pages/how-tos/installation.adoc
@@ -13,6 +13,9 @@ The Kubernetes Service Account token is required to have `get` permissions on `N
 
 * `buckets.exoscale.crossplane.io`
 * `postgresqls.exoscale.crossplane.io`
+* `mysqls.exoscale.crossplane.io`
+* `redis.exoscale.crossplane.io`
+* `kafkas.exoscale.crossplane.io`
 
 The Access Key (IAM Key) from an Exoscale organization is required to have read access across all managed resources (e.g. SOS buckets, Postgres, etc).
 
@@ -25,7 +28,11 @@ applications:
 
 parameters:
   exoscale_metrics_collector:
-    schedule: '10 10,16,20 * * *'
+    objectStorage:
+      schedule: '10 10,16,20 * * *'
+    dbaas:
+      enabled: true # must be manually enabled to scrape DBaas metrics
+      schedule: '*/15 * * * *'
 ----
 
 See the xref:references/parameters.adoc[parameters] reference for a full list of parameters.
diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc
@@ -19,13 +19,29 @@ default:: https://github.com/vshn/exoscale-metrics-collector/blob/master/compone
 
 Dictionary containing the container images used by this component.
 
-== `schedule`
+== `objectStorage.schedule`
 
 [horizontal]
 type:: string
 default:: `10 10,16,20 * * *`
 
-The cron schedule at which the metrics collection job is spawned.
+The cron schedule at which the object storage metrics collection job is spawned.
+
+== `dbaas.enabled`
+
+[horizontal]
+type:: bool
+default:: `false`
+
+Whether to add the DBaaS cronjob, too.
+
+== `dbaas.schedule`
+
+[horizontal]
+type:: string
+default:: `*/15 * * * *`
+
+The cron schedule at which the DBaaS metrics collection job is spawned, if enabled.
 
 == `secrets.credentials.stringData.EXOSCALE_API_KEY`