From 6ab71f7b2d856ff283abd6812a6bedf20bbef389 Mon Sep 17 00:00:00 2001 From: Michael Weibel Date: Wed, 7 Dec 2022 15:19:38 +0100 Subject: [PATCH] add dbaas cronjob --- component/class/defaults.yml | 13 +- component/component/main.jsonnet | 143 ++++++++++-------- .../tests/exoscale-metrics-collector.yml | 3 + ...cronjob.yaml => objectStorageCronjob.yaml} | 2 +- .../dbaasCronjob.yaml | 42 +++++ ...cronjob.yaml => objectStorageCronjob.yaml} | 2 +- 6 files changed, 136 insertions(+), 69 deletions(-) rename component/tests/golden/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/{cronjob.yaml => objectStorageCronjob.yaml} (96%) create mode 100644 component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/dbaasCronjob.yaml rename component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/{cronjob.yaml => objectStorageCronjob.yaml} (96%) diff --git a/component/class/defaults.yml b/component/class/defaults.yml index 77c6e61..20c5a7f 100644 --- a/component/class/defaults.yml +++ b/component/class/defaults.yml @@ -15,5 +15,14 @@ parameters: repository: 'vshn/exoscale-metrics-collector' tag: 'v0.3.0' # Times in UTC! Don't run job around midnight as exoscale API may return incomplete data - # default: Every day at minute 10 past hour 10, 16 and 20. - schedule: '10 10,16,20 * * *' + + objectStorage: + # schedule for objectstorage cronjob + # default: Every day at minute 10 past hour 10, 16 and 20. + schedule: '10 10,16,20 * * *' + + dbaas: + # enable DBaaS cronjob in addition to objectstorage cronjob. + enabled: false + # schedule for DBaaS cronjob every 15min + schedule: '*/15 * * * *' diff --git a/component/component/main.jsonnet b/component/component/main.jsonnet index cde7238..4996f4d 100644 --- a/component/component/main.jsonnet +++ b/component/component/main.jsonnet @@ -28,6 +28,82 @@ local secrets = [ for s in std.objectFields(params.secrets) ]; +local cronjob(name, subcommand, schedule) = { + kind: 'CronJob', + apiVersion: 'batch/v1', + metadata: { + name: name, + namespace: paramsACR.namespace, + labels+: labels, + }, + spec: { + concurrencyPolicy: 'Forbid', + failedJobsHistoryLimit: 5, + jobTemplate: { + spec: { + template: { + spec: { + restartPolicy: 'OnFailure', + containers: [ + { + name: 'exoscale-metrics-collector-backfill', + image: collectorImage, + args: [ + subcommand, + ], + envFrom: [ + { + secretRef: { + name: credentials_secret_name, + }, + }, + ], + env: [ + { + name: 'password', + valueFrom: { + secretKeyRef: { + key: 'password', + name: 'reporting-db', + }, + }, + }, + { + name: 'username', + valueFrom: { + secretKeyRef: { + key: 'username', + name: 'reporting-db', + }, + }, + }, + { + name: 'ACR_DB_URL', + value: 'postgres://$(username):$(password)@%(host)s:%(port)s/%(name)s?%(parameters)s' % paramsACR.database, + }, + ], + resources: {}, + }, + ], + }, + }, + }, + }, + schedule: schedule, + successfulJobsHistoryLimit: 3, + }, +}; + +local objectStorageSchedule = + if std.objectHas(params, 'schedule') then + std.trace( + 'Parameter `schedule` is deprecated,' + + ' please use parameter `objectStorage.schedule` instead', + params.schedule + ) + else + params.objectStorage.schedule; + { assert params.secrets != null : 'secrets must be set.', assert params.secrets.credentials != null : 'secrets.credentials must be set.', @@ -39,69 +115,6 @@ local secrets = [ secrets: std.filter(function(it) it != null, secrets), - cronjob: { - kind: 'CronJob', - apiVersion: 'batch/v1', - metadata: { - name: alias, - namespace: paramsACR.namespace, - labels+: labels, - }, - spec: { - concurrencyPolicy: 'Forbid', - failedJobsHistoryLimit: 5, - jobTemplate: { - spec: { - template: { - spec: { - restartPolicy: 'OnFailure', - containers: [ - { - name: 'exoscale-metrics-collector-backfill', - image: collectorImage, - args: [ - 'objectstorage', - ], - envFrom: [ - { - secretRef: { - name: credentials_secret_name - } - } - ], - env: [ - { - name: 'password', - valueFrom: { - secretKeyRef: { - key: 'password', - name: 'reporting-db', - }, - }, - }, - { - name: 'username', - valueFrom: { - secretKeyRef: { - key: 'username', - name: 'reporting-db', - }, - }, - }, - { - name: 'ACR_DB_URL', - value: 'postgres://$(username):$(password)@%(host)s:%(port)s/%(name)s?%(parameters)s' % paramsACR.database, - }, - ], - resources: {}, - }, - ], - }, - }, - }, - }, - schedule: params.schedule, - successfulJobsHistoryLimit: 3, - }, - }, + objectStorageCronjob: cronjob(alias + '-objectstorage', 'objectstorage', objectStorageSchedule), + [if params.dbaas.enabled then 'dbaasCronjob']: cronjob(alias + '-dbaas', 'dbaas', params.dbaas.schedule), } diff --git a/component/tests/exoscale-metrics-collector.yml b/component/tests/exoscale-metrics-collector.yml index d94263c..cdfaf79 100644 --- a/component/tests/exoscale-metrics-collector.yml +++ b/component/tests/exoscale-metrics-collector.yml @@ -7,3 +7,6 @@ parameters: parameters: 'sslmode=disable' password: 'passw0rd' port: 5432 + exoscale_metrics_collector: + dbaas: + enabled: true diff --git a/component/tests/golden/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/cronjob.yaml b/component/tests/golden/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/objectStorageCronjob.yaml similarity index 96% rename from component/tests/golden/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/cronjob.yaml rename to component/tests/golden/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/objectStorageCronjob.yaml index d425c30..94be726 100644 --- a/component/tests/golden/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/cronjob.yaml +++ b/component/tests/golden/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/objectStorageCronjob.yaml @@ -6,7 +6,7 @@ metadata: app.kubernetes.io/managed-by: commodore app.kubernetes.io/name: exoscale-metrics-collector app.kubernetes.io/part-of: appuio-cloud-reporting - name: collector-cloudscale-lpg-2 + name: collector-cloudscale-lpg-2-objectstorage namespace: appuio-cloud-reporting spec: concurrencyPolicy: Forbid diff --git a/component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/dbaasCronjob.yaml b/component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/dbaasCronjob.yaml new file mode 100644 index 0000000..74aa6bb --- /dev/null +++ b/component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/dbaasCronjob.yaml @@ -0,0 +1,42 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + labels: + app.kubernetes.io/component: exoscale-metrics-collector + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: exoscale-metrics-collector + app.kubernetes.io/part-of: appuio-cloud-reporting + name: exoscale-metrics-collector-dbaas + namespace: appuio-cloud-reporting +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + template: + spec: + containers: + - args: + - dbaas + env: + - name: password + valueFrom: + secretKeyRef: + key: password + name: reporting-db + - name: username + valueFrom: + secretKeyRef: + key: username + name: reporting-db + - name: ACR_DB_URL + value: postgres://$(username):$(password)@reporting-db.appuio-reporting.svc:5432/reporting?sslmode=disable + envFrom: + - secretRef: + name: credentials-exoscale-metrics-collector + image: ghcr.io/vshn/exoscale-metrics-collector:v0.3.0 + name: exoscale-metrics-collector-backfill + resources: {} + restartPolicy: OnFailure + schedule: '*/15 * * * *' + successfulJobsHistoryLimit: 3 diff --git a/component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/cronjob.yaml b/component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/objectStorageCronjob.yaml similarity index 96% rename from component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/cronjob.yaml rename to component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/objectStorageCronjob.yaml index e2005aa..1c3d78d 100644 --- a/component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/cronjob.yaml +++ b/component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/objectStorageCronjob.yaml @@ -6,7 +6,7 @@ metadata: app.kubernetes.io/managed-by: commodore app.kubernetes.io/name: exoscale-metrics-collector app.kubernetes.io/part-of: appuio-cloud-reporting - name: exoscale-metrics-collector + name: exoscale-metrics-collector-objectstorage namespace: appuio-cloud-reporting spec: concurrencyPolicy: Forbid