Release/2.3.2

.gitignore

@@ -169,3 +169,6 @@ observability_ui/node_modules/
 
 # Terraform
 .terraform*
+
+# Helm
+**/*.tgz

common/apscheduler_extensions.py

@@ -58,7 +58,7 @@ def validate_cron_expression(expression: str) -> list[str]:
     if len(values) != 5:
         errors.append(f"Got {len(values)} fields. Expected 5.")
 
-    for (field_name, field_class), value in zip(CRON_EXPRESSION_FIELDS, values, strict=True):
+    for (field_name, field_class), value in zip(CRON_EXPRESSION_FIELDS, values, strict=False):
         try:
             field_class(field_name, value)
         except ValueError:

common/entity_services/project_service.py

@@ -58,7 +58,7 @@ def get_test_outcomes_with_rules(
         query = TestOutcome.select(TestOutcome).where(TestOutcome.component.in_(project.components))
         if rules.search is not None:
             query = query.where(
-                ((TestOutcome.name ** f"%{rules.search}%") | (TestOutcome.description ** f"%{rules.search}%"))
+                (TestOutcome.name ** f"%{rules.search}%") | (TestOutcome.description ** f"%{rules.search}%")
             )
         if filters:
             if statuses := filters.statuses:

deploy/charts/observability-app/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: dataops-observability-app
 type: application
 appVersion: "2.x.x"
-version: "2.1.0"
+version: "2.2.0"
 
 description: DataOps Observability
 home: https://datakitchen.io

deploy/charts/observability-app/templates/agent-api.yaml

@@ -61,6 +61,10 @@ spec:
                   name: {{ .Values.observability.keys_secrets_name | quote }}
                   key: AGENT_API_KEY_FLASK_SECRET
             {{- end }}
+          {{- with .Values.extraVolumeMounts }}
+          volumeMounts:
+            {{ toYaml . | nindent 12 }}
+          {{- end }}
           command: [ "/dk/bin/gunicorn" ]
           args: [ "-c", "/dk/gunicorn.conf.py", "agent_api.app:app" ]
           readinessProbe:
@@ -74,6 +78,10 @@ spec:
                 - name: Host
                   value: {{ tpl .Values.agent_api.hostname . | quote }}
                 {{- end }}
+      {{- with .Values.extraVolumes }}
+      volumes:
+        {{ toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.agent_api.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

deploy/charts/observability-app/templates/cli-tool-hook.yaml

@@ -1,22 +1,23 @@
-{{- range .Values.cli_hook.enable }}
+{{- range $index, $job := .Values.cli_hook.enable }}
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: "cli-job"
+  name: cli-job-{{ default $index .name }}
   annotations:
     "helm.sh/hook": {{ join ", " .phases }}
     "helm.sh/hook-weight": {{ default 0 .weight | quote }}
     "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation
 spec:
   template:
     metadata:
-      name: "cli-job"
+      name: cli-job-{{ default $index .name }}
     spec:
       restartPolicy: Never
       {{- with $.Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      serviceAccountName: {{ include "observability.serviceAccountName" $ }}
       containers:
         - name: cli-hook
           image: {{ include "observability.cli_hook.image" $ | quote }}
@@ -25,9 +26,21 @@ spec:
             {{- include "observability.environment.base" $ | nindent 12 }}
             {{- include "observability.environment.database" $ | nindent 12 }}
             {{- include "observability.environment.pythonpath" $ | nindent 12 }}
+          {{- with $.Values.extraVolumeMounts }}
+          volumeMounts:
+            {{ toYaml . | nindent 12 }}
+          {{- end }}
           command:
             {{- range .command }}
             - {{ . | quote -}}
             {{- end }}
+          args:
+            {{- range .args }}
+            - {{ . | quote -}}
+            {{- end }}
+      {{- with $.Values.extraVolumes }}
+      volumes:
+        {{ toYaml . | nindent 8 }}
+      {{- end }}
 ---
 {{- end }}

deploy/charts/observability-app/templates/event-api.yaml

@@ -62,6 +62,10 @@ spec:
                   name: {{ .Values.observability.keys_secrets_name | quote }}
                   key: EVENTS_KEY_FLASK_SECRET
             {{- end }}
+          {{- with .Values.extraVolumeMounts }}
+          volumeMounts:
+            {{ toYaml . | nindent 12 }}
+          {{- end }}
           command: [ "/dk/bin/gunicorn" ]
           args: [ "-c", "/dk/gunicorn.conf.py", "event_api.app:app" ]
           readinessProbe:
@@ -75,7 +79,10 @@ spec:
                 - name: Host
                   value: {{ tpl .Values.event_api.hostname . | quote }}
                 {{- end }}
-
+      {{- with .Values.extraVolumes }}
+      volumes:
+        {{ toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.event_api.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

deploy/charts/observability-app/templates/observability-api.yaml

@@ -61,6 +61,10 @@ spec:
                   name: {{ .Values.observability.keys_secrets_name | quote }}
                   key: OBSERVABILITY_KEY_FLASK_SECRET
             {{- end }}
+          {{- with .Values.extraVolumeMounts }}
+          volumeMounts:
+            {{ toYaml . | nindent 12 }}
+          {{- end }}
           command: [ "/dk/bin/gunicorn" ]
           args: [ "-c", "/dk/gunicorn.conf.py", "observability_api.app:app" ]
           readinessProbe:
@@ -74,6 +78,10 @@ spec:
                 - name: Host
                   value: {{ tpl .Values.observability_api.hostname . | quote }}
                 {{- end }}
+      {{- with .Values.extraVolumes }}
+      volumes:
+        {{ toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.observability_api.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

deploy/charts/observability-app/templates/rules-engine.yaml

@@ -45,8 +45,16 @@ spec:
             {{- include "observability.environment.database" . | nindent 12 }}
             {{- include "observability.environment.kafka" . | nindent 12 }}
             {{- include "observability.environment.smtp" . | nindent 12 }}
+          {{- with .Values.extraVolumeMounts }}
+          volumeMounts:
+            {{ toYaml . | nindent 12 }}
+          {{- end }}
           command: [ "/dk/bin/rules-engine" ]
           {{- include "observability.probes.readiness_cmd" "rules-engine" | nindent 10 -}}
+      {{- with .Values.extraVolumes }}
+      volumes:
+        {{ toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.rules_engine.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

deploy/charts/observability-app/templates/run-manager.yaml

@@ -44,8 +44,16 @@ spec:
             {{- include "observability.environment.base" . | nindent 12 }}
             {{- include "observability.environment.database" . | nindent 12 }}
             {{- include "observability.environment.kafka" . | nindent 12 }}
+          {{- with .Values.extraVolumeMounts }}
+          volumeMounts:
+            {{ toYaml . | nindent 12 }}
+          {{- end }}
           command: ["/dk/bin/run-manager"]
           {{- include "observability.probes.readiness_cmd" "run-manager" | nindent 10 -}}
+      {{- with .Values.extraVolumes }}
+      volumes:
+        {{ toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.run_manager.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

deploy/charts/observability-app/templates/scheduler.yaml

@@ -44,8 +44,16 @@ spec:
             {{- include "observability.environment.base" . | nindent 12 }}
             {{- include "observability.environment.database" . | nindent 12 }}
             {{- include "observability.environment.kafka" . | nindent 12 }}
+          {{- with .Values.extraVolumeMounts }}
+          volumeMounts:
+            {{ toYaml . | nindent 12 }}
+          {{- end }}
           command: [ "/dk/bin/scheduler" ]
           {{- include "observability.probes.readiness_cmd" "scheduler" | nindent 10 -}}
+      {{- with .Values.extraVolumes }}
+      volumes:
+        {{ toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.scheduler.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

observability_ui/apps/shell/src/app/projects/instances/instance-tests/instance-tests.component.html

@@ -183,7 +183,7 @@
       <span headerLabel="start_time">Start Time</span>
     </ng-container>
     <ng-container *matCellDef="let test">
-      <div dkTooltip="{{test.start_time | date: 'medium'}}">{{test.start_time | date: 'hh:mm:ss a'}}</div>
+      <div dkTooltip="{{test.start_time | date: 'medium'}}">{{test.start_time | date: 'h:mm:ss a'}}</div>
     </ng-container>
   </ng-container>
   <ng-container matColumnDef="end_time"
@@ -193,7 +193,7 @@
       <span headerLabel="end_time">End Time</span>
     </ng-container>
     <ng-container *matCellDef="let test">
-      <div dkTooltip="{{test.end_time | date: 'medium'}}">{{test.end_time | date: 'hh:mm:ss a'}}</div>
+      <div dkTooltip="{{test.end_time | date: 'medium'}}">{{test.end_time | date: 'h:mm:ss a'}}</div>
     </ng-container>
   </ng-container>
   <ng-container matColumnDef="actions"

observability_ui/apps/shell/src/app/projects/instances/instance-timeline/instance-timeline.component.html

@@ -12,8 +12,7 @@
       [start]="ganttStartDate$ | async"
       [end]="ganttEndDate$ | async"
       [nowBar]="instanceIsActive$ | async"
-      [timeformat]="timeformat$ | async"
-      [showDays]="spansMultipleDays$ | async">
+      [timeformat]="timeformat$ | async">
 
       <ng-template ganttLabel
         let-taskGroup="group">
@@ -49,8 +48,10 @@
         <ng-template #barTooltip>
           <div class="gantt-bar-tooltip">
             <span class="gantt-bar-tooltip--line gantt-bar-tooltip--name">{{ run.pipeline.display_name }}</span>
+            <span *ngIf="run.pipeline.key !== run.pipeline.display_name"
+              class="gantt-bar-tooltip--line gantt-bar-tooltip--key">Key: {{ run.pipeline.key ?? '-' }}</span>
             <span class="gantt-bar-tooltip--line gantt-bar-tooltip--status">{{ 'runStatus.' + run.status | translate }}</span>
-            <span class="gantt-bar-tooltip--line gantt-bar-tooltip--key">Key: {{ run.key ?? '-' }}</span>
+            <span class="gantt-bar-tooltip--line gantt-bar-tooltip--key">Run Key: {{ run.key ?? '-' }}</span>
             <span *ngIf="run.hasTime"
               class="gantt-bar-tooltip--line gantt-bar-tooltip--start">Start Time: {{ run.start_time | date: (timeformat$ | async) }}</span>
             <span *ngIf="run.hasTime && run.status !== 'RUNNING'"

observability_ui/apps/shell/src/app/projects/instances/instance-timeline/instance-timeline.component.ts

@@ -56,16 +56,11 @@ export class InstanceTimelineComponent implements OnInit, OnDestroy {
     map(([ start, end ]) => {
       const timespan = end.getTime() - start.getTime();
       if (timespan <= this.minTimespanInMilliseconds) {
-        return 'hh:mm:ss a';
+        return 'MMM d, h:mm:ss a';
       }
-      return 'hh:mm a';
-    }),
-    startWith('hh:mm a'),
-  );
-  spansMultipleDays$ = combineLatest([ this.ganttStartDate$, this.ganttEndDate$ ]).pipe(
-    map(([ start, end ]) => {
-      return (end.getTime() - start.getTime()) > 24 * 60 * 60 * 1000;
+      return 'MMM d, h:mm a';
     }),
+    startWith('MMM d, h:mm a'),
   );
 
   private minTimespanInMilliseconds: number = 1 * 60 * 1000;

observability_ui/apps/shell/src/app/projects/overview/overview.component.html

@@ -145,22 +145,22 @@ <h1 class="page-title">{{ 'overview' | translate }}</h1>
                   <span class="gantt-bar-tooltip--line">{{ instance.errorAlertsCount }}
                     errors, {{ instance.warningAlertsCount }} warnings</span>
                   <span class="gantt-bar-tooltip--line gantt-bar-tooltip--runs">Total Runs: {{ instance.runsCount }}</span>
-                  <span class="gantt-bar-tooltip--line gantt-bar-tooltip--start">Start Time: {{ instance.start_time | date:'MMM d, hh:mm a' }}</span>
+                  <span class="gantt-bar-tooltip--line gantt-bar-tooltip--start">Start Time: {{ instance.start_time | date:'MMM d, h:mm a' }}</span>
                   <span *ngIf="!instance.active"
-                    class="gantt-bar-tooltip--line gantt-bar-tooltip--end">End Time: {{ instance.end_time | date:'MMM d, hh:mm a' }}</span>
+                    class="gantt-bar-tooltip--line gantt-bar-tooltip--end">End Time: {{ instance.end_time | date:'MMM d, h:mm a' }}</span>
                   <span class="gantt-bar-tooltip--line gantt-bar-tooltip--duration">Duration: {{ instance.start_time | duration:instance.end_time }}</span>
                 </ng-container>
 
                 <ng-container *ngIf="instance.status === 'UPCOMING'">
 
                 <span *ngIf="instance.expected_start_time"
                   class="gantt-bar-tooltip--line gantt-bar-tooltip--start">
-                  Expected Start Time: {{ instance.start_time | date:'MMM d, hh:mm a' }}
+                  Expected Start Time: {{ instance.start_time | date:'MMM d, h:mm a' }}
                 </span>
 
                   <span *ngIf="instance.expected_end_time"
                     class="gantt-bar-tooltip--line gantt-bar-tooltip--end">
-                  Expected End Time: {{ instance.end_time | date:'MMM d, hh:mm a' }}
+                  Expected End Time: {{ instance.end_time | date:'MMM d, h:mm a' }}
                 </span>
 
                 </ng-container>
@@ -246,22 +246,22 @@ <h1 class="page-title">{{ 'overview' | translate }}</h1>
                         <span class="gantt-bar-tooltip--line gantt-bar-tooltip--runs">{{ directive.value.active ? 'Active' : 'Ended' }}</span>
                         <span class="gantt-bar-tooltip--line gantt-bar-tooltip--status">{{ directive.value.errorAlertsCount }}
                           errors, {{ directive.value.warningAlertsCount }} warnings</span>
-                        <span class="gantt-bar-tooltip--line gantt-bar-tooltip--start">Start Time: {{ directive.value.start_time | date:'MMM d, hh:mm a' }}</span>
+                        <span class="gantt-bar-tooltip--line gantt-bar-tooltip--start">Start Time: {{ directive.value.start_time | date:'MMM d, h:mm a' }}</span>
                         <span *ngIf="!directive.value.active"
-                          class="gantt-bar-tooltip--line gantt-bar-tooltip--end">End Time: {{ directive.value.end_time | date:'MMM d, hh:mm a' }}</span>
+                          class="gantt-bar-tooltip--line gantt-bar-tooltip--end">End Time: {{ directive.value.end_time | date:'MMM d, h:mm a' }}</span>
                         <span class="gantt-bar-tooltip--line gantt-bar-tooltip--duration">Duration: {{ directive.value.start_time | duration:directive.value.end_time }}</span>
                         <span class="gantt-bar-tooltip--line gantt-bar-tooltip--runs">Total Runs: {{ directive.value.runsCount }}</span>
                       </ng-container>
 
                       <ng-container *ngIf="directive.value.status === 'UPCOMING'">
                       <span *ngIf="directive.value.expected_start_time"
                         class="gantt-bar-tooltip--line gantt-bar-tooltip--start mt-2">
-                        Expected Start Time: {{ directive.value.start_time | date:'MMM d, hh:mm a' }}
+                        Expected Start Time: {{ directive.value.start_time | date:'MMM d, h:mm a' }}
                       </span>
 
                         <span *ngIf="directive.value.expected_end_time"
                           class="gantt-bar-tooltip--line gantt-bar-tooltip--end">
-                        Expected End Time: {{ directive.value.end_time | date:'MMM d, hh:mm a' }}
+                        Expected End Time: {{ directive.value.end_time | date:'MMM d, h:mm a' }}
                       </span>
 
                       </ng-container>
@@ -290,9 +290,9 @@ <h1 class="page-title">{{ 'overview' | translate }}</h1>
                           <span class="gantt-bar-tooltip--line gantt-bar-tooltip--status">{{ 'runStatus.' + run.status | translate }}</span>
                           <span class="gantt-bar-tooltip--line gantt-bar-tooltip--key">Key: {{ run.key ?? '-' }}</span>
                           <span *ngIf="run.hasTime"
-                            class="gantt-bar-tooltip--line gantt-bar-tooltip--start">Start Time: {{ run.start_time | date: 'MMM d, hh:mm a' }}</span>
+                            class="gantt-bar-tooltip--line gantt-bar-tooltip--start">Start Time: {{ run.start_time | date: 'MMM d, h:mm a' }}</span>
                           <span *ngIf="run.hasTime && run.status !== 'RUNNING'"
-                            class="gantt-bar-tooltip--line gantt-bar-tooltip--end">End Time: {{ run.end_time | date: 'MMM d, hh:mm a' }}</span>
+                            class="gantt-bar-tooltip--line gantt-bar-tooltip--end">End Time: {{ run.end_time | date: 'MMM d, h:mm a' }}</span>
                           <span *ngIf="run.hasTime"
                             class="gantt-bar-tooltip--line gantt-bar-tooltip--duration">Duration: {{ run.start_time | duration: (run.end_time ?? (now$ | async)) }}</span>
                         </div>