Merge branch 'feature/keycloak-middleware-clean' into enhancement/openremote-client-service-register

Author: dominiquekleeven

.github/workflows/ci.yml

@@ -6,18 +6,17 @@ on:
   pull_request:
     branches: [ main ]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
 
   quality-checks:
     runs-on: ubuntu-latest
     timeout-minutes: 5
 
     steps:
-    - name: Cancel previous runs
-      uses: styfle/cancel-workflow-action@85880fa0301c86cca9da44039ee3bb12d3bedbfa # 0.12.1
-      with:
-        access_token: ${{ github.token }}
-
     - name: Checkout
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 

frontend/src/components/custom-duration-input.ts

@@ -26,17 +26,37 @@ export enum DurationInputType {
 }
 
 // ISO 8601
-enum TimeDurationUnit {
-    MINUTE = 'M',
-    HOUR = 'H'
+export enum TimeDurationUnit {
+    MINUTE = 'PT%M',
+    HOUR = 'PT%H',
+    DAY = 'P%D',
+    WEEK = 'P%W',
+    MONTH = 'P%M',
+    YEAR = 'P%Y'
 }
 
+// Display name for the ISO 8601 units
+const TimeDurationUnitDisplay: Record<TimeDurationUnit, string> = {
+    [TimeDurationUnit.MINUTE]: 'Minutes',
+    [TimeDurationUnit.HOUR]: 'Hours',
+    [TimeDurationUnit.DAY]: 'Days',
+    [TimeDurationUnit.WEEK]: 'Weeks',
+    [TimeDurationUnit.MONTH]: 'Months',
+    [TimeDurationUnit.YEAR]: 'Years'
+};
+
 // Pandas Frequency
-enum PandasTimeUnit {
-    MINUTE = 'min',
-    HOUR = 'h'
+export enum PandasTimeUnit {
+    MINUTE = '%min',
+    HOUR = '%h'
 }
 
+// Display name for the Pandas units
+const PandasTimeUnitDisplay: Record<PandasTimeUnit, string> = {
+    [PandasTimeUnit.MINUTE]: 'Minutes',
+    [PandasTimeUnit.HOUR]: 'Hours'
+};
+
 // This is a input component that renders both a number input and a dropdown for the unit of the duration
 @customElement('custom-duration-input')
 export class CustomDurationInput extends LitElement {
@@ -79,12 +99,17 @@ export class CustomDurationInput extends LitElement {
                 break;
         }
 
+        if (!this.unit || !this.number) {
+            return;
+        }
+
+        // Replace the % in the unit with the number and set the value
         switch (this.type) {
             case DurationInputType.ISO_8601:
-                this.value = `PT${this.number}${this.unit}`;
+                this.value = `${this.unit.replace('%', this.number.toString())}`;
                 break;
             case DurationInputType.PANDAS_FREQ:
-                this.value = `${this.number}${this.unit}`;
+                this.value = `${this.unit.replace('%', this.number.toString())}`;
         }
 
         // Fire event to notify parent component that the value has changed
@@ -105,20 +130,29 @@ export class CustomDurationInput extends LitElement {
     @property({ type: String })
     public value: string = '';
 
+    @property({ type: Array })
+    public iso_units: TimeDurationUnit[] = [TimeDurationUnit.MINUTE, TimeDurationUnit.HOUR];
+
+    @property({ type: Array })
+    public pandas_units: PandasTimeUnit[] = [PandasTimeUnit.MINUTE, PandasTimeUnit.HOUR];
+
     protected number: number | null = null;
 
     protected unit: TimeDurationUnit | PandasTimeUnit | null = null;
 
     // Extract the number from the ISO 8601 Duration string
     getNumberFromDuration(duration: string): number | null {
-        const match = /PT(\d+)([HM])/.exec(duration);
+        const match = /P(?:T)?(\d+)([HMDWMOY]+)/.exec(duration);
         return match ? parseInt(match[1], 10) : null;
     }
 
     // Extract the unit from the ISO 8601 Duration string
     getUnitFromDuration(duration: string): TimeDurationUnit | null {
-        const match = /PT(\d+)([HM])/.exec(duration);
-        return match ? (match[2] as TimeDurationUnit) : null;
+        const match = /P(?:T)?(\d+)([HMDWMOY]+)/.exec(duration);
+
+        // replace the number in the full match with % to get the unit
+        const unit = match?.[0]?.replace(match?.[1], '%');
+        return unit as TimeDurationUnit;
     }
 
     // Extract the number from the Pandas Frequency string
@@ -130,7 +164,21 @@ export class CustomDurationInput extends LitElement {
     // Extract the unit from the Pandas Frequency string
     getUnitFromPandasFrequency(freq: string): PandasTimeUnit | null {
         const match = /(\d+)(min|h)/.exec(freq);
-        return match ? (match[2] as PandasTimeUnit) : null;
+
+        // replace the number in the full match with % to get the unit
+        const unit = match?.[0]?.replace(match?.[1], '%');
+        return unit as PandasTimeUnit;
+    }
+
+    // Get available unit options based on the units property or defaults
+    getUnitOptions(): [string, string][] {
+        if (this.type === DurationInputType.ISO_8601) {
+            return this.iso_units.map((unit) => [unit, TimeDurationUnitDisplay[unit]]);
+        } else if (this.type === DurationInputType.PANDAS_FREQ) {
+            return this.pandas_units.map((unit) => [unit, PandasTimeUnitDisplay[unit]]);
+        }
+
+        return [];
     }
 
     render() {
@@ -153,15 +201,7 @@ export class CustomDurationInput extends LitElement {
                 label="Unit"
                 .value=${this.unit}
                 @or-mwc-input-changed=${this.onInput}
-                .options="${this.type === DurationInputType.ISO_8601
-                    ? [
-                          [TimeDurationUnit.MINUTE, 'Minutes'],
-                          [TimeDurationUnit.HOUR, 'Hours']
-                      ]
-                    : [
-                          [PandasTimeUnit.MINUTE, 'Minutes'],
-                          [PandasTimeUnit.HOUR, 'Hours']
-                      ]}"
+                .options="${this.getUnitOptions()}"
             ></or-mwc-input>
         `;
     }

frontend/src/pages/pages-config-editor.ts

@@ -26,7 +26,7 @@ import { Router, RouterLocation } from '@vaadin/router';
 import { InputType, OrInputChangedEvent } from '@openremote/or-mwc-components/or-mwc-input';
 import { showSnackbar } from '@openremote/or-mwc-components/or-mwc-snackbar';
 import { getRootPath } from '../common/util';
-import { DurationInputType } from '../components/custom-duration-input';
+import { DurationInputType, TimeDurationUnit } from '../components/custom-duration-input';
 import { consume } from '@lit/context';
 import { realmContext } from './app-layout';
 
@@ -137,7 +137,7 @@ export class PageConfigEditor extends LitElement {
         target: {
             asset_id: '',
             attribute_name: '',
-            cutoff_timestamp: new Date().getTime()
+            training_data_period: 'P6M'
         },
         regressors: null,
         forecast_interval: 'PT1H',
@@ -342,7 +342,7 @@ export class PageConfigEditor extends LitElement {
         this.formData.regressors.push({
             asset_id: '',
             attribute_name: '',
-            cutoff_timestamp: new Date().getTime()
+            training_data_period: 'P6M'
         });
         this.requestUpdate();
     }
@@ -402,14 +402,14 @@ export class PageConfigEditor extends LitElement {
                             `
                         )}
 
-                        <or-mwc-input
-                            type="${InputType.DATETIME}"
-                            name="cutoff_timestamp"
-                            @or-mwc-input-changed="${(e: OrInputChangedEvent) => this.handleRegressorInput(e, index)}"
-                            label="Use datapoints since"
-                            .value="${regressor.cutoff_timestamp}"
-                            required
-                        ></or-mwc-input>
+                        <custom-duration-input
+                            name="training_data_period"
+                            .type="${DurationInputType.ISO_8601}"
+                            @value-changed="${(e: OrInputChangedEvent) => this.handleRegressorInput(e, index)}"
+                            label="Training data period"
+                            .iso_units="${[TimeDurationUnit.DAY, TimeDurationUnit.WEEK, TimeDurationUnit.MONTH, TimeDurationUnit.YEAR]}"
+                            .value="${regressor.training_data_period}"
+                        ></custom-duration-input>
 
                         <or-mwc-input
                             style="max-width: 48px;"
@@ -607,14 +607,15 @@ export class PageConfigEditor extends LitElement {
                                 `
                             )}
 
-                            <or-mwc-input
-                                type="${InputType.DATETIME}"
-                                name="target.cutoff_timestamp"
-                                @or-mwc-input-changed="${this.handleTargetInput}"
-                                label="Use datapoints since"
-                                .value="${this.formData.target.cutoff_timestamp}"
-                                required
-                            ></or-mwc-input>
+                            <!-- target.training_data_period -->
+                            <custom-duration-input
+                                name="target.training_data_period"
+                                .type="${DurationInputType.ISO_8601}"
+                                @value-changed="${this.handleTargetInput}"
+                                label="Training data period"
+                                .iso_units="${[TimeDurationUnit.DAY, TimeDurationUnit.WEEK, TimeDurationUnit.MONTH, TimeDurationUnit.YEAR]}"
+                                .value="${this.formData.target.training_data_period}"
+                            ></custom-duration-input>
                         </div>
                     </div>
                 </or-panel>

frontend/src/services/models.ts

@@ -41,10 +41,10 @@ export interface TargetFeature {
      */
     attribute_name: string;
     /**
-     * Timestamp in milliseconds since epoch. All data after this timestamp will be used for forecasting.
-     * Constraints: gt=0
+     * ISO 8601 duration string, this duration period will be used for retrieving training data.
+     * E.g. 'P6M' for data from the last 6 months.
      */
-    cutoff_timestamp: number;
+    training_data_period: string;
 }
 
 /**
@@ -62,10 +62,10 @@ export interface RegressorFeature {
      */
     attribute_name: string;
     /**
-     * Timestamp in milliseconds since epoch. All data after this timestamp will be used for forecasting.
-     * Constraints: gt=0
+     * ISO 8601 duration string, this duration period will be used for retrieving training data.
+     * E.g. 'P6M' for data from the last 6 months.
      */
-    cutoff_timestamp: number;
+    training_data_period: string;
 }
 
 /**

src/service_ml_forecast/common/time_util.py

@@ -17,6 +17,7 @@
 
 import logging
 import time
+from datetime import timedelta
 
 import pandas as pd
 from isodate import Duration, parse_duration
@@ -35,6 +36,12 @@ def get_timestamp_ms() -> int:
         millis = TimeUtil.sec_to_ms(timestamp)
         return millis
 
+    @staticmethod
+    def get_timestamp_sec() -> int:
+        """Get the current timestamp in seconds."""
+        timestamp = int(time.time())
+        return timestamp
+
     @staticmethod
     def parse_iso_duration(duration: str) -> int:
         """Parse the given time duration String and returns the corresponding number of seconds.
@@ -46,11 +53,22 @@ def parse_iso_duration(duration: str) -> int:
             The number of seconds.
         """
         # Parse the ISO 8601 duration string to a timedelta object
-        duration_obj: Duration = parse_duration(duration, as_timedelta_if_possible=False)
+        parsed: timedelta | Duration = parse_duration(duration)
+
+        seconds = 0
+
+        # Handle ISO 8601 strings that do not contain MONTH or YEAR (Timedelta)
+        if isinstance(parsed, timedelta):
+            seconds = int(parsed.total_seconds())
 
-        # Convert the duration object to seconds
-        duration_seconds = int(duration_obj.total_seconds())
-        return duration_seconds
+        # Edge case, the library returns a Duration object for ISO 8601 strings that contain MONTH or YEAR
+        # So we need to handle the conversion to seconds manually
+        if isinstance(parsed, Duration):
+            seconds = int(parsed.months * 30 * 24 * 60 * 60)
+            seconds += int(parsed.years * 365 * 24 * 60 * 60)  # Not taking into account leap years is OK
+            seconds += int(parsed.total_seconds())
+
+        return seconds
 
     @staticmethod
     def pd_future_timestamp(periods: int, frequency: str) -> int:
@@ -69,6 +87,31 @@ def pd_future_timestamp(periods: int, frequency: str) -> int:
 
         return millis
 
+    @staticmethod
+    def get_period_start_timestamp(period: str) -> int:
+        """Get the start timestamp for the period based on the provided ISO 8601 duration string.
+
+        Args:
+            period: The ISO 8601 duration string.
+
+        Returns:
+            The start timestamp in seconds.
+        """
+        start_timestamp = TimeUtil.get_timestamp_sec() - TimeUtil.parse_iso_duration(period)
+        return start_timestamp
+
+    @staticmethod
+    def get_period_start_timestamp_ms(period: str) -> int:
+        """Get the start timestamp for the period based on the provided ISO 8601 duration string.
+
+        Args:
+            period: The ISO 8601 duration string.
+
+        Returns:
+        """
+        start_timestamp = TimeUtil.get_period_start_timestamp(period)
+        return TimeUtil.sec_to_ms(start_timestamp)
+
     @staticmethod
     def sec_to_ms(timestamp: int) -> int:
         """Convert the epoch timestamp in seconds to milliseconds.
@@ -80,3 +123,42 @@ def sec_to_ms(timestamp: int) -> int:
             The epoch timestamp in milliseconds. (last 3 digits will be 000)
         """
         return int(timestamp * 1000)
+
+    @staticmethod
+    def months_between_timestamps(from_timestamp_ms: int, to_timestamp_ms: int) -> int:
+        """Calculate the number of months between two epoch millisecond timestamps.
+
+        Args:
+            from_timestamp_ms: The start timestamp in milliseconds.
+            to_timestamp_ms: The end timestamp in milliseconds.
+
+        Returns:
+            The number of months between the timestamps.
+        """
+        # Convert milliseconds to pandas timestamps
+        from_dt = pd.to_datetime(from_timestamp_ms, unit="ms")
+        to_dt = pd.to_datetime(to_timestamp_ms, unit="ms")
+
+        # Calculate the difference in months
+        months_diff = (to_dt.year - from_dt.year) * 12 + (to_dt.month - from_dt.month)
+        return months_diff
+
+    @staticmethod
+    def add_months_to_timestamp(timestamp_ms: int, months: int) -> int:
+        """Add a specified number of months to a timestamp in milliseconds.
+
+        Args:
+            timestamp_ms: The timestamp in milliseconds.
+            months: The number of months to add.
+
+        Returns:
+            The new timestamp in milliseconds.
+        """
+        # Convert milliseconds to pandas timestamp
+        dt = pd.to_datetime(timestamp_ms, unit="ms")
+
+        # Add months
+        new_dt = dt + pd.DateOffset(months=months)
+
+        # Convert back to milliseconds
+        return int(new_dt.timestamp() * 1000)