2.19 based feature branch for serverless

.github/workflows/changelog_verifier.yml

@@ -0,0 +1,23 @@
+name: "Changelog Verifier"
+on:
+  push:
+    branches-ignore:
+      - 'whitesource-remediate/**'
+      - 'backport/**'
+  pull_request:
+    types: [opened, edited, review_requested, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+jobs:
+  # Enforces the update of a changelog file on every pull request
+  verify-changelog:
+    if: github.repository == 'opensearch-project/anomaly-detection'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: dangoslen/changelog-enforcer@v3
+        with:
+          skipLabels: "autocut, skip-changelog"

.github/workflows/labeler.yml

@@ -6,9 +6,6 @@ on:
     types:
       - opened
 
-env:
-  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-
 jobs:
   label:
     runs-on: ubuntu-latest

.github/workflows/maven-publish.yml

@@ -4,9 +4,9 @@ on:
   workflow_dispatch:
   push:
     branches:
-        - 'main'
-        - '1.*'
-        - '2.*'
+      - 'main'
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
 
 jobs:
   build-and-publish-snapshots:

.github/workflows/test_build_multi_platform.yml

@@ -18,10 +18,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      # Spotless requires JDK 17+
       - uses: actions/setup-java@v4
         with:
-          java-version: 17
+          java-version: 21
           distribution: temurin
       - name: Spotless Check
         run: ./gradlew spotlessCheck
@@ -30,7 +29,7 @@ jobs:
     needs: spotless
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 21 ]
     name: Build and Test Anomaly Detection Plugin on Windows
     runs-on: windows-latest
     env:
@@ -58,7 +57,7 @@ jobs:
     needs: [Get-CI-Image-Tag, spotless]
     strategy:
       matrix:
-        java: [11, 17, 21]
+        java: [21]
       fail-fast: false
     name: Build and Test Anomaly detection Plugin on Linux
     runs-on: ubuntu-latest
@@ -96,7 +95,7 @@ jobs:
     needs: spotless
     strategy:
       matrix:
-        java: [11,17,21]
+        java: [21]
       fail-fast: false
 
     name: Build and Test Anomaly detection Plugin on MacOS

.github/workflows/test_security.yml

@@ -1,85 +1,49 @@
-name: Security test workflow for Anomaly Detection
+name: Security test workflow
+# This workflow is triggered on pull requests to main
 on:
-  push:
-    branches:
-      - "*"
   pull_request:
     branches:
-      - "*"
-
-env:
-  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+      - "**"
+  push:
+    branches:
+      - "**"
 
 jobs:
-  Build-ad:
-    strategy:
-      matrix:
-        java: [11,17,21]
-      fail-fast: false
-
-    name: Security test workflow for Anomaly Detection
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  security-test:
+    needs: Get-CI-Image-Tag
+    # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-      - name: Setup Java ${{ matrix.java }}
-        uses: actions/setup-java@v3
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK
+        uses: actions/setup-java@v4
         with:
-          distribution: 'temurin'
-          java-version: ${{ matrix.java }}
-
-      # time-series-analytics
-      - name: Checkout AD
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 21
+      # index-management
+      - name: Checkout Branch
         uses: actions/checkout@v4
-
-      - name: Build Anomaly Detection
-        run: |
-          ./gradlew assemble
-      # example of variables:
-      # plugin = opensearch-time-series-analytics-2.10.0.0-SNAPSHOT.zip
-      # version = 2.10.0, plugin_version = 2.10.0.0, qualifier = SNAPSHOT
-      - name: Pull and Run Docker
+      - name: Run integration tests
         run: |
-          plugin=`basename $(ls build/distributions/*.zip)`
-          version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
-          plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
-          qualifier=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1`
-
-          if $qualifier!=SNAPSHOT
-          then
-            docker_version=$version-$qualifier
-          else
-            docker_version=$version
-          fi
-          echo plugin version plugin_version qualifier docker_version
-          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
-
-          cd ..
-          if docker pull opensearchstaging/opensearch:$docker_version
-          then
-            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-anomaly-detection ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-anomaly-detection; fi" >> Dockerfile
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-time-series-analytics ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-time-series-analytics; fi" >> Dockerfile
-            echo "ADD anomaly-detection/build/distributions/$plugin /tmp/" >> Dockerfile
-            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
-            docker build -t opensearch-ad:test .
-            echo "imagePresent=true" >> $GITHUB_ENV
-          else
-            echo "imagePresent=false" >> $GITHUB_ENV
-          fi
-      - name: Run Docker Image
-        if: env.imagePresent == 'true'
-        run: |
-          cd ..
-          docker run -p 9200:9200 -d -p 9600:9600 -e "OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123!" -e "discovery.type=single-node" opensearch-ad:test
-          sleep 90
-      - name: Run AD Test
-        if: env.imagePresent == 'true'
-        run: |
-          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:myStrongPassword123! --insecure |grep opensearch-security|wc -l`
-          if [ $security -gt 0 ]
-          then
-            echo "Security plugin is available"
-            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=myStrongPassword123!
-          else
-            echo "Security plugin is NOT available, skipping integration tests"
-          fi
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'"
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: logs
+          overwrite: 'true'
+          path: build/testclusters/integTest-*/logs/*

CHANGELOG.md

@@ -0,0 +1,30 @@
+# CHANGELOG
+All notable changes to this project are documented in this file.
+
+Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
+
+## [Unreleased 3.x](https://github.com/opensearch-project/anomaly-detection/compare/3.0...HEAD)
+### Features
+### Enhancements
+- Support >1 hr intervals ([#1513](https://github.com/opensearch-project/anomaly-detection/pull/1513))
+- Introduce state machine, separate config index, improve suggest/validate APIs, and persist cold-start results for run-once visualization ([#1479](https://github.com/opensearch-project/anomaly-detection/pull/1479))
+
+
+### Bug Fixes
+- Fixing concurrency bug on writer ([#1508](https://github.com/opensearch-project/anomaly-detection/pull/1508))
+- fix(forecast): advance past current interval & anchor on now ([#1528](https://github.com/opensearch-project/anomaly-detection/pull/1528))
+- Changing search calls on interval calculation ([#1535](https://github.com/opensearch-project/anomaly-detection/pull/1535))
+
+### Infrastructure
+### Documentation
+### Maintenance
+### Refactoring
+
+## [Unreleased 2.x](https://github.com/opensearch-project/anomaly-detection/compare/2.19...2.x)
+### Features
+### Enhancements
+### Bug Fixes
+### Infrastructure
+### Documentation
+### Maintenance
+### Refactoring

DEVELOPER_GUIDE.md

@@ -40,16 +40,19 @@ Currently we just put RCF jar in lib as dependency. Plan to publish to Maven and
 
 1. `./gradlew build` builds and tests
 2. `./gradlew :run` launches a single node cluster with anomaly-detection (and job-scheduler) plugin installed
-3. `./gradlew :integTest` launches a single node cluster with anomaly-detection (and job-scheduler) plugin installed and runs all integration tests except security
-4. ` ./gradlew :integTest --tests="**.test execute foo"` runs a single integration test class or method
-5. `./gradlew integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=<admin-password>` launches integration tests against a local cluster and run tests with security
-6. `./gradlew spotlessApply` formats code. And/or import formatting rules in `.eclipseformat.xml` with IDE.
-7. `./gradlew adBwcCluster#mixedClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by upgrading one of the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler creating a mixed cluster.
-8. `./gradlew adBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with anomaly-detection and job-scheduler.
-9. `./gradlew adBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing a full restart on the cluster upgrading all the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler.
-10. `./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined.
-11. `./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true` launches HCAD model performance tests and logs the result in the standard output
-12. `./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true` launches single stream AD model performance tests and logs the result in the standard output
+3. `./gradlew :run -PdualCluster=true` launches 2 single node clusters with anomaly-detection (and job-scheduler) plugin installed, one cluster is on localhost:9200 and the other on localhost:9200
+4. `./gradlew :integTest` launches a single node cluster with anomaly-detection (and job-scheduler) plugin installed and runs all integration tests except security
+5. ` ./gradlew :integTest --tests="**.test execute foo"` runs a single integration test class or method
+6. `./gradlew integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=<admin-password>` launches integration tests against a local cluster and run tests with security
+7. `./gradlew spotlessApply` formats code. And/or import formatting rules in `.eclipseformat.xml` with IDE.
+8. `./gradlew adBwcCluster#mixedClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by upgrading one of the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler creating a mixed cluster.
+9. `./gradlew adBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with anomaly-detection and job-scheduler.
+10. `./gradlew adBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing a full restart on the cluster upgrading all the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler.
+11. `./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined.
+12. `./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true` launches HCAD model performance tests and logs the result in the standard output
+13. `./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true` launches single stream AD model performance tests and logs the result in the standard output
+14. `./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'` runs integration tests against a secure cluster
+
 
 When launching a cluster using one of the above commands logs are placed in `/build/cluster/run node0/opensearch-<version>/logs`. Though the logs are teed to the console, in practices it's best to check the actual log file.
 

build-tools/coverage.gradle

@@ -6,7 +6,7 @@
 apply plugin: 'jacoco'
 
 jacoco {
-    toolVersion = "0.8.10"
+    toolVersion = "0.8.12"
 }
 
 /**