Merge branch 'main' into zmoog/fix-azure-storage-account-dashboard

.buildkite/hooks/pre-command

@@ -33,7 +33,6 @@ AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
 PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
 
 BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
-GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 
 EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
 EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
@@ -101,7 +100,7 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "publish-benchmarks" ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
-        GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}")
+        GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
         export GITHUB_TOKEN
     fi
 
@@ -147,12 +146,10 @@ fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
-    GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username "${GITHUB_TOKEN_VAULT_PATH}")
-    export GITHUB_USERNAME_SECRET
-    GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email "${GITHUB_TOKEN_VAULT_PATH}")
-    export GITHUB_EMAIL_SECRET
-    GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}")
-    export GITHUB_TOKEN
+    GITHUB_USERNAME="elastic-vault-github-plugin-prod"
+    GITHUB_EMAIL="elasticmachine@elastic.co"
+    GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
+    export GITHUB_TOKEN GITHUB_EMAIL GITHUB_USERNAME
   fi
 fi
 

.buildkite/pipeline.schedule-daily.yml

@@ -51,6 +51,18 @@ steps:
       - step: "check"
         allow_failure: false
 
+  - label: "Check integrations local stacks - Stack Version v9.0"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        STACK_VERSION: 9.0.0-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "false"
+    depends_on:
+      - step: "check"
+        allow_failure: false
+
   - label: "Check integrations in serverless - project: Observability"
     key: "trigger-integrations-serverless-obs"
     trigger: "integrations-serverless"

.buildkite/pipeline.schedule-weekly.yml

@@ -27,3 +27,16 @@ steps:
     depends_on:
       - step: "check"
         allow_failure: false
+
+  - label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.0"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        STACK_VERSION: 9.0.0-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "false"
+        ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
+    depends_on:
+      - step: "check"
+        allow_failure: false

.buildkite/pipeline.yml

@@ -102,5 +102,8 @@ steps:
     # not fail build if this step fails
     soft_fail: true
     # run this step when if it is triggered by the daily job
-    if: >
-      build.source == "trigger_job" &&  build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"
+    # skipped for now notification for tests related to STACK_VERSION: 9
+    if: |
+      build.source == "trigger_job" &&
+      build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" &&
+      build.env('STACK_VERSION') !~ /^9/

.buildkite/scripts/backport_branch.sh

@@ -106,10 +106,9 @@ removeOtherPackages() {
 
 update_git_config() {
     pushd $WORKSPACE > /dev/null
-    git config --global user.name "${GITHUB_USERNAME_SECRET}"
-    git config --global user.email "${GITHUB_EMAIL_SECRET}"
+    git config --global user.name "${GITHUB_USERNAME}"
+    git config --global user.email "${GITHUB_EMAIL}"
 
-    git config remote.origin.url "https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN}@github.com/elastic/integrations.git"
     popd > /dev/null
 }
 

.buildkite/scripts/build_packages.sh

@@ -43,7 +43,7 @@ report_build_failure() {
 
     # if running in Buildkite , add an annotation
     if [ -n "${BUILDKITE_BRANCH+x}" ]; then
-        buildkite-agent annotate "Build package ${package} failed, not published." --ctx "ctx-build-${package}" --style "warning"
+        buildkite-agent annotate "Build package ${package} failed, not published." --context "ctx-build-${package}" --style "warning"
     fi
 }
 

.buildkite/scripts/common.sh

@@ -126,8 +126,6 @@ with_mage() {
 
     local install_packages=(
             "github.com/magefile/mage"
-            "github.com/elastic/go-licenser"
-            "golang.org/x/tools/cmd/goimports"
             "github.com/jstemmer/go-junit-report"
             "gotest.tools/gotestsum"
     )
@@ -464,12 +462,19 @@ is_supported_stack() {
     if [ "${kibana_version}" == "null" ]; then
         return 0
     fi
-    if [[ ! "${kibana_version}" =~ \^7\. && "${STACK_VERSION}" =~ ^7\. ]]; then
+    if [[ ( ! "${kibana_version}" =~ \^7\. ) && "${STACK_VERSION}" =~ ^7\. ]]; then
         return 1
     fi
-    if [[ ! "${kibana_version}" =~ \^8\. && "${STACK_VERSION}" =~ ^8\. ]]; then
+    if [[ ( ! "${kibana_version}" =~ \^8\. ) && "${STACK_VERSION}" =~ ^8\. ]]; then
         return 1
     fi
+
+    # TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too.
+    # This workaround should be removed once packages have been updated their constraints for 9.0 stack.
+    if [[ ( ! ( "${kibana_version}" =~ \^9\. || "${kibana_version}" =~ \^8\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then
+        return 1
+    fi
+
     return 0
 }
 

.buildkite/scripts/find_oldest_supported_version.py

@@ -67,9 +67,9 @@ def find_oldest_supported_version(kibana_version_condition: str) -> str:
         if len(available_parts) < 2:
             continue
 
-        available_major = available_parts[0]
-        available_minor = available_parts[1]
-        if major == available_major and minor > available_minor:
+        available_major = int(available_parts[0])
+        available_minor = int(available_parts[1])
+        if int(major) == available_major and int(minor) > available_minor:
             older = False
             break
     if older:
@@ -181,6 +181,7 @@ class TestFindOldestSupportVersion(unittest.TestCase):
             "8.11.0-SNAPSHOT"
         ],
         "aliases": [
+            "7.x-SNAPSHOT",
             "7.17-SNAPSHOT",
             "7.17",
             "8.7",
@@ -225,6 +226,14 @@ def test_available_version_contains_kibana_version(self):
     def test_too_old_to_be_in_api(self):
         self.assertEqual(find_oldest_supported_version("7.16.0"), "7.16.0")
         self.assertEqual(find_oldest_supported_version("8.6.0"), "8.6.0")
+        self.assertEqual(find_oldest_supported_version("7.6.0"), "7.6.0")
+
+    def test_no_version_available_no_next_minor_in_current_major(self):
+        # returns the version as in the manifest
+        self.assertEqual(find_oldest_supported_version("8.11.3"), "8.11.3")
+
+    def test_available_next_minor_in_current_major(self):
+        self.assertEqual(find_oldest_supported_version("7.19.0"), "7.x-SNAPSHOT")
 
     def test_or(self):
         self.assertEqual(find_oldest_supported_version("8.6.0||8.7.0"), "8.6.0")

.github/CODEOWNERS

@@ -171,6 +171,7 @@
 /packages/digital_guardian @elastic/security-service-integrations
 /packages/docker @elastic/obs-cloudnative-monitoring
 /packages/elastic_agent @elastic/elastic-agent
+/packages/elastic_connectors @elastic/ingestion-team
 /packages/elastic_package_registry @elastic/ecosystem
 /packages/elasticsearch @elastic/stack-monitoring
 /packages/enterprisesearch @elastic/stack-monitoring
@@ -311,6 +312,7 @@
 /packages/prometheus_input @elastic/obs-infraobs-integrations
 /packages/proofpoint_on_demand @elastic/security-service-integrations
 /packages/proofpoint_tap @elastic/security-service-integrations
+/packages/proxysg @elastic/sec-deployment-and-devices
 /packages/pulse_connect_secure @elastic/security-service-integrations
 /packages/qualys_vmdr @elastic/security-service-integrations
 /packages/qnap_nas @elastic/sec-deployment-and-devices
@@ -344,6 +346,7 @@
 /packages/symantec_endpoint_security @elastic/security-service-integrations
 /packages/synthetics @elastic/obs-ux-infra_services-team
 /packages/synthetics_dashboards @elastic/obs-ux-infra_services-team
+/packages/sysdig @elastic/security-service-integrations
 /packages/sysmon_linux @elastic/sec-linux-platform
 /packages/system @elastic/obs-infraobs-integrations
 /packages/system/changelog.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
@@ -386,6 +389,7 @@
 /packages/ti_otx @elastic/security-service-integrations
 /packages/ti_rapid7_threat_command @elastic/security-service-integrations
 /packages/ti_recordedfuture @elastic/security-service-integrations
+/packages/ti_custom @elastic/security-service-integrations
 /packages/ti_threatconnect @elastic/security-service-integrations
 /packages/ti_threatq @elastic/security-service-integrations
 /packages/ti_util @elastic/security-service-integrations

.github/ISSUE_TEMPLATE/integration_bug.yml

@@ -101,6 +101,7 @@ body:
         - Digital Guardian [digital_guardian]
         - Docker [docker]
         - Elastic Agent [elastic_agent]
+        - Elastic Connectors [elastic_connectors]
         - Elastic Package Registry [elastic_package_registry]
         - Elasticsearch [elasticsearch]
         - Enterprise Search [enterprisesearch]
@@ -218,6 +219,7 @@ body:
         - Prometheus Input [prometheus_input]
         - Proofpoint On Demand [proofpoint_on_demand]
         - Proofpoint TAP [proofpoint_tap]
+        - Broadcom ProxySG [proxysg]
         - Pulse Connect Secure [pulse_connect_secure]
         - QNAP NAS [qnap_nas]
         - Qualys VMDR [qualys_vmdr]
@@ -251,6 +253,7 @@ body:
         - Symantec Endpoint Security [symantec_endpoint_security]
         - Elastic Synthetics [synthetics]
         - Elastic Synthetics Dashboards [synthetics_dashboards]
+        - Sysdig [sysdig]
         - Sysmon for Linux [sysmon_linux]
         - System [system]
         - System Audit [system_audit]
@@ -264,6 +267,7 @@ body:
         - Anomali [ti_anomali]
         - Collective Intelligence Framework v3 [ti_cif3]
         - CrowdStrike Falcon Intelligence [ti_crowdstrike]
+        - Custom Threat Intelligence [ti_custom]
         - Cybersixgill [ti_cybersixgill]
         - EclecticIQ [ti_eclecticiq]
         - ESET Threat Intelligence [ti_eset]

.github/workflows/updatecli/updatecli.d/bump-latest-snapshot-version.yml → .github/workflows/updatecli/updatecli.d/bump-latest-8x-snapshot-version.yml

@@ -1,6 +1,6 @@
 ---
-name: Bump latest elastic-stack test version
-pipelineid: 'bump-latest-elastic-stack-version'
+name: Bump latest 8.x-SNAPSHOT elastic-stack test version
+pipelineid: 'bump-latest-8x-elastic-stack-version'
 
 actions:
   default:
@@ -29,13 +29,13 @@ sources:
     name: Get latest snapshot
     kind: json
     spec:
-      file: https://storage.googleapis.com/artifacts-api/snapshots/main.json
+      file: https://storage.googleapis.com/artifacts-api/snapshots/8.x.json
       key: .version
   latestSnapshotMajorMinor:
     name: Get latest snapshort major and minor
     kind: json
     spec:
-      file: https://storage.googleapis.com/artifacts-api/snapshots/main.json
+      file: https://storage.googleapis.com/artifacts-api/snapshots/8.x.json
       key: .version
     transformers:
       - findsubmatch:

.github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml

@@ -0,0 +1,84 @@
+---
+name: Bump latest 9.x-SNAPSHOT elastic-stack test version
+pipelineid: 'bump-latest-9x-elastic-stack-version'
+
+actions:
+  default:
+    title: '[updatecli] Update latest snapshot to {{ source "latestSnapshot" }}'
+    kind: github/pullrequest
+    scmid: default
+    spec:
+      labels:
+        - automation
+        - dependency
+
+scms:
+  default:
+    kind: github
+    spec:
+      owner: '{{ .scm.owner }}'
+      repository: '{{ .scm.repository }}'
+      user: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      username: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      token: '{{ requiredEnv "GITHUB_TOKEN" }}'
+      commitusingapi: true
+      branch: main
+
+sources:
+  latestSnapshot:
+    name: Get latest snapshot
+    kind: json
+    spec:
+      file: https://storage.googleapis.com/artifacts-api/snapshots/main.json
+      key: .version
+  latestSnapshotMajorMinor:
+    name: Get latest snapshort major and minor
+    kind: json
+    spec:
+      file: https://storage.googleapis.com/artifacts-api/snapshots/main.json
+      key: .version
+    transformers:
+      - findsubmatch:
+          pattern: '(^\d+\.\d+)\.[^\s]+$'
+          captureindex: 1
+
+targets:
+  update-snapshot-daily:
+    name: '[updatecli] [daily] Update latest snapshot to {{ source "latestSnapshot" }}'
+    kind: file
+    sourceid: latestSnapshot
+    scmid: default
+    spec:
+      file: '.buildkite/pipeline.schedule-daily.yml'
+      matchpattern: '(STACK_VERSION:) 9\.[^\s]*\.[^\s]*'
+      replacepattern: '$1 {{ source "latestSnapshot" }}'
+
+  update-snapshot-label-daily:
+    name: '[updatecli] [daily] Update latest snapshot label step to {{ source "latestSnapshotMajorMinor" }}'
+    kind: file
+    sourceid: latestSnapshotMajorMinor
+    scmid: default
+    spec:
+      file: '.buildkite/pipeline.schedule-daily.yml'
+      matchpattern: '(Stack Version) v9\.\d+"'
+      replacepattern: 'Stack Version v{{ source "latestSnapshotMajorMinor" }}"'
+
+  update-snapshot-weekly:
+    name: '[updatecli] [weekly] Update latest snapshot to {{ source "latestSnapshot" }}'
+    kind: file
+    sourceid: latestSnapshot
+    scmid: default
+    spec:
+      file: '.buildkite/pipeline.schedule-weekly.yml'
+      matchpattern: '(STACK_VERSION:) 9\.[^\s]*\.[^\s]*'
+      replacepattern: '$1 {{ source "latestSnapshot" }}'
+
+  update-snapshot-label-weekly:
+    name: '[updatecli] [weekly] Update latest snapshot label step to {{ source "latestSnapshotMajorMinor" }}'
+    kind: file
+    sourceid: latestSnapshotMajorMinor
+    scmid: default
+    spec:
+      file: '.buildkite/pipeline.schedule-weekly.yml'
+      matchpattern: '(Stack Version) v9\.\d+"'
+      replacepattern: 'Stack Version v{{ source "latestSnapshotMajorMinor" }}"'

catalog-info.yaml

@@ -142,7 +142,7 @@ spec:
       schedules:
         main_weekly:
           branch: "main"
-          cronline: "30 3 * * 1"
+          cronline: "00 5 * * 1"
           message: "Run the weekly jobs"
       provider_settings:
         trigger_mode: none # don't trigger jobs from github activity