Merge remote-tracking branch 'origin/master' into monitoring-ring-buffer

elastic · Jan 28, 2022 · 6c8797f · 6c8797f
2 parents 6006ffc + 70ce544
commit 6c8797f
Show file tree

Hide file tree

Showing 643 changed files with 21,282 additions and 9,847 deletions.
diff --git a/.ci/beats-tester-bc.groovy b/.ci/beats-tester-bc.groovy
@@ -7,7 +7,7 @@ pipeline {
   environment {
     BASE_DIR = 'src/github.com/elastic/beats'
     PIPELINE_LOG_LEVEL = "INFO"
-    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/master'
+    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/main'
     BASE_URL = "https://staging.elastic.co/${params.version}/downloads"
     APM_BASE_URL = "${env.BASE_URL}/apm-server"
     BEATS_BASE_URL = "${env.BASE_URL}/beats"

diff --git a/.ci/beats-tester.groovy b/.ci/beats-tester.groovy
@@ -7,7 +7,7 @@ pipeline {
   environment {
     BASE_DIR = 'src/github.com/elastic/beats'
     PIPELINE_LOG_LEVEL = "INFO"
-    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/master'
+    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/main'
   }
   options {
     timeout(time: 1, unit: 'HOURS')

diff --git a/.ci/jobs/apm-beats-update.yml b/.ci/jobs/apm-beats-update.yml
@@ -18,7 +18,7 @@
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
         discover-tags: true
-        head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         disable-pr-notifications: true
         notification-context: 'apm-beats-update'
         repo: 'beats'
@@ -36,10 +36,7 @@
                 name: 'master'
                 case-sensitive: true
             - regex-name:
-                regex: '7\.[x789]'
-                case-sensitive: true
-            - regex-name:
-                regex: '7\.1\d'
+                regex: '7\.1[6789]'
                 case-sensitive: true
             - regex-name:
                 regex: '8\.\d+'

diff --git a/.ci/jobs/beats-release-changelog.yml b/.ci/jobs/beats-release-changelog.yml
@@ -17,4 +17,4 @@
             credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
             reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
             branches:
-                - master
+                - main
diff --git a/.ci/jobs/beats-release-minor-major.yml b/.ci/jobs/beats-release-minor-major.yml
@@ -17,4 +17,4 @@
             credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
             reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
             branches:
-                - master
+                - main
diff --git a/.ci/jobs/beats-release-patch.yml b/.ci/jobs/beats-release-patch.yml
@@ -17,4 +17,4 @@
             credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
             reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
             branches:
-                - master
+                - main
diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml
@@ -14,7 +14,7 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          head-filter-regex: '(master|7\.([x9]|1\d+)|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+          head-filter-regex: '(master|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
           disable-pr-notifications: true
           notification-context: 'beats-tester'
           repo: 'beats'
@@ -32,7 +32,7 @@
                   name: 'master'
                   case-sensitive: true
               - regex-name:
-                  regex: '7\.([x9]|1\d+)'
+                  regex: '7\.1[6789]'
                   case-sensitive: true
               - regex-name:
                   regex: '8\.\d+'

diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml
@@ -17,7 +17,7 @@
         discover-pr-forks-strategy: 'merge-current'
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
-        head-filter-regex: '(master|6\.[89]|7\.16|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|6\.[89]|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         discover-tags: true
         notification-context: "beats-ci"
         repo: 'beats'
@@ -39,10 +39,7 @@
                 regex: '6\.[89]'
                 case-sensitive: true
             - regex-name:
-                regex: '7\.[x789]'
-                case-sensitive: true
-            - regex-name:
-                regex: '7\.1\d'
+                regex: '7\.1[6789]'
                 case-sensitive: true
             - regex-name:
                 regex: '8\.\d+'

diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml
@@ -14,7 +14,7 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+          head-filter-regex: '(master|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
           disable-pr-notifications: true
           notification-context: 'beats-packaging'
           repo: 'beats'
@@ -32,10 +32,7 @@
                   name: 'master'
                   case-sensitive: true
               - regex-name:
-                  regex: '7\.[x789]'
-                  case-sensitive: true
-              - regex-name:
-                  regex: '7\.1\d'
+                  regex: '7\.1[6789]'
                   case-sensitive: true
               - regex-name:
                   regex: '8\.\d+'

diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -307,8 +307,6 @@ def tagAndPush(Map args = [:]) {
     tagName = "pr-${env.CHANGE_ID}"
   }
 
-  dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-
   // supported tags
   def tags = [tagName, "${env.GIT_BASE_COMMIT}"]
   if (!isPR() && aliasVersion != "") {
@@ -386,6 +384,7 @@ def release(){
     withEnv([
       "DEV=true"
     ]) {
+      dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
       dir("${env.BEATS_FOLDER}") {
         sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
       }

diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh
@@ -21,7 +21,7 @@ function dockerPullCommonImages() {
   docker.elastic.co/observability-ci/database-enterprise:12.2.0.1
   docker.elastic.co/beats-dev/fpm:1.11.0
   golang:1.14.12-stretch
-  centos:7
+  ubuntu:20.04
   "
   for image in ${DOCKER_IMAGES} ; do
     (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing."

diff --git a/.ci/schedule-daily.groovy b/.ci/schedule-daily.groovy
@@ -20,12 +20,7 @@ pipeline {
   stages {
     stage('Nighly beats builds') {
       steps {
-        runBuild(quietPeriod: 0, job: 'Beats/beats/master')
-        // This should be `current_8` bump.getCurrentMinorReleaseFor8
-        runBuild(quietPeriod: 2000, job: 'Beats/beats/8.0')
-        // This should be `current_7` bump.getCurrentMinorReleaseFor7 or
-        // `next_minor_7`  bump.getNextMinorReleaseFor7
-        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.16')
+        runBuilds(quietPeriodFactor: 2000, branches: ['master', '8.<minor>', '7.<minor>', '7.<next-minor>'])
       }
     }
   }
@@ -36,7 +31,37 @@ pipeline {
   }
 }
 
-def runBuild(Map args = [:]) {
-  def jobName = args.job
-  build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false)
+def runBuilds(Map args = [:]) {
+  def branches = []
+  // Expand macros and filter duplicated matches.
+  args.branches.each { branch ->
+    def branchName = getBranchName(branch)
+    if (!branches.contains(branchName)) {
+      branches << branchName
+    }
+  }
+
+  def quietPeriod = 0
+  branches.each { branch ->
+    build(quietPeriod: quietPeriod, job: "Beats/beats/${branch}", parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false)
+    // Increate the quiet period for the next iteration
+    quietPeriod += args.quietPeriodFactor
+  }
+}
+
+def getBranchName(branch) {
+  // special macro to look for the latest minor version
+  if (branch.contains('8.<minor>')) {
+   return bumpUtils.getMajorMinor(bumpUtils.getCurrentMinorReleaseFor8())
+  }
+  if (branch.contains('8.<next-minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getNextMinorReleaseFor8())
+  }
+  if (branch.contains('7.<minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getCurrentMinorReleaseFor7())
+  }
+  if (branch.contains('7.<next-minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getNextMinorReleaseFor7())
+  }
+  return branch
 }
diff --git a/.ci/schedule-weekly.groovy b/.ci/schedule-weekly.groovy
@@ -20,12 +20,7 @@ pipeline {
   stages {
     stage('Weekly beats builds') {
       steps {
-        runBuild(quietPeriod: 0, job: 'Beats/beats/master')
-        // This should be `current_8` bump.getCurrentMinorReleaseFor8
-        runBuild(quietPeriod: 1000, job: 'Beats/beats/8.0')
-        // This should be `current_7` bump.getCurrentMinorReleaseFor7 or
-        // `next_minor_7`  bump.getNextMinorReleaseFor7
-        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.16')
+        runBuilds(quietPeriodFactor: 1000, branches: ['master', '8.<minor>', '7.<minor>', '7.<next-minor>'])
       }
     }
   }
@@ -36,7 +31,37 @@ pipeline {
   }
 }
 
-def runBuild(Map args = [:]) {
-  def jobName = args.job
-  build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+def runBuilds(Map args = [:]) {
+  def branches = []
+  // Expand macros and filter duplicated matches.
+  args.branches.each { branch ->
+    def branchName = getBranchName(branch)
+    if (!branches.contains(branchName)) {
+      branches << branchName
+    }
+  }
+
+  def quietPeriod = 0
+  branches.each { branch ->
+    build(quietPeriod: quietPeriod, job: "Beats/beats/${branch}", parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+    // Increate the quiet period for the next iteration
+    quietPeriod += args.quietPeriodFactor
+  }
+}
+
+def getBranchName(branch) {
+  // special macro to look for the latest minor version
+  if (branch.contains('8.<minor>')) {
+   return bumpUtils.getMajorMinor(bumpUtils.getCurrentMinorReleaseFor8())
+  }
+  if (branch.contains('8.<next-minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getNextMinorReleaseFor8())
+  }
+  if (branch.contains('7.<minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getCurrentMinorReleaseFor7())
+  }
+  if (branch.contains('7.<next-minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getNextMinorReleaseFor7())
+  }
+  return branch
 }
diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat
@@ -16,21 +16,15 @@ SET PREVIOUS_USERPROFILE=%USERPROFILE%
 SET USERPROFILE=%OLD_USERPROFILE%
 IF NOT EXIST C:\Python38\python.exe (
     REM Install python 3.8
-    choco install python -y -r --no-progress --version 3.8.5
-    IF NOT ERRORLEVEL 0 (
-        exit /b 1
-    )
+    choco install python -y -r --no-progress --version 3.8.5 || exit /b 1
 )
 python --version
 where python
 
-where /q gcc
-IF ERRORLEVEL 1 (
+WHERE /q gcc
+IF %ERRORLEVEL% NEQ 0 (
     REM Install mingw 5.3.0
-    choco install mingw -y -r --no-progress --version 5.3.0
-    IF NOT ERRORLEVEL 0 (
-        exit /b 1
-    )
+    choco install mingw -y -r --no-progress --version 5.3.0 || exit /b 1
 )
 gcc --version
 where gcc

diff --git a/.ci/scripts/kind-setup.sh b/.ci/scripts/kind-setup.sh
@@ -1,23 +1,5 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-kind create cluster --image kindest/node:${K8S_VERSION} --config - <<EOF
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  kubeadmConfigPatches:
-  - |
-    kind: ClusterConfiguration
-    scheduler:
-      extraArgs:
-        bind-address: "0.0.0.0"
-        port: "10251"
-        secure-port: "10259"
-    controllerManager:
-      extraArgs:
-        bind-address: "0.0.0.0"
-        port: "10252"
-        secure-port: "10257"
-EOF
+kind create cluster --image kindest/node:${K8S_VERSION}
 kubectl cluster-info
diff --git a/.github/stale.yml b/.github/stale.yml
@@ -11,10 +11,11 @@ daysUntilClose: 30
 onlyLabels: []
 
 # Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
-exemptLabels: []
+exemptLabels:
+  - flaky-test
 
 # Set to true to ignore issues in a project (defaults to false)
-exemptProjects: true
+exemptProjects: false
 
 # Set to true to ignore issues in a milestone (defaults to false)
 exemptMilestones: true
@@ -47,7 +48,7 @@ limitPerRun: 30
 
 # Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
 pulls:
-  daysUntilStale: 30
+  daysUntilStale: 90
   daysUntilClose: 30
   markComment: >
     Hi!
@@ -79,4 +80,4 @@ pulls:
 
 # issues:
 #   exemptLabels:
-#     - confirmed
+#     - confirmed
diff --git a/.github/workflows/macos-auditbeat.yml b/.github/workflows/macos-auditbeat.yml
@@ -0,0 +1,31 @@
+name: auditbeat
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/macos-auditbeat.yml'
+  push:
+    branches:
+      - master
+      - 7.1*
+      - 8.*
+
+env:
+  BEAT_MODULE: 'auditbeat'
+
+jobs:
+  macos:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Fetch Go version from .go-version
+      run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ${{ env.GO_VERSION }}
+    - name: Install dependencies
+      run:  go get -u github.com/magefile/mage
+    - name: Run build
+      run: cd ${{ env.BEAT_MODULE }} && mage build
+    - name: Run test
+      run: cd ${{ env.BEAT_MODULE }} && mage unitTest
diff --git a/.github/workflows/macos-filebeat.yml b/.github/workflows/macos-filebeat.yml
@@ -0,0 +1,31 @@
+name: filebeat
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/macos-filebeat.yml'
+  push:
+    branches:
+      - master
+      - 7.1*
+      - 8.*
+
+env:
+  BEAT_MODULE: 'filebeat'
+
+jobs:
+  macos:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Fetch Go version from .go-version
+      run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ${{ env.GO_VERSION }}
+    - name: Install dependencies
+      run:  go get -u github.com/magefile/mage
+    - name: Run build
+      run: cd ${{ env.BEAT_MODULE }} && mage build
+    - name: Run test
+      run: cd ${{ env.BEAT_MODULE }} && mage unitTest