Skip to content

Commit

Permalink
Backport some features in Jenkinsfile to 7.x (#18040)
Browse files Browse the repository at this point in the history 
Backport some features added to Jenkinsfile to 7.x branch:
* Dry run option.
* Docker login.
* Git config for generator tests.
* Filter changes using go list.

These are the cherry-picked changes:
* fix: login into the docker registry (#17620)
* feat: filter changes using go list output (#17397)
* fix: disable workaround on macos (#17750)
* ci: set git user configuration if it is not set (#17782)
* fix: mount Docker credentials (#17798)
* Review dependency patterns collection in Jenkins (#18004)

Co-authored-by: Ivan Fernandez Calvo <kuisathaverat@users.noreply.github.com>
Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>
Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>
  • Loading branch information
@jsoriano @kuisathaverat
@v1v @andrewkroh
4 people authored Apr 28, 2020
1 parent 784e91f commit d6ce568
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 53 deletions.
141 changes: 88 additions & 53 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ pipeline {
BASE_DIR = 'src/github.com/elastic/beats'
GOX_FLAGS = "-arch amd64"
DOCKER_COMPOSE_VERSION = "1.21.0"
PIPELINE_LOG_LEVEL = "INFO"
DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
DOCKER_REGISTRY = 'docker.elastic.co'
}
options {
timeout(time: 2, unit: 'HOURS')
Expand All @@ -17,7 +20,6 @@ pipeline {
disableResume()
durabilityHint('PERFORMANCE_OPTIMIZED')
disableConcurrentBuilds()
// checkoutToSubdirectory "${env.BASE_DIR}"
}
triggers {
issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
Expand All @@ -27,6 +29,7 @@ pipeline {
booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.')
booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
}
stages {
/**
Expand All @@ -37,13 +40,13 @@ pipeline {
steps {
deleteDir()
gitCheckout(basedir: "${BASE_DIR}")
stash allowEmpty: true, name: 'source', useDefaultExcludes: false
dir("${BASE_DIR}"){
loadConfigEnvVars()
}
whenTrue(params.debug){
dumpFilteredEnvironment()
}
stash allowEmpty: true, name: 'source', useDefaultExcludes: false
}
}
stage('Lint'){
Expand Down Expand Up @@ -644,12 +647,20 @@ def withBeatsEnv(boolean archive, Closure body) {
]) {
deleteDir()
unstash 'source'
if(isDockerInstalled()){
dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
}
dir("${env.BASE_DIR}") {
sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh")
sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh")
sh(label: "Install Mage", script: "make mage")
// TODO (2020-04-07): This is a work-around to fix the Beat generator tests.
// See https://github.com/elastic/beats/issues/17787.
setGitConfig()
try {
body()
if(!params.dry_run){
body()
}
} finally {
if (archive) {
catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
Expand Down Expand Up @@ -682,7 +693,9 @@ def withBeatsEnvWin(Closure body) {
dir("${env.BASE_DIR}"){
bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat")
try {
body()
if(!params.dry_run){
body()
}
} finally {
catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml")
Expand Down Expand Up @@ -818,19 +831,20 @@ def isChanged(patterns){
}

def isChangedOSSCode(patterns) {
def always = [
def allPatterns = [
"^Jenkinsfile",
"^vendor/.*",
"^libbeat/.*",
"^testing/.*",
"^dev-tools/.*",
"^\\.ci/.*",
]
return isChanged(always + patterns)
allPatterns.addAll(patterns)
return isChanged(allPatterns)
}

def isChangedXPackCode(patterns) {
def always = [
def allPatterns = [
"^Jenkinsfile",
"^vendor/.*",
"^libbeat/.*",
Expand All @@ -839,91 +853,112 @@ def isChangedXPackCode(patterns) {
"^x-pack/libbeat/.*",
"^\\.ci/.*",
]
return isChanged(always + patterns)
allPatterns.addAll(patterns)
return isChanged(allPatterns)
}

def loadConfigEnvVars(){
def empty = []
env.GO_VERSION = readFile(".go-version").trim()

withEnv(["HOME=${env.WORKSPACE}"]) {
sh(label: "Install Go ${env.GO_VERSION}", script: ".ci/scripts/install-go.sh")
}

// Libbeat is the core framework of Beats. It has no additional dependencies
// on other projects in the Beats repository.
env.BUILD_LIBBEAT = isChangedOSSCode([])
env.BUILD_LIBBEAT_XPACK = env.BUILD_LIBBEAT || isChangedXPackCode([])
env.BUILD_LIBBEAT = isChangedOSSCode(empty)
env.BUILD_LIBBEAT_XPACK = isChangedXPackCode(empty)

// Auditbeat depends on metricbeat as framework, but does not include any of
// the modules from Metricbeat.
// The Auditbeat x-pack build contains all functionality from OSS Auditbeat.
env.BUILD_AUDITBEAT = isChangedOSSCode([
"^metricbeat/.*",
"^auditbeat/.*",
])
env.BUILD_AUDITBEAT_XPACK = env.BUILD_AUDITBEAT || isChangedXPackCode([
"^x-pack/auditbeat/.*",
])
env.BUILD_AUDITBEAT = isChangedOSSCode(getVendorPatterns('auditbeat'))
env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/auditbeat'))

// Dockerlogbeat is a standalone Beat that only relies on libbeat.
env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode([
"^x-pack/dockerlogbeat/.*",
])
env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/dockerlogbeat'))

// Filebeat depends on libbeat only.
// The Filebeat x-pack build contains all functionality from OSS Filebeat.
env.BUILD_FILEBEAT = isChangedOSSCode(["^filebeat/.*"])
env.BUILD_FILEBEAT_XPACK = env.BUILD_FILEBEAT || isChangedXPackCode([
"^x-pack/filebeat/.*",
])
env.BUILD_FILEBEAT = isChangedOSSCode(getVendorPatterns('filebeat'))
env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/filebeat'))

// Metricbeat depends on libbeat only.
// The Metricbeat x-pack build contains all functionality from OSS Metricbeat.
env.BUILD_METRICBEAT = isChangedOSSCode(["^metricbeat/.*"])
env.BUILD_METRICBEAT_XPACK = env.BUILD_METRICBEAT || isChangedXPackCode([
"^x-pack/metricbeat/.*",
])
env.BUILD_METRICBEAT = isChangedOSSCode(getVendorPatterns('metricbeat'))
env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/metricbeat'))

// Functionbeat is a standalone beat that depends on libbeat only.
// Functionbeat is available as x-pack build only.
env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode([
"^x-pack/functionbeat/.*",
])
env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/functionbeat'))

// Heartbeat depends on libbeat only.
// The Heartbeat x-pack build contains all functionality from OSS Heartbeat.
env.BUILD_HEARTBEAT = isChangedOSSCode(["^heartbeat/.*"])
env.BUILD_HEARTBEAT_XPACK = env.BUILD_HEARTBEAT || isChangedXPackCode([
"^x-pack/heartbeat/.*",
])

env.BUILD_HEARTBEAT = isChangedOSSCode(getVendorPatterns('heartbeat'))
env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/heartbeat'))

// Journalbeat depends on libbeat only.
// The Journalbeat x-pack build contains all functionality from OSS Journalbeat.
env.BUILD_JOURNALBEAT = isChangedOSSCode(["^journalbeat/.*"])
env.BUILD_JOURNALBEAT_XPACK = env.BUILD_JOURNALBEAT || isChangedXPackCode([
"^x-pack/journalbeat/.*",
])
env.BUILD_JOURNALBEAT = isChangedOSSCode(getVendorPatterns('journalbeat'))
env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/journalbeat'))

// Packetbeat depends on libbeat only.
// The Packetbeat x-pack build contains all functionality from OSS Packetbeat.
env.BUILD_PACKETBEAT = isChangedOSSCode(["^packetbeat/.*"])
env.BUILD_PACKETBEAT_XPACK = env.BUILD_PACKETBEAT || isChangedXPackCode([
"^x-pack/packetbeat/.*",
])
env.BUILD_PACKETBEAT = isChangedOSSCode(getVendorPatterns('packetbeat'))
env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/packetbeat'))

// Winlogbeat depends on libbeat only.
// The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat.
env.BUILD_WINLOGBEAT = isChangedOSSCode(["^winlogbeat/.*"])
env.BUILD_WINLOGBEAT_XPACK = env.BUILD_WINLOGBEAT || isChangedXPackCode([
"^x-pack/winlogbeat/.*",
])
env.BUILD_WINLOGBEAT = isChangedOSSCode(getVendorPatterns('winlogbeat'))
env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/winlogbeat'))

// Elastic-agent is a self-contained product, that depends on libbeat only.
// The agent acts as a supervisor for other Beats like Filebeat or Metricbeat.
// The agent is available as x-pack build only.
env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/elastic-agent'))

// The Kubernetes test use Filebeat and Metricbeat, but only need to be run
// if the deployment scripts have been updated. No Beats specific testing is
// involved.
env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"])

env.BUILD_GENERATOR = isChangedOSSCode(["^generator/.*"])
def generatorPatterns = ['^generator/.*']
generatorPatterns.addAll(getVendorPatterns('generator/common/beatgen'))
generatorPatterns.addAll(getVendorPatterns('metricbeat/beater'))
env.BUILD_GENERATOR = isChangedOSSCode(generatorPatterns)
}

/**
This method grab the dependencies of a Go module and transform them on regexp
*/
def getVendorPatterns(beatName){
def os = goos()
def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64"
def output = ""

env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode([
"^x-pack/elastic-agent/.*",
])
withEnv([
"HOME=${env.WORKSPACE}/${env.BASE_DIR}",
"PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}",
]) {
output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """
go list -mod=vendor -f '{{ .ImportPath }}{{ "\\n" }}{{ join .Deps "\\n" }}' ./${beatName}\
|awk '{print \$1"/.*"}'\
|sed -e "s#github.com/elastic/beats/v7/##g"
""")
}
return output?.split('\n').collect{ item -> item as String }
}

env.GO_VERSION = readFile(".go-version").trim()
def setGitConfig(){
sh(label: 'check git config', script: '''
if [ -z "$(git config --get user.email)" ]; then
git config user.email "beatsmachine@users.noreply.github.com"
git config user.name "beatsmachine"
fi
''')
}

def isDockerInstalled(){
return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true)
}
1 change: 1 addition & 0 deletions x-pack/metricbeat/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ services:
volumes:
- ${PWD}/../..:/go/src/github.com/elastic/beats/
- /var/run/docker.sock:/var/run/docker.sock
- ${HOME}/.docker:/root/.docker:ro
network_mode: host
command: make

Expand Down

0 comments on commit d6ce568

Please sign in to comment.