Skip to content

internat(QTDI-1487) Reduce log content #1039

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

internat(QTDI-1487) Reduce log content #1039

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d1afd4e
chore(QTDI-1475): Update Jenkinsfile to adjust build discarder settings
acatoire Apr 16, 2025
69dc66d
chore(QTDI-1475): Update Jenkinsfile to log Maven build and deploy ou…
acatoire Apr 16, 2025
e6700fd
chore(Jenkinsfile): Refactor log file handling and archive the maven …
acatoire Apr 22, 2025
e728bb9
Refactor Trivy build
acatoire Apr 22, 2025
6017e93
chore(Jenkinsfile): Enhance log filtering and structure for build and…
acatoire Apr 22, 2025
dee2794
Merge branch 'master' into acatoire/QTDI-1487-reduce-log-content
acatoire Apr 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 57 additions & 39 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ final String buildTimestamp = String.format('-%tY%<tm%<td%<tH%<tM%<tS', LocalDat
final String artifactoryAddr = "https://artifactory.datapwn.com"
final String artifactoryPath = "tlnd-docker-dev/talend/common/tacokit"

final String BUILD_LOG_FILE = "build.log"
final String DEPLOY_LOG_FILE = "deploy.log"
final String MAVEN_GREP_FILTER = "WARN|ERROR|-----------[| :: |-------<|\\[[0-9]+/[0-9]+\\]|\\[INFO\\] Reactor|\\[INFO\\] BUILD"

// Job variables declaration
String branch_user
String branch_ticket
Expand All @@ -80,8 +84,9 @@ Boolean devBranch_mavenDeploy = false
Boolean devBranch_dockerPush = false


String skipOptions = "-Dspotless.apply.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipTests -Dinvoker.skip=true"
String deployOptions = "$skipOptions -Possrh -Prelease -Pgpg2 -Denforcer.skip=true"
final String BUILD_ARGS = "-Dgpg.skip=true -Denforcer.skip=true"
final String skipOptions = "-Dspotless.apply.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipTests -Dinvoker.skip=true"
final String deployOptions = "$skipOptions -Possrh -Prelease -Pgpg2 -Denforcer.skip=true"


pipeline {
Expand All @@ -97,15 +102,16 @@ pipeline {

environment {
MAVEN_OPTS = "-Dformatter.skip=true -Dmaven.artifact.threads=256"
BUILD_ARGS = "-Dgpg.skip=true -Denforcer.skip=true"
ARTIFACTORY_REGISTRY = "artifactory.datapwn.com"
VERACODE_APP_NAME = 'Talend Component Kit'
VERACODE_SANDBOX = 'component-runtime'
APP_ID = '579232'
}

options {
buildDiscarder(logRotator(artifactNumToKeepStr: '10', numToKeepStr: branch_name == 'master' ? '15' : '10'))
buildDiscarder(logRotator(artifactNumToKeepStr: '5',
numToKeepStr: '5',
daysToKeepStr: '10'))
timeout(time: 180, unit: 'MINUTES')
skipStagesAfterUnstable()
}
Expand Down Expand Up @@ -303,16 +309,14 @@ pipeline {
- We do not want to deploy on dev branch
""".stripIndent()
finalVersion = pomVersion
}
else {
} else {
branch_user = ""
branch_ticket = ""
branch_description = ""
if (params.VERSION_QUALIFIER != ("DEFAULT")) {
// If the qualifier is given, use it
println """No need to add qualifier, use the given one: "$params.VERSION_QUALIFIER" """
}
else {
} else {
println "Validate the branch name"

(branch_user,
Expand Down Expand Up @@ -387,6 +391,7 @@ pipeline {
}
}
}

stage('Post login') {
steps {
withCredentials([gitCredentials,
Expand All @@ -411,22 +416,29 @@ pipeline {
}
}
}

stage('Maven validate to install') {
when { expression { params.ACTION != 'RELEASE' } }
steps {
withCredentials([ossrhCredentials,
nexusCredentials]) {
sh """\
#!/usr/bin/env bash
set -xe
mvn clean install $BUILD_ARGS \
$extraBuildParams \
--settings .jenkins/settings.xml
""".stripIndent()
#!/usr/bin/env bash
set -xe
# Run the Maven command, save full output to the log file, and filter live output
mvn clean install $BUILD_ARGS \
$extraBuildParams \
--settings .jenkins/settings.xml \
2>&1 | tee $BUILD_LOG_FILE | grep -E '$MAVEN_GREP_FILTER'
""".stripIndent()
}
}
post {
always {
script {
jenkinsJobTools.job_description_append("You will find full build logs in artefacts as $BUILD_LOG_FILE")
archiveArtifacts artifacts: "${"**/$BUILD_LOG_FILE"}", allowEmptyArchive: false, onlyIfSuccessful: false
}
recordIssues(
enabledForFailure: false,
tools: [
Expand All @@ -440,6 +452,7 @@ pipeline {
}
}
}

stage('Maven deploy') {
when {
anyOf {
Expand All @@ -457,7 +470,8 @@ pipeline {
set -xe
bash mvn deploy $deployOptions \
$extraBuildParams \
--settings .jenkins/settings.xml
--settings .jenkins/settings.xml \
2>&1 | tee $DEPLOY_LOG_FILE | grep -E '$MAVEN_GREP_FILTER'
""".stripIndent()
}
}
Expand All @@ -467,16 +481,24 @@ pipeline {
if (devBranch_mavenDeploy) {
repo = ['artifacts-zl.talend.com',
'https://artifacts-zl.talend.com/nexus/content/repositories/snapshots/org/talend/sdk/component']
}
else {
} else {
repo = ['oss.sonatype.org',
'https://oss.sonatype.org/content/repositories/snapshots/org/talend/sdk/component/']
}

jenkinsJobTools.job_description_append("Maven artefact deployed as ${finalVersion} on [${repo[0]}](${repo[1]}) ")
}
}
post {
always {
script {
jenkinsJobTools.job_description_append("You will find full deploy logs in artefacts as $DEPLOY_LOG_FILE")
archiveArtifacts artifacts: "${"**/$DEPLOY_LOG_FILE"}", allowEmptyArchive: false, onlyIfSuccessful: false
}
}
}
}

stage('Docker build/push') {
when {
anyOf {
Expand All @@ -492,39 +514,34 @@ pipeline {
if (isStdBranch) {
// Build and push all images
jenkinsJobTools.job_description_append("Docker images deployed: component-server, component-starter-server and remote-engine-customizer ")
}
else {
} else {
String image_list
if (params.DOCKER_CHOICE == 'All') {
images_options = 'false'
}
else {
} else {
images_options = 'false ' + params.DOCKER_CHOICE
}

if (params.DOCKER_CHOICE == 'All') {
jenkinsJobTools.job_description_append("All docker images deployed ")

jenkinsJobTools.job_description_append("As ${finalVersion}${buildTimestamp} on " +
"[artifactory.datapwn.com]" +
"($artifactoryAddr/$artifactoryPath) ")
"[artifactory.datapwn.com]" +
"($artifactoryAddr/$artifactoryPath) ")
jenkinsJobTools.job_description_append("docker pull $artifactoryAddr/$artifactoryPath" +
"/component-server:${finalVersion}${buildTimestamp} ")
"/component-server:${finalVersion}${buildTimestamp} ")
jenkinsJobTools.job_description_append("docker pull $artifactoryAddr/$artifactoryPath" +
"/component-starter-server:${finalVersion}${buildTimestamp} ")
"/component-starter-server:${finalVersion}${buildTimestamp} ")
jenkinsJobTools.job_description_append("docker pull $artifactoryAddr/$artifactoryPath" +
"/remote-engine-customize:${finalVersion}${buildTimestamp} ")
"/remote-engine-customize:${finalVersion}${buildTimestamp} ")

}
else {
} else {
jenkinsJobTools.job_description_append("Docker images deployed: $params.DOCKER_CHOICE ")
jenkinsJobTools.job_description_append("As ${finalVersion}${buildTimestamp} on " +
"[artifactory.datapwn.com]($artifactoryAddr/$artifactoryPath) ")
"[artifactory.datapwn.com]($artifactoryAddr/$artifactoryPath) ")
jenkinsJobTools.job_description_append("docker pull $artifactoryAddr/$artifactoryPath/$params.DOCKER_CHOICE:" +
"${finalVersion}${buildTimestamp} ")

"${finalVersion}${buildTimestamp} ")
}

}

// Build and push specific image
Expand All @@ -534,10 +551,10 @@ pipeline {
${images_options}
"""
}

}
}
}

stage('Documentation') {
when {
expression {
Expand All @@ -559,6 +576,7 @@ pipeline {
}
}
}

stage('OSS security analysis') {
when {
anyOf {
Expand Down Expand Up @@ -589,6 +607,7 @@ pipeline {
}
}
}

stage('Deps report') {
when {
anyOf {
Expand Down Expand Up @@ -628,6 +647,7 @@ pipeline {
}
}
}

stage('Sonar') {
when {
expression { (params.ACTION != 'RELEASE') && !params.DISABLE_SONAR }
Expand All @@ -648,8 +668,7 @@ pipeline {
'${pull_request_id}' \
${extraBuildParams}
"""
}
else {
} else {
echo 'Run analysis for branch'
sh """
bash .jenkins/scripts/mvn_sonar_branch.sh \
Expand All @@ -661,6 +680,7 @@ pipeline {
}
}
}

stage('Release') {
when {
allOf {
Expand Down Expand Up @@ -838,12 +858,10 @@ private static String add_qualifier_to_version(String version, String ticket, St
if (user_qualifier.contains("DEFAULT")) {
if (version.contains("-SNAPSHOT")) {
new_version = version.replace("-SNAPSHOT", "-$ticket-SNAPSHOT" as String)
}
else {
} else {
new_version = "$version-$ticket".toString()
}
}
else {
} else {
new_version = version.replace("-SNAPSHOT", "-$user_qualifier-SNAPSHOT" as String)
}
return new_version
Expand Down
Loading