Issue 23631 create global log4j

.github/actions/run-postman-tests/dist/index.js

@@ -186,7 +186,7 @@ exports.runTests = runTests;
  */
 const copyOutputs = () => __awaiter(void 0, void 0, void 0, function* () {
     const logFile = path.join(dotCmsRoot, `${normalize(parallelCollection)}.log`);
-    yield execCmd(toCommand('docker', ['cp', 'docker_dotcms-app_1:/srv/dotserver/tomcat-9.0.60/logs/dotcms.log', logFile]));
+    yield execCmd(toCommand('docker', ['cp', 'dotcms-app_1:/srv/dotserver/tomcat-9.0.60/logs/dotcms.log', logFile]));
 });
 const copyHeaderAndFooter = () => __awaiter(void 0, void 0, void 0, function* () {
     yield execCmd(toCommand('cp', [path.join(resourcesFolder, 'postman-results-header.html'), reportFolder]));

.github/actions/run-postman-tests/src/postman.ts

@@ -106,9 +106,7 @@ export const runTests = async (): Promise<PostmanTestsResult> => {
  */
 const copyOutputs = async () => {
   const logFile = path.join(dotCmsRoot, `${normalize(parallelCollection)}.log`)
-  await execCmd(
-    toCommand('docker', ['cp', 'docker_dotcms-app_1:/srv/dotserver/tomcat-9.0.60/logs/dotcms.log', logFile])
-  )
+  await execCmd(toCommand('docker', ['cp', 'dotcms-app_1:/srv/dotserver/tomcat-9.0.60/logs/dotcms.log', logFile]))
 }
 
 const copyHeaderAndFooter = async () => {

bin/setclasspath.bat

@@ -0,0 +1,104 @@
+@echo off
+rem Licensed to the Apache Software Foundation (ASF) under one or more
+rem contributor license agreements.  See the NOTICE file distributed with
+rem this work for additional information regarding copyright ownership.
+rem The ASF licenses this file to You under the Apache License, Version 2.0
+rem (the "License"); you may not use this file except in compliance with
+rem the License.  You may obtain a copy of the License at
+rem
+rem     http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem Unless required by applicable law or agreed to in writing, software
+rem distributed under the License is distributed on an "AS IS" BASIS,
+rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem See the License for the specific language governing permissions and
+rem limitations under the License.
+
+rem ---------------------------------------------------------------------------
+rem Set JAVA_HOME or JRE_HOME if not already set, ensure any provided settings
+rem are valid and consistent with the selected start-up options and set up the
+rem endorsed directory.
+rem ---------------------------------------------------------------------------
+
+rem Make sure prerequisite environment variables are set
+
+rem In debug mode we need a real JDK (JAVA_HOME)
+if ""%1"" == ""debug"" goto needJavaHome
+
+rem Otherwise either JRE or JDK are fine
+if not "%JRE_HOME%" == "" goto gotJreHome
+if not "%JAVA_HOME%" == "" goto gotJavaHome
+echo Neither the JAVA_HOME nor the JRE_HOME environment variable is defined
+echo At least one of these environment variable is needed to run this program
+goto exit
+
+:needJavaHome
+rem Check if we have a usable JDK
+if "%JAVA_HOME%" == "" goto noJavaHome
+if not exist "%JAVA_HOME%\bin\java.exe" goto noJavaHome
+if not exist "%JAVA_HOME%\bin\jdb.exe" goto noJavaHome
+if not exist "%JAVA_HOME%\bin\javac.exe" goto noJavaHome
+set "JRE_HOME=%JAVA_HOME%"
+goto okJava
+
+:noJavaHome
+echo The JAVA_HOME environment variable is not defined correctly.
+echo It is needed to run this program in debug mode.
+echo NB: JAVA_HOME should point to a JDK not a JRE.
+goto exit
+
+:gotJavaHome
+rem No JRE given, use JAVA_HOME as JRE_HOME
+set "JRE_HOME=%JAVA_HOME%"
+
+:gotJreHome
+rem Check if we have a usable JRE
+if not exist "%JRE_HOME%\bin\java.exe" goto noJreHome
+goto okJava
+
+:noJreHome
+rem Needed at least a JRE
+echo The JRE_HOME environment variable is not defined correctly
+echo This environment variable is needed to run this program
+goto exit
+
+:okJava
+rem Don't override the endorsed dir if the user has set it previously
+if not "%JAVA_ENDORSED_DIRS%" == "" goto gotEndorseddir
+rem Java 9 no longer supports the java.endorsed.dirs
+rem system property. Only try to use it if
+rem CATALINA_HOME/endorsed exists.
+if not exist "%CATALINA_HOME%\endorsed" goto gotEndorseddir
+set "JAVA_ENDORSED_DIRS=%CATALINA_HOME%\endorsed"
+:gotEndorseddir
+
+rem Don't override _RUNJAVA if the user has set it previously
+if not "%_RUNJAVA%" == "" goto gotRunJava
+rem Set standard command for invoking Java.
+rem Also note the quoting as JRE_HOME may contain spaces.
+set _RUNJAVA="%JRE_HOME%\bin\java.exe"
+:gotRunJava
+
+rem Don't override _RUNJDB if the user has set it previously
+rem Also note the quoting as JAVA_HOME may contain spaces.
+if not "%_RUNJDB%" == "" goto gotRunJdb
+set _RUNJDB="%JAVA_HOME%\bin\jdb.exe"
+:gotRunJdb
+
+if not "%USE_GLOBAL_LOG4J" == "" goto gotGlobalLog4j
+if [ "$USE_GLOBAL_LOG4J" = true ]; then
+  echo Using Global Log4j
+  set CLASSPATH="%CATALINA_HOME%\log4j2\lib\*;%CATALINA_HOME%\log4j2\conf$CLASSPATH"
+  set JAVA_OPTS=" -DLog4jContextSelector=org.apache.logging.log4j.core.async.BasicAsyncLoggerContextSelector %JAVA_OPTS%"
+  set JAVA_OPTS=" -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager %JAVA_OPTS%"
+fi
+:gotGlobalLog4j
+
+
+goto end
+
+:exit
+exit /b 1
+
+:end
+exit /b 0

bin/setclasspath.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -----------------------------------------------------------------------------
+#  Set JAVA_HOME or JRE_HOME if not already set, ensure any provided settings
+#  are valid and consistent with the selected start-up options and set up the
+#  endorsed directory.
+# -----------------------------------------------------------------------------
+
+# Make sure prerequisite environment variables are set
+if [ -z "$JAVA_HOME" ] && [ -z "$JRE_HOME" ]; then
+  if $darwin; then
+    # Bugzilla 54390
+    if [ -x '/usr/libexec/java_home' ] ; then
+      export JAVA_HOME=`/usr/libexec/java_home`
+    # Bugzilla 37284 (reviewed).
+    elif [ -d "/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home" ]; then
+      export JAVA_HOME="/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home"
+    fi
+  else
+    JAVA_PATH=`which java 2>/dev/null`
+    if [ "x$JAVA_PATH" != "x" ]; then
+      JAVA_PATH=`dirname "$JAVA_PATH" 2>/dev/null`
+      JRE_HOME=`dirname "$JAVA_PATH" 2>/dev/null`
+    fi
+    if [ "x$JRE_HOME" = "x" ]; then
+      # XXX: Should we try other locations?
+      if [ -x /usr/bin/java ]; then
+        JRE_HOME=/usr
+      fi
+    fi
+  fi
+  if [ -z "$JAVA_HOME" ] && [ -z "$JRE_HOME" ]; then
+    echo "Neither the JAVA_HOME nor the JRE_HOME environment variable is defined"
+    echo "At least one of these environment variable is needed to run this program"
+    exit 1
+  fi
+fi
+if [ -z "$JAVA_HOME" ] && [ "$1" = "debug" ]; then
+  echo "JAVA_HOME should point to a JDK in order to run in debug mode."
+  exit 1
+fi
+if [ -z "$JRE_HOME" ]; then
+  JRE_HOME="$JAVA_HOME"
+fi
+
+# If we're running under jdb, we need a full jdk.
+if [ "$1" = "debug" ] ; then
+  if [ "$os400" = "true" ]; then
+    if [ ! -x "$JAVA_HOME"/bin/java ] || [ ! -x "$JAVA_HOME"/bin/javac ]; then
+      echo "The JAVA_HOME environment variable is not defined correctly"
+      echo "This environment variable is needed to run this program"
+      echo "NB: JAVA_HOME should point to a JDK not a JRE"
+      exit 1
+    fi
+  else
+    if [ ! -x "$JAVA_HOME"/bin/java ] || [ ! -x "$JAVA_HOME"/bin/jdb ] || [ ! -x "$JAVA_HOME"/bin/javac ]; then
+      echo "The JAVA_HOME environment variable is not defined correctly"
+      echo "This environment variable is needed to run this program"
+      echo "NB: JAVA_HOME should point to a JDK not a JRE"
+      exit 1
+    fi
+  fi
+fi
+
+# Don't override the endorsed dir if the user has set it previously
+if [ -z "$JAVA_ENDORSED_DIRS" ]; then
+  # Java 9 no longer supports the java.endorsed.dirs
+  # system property. Only try to use it if
+  # CATALINA_HOME/endorsed exists.
+  if [ -d "$CATALINA_HOME"/endorsed ]; then
+    JAVA_ENDORSED_DIRS="$CATALINA_HOME"/endorsed
+  fi
+fi
+
+# Set standard commands for invoking Java, if not already set.
+if [ -z "$_RUNJAVA" ]; then
+  _RUNJAVA="$JRE_HOME"/bin/java
+fi
+if [ "$os400" != "true" ]; then
+  if [ -z "$_RUNJDB" ]; then
+    _RUNJDB="$JAVA_HOME"/bin/jdb
+  fi
+fi
+ echo In set classpath
+if [ "$USE_GLOBAL_LOG4J" = true ]; then
+  echo Using Global Log4j
+  CLASSPATH="$CATALINA_HOME/log4j2/lib/*:$CATALINA_HOME/log4j2/conf$CLASSPATH"
+  JAVA_OPTS=" -DLog4jContextSelector=org.apache.logging.log4j.core.async.BasicAsyncLoggerContextSelector $JAVA_OPTS"
+  JAVA_OPTS=" -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager $JAVA_OPTS"
+fi

dotCMS/build.gradle

@@ -493,12 +493,37 @@ class DeployWarTomcatTask extends DefaultTask{
         }
 
         project.copy{
-            from '../libs/buildlibs'
+            from project.file("../libs/buildlibs")
             into buildLibLocation + '/lib'
             include 'mail.jar'
             include 'javax.activation-1.2.0.jar'
         }
 
+
+        project.copy {
+            from "$project.buildDir/resources/main"
+            into buildLibLocation + '/log4j2/conf'
+            include "log4j2-tomcat*.xml"
+        }
+
+        project.copy{
+            from project.configurations.compileClasspath
+            into buildLibLocation + '/log4j2/lib'
+            include 'commons-logging-*.jar'
+            include 'log4j-*.jar'
+            include 'disruptor-*.jar'
+            include 'slf4j-api-*.jar'
+            exclude 'log4j-web-*.jar'
+        }
+
+        project.copy{
+            from project.file("../bin")
+            into buildLibLocation + '/bin'
+            include 'setclasspath.sh'
+            include 'setclasspath.bat'
+        }
+
+
     }
 }
 
@@ -904,6 +929,12 @@ task createDistPrep(dependsOn: ['clonePullTomcatDist', 'deployWarTomcatDist']) {
             exclude "com", "org", "release.properties"
         }
 
+        copy {
+            from "$rootDir/../conf/log4j"
+            into "$distLocation$log4j2ConfDistLocation"
+            include "log4j*.xml"
+        }
+
         copy {
             into "$distLocation$pluginsDistLocation"
             from "src/main/plugins"

dotCMS/dependencies.gradle

@@ -5,6 +5,8 @@ dependencies {
 
     def immutablesVersion = "2.9.1"
     def byteBuddyVersion = "1.12.6"
+    def log4jVersion = "2.19.0"
+
     annotationProcessor "org.immutables:value:$immutablesVersion" // <--- this is important
 
     implementation('junit:junit:4.13.2') {
@@ -49,15 +51,19 @@ dependencies {
     implementation group: 'com.dotcms.lib', name: 'dot.counter-ejb', version:'ukv_2'
     implementation group: 'com.dotcms.lib', name: 'dot.dwr', version:'3rc2modified_3'
 
-    implementation group: 'com.lmax', name: 'disruptor', version: '3.3.4'
+
     implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
     implementation group: 'org.apache.commons', name: 'commons-numbers-gamma', version: '1.0'
-    implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.1'
-    implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1'
-    implementation group: 'org.apache.logging.log4j', name: 'log4j-1.2-api', version: '2.17.1'
-    implementation group: 'org.apache.logging.log4j', name: 'log4j-jcl', version: '2.17.1'
-    implementation group: 'org.apache.logging.log4j', name: 'log4j-web', version: '2.17.1'
-    implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.17.1'
+
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: log4jVersion
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: log4jVersion
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-1.2-api', version: log4jVersion
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-jcl', version: log4jVersion
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-web', version: log4jVersion
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: log4jVersion
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-jul', version: log4jVersion
+    implementation group: 'org.apache.logging.log4j', name: 'log4j-appserver', version: log4jVersion
+    implementation group: 'com.lmax', name: 'disruptor', version: '3.3.4'
 
     implementation group: 'com.dotcms.lib', name: 'dot.fileupload-ext', version:'ukv_3'
     implementation group: 'fop', name: 'fop', version:'0.20.3'

dotCMS/gradle.properties

@@ -29,6 +29,7 @@ distBinLocation=/bin
 pluginsDistLocation=/plugins
 docsDistLocation=/docs
 confDistLocation=/bin/system/src-conf
+log4j2ConfDistLocation=/log4j2/conf
 tomcatDistInstallLocation=dotserver/tomcat
 outputDistLocation=../dist-output/
 distUpdate=false

dotCMS/src/integration-test/java/com/dotcms/IntegrationTestBase.java

@@ -132,7 +132,7 @@ public void after ()  {
         //Closing the session
         try {
             HibernateUtil.closeAndCommitTransaction();
-            if (null != HibernateUtil.getSession()) {
+            if (null != HibernateUtil.getSession(false)) {
                 HibernateUtil.getSession().connection().close();
                 HibernateUtil.getSession().close();
             }

dotCMS/src/integration-test/java/com/dotcms/MainSuite.java

@@ -159,7 +159,6 @@
 import com.dotmarketing.quartz.DotStatefulJobTest;
 import com.dotmarketing.quartz.job.CleanUpFieldReferencesJobTest;
 import com.dotmarketing.quartz.job.IntegrityDataGenerationJobTest;
-import com.dotmarketing.quartz.job.StartEndScheduledExperimentsJobTest;
 import com.dotmarketing.startup.StartupTasksExecutorTest;
 import com.dotmarketing.startup.runalways.Task00050LoadAppsSecretsTest;
 import com.dotmarketing.startup.runonce.Task05195CreatesDestroyActionAndAssignDestroyDefaultActionsToTheSystemWorkflowTest;
@@ -227,7 +226,6 @@
 
 @RunWith(MainBaseSuite.class)
 @SuiteClasses({
-        StartEndScheduledExperimentsJobTest.class,
         RulesAPIImplIntegrationTest.class,
         Task220825CreateVariantFieldTest.class,
 //        AnalyticsAPIImplTest.class,

dotCMS/src/integration-test/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImplTest.java

@@ -56,6 +56,7 @@
 import com.dotmarketing.portlets.structure.model.Structure;
 import com.dotmarketing.portlets.templates.model.Template;
 import com.dotmarketing.sitesearch.business.SiteSearchAPI;
+import com.dotmarketing.util.Config;
 import com.dotmarketing.util.Logger;
 import com.dotmarketing.util.ThreadUtils;
 import com.dotmarketing.util.UUIDGenerator;
@@ -849,7 +850,9 @@ public void test_that_live_and_working_content_makes_it_into_the_index() throws
     indexAPI.removeContentFromIndex(content);
     ReindexThread.startThread();
     APILocator.getReindexQueueAPI().addContentletReindex(content);
-    ThreadUtils.sleep(10000);
+    int waitTime=60;
+    boolean  queueEmpty = APILocator.getReindexQueueAPI().waitForEmptyQueue(waitTime);
+    assertTrue("Index queue not empty after "+waitTime,queueEmpty);
     liveSearch = contentletAPI.searchIndex("+identifier:" + content.getIdentifier() + " +live:true", 1, 0, "modDate", user, false);
     workingSearch = contentletAPI.searchIndex("+identifier:" + content.getIdentifier() + " +live:false", 1, 0, "modDate", user, false);
     assert(liveSearch.size()>0);