[#1552] improvement: Migrate from log4j1 to log4j2 (#1553)

### What changes were proposed in this pull request? Migrate from log4j1 to log4j2 ### Why are the changes needed? For #1552 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing UTs.
apache · Mar 4, 2024 · d8aedf3 · d8aedf3
1 parent 9f4ccde
commit d8aedf3
Show file tree

Hide file tree

Showing 30 changed files with 420 additions and 132 deletions.
diff --git a/bin/start-coordinator.sh b/bin/start-coordinator.sh
@@ -28,9 +28,8 @@ cd "$RSS_HOME"
 
 COORDINATOR_CONF_FILE="${RSS_CONF_DIR}/coordinator.conf"
 JAR_DIR="${RSS_HOME}/jars"
-LOG_CONF_FILE="${RSS_CONF_DIR}/log4j.properties"
+LOG_CONF_FILE="${RSS_CONF_DIR}/log4j2.xml"
 LOG_PATH="${RSS_LOG_DIR}/coordinator.log"
-OUT_PATH="${RSS_LOG_DIR}/coordinator.out"
 
 MAIN_CLASS="org.apache.uniffle.coordinator.CoordinatorServer"
 
@@ -87,13 +86,13 @@ JAVA11_EXTRA_ARGS=" -XX:+IgnoreUnrecognizedVMOptions \
 ARGS=""
 
 if [ -f ${LOG_CONF_FILE} ]; then
-  ARGS="$ARGS -Dlog4j.configuration=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH}"
+  ARGS="$ARGS -Dlog4j2.configurationFile=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH}"
 else
   echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
   exit 1
 fi
 
-$RUNNER $ARGS $JVM_ARGS $JAVA11_EXTRA_ARGS -cp $CLASSPATH $MAIN_CLASS --conf "$COORDINATOR_CONF_FILE" $@ &> $OUT_PATH &
+$RUNNER $ARGS $JVM_ARGS $JAVA11_EXTRA_ARGS -cp $CLASSPATH $MAIN_CLASS --conf "$COORDINATOR_CONF_FILE" $@ &
 
 get_pid_file_name coordinator
 echo $! >${RSS_PID_DIR}/${pid_file}
diff --git a/bin/start-dashboard.sh b/bin/start-dashboard.sh
@@ -28,9 +28,8 @@ cd "$RSS_HOME"
 
 DASHBOARD_CONF_FILE="${RSS_CONF_DIR}/dashboard.conf"
 JAR_DIR="${RSS_HOME}/jars"
-LOG_CONF_FILE="${RSS_CONF_DIR}/log4j.properties"
+LOG_CONF_FILE="${RSS_CONF_DIR}/log4j2.xml"
 LOG_PATH="${RSS_LOG_DIR}/dashboard.log"
-OUT_PATH="${RSS_LOG_DIR}/dashboard.out"
 
 MAIN_CLASS="org.apache.uniffle.dashboard.web.JettyServerFront"
 
@@ -74,13 +73,13 @@ JAVA11_EXTRA_ARGS=" -XX:+IgnoreUnrecognizedVMOptions \
 ARGS=""
 
 if [ -f ${LOG_CONF_FILE} ]; then
-  ARGS="$ARGS -Dlog4j.configuration=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH}"
+  ARGS="$ARGS -Dlog4j2.configurationFile=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH}"
 else
   echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
   exit 1
 fi
 
-$RUNNER $ARGS $JVM_ARGS $JAVA11_EXTRA_ARGS -cp $CLASSPATH $MAIN_CLASS --conf "$DASHBOARD_CONF_FILE" $@ &> $OUT_PATH &
+$RUNNER $ARGS $JVM_ARGS $JAVA11_EXTRA_ARGS -cp $CLASSPATH $MAIN_CLASS --conf "$DASHBOARD_CONF_FILE" $@ &
 
 get_pid_file_name dashboard
 echo $! >${RSS_PID_DIR}/${pid_file}
diff --git a/bin/start-shuffle-server.sh b/bin/start-shuffle-server.sh
@@ -28,9 +28,8 @@ cd "$RSS_HOME"
 
 SHUFFLE_SERVER_CONF_FILE="${RSS_CONF_DIR}/server.conf"
 JAR_DIR="${RSS_HOME}/jars"
-LOG_CONF_FILE="${RSS_CONF_DIR}/log4j.properties"
+LOG_CONF_FILE="${RSS_CONF_DIR}/log4j2.xml"
 LOG_PATH="${RSS_LOG_DIR}/shuffle_server.log"
-OUT_PATH="${RSS_LOG_DIR}/shuffle_server.out"
 
 if [ -z "${XMX_SIZE:-}" ]; then
   echo "No env XMX_SIZE."
@@ -124,13 +123,13 @@ JAVA11_EXTRA_ARGS=" -XX:+IgnoreUnrecognizedVMOptions \
 ARGS=""
 
 if [ -f ${LOG_CONF_FILE} ]; then
-  ARGS="$ARGS -Dlog4j.configuration=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH}"
+  ARGS="$ARGS -Dlog4j2.configurationFile=file:${LOG_CONF_FILE} -Dlog.path=${LOG_PATH}"
 else
   echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
   exit 1
 fi
 
-$RUNNER $ARGS $JVM_ARGS $JAVA11_EXTRA_ARGS $JAVA_LIB_PATH -cp $CLASSPATH $MAIN_CLASS --conf "$SHUFFLE_SERVER_CONF_FILE" $@ &> $OUT_PATH &
+$RUNNER $ARGS $JVM_ARGS $JAVA11_EXTRA_ARGS $JAVA_LIB_PATH -cp $CLASSPATH $MAIN_CLASS --conf "$SHUFFLE_SERVER_CONF_FILE" $@ &
 
 get_pid_file_name shuffle-server
 echo $! >${RSS_PID_DIR}/${pid_file}
diff --git a/bin/uniffle-function.sh b/bin/uniffle-function.sh
@@ -384,7 +384,7 @@ function uniffle_basic_init
   UNIFFLE_LOGFILE=${UNIFFLE_LOGFILE:-uniffle.log}
   UNIFFLE_STOP_TIMEOUT=${UNIFFLE_STOP_TIMEOUT:-60}
   UNIFFLE_PID_DIR=${RSS_PID_DIR:-"${RSS_HOME}"}
-  UNIFFLE_LOG_CONF_FILE=${LOG_CONF_FILE:-"${RSS_CONF_DIR}/log4j.properties"}
+  UNIFFLE_LOG_CONF_FILE=${LOG_CONF_FILE:-"${RSS_CONF_DIR}/log4j2.xml"}
 
   uniffle_debug "USER: "$USER
   uniffle_debug "UNIFFLE_IDENT_STRING: "$UNIFFLE_IDENT_STRING
@@ -793,7 +793,7 @@ function uniffle_mkdir
 function uniffle_finalize_uniffle_opts
 {
   set +u
-  uniffle_add_param UNIFFLE_OPTS log4j.configuration "-Dlog4j.configuration=file:${UNIFFLE_LOG_CONF_FILE}"
+  uniffle_add_param UNIFFLE_OPTS log4j2.configurationFile "-Dlog4j2.configurationFile=file:${UNIFFLE_LOG_CONF_FILE}"
   uniffle_add_param UNIFFLE_OPTS uniffle.log.dir "-Duniffle.log.dir=${UNIFFLE_LOG_DIR}"
   uniffle_add_param UNIFFLE_OPTS uniffle.home.dir "-Duniffle.home.dir=${RSS_HOME}"
   uniffle_add_param UNIFFLE_OPTS uniffle.id.str "-Duniffle.id.str=${UNIFFLE_IDENT_STRING}"

diff --git a/client-mr/core/pom.xml b/client-mr/core/pom.xml
@@ -57,6 +57,10 @@
                     <groupId>com.google.protobuf</groupId>
                     <artifactId>protobuf-java</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -68,6 +72,10 @@
                     <groupId>com.sun.jersey</groupId>
                     <artifactId>jersey-json</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

diff --git a/client-mr/core/src/test/resources/log4j.properties b/client-mr/core/src/test/resources/log4j.properties
diff --git a/client-mr/core/src/test/resources/log4j2.xml b/client-mr/core/src/test/resources/log4j2.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+<Configuration status="WARN" monitorInterval="30">
+  <Appenders>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%t] [%p] %c{1}.%M - %m%n%ex"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="info">
+      <AppenderRef ref="Console"/>
+    </Root>
+  </Loggers>
+</Configuration>
diff --git a/client-mr/hadoop2.8/pom.xml b/client-mr/hadoop2.8/pom.xml
@@ -38,6 +38,10 @@
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -49,6 +53,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-mapreduce-client-app</artifactId>
       <version>${hadoop.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
   </dependencies>
 </project>
diff --git a/client-mr/hadoop3.2/pom.xml b/client-mr/hadoop3.2/pom.xml
@@ -42,6 +42,10 @@
           <groupId>com.google.protobuf</groupId>
           <artifactId>protobuf-java</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -53,6 +57,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-mapreduce-client-app</artifactId>
       <version>${hadoop.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
   </dependencies>
 </project>
diff --git a/client-spark/common/pom.xml b/client-spark/common/pom.xml
@@ -51,6 +51,12 @@
             <artifactId>spark-core_${scala.binary.version}</artifactId>
             <version>${spark.version}</version>
             <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.uniffle</groupId>
@@ -72,6 +78,10 @@
                     <groupId>javax.servlet</groupId>
                     <artifactId>servlet-api</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

diff --git a/client-spark/spark2/pom.xml b/client-spark/spark2/pom.xml
@@ -46,10 +46,12 @@
       <artifactId>spark-core_${scala.binary.version}</artifactId>
       <version>${spark.version}</version>
       <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
@@ -60,6 +62,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-minicluster</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>io.netty</groupId>

diff --git a/client-spark/spark3/pom.xml b/client-spark/spark3/pom.xml
@@ -42,6 +42,12 @@
             <artifactId>spark-core_${scala.binary.version}</artifactId>
             <version>${spark.version}</version>
             <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.uniffle</groupId>
@@ -79,6 +85,12 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-minicluster</artifactId>
             <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>

diff --git a/client-tez/pom.xml b/client-tez/pom.xml
@@ -99,6 +99,10 @@
             <groupId>com.sun.jersey</groupId>
             <artifactId>jersey-json</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -128,6 +132,12 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-minicluster</artifactId>
           <scope>test</scope>
+          <exclusions>
+            <exclusion>
+              <groupId>org.slf4j</groupId>
+              <artifactId>slf4j-log4j12</artifactId>
+            </exclusion>
+          </exclusions>
         </dependency>
     </dependencies>
 

diff --git a/client-tez/src/test/resources/log4j.properties b/client-tez/src/test/resources/log4j.properties
diff --git a/client-tez/src/test/resources/log4j2.xml b/client-tez/src/test/resources/log4j2.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+<Configuration status="WARN" monitorInterval="30">
+  <Appenders>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%t] [%p] %c{1}.%M - %m%n%ex"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="info">
+      <AppenderRef ref="Console"/>
+    </Root>
+  </Loggers>
+</Configuration>