[Enhancement] Enhance FE/BE to output logs to console directly (#20967)

1. LOG_CONSOLE env var is checked in fe/be/cn entrypoint script and pass `--logconsole` parameter to start_<fe|be|cn>.sh 2. start_fe.sh force set `syslog_to_console = true` in fe.conf so all logs are written to console stderr 3. start_be.sh/start_cn.sh force set `GLOG_logtostderr=1` env var to redirect glog output to console stderr 4. remove writing to pidfile, FE/BE application code will take care it. Signed-off-by: Kevin Xiaohua Cai <caixiaohua@starrocks.com>
StarRocks · Apr 6, 2023 · 35044f9 · 35044f9
1 parent 3ff71b9
commit 35044f9
Show file tree

Hide file tree

Showing 7 changed files with 84 additions and 18 deletions.
diff --git a/bin/start_backend.sh b/bin/start_backend.sh
@@ -30,6 +30,7 @@ OPTS=$(getopt \
     -l 'daemon' \
     -l 'cn' \
     -l 'be' \
+    -l 'logconsole' \
     -l numa: \
 -- "$@")
 
@@ -39,12 +40,14 @@ RUN_DAEMON=0
 RUN_CN=0
 RUN_BE=0
 RUN_NUMA="-1"
+RUN_LOG_CONSOLE=0
 
 while true; do
     case "$1" in
         --daemon) RUN_DAEMON=1 ; shift ;;
         --cn) RUN_CN=1; RUN_BE=0; shift ;;
         --be) RUN_BE=1; RUN_CN=0; shift ;;
+        --logconsole) RUN_LOG_CONSOLE=1 ; shift ;;
         --numa) RUN_NUMA=$2; shift 2 ;;
         --) shift ;  break ;;
         *) echo "Internal error" ; exit 1 ;;
@@ -177,9 +180,17 @@ if [ ${RUN_CN} -eq 1 ]; then
     LOG_FILE=${LOG_DIR}/cn.out
 fi
 
-echo "start time: "$(date) >> ${LOG_FILE}
+if [ ${RUN_LOG_CONSOLE} -eq 1 ] ; then
+    # force glog output to console (stderr)
+    export GLOG_logtostderr=1
+else
+    # redirect stdout/stderr to ${LOG_FILE}
+    exec &>> ${LOG_FILE}
+fi
+
+echo "start time: "$(date)
 if [ ${RUN_DAEMON} -eq 1 ]; then
-    nohup ${START_BE_CMD} "$@" >> ${LOG_FILE} 2>&1 </dev/null &
+    nohup ${START_BE_CMD} "$@" </dev/null &
 else
-    ${START_BE_CMD} "$@" >> ${LOG_FILE} 2>&1 </dev/null
+    exec ${START_BE_CMD} "$@" </dev/null
 fi
diff --git a/bin/start_fe.sh b/bin/start_fe.sh
@@ -26,6 +26,7 @@ OPTS=$(getopt \
   -l 'helper:' \
   -l 'host_type:' \
   -l 'debug' \
+  -l 'logconsole' \
   -- "$@")
 
 eval set -- "$OPTS"
@@ -34,12 +35,14 @@ RUN_DAEMON=0
 HELPER=
 HOST_TYPE=
 ENABLE_DEBUGGER=0
+RUN_LOG_CONSOLE=0
 while true; do
     case "$1" in
         --daemon) RUN_DAEMON=1 ; shift ;;
         --helper) HELPER=$2 ; shift 2 ;;
         --host_type) HOST_TYPE=$2 ; shift 2 ;;
         --debug) ENABLE_DEBUGGER=1 ; shift ;;
+        --logconsole) RUN_LOG_CONSOLE=1 ; shift ;;
         --) shift ;  break ;;
         *) echo "Internal error" ; exit 1 ;;
     esac
@@ -86,7 +89,7 @@ JAVA_VERSION=$(jdk_version)
 final_java_opt=$JAVA_OPTS
 if [[ "$JAVA_VERSION" -gt 8 ]]; then
     if [ -z "$JAVA_OPTS_FOR_JDK_9" ]; then
-        echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" >> $LOG_DIR/fe.out
+        echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf"
         exit -1
     fi 
     final_java_opt=$JAVA_OPTS_FOR_JDK_9
@@ -107,9 +110,6 @@ if [ ! -d $LOG_DIR ]; then
     mkdir -p $LOG_DIR
 fi
 
-echo "using java version $JAVA_VERSION" >> $LOG_DIR/fe.out
-echo $final_java_opt >> $LOG_DIR/fe.out
-
 # add libs to CLASSPATH
 for f in $STARROCKS_HOME/lib/*.jar; do
   CLASSPATH=$f:${CLASSPATH};
@@ -131,8 +131,6 @@ else
     LIMIT=/bin/limit
 fi
 
-echo `date` >> $LOG_DIR/fe.out
-
 if [ x"$HELPER" != x"" ]; then
     # change it to '-helper' to be compatible with code in Frontend
     HELPER="-helper $HELPER"
@@ -143,10 +141,28 @@ if [ x"$HOST_TYPE" != x"" ]; then
     HOST_TYPE="-host_type $HOST_TYPE"
 fi
 
-if [ ${RUN_DAEMON} -eq 1 ]; then
-    nohup $LIMIT $JAVA $final_java_opt com.starrocks.StarRocksFE ${HELPER} ${HOST_TYPE} "$@" >> $LOG_DIR/fe.out 2>&1 </dev/null &
+LOG_FILE=$LOG_DIR/fe.out
+
+if [ ${RUN_LOG_CONSOLE} -eq 1 ] ; then
+    if [ ! -w $STARROCKS_HOME/conf/fe.conf ] ; then
+        # workaround configmap readonly, can't change its content
+        mv $STARROCKS_HOME/conf/fe.conf $STARROCKS_HOME/conf/fe.conf.readonly
+        cp $STARROCKS_HOME/conf/fe.conf.readonly $STARROCKS_HOME/conf/fe.conf
+    fi
+    # force sys_log_to_console = true
+    echo "sys_log_to_console = true" >> $STARROCKS_HOME/conf/fe.conf
 else
-    $LIMIT $JAVA $final_java_opt com.starrocks.StarRocksFE ${HELPER} ${HOST_TYPE} "$@" >> $LOG_DIR/fe.out 2>&1 </dev/null
+    # redirect all subsequent commands' stdout/stderr into $LOG_FILE
+    exec &>> $LOG_FILE
 fi
 
-echo $! > $pidfile
+echo "using java version $JAVA_VERSION"
+echo $final_java_opt
+echo `date`
+
+# StarRocksFE java process will write its process id into $pidfile
+if [ ${RUN_DAEMON} -eq 1 ]; then
+    nohup $LIMIT $JAVA $final_java_opt com.starrocks.StarRocksFE ${HELPER} ${HOST_TYPE} "$@" </dev/null &
+else
+    exec $LIMIT $JAVA $final_java_opt com.starrocks.StarRocksFE ${HELPER} ${HOST_TYPE} "$@" </dev/null
+fi
diff --git a/docker/dockerfiles/be/be_entrypoint.sh b/docker/dockerfiles/be/be_entrypoint.sh
@@ -108,4 +108,10 @@ update_conf_from_configmap
 collect_env_info
 add_self $svc_name || exit $?
 log_stderr "run start_be.sh"
-$STARROCKS_HOME/bin/start_be.sh
+
+addition_args=
+if [[ "x$LOG_CONSOLE" == "x1" ]] ; then
+    # env var `LOG_CONSOLE=1` can be added to enable logging to console
+    addition_args="--logconsole"
+fi
+$STARROCKS_HOME/bin/start_be.sh $addition_args
diff --git a/docker/dockerfiles/be/cn_entrypoint.sh b/docker/dockerfiles/be/cn_entrypoint.sh
@@ -166,4 +166,10 @@ trap exit_clean SIGTERM
 
 update_conf_from_configmap
 log_stderr "run start_cn.sh"
-$STARROCKS_HOME/bin/start_cn.sh
+
+addition_args=
+if [[ "x$LOG_CONSOLE" == "x1" ]] ; then
+    # env var `LOG_CONSOLE=1` can be added to enable logging to console
+    addition_args="--logconsole"
+fi
+$STARROCKS_HOME/bin/start_cn.sh $addition_args
diff --git a/docker/dockerfiles/fe/fe_entrypoint.sh b/docker/dockerfiles/fe/fe_entrypoint.sh
@@ -223,6 +223,9 @@ start_fe_no_meta()
         done
     fi
 
+    if [[ "x$LOG_CONSOLE" == "x1" ]] ; then
+        opts+=" --logconsole"
+    fi
     log_stderr "first start with no meta run start_fe.sh with additional options: '$opts'"
     $STARROCKS_HOME/bin/start_fe.sh $opts
 }
@@ -234,6 +237,9 @@ start_fe_with_meta()
         opts+=" --host_type $HOST_TYPE"
     fi
 
+    if [[ "x$LOG_CONSOLE" == "x1" ]] ; then
+        opts+=" --logconsole"
+    fi
     log_stderr "start with meta run start_fe.sh with additional options: '$opts'"
     $STARROCKS_HOME/bin/start_fe.sh $opts
 }

diff --git a/fe/fe-core/src/main/java/com/starrocks/common/Config.java b/fe/fe-core/src/main/java/com/starrocks/common/Config.java
@@ -90,6 +90,11 @@ public class Config extends ConfigBase {
     @Deprecated
     @ConfField
     public static String sys_log_roll_mode = "SIZE-MB-1024";
+    /**
+     * Log to file by default. set to `true` if want to log to console
+     */
+    @ConfField
+    public static boolean sys_log_to_console = false;
 
     /**
      * audit_log_dir:

diff --git a/fe/fe-core/src/main/java/com/starrocks/common/Log4jConfig.java b/fe/fe-core/src/main/java/com/starrocks/common/Log4jConfig.java
@@ -52,10 +52,13 @@
 public class Log4jConfig extends XmlConfiguration {
     private static final long serialVersionUID = 1L;
 
-    private static String xmlConfTemplate = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
+    private static String xmlConfTemplateAppenders = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
             "\n" +
             "<Configuration status=\"info\" packages=\"com.starrocks.common\">\n" +
             "  <Appenders>\n" +
+            "    <Console name=\"ConsoleErr\" target=\"SYSTEM_ERR\" follow=\"true\">\n" +
+            "      <PatternLayout pattern=\"%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid) [%C{1}.%M():%L] %m%n\"/>\n" +
+            "    </Console>\n" +
             "    <RollingFile name=\"Sys\" fileName=\"${sys_log_dir}/fe.log\" filePattern=\"${sys_log_dir}/fe.log.${sys_file_pattern}-%i\">\n" +
             "      <PatternLayout charset=\"UTF-8\">\n" +
             "        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid) [%C{1}.%M():%L] %m%n</Pattern>\n" +
@@ -131,7 +134,10 @@ public class Log4jConfig extends XmlConfiguration {
             "        </Delete>\n" +
             "      </DefaultRolloverStrategy>\n" +
             "    </RollingFile>\n" +
-            "  </Appenders>\n" +
+            "  </Appenders>\n";
+
+    // Predefined loggers to write log to file
+    private static String xmlConfTemplateFileLoggers =
             "  <Loggers>\n" +
             "    <Root level=\"${sys_log_level}\">\n" +
             "      <AppenderRef ref=\"Sys\"/>\n" +
@@ -162,6 +168,15 @@ public class Log4jConfig extends XmlConfiguration {
             "  </Loggers>\n" +
             "</Configuration>";
 
+    // Predefined console logger, all logs will be written to console
+    private static String xmlConfTemplateConsoleLoggers =
+            "  <Loggers>\n" +
+            "    <Root level=\"${sys_log_level}\">\n" +
+            "      <AppenderRef ref=\"ConsoleErr\"/>\n" +
+            "    </Root>\n" +
+            "    <!--REPLACED BY AUDIT AND VERBOSE MODULE NAMES-->\n" +
+            "  </Loggers>\n" +
+            "</Configuration>";
     private static StrSubstitutor strSub;
     private static String sysLogLevel;
     private static String[] verboseModules;
@@ -170,7 +185,8 @@ public class Log4jConfig extends XmlConfiguration {
     private static String[] bigQueryModules;
 
     private static void reconfig() throws IOException {
-        String newXmlConfTemplate = xmlConfTemplate;
+        String newXmlConfTemplate = xmlConfTemplateAppenders;
+        newXmlConfTemplate += Config.sys_log_to_console ? xmlConfTemplateConsoleLoggers : xmlConfTemplateFileLoggers;
 
         // sys log config
         String sysLogDir = Config.sys_log_dir;