[opt](log) refactor the log dir config (apache#32933)

Refactor the config for log dir of FE and BE

TLDR:
- Use env variable `LOG_DIR` to set root log dir
- Remove `sys_log_dir` for FE and BE

Details:

1. FE

    1. The root log dir is set by env variable `LOG_DIR` in `fe.conf`
    2. The default value of `audit_log_dir` is same as `${LOG_DIR}/`
    3. The default value of `spark_launcher_log_dir` is `${LOG_DIR}/spark_launcher_log`
    4. The default value of `nereids_trace_log_dir` is `${LOG_DIR}/nereids_trace_log`
    5. The origin `sys_log_dir` is deprecated, and default value is `""`.
        But for compatibility, if user already set `sys_log_dir` before, Doris will still use it as root log dir.

2. BE

     1. The root log dir is set by env variable `LOG_DIR` in `be.conf`
     2. Remove `pipeline_tracing_log_dir`, use `${LOG_DIR}` directly.
     3. The origin `sys_log_dir` is deprecated, and default value is `""`.
         But for compatibility, if user already set `sys_log_dir` before, Doris will still use it as root log dir.

be/src/common/config.cpp

@@ -203,10 +203,8 @@ DEFINE_mInt32(download_low_speed_limit_kbps, "50");
 // download low speed time(seconds)
 DEFINE_mInt32(download_low_speed_time, "300");
 
-// log dir
-DEFINE_String(sys_log_dir, "${DORIS_HOME}/log");
+DEFINE_String(sys_log_dir, "");
 DEFINE_String(user_function_dir, "${DORIS_HOME}/lib/udf");
-DEFINE_String(pipeline_tracing_log_dir, "${DORIS_HOME}/log/tracing");
 // INFO, WARNING, ERROR, FATAL
 DEFINE_mString(sys_log_level, "INFO");
 // TIME-DAY, TIME-HOUR, SIZE-MB-nnn

be/src/common/config.h

@@ -251,10 +251,10 @@ DECLARE_mInt32(download_low_speed_limit_kbps);
 // download low speed time(seconds)
 DECLARE_mInt32(download_low_speed_time);
 
-// log dir
+// deprecated, use env var LOG_DIR in be.conf
 DECLARE_String(sys_log_dir);
+// for udf
 DECLARE_String(user_function_dir);
-DECLARE_String(pipeline_tracing_log_dir);
 // INFO, WARNING, ERROR, FATAL
 DECLARE_String(sys_log_level);
 // TIME-DAY, TIME-HOUR, SIZE-MB-nnn

be/src/common/logconfig.cpp

@@ -63,7 +63,12 @@ bool init_glog(const char* basename) {
     // so fatal log can output to be.out .
     FLAGS_stderrthreshold = google::FATAL;
     // set glog log dir
-    FLAGS_log_dir = config::sys_log_dir;
+    // ATTN: sys_log_dir is deprecated, this is just for compatibility
+    std::string log_dir = config::sys_log_dir;
+    if (log_dir == "") {
+        log_dir = getenv("LOG_DIR");
+    }
+    FLAGS_log_dir = log_dir;
     // 0 means buffer INFO only
     FLAGS_logbuflevel = 0;
     // buffer log messages for at most this many seconds

be/src/olap/compaction.cpp

@@ -524,7 +524,8 @@ Status CompactionMixin::do_inverted_index_compaction() {
         auto write_json_to_file = [&](const nlohmann::json& json_obj,
                                       const std::string& file_name) {
             io::FileWriterPtr file_writer;
-            std::string file_path = fmt::format("{}/{}.json", config::sys_log_dir, file_name);
+            std::string file_path =
+                    fmt::format("{}/{}.json", std::string(getenv("LOG_DIR")), file_name);
             RETURN_IF_ERROR(io::global_local_filesystem()->create_file(file_path, &file_writer));
             RETURN_IF_ERROR(file_writer->append(json_obj.dump()));
             RETURN_IF_ERROR(file_writer->append("\n"));

be/src/pipeline/pipeline_tracing.cpp

@@ -95,10 +95,11 @@ void PipelineTracerContext::_dump(TUniqueId query_id) {
         return;
     }
 
+    std::filesystem::path log_dir = fmt::format("{}/pipe_tracing", getenv("LOG_DIR"));
     //TODO: when dump, now could append records but can't add new query. try use better grained locks.
     std::unique_lock<std::mutex> l(_data_lock); // can't rehash
     if (_dump_type == RecordType::PerQuery) {
-        auto path = _dir / fmt::format("query{}", to_string(query_id));
+        auto path = log_dir / fmt::format("query{}", to_string(query_id));
         int fd = ::open(
                 path.c_str(), O_CREAT | O_WRONLY | O_TRUNC,
                 S_ISGID | S_ISUID | S_IWUSR | S_IRUSR | S_IWGRP | S_IRGRP | S_IWOTH | S_IROTH);
@@ -123,8 +124,9 @@ void PipelineTracerContext::_dump(TUniqueId query_id) {
         THROW_IF_ERROR(writer.finalize());
         THROW_IF_ERROR(writer.close());
     } else if (_dump_type == RecordType::Periodic) {
-        auto path = _dir / fmt::format("until{}",
-                                       std::chrono::steady_clock::now().time_since_epoch().count());
+        auto path =
+                log_dir /
+                fmt::format("until{}", std::chrono::steady_clock::now().time_since_epoch().count());
         int fd = ::open(
                 path.c_str(), O_CREAT | O_WRONLY | O_TRUNC,
                 S_ISGID | S_ISUID | S_IWUSR | S_IRUSR | S_IWGRP | S_IRGRP | S_IWOTH | S_IROTH);

be/src/pipeline/pipeline_tracing.h

@@ -52,6 +52,7 @@ struct ScheduleRecord {
 using OneQueryTraces = moodycamel::ConcurrentQueue<ScheduleRecord>;
 
 // belongs to exec_env, for all query, if enable
+// curl http://{host}:{web_server_port}/api/running_pipeline_tasks
 class PipelineTracerContext {
 public:
     enum class RecordType {
@@ -75,7 +76,6 @@ class PipelineTracerContext {
     phmap::flat_hash_map<TUniqueId, uint64_t> _id_to_workload_group;
 
     RecordType _dump_type = RecordType::None;
-    std::filesystem::path _dir = config::pipeline_tracing_log_dir;
     decltype(MonotonicSeconds()) _last_dump_time;
     decltype(MonotonicSeconds()) _dump_interval_s =
             60; // effective iff Periodic mode. 1 minute default.

bin/start_be.sh

@@ -208,6 +208,8 @@ if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then
     source "${DORIS_HOME}/bin/palo_env.sh"
 fi
 
+export PPROF_TMPDIR="${LOG_DIR}"
+
 if [[ -z "${JAVA_HOME}" ]]; then
     echo "The JAVA_HOME environment variable is not defined correctly"
     echo "This environment variable is needed to run this program"

conf/be.conf

@@ -17,13 +17,14 @@
 
 CUR_DATE=`date +%Y%m%d-%H%M%S`
 
-PPROF_TMPDIR="$DORIS_HOME/log/"
+# Log dir
+LOG_DIR="${DORIS_HOME}/log/"
 
 # For jdk 8
-JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
+JAVA_OPTS="-Xmx1024m -DlogPath=$LOG_DIR/jni.log -Xloggc:$LOG_DIR/be.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
 
 # For jdk 17, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc*:$DORIS_HOME/log/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED"
+JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$LOG_DIR/jni.log -Xlog:gc*:$LOG_DIR/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED"
 
 # Set your own JAVA_HOME
 # JAVA_HOME=/path/to/jdk/
@@ -33,9 +34,6 @@ JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc*:$DOR
 JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
 JEMALLOC_PROF_PRFIX=""
 
-# INFO, WARNING, ERROR, FATAL
-sys_log_level = INFO
-
 # ports for admin, web, heartbeat service 
 be_port = 9060
 webserver_port = 8040
@@ -74,7 +72,8 @@ ssl_private_key_path = "$DORIS_HOME/conf/key.pem"
 # jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers
 
 # Advanced configurations
-# sys_log_dir = ${DORIS_HOME}/log
+# INFO, WARNING, ERROR, FATAL
+sys_log_level = INFO
 # sys_log_roll_mode = SIZE-MB-1024
 # sys_log_roll_num = 10
 # sys_log_verbose_modules = *

conf/fe.conf

@@ -23,14 +23,14 @@
 
 CUR_DATE=`date +%Y%m%d-%H%M%S`
 
-# the output dir of stderr and stdout 
+# Log dir
 LOG_DIR = ${DORIS_HOME}/log
 
 # For jdk 8
-JAVA_OPTS="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Dlog4j2.formatMsgNoLookups=true"
+JAVA_OPTS="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:$LOG_DIR/log/fe.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Dlog4j2.formatMsgNoLookups=true"
 
 # For jdk 17, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$DORIS_HOME/log/ -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.ref=ALL-UNNAMED"
+JAVA_OPTS_FOR_JDK_17="-Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$LOG_DIR -Xlog:gc*:$LOG_DIR/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.ref=ALL-UNNAMED"
 
 # Set your own JAVA_HOME
 # JAVA_HOME=/path/to/jdk/
@@ -39,12 +39,6 @@ JAVA_OPTS_FOR_JDK_17="-Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m
 ## the lowercase properties are read by main program.
 ##
 
-# INFO, WARN, ERROR, FATAL
-sys_log_level = INFO
-
-# NORMAL, BRIEF, ASYNC
-sys_log_mode = NORMAL
-
 # store metadata, must be created before start FE.
 # Default value is ${DORIS_HOME}/doris-meta
 # meta_dir = ${DORIS_HOME}/doris-meta
@@ -67,10 +61,13 @@ arrow_flight_sql_port = -1
 
 # Advanced configurations 
 # log_roll_size_mb = 1024
-# sys_log_dir = ${DORIS_HOME}/log
+# INFO, WARN, ERROR, FATAL
+sys_log_level = INFO
+# NORMAL, BRIEF, ASYNC
+sys_log_mode = NORMAL
 # sys_log_roll_num = 10
 # sys_log_verbose_modules = org.apache.doris
-# audit_log_dir = ${DORIS_HOME}/log
+# audit_log_dir = $LOG_DIR
 # audit_log_modules = slow_query, query
 # audit_log_roll_num = 10
 # meta_delay_toleration_second = 10

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -62,9 +62,10 @@ public class Config extends ConfigBase {
      * sys_log_enable_compress:
      *      default is false. if true, will compress fe.log & fe.warn.log by gzip
      */
+    @Deprecated // use env var LOG_DIR instead
     @ConfField(description = {"FE 日志文件的存放路径，用于存放 fe.log。",
             "The path of the FE log file, used to store fe.log"})
-    public static String sys_log_dir = System.getenv("DORIS_HOME") + "/log";
+    public static String sys_log_dir = "";
 
     @ConfField(description = {"FE 日志的级别", "The level of FE log"}, options = {"INFO", "WARN", "ERROR", "FATAL"})
     public static String sys_log_level = "INFO";
@@ -101,7 +102,7 @@ public class Config extends ConfigBase {
 
     @ConfField(description = {"FE 审计日志文件的存放路径，用于存放 fe.audit.log。",
             "The path of the FE audit log file, used to store fe.audit.log"})
-    public static String audit_log_dir = System.getenv("DORIS_HOME") + "/log";
+    public static String audit_log_dir = System.getenv("LOG_DIR");
     @ConfField(description = {"FE 审计日志文件的最大数量。超过这个数量后，最老的日志文件会被删除",
             "The maximum number of FE audit log files. "
                     + "After exceeding this number, the oldest log file will be deleted"})
@@ -570,7 +571,7 @@ public class Config extends ConfigBase {
     public static String spark_resource_path = "";
 
     @ConfField(description = {"Spark launcher 日志路径", "Spark launcher log dir"})
-    public static String spark_launcher_log_dir = sys_log_dir + "/spark_launcher_log";
+    public static String spark_launcher_log_dir = System.getenv("LOG_DIR") + "/spark_launcher_log";
 
     @ConfField(description = {"Yarn client 的路径", "Yarn client path"})
     public static String yarn_client_path = System.getenv("DORIS_HOME") + "/lib/yarn-client/hadoop/bin/yarn";
@@ -2469,7 +2470,7 @@ public class Config extends ConfigBase {
 
     @ConfField(description = {"nereids trace文件的存放路径。",
             "The path of the nereids trace file."})
-    public static String nereids_trace_log_dir = System.getenv("DORIS_HOME") + "/log/nereids_trace";
+    public static String nereids_trace_log_dir = System.getenv("LOG_DIR") + "/nereids_trace";
 
     @ConfField(mutable = true, masterOnly = true, description = {
             "备份过程中，分配给每个be的upload任务最大个数，默认值为3个。",

fe/fe-common/src/main/java/org/apache/doris/common/io/DiskUtils.java

@@ -44,7 +44,6 @@ public static Df df(String dir) {
             return df;
         }
 
-
         Process process;
         try {
             process = Runtime.getRuntime().exec("df -k " + dir);

fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java

@@ -19,6 +19,7 @@
 
 import org.apache.doris.httpv2.config.SpringLog4j2Config;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.Maps;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.LoggerContext;
@@ -136,7 +137,9 @@ private static void reconfig() throws IOException {
         String newXmlConfTemplate = xmlConfTemplate;
 
         // sys log config
-        String sysLogDir = Config.sys_log_dir;
+        // ATTN, sys_log_dir is deprecated, use LOG_DIR instead
+        String sysLogDir = Strings.isNullOrEmpty(Config.sys_log_dir) ? System.getenv("LOG_DIR") :
+                Config.sys_log_dir;
         String sysRollNum = String.valueOf(Config.sys_log_roll_num);
         String sysDeleteAge = String.valueOf(Config.sys_log_delete_age);
         boolean compressSysLog = Config.sys_log_enable_compress;

fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/LogController.java

@@ -109,7 +109,8 @@ private void appendLogConf(Map<String, Map<String, String>> content, String addV
     private void appendLogInfo(Map<String, Map<String, String>> content) {
         Map<String, String> map = new HashMap<>();
 
-        final String logPath = Config.sys_log_dir + "/fe.warn.log";
+        String logDir = Strings.isNullOrEmpty(Config.sys_log_dir) ? System.getenv("LOG_DIR") : Config.sys_log_dir;
+        final String logPath = logDir + "/fe.warn.log";
         map.put("logPath", logPath);
 
         RandomAccessFile raf = null;

fe/fe-core/src/main/java/org/apache/doris/service/ExecuteEnv.java

@@ -22,6 +22,8 @@
 import org.apache.doris.qe.ConnectScheduler;
 import org.apache.doris.qe.MultiLoadMgr;
 
+import com.google.common.base.Strings;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -40,9 +42,11 @@ private ExecuteEnv() {
         scheduler = new ConnectScheduler(Config.qe_max_connection);
         startupTime = System.currentTimeMillis();
         processUUID = System.currentTimeMillis();
+        String logDir = Strings.isNullOrEmpty(Config.sys_log_dir) ? System.getenv("LOG_DIR") :
+                Config.sys_log_dir;
         diskInfos = new ArrayList<FeDiskInfo>() {{
                 add(new FeDiskInfo("meta", Config.meta_dir, DiskUtils.df(Config.meta_dir)));
-                add(new FeDiskInfo("log", Config.sys_log_dir, DiskUtils.df(Config.sys_log_dir)));
+                add(new FeDiskInfo("log", logDir, DiskUtils.df(logDir)));
                 add(new FeDiskInfo("audit-log", Config.audit_log_dir, DiskUtils.df(Config.audit_log_dir)));
                 add(new FeDiskInfo("temp", Config.tmp_dir, DiskUtils.df(Config.tmp_dir)));
                 add(new FeDiskInfo("deploy", System.getenv("DORIS_HOME"), DiskUtils.df(System.getenv("DORIS_HOME"))));