Allow dynamically setting the log level even for loggers that aren't already explicitly configured

pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerLogger.java

@@ -68,7 +68,7 @@ public class PinotBrokerLogger {
   @Produces(MediaType.APPLICATION_JSON)
   @ApiOperation(value = "Get all the loggers", notes = "Return all the logger names")
   public List<String> getLoggers() {
-    return LoggerUtils.getAllLoggers();
+    return LoggerUtils.getAllConfiguredLoggers();
   }
 
   @GET

pinot-common/src/main/java/org/apache/pinot/common/utils/LoggerUtils.java

@@ -29,6 +29,7 @@
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.config.Configuration;
 import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.spi.AbstractLogger;
 
 
 /**
@@ -45,22 +46,40 @@ private LoggerUtils() {
 
   /**
    * Set logger level at runtime.
-   * @param loggerName
-   * @param logLevel
+   * @param loggerName name of the logger whose level is to be changed
+   * @param logLevel the new log level
    * @return logger info
    */
   public static Map<String, String> setLoggerLevel(String loggerName, String logLevel) {
-    LoggerContext context = (LoggerContext) LogManager.getContext(false);
-    Configuration config = context.getConfiguration();
-    if (!getAllLoggers().contains(loggerName)) {
-      throw new RuntimeException("Logger - " + loggerName + " not found");
-    }
-    LoggerConfig loggerConfig = getLoggerConfig(config, loggerName);
+    Level level;
     try {
-      loggerConfig.setLevel(Level.valueOf(logLevel));
+      level = Level.valueOf(logLevel);
     } catch (Exception e) {
       throw new RuntimeException("Unrecognized logger level - " + logLevel, e);
     }
+    LoggerContext context = (LoggerContext) LogManager.getContext(false);
+    Configuration config = context.getConfiguration();
+    LoggerConfig loggerConfig;
+    if (getAllConfiguredLoggers().contains(loggerName)) {
+      loggerConfig = getLoggerConfig(config, loggerName);
+      loggerConfig.setLevel(level);
+    } else {
+      // Check if the loggerName exists by comparing it to all known loggers in the context
+      if (getAllLoggers().stream().noneMatch(logger -> {
+        if (!logger.startsWith(loggerName)) {
+          return false;
+        }
+        if (logger.equals(loggerName)) {
+          return true;
+        }
+        // Check if loggerName is a valid parent / descendant logger for any known logger
+        return logger.substring(loggerName.length()).startsWith(".");
+      })) {
+        throw new RuntimeException("Logger - " + loggerName + " not found");
+      }
+      loggerConfig = new LoggerConfig(loggerName, level, true);
+      config.addLogger(loggerName, loggerConfig);
+    }
     // This causes all Loggers to re-fetch information from their LoggerConfig.
     context.updateLoggers();
     return getLoggerResponse(loggerConfig);
@@ -75,22 +94,30 @@ public static Map<String, String> setLoggerLevel(String loggerName, String logLe
   public static Map<String, String> getLoggerInfo(String loggerName) {
     LoggerContext context = (LoggerContext) LogManager.getContext(false);
     Configuration config = context.getConfiguration();
-    if (!getAllLoggers().contains(loggerName)) {
+    if (!getAllConfiguredLoggers().contains(loggerName)) {
       return null;
     }
     LoggerConfig loggerConfig = getLoggerConfig(config, loggerName);
     return getLoggerResponse(loggerConfig);
   }
 
   /**
-   * @return a list of all the logger names
+   * @return a list of all the configured logger names
    */
-  public static List<String> getAllLoggers() {
+  public static List<String> getAllConfiguredLoggers() {
     LoggerContext context = (LoggerContext) LogManager.getContext(false);
     Configuration config = context.getConfiguration();
     return config.getLoggers().values().stream().map(LoggerConfig::toString).collect(Collectors.toList());
   }
 
+  /**
+   * @return a list of all the logger names
+   */
+  public static List<String> getAllLoggers() {
+    LoggerContext context = (LoggerContext) LogManager.getContext(false);
+    return context.getLoggers().stream().map(AbstractLogger::getName).collect(Collectors.toList());
+  }
+
   private static LoggerConfig getLoggerConfig(Configuration config, String loggerName) {
     return loggerName.equalsIgnoreCase(ROOT) ? config.getRootLogger() : config.getLoggerConfig(loggerName);
   }

pinot-common/src/test/java/org/apache/pinot/common/utils/LoggerUtilsTest.java

@@ -21,6 +21,8 @@
 import com.google.common.collect.ImmutableList;
 import java.util.List;
 import java.util.Map;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LoggerContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testng.annotations.Test;
@@ -34,8 +36,8 @@ public class LoggerUtilsTest {
   private static final String PINOT = "org.apache.pinot";
 
   @Test
-  public void testGetAllLoggers() {
-    List<String> allLoggers = LoggerUtils.getAllLoggers();
+  public void testGetAllConfiguredLoggers() {
+    List<String> allLoggers = LoggerUtils.getAllConfiguredLoggers();
     assertEquals(allLoggers.size(), 2);
     assertTrue(allLoggers.contains(ROOT));
     assertTrue(allLoggers.contains(PINOT));
@@ -59,7 +61,7 @@ public void testGetLoggerInfo() {
   }
 
   @Test
-  public void testChangeLoggerLevel() {
+  public void testChangeConfiguredLoggerLevel() {
     Map<String, String> pinotLoggerInfo = LoggerUtils.getLoggerInfo(PINOT);
     assertNotNull(pinotLoggerInfo);
     assertEquals(pinotLoggerInfo.get("level"), "WARN");
@@ -72,6 +74,44 @@ public void testChangeLoggerLevel() {
     }
   }
 
+  @Test
+  public void testChangeNonConfiguredLoggerLevel() {
+    String loggerName = getClass().getCanonicalName();
+    // The logger for this test class is not explicitly configured and inherits the root logger's config
+    assertNull(LoggerUtils.getLoggerInfo(loggerName));
+
+    Map<String, String> loggerInfo = LoggerUtils.setLoggerLevel(loggerName, "DEBUG");
+    assertNotNull(loggerInfo);
+    assertEquals(loggerInfo.get("level"), "DEBUG");
+
+    // Verify that the logger for this test class now shows up in the configured loggers
+    loggerInfo = LoggerUtils.getLoggerInfo(loggerName);
+    assertNotNull(loggerInfo);
+    assertEquals(loggerInfo.get("level"), "DEBUG");
+
+    // Remove the logger configuration so that other tests aren't affected
+    ((LoggerContext) LogManager.getContext(false)).getConfiguration().removeLogger(loggerName);
+  }
+
+  @Test
+  public void testChangeNonConfiguredAncestorLoggerLevel() {
+    String loggerName = getClass().getPackageName();
+    // The logger for this package is not explicitly configured and inherits the root logger's config
+    assertNull(LoggerUtils.getLoggerInfo(loggerName));
+
+    Map<String, String> loggerInfo = LoggerUtils.setLoggerLevel(loggerName, "DEBUG");
+    assertNotNull(loggerInfo);
+    assertEquals(loggerInfo.get("level"), "DEBUG");
+
+    // Verify that the logger for this package now shows up in the configured loggers
+    loggerInfo = LoggerUtils.getLoggerInfo(loggerName);
+    assertNotNull(loggerInfo);
+    assertEquals(loggerInfo.get("level"), "DEBUG");
+
+    // Remove the logger configuration so that other tests aren't affected
+    ((LoggerContext) LogManager.getContext(false)).getConfiguration().removeLogger(loggerName);
+  }
+
   @Test
   public void testChangeLoggerLevelWithExceptions() {
     try {

pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerLogger.java

@@ -92,7 +92,7 @@ public class PinotControllerLogger {
   @Produces(MediaType.APPLICATION_JSON)
   @ApiOperation(value = "Get all the loggers", notes = "Return all the logger names")
   public List<String> getLoggers() {
-    return LoggerUtils.getAllLoggers();
+    return LoggerUtils.getAllConfiguredLoggers();
   }
 
   @GET

pinot-minion/src/main/java/org/apache/pinot/minion/api/resources/PinotMinionLogger.java

@@ -64,7 +64,7 @@ public class PinotMinionLogger {
   @Produces(MediaType.APPLICATION_JSON)
   @ApiOperation(value = "Get all the loggers", notes = "Return all the logger names")
   public List<String> getLoggers() {
-    return LoggerUtils.getAllLoggers();
+    return LoggerUtils.getAllConfiguredLoggers();
   }
 
   @GET

pinot-server/src/main/java/org/apache/pinot/server/api/resources/PinotServerLogger.java

@@ -64,7 +64,7 @@ public class PinotServerLogger {
   @Produces(MediaType.APPLICATION_JSON)
   @ApiOperation(value = "Get all the loggers", notes = "Return all the logger names")
   public List<String> getLoggers() {
-    return LoggerUtils.getAllLoggers();
+    return LoggerUtils.getAllConfiguredLoggers();
   }
 
   @GET

pinot-tools/src/main/java/org/apache/pinot/tools/service/api/resources/PinotServiceManagerLogger.java

@@ -56,7 +56,7 @@ public class PinotServiceManagerLogger {
   @Produces(MediaType.APPLICATION_JSON)
   @ApiOperation(value = "Get all the loggers", notes = "Return all the logger names")
   public List<String> getLoggers() {
-    return LoggerUtils.getAllLoggers();
+    return LoggerUtils.getAllConfiguredLoggers();
   }
 
   @GET