[enhancement](ldap) optimize LDAP authentication. (apache#11948)

* [enhancement](ldap) optimize LDAP authentication. 1. Support caching LDAP user information. 2. HTTP authentication supports LDAP. 3. LDAP temporary users support default user property. 4. LDAP configuration supports the `admin show config` and `admin set config` commands.
jackwener · Aug 24, 2022 · b619bb2 · b619bb2
1 parent d87ab69
commit b619bb2
Show file tree

Hide file tree

Showing 21 changed files with 593 additions and 242 deletions.
diff --git a/build.sh b/build.sh
@@ -432,6 +432,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
 
     cp -r -p "${DORIS_HOME}/bin"/*_fe.sh "${DORIS_OUTPUT}/fe/bin"/
     cp -r -p "${DORIS_HOME}/conf/fe.conf" "${DORIS_OUTPUT}/fe/conf"/
+    cp -r -p "${DORIS_HOME}/conf/ldap.conf" "${DORIS_OUTPUT}/fe/conf"/
     cp -r -p "${DORIS_HOME}/conf"/*.xml "${DORIS_OUTPUT}/fe/conf"/
     rm -rf "${DORIS_OUTPUT}/fe/lib"/*
     cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/

diff --git a/fe/conf/ldap.conf → conf/ldap.conf b/fe/conf/ldap.conf → conf/ldap.conf
@@ -41,14 +41,16 @@ ldap_user_basedn = ou=people,dc=domain,dc=com
 ldap_user_filter = (&(uid={login}))
 ldap_group_basedn = ou=group,dc=domain,dc=com
 
+# ldap_cache_time_out_s = 12 * 60 * 60;
+
 # LDAP pool configuration
 # https://docs.spring.io/spring-ldap/docs/2.3.3.RELEASE/reference/#pool-configuration
-#max_active = 8
-#max_total = -1
-#max_idle = 8
-#min_idle = 0
-#max_wait = -1
-#when_exhausted = 1
-#test_on_borrow = false
-#test_on_return = false
-#test_while_idle = false
+# ldap_pool_max_active = 8
+# ldap_pool_max_total = -1
+# ldap_pool_max_idle = 8
+# ldap_pool_min_idle = 0
+# ldap_pool_max_wait = -1
+# ldap_pool_when_exhausted = 1
+# ldap_pool_test_on_borrow = false
+# ldap_pool_test_on_return = false
+# ldap_pool_test_while_idle = false
diff --git a/docs/en/docs/admin-manual/privilege-ldap/ldap.md b/docs/en/docs/admin-manual/privilege-ldap/ldap.md
@@ -171,5 +171,3 @@ If jack also belongs to the LDAP groups doris_qa, doris_pm; Doris exists roles:
 ## Limitations of LDAP authentication
 
 * The current LDAP feature of Doris only supports plaintext password authentication, that is, when a user logs in, the password is transmitted in plaintext between client and fe and between fe and LDAP service.
-* The current LDAP authentication only supports password authentication under mysql protocol. If you use the Http interface, you cannot use LDAP users for authentication.
-* Temporary users do not have user properties.
diff --git a/docs/zh-CN/docs/admin-manual/privilege-ldap/ldap.md b/docs/zh-CN/docs/admin-manual/privilege-ldap/ldap.md
@@ -188,5 +188,3 @@ member: uid=jack,ou=aidp,dc=domain,dc=com
 ## LDAP验证的局限
 
 - 目前Doris的LDAP功能只支持明文密码验证，即用户登录时，密码在client与fe之间、fe与LDAP服务之间以明文的形式传输。
-- 当前的LDAP验证只支持在mysql协议下进行密码验证，如果使用Http接口则无法使用LDAP用户进行验证。
-- 临时用户不具有用户属性。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java b/fe/fe-core/src/main/java/org/apache/doris/common/ConfigBase.java
@@ -75,11 +75,12 @@ public void handle(Field field, String confVal) throws Exception {
     private static String ldapCustomConfFile;
     public static Class<? extends ConfigBase> ldapConfClass;
 
+    public static Map<String, Field> ldapConfFields = Maps.newHashMap();
+
     private boolean isLdapConfig = false;
 
     public void init(String configFile) throws Exception {
-        this.isLdapConfig = (this instanceof LdapConfig);
-        if (!isLdapConfig) {
+        if (this instanceof Config) {
             confClass = this.getClass();
             confFile = configFile;
             confFields = Maps.newHashMap();
@@ -92,9 +93,17 @@ public void init(String configFile) throws Exception {
             }
 
             initConf(confFile);
-        } else {
+        } else if (this instanceof LdapConfig) {
+            isLdapConfig = true;
             ldapConfClass = this.getClass();
             ldapConfFile = configFile;
+            for (Field field : ldapConfClass.getFields()) {
+                ConfField confField = field.getAnnotation(ConfField.class);
+                if (confField == null) {
+                    continue;
+                }
+                ldapConfFields.put(confField.value().equals("") ? field.getName() : confField.value(), field);
+            }
             initConf(ldapConfFile);
         }
     }
@@ -292,7 +301,11 @@ private static boolean isBoolean(String s) {
     public static synchronized void setMutableConfig(String key, String value) throws DdlException {
         Field field = confFields.get(key);
         if (field == null) {
-            throw new DdlException("Config '" + key + "' does not exist");
+            if (ldapConfFields.containsKey(key)) {
+                field = ldapConfFields.get(key);
+            } else {
+                throw new DdlException("Config '" + key + "' does not exist");
+            }
         }
 
         ConfField anno = field.getAnnotation(ConfField.class);
@@ -313,7 +326,10 @@ public static synchronized void setMutableConfig(String key, String value) throw
     }
 
     public static synchronized List<List<String>> getConfigInfo(PatternMatcher matcher) {
-        return confFields.entrySet().stream().sorted(Map.Entry.comparingByKey()).flatMap(e -> {
+        Map<String, Field> allConfFields = Maps.newHashMap();
+        allConfFields.putAll(confFields);
+        allConfFields.putAll(ldapConfFields);
+        return allConfFields.entrySet().stream().sorted(Map.Entry.comparingByKey()).flatMap(e -> {
             String confKey = e.getKey();
             Field f = e.getValue();
             ConfField anno = f.getAnnotation(ConfField.class);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/LdapConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/LdapConfig.java
@@ -66,10 +66,18 @@ public class LdapConfig extends ConfigBase {
     public static String ldap_group_basedn = "";
 
     /**
-     * Maximum number of user connections. This value should be between 1 and 10000.
+     * The user LDAP information cache time.
+     * After timeout, the user information will be retrieved from the LDAP service again.
      */
-    @ConfigBase.ConfField
-    public static long user_max_connections = 100L;
+    @ConfigBase.ConfField(mutable = true)
+    public static long ldap_user_cache_timeout_s = 12 * 60 * 60;
+
+    /**
+     * System LDAP information cache time.
+     * After timeout, clear all user information in the cache.
+     */
+    @ConfigBase.ConfField(mutable = true)
+    public static long ldap_cache_timeout_day = 30;
 
     /**
      * LDAP pool configuration:
@@ -80,35 +88,35 @@ public class LdapConfig extends ConfigBase {
      * from this pool at the same time. You can use a non-positive number for no limit.
      */
     @ConfigBase.ConfField
-    public static int max_active = 8;
+    public static int ldap_pool_max_active = 8;
 
     /**
      * The overall maximum number of active connections (for all types) that can be allocated from this pool
      * at the same time. You can use a non-positive number for no limit.
      */
     @ConfigBase.ConfField
-    public static int max_total = -1;
+    public static int ldap_pool_max_total = -1;
 
     /**
      * The maximum number of active connections of each type (read-only or read-write) that can remain idle
      * in the pool without extra connections being released. You can use a non-positive number for no limit.
      */
     @ConfigBase.ConfField
-    public static int max_idle = 8;
+    public static int ldap_pool_max_idle = 8;
 
     /**
      * The minimum number of active connections of each type (read-only or read-write) that can remain idle
      * in the pool without extra connections being created. You can use zero (the default) to create none.
      */
     @ConfigBase.ConfField
-    public static int min_idle = 0;
+    public static int ldap_pool_min_idle = 0;
 
     /**
      * The maximum number of milliseconds that the pool waits (when no connections are available) for a connection
      * to be returned before throwing an exception. You can use a non-positive number to wait indefinitely.
      */
     @ConfigBase.ConfField
-    public static int max_wait = -1;
+    public static int ldap_pool_max_wait = -1;
 
     /**
      * Specifies the behavior when the pool is exhausted.
@@ -121,25 +129,25 @@ public class LdapConfig extends ConfigBase {
      * The '2' option creates and returns a new object (essentially making max-active meaningless).
      */
     @ConfigBase.ConfField
-    public static byte when_exhausted = 1;
+    public static byte ldap_pool_when_exhausted = 1;
 
     /**
      * Whether objects are validated before being borrowed from the pool. If the object fails to validate,
      * it is dropped from the pool, and an attempt to borrow another is made.
      */
     @ConfigBase.ConfField
-    public static boolean test_on_borrow = false;
+    public static boolean ldap_pool_test_on_borrow = false;
 
     /**
      * Whether objects are validated before being returned to the pool.
      */
     @ConfigBase.ConfField
-    public static boolean test_on_return = false;
+    public static boolean ldap_pool_test_on_return = false;
 
     /**
      * Whether objects are validated by the idle object evictor (if any). If an object fails to validate,
      * it is dropped from the pool.
      */
     @ConfigBase.ConfField
-    public static boolean test_while_idle = false;
+    public static boolean ldap_pool_test_while_idle = false;
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/ldap/LdapAuthenticate.java b/fe/fe-core/src/main/java/org/apache/doris/ldap/LdapAuthenticate.java
@@ -22,17 +22,12 @@
 import org.apache.doris.cluster.ClusterNamespace;
 import org.apache.doris.common.ErrorCode;
 import org.apache.doris.common.ErrorReport;
-import org.apache.doris.common.LdapConfig;
-import org.apache.doris.mysql.privilege.PaloRole;
 import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.util.List;
-
 /**
  * This class is used for LDAP authentication login and LDAP group authorization.
  * This means that users can log in to Doris with a user name and LDAP password,
@@ -41,20 +36,6 @@
 public class LdapAuthenticate {
     private static final Logger LOG = LogManager.getLogger(LdapAuthenticate.class);
 
-    private static final String LDAP_GROUPS_PRIVS_NAME = "ldapGroupsPrivs";
-
-    // Maximum number of the user LDAP authentication login connections.
-    private static long userMaxConn = 100;
-
-    {
-        if (LdapConfig.user_max_connections <= 0 || LdapConfig.user_max_connections > 10000) {
-            LOG.warn("Ldap config user_max_connections is invalid. It should be set between 1 and 10000. "
-                    + "And now, it is set to the default value.");
-        } else {
-            userMaxConn = LdapConfig.user_max_connections;
-        }
-    }
-
     /**
      * The LDAP authentication process is as follows:
      * step1: Check the LDAP password.
@@ -70,8 +51,8 @@ public static boolean authenticate(ConnectContext context, String password, Stri
 
         // check user password by ldap server.
         try {
-            if (!LdapClient.checkPassword(userName, password)) {
-                LOG.debug("user:{} use error LDAP password.", userName);
+            if (!Env.getCurrentEnv().getAuth().getLdapManager().checkUserPasswd(qualifiedUser, password)) {
+                LOG.info("user:{} use check LDAP password failed.", userName);
                 ErrorReport.report(ErrorCode.ERR_ACCESS_DENIED_ERROR, qualifiedUser, context.getRemoteIP(), usePasswd);
                 return false;
             }
@@ -80,15 +61,6 @@ public static boolean authenticate(ConnectContext context, String password, Stri
             return false;
         }
 
-        // Get the LDAP groups privileges as a role.
-        PaloRole ldapGroupsPrivs;
-        try {
-            ldapGroupsPrivs = getLdapGroupsPrivs(userName, clusterName);
-        } catch (Exception e) {
-            LOG.error("Get ldap groups error.", e);
-            return false;
-        }
-
         String remoteIp = context.getMysqlChannel().getRemoteIp();
         UserIdentity tempUserIdentity = UserIdentity.createAnalyzedUserIdentWithIp(qualifiedUser, remoteIp);
         // Search the user in doris.
@@ -97,48 +69,11 @@ public static boolean authenticate(ConnectContext context, String password, Stri
             userIdentity = tempUserIdentity;
             LOG.debug("User:{} does not exists in doris, login as temporary users.", userName);
             context.setIsTempUser(true);
-            if (ldapGroupsPrivs == null) {
-                ldapGroupsPrivs = new PaloRole(LDAP_GROUPS_PRIVS_NAME);
-            }
-            LdapPrivsChecker.grantDefaultPrivToTempUser(ldapGroupsPrivs, clusterName);
         }
 
         context.setCurrentUserIdentity(userIdentity);
         context.setRemoteIP(remoteIp);
-        context.setLdapGroupsPrivs(ldapGroupsPrivs);
-        LOG.debug("ldap authentication success: identity:{}, privs:{}",
-                context.getCurrentUserIdentity(), context.getLdapGroupsPrivs());
+        LOG.debug("ldap authentication success: identity:{}", context.getCurrentUserIdentity());
         return true;
     }
-
-    /**
-     * Step1: get ldap groups from ldap server;
-     * Step2: get roles by ldap groups;
-     * Step3: merge the roles;
-     */
-    private static PaloRole getLdapGroupsPrivs(String userName, String clusterName) {
-        //get user ldap group. the ldap group name should be the same as the doris role name
-        List<String> ldapGroups = LdapClient.getGroups(userName);
-        List<String> rolesNames = Lists.newArrayList();
-        for (String group : ldapGroups) {
-            String qualifiedRole = ClusterNamespace.getFullName(clusterName, group);
-            if (Env.getCurrentEnv().getAuth().doesRoleExist(qualifiedRole)) {
-                rolesNames.add(qualifiedRole);
-            }
-        }
-        LOG.debug("get user:{} ldap groups:{} and doris roles:{}", userName, ldapGroups, rolesNames);
-
-        // merge the roles
-        if (rolesNames.isEmpty()) {
-            return null;
-        } else {
-            PaloRole ldapGroupsPrivs = new PaloRole(LDAP_GROUPS_PRIVS_NAME);
-            Env.getCurrentEnv().getAuth().mergeRolesNoCheckName(rolesNames, ldapGroupsPrivs);
-            return ldapGroupsPrivs;
-        }
-    }
-
-    public static long getMaxConn() {
-        return userMaxConn;
-    }
 }