fix bugs: (#136)

1. rename database does not modify the coresponding names in cluster. 2. Result of SHOW BROKERS does not match that of in SHOW PROC '/brokers'. 3. Bugs when set property for other user using admin user.
apache · Nov 12, 2017 · 05b86e8 · 05b86e8
1 parent 0400fd1
commit 05b86e8
Show file tree

Hide file tree

Showing 7 changed files with 75 additions and 46 deletions.
diff --git a/docs/help/Contents/Data Manipulation/manipulation_stmt.md b/docs/help/Contents/Data Manipulation/manipulation_stmt.md
@@ -101,6 +101,8 @@
         max_filter_ratio：最大容忍可过滤（数据不规范等原因）的数据比例。默认零容忍。
         load_delete_flag：指定该导入是否通过导入key列的方式删除数据，仅适用于UNIQUE KEY，
                           导入时可不指定value列。默认为false。
+        exec_mem_limit:   当使用 broker 方式导入时，可以指定导入作业在单个 BE 上的内存大小限制。
+                          单位是字节。（默认是 2147483648，即 2G）
 
     5. 导入数据格式样例
 

diff --git a/fe/src/com/baidu/palo/analysis/SetUserPropertyStmt.java b/fe/src/com/baidu/palo/analysis/SetUserPropertyStmt.java
@@ -23,6 +23,7 @@
 import com.baidu.palo.cluster.ClusterNamespace;
 import com.baidu.palo.common.AnalysisException;
 import com.baidu.palo.common.InternalException;
+
 import com.google.common.base.Strings;
 
 import java.util.List;
@@ -48,9 +49,12 @@ public List<SetVar> getPropertyList() {
     public void analyze(Analyzer analyzer) throws AnalysisException, InternalException {
         super.analyze(analyzer);
         if (Strings.isNullOrEmpty(user)) {
+            // If param 'user' is not set, use the login user name.
+            // The login user name is full-qualified with cluster name.
             user = analyzer.getUser();
         } else {
-            if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) {
+            // If param 'user' is set, check if it need to be full-qualified
+            if (!analyzer.getCatalog().getUserMgr().isAdmin(user)) {
                 user = ClusterNamespace.getFullName(getClusterName(), user);
             }
         }

diff --git a/fe/src/com/baidu/palo/analysis/ShowBrokerStmt.java b/fe/src/com/baidu/palo/analysis/ShowBrokerStmt.java
@@ -24,7 +24,7 @@ public class ShowBrokerStmt extends ShowStmt {
     private static final ShowResultSetMetaData META_DATA =
             ShowResultSetMetaData.builder()
                     .addColumn(new Column("Broker", ColumnType.createVarchar(20)))
-                    .addColumn(new Column("Description", ColumnType.createVarchar(200)))
+                    .addColumn(new Column("Instances", ColumnType.createVarchar(200)))
                     .build();
 
     public ShowBrokerStmt() {

diff --git a/fe/src/com/baidu/palo/catalog/BrokerMgr.java b/fe/src/com/baidu/palo/catalog/BrokerMgr.java
@@ -25,6 +25,7 @@
 import com.baidu.palo.common.proc.ProcNodeInterface;
 import com.baidu.palo.common.proc.ProcResult;
 
+import com.google.common.base.Joiner;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -329,6 +330,18 @@ public void replayDropAllBroker(String name) {
         }
     }
 
+    public List<List<String>> getBrokersInfo() {
+        lock.lock();
+        try {
+            if (procNode == null) {
+                procNode = new BrokerProcNode();
+            }
+            return procNode.fetchResult().getRows();
+        } finally {
+            lock.unlock();
+        }
+    }
+
     public BrokerProcNode getProcNode() {
         lock.lock();
         try {
@@ -343,16 +356,19 @@ public BrokerProcNode getProcNode() {
 
     public class BrokerProcNode implements ProcNodeInterface {
         @Override
-        public ProcResult fetchResult() throws AnalysisException {
+        public ProcResult fetchResult() {
             BaseProcResult result = new BaseProcResult();
             result.setNames(BROKER_PROC_NODE_TITLE_NAMES);
 
             lock.lock();
             try {
                 for (Map.Entry<String, ArrayListMultimap<String, BrokerAddress>> entry : brokersMap.entrySet()) {
+                    String brokerName = entry.getKey();
+                    List<String> brokerAddrs = Lists.newArrayList();
                     for (BrokerAddress address : entry.getValue().values()) {
-                        result.addRow(Lists.newArrayList(entry.getKey(), address.toString()));
+                        brokerAddrs.add(address.toString());
                     }
+                    result.addRow(Lists.newArrayList(brokerName, Joiner.on(", ").join(brokerAddrs)));
                 }
             } finally {
                 lock.unlock();

diff --git a/fe/src/com/baidu/palo/catalog/Catalog.java b/fe/src/com/baidu/palo/catalog/Catalog.java
@@ -2279,19 +2279,18 @@ public void renameDatabase(AlterDatabaseRename stmt) throws DdlException {
     public void replayRenameDatabase(String dbName, String newDbName) {
         writeLock();
         try {
-            Database db = getDb(dbName);
-            db.setNameWithLock(newDbName);
-            db = fullNameToDb.get(dbName);
-            final Cluster cluster = nameToCluster.get(db.getClusterName());
+            Database db = fullNameToDb.get(dbName);
+            Cluster cluster = nameToCluster.get(db.getClusterName());
             cluster.removeDb(db.getFullName(), db.getId());
+            db.setName(newDbName);
             cluster.addDb(newDbName, db.getId());
             fullNameToDb.remove(dbName);
             fullNameToDb.put(newDbName, db);
         } finally {
             writeUnlock();
         }
 
-        LOG.info("replay rename database[{}] to {}", dbName, newDbName);
+        LOG.info("replay rename database {} to {}", dbName, newDbName);
     }
 
     public void createTable(CreateTableStmt stmt) throws DdlException {

diff --git a/fe/src/com/baidu/palo/common/proc/BackendsProcDir.java b/fe/src/com/baidu/palo/common/proc/BackendsProcDir.java
@@ -20,33 +20,36 @@
 
 package com.baidu.palo.common.proc;
 
-import com.baidu.palo.alter.DecommissionBackendJob.DecommissionType;
-import com.baidu.palo.catalog.Catalog;
+import com.baidu.palo.alter.DecommissionBackendJob.DecommissionType;
+import com.baidu.palo.catalog.Catalog;
 import com.baidu.palo.cluster.Cluster;
-import com.baidu.palo.common.AnalysisException;
-import com.baidu.palo.common.util.ListComparator;
-import com.baidu.palo.common.util.TimeUtils;
-import com.baidu.palo.system.Backend;
-import com.baidu.palo.system.SystemInfoService;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
+import com.baidu.palo.common.AnalysisException;
+import com.baidu.palo.common.util.ListComparator;
+import com.baidu.palo.common.util.TimeUtils;
+import com.baidu.palo.system.Backend;
+import com.baidu.palo.system.SystemInfoService;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
 
 public class BackendsProcDir implements ProcDirInterface {
-    public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>().add("Cluster")
-            .add("BackendId").add("IP").add("HostName").add("HeartbeatPort").add("BePort").add("HttpPort")
-            .add("LastStartTime").add("LastHeartbeat").add("Alive").add("SystemDecommissioned")
-            .add("ClusterDecommissioned").add("TabletNum").build();
-    public static final int IP_INDEX = 1;
-    public static final int HOSTNAME_INDEX = 2;
+    public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
+            .add("BackendId").add("Cluster").add("IP").add("HostName").add("HeartbeatPort")
+            .add("BePort").add("HttpPort").add("LastStartTime").add("LastHeartbeat").add("Alive")
+            .add("SystemDecommissioned").add("ClusterDecommissioned").add("TabletNum").add("FreeSpace")
+            .build();
+
+    public static final int IP_INDEX = 2;
+    public static final int HOSTNAME_INDEX = 3;
 
     private SystemInfoService clusterInfoService;
 
@@ -120,8 +123,8 @@ public static List<List<String>> getClusterBackendInfos(String clusterName) {
 
             Integer tabletNum = Catalog.getCurrentInvertedIndex().getTabletNumByBackendId(backendId);
             List<Comparable> backendInfo = Lists.newArrayList();
-            backendInfo.add(backend.getOwnerClusterName());
             backendInfo.add(String.valueOf(backendId));
+            backendInfo.add(backend.getOwnerClusterName());
             backendInfo.add(backend.getHost());
             if (Strings.isNullOrEmpty(clusterName)) {
                 backendInfo.add(hostName);
@@ -144,11 +147,20 @@ public static List<List<String>> getClusterBackendInfos(String clusterName) {
                 backendInfo.add(String.valueOf("false"));
             }
             backendInfo.add(tabletNum.toString());
+
+            double free = 0.0;
+            if (backend.getTotalCapacityB() <= 0) {
+                free = 0.0;
+            } else {
+                free = (double) backend.getAvailableCapacityB() * 100 / backend.getTotalCapacityB();
+            }
+            backendInfo.add(String.format("%.2f", free) + " %");
+
             comparableBackendInfos.add(backendInfo);
         }
 
-        // sort by id, ip hostName
-        ListComparator<List<Comparable>> comparator = new ListComparator<List<Comparable>>(0, 1, 2);
+        // sort by cluster name, host name
+        ListComparator<List<Comparable>> comparator = new ListComparator<List<Comparable>>(1, 3);
         Collections.sort(comparableBackendInfos, comparator);
 
         for (List<Comparable> backendInfo : comparableBackendInfos) {
@@ -168,16 +180,16 @@ public boolean register(String name, ProcNodeInterface node) {
     }
 
     @Override
-    public ProcNodeInterface lookup(String name) throws AnalysisException {
-        if (Strings.isNullOrEmpty(name)) {
+    public ProcNodeInterface lookup(String beIdStr) throws AnalysisException {
+        if (Strings.isNullOrEmpty(beIdStr)) {
             throw new AnalysisException("Backend id is null");
         }
 
         long backendId = -1L;
         try {
-            backendId = Long.valueOf(name);
+            backendId = Long.valueOf(beIdStr);
         } catch (NumberFormatException e) {
-            throw new AnalysisException("Invalid backend id format: " + name);
+            throw new AnalysisException("Invalid backend id format: " + beIdStr);
         }
 
         Backend backend = clusterInfoService.getBackend(backendId);

diff --git a/fe/src/com/baidu/palo/qe/ShowExecutor.java b/fe/src/com/baidu/palo/qe/ShowExecutor.java
@@ -82,12 +82,12 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.util.ArrayList;
 import java.util.Iterator;
-
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -847,13 +847,10 @@ private void handleShowRestore() throws AnalysisException {
         resultSet = new ShowResultSet(showStmt.getMetaData(), showStmt.getResultRows());
     }
 
-    // Handle show engines
+    // Handle show brokers
     private void handleShowBroker() {
         ShowBrokerStmt showStmt = (ShowBrokerStmt) stmt;
-        List<List<String>> rowSet = Lists.newArrayList();
-        for (String broker : Catalog.getInstance().getBrokerMgr().getBrokerNames()) {
-            rowSet.add(Lists.newArrayList(broker, ""));
-        }
+        List<List<String>> rowSet = Catalog.getInstance().getBrokerMgr().getBrokersInfo();
 
         // Only success
         resultSet = new ShowResultSet(showStmt.getMetaData(), rowSet);
@@ -882,7 +879,6 @@ private void handleShowExport() throws AnalysisException {
         List<List<String>> rows = Lists.newArrayList();
         for (List<Comparable> loadInfo : infos) {
             List<String> oneInfo = new ArrayList<String>(loadInfo.size());
-
             for (Comparable element : loadInfo) {
                 oneInfo.add(element.toString());
             }