From 05b86e82b21301fd98e2fed01a7c22f0774abf73 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 11 Nov 2017 20:26:29 -0600 Subject: [PATCH] fix bugs: (#136) 1. rename database does not modify the coresponding names in cluster. 2. Result of SHOW BROKERS does not match that of in SHOW PROC '/brokers'. 3. Bugs when set property for other user using admin user. --- .../Data Manipulation/manipulation_stmt.md | 2 + .../palo/analysis/SetUserPropertyStmt.java | 6 +- .../baidu/palo/analysis/ShowBrokerStmt.java | 2 +- fe/src/com/baidu/palo/catalog/BrokerMgr.java | 20 +++++- fe/src/com/baidu/palo/catalog/Catalog.java | 9 ++- .../palo/common/proc/BackendsProcDir.java | 72 +++++++++++-------- fe/src/com/baidu/palo/qe/ShowExecutor.java | 10 +-- 7 files changed, 75 insertions(+), 46 deletions(-) diff --git a/docs/help/Contents/Data Manipulation/manipulation_stmt.md b/docs/help/Contents/Data Manipulation/manipulation_stmt.md index b1aa2b4bd79495..9e6ed5218810d3 100644 --- a/docs/help/Contents/Data Manipulation/manipulation_stmt.md +++ b/docs/help/Contents/Data Manipulation/manipulation_stmt.md @@ -101,6 +101,8 @@ max_filter_ratio：最大容忍可过滤（数据不规范等原因）的数据比例。默认零容忍。 load_delete_flag：指定该导入是否通过导入key列的方式删除数据，仅适用于UNIQUE KEY， 导入时可不指定value列。默认为false。 + exec_mem_limit: 当使用 broker 方式导入时，可以指定导入作业在单个 BE 上的内存大小限制。 + 单位是字节。（默认是 2147483648，即 2G） 5. 导入数据格式样例 diff --git a/fe/src/com/baidu/palo/analysis/SetUserPropertyStmt.java b/fe/src/com/baidu/palo/analysis/SetUserPropertyStmt.java index 1d5c52aef7c5c3..5c4f15687b5351 100644 --- a/fe/src/com/baidu/palo/analysis/SetUserPropertyStmt.java +++ b/fe/src/com/baidu/palo/analysis/SetUserPropertyStmt.java @@ -23,6 +23,7 @@ import com.baidu.palo.cluster.ClusterNamespace; import com.baidu.palo.common.AnalysisException; import com.baidu.palo.common.InternalException; + import com.google.common.base.Strings; import java.util.List; @@ -48,9 +49,12 @@ public List getPropertyList() { public void analyze(Analyzer analyzer) throws AnalysisException, InternalException { super.analyze(analyzer); if (Strings.isNullOrEmpty(user)) { + // If param 'user' is not set, use the login user name. + // The login user name is full-qualified with cluster name. user = analyzer.getUser(); } else { - if (!analyzer.getCatalog().getUserMgr().isAdmin(analyzer.getUser())) { + // If param 'user' is set, check if it need to be full-qualified + if (!analyzer.getCatalog().getUserMgr().isAdmin(user)) { user = ClusterNamespace.getFullName(getClusterName(), user); } } diff --git a/fe/src/com/baidu/palo/analysis/ShowBrokerStmt.java b/fe/src/com/baidu/palo/analysis/ShowBrokerStmt.java index 1572e1bd17029b..f5d10d6a62c82c 100644 --- a/fe/src/com/baidu/palo/analysis/ShowBrokerStmt.java +++ b/fe/src/com/baidu/palo/analysis/ShowBrokerStmt.java @@ -24,7 +24,7 @@ public class ShowBrokerStmt extends ShowStmt { private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Broker", ColumnType.createVarchar(20))) - .addColumn(new Column("Description", ColumnType.createVarchar(200))) + .addColumn(new Column("Instances", ColumnType.createVarchar(200))) .build(); public ShowBrokerStmt() { diff --git a/fe/src/com/baidu/palo/catalog/BrokerMgr.java b/fe/src/com/baidu/palo/catalog/BrokerMgr.java index 45fc762c92ed35..83dbff0ee36c75 100644 --- a/fe/src/com/baidu/palo/catalog/BrokerMgr.java +++ b/fe/src/com/baidu/palo/catalog/BrokerMgr.java @@ -25,6 +25,7 @@ import com.baidu.palo.common.proc.ProcNodeInterface; import com.baidu.palo.common.proc.ProcResult; +import com.google.common.base.Joiner; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -329,6 +330,18 @@ public void replayDropAllBroker(String name) { } } + public List> getBrokersInfo() { + lock.lock(); + try { + if (procNode == null) { + procNode = new BrokerProcNode(); + } + return procNode.fetchResult().getRows(); + } finally { + lock.unlock(); + } + } + public BrokerProcNode getProcNode() { lock.lock(); try { @@ -343,16 +356,19 @@ public BrokerProcNode getProcNode() { public class BrokerProcNode implements ProcNodeInterface { @Override - public ProcResult fetchResult() throws AnalysisException { + public ProcResult fetchResult() { BaseProcResult result = new BaseProcResult(); result.setNames(BROKER_PROC_NODE_TITLE_NAMES); lock.lock(); try { for (Map.Entry> entry : brokersMap.entrySet()) { + String brokerName = entry.getKey(); + List brokerAddrs = Lists.newArrayList(); for (BrokerAddress address : entry.getValue().values()) { - result.addRow(Lists.newArrayList(entry.getKey(), address.toString())); + brokerAddrs.add(address.toString()); } + result.addRow(Lists.newArrayList(brokerName, Joiner.on(", ").join(brokerAddrs))); } } finally { lock.unlock(); diff --git a/fe/src/com/baidu/palo/catalog/Catalog.java b/fe/src/com/baidu/palo/catalog/Catalog.java index 013a25ef2578bf..0791f0322bf272 100644 --- a/fe/src/com/baidu/palo/catalog/Catalog.java +++ b/fe/src/com/baidu/palo/catalog/Catalog.java @@ -2279,11 +2279,10 @@ public void renameDatabase(AlterDatabaseRename stmt) throws DdlException { public void replayRenameDatabase(String dbName, String newDbName) { writeLock(); try { - Database db = getDb(dbName); - db.setNameWithLock(newDbName); - db = fullNameToDb.get(dbName); - final Cluster cluster = nameToCluster.get(db.getClusterName()); + Database db = fullNameToDb.get(dbName); + Cluster cluster = nameToCluster.get(db.getClusterName()); cluster.removeDb(db.getFullName(), db.getId()); + db.setName(newDbName); cluster.addDb(newDbName, db.getId()); fullNameToDb.remove(dbName); fullNameToDb.put(newDbName, db); @@ -2291,7 +2290,7 @@ public void replayRenameDatabase(String dbName, String newDbName) { writeUnlock(); } - LOG.info("replay rename database[{}] to {}", dbName, newDbName); + LOG.info("replay rename database {} to {}", dbName, newDbName); } public void createTable(CreateTableStmt stmt) throws DdlException { diff --git a/fe/src/com/baidu/palo/common/proc/BackendsProcDir.java b/fe/src/com/baidu/palo/common/proc/BackendsProcDir.java index e59c09f8d4e832..caefa5ce3f37ac 100644 --- a/fe/src/com/baidu/palo/common/proc/BackendsProcDir.java +++ b/fe/src/com/baidu/palo/common/proc/BackendsProcDir.java @@ -20,33 +20,36 @@ package com.baidu.palo.common.proc; -import com.baidu.palo.alter.DecommissionBackendJob.DecommissionType; -import com.baidu.palo.catalog.Catalog; +import com.baidu.palo.alter.DecommissionBackendJob.DecommissionType; +import com.baidu.palo.catalog.Catalog; import com.baidu.palo.cluster.Cluster; -import com.baidu.palo.common.AnalysisException; -import com.baidu.palo.common.util.ListComparator; -import com.baidu.palo.common.util.TimeUtils; -import com.baidu.palo.system.Backend; -import com.baidu.palo.system.SystemInfoService; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; +import com.baidu.palo.common.AnalysisException; +import com.baidu.palo.common.util.ListComparator; +import com.baidu.palo.common.util.TimeUtils; +import com.baidu.palo.system.Backend; +import com.baidu.palo.system.SystemInfoService; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; import java.util.List; public class BackendsProcDir implements ProcDirInterface { - public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder().add("Cluster") - .add("BackendId").add("IP").add("HostName").add("HeartbeatPort").add("BePort").add("HttpPort") - .add("LastStartTime").add("LastHeartbeat").add("Alive").add("SystemDecommissioned") - .add("ClusterDecommissioned").add("TabletNum").build(); - public static final int IP_INDEX = 1; - public static final int HOSTNAME_INDEX = 2; + public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() + .add("BackendId").add("Cluster").add("IP").add("HostName").add("HeartbeatPort") + .add("BePort").add("HttpPort").add("LastStartTime").add("LastHeartbeat").add("Alive") + .add("SystemDecommissioned").add("ClusterDecommissioned").add("TabletNum").add("FreeSpace") + .build(); + + public static final int IP_INDEX = 2; + public static final int HOSTNAME_INDEX = 3; private SystemInfoService clusterInfoService; @@ -120,8 +123,8 @@ public static List> getClusterBackendInfos(String clusterName) { Integer tabletNum = Catalog.getCurrentInvertedIndex().getTabletNumByBackendId(backendId); List backendInfo = Lists.newArrayList(); - backendInfo.add(backend.getOwnerClusterName()); backendInfo.add(String.valueOf(backendId)); + backendInfo.add(backend.getOwnerClusterName()); backendInfo.add(backend.getHost()); if (Strings.isNullOrEmpty(clusterName)) { backendInfo.add(hostName); @@ -144,11 +147,20 @@ public static List> getClusterBackendInfos(String clusterName) { backendInfo.add(String.valueOf("false")); } backendInfo.add(tabletNum.toString()); + + double free = 0.0; + if (backend.getTotalCapacityB() <= 0) { + free = 0.0; + } else { + free = (double) backend.getAvailableCapacityB() * 100 / backend.getTotalCapacityB(); + } + backendInfo.add(String.format("%.2f", free) + " %"); + comparableBackendInfos.add(backendInfo); } - // sort by id, ip hostName - ListComparator> comparator = new ListComparator>(0, 1, 2); + // sort by cluster name, host name + ListComparator> comparator = new ListComparator>(1, 3); Collections.sort(comparableBackendInfos, comparator); for (List backendInfo : comparableBackendInfos) { @@ -168,16 +180,16 @@ public boolean register(String name, ProcNodeInterface node) { } @Override - public ProcNodeInterface lookup(String name) throws AnalysisException { - if (Strings.isNullOrEmpty(name)) { + public ProcNodeInterface lookup(String beIdStr) throws AnalysisException { + if (Strings.isNullOrEmpty(beIdStr)) { throw new AnalysisException("Backend id is null"); } long backendId = -1L; try { - backendId = Long.valueOf(name); + backendId = Long.valueOf(beIdStr); } catch (NumberFormatException e) { - throw new AnalysisException("Invalid backend id format: " + name); + throw new AnalysisException("Invalid backend id format: " + beIdStr); } Backend backend = clusterInfoService.getBackend(backendId); diff --git a/fe/src/com/baidu/palo/qe/ShowExecutor.java b/fe/src/com/baidu/palo/qe/ShowExecutor.java index c7d3005e6cc3f3..da8bda3381700e 100644 --- a/fe/src/com/baidu/palo/qe/ShowExecutor.java +++ b/fe/src/com/baidu/palo/qe/ShowExecutor.java @@ -82,12 +82,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.Iterator; - import java.util.List; import java.util.Map; import java.util.Set; @@ -847,13 +847,10 @@ private void handleShowRestore() throws AnalysisException { resultSet = new ShowResultSet(showStmt.getMetaData(), showStmt.getResultRows()); } - // Handle show engines + // Handle show brokers private void handleShowBroker() { ShowBrokerStmt showStmt = (ShowBrokerStmt) stmt; - List> rowSet = Lists.newArrayList(); - for (String broker : Catalog.getInstance().getBrokerMgr().getBrokerNames()) { - rowSet.add(Lists.newArrayList(broker, "")); - } + List> rowSet = Catalog.getInstance().getBrokerMgr().getBrokersInfo(); // Only success resultSet = new ShowResultSet(showStmt.getMetaData(), rowSet); @@ -882,7 +879,6 @@ private void handleShowExport() throws AnalysisException { List> rows = Lists.newArrayList(); for (List loadInfo : infos) { List oneInfo = new ArrayList(loadInfo.size()); - for (Comparable element : loadInfo) { oneInfo.add(element.toString()); }