diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 979320815bca9d..80adbfd67b6a86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -399,7 +399,7 @@ private void init() { db.setRemoteName(remoteDbName); } tmpIdToDb.put(dbId, db); - initCatalogLog.addRefreshDb(dbId); + initCatalogLog.addRefreshDb(dbId, remoteDbName); } else { dbId = Env.getCurrentEnv().getNextId(); tmpDbNameToId.put(localDbName, dbId); @@ -734,8 +734,12 @@ private void removeAccessController() { } public void replayInitCatalog(InitCatalogLog log) { - // If the remote name is missing during upgrade, all databases in the Map will be reinitialized. - if (log.getCreateCount() > 0 && (log.getRemoteDbNames() == null || log.getRemoteDbNames().isEmpty())) { + // If the remote name is missing during upgrade, or + // the refresh db's remote name is empty, + // all databases in the Map will be reinitialized. + if ((log.getCreateCount() > 0 && (log.getRemoteDbNames() == null || log.getRemoteDbNames().isEmpty())) + || (log.getRefreshCount() > 0 + && (log.getRefreshRemoteDbNames() == null || log.getRefreshRemoteDbNames().isEmpty()))) { dbNameToId = Maps.newConcurrentMap(); idToDb = Maps.newConcurrentMap(); lastUpdateTime = log.getLastUpdateTime(); @@ -755,6 +759,7 @@ public void replayInitCatalog(InitCatalogLog log) { log.getRefreshDbIds().get(i), name); continue; } + db.get().setRemoteName(log.getRefreshRemoteDbNames().get(i)); Preconditions.checkNotNull(db.get()); tmpDbNameToId.put(db.get().getFullName(), db.get().getId()); tmpIdToDb.put(db.get().getId(), db.get()); @@ -771,6 +776,18 @@ public void replayInitCatalog(InitCatalogLog log) { db.getFullName(), db.getId(), log.getRemoteDbNames().get(i)); } } + // Check whether the remoteName of db in tmpIdToDb is empty + for (ExternalDatabase db : tmpIdToDb.values()) { + if (Strings.isNullOrEmpty(db.getRemoteName())) { + LOG.info("Database [{}] remoteName is empty in catalog [{}], mark as uninitialized", + db.getFullName(), name); + dbNameToId = Maps.newConcurrentMap(); + idToDb = Maps.newConcurrentMap(); + lastUpdateTime = log.getLastUpdateTime(); + initialized = false; + return; + } + } dbNameToId = tmpDbNameToId; idToDb = tmpIdToDb; lastUpdateTime = log.getLastUpdateTime(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index cbacf563c32424..ea02a20a07b538 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -191,7 +191,9 @@ public final synchronized void makeSureInitialized() { public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { // If the remote name is missing during upgrade, all tables in the Map will be reinitialized. - if (log.getCreateCount() > 0 && (log.getRemoteTableNames() == null || log.getRemoteTableNames().isEmpty())) { + if ((log.getCreateCount() > 0 && (log.getRemoteTableNames() == null || log.getRemoteTableNames().isEmpty())) + || (log.getRefreshCount() > 0 + && (log.getRefreshRemoteTableNames() == null || log.getRefreshRemoteTableNames().isEmpty()))) { tableNameToId = Maps.newConcurrentMap(); idToTbl = Maps.newConcurrentMap(); lastUpdateTime = log.getLastUpdateTime(); @@ -209,6 +211,7 @@ public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { // So we need add a validation here to avoid table(s) not found, this is just a temporary solution // because later we will remove all the logics about InitCatalogLog/InitDatabaseLog. if (table.isPresent()) { + table.get().setRemoteName(log.getRefreshRemoteTableNames().get(i)); tmpTableNameToId.put(table.get().getName(), table.get().getId()); tmpIdToTbl.put(table.get().getId(), table.get()); @@ -234,6 +237,19 @@ public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { LOG.info("Synchronized table (create): [Name: {}, ID: {}, Remote Name: {}]", table.getName(), table.getId(), log.getRemoteTableNames().get(i)); } + // Check whether the remoteName and db Tbl db in idToTbl is empty + for (T table : idToTbl.values()) { + if (Strings.isNullOrEmpty(table.getRemoteName()) + || table.getDb() == null) { + LOG.info("Table [{}] remoteName or database is empty, mark as uninitialized", + table.getName()); + tableNameToId = Maps.newConcurrentMap(); + idToTbl = Maps.newConcurrentMap(); + lastUpdateTime = log.getLastUpdateTime(); + initialized = false; + return; + } + } tableNameToId = tmpTableNameToId; idToTbl = tmpIdToTbl; lastUpdateTime = log.getLastUpdateTime(); @@ -266,7 +282,7 @@ private void init() { table.setDb(this); } tmpIdToTbl.put(tblId, table); - initDatabaseLog.addRefreshTable(tblId); + initDatabaseLog.addRefreshTable(tblId, remoteTableName); } else { tblId = Env.getCurrentEnv().getNextId(); tmpTableNameToId.put(localTableName, tblId); @@ -620,14 +636,22 @@ public void gsonPostProcess() throws IOException { case "ExternalInfoSchemaTable": ExternalInfoSchemaTable infoSchemaTable = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(obj), ExternalInfoSchemaTable.class); + if (infoSchemaTable.getDb() == null) { + infoSchemaTable.setDb(this); + } tmpIdToTbl.put(infoSchemaTable.getId(), (T) infoSchemaTable); tableNameToId.put(infoSchemaTable.getName(), infoSchemaTable.getId()); + lowerCaseToTableName.put(infoSchemaTable.getName().toLowerCase(), infoSchemaTable.getName()); break; case "ExternalMysqlTable": ExternalMysqlTable mysqlTable = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(obj), ExternalMysqlTable.class); + if (mysqlTable.getDb() == null) { + mysqlTable.setDb(this); + } tmpIdToTbl.put(mysqlTable.getId(), (T) mysqlTable); tableNameToId.put(mysqlTable.getName(), mysqlTable.getId()); + lowerCaseToTableName.put(mysqlTable.getName().toLowerCase(), mysqlTable.getName()); break; default: break; @@ -640,6 +664,14 @@ public void gsonPostProcess() throws IOException { ((ExternalTable) obj).getName()); } } + // Check whether the remoteName and db Tbl db in idToTbl is empty + for (T table : idToTbl.values()) { + if (Strings.isNullOrEmpty(table.getRemoteName()) + || table.getDb() == null) { + initialized = false; + break; + } + } idToTbl = tmpIdToTbl; rwLock = new MonitoredReentrantReadWriteLock(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java index 023eecc2fa4c00..ac262764de64c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java @@ -58,6 +58,9 @@ public enum Type { @SerializedName(value = "refreshDbIds") private List refreshDbIds; + @SerializedName(value = "refreshRemoteDbNames") + private List refreshRemoteDbNames; + @SerializedName(value = "createDbIds") private List createDbIds; @@ -78,15 +81,17 @@ public InitCatalogLog() { createCount = 0; catalogId = 0; refreshDbIds = Lists.newArrayList(); + refreshRemoteDbNames = Lists.newArrayList(); createDbIds = Lists.newArrayList(); createDbNames = Lists.newArrayList(); remoteDbNames = Lists.newArrayList(); type = Type.UNKNOWN; } - public void addRefreshDb(long id) { + public void addRefreshDb(long id, String remoteName) { refreshCount += 1; refreshDbIds.add(id); + refreshRemoteDbNames.add(remoteName); } public void addCreateDb(long id, String name, String remoteName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java index 44ee0a39c56943..d1ea04a16b03e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java @@ -62,6 +62,9 @@ public enum Type { @SerializedName(value = "refreshTableIds") private List refreshTableIds; + @SerializedName(value = "refreshRemoteTableNames") + private List refreshRemoteTableNames; + @SerializedName(value = "createTableIds") private List createTableIds; @@ -83,15 +86,17 @@ public InitDatabaseLog() { catalogId = 0; dbId = 0; refreshTableIds = Lists.newArrayList(); + refreshRemoteTableNames = Lists.newArrayList(); createTableIds = Lists.newArrayList(); createTableNames = Lists.newArrayList(); remoteTableNames = Lists.newArrayList(); type = Type.UNKNOWN; } - public void addRefreshTable(long id) { + public void addRefreshTable(long id, String remoteName) { refreshCount += 1; refreshTableIds.add(id); + refreshRemoteTableNames.add(remoteName); } public void addCreateTable(long id, String name, String remoteName) {