Skip to content

Commit

Permalink
Merge branch 'release-7.1' into cherry-pick-9274-to-release-7.1
Browse files Browse the repository at this point in the history
  • Loading branch information
@JaySon-Huang
JaySon-Huang authored Sep 26, 2024
2 parents 7fc0f6b + 0ad1189 commit 233e882
Show file tree
Hide file tree
Showing 24 changed files with 838 additions and 470 deletions.
1 change: 1 addition & 0 deletions dbms/src/Common/FailPoint.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ namespace DB
M(skip_check_segment_update) \
M(force_set_page_file_write_errno) \
M(force_split_io_size_4k) \
M(force_set_num_regions_for_table) \
M(minimum_block_size_for_cross_join) \
M(random_exception_after_dt_write_done) \
M(random_slow_page_storage_write) \
Expand Down
72 changes: 61 additions & 11 deletions dbms/src/Debug/MockTiDB.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,17 +64,47 @@ MockTiDB::MockTiDB()
databases["default"] = 0;
}

TablePtr MockTiDB::dropTableInternal(Context & context, const String & database_name, const String & table_name, bool drop_regions)
TablePtr MockTiDB::dropTableByNameImpl(Context & context, const String & database_name, const String & table_name, bool drop_regions)
{
String qualified_name = database_name + "." + table_name;
auto it_by_name = tables_by_name.find(qualified_name);
if (it_by_name == tables_by_name.end())
return nullptr;
auto table = it_by_name->second;
dropTableInternal(context, table, drop_regions);

tables_by_name.erase(it_by_name);
return table;
}

TablePtr MockTiDB::dropTableByIdImpl(Context & context, const TableID table_id, bool drop_regions)
{
auto iter = tables_by_id.find(table_id);
if (iter == tables_by_id.end())
return nullptr;

auto table = iter->second;
dropTableInternal(context, table, drop_regions);

// erase from `tables_by_name`
for (auto iter_by_name = tables_by_name.begin(); iter_by_name != tables_by_name.end(); /* empty */)
{
if (table != iter_by_name->second)
{
++iter_by_name;
continue;
}
LOG_INFO(Logger::get(), "removing table from MockTiDB, name={} table_id={}", iter_by_name->first, table_id);
iter_by_name = tables_by_name.erase(iter_by_name);
}
return table;
}

TablePtr MockTiDB::dropTableInternal(Context & context, const TablePtr & table, bool drop_regions)
{
auto & kvstore = context.getTMTContext().getKVStore();
auto & region_table = context.getTMTContext().getRegionTable();

auto table = it_by_name->second;
if (table->isPartitionTable())
{
for (const auto & partition : table->table_info.partition.definitions)
Expand All @@ -90,15 +120,12 @@ TablePtr MockTiDB::dropTableInternal(Context & context, const String & database_
}
tables_by_id.erase(table->id());

tables_by_name.erase(it_by_name);

if (drop_regions)
{
for (auto & e : region_table.getRegionsByTable(NullspaceID, table->id()))
kvstore->mockRemoveRegion(e.first, region_table);
region_table.removeTable(NullspaceID, table->id());
}

return table;
}

Expand All @@ -113,7 +140,7 @@ void MockTiDB::dropDB(Context & context, const String & database_name, bool drop
});

for (const auto & table_name : table_names)
dropTableInternal(context, database_name, table_name, drop_regions);
dropTableByNameImpl(context, database_name, table_name, drop_regions);

version++;

Expand All @@ -132,8 +159,25 @@ void MockTiDB::dropDB(Context & context, const String & database_name, bool drop
void MockTiDB::dropTable(Context & context, const String & database_name, const String & table_name, bool drop_regions)
{
std::lock_guard lock(tables_mutex);
auto table = dropTableByNameImpl(context, database_name, table_name, drop_regions);
if (!table)
return;

version++;

auto table = dropTableInternal(context, database_name, table_name, drop_regions);
SchemaDiff diff;
diff.type = SchemaActionType::DropTable;
diff.schema_id = table->database_id;
diff.table_id = table->id();
diff.version = version;
version_diff[version] = diff;
}

void MockTiDB::dropTableById(Context & context, const TableID & table_id, bool drop_regions)
{
std::lock_guard lock(tables_mutex);

auto table = dropTableByIdImpl(context, table_id, drop_regions);
if (!table)
return;

Expand Down Expand Up @@ -273,13 +317,15 @@ TableID MockTiDB::newTable(
return addTable(database_name, std::move(*table_info));
}

int MockTiDB::newTables(
std::vector<TableID> MockTiDB::newTables(
const String & database_name,
const std::vector<std::tuple<String, ColumnsDescription, String>> & tables,
Timestamp tso,
const String & engine_type)
{
std::lock_guard lock(tables_mutex);
std::scoped_lock lock(tables_mutex);
std::vector<TableID> table_ids;
table_ids.reserve(tables.size());
if (databases.find(database_name) == databases.end())
{
throw Exception("MockTiDB not found db: " + database_name, ErrorCodes::LOGICAL_ERROR);
Expand All @@ -300,7 +346,8 @@ int MockTiDB::newTables(
table_info.id = table_id_allocator++;
table_info.update_timestamp = tso;

auto table = std::make_shared<Table>(database_name, databases[database_name], table_info.name, std::move(table_info));
auto table
= std::make_shared<Table>(database_name, databases[database_name], table_info.name, std::move(table_info));
tables_by_id.emplace(table->table_info.id, table);
tables_by_name.emplace(qualified_name, table);

Expand All @@ -310,6 +357,8 @@ int MockTiDB::newTables(
opt.old_schema_id = table->database_id;
opt.old_table_id = table->id();
diff.affected_opts.push_back(std::move(opt));

table_ids.push_back(table->id());
}

if (diff.affected_opts.empty())
Expand All @@ -318,7 +367,8 @@ int MockTiDB::newTables(
diff.schema_id = diff.affected_opts[0].schema_id;
diff.version = version;
version_diff[version] = diff;
return 0;

return table_ids;
}

TableID MockTiDB::addTable(const String & database_name, TiDB::TableInfo && table_info)
Expand Down
8 changes: 6 additions & 2 deletions dbms/src/Debug/MockTiDB.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#pragma once

#include <Storages/ColumnsDescription.h>
#include <Storages/IStorage.h>
#include <Storages/Transaction/TiDB.h>
#include <Storages/Transaction/Types.h>
#include <TiDB/Schema/SchemaGetter.h>
Expand Down Expand Up @@ -82,7 +83,7 @@ class MockTiDB : public ext::Singleton<MockTiDB>
const String & handle_pk_name,
const String & engine_type);

int newTables(
std::vector<TableID> newTables(
const String & database_name,
const std::vector<std::tuple<String, ColumnsDescription, String>> & tables,
Timestamp tso,
Expand All @@ -104,6 +105,7 @@ class MockTiDB : public ext::Singleton<MockTiDB>
void dropPartition(const String & database_name, const String & table_name, TableID partition_id);

void dropTable(Context & context, const String & database_name, const String & table_name, bool drop_regions);
void dropTableById(Context & context, const TableID & table_id, bool drop_regions);

void dropDB(Context & context, const String & database_name, bool drop_regions);

Expand Down Expand Up @@ -151,7 +153,9 @@ class MockTiDB : public ext::Singleton<MockTiDB>

private:
TableID newPartitionImpl(const TablePtr & logical_table, TableID partition_id, const String & partition_name, Timestamp tso, bool is_add_part);
TablePtr dropTableInternal(Context & context, const String & database_name, const String & table_name, bool drop_regions);
TablePtr dropTableByNameImpl(Context & context, const String & database_name, const String & table_name, bool drop_regions);
TablePtr dropTableByIdImpl(Context & context, TableID table_id, bool drop_regions);
TablePtr dropTableInternal(Context & context, const TablePtr & table, bool drop_regions);
TablePtr getTableByNameInternal(const String & database_name, const String & table_name);
TablePtr getTableByID(TableID table_id);

Expand Down
12 changes: 8 additions & 4 deletions dbms/src/Debug/dbgFuncSchema.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,16 +86,20 @@ void dbgFuncRefreshSchemas(Context & context, const ASTs &, DBGInvoker::Printer

// Trigger gc on all databases / tables.
// Usage:
// ./storage-client.sh "DBGInvoke gc_schemas([gc_safe_point])"
// ./storage-client.sh "DBGInvoke gc_schemas([gc_safe_point, ignore_remain_regions])"
void dbgFuncGcSchemas(Context & context, const ASTs & args, DBGInvoker::Printer output)
{
auto & service = context.getSchemaSyncService();
Timestamp gc_safe_point = 0;
bool ignore_remain_regions = false;
if (args.empty())
gc_safe_point = PDClientHelper::getGCSafePointWithRetry(context.getTMTContext().getPDClient());
else
if (!args.empty())
gc_safe_point = safeGet<Timestamp>(typeid_cast<const ASTLiteral &>(*args[0]).value);
service->gc(gc_safe_point, NullspaceID);
if (args.size() >= 2)
ignore_remain_regions = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[1]).value) == "true";
// Note that only call it in tests, we need to ignore remain regions
service->gcImpl(gc_safe_point, NullspaceID, ignore_remain_regions);

output("schemas gc done");
}
Expand Down Expand Up @@ -139,4 +143,4 @@ void dbgFuncIsTombstone(Context & context, const ASTs & args, DBGInvoker::Printe
}


} // namespace DB
} // namespace DB
2 changes: 1 addition & 1 deletion dbms/src/Debug/dbgFuncSchema.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ void dbgFuncRefreshSchemas(Context & context, const ASTs & args, DBGInvoker::Pri

// Trigger gc on all databases / tables.
// Usage:
// ./storage-client.sh "DBGInvoke gc_schemas([gc_safe_point])"
// ./storage-client.sh "DBGInvoke gc_schemas([gc_safe_point, ignore_remain_regions])"
void dbgFuncGcSchemas(Context & context, const ASTs & args, DBGInvoker::Printer output);

// Reset schemas.
Expand Down
16 changes: 8 additions & 8 deletions dbms/src/Flash/Coprocessor/DAGUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,23 +140,23 @@ const std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
{tipb::ScalarFuncSig::LTDecimal, "less"},
{tipb::ScalarFuncSig::LTTime, "less"},
{tipb::ScalarFuncSig::LTDuration, "less"},
{tipb::ScalarFuncSig::LTJson, "less"},
//{tipb::ScalarFuncSig::LTJson, "less"},

{tipb::ScalarFuncSig::LEInt, "lessOrEquals"},
{tipb::ScalarFuncSig::LEReal, "lessOrEquals"},
{tipb::ScalarFuncSig::LEString, "lessOrEquals"},
{tipb::ScalarFuncSig::LEDecimal, "lessOrEquals"},
{tipb::ScalarFuncSig::LETime, "lessOrEquals"},
{tipb::ScalarFuncSig::LEDuration, "lessOrEquals"},
{tipb::ScalarFuncSig::LEJson, "lessOrEquals"},
//{tipb::ScalarFuncSig::LEJson, "lessOrEquals"},

{tipb::ScalarFuncSig::GTInt, "greater"},
{tipb::ScalarFuncSig::GTReal, "greater"},
{tipb::ScalarFuncSig::GTString, "greater"},
{tipb::ScalarFuncSig::GTDecimal, "greater"},
{tipb::ScalarFuncSig::GTTime, "greater"},
{tipb::ScalarFuncSig::GTDuration, "greater"},
{tipb::ScalarFuncSig::GTJson, "greater"},
//{tipb::ScalarFuncSig::GTJson, "greater"},

{tipb::ScalarFuncSig::GreatestInt, "tidbGreatest"},
{tipb::ScalarFuncSig::GreatestReal, "tidbGreatest"},
Expand All @@ -179,23 +179,23 @@ const std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
{tipb::ScalarFuncSig::GEDecimal, "greaterOrEquals"},
{tipb::ScalarFuncSig::GETime, "greaterOrEquals"},
{tipb::ScalarFuncSig::GEDuration, "greaterOrEquals"},
{tipb::ScalarFuncSig::GEJson, "greaterOrEquals"},
//{tipb::ScalarFuncSig::GEJson, "greaterOrEquals"},

{tipb::ScalarFuncSig::EQInt, "equals"},
{tipb::ScalarFuncSig::EQReal, "equals"},
{tipb::ScalarFuncSig::EQString, "equals"},
{tipb::ScalarFuncSig::EQDecimal, "equals"},
{tipb::ScalarFuncSig::EQTime, "equals"},
{tipb::ScalarFuncSig::EQDuration, "equals"},
{tipb::ScalarFuncSig::EQJson, "equals"},
//{tipb::ScalarFuncSig::EQJson, "equals"},

{tipb::ScalarFuncSig::NEInt, "notEquals"},
{tipb::ScalarFuncSig::NEReal, "notEquals"},
{tipb::ScalarFuncSig::NEString, "notEquals"},
{tipb::ScalarFuncSig::NEDecimal, "notEquals"},
{tipb::ScalarFuncSig::NETime, "notEquals"},
{tipb::ScalarFuncSig::NEDuration, "notEquals"},
{tipb::ScalarFuncSig::NEJson, "notEquals"},
//{tipb::ScalarFuncSig::NEJson, "notEquals"},

//{tipb::ScalarFuncSig::NullEQInt, "cast"},
//{tipb::ScalarFuncSig::NullEQReal, "cast"},
Expand Down Expand Up @@ -312,7 +312,7 @@ const std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
{tipb::ScalarFuncSig::StringIsNull, "isNull"},
{tipb::ScalarFuncSig::TimeIsNull, "isNull"},
{tipb::ScalarFuncSig::IntIsNull, "isNull"},
{tipb::ScalarFuncSig::JsonIsNull, "isNull"},
//{tipb::ScalarFuncSig::JsonIsNull, "isNull"},

{tipb::ScalarFuncSig::BitAndSig, "bitAnd"},
{tipb::ScalarFuncSig::BitOrSig, "bitOr"},
Expand Down Expand Up @@ -353,7 +353,7 @@ const std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
{tipb::ScalarFuncSig::InDecimal, "tidbIn"},
{tipb::ScalarFuncSig::InTime, "tidbIn"},
{tipb::ScalarFuncSig::InDuration, "tidbIn"},
{tipb::ScalarFuncSig::InJson, "tidbIn"},
//{tipb::ScalarFuncSig::InJson, "tidbIn"},

{tipb::ScalarFuncSig::IfNullInt, "ifNull"},
{tipb::ScalarFuncSig::IfNullReal, "ifNull"},
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Functions/FunctionsNull.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ class FunctionCoalesce : public IFunction
public:
static constexpr auto name = "coalesce";
static FunctionPtr create(const Context & context);
FunctionCoalesce(const Context & context)
explicit FunctionCoalesce(const Context & context)
: context(context)
{}

Expand Down
Loading

0 comments on commit 233e882

Please sign in to comment.