[yugabyte#15767] DocDB: Revert vtables lib as it results in circular …

…dependencies between master and vtables lib

Summary: This change reverts commit e8ce3d7 as this change resulted in circular dependencies between vtables and master lib. Circular dependency is only caught in `no-tcmalloc` build for which we don't allow any undefined symbols.

Test Plan: Jenkins

Reviewers: dfelsing, zdrudi, skedia, sergei, rthallam

Reviewed By: rthallam

Subscribers: sergei, skedia, rthallam, bogdan

Differential Revision: https://phabricator.dev.yugabyte.com/D22414

CMakeLists.txt

@@ -808,7 +808,6 @@ add_subdirectory(src/yb/rocksutil)
 add_subdirectory(src/yb/docdb)
 add_subdirectory(src/yb/yql)
 add_subdirectory(src/yb/cdc)
-add_subdirectory(src/yb/vtables)
 
 include("${YB_SRC_ROOT}/ent/CMakeLists.txt")
 

src/yb/master/CMakeLists.txt

@@ -115,30 +115,49 @@ set(MASTER_SRCS
   tasks_tracker.cc
   ts_descriptor.cc
   ts_manager.cc
+  util/yql_vtable_helpers.cc
   xcluster/xcluster_consumer_metrics.cc
   xcluster/xcluster_safe_time_service.cc
+  yql_aggregates_vtable.cc
+  yql_auth_resource_role_permissions_index.cc
+  yql_auth_role_permissions_vtable.cc
+  yql_auth_roles_vtable.cc
+  yql_columns_vtable.cc
+  yql_empty_vtable.cc
+  yql_functions_vtable.cc
+  yql_indexes_vtable.cc
+  yql_keyspaces_vtable.cc
+  yql_local_vtable.cc
+  yql_partitions_vtable.cc
+  yql_peers_vtable.cc
+  yql_size_estimates_vtable.cc
+  yql_tables_vtable.cc
+  yql_triggers_vtable.cc
+  yql_types_vtable.cc
+  yql_views_vtable.cc
+  yql_virtual_table.cc
+  yql_vtable_iterator.cc
   ysql_tablegroup_manager.cc
   ysql_tablespace_manager.cc
   ysql_transaction_ddl.cc
   ${MASTER_SRCS_EXTENSIONS})
 
 set(MASTER_DEPS
-  ${MASTER_PROTO_LIBS}
-  gutil
-  master_util
-  vtables
-  rpc_header_proto
+  yb_common
+  tablet
   server_common
   server_process
-  tablet
+  yrpc
+  gutil
+  yb_util
   tserver
   tserver_service_proto
+  ${MASTER_PROTO_LIBS}
+  rpc_header_proto
+  master_util
   version_info_proto
-  yb_common
-  yb_pggate
   yb_pggate_flags
-  yb_util
-  yrpc
+  yb_pggate
 )
 
 ADD_YB_LIBRARY(master

src/yb/master/catalog_manager.cc

@@ -109,8 +109,8 @@
 #include "yb/gutil/walltime.h"
 
 #include "yb/master/master_fwd.h"
-#include "yb/master/async_rpc_tasks.h"
 #include "yb/master/auto_flags_orchestrator.h"
+#include "yb/master/async_rpc_tasks.h"
 #include "yb/master/backfill_index.h"
 #include "yb/master/catalog_entity_info.h"
 #include "yb/master/catalog_loaders.h"
@@ -136,6 +136,23 @@
 #include "yb/master/sys_catalog_constants.h"
 #include "yb/master/ts_descriptor.h"
 #include "yb/master/xcluster/xcluster_safe_time_service.h"
+#include "yb/master/yql_aggregates_vtable.h"
+#include "yb/master/yql_auth_resource_role_permissions_index.h"
+#include "yb/master/yql_auth_role_permissions_vtable.h"
+#include "yb/master/yql_auth_roles_vtable.h"
+#include "yb/master/yql_columns_vtable.h"
+#include "yb/master/yql_empty_vtable.h"
+#include "yb/master/yql_functions_vtable.h"
+#include "yb/master/yql_indexes_vtable.h"
+#include "yb/master/yql_keyspaces_vtable.h"
+#include "yb/master/yql_local_vtable.h"
+#include "yb/master/yql_partitions_vtable.h"
+#include "yb/master/yql_peers_vtable.h"
+#include "yb/master/yql_size_estimates_vtable.h"
+#include "yb/master/yql_tables_vtable.h"
+#include "yb/master/yql_triggers_vtable.h"
+#include "yb/master/yql_types_vtable.h"
+#include "yb/master/yql_views_vtable.h"
 #include "yb/master/ysql_tablegroup_manager.h"
 #include "yb/master/ysql_transaction_ddl.h"
 
@@ -184,8 +201,6 @@
 #include "yb/util/tsan_util.h"
 #include "yb/util/uuid.h"
 
-#include "yb/vtables/yql_all_vtables.h"
-
 #include "yb/yql/pgwrapper/pg_wrapper.h"
 #include "yb/yql/redis/redisserver/redis_constants.h"
 
@@ -1543,45 +1558,45 @@ Status CatalogManager::PrepareSystemTables(int64_t term) {
   RETURN_NOT_OK(PrepareSysCatalogTable(term));
 
   // Create the required system tables here.
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::PeersVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<PeersVTable>(
       kSystemPeersTableName, kSystemNamespaceName, kSystemNamespaceId, term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::LocalVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<LocalVTable>(
       kSystemLocalTableName, kSystemNamespaceName, kSystemNamespaceId, term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLKeyspacesVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLKeyspacesVTable>(
       kSystemSchemaKeyspacesTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId,
       term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLTablesVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLTablesVTable>(
       kSystemSchemaTablesTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId, term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLColumnsVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLColumnsVTable>(
       kSystemSchemaColumnsTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId, term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLSizeEstimatesVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLSizeEstimatesVTable>(
       kSystemSizeEstimatesTableName, kSystemNamespaceName, kSystemNamespaceId, term)));
 
   // Empty tables.
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLAggregatesVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLAggregatesVTable>(
       kSystemSchemaAggregatesTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId,
       term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLFunctionsVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLFunctionsVTable>(
       kSystemSchemaFunctionsTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId,
       term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLIndexesVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLIndexesVTable>(
       kSystemSchemaIndexesTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId, term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLTriggersVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLTriggersVTable>(
       kSystemSchemaTriggersTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId, term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLViewsVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLViewsVTable>(
       kSystemSchemaViewsTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId, term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::QLTypesVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<QLTypesVTable>(
       kSystemSchemaTypesTableName, kSystemSchemaNamespaceName, kSystemSchemaNamespaceId, term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLPartitionsVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLPartitionsVTable>(
       kSystemPartitionsTableName, kSystemNamespaceName, kSystemNamespaceId, term)));
 
   // System auth tables.
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLAuthRolesVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLAuthRolesVTable>(
       kSystemAuthRolesTableName, kSystemAuthNamespaceName, kSystemAuthNamespaceId, term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLAuthRolePermissionsVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLAuthRolePermissionsVTable>(
       kSystemAuthRolePermissionsTableName, kSystemAuthNamespaceName, kSystemAuthNamespaceId,
       term)));
-  RETURN_NOT_OK((PrepareSystemTableTemplate<yb::vtables::YQLAuthResourceRolePermissionsIndexVTable>(
+  RETURN_NOT_OK((PrepareSystemTableTemplate<YQLAuthResourceRolePermissionsIndexVTable>(
       kSystemAuthResourceRolePermissionsIndexTableName, kSystemAuthNamespaceName,
       kSystemAuthNamespaceId, term)));
 
@@ -1659,7 +1674,7 @@ Status CatalogManager::PrepareSystemTableTemplate(const TableName& table_name,
                                                   const NamespaceName& namespace_name,
                                                   const NamespaceId& namespace_id,
                                                   int64_t term) {
-  yb::vtables::YQLVirtualTable* vtable = new T(table_name, namespace_name, master_);
+  YQLVirtualTable* vtable = new T(table_name, namespace_name, master_);
   return PrepareSystemTable(
       table_name, namespace_name, namespace_id, vtable->schema(), term, vtable);
 }
@@ -1669,8 +1684,8 @@ Status CatalogManager::PrepareSystemTable(const TableName& table_name,
                                           const NamespaceId& namespace_id,
                                           const Schema& schema,
                                           int64_t term,
-                                          yb::vtables::YQLVirtualTable* vtable) {
-  std::unique_ptr<yb::vtables::YQLVirtualTable> yql_storage(vtable);
+                                          YQLVirtualTable* vtable) {
+  std::unique_ptr<YQLVirtualTable> yql_storage(vtable);
 
   scoped_refptr<TableInfo> table = FindPtrOrNull(table_names_map_,
                                                  std::make_pair(namespace_id, table_name));
@@ -4831,7 +4846,7 @@ Result<bool> CatalogManager::IsCreateTableDone(const TableInfoPtr& table) {
   if (DCHECK_IS_ON() &&
       result &&
       IsYcqlTable(*table) &&
-      yb::vtables::YQLPartitionsVTable::GeneratePartitionsVTableOnChanges() &&
+      YQLPartitionsVTable::GeneratePartitionsVTableOnChanges() &&
       FLAGS_TEST_catalog_manager_check_yql_partitions_exist_for_is_create_table_done) {
     Schema schema;
     RETURN_NOT_OK(table->GetSchema(&schema));
@@ -12141,13 +12156,13 @@ Status CatalogManager::GetYQLPartitionsVTable(std::shared_ptr<SystemTablet>* tab
 }
 
 void CatalogManager::RebuildYQLSystemPartitions() {
-  if (yb::vtables::YQLPartitionsVTable::GeneratePartitionsVTableWithBgTask() ||
-      yb::vtables::YQLPartitionsVTable::GeneratePartitionsVTableOnChanges()) {
+  if (YQLPartitionsVTable::GeneratePartitionsVTableWithBgTask() ||
+      YQLPartitionsVTable::GeneratePartitionsVTableOnChanges()) {
     SCOPED_LEADER_SHARED_LOCK(l, this);
     if (l.IsInitializedAndIsLeader()) {
       if (system_partitions_tablet_ != nullptr) {
         Status s;
-        if (yb::vtables::YQLPartitionsVTable::GeneratePartitionsVTableWithBgTask()) {
+        if (YQLPartitionsVTable::GeneratePartitionsVTableWithBgTask()) {
           // If we are not generating the vtable on changes, then we need to do a full refresh.
           s = ResultToStatus(GetYqlPartitionsVtable().GenerateAndCacheData());
         } else {
@@ -12279,8 +12294,8 @@ void CatalogManager::CheckTableDeleted(const TableInfoPtr& table) {
   }), "Failed to submit update table task");
 }
 
-const yb::vtables::YQLPartitionsVTable& CatalogManager::GetYqlPartitionsVtable() const {
-  return down_cast<const yb::vtables::YQLPartitionsVTable&>(system_partitions_tablet_->QLStorage());
+const YQLPartitionsVTable& CatalogManager::GetYqlPartitionsVtable() const {
+  return down_cast<const YQLPartitionsVTable&>(system_partitions_tablet_->QLStorage());
 }
 
 void CatalogManager::InitializeTableLoadState(

src/yb/master/catalog_manager.h

@@ -1114,7 +1114,7 @@ class CatalogManager : public tserver::TabletPeerLookupIf,
                             const NamespaceId& namespace_id,
                             const Schema& schema,
                             int64_t term,
-                            yb::vtables::YQLVirtualTable* vtable) REQUIRES(mutex_);
+                            YQLVirtualTable* vtable) REQUIRES(mutex_);
 
   Status PrepareNamespace(YQLDatabase db_type,
                           const NamespaceName& name,
@@ -1983,7 +1983,7 @@ class CatalogManager : public tserver::TabletPeerLookupIf,
 
   TSDescriptorVector GetAllLiveNotBlacklistedTServers() const;
 
-  const yb::vtables::YQLPartitionsVTable& GetYqlPartitionsVtable() const;
+  const YQLPartitionsVTable& GetYqlPartitionsVtable() const;
 
   void InitializeTableLoadState(
       const TableId& table_id, TSDescriptorVector ts_descs, CMPerTableLoadState* state);

src/yb/master/master_fwd.h

@@ -39,13 +39,6 @@ namespace yb {
 class HostPort;
 struct HostPortHash;
 
-namespace vtables {
-
-class YQLPartitionsVTable;
-class YQLVirtualTable;
-
-}  // namespace vtables
-
 namespace master {
 
 class TSDescriptor;
@@ -87,6 +80,8 @@ class TSDescriptor;
 class TSManager;
 class UDTypeInfo;
 class XClusterSafeTimeService;
+class YQLPartitionsVTable;
+class YQLVirtualTable;
 class YsqlTablegroupManager;
 class YsqlTablespaceManager;
 class YsqlTransactionDdl;

src/yb/master/system_tablet.cc

@@ -20,20 +20,19 @@
 #include "yb/docdb/doc_read_context.h"
 
 #include "yb/master/sys_catalog_constants.h"
-
-#include "yb/vtables/yql_virtual_table.h"
+#include "yb/master/yql_virtual_table.h"
 
 namespace yb {
 namespace master {
 
-SystemTablet::SystemTablet(
-    const Schema& schema, std::unique_ptr<yb::vtables::YQLVirtualTable> yql_virtual_table,
-    const TabletId& tablet_id)
-    : log_prefix_(Format("T $0: ", tablet_id)),  // Don't have UUID here to log in T XX P YY format.
-      doc_read_context_(
-          std::make_shared<docdb::DocReadContext>(log_prefix_, schema, kSysCatalogSchemaVersion)),
+SystemTablet::SystemTablet(const Schema& schema, std::unique_ptr<YQLVirtualTable> yql_virtual_table,
+                           const TabletId& tablet_id)
+    : log_prefix_(Format("T $0: ", tablet_id)), // Don't have UUID here to log in T XX P YY format.
+      doc_read_context_(std::make_shared<docdb::DocReadContext>(
+          log_prefix_, schema, kSysCatalogSchemaVersion)),
       yql_virtual_table_(std::move(yql_virtual_table)),
-      tablet_id_(tablet_id) {}
+      tablet_id_(tablet_id) {
+}
 
 docdb::DocReadContextPtr SystemTablet::GetDocReadContext(const std::string& table_id) const {
   // table_id is ignored. It should match the system table's id.

src/yb/master/system_tablet.h

@@ -25,9 +25,8 @@ namespace master {
 // This is a virtual tablet that is used for our virtual tables in the system namespace.
 class SystemTablet : public tablet::AbstractTablet {
  public:
-  SystemTablet(
-      const Schema& schema, std::unique_ptr<yb::vtables::YQLVirtualTable> yql_virtual_table,
-      const TabletId& tablet_id);
+  SystemTablet(const Schema& schema, std::unique_ptr<YQLVirtualTable> yql_virtual_table,
+               const TabletId& tablet_id);
 
   docdb::DocReadContextPtr GetDocReadContext(const std::string& table_id = "") const override;
 
@@ -100,7 +99,7 @@ class SystemTablet : public tablet::AbstractTablet {
 
   const std::string log_prefix_;
   docdb::DocReadContextPtr doc_read_context_;
-  std::unique_ptr<yb::vtables::YQLVirtualTable> yql_virtual_table_;
+  std::unique_ptr<YQLVirtualTable> yql_virtual_table_;
   TabletId tablet_id_;
 };
 

src/yb/vtables/yql_vtable_helpers.cc → src/yb/master/util/yql_vtable_helpers.cc

@@ -11,7 +11,7 @@
 // under the License.
 //
 
-#include "yb/vtables/yql_vtable_helpers.h"
+#include "yb/master/util/yql_vtable_helpers.h"
 
 #include <future>
 
@@ -28,7 +28,7 @@
 #include "yb/util/yb_partition.h"
 
 namespace yb {
-namespace vtables {
+namespace master {
 namespace util {
 
 // Ideally, we want clients to use YB's own load-balancing policy for Cassandra to route the
@@ -63,7 +63,7 @@ bool RemoteEndpointMatchesList(const google::protobuf::RepeatedPtrField<HostPort
   return false;
 }
 
-bool RemoteEndpointMatchesTServer(const yb::master::TSInformationPB& ts_info,
+bool RemoteEndpointMatchesTServer(const TSInformationPB& ts_info,
                                   const InetAddress& remote_endpoint) {
   const auto& common = ts_info.registration().common();
   if (RemoteEndpointMatchesList(common.private_rpc_addresses(), remote_endpoint)) {
@@ -95,7 +95,7 @@ QLValuePB GetReplicationValue(size_t replication_factor) {
 }
 
 PublicPrivateIPFutures GetPublicPrivateIPFutures(
-    const yb::master::TSInformationPB& ts_info, DnsResolver* resolver) {
+    const TSInformationPB& ts_info, DnsResolver* resolver) {
   const auto& common = ts_info.registration().common();
   PublicPrivateIPFutures result;
 
@@ -200,5 +200,5 @@ QLValuePB GetValueHelper<bool>::Apply(const bool bool_val, const DataType data_t
 }
 
 }  // namespace util
-}  // namespace vtables
+}  // namespace master
 }  // namespace yb

src/yb/vtables/yql_vtable_helpers.h → src/yb/master/util/yql_vtable_helpers.h

@@ -30,7 +30,7 @@
 #include "yb/util/uuid.h"
 
 namespace yb {
-namespace vtables {
+namespace master {
 namespace util {
 
 template<class T> struct GetValueHelper;
@@ -78,17 +78,17 @@ QLValuePB GetTokensValue(size_t index, size_t node_count);
 
 QLValuePB GetReplicationValue(size_t replication_factor);
 
-bool RemoteEndpointMatchesTServer(
-    const yb::master::TSInformationPB& ts_info, const InetAddress& remote_endpoint);
+bool RemoteEndpointMatchesTServer(const TSInformationPB& ts_info,
+                                  const InetAddress& remote_endpoint);
 
 struct PublicPrivateIPFutures {
   std::shared_future<Result<IpAddress>> private_ip_future;
   std::shared_future<Result<IpAddress>> public_ip_future;
 };
 
 PublicPrivateIPFutures GetPublicPrivateIPFutures(
-    const master::TSInformationPB& ts_info, DnsResolver* resolver);
+    const TSInformationPB& ts_info, DnsResolver* resolver);
 
 }  // namespace util
-}  // namespace vtables
+}  // namespace master
 }  // namespace yb