external table content has been decoded for viewer

Author: dorooleg

ydb/core/base/appdata.h

@@ -11,6 +11,7 @@
 #include "event_filter.h"
 
 #include <ydb/core/control/immediate_control_board_impl.h>
+#include <ydb/core/external_sources/external_source_factory.h>
 #include <ydb/core/grpc_services/grpc_helper.h>
 #include <ydb/core/protos/auth.pb.h>
 #include <ydb/core/protos/cms.pb.h>

ydb/core/base/appdata_fwd.h

@@ -102,6 +102,10 @@ namespace NYdb {
 
 namespace NKikimr {
 
+namespace NExternalSource {
+    struct IExternalSourceFactory;
+}
+
 namespace NScheme {
     class TTypeRegistry;
 }
@@ -148,6 +152,8 @@ struct TAppData {
     const NDataShard::IExportFactory *DataShardExportFactory = nullptr;
     const TFormatFactory* FormatFactory = nullptr;
     const NSQS::IEventsWriterFactory* SqsEventsWriterFactory = nullptr;
+    TIntrusivePtr<NExternalSource::IExternalSourceFactory> ExternalSourceFactory = nullptr;
+    
 
     NSQS::IAuthFactory* SqsAuthFactory = nullptr;
 

ydb/core/driver_lib/run/run.cpp

@@ -1004,6 +1004,13 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig)
     AppData->PersQueueGetReadSessionsInfoWorkerFactory = ModuleFactories ? ModuleFactories->PQReadSessionsInfoWorkerFactory.get() : nullptr;
     AppData->IoContextFactory = ModuleFactories ? ModuleFactories->IoContextFactory.get() : nullptr;
 
+    std::vector<TString> hostnamePatterns;
+    if (runConfig.AppConfig.HasQueryServiceConfig()) {
+        const auto& patterns = runConfig.AppConfig.GetQueryServiceConfig().GetHostnamePatterns();
+        hostnamePatterns = {patterns.begin(), patterns.end()};
+    }
+    AppData->ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory(hostnamePatterns);
+
     AppData->SqsAuthFactory = ModuleFactories
         ? ModuleFactories->SqsAuthFactory.get()
         : nullptr;

ydb/core/kqp/host/kqp_host.cpp

@@ -1517,8 +1517,8 @@ class TKqpHost : public IKqpHost {
 
         auto queryExecutor = MakeIntrusive<TKqpQueryExecutor>(Gateway, Cluster, SessionCtx, KqpRunner);
         auto kikimrDataSource = CreateKikimrDataSource(*FuncRegistry, *TypesCtx, gatewayProxy, SessionCtx,
-            ExternalSourceFactory, IsInternalCall);
-        auto kikimrDataSink = CreateKikimrDataSink(*FuncRegistry, *TypesCtx, gatewayProxy, SessionCtx, ExternalSourceFactory, queryExecutor);
+            AppData()->ExternalSourceFactory, IsInternalCall);
+        auto kikimrDataSink = CreateKikimrDataSink(*FuncRegistry, *TypesCtx, gatewayProxy, SessionCtx, AppData()->ExternalSourceFactory, queryExecutor);
 
         FillSettings.AllResultsBytesLimit = Nothing();
         FillSettings.RowsLimitPerWrite = SessionCtx->Config()._ResultRowsLimit.Get();
@@ -1649,7 +1649,6 @@ class TKqpHost : public IKqpHost {
 
     TIntrusivePtr<TExecuteContext> ExecuteCtx;
     TIntrusivePtr<IKqpRunner> KqpRunner;
-    NExternalSource::IExternalSourceFactory::TPtr ExternalSourceFactory{NExternalSource::CreateExternalSourceFactory({})};
 
     TKqpTempTablesState::TConstPtr TempTablesState;
     NActors::TActorSystem* ActorSystem = nullptr;

ydb/core/tx/schemeshard/schemeshard_impl.cpp

@@ -4271,6 +4271,7 @@ void TSchemeShard::OnActivateExecutor(const TActorContext &ctx) {
     EnableStatistics = appData->FeatureFlags.GetEnableStatistics();
     EnableTablePgTypes = appData->FeatureFlags.GetEnableTablePgTypes();
     EnableServerlessExclusiveDynamicNodes = appData->FeatureFlags.GetEnableServerlessExclusiveDynamicNodes();
+    ExternalSourceFactory = appData->ExternalSourceFactory;
 
     ConfigureCompactionQueues(appData->CompactionConfig, ctx);
     ConfigureStatsBatching(appData->SchemeShardConfig, ctx);
@@ -6719,11 +6720,6 @@ void TSchemeShard::ApplyConsoleConfigs(const NKikimrConfig::TAppConfig& appConfi
         LoadTableProfiles(nullptr, ctx);
     }
 
-    if (appConfig.HasQueryServiceConfig()) {
-        const auto& hostnamePatterns = appConfig.GetQueryServiceConfig().GetHostnamePatterns();
-        ExternalSourceFactory = NExternalSource::CreateExternalSourceFactory(std::vector<TString>(hostnamePatterns.begin(), hostnamePatterns.end()));
-    }
-
     if (IsSchemeShardConfigured()) {
         StartStopCompactionQueues();
     }

ydb/core/tx/schemeshard/schemeshard_impl.h

@@ -310,7 +310,7 @@ class TSchemeShard
     TActorId DelayedInitTenantDestination;
     TAutoPtr<TEvSchemeShard::TEvInitTenantSchemeShardResult> DelayedInitTenantReply;
 
-    NExternalSource::IExternalSourceFactory::TPtr ExternalSourceFactory{NExternalSource::CreateExternalSourceFactory({})};
+    NExternalSource::IExternalSourceFactory::TPtr ExternalSourceFactory;
 
     THolder<TProposeResponse> IgniteOperation(TProposeRequest& request, TOperationContext& context);
     THolder<TEvDataShard::TEvProposeTransaction> MakeDataShardProposal(const TPathId& pathId, const TOperationId& opId,

ydb/core/viewer/json_describe.h

@@ -268,6 +268,7 @@ class TJsonDescribe : public TViewerPipeClient<TJsonDescribe> {
                 headers = HTTPFORBIDDENJSON;
             }
             TProtoToJson::ProtoToJson(json, *DescribeResult, JsonSettings);
+            DecodeExternalTableContent(json);
         } else {
             json << "null";
         }
@@ -276,6 +277,41 @@ class TJsonDescribe : public TViewerPipeClient<TJsonDescribe> {
         PassAway();
     }
 
+    void DecodeExternalTableContent(TStringStream& json) const {
+        if (!DescribeResult) {
+            return;
+        }
+
+        if (!DescribeResult->GetPathDescription().HasExternalTableDescription()) {
+            return;
+        }
+
+        const auto& content = DescribeResult->GetPathDescription().GetExternalTableDescription().GetContent();
+        if (!content) {
+            return;
+        }
+
+        NJson::TJsonValue root;
+        const auto& sourceType = DescribeResult->GetPathDescription().GetExternalTableDescription().GetSourceType();
+        try {
+            NJson::ReadJsonTree(json.Str(), &root);
+            root["PathDescription"]["ExternalTableDescription"].EraseValue("Content");
+            auto source = AppData()->ExternalSourceFactory->GetOrCreate(sourceType);
+            auto parameters = source->GetParameters(content);
+            for (const auto& [key, items]: parameters) {
+                NJson::TJsonValue array{NJson::EJsonValueType::JSON_ARRAY};
+                for (const auto& item: items) {
+                    array.AppendValue(item);
+                }
+                root["PathDescription"]["ExternalTableDescription"]["Content"][key] = array;
+            }
+        } catch (...) {
+            BLOG_CRIT("Сan't unpack content for external table: " << sourceType << ", error: " << CurrentExceptionMessage());
+        }
+        json.Clear();
+        json << root;
+    }
+
     void HandleTimeout() {
         Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPGATEWAYTIMEOUT(Event->Get()), 0, NMon::IEvHttpInfoRes::EContentType::Custom));
         PassAway();