Merge #9778 #10161 (#10202)

Author: vitalyisaev2

ydb/core/kqp/ut/federated_query/generic_ut/kqp_generic_provider_ut.cpp

@@ -125,6 +125,7 @@ namespace NKikimr::NKqp {
             // step 3: ReadSplits
             std::vector<ui16> colData = {10, 20, 30, 40, 50};
             clientMock->ExpectReadSplits()
+                .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
                 .Split()
                     .Description("some binary description")
                     .Select()
@@ -221,6 +222,7 @@ namespace NKikimr::NKqp {
 
             // step 3: ReadSplits
             clientMock->ExpectReadSplits()
+                .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
                 .Split()
                     .Description("some binary description")
                     .Select()
@@ -313,6 +315,7 @@ namespace NKikimr::NKqp {
 
             // step 3: ReadSplits
             clientMock->ExpectReadSplits()
+                .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
                 .Split()
                     .Description("some binary description")
                     .Select()
@@ -418,6 +421,7 @@ namespace NKikimr::NKqp {
             std::vector<i32> filterColumnData = {42, 24};
             // clang-format off
             clientMock->ExpectReadSplits()
+                .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL)
                 .Split()
                     .Description("some binary description")
                     .Select(select)

ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp

@@ -128,6 +128,7 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) {
 
         connectorMock->ExpectReadSplits()
             .DataSourceInstance(dsi)
+            .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY)
             .Split()
                 .Description("Actual split info is not important")
                 .Done()

ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp

@@ -162,6 +162,7 @@ namespace NYql::NDq {
 
             *readRequest.add_splits() = split;
             readRequest.Setformat(NConnector::NApi::TReadSplitsRequest_EFormat::TReadSplitsRequest_EFormat_ARROW_IPC_STREAMING);
+            readRequest.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY);
             Connector->ReadSplits(readRequest).Subscribe([actorSystem = TActivationContext::ActorSystem(), selfId = SelfId()](const NConnector::TReadSplitsStreamIteratorAsyncResult& asyncResult) {
                 YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << selfId << " Got ReadSplitsStreamIterator from Connector";
                 auto result = ExtractFromConstFuture(asyncResult);

ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp

@@ -199,6 +199,7 @@ namespace NYql::NDq {
             // Prepare request
             NConnector::NApi::TReadSplitsRequest request;
             request.set_format(NConnector::NApi::TReadSplitsRequest::ARROW_IPC_STREAMING);
+            request.set_filtering(NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL);
             request.mutable_splits()->Reserve(Splits_.size());
 
             for (const auto& split : Splits_) {

ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto

@@ -235,6 +235,22 @@ message TReadSplitsRequest {
     // If empty, the connector will return the split data from the very beginning.
     TContinuation continuation = 6;
 
+    enum EFiltering {
+        FILTERING_UNSPECIFIED = 0;
+        // If Connector cannot push down the predicate to the data source completely
+        // (due to the lack of data type support, for example), it doesn't apply filter at all
+        // and returns the full result of `SELECT columns FROM table` (no WHERE clause).
+        // It's YDB's duty to filter the output on its own side. 
+        FILTERING_OPTIONAL = 1;
+        // If Connector cannot push down the predicate to the data source completely,
+        // it terminates the request and returns an error.
+        FILTERING_MANDATORY = 2;
+    }
+
+    // Determines various modes of server behavior in the context of predicate pushdown.
+    // If not set, the default value is `FILTERING_OPTIONAL`. 
+    EFiltering filtering = 7;
+
     reserved 5;
 }
 

ydb/library/yql/providers/generic/connector/libcpp/ut_helpers/connector_client_mock.h

@@ -745,6 +745,7 @@ namespace NYql::NConnector::NTest {
             DATA_SOURCE_INSTANCE_SUBBUILDER();
             SUBPROTO_BUILDER(Split, add_splits, NApi::TSplit, TSplitBuilder<TBuilder>);
             SETTER(Format, format);
+            SETTER(Filtering, filtering);
 
             TReadSplitsResultBuilder<TBuilder> Result() {
                 return TReadSplitsResultBuilder<TBuilder>(ResponseResult_, this);

ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml

@@ -12,7 +12,7 @@ services:
     - 8123
   fq-connector-go:
     container_name: fq-tests-ch-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
     - 2130
     volumes:

ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   fq-connector-go:
     container_name: fq-tests-pg-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
     - 2130
     volumes:

ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml

@@ -5,7 +5,7 @@ services:
         echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts;
         /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
     container_name: fq-tests-ydb-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
     - 2130
     volumes:

ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml

@@ -12,7 +12,7 @@ services:
     - 8123
   fq-connector-go:
     container_name: fq-tests-join-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.7-rc.1@sha256:f12475f346105d7bc630e7b85f51dce980bf9833f6ce0625c6f1191b1a1de923
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.5.11-rc.1@sha256:c018b2f1151fac8a86aaf79950ccdffaa72150785f303096d9a466f50eb62498
     ports:
     - 2130
     volumes: