YDB FQ: integration tests for YDB as an external data source

ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml

@@ -6,13 +6,13 @@ services:
       CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
       CLICKHOUSE_PASSWORD: password
       CLICKHOUSE_USER: user
-    image: clickhouse/clickhouse-server:23-alpine@sha256:b078c1cd294632afa2aeba3530e7ba2e568513da23304354f455a25fab575c06
+    image: mirror.gcr.io/clickhouse/clickhouse-server:23-alpine@sha256:d75017307e76d1bca81a5ac7ada94620567782c0610541f525d1e443e23f76e3
     ports:
     - 9000
     - 8123
   fq-connector-go:
     container_name: fq-tests-ch-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.2.20@sha256:a1771f348dc8be6219865e332f788429907cdfec3677b3e98f0bc6f7dd542dc6
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.4.1@sha256:61ad8d7b7b981b9ef5709a0ce71d10702f49396bd307437c9efd876c6ad6e90e
     ports:
     - 2130
     volumes:

ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   fq-connector-go:
     container_name: fq-tests-pg-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.2.20@sha256:a1771f348dc8be6219865e332f788429907cdfec3677b3e98f0bc6f7dd542dc6
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.4.1@sha256:61ad8d7b7b981b9ef5709a0ce71d10702f49396bd307437c9efd876c6ad6e90e
     ports:
     - 2130
     volumes:
@@ -20,7 +20,7 @@ services:
       POSTGRES_DB: db
       POSTGRES_PASSWORD: password
       POSTGRES_USER: user
-    image: postgres:15-bullseye@sha256:3411b9f2e5239cd7867f34fcf22fe964230f7d447a71d63c283e3593d3f84085
+    image: mirror.gcr.io/library/postgres:15-bullseye@sha256:2e89ed90224245851ea2b01e0b20c4b893e69141eb36e7a1cece7fb9e19f21f0
     ports:
     - 5432
 version: "3.4"

ydb/library/yql/providers/generic/connector/tests/datasource/ydb/collection.py

@@ -2,8 +2,9 @@
 
 from ydb.library.yql.providers.generic.connector.api.common.data_source_pb2 import EDataSourceKind
 
-# import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database
-# import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_database as select_missing_database
+import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_missing_table as select_missing_table
+import select_positive
 import ydb.library.yql.providers.generic.connector.tests.common_test_cases.select_positive_common as select_positive_common
 
 from ydb.library.yql.providers.generic.connector.tests.utils.settings import Settings
@@ -14,9 +15,10 @@ class Collection(object):
 
     def __init__(self, ss: Settings):
         self._test_cases = {
-            # 'select_missing_database': select_missing_database.Factory().make_test_cases(EDataSourceKind.YDB),
-            # 'select_missing_table': select_missing_table.Factory().make_test_cases(EDataSourceKind.YDB),
-            'select_positive': select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.YDB),
+            'select_missing_database': select_missing_database.Factory().make_test_cases(EDataSourceKind.YDB),
+            'select_missing_table': select_missing_table.Factory().make_test_cases(EDataSourceKind.YDB),
+            'select_positive': select_positive_common.Factory(ss).make_test_cases(EDataSourceKind.YDB)
+            + select_positive.Factory().make_test_cases(),
         }
 
     def get(self, key: str) -> Sequence:

ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml

@@ -5,7 +5,7 @@ services:
         echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts;
         /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml"
     container_name: fq-tests-ydb-fq-connector-go
-    image: ghcr.io/ydb-platform/fq-connector-go:v0.2.20@sha256:a1771f348dc8be6219865e332f788429907cdfec3677b3e98f0bc6f7dd542dc6
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.4.1@sha256:61ad8d7b7b981b9ef5709a0ce71d10702f49396bd307437c9efd876c6ad6e90e
     ports:
     - 2130
     volumes:

ydb/library/yql/providers/generic/connector/tests/datasource/ydb/init/01_basic.sh

@@ -44,6 +44,106 @@ set -ex
       (1, 2),
       (10, 20);
     COMMIT;
+
+    CREATE TABLE primitive_types_NATIVE (
+      col_00_id Int32,
+      col_01_bool Bool,
+      col_02_int8 Int8,
+      col_03_int16 Int16,
+      col_04_int32 Int32,
+      col_05_int64 Int64,
+      col_06_uint8 Uint8,
+      col_07_uint16 Uint16,
+      col_08_uint32 Uint32,
+      col_09_uint64 Uint64,
+      col_10_float Float,
+      col_11_double Double,
+      col_12_string String,
+      col_13_utf8 Utf8,
+      col_14_date Date,
+      col_15_datetime Datetime,
+      col_16_timestamp Timestamp,
+      PRIMARY KEY (col_00_id)
+    );
+    COMMIT;
+    INSERT INTO
+    primitive_types_NATIVE 
+                (col_00_id, col_01_bool, col_02_int8, col_03_int16, col_04_int32, col_05_int64, col_06_uint8, 
+                col_07_uint16, col_08_uint32, col_09_uint64, col_10_float, col_11_double, col_12_string,
+                col_13_utf8, col_14_date, col_15_datetime, col_16_timestamp)
+    VALUES (1, false, 2, 3, 4, 5, 6, 7, 8, 9, 10.10f, 11.11f, "аз", "az",
+            Date("1988-11-20"), Datetime("1988-11-20T12:55:28Z"), Timestamp("1988-11-20T12:55:28.111Z")),
+           (2, true, -2, -3, -4, -5, 6, 7, 8, 9, -10.10f, -11.11f, "буки", "buki",
+            Date("2024-05-27"), Datetime("2024-05-27T18:43:32Z"), Timestamp("2024-05-27T18:43:32.123456Z"));
+    COMMIT;
+
+    CREATE TABLE optional_types_NATIVE (
+      col_00_id Int32 NOT NULL,
+      col_01_bool Bool,
+      col_02_int8 Int8,
+      col_03_int16 Int16,
+      col_04_int32 Int32,
+      col_05_int64 Int64,
+      col_06_uint8 Uint8,
+      col_07_uint16 Uint16,
+      col_08_uint32 Uint32,
+      col_09_uint64 Uint64,
+      col_10_float Float,
+      col_11_double Double,
+      col_12_string String,
+      col_13_utf8 Utf8,
+      col_14_date Date,
+      col_15_datetime Datetime,
+      col_16_timestamp Timestamp,
+      PRIMARY KEY (col_00_id)
+    );
+    COMMIT;
+    INSERT INTO
+    optional_types_NATIVE 
+                (col_00_id, col_01_bool, col_02_int8, col_03_int16, col_04_int32, col_05_int64, col_06_uint8, 
+                col_07_uint16, col_08_uint32, col_09_uint64, col_10_float, col_11_double, col_12_string,
+                col_13_utf8, col_14_date, col_15_datetime, col_16_timestamp)
+    VALUES (1, false, 2, 3, 4, 5, 6, 7, 8, 9, 10.10f, 11.11f, "аз", "az",
+            Date("1988-11-20"), Datetime("1988-11-20T12:55:28Z"), Timestamp("1988-11-20T12:55:28.111Z")),
+           (2, true, -2, -3, -4, -5, 6, 7, 8, 9, -10.10f, -11.11f, "буки", "buki",
+            Date("2024-05-27"), Datetime("2024-05-27T18:43:32Z"), Timestamp("2024-05-27T18:43:32.123456Z")),
+           (3, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+            NULL, NULL, NULL);
+    COMMIT;
+
+    CREATE TABLE constant_NATIVE (col_00_id Int32 NOT NULL, PRIMARY KEY (col_00_id));
+    COMMIT;
+    INSERT INTO constant_NATIVE (col_00_id) VALUES
+      (1),
+      (2),
+      (3);
+    COMMIT;
+
+    CREATE TABLE count_NATIVE (col_00_id Int32 NOT NULL, PRIMARY KEY (col_00_id));
+    COMMIT;
+    INSERT INTO count_NATIVE (col_00_id) VALUES
+      (1),
+      (2),
+      (3),
+      (4);
+    COMMIT;
+
+    CREATE TABLE pushdown_NATIVE (col_00_id Int32 NOT NULL, col_01_string String, PRIMARY KEY (col_00_id));
+    COMMIT;
+    INSERT INTO pushdown_NATIVE (col_00_id, col_01_string) VALUES
+      (1, "one"),
+      (2, "two"),
+      (3, "three");
+    COMMIT;
+
+    -- As of 2024.05.31, INTERVAL type is not supported, so we use it to check behavior of connector
+    -- when reading table containing usupported type columns.
+    CREATE TABLE unsupported_types_NATIVE (col_00_id Int32 NOT NULL, col_01_interval INTERVAL, PRIMARY KEY (col_00_id));
+    COMMIT;
+    INSERT INTO unsupported_types_NATIVE (col_00_id, col_01_interval) VALUES
+      (1, DATE("2024-01-01") - DATE("2023-01-01")),
+      (2, DATE("2022-01-01") - DATE("2023-01-01"));
+    COMMIT;
   '
 
 echo $(date +"%T.%6N") "SUCCESS"