tests/library: separate fixtures from harness (#15437)

ijon · blinkov · commit 65c4425a0ed4 · 2025-03-21T09:56:16.000+07:00
`ydb/tests/library/harness/ydb_fixtures` -&gt; `ydb/tests/library/fixtures`.

Move module with pytest fixtures out of `ydb/tests/library/harness` and make it a separate (but dependent) library.
Part of the `harness` (namely `ydbd` launching framework: `Daemon`, runner etc), is widely used outside of ydb tests, and some of that uses still require py2 compatibility (so that `harness` must run under py2).

ydb's own tests (`ydb/tests/functional`) are all pure py3, so requirement for the test fixtures to be py2 compatible is excessive.
diff --git a/ydb/tests/functional/audit/conftest.py b/ydb/tests/functional/audit/conftest.py
@@ -2,4 +2,4 @@
 # but somehow it does not
 #
 # for ydb_{cluster, database, ...} fixture family
-pytest_plugins = 'ydb.tests.library.harness.ydb_fixtures'
+pytest_plugins = 'ydb.tests.library.fixtures'
diff --git a/ydb/tests/functional/audit/test_auditlog.py b/ydb/tests/functional/audit/test_auditlog.py
@@ -13,7 +13,7 @@
 from ydb import Driver, DriverConfig, SessionPool
 from ydb.draft import DynamicConfigClient
 from ydb.tests.library.harness.util import LogLevels
-from ydb.tests.library.harness.ydb_fixtures import ydb_database_ctx
+from ydb.tests.library.fixtures import ydb_database_ctx
 
 logger = logging.getLogger(__name__)
 
diff --git a/ydb/tests/functional/audit/ya.make b/ydb/tests/functional/audit/ya.make
@@ -23,6 +23,7 @@ DEPENDS(
 
 PEERDIR(
     ydb/tests/library
+    ydb/tests/library/fixtures
     ydb/tests/oss/ydb_sdk_import
     ydb/public/sdk/python
 )
diff --git a/ydb/tests/functional/rename/conftest.py b/ydb/tests/functional/rename/conftest.py
@@ -2,4 +2,4 @@
 # but somehow it does not
 #
 # for ydb_{cluster, database, ...} fixture family
-pytest_plugins = 'ydb.tests.library.harness.ydb_fixtures'
+pytest_plugins = 'ydb.tests.library.fixtures'
diff --git a/ydb/tests/functional/rename/ya.make b/ydb/tests/functional/rename/ya.make
@@ -32,6 +32,7 @@ DEPENDS(
 
 PEERDIR(
     ydb/tests/library
+    ydb/tests/library/fixtures
     ydb/tests/oss/ydb_sdk_import
     ydb/public/sdk/python
     contrib/python/tornado/tornado-4
diff --git a/ydb/tests/functional/serverless/conftest.py b/ydb/tests/functional/serverless/conftest.py
@@ -9,7 +9,7 @@
 # but somehow it does not
 #
 # for ydb_{cluster, database, ...} fixture family
-pytest_plugins = 'ydb.tests.library.harness.ydb_fixtures'
+pytest_plugins = 'ydb.tests.library.fixtures'
 
 
 logger = logging.getLogger(__name__)
diff --git a/ydb/tests/functional/serverless/test_serverless.py b/ydb/tests/functional/serverless/test_serverless.py
@@ -65,7 +65,7 @@ def enable_alter_database_create_hive_first(request):
     return request.param
 
 
-# ydb_fixtures.ydb_cluster_configuration local override
+# fixtures.ydb_cluster_configuration local override
 @pytest.fixture(scope='module')
 def ydb_cluster_configuration(enable_alter_database_create_hive_first):
     conf = copy.deepcopy(CLUSTER_CONFIG)
diff --git a/ydb/tests/functional/serverless/ya.make b/ydb/tests/functional/serverless/ya.make
@@ -22,6 +22,7 @@ DEPENDS(
 PEERDIR(
     contrib/python/tornado/tornado-4
     ydb/tests/library
+    ydb/tests/library/fixtures
     ydb/tests/oss/ydb_sdk_import
     ydb/public/sdk/python
 )
diff --git a/ydb/tests/functional/tenants/conftest.py b/ydb/tests/functional/tenants/conftest.py
@@ -7,7 +7,7 @@
 # but somehow it does not
 #
 # for ydb_{cluster, database, ...} fixture family
-pytest_plugins = 'ydb.tests.library.harness.ydb_fixtures'
+pytest_plugins = 'ydb.tests.library.fixtures'
 
 
 @pytest.fixture(scope='module')
diff --git a/ydb/tests/functional/tenants/test_auth_system_views.py b/ydb/tests/functional/tenants/test_auth_system_views.py
@@ -49,7 +49,7 @@
 )
 
 
-# ydb_fixtures.ydb_cluster_configuration local override
+# fixtures.ydb_cluster_configuration local override
 @pytest.fixture(scope='module')
 def ydb_cluster_configuration():
     conf = copy.deepcopy(CLUSTER_CONFIG)
diff --git a/ydb/tests/functional/tenants/test_db_counters.py b/ydb/tests/functional/tenants/test_db_counters.py
@@ -15,7 +15,7 @@
 from ydb.tests.library.harness.kikimr_runner import KiKiMR
 from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
 from ydb.tests.library.harness.util import LogLevels
-from ydb.tests.library.harness.ydb_fixtures import ydb_database_ctx
+from ydb.tests.library.fixtures import ydb_database_ctx
 from ydb.tests.library.matchers.response_matchers import ProtobufWithStatusMatcher
 from ydb.tests.oss.ydb_sdk_import import ydb
 
diff --git a/ydb/tests/functional/tenants/test_dynamic_tenants.py b/ydb/tests/functional/tenants/test_dynamic_tenants.py
@@ -50,7 +50,7 @@ def enable_alter_database_create_hive_first(request):
     return request.param
 
 
-# ydb_fixtures.ydb_cluster_configuration local override
+# fixtures.ydb_cluster_configuration local override
 @pytest.fixture(scope='module')
 def ydb_cluster_configuration(enable_alter_database_create_hive_first):
     conf = copy.deepcopy(CLUSTER_CONFIG)
diff --git a/ydb/tests/functional/tenants/test_tenants.py b/ydb/tests/functional/tenants/test_tenants.py
@@ -10,7 +10,7 @@
 
 from ydb.tests.oss.ydb_sdk_import import ydb
 from ydb.tests.library.harness.util import LogLevels
-from ydb.tests.library.harness.ydb_fixtures import ydb_database_ctx
+from ydb.tests.library.fixtures import ydb_database_ctx
 
 
 logger = logging.getLogger(__name__)
@@ -45,7 +45,7 @@ def enable_alter_database_create_hive_first(request):
     return request.param
 
 
-# ydb_fixtures.ydb_cluster_configuration local override
+# fixtures.ydb_cluster_configuration local override
 @pytest.fixture(scope='module')
 def ydb_cluster_configuration(enable_alter_database_create_hive_first):
     conf = copy.deepcopy(CLUSTER_CONFIG)
diff --git a/ydb/tests/functional/tenants/test_users_groups_with_acl.py b/ydb/tests/functional/tenants/test_users_groups_with_acl.py
@@ -3,7 +3,7 @@
 import logging
 import pytest
 from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
-from ydb.tests.library.harness.ydb_fixtures import ydb_database_ctx
+from ydb.tests.library.fixtures import ydb_database_ctx
 from ydb.tests.oss.ydb_sdk_import import ydb
 
 
diff --git a/ydb/tests/functional/tenants/ya.make b/ydb/tests/functional/tenants/ya.make
@@ -25,6 +25,7 @@ DEPENDS(
 PEERDIR(
     contrib/python/requests
     ydb/tests/library
+    ydb/tests/library/fixtures
     ydb/tests/library/clients
     ydb/tests/oss/ydb_sdk_import
     ydb/public/sdk/python
diff --git a/ydb/tests/functional/ydb_cli/conftest.py b/ydb/tests/functional/ydb_cli/conftest.py
@@ -2,4 +2,4 @@
 # but somehow it does not
 #
 # for ydb_{cluster, database, ...} fixture family
-pytest_plugins = 'ydb.tests.library.harness.ydb_fixtures'
+pytest_plugins = 'ydb.tests.library.fixtures'
diff --git a/ydb/tests/functional/ydb_cli/ya.make b/ydb/tests/functional/ydb_cli/ya.make
@@ -30,6 +30,7 @@ DEPENDS(
 PEERDIR(
     contrib/python/pyarrow
     ydb/tests/library
+    ydb/tests/library/fixtures
     ydb/tests/oss/canonical
     ydb/tests/oss/ydb_sdk_import
 )
diff --git a/ydb/tests/library/fixtures/__init__.py b/ydb/tests/library/fixtures/__init__.py
@@ -114,21 +114,15 @@ def _ydb_database(cluster, database_path_base, unique_name):
 
 @pytest.fixture(scope='function')
 def ydb_database(ydb_cluster, ydb_root, ydb_safe_test_name):
-    # FIXME: PY2 syntax compatibility quirk: "yield from" emulation
-    # Can't use py3 syntax here cause there are nasty dependencies on
-    # tests/library/harness from some py2-only code in some other repositories
-    for i in _ydb_database(ydb_cluster, ydb_root, ydb_safe_test_name):
-        yield i
+    yield from _ydb_database(ydb_cluster, ydb_root, ydb_safe_test_name)
 
 
 @pytest.fixture(scope='module')
 def ydb_database_module_scope(ydb_cluster, ydb_root, request):
     # make unique database name from the test module name, ensuring that
     # it does not contains the dots
     unique_name = request.module.__name__.split('.')[-1]
-    # FIXME: PY2 syntax compatibility quirk: "yield from" emulation
-    for i in _ydb_database(ydb_cluster, ydb_root, unique_name):
-        yield i
+    yield from _ydb_database(ydb_cluster, ydb_root, unique_name)
 
 
 @pytest.fixture(scope='module')
diff --git a/ydb/tests/library/fixtures/__init__.py_ b/ydb/tests/library/fixtures/__init__.py_
@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+import contextlib
+import logging
+import os
+
+import pytest
+
+from ydb import Driver, DriverConfig, SessionPool
+
+from ydb.tests.library.common.types import Erasure
+from ydb.tests.library.harness.kikimr_runner import KiKiMR
+from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
+from ydb.tests.library.harness.util import LogLevels
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_CLUSTER_CONFIG = dict(
+    erasure=Erasure.NONE,
+    nodes=1,
+    additional_log_configs={
+        'FLAT_TX_SCHEMESHARD': LogLevels.DEBUG,
+        'SCHEME_BOARD_POPULATOR': LogLevels.WARN,
+        'SCHEME_BOARD_SUBSCRIBER': LogLevels.WARN,
+        'TX_DATASHARD': LogLevels.DEBUG,
+        'CHANGE_EXCHANGE': LogLevels.DEBUG,
+    },
+)
+
+
+@pytest.fixture(scope='module')
+def ydb_cluster_configuration(request):
+    conf = getattr(request.module, 'CLUSTER_CONFIG', DEFAULT_CLUSTER_CONFIG)
+    return conf
+
+
+@pytest.fixture(scope='module')
+def ydb_configurator(ydb_cluster_configuration):
+    return KikimrConfigGenerator(**ydb_cluster_configuration)
+
+
+@pytest.fixture(scope='module')
+def ydb_cluster(ydb_configurator, request):
+    module_name = request.module.__name__
+
+    logger.info("setup ydb_cluster for %s", module_name)
+
+    logger.info("setup ydb_cluster as local")
+    cluster = KiKiMR(
+        configurator=ydb_configurator,
+    )
+    cluster.is_local_test = True
+
+    cluster.start()
+
+    yield cluster
+
+    logger.info("destroy ydb_cluster for %s", module_name)
+    cluster.stop()
+
+
+@pytest.fixture(scope='module')
+def ydb_root(ydb_cluster):
+    return os.path.join("/", ydb_cluster.domain_name)
+
+
+@pytest.fixture(scope='module')
+def ydb_private_client(ydb_cluster):
+    return ydb_cluster.client
+
+
+@pytest.fixture(scope='function')
+def ydb_safe_test_name(request):
+    return request.node.name.replace("[", "_").replace("]", "_")
+
+
+@contextlib.contextmanager
+def ydb_database_ctx(ydb_cluster, database_path, node_count=1, timeout_seconds=20, storage_pools={'hdd': 1}):
+    '''???'''
+    assert os.path.abspath(database_path), 'database_path should be an (absolute) path, not a database name'
+
+    token = ydb_cluster.config.default_clusteradmin
+
+    ydb_cluster.remove_database(database_path, timeout_seconds=timeout_seconds, token=token)
+
+    logger.debug("create database %s: create path and declare internals", database_path)
+
+    ydb_cluster.create_database(database_path, storage_pool_units_count=storage_pools, timeout_seconds=timeout_seconds, token=token)
+
+    logger.debug("create database %s: start nodes and construct internals", database_path)
+    database_nodes = ydb_cluster.register_and_start_slots(database_path, node_count)
+
+    logger.debug("create database %s: wait construction done", database_path)
+    ydb_cluster.wait_tenant_up(database_path, token=token)
+
+    logger.debug("create database %s: database up", database_path)
+    yield database_path
+
+    logger.debug("destroy database %s: remove path and dismantle internals", database_path)
+    ydb_cluster.remove_database(database_path, timeout_seconds=timeout_seconds, token=token)
+
+    logger.debug("destroy database %s: stop nodes", database_path)
+    ydb_cluster.unregister_and_stop_slots(database_nodes)
+
+    logger.debug("destroy database %s: database down", database_path)
+
+
+def _ydb_database(cluster, database_path_base, unique_name):
+    database = os.path.join(database_path_base, unique_name)
+
+    with ydb_database_ctx(cluster, database):
+        yield database
+
+@pytest.fixture(scope='function')
+def ydb_database(ydb_cluster, ydb_root, ydb_safe_test_name):
+    yield from _ydb_database(ydb_cluster, ydb_root, ydb_safe_test_name)
+
+
+@pytest.fixture(scope='module')
+def ydb_database_module_scope(ydb_cluster, ydb_root, request):
+    # make unique database name from the test module name, ensuring that
+    # it does not contains the dots
+    unique_name = request.module.__name__.split('.')[-1]
+    yield from _ydb_database(ydb_cluster, ydb_root, unique_name)
+
+
+@pytest.fixture(scope='module')
+def ydb_endpoint(ydb_cluster):
+    return "%s:%s" % (ydb_cluster.nodes[1].host, ydb_cluster.nodes[1].port)
+
+
+@pytest.fixture(scope='function')
+def ydb_client(ydb_endpoint, request):
+    def _make_driver(database_path, **kwargs):
+        driver_config = DriverConfig(ydb_endpoint, database_path, **kwargs)
+        driver = Driver(driver_config)
+
+        def stop_driver():
+            driver.stop()
+
+        request.addfinalizer(stop_driver)
+        return driver
+
+    return _make_driver
+
+
+@pytest.fixture(scope='function')
+def ydb_client_session(ydb_client, request):
+    def _make_pool(database_path, **kwargs):
+        driver = ydb_client(database_path, **kwargs)
+        pool = SessionPool(driver)
+
+        def stop_pool():
+            pool.stop()
+
+        request.addfinalizer(stop_pool)
+        return pool
+
+    return _make_pool
+
+
+# possible replacement for both ydb_client and ydb_client_session
+# @pytest.fixture(scope='function')
+# def ydb_database_and_client(ydb_database, ydb_endpoint):
+#     database_path = ydb_database
+#     with Driver(DriverConfig(ydb_endpoint, database_path)) as driver:
+#         with SessionPool(driver) as pool:
+#             yield database_path, pool
diff --git a/ydb/tests/library/fixtures/ya.make b/ydb/tests/library/fixtures/ya.make
@@ -0,0 +1,12 @@
+PY3_LIBRARY()
+
+PY_SRCS(
+    __init__.py
+)
+
+PEERDIR(
+    ydb/tests/library
+    ydb/public/sdk/python
+)
+
+END()
diff --git a/ydb/tests/library/ya.make b/ydb/tests/library/ya.make
@@ -32,7 +32,6 @@ PY_SRCS(
     harness/param_constants.py
     harness/util.py
     harness/tls_tools.py
-    harness/ydb_fixtures.py
     matchers/__init__.py
     matchers/collection.py
     matchers/datashard_matchers.py

Original file line number	Diff line number	Diff line change
`@@ -2,4 +2,4 @@`
`2`	`2`	`# but somehow it does not`
`3`	`3`	`#`
`4`	`4`	`# for ydb_{cluster, database, ...} fixture family`
`5`		`-pytest_plugins = 'ydb.tests.library.harness.ydb_fixtures'`
	`5`	`+pytest_plugins = 'ydb.tests.library.fixtures'`
Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@ DEPENDS(`
`23`	`23`
`24`	`24`	`PEERDIR(`
`25`	`25`	`ydb/tests/library`
	`26`	`+ ydb/tests/library/fixtures`
`26`	`27`	`ydb/tests/oss/ydb_sdk_import`
`27`	`28`	`ydb/public/sdk/python`
`28`	`29`	`)`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@`
`9`	`9`	`# but somehow it does not`
`10`	`10`	`#`
`11`	`11`	`# for ydb_{cluster, database, ...} fixture family`
`12`		`-pytest_plugins = 'ydb.tests.library.harness.ydb_fixtures'`
	`12`	`+pytest_plugins = 'ydb.tests.library.fixtures'`
`13`	`13`
`14`	`14`
`15`	`15`	`logger = logging.getLogger(__name__)`
Original file line number	Diff line number	Diff line change
`@@ -22,6 +22,7 @@ DEPENDS(`
`22`	`22`	`PEERDIR(`
`23`	`23`	`contrib/python/tornado/tornado-4`
`24`	`24`	`ydb/tests/library`
	`25`	`+ ydb/tests/library/fixtures`
`25`	`26`	`ydb/tests/oss/ydb_sdk_import`
`26`	`27`	`ydb/public/sdk/python`
`27`	`28`	`)`
Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,7 @@`
`7`	`7`	`# but somehow it does not`
`8`	`8`	`#`
`9`	`9`	`# for ydb_{cluster, database, ...} fixture family`
`10`		`-pytest_plugins = 'ydb.tests.library.harness.ydb_fixtures'`
	`10`	`+pytest_plugins = 'ydb.tests.library.fixtures'`
`11`	`11`
`12`	`12`
`13`	`13`	`@pytest.fixture(scope='module')`
Original file line number	Diff line number	Diff line change
`@@ -49,7 +49,7 @@`
`49`	`49`	`)`
`50`	`50`
`51`	`51`
`52`		`-# ydb_fixtures.ydb_cluster_configuration local override`
	`52`	`+# fixtures.ydb_cluster_configuration local override`
`53`	`53`	`@pytest.fixture(scope='module')`
`54`	`54`	`def ydb_cluster_configuration():`
`55`	`55`	`conf = copy.deepcopy(CLUSTER_CONFIG)`