Support protubuf 3 and protobuf 4 same time

.github/workflows/tests.yaml

@@ -2,6 +2,7 @@ name: Functional tests
 
 on:
   push:
+    - master
   pull_request:
 
 jobs:
@@ -11,7 +12,7 @@ jobs:
       max-parallel: 4
       matrix:
         python-version: [3.8]
-        environment: [py38, py38-tls]
+        environment: [py38, py38-tls, py38-proto3, py38-tls-proto3]
 
     steps:
     - uses: actions/checkout@v1

Makefile

@@ -1,3 +1,9 @@
-protobuf:
-	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env
-	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env python generate_protoc.py
+protobuf-3:
+	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-3 --build-arg GRPCIO_VER=1.39.0 --build-arg PY_PROTOBUF_VER=3.20.3
+	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-3 python generate_protoc.py --target-version=v3
+
+protobuf-4:
+	docker build -f generate-protobuf.Dockerfile . -t ydb-python-sdk-proto-generator-env-4
+	docker run --rm -it -v $${PWD}:$${PWD} -w $${PWD} ydb-python-sdk-proto-generator-env-4 python generate_protoc.py
+
+protobuf: protobuf-3 protobuf-4

generate-protobuf.Dockerfile

@@ -1,10 +1,13 @@
 FROM python:3.9.15
+ENV GRPCLIB_VER=0.4.3
+ARG GRPCIO_VER=1.50.0
+ARG PY_PROTOBUF_VER=4.21.9
 RUN \
     python -m pip install --upgrade pip && \
-    python -m pip install grpcio==1.39.0 && \
-    python -m pip install grpclib && \
-    python -m pip install protobuf==3.20.3 && \
-    python -m pip install grpcio-tools==1.38.0
+    python -m pip install grpcio==${GRPCIO_VER} && \
+    python -m pip install grpclib==${GRPCLIB_VER} && \
+    python -m pip install grpcio-tools==${GRPCIO_VER} && \
+    python -m pip install protobuf==${PY_PROTOBUF_VER}
 
 ENV PROTOC_VER=21.8
 RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/protoc-${PROTOC_VER}-linux-x86_64.zip && \

generate_protoc.py

@@ -1,7 +1,9 @@
 import os
 import pathlib
 import shutil
+
 from typing import List
+from argparse import ArgumentParser
 
 from grpc_tools import command
 
@@ -35,8 +37,9 @@ def remove_protos(rootdirpath: str):
                 os.remove(os.path.join(root, file))
 
 
-def fix_file_contents(rootdir: str):
+def fix_file_contents(rootdir, protobuf_version: str):
     flake_ignore_line = "# flake8: " + "noqa"  # prevent ignore the file
+    package_path = "ydb._grpc." + protobuf_version + ".protos"
 
     for dirpath, _, fnames in os.walk(rootdir):
         for fname in fnames:
@@ -47,24 +50,30 @@ def fix_file_contents(rootdir: str):
                 content = f.read()
 
                 # Fix imports
-                content = content.replace("from protos", "from ydb._grpc.protos")
+                content = content.replace("from protos", "from " + package_path)
 
                 # Add ignore style check
                 content = content.replace("# -*- coding: utf-8 -*-", "# -*- coding: utf-8 -*-\n" + flake_ignore_line)
                 f.seek(0)
                 f.write(content)
 
 
-def generate_protobuf(src_proto_dir: str, dst_dir: str):
+def generate_protobuf(src_proto_dir: str, dst_dir, protobuf_version: str):
     shutil.rmtree(dst_dir, ignore_errors=True)
 
     shutil.copytree(src_proto_dir, dst_dir, ignore=files_filter)
-    create_init_files(dst_dir)
 
     command.build_package_protos(dst_dir)
     remove_protos(dst_dir)
-    fix_file_contents(dst_dir)
+    create_init_files(dst_dir)
+    fix_file_contents(dst_dir, protobuf_version)
 
 
 if __name__ == '__main__':
-    generate_protobuf("ydb-api-protos", "ydb/_grpc")
+    parser = ArgumentParser()
+    parser.add_argument("--target-version", default="v4", help="Protobuf version")
+
+    args = parser.parse_args()
+
+    target_dir = "ydb/_grpc/" + args.target_version
+    generate_protobuf("ydb-api-protos", target_dir, args.target_version)

requirements.txt

@@ -1,4 +1,5 @@
 grpcio==1.39.0
-protobuf>3.17.3,<4.0.0
+packaging
+protobuf>3.13.0,<5.0.0
 pytest==6.2.4
 aiohttp==3.7.4

setup.py

@@ -24,7 +24,7 @@
         "Programming Language :: Python :: 3.6",
     ],
     install_requires=(
-        "protobuf>=3.13.0,<4.0.0",
+        "protobuf>=3.13.0",
         "grpcio>=1.5.0",
         "enum-compat>=0.0.1",
     ),

test-requirements.txt

@@ -10,16 +10,16 @@ docker==5.0.0
 docker-compose==1.29.2
 dockerpty==0.4.1
 docopt==0.6.2
-grpcio==1.39.0
-grpcio-tools==1.39.0
+grpcio
+grpcio-tools
 idna==3.2
 importlib-metadata==4.6.1
 iniconfig==1.1.1
 jsonschema==3.2.0
 packaging==21.0
 paramiko==2.10.1
 pluggy==0.13.1
-protobuf>3.17.3,<4.0.0
+protobuf>3.13.0,<5.0.0
 py==1.10.0
 pycparser==2.20
 PyNaCl==1.4.0

tox.ini

@@ -13,27 +13,37 @@ setenv =
 deps =
     -r{toxinidir}/test-requirements.txt
 
-[testenv:protoc]
+[testenv:dev]
 commands =
-    python3 generate_protoc.py
 
-
-[testenv:py36]
+[testenv:dev-proto3]
 commands =
-    pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs}
+deps =
+    -r{toxinidir}/test-requirements.txt
+    protobuf<4.0.0
 
-[testenv:py36-tls]
+[testenv:py38]
 commands =
-    pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs}
+    pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs}
 
-[testenv:py38]
+[testenv:py38-proto3]
 commands =
     pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs}
+deps =
+    -r{toxinidir}/test-requirements.txt
+    protobuf<4.0.0
 
 [testenv:py38-tls]
 commands =
     pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs}
 
+[testenv:py38-tls-proto3]
+commands =
+    pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs}
+deps =
+    -r{toxinidir}/test-requirements.txt
+    protobuf<4.0.0
+
 [testenv:py310]
 commands =
     pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs}

ydb/_apis.py

@@ -1,20 +1,20 @@
 # -*- coding: utf-8 -*-
-from ydb._grpc import (
+from ydb._grpc.common import (
     ydb_cms_v1_pb2_grpc,
     ydb_discovery_v1_pb2_grpc,
     ydb_scheme_v1_pb2_grpc,
     ydb_table_v1_pb2_grpc,
 )
-from ydb._grpc.protos import (
+from ydb._grpc.common.protos import (
     ydb_status_codes_pb2,
     ydb_discovery_pb2,
     ydb_scheme_pb2,
     ydb_table_pb2,
     ydb_value_pb2,
 )
-from ydb._grpc.protos import ydb_operation_pb2
-from ydb._grpc.protos import ydb_common_pb2
-from ydb._grpc import ydb_operation_v1_pb2_grpc
+from ydb._grpc.common.protos import ydb_operation_pb2
+from ydb._grpc.common.protos import ydb_common_pb2
+from ydb._grpc.common import ydb_operation_v1_pb2_grpc
 
 
 StatusIds = ydb_status_codes_pb2.StatusIds

ydb/_grpc/common/__init__.py

@@ -0,0 +1,20 @@
+import sys
+
+import google.protobuf
+from packaging.version import Version
+
+# generated files are incompatible between 3 and 4 protobuf versions
+# import right generated version for current protobuf lib
+# sdk code must always import from ydb._grpc.common
+protobuf_version = Version(google.protobuf.__version__)
+
+if protobuf_version < Version("4.0"):
+    from ydb._grpc.v3 import * # noqa
+    from ydb._grpc.v3 import protos # noqa
+    sys.modules["ydb._grpc.common"] = sys.modules["ydb._grpc.v3"]
+    sys.modules["ydb._grpc.common.protos"] = sys.modules["ydb._grpc.v3.protos"]
+else:
+    from ydb._grpc.v4 import * # noqa
+    from ydb._grpc.v4 import protos # noqa
+    sys.modules["ydb._grpc.common"] = sys.modules["ydb._grpc.v4"]
+    sys.modules["ydb._grpc.common.protos"] = sys.modules["ydb._grpc.v4.protos"]