Skip to content

Commit

Permalink
schema_registry/tests: Use protobuf/payload.proto for python kafka-serde
Browse files Browse the repository at this point in the history
The proto has been generated by protoc. I'd prefer to have put it as
part of the build system, but I can't. Maybe somebody else can.

```
cd tests/protobuf
../../vbuild/debug/clang/rp_deps_install/bin/protoc --python_out=../rptest/remote_scripts payload.proto
```

Signed-off-by: Ben Pope <ben@redpanda.com>
  • Loading branch information
BenPope committed Apr 18, 2023
1 parent 9d8e40b commit 9c095b9
Show file tree
Hide file tree
Showing 5 changed files with 48 additions and 13 deletions.
1 change: 1 addition & 0 deletions .yapfignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@ build/*
vtools/*
depot_tools/*
src/v/dashboard/node_modules/*
tests/rptest/remote_scripts/payload_pb2*
go/*
.git/*
26 changes: 26 additions & 0 deletions tests/rptest/remote_scripts/payload_pb2.py

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,7 @@
from confluent_kafka.schema_registry.avro import AvroDeserializer, AvroSerializer
from confluent_kafka.schema_registry.protobuf import ProtobufDeserializer, ProtobufSerializer

from google.protobuf.descriptor_pb2 import FieldDescriptorProto
from google.protobuf import proto_builder
from payload_pb2 import Payload as ProtobufPayloadClass


class SchemaType(IntEnum):
Expand All @@ -50,10 +49,6 @@ def val(self):
}
'''

ProtobufPayloadClass = proto_builder.MakeSimpleProtoClass(
OrderedDict([('val', FieldDescriptorProto.TYPE_INT64)]),
full_name="example.Payload")


def make_protobuf_payload(val: int):
p = ProtobufPayloadClass()
Expand Down
1 change: 1 addition & 0 deletions tests/rptest/services/serde_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ def __init__(self,
def _worker(self, idx, node):

if self._serde_client_type == SerdeClientType.Python:
inject_remote_script(node, "payload_pb2.py")
script_path = inject_remote_script(
node, "python_librdkafka_serde_client.py")
script_path = f"{PYTHON_EXEC} {script_path}"
Expand Down
26 changes: 19 additions & 7 deletions tests/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,29 @@
author_email='hi@vectorized.io',
description='vectorized internal testing',
packages=find_packages(),
setup_requires=['setuptools'],
setup_requires=['setuptools', 'grpcio-tools==1.53'],
package_data={'': ['*.md']},
include_package_data=True,
install_requires=[
'ducktape@git+https://github.com/redpanda-data/ducktape.git@5b4c31648d8f94d3f28113a6fa0377339aa222ed',
'prometheus-client==0.9.0', 'pyyaml==6.0', 'kafka-python==2.0.2',
'crc32c==2.2', 'confluent-kafka==2.0.2', 'zstandard==0.15.2',
'xxhash==2.0.2', 'protobuf==3.19.5', 'fastavro==1.4.9',
'psutil==5.9.0', 'numpy==1.22.3', 'pygal==3.0', 'pytest==7.1.2',
'jump-consistent-hash==3.2.0', 'azure-storage-blob==12.14.1',
'kafkatest@git+https://github.com/apache/kafka.git@058589b03db686803b33052d574ce887fb5cfbd1#egg=kafkatest&subdirectory=tests'
'prometheus-client==0.9.0',
'pyyaml==6.0',
'kafka-python==2.0.2',
'crc32c==2.2',
'confluent-kafka==2.0.2',
'zstandard==0.15.2',
'xxhash==2.0.2',
'protobuf==4.21.8',
'fastavro==1.4.9',
'psutil==5.9.0',
'numpy==1.22.3',
'pygal==3.0',
'pytest==7.1.2',
'jump-consistent-hash==3.2.0',
'azure-storage-blob==12.14.1',
'kafkatest@git+https://github.com/apache/kafka.git@058589b03db686803b33052d574ce887fb5cfbd1#egg=kafkatest&subdirectory=tests',
'grpcio==1.53.0',
'grpcio-tools==1.53',
],
scripts=[],
)

0 comments on commit 9c095b9

Please sign in to comment.