Kickoff 5.2.0 development [API-1564] (#609)

* Kickoff 5.2.0 development

This PR contains various fixes to make the client compatible with
the 5.2 server.

- Server version used in tests bumped to 5.2.2
- For Compact, field kinds are updated with the up-to-date enum values
and they are moved to the serialization.api from compact, to match
with the Java side
- Dependency on the enterprise-tests JAR is removed and the self-signed
certificates that we depend on moved to the client-side, instead
of adding a dependency to the private test artifacts.
- Some reactor related tests that fail on macOS are fixed

* add unsupported field kinds

* fix broken test

Author: mdumandag

hazelcast/serialization/api.py

@@ -4,6 +4,7 @@
 import abc
 import datetime
 import decimal
+import enum
 import typing
 
 from hazelcast.serialization.portable.classdef import FieldType
@@ -1381,7 +1382,7 @@ class CompactReader(abc.ABC):
     """
 
     @abc.abstractmethod
-    def get_field_kind(self, field_name):
+    def get_field_kind(self, field_name: str) -> "FieldKind":
         """Returns the FieldKind for the given field.
 
         Args:
@@ -2593,3 +2594,61 @@ def get_type_name(self) -> str:
         Returns:
             The type name.
         """
+
+
+class FieldKind(enum.IntEnum):
+    """
+    Represents the types of the fields used in the Compact serialization.
+    """
+
+    NOT_AVAILABLE = 0
+    """
+    Represents fields that do not exist.
+    """
+
+    BOOLEAN = 1
+    ARRAY_OF_BOOLEAN = 2
+    INT8 = 3
+    ARRAY_OF_INT8 = 4
+    CHAR = 5
+    ARRAY_OF_CHAR = 6
+    INT16 = 7
+    ARRAY_OF_INT16 = 8
+    INT32 = 9
+    ARRAY_OF_INT32 = 10
+    INT64 = 11
+    ARRAY_OF_INT64 = 12
+    FLOAT32 = 13
+    ARRAY_OF_FLOAT32 = 14
+    FLOAT64 = 15
+    ARRAY_OF_FLOAT64 = 16
+    STRING = 17
+    ARRAY_OF_STRING = 18
+    DECIMAL = 19
+    ARRAY_OF_DECIMAL = 20
+    TIME = 21
+    ARRAY_OF_TIME = 22
+    DATE = 23
+    ARRAY_OF_DATE = 24
+    TIMESTAMP = 25
+    ARRAY_OF_TIMESTAMP = 26
+    TIMESTAMP_WITH_TIMEZONE = 27
+    ARRAY_OF_TIMESTAMP_WITH_TIMEZONE = 28
+    COMPACT = 29
+    ARRAY_OF_COMPACT = 30
+    PORTABLE = 31
+    ARRAY_OF_PORTABLE = 32
+    NULLABLE_BOOLEAN = 33
+    ARRAY_OF_NULLABLE_BOOLEAN = 34
+    NULLABLE_INT8 = 35
+    ARRAY_OF_NULLABLE_INT8 = 36
+    NULLABLE_INT16 = 37
+    ARRAY_OF_NULLABLE_INT16 = 38
+    NULLABLE_INT32 = 39
+    ARRAY_OF_NULLABLE_INT32 = 40
+    NULLABLE_INT64 = 41
+    ARRAY_OF_NULLABLE_INT64 = 42
+    NULLABLE_FLOAT32 = 43
+    ARRAY_OF_NULLABLE_FLOAT32 = 44
+    NULLABLE_FLOAT64 = 45
+    ARRAY_OF_NULLABLE_FLOAT64 = 46

hazelcast/serialization/compact.py

@@ -2,7 +2,6 @@
 import collections
 import datetime
 import decimal
-import enum
 import typing
 
 from hazelcast.errors import HazelcastError, HazelcastSerializationError, IllegalStateError
@@ -24,6 +23,7 @@
     CompactWriter,
     CompactReader,
     ObjectDataInput,
+    FieldKind,
 )
 from hazelcast.serialization.input import _ObjectDataInput
 from hazelcast.serialization.output import _ObjectDataOutput
@@ -1762,8 +1762,6 @@ def _init(self):
 
         for field in self.fields_list:
             kind = field.kind
-            if kind < 0 or kind >= FieldKind.NOT_AVAILABLE:
-                raise HazelcastSerializationError(f"Invalid field kind: {kind}")
             if FIELD_OPERATIONS[field.kind].is_var_sized():
                 var_sized_fields.append(field)
             elif FieldKind.BOOLEAN == kind:
@@ -2004,52 +2002,6 @@ def read(self, inp: _ObjectDataInput, start_position: int, index: int) -> int:
 _INT32_POSITION_READER_INSTANCE = Int32PositionReader()
 
 
-class FieldKind(enum.IntEnum):
-    BOOLEAN = 0
-    ARRAY_OF_BOOLEAN = 1
-    INT8 = 2
-    ARRAY_OF_INT8 = 3
-    INT16 = 6
-    ARRAY_OF_INT16 = 7
-    INT32 = 8
-    ARRAY_OF_INT32 = 9
-    INT64 = 10
-    ARRAY_OF_INT64 = 11
-    FLOAT32 = 12
-    ARRAY_OF_FLOAT32 = 13
-    FLOAT64 = 14
-    ARRAY_OF_FLOAT64 = 15
-    STRING = 16
-    ARRAY_OF_STRING = 17
-    DECIMAL = 18
-    ARRAY_OF_DECIMAL = 19
-    TIME = 20
-    ARRAY_OF_TIME = 21
-    DATE = 22
-    ARRAY_OF_DATE = 23
-    TIMESTAMP = 24
-    ARRAY_OF_TIMESTAMP = 25
-    TIMESTAMP_WITH_TIMEZONE = 26
-    ARRAY_OF_TIMESTAMP_WITH_TIMEZONE = 27
-    COMPACT = 28
-    ARRAY_OF_COMPACT = 29
-    NULLABLE_BOOLEAN = 32
-    ARRAY_OF_NULLABLE_BOOLEAN = 33
-    NULLABLE_INT8 = 34
-    ARRAY_OF_NULLABLE_INT8 = 35
-    NULLABLE_INT16 = 36
-    ARRAY_OF_NULLABLE_INT16 = 37
-    NULLABLE_INT32 = 38
-    ARRAY_OF_NULLABLE_INT32 = 39
-    NULLABLE_INT64 = 40
-    ARRAY_OF_NULLABLE_INT64 = 41
-    NULLABLE_FLOAT32 = 42
-    ARRAY_OF_NULLABLE_FLOAT32 = 43
-    NULLABLE_FLOAT64 = 44
-    ARRAY_OF_NULLABLE_FLOAT64 = 45
-    NOT_AVAILABLE = 46
-
-
 class FieldKindOperations(abc.ABC):
     _VAR_SIZE = -1
 
@@ -2236,6 +2188,7 @@ class ArrayOfNullableFloat64Operations(FieldKindOperations):
 
 
 FIELD_OPERATIONS: typing.List[typing.Optional[FieldKindOperations]] = [
+    None,
     BooleanOperations(),
     ArrayOfBooleanOperations(),
     Int8Operations(),
@@ -2282,5 +2235,4 @@ class ArrayOfNullableFloat64Operations(FieldKindOperations):
     ArrayOfNullableFloat32Operations(),
     NullableFloat64Operations(),
     ArrayOfNullableFloat64Operations(),
-    None,
 ]

start_rc.py

@@ -3,7 +3,7 @@
 import sys
 from os.path import isfile
 
-SERVER_VERSION = "5.1"
+SERVER_VERSION = "5.2.2"
 RC_VERSION = "0.8-SNAPSHOT"
 
 RELEASE_REPO = "https://repo1.maven.apache.org/maven2"
@@ -74,15 +74,10 @@ def start_rc(stdout=None, stderr=None):
 
     if enterprise_key:
         server = download_if_necessary(ENTERPRISE_REPO, "hazelcast-enterprise", SERVER_VERSION)
-        ep_tests = download_if_necessary(
-            ENTERPRISE_REPO, "hazelcast-enterprise", SERVER_VERSION, True
-        )
-
-        artifacts.append(server)
-        artifacts.append(ep_tests)
     else:
         server = download_if_necessary(REPO, "hazelcast", SERVER_VERSION)
-        artifacts.append(server)
+
+    artifacts.append(server)
 
     class_path = CLASS_PATH_SEPARATOR.join(artifacts)
 

tests/integration/backward_compatible/serialization/compact_compatibility/compact_compatibility_test.py

@@ -48,6 +48,9 @@ def __eq__(self, o: object) -> bool:
     def __hash__(self) -> int:
         return hash(self.string_field)
 
+    def __repr__(self):
+        return f"InnerCompact(string_field={self.string_field})"
+
 
 class OuterCompact:
     def __init__(self, int_field: int, inner_field: InnerCompact):
@@ -64,6 +67,9 @@ def __eq__(self, o: object) -> bool:
     def __hash__(self) -> int:
         return hash((self.int_field, self.inner_field))
 
+    def __repr__(self):
+        return f"OuterCompact(int_field={self.int_field}, inner_field={self.inner_field})"
+
 
 class InnerSerializer(CompactSerializer[InnerCompact]):
     def read(self, reader: CompactReader) -> InnerCompact:
@@ -272,7 +278,7 @@ def get_class(self):
 
 
 @unittest.skipIf(
-    compare_client_version("5.1") < 0, "Tests the features added in 5.1 version of the client"
+    compare_client_version("5.2") < 0, "Tests the features added in 5.2 version of the client"
 )
 class CompactCompatibilityBase(HazelcastTestCase):
     rc = None
@@ -282,24 +288,15 @@ class CompactCompatibilityBase(HazelcastTestCase):
     @classmethod
     def setUpClass(cls) -> None:
         cls.rc = cls.create_rc()
-        if compare_server_version_with_rc(cls.rc, "5.1") < 0:
+        if compare_server_version_with_rc(cls.rc, "5.2") < 0:
             cls.rc.exit()
-            raise unittest.SkipTest("Compact serialization requires 5.1 server")
-
-        if compare_server_version_with_rc(cls.rc, "5.2") >= 0 and compare_client_version("5.2") < 0:
-            cls.rc.exit()
-            raise unittest.SkipTest(
-                "Compact serialization 5.2 server is not compatible with clients older than 5.2"
-            )
+            raise unittest.SkipTest("Compact serialization requires 5.2 server")
 
         config = f"""
 <hazelcast xmlns="http://www.hazelcast.com/schema/config"
            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
            xsi:schemaLocation="http://www.hazelcast.com/schema/config
-           http://www.hazelcast.com/schema/config/hazelcast-config-5.1.xsd">
-    <serialization>
-        <compact-serialization enabled="true" />
-    </serialization>
+           http://www.hazelcast.com/schema/config/hazelcast-config-5.2.xsd">
     <jet enabled="true" />
 </hazelcast>
         """
@@ -1777,6 +1774,6 @@ def _put_from_another_client(self, key, value):
 class PartitionServiceCompactCompatibilityTest(CompactCompatibilityBase):
     def test_partition_service(self):
         self.assertEqual(
-            268,
+            267,
             self.client.partition_service.get_partition_id(OUTER_COMPACT_INSTANCE),
         )

tests/integration/backward_compatible/serialization/compact_test.py

@@ -25,8 +25,9 @@
         CompactSerializer,
         CompactReader,
         CompactWriter,
+        FieldKind,
     )
-    from hazelcast.serialization.compact import FIELD_OPERATIONS, FieldKind
+    from hazelcast.serialization.compact import FIELD_OPERATIONS
 
     _COMPACT_AVAILABLE = True
 except ImportError:
@@ -91,7 +92,7 @@ class FieldKind(enum.Enum):
 
 
 @unittest.skipIf(
-    compare_client_version("5.1") < 0, "Tests the features added in 5.1 version of the client"
+    compare_client_version("5.2") < 0, "Tests the features added in 5.2 version of the client"
 )
 class CompactTestBase(HazelcastTestCase):
     rc = None
@@ -101,28 +102,11 @@ class CompactTestBase(HazelcastTestCase):
     @classmethod
     def setUpClass(cls) -> None:
         cls.rc = cls.create_rc()
-        if compare_server_version_with_rc(cls.rc, "5.1") < 0:
+        if compare_server_version_with_rc(cls.rc, "5.2") < 0:
             cls.rc.exit()
-            raise unittest.SkipTest("Compact serialization requires 5.1 server")
+            raise unittest.SkipTest("Compact serialization requires 5.2 server")
 
-        if compare_server_version_with_rc(cls.rc, "5.2") >= 0 and compare_client_version("5.2") < 0:
-            cls.rc.exit()
-            raise unittest.SkipTest(
-                "Compact serialization 5.2 server is not compatible with clients older than 5.2"
-            )
-
-        config = """
-        <hazelcast xmlns="http://www.hazelcast.com/schema/config"
-           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-           xsi:schemaLocation="http://www.hazelcast.com/schema/config
-           http://www.hazelcast.com/schema/config/hazelcast-config-5.1.xsd">
-            <serialization>
-                    <compact-serialization enabled="true" />
-            </serialization>
-        </hazelcast>
-        """
-
-        cls.cluster = cls.create_cluster(cls.rc, config)
+        cls.cluster = cls.create_cluster(cls.rc, None)
         cls.member = cls.cluster.start_member()
 
     @classmethod

tests/integration/backward_compatible/ssl_tests/hazelcast-ma-optional.xml

@@ -5,13 +5,12 @@
     <network>
         <ssl enabled="true">
             <factory-class-name>
-                com.hazelcast.nio.ssl.ClasspathSSLContextFactory
+                com.hazelcast.nio.ssl.BasicSSLContextFactory
             </factory-class-name>
             <properties>
-                <property name="keyStore">com/hazelcast/nio/ssl-mutual-auth/server1.keystore</property>
+                <property name="keyStore">%s</property>
                 <property name="keyStorePassword">password</property>
-                <property name="trustStore">com/hazelcast/nio/ssl-mutual-auth/server1_knows_client1/server1.truststore
-                </property>
+                <property name="trustStore">%s</property>
                 <property name="trustStorePassword">password</property>
                 <property name="trustManagerAlgorithm">SunX509</property>
                 <property name="javax.net.ssl.mutualAuthentication">OPTIONAL</property>

tests/integration/backward_compatible/ssl_tests/hazelcast-ma-required.xml

@@ -5,13 +5,12 @@
     <network>
         <ssl enabled="true">
             <factory-class-name>
-                com.hazelcast.nio.ssl.ClasspathSSLContextFactory
+                com.hazelcast.nio.ssl.BasicSSLContextFactory
             </factory-class-name>
             <properties>
-                <property name="keyStore">com/hazelcast/nio/ssl-mutual-auth/server1.keystore</property>
+                <property name="keyStore">%s</property>
                 <property name="keyStorePassword">password</property>
-                <property name="trustStore">com/hazelcast/nio/ssl-mutual-auth/server1_knows_client1/server1.truststore
-                </property>
+                <property name="trustStore">%s</property>
                 <property name="trustStorePassword">password</property>
                 <property name="trustManagerAlgorithm">SunX509</property>
                 <property name="javax.net.ssl.mutualAuthentication">REQUIRED</property>

tests/integration/backward_compatible/ssl_tests/hazelcast-ssl.xml

@@ -5,10 +5,10 @@
     <network>
         <ssl enabled="true">
             <factory-class-name>
-                com.hazelcast.nio.ssl.ClasspathSSLContextFactory
+                com.hazelcast.nio.ssl.BasicSSLContextFactory
             </factory-class-name>
             <properties>
-                <property name="keyStore">com/hazelcast/nio/ssl-mutual-auth/server1.keystore</property>
+                <property name="keyStore">%s</property>
                 <property name="keyStorePassword">password</property>
                 <property name="keyManagerAlgorithm">SunX509</property>
                 <property name="protocol">TLSv1.2</property>