Issue 212: fix py2 utf8 support

.github/workflows/tests.yml

@@ -88,3 +88,28 @@ jobs:
       - name: Stop the docker
         run: docker container stop ndts
 
+  python2_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [debian10]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+
+      - name: Build the docker
+        env:
+          OS: ${{ matrix.os }}
+        run: docker build -t ndts .ci/${OS}_py2
+
+      - name: Run the docker
+        run: docker run --name  ndts -d -it -v `pwd`:/home/tango  ndts
+
+      - name: install python-pninexus
+        run: .ci/install.sh 2
+
+      - name: run tests
+        run: .ci/run.sh 2
+
+      - name: Stop the docker
+        run: docker container stop ndts

doc/sphinx/conf.py

@@ -160,7 +160,7 @@
 
 html_context = {
     "docs_versions" : [
-        "v3.2.3", "v3.2.2", "v3.2.1", "v3.2.0", "v3.1.0",
+        "v3.2.4", "v3.2.3", "v3.2.2", "v3.2.1", "v3.2.0", "v3.1.0",
         "v3.0.3", "v3.0.2", "v3.0.1", "v3.0.0", "v2.0.0", "v1.3.4"]
 }
 

setup.py

@@ -20,8 +20,8 @@
     BuildDoc = None
 
 name = "pninexus"
-version = "3.2.3"
-release = "3.2.3"
+version = "3.2.4"
+release = "3.2.4"
 # release = "3.2"
 
 if release.count(".") == 1:

src/pninexus/h5cpp/attribute/__init__.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
 import numpy
+import sys
 
 from pninexus.h5cpp import property
 from pninexus.h5cpp._attribute import AttributeManager
@@ -8,6 +9,10 @@
 __all__ = ["property", "AttributeManager", "Attribute"]
 
 
+if sys.version_info > (3,):
+    unicode = str
+
+
 def attribute__getitem__(self, index):
 
     data = self.read()
@@ -29,11 +34,11 @@ def attribute_write(self, data):
                 if len(shape) > 1:
                     data = data.flatten()
                 write_data = numpy.array(
-                    [bytes(str(dt).encode('utf-8')) for dt in data])
+                    [bytes(unicode(dt).encode('utf-8')) for dt in data])
                 if len(shape) > 1:
                     write_data = write_data.reshape(shape)
             else:
-                write_data = numpy.array(str(data).encode('utf-8'))
+                write_data = numpy.array(unicode(data).encode('utf-8'))
     elif write_data.dtype == 'bool':
         write_data = write_data.astype("int8")
 

src/pninexus/h5cpp/node/__init__.py

@@ -5,6 +5,7 @@
 from pninexus.h5cpp import datatype
 from pninexus.h5cpp.filter import ExternalFilters
 import numpy
+import sys
 # from collections import OrderedDict
 
 #
@@ -45,6 +46,10 @@
     VDSAvailable = False
 
 
+if sys.version_info > (3,):
+    unicode = str
+
+
 def copy(node, base, path=None, link_creation_list=property.LinkCreationList(),
          object_copy_list=property.ObjectCopyList()):
     """Copy an object within the HDF5 tree
@@ -283,11 +288,11 @@ def dataset_write(self, data, selection=None):
                 if len(shape) > 1:
                     data = data.flatten()
                 data = numpy.array(
-                    [bytes(str(dt).encode('utf-8')) for dt in data])
+                    [bytes(unicode(dt).encode('utf-8')) for dt in data])
                 if len(shape) > 1:
                     data = data.reshape(shape)
             else:
-                data = numpy.array(str(data).encode('utf-8'))
+                data = numpy.array(unicode(data).encode('utf-8'))
     #
     # determine memory datatype and dataspace
     # - if the file type is a variable length string we have to adjust the

test/h5cpp_tests/attribute_tests/attribute_io_test.py

@@ -24,6 +24,7 @@
 import unittest
 import numpy
 import os
+import sys
 import numpy.testing as npt
 from pninexus import h5cpp
 from pninexus.h5cpp.file import AccessFlags
@@ -130,13 +131,17 @@ def testStringScalarVariableLength(self):
 
     def testStringUTF8ScalarVariableLength(self):
 
-        data = u"µm"
+        data = u"\u03bcm"
+        bdata = b"\xce\xbcm"
         dtype = String.variable()
         dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8
         a = self.root.attributes.create("StringUTF8ScalarVLength", dtype)
         a.write(data)
         r = a.read()
-        self.assertEqual(r, data)
+        if sys.version_info > (3,):
+            self.assertEqual(r, data)
+        else:
+            self.assertEqual(r, bdata)
 
     def testStringArray(self):
 
@@ -159,14 +164,18 @@ def testStringArrayVariableLength(self):
 
     def testStringUTF8ArrayVariableLength(self):
 
-        data = numpy.array([u"µm", u"µA"])
+        data = numpy.array([u"\u03bcm", u"\u03bcA"])
+        bdata = numpy.array([b'\xce\xbcm', b'\xce\xbcA'])
         dtype = String.variable()
         dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8
         a = self.root.attributes.create(
             "StringUTF8ArrayVLength", dtype, (2,))
         a.write(data)
         r = a.read()
-        npt.assert_array_equal(r, data)
+        if sys.version_info > (3,):
+            npt.assert_array_equal(r, data)
+        else:
+            npt.assert_array_equal(r, bdata)
 
     def testIntArray(self):
 

test/h5cpp_tests/node_tests/dataset_io_test.py

@@ -23,6 +23,7 @@
 from __future__ import print_function
 import unittest
 import os
+import sys
 from pninexus import h5cpp
 from pninexus.h5cpp.file import AccessFlags
 from pninexus.h5cpp.node import Dataset
@@ -116,15 +117,19 @@ def testWriteVariableLengthScalar(self):
         self.assertEqual(read, "hello world")
 
     def testWriteVariableLengthUTF8Scalar(self):
-        data = u"µm"
+        data = u"\u03bcm"
+        bdata = b"\xce\xbcm"
         dtype = h5cpp.datatype.String.variable()
         dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8
         dataset = Dataset(
             self.root, h5cpp.Path("VariableLengthStringUTF8Scalar"),
             dtype, Scalar())
         dataset.write(data)
         read = dataset.read()
-        self.assertEqual(read, data)
+        if sys.version_info > (3,):
+            self.assertEqual(read, data)
+        else:
+            self.assertEqual(read, bdata)
 
     def testWriteIntegerArray(self):
 
@@ -236,7 +241,8 @@ def testWriteVariableLengthStringArray(self):
 
     def testWriteVariableLengthStringUTF8Array(self):
 
-        data = numpy.array([u"µm", u"µA"])
+        data = numpy.array([u"\u03bcm", u"\u03bcA"])
+        bdata = numpy.array([b'\xce\xbcm', b'\xce\xbcA'])
         dtype = h5cpp.datatype.String.variable()
         dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8
         dataset = Dataset(
@@ -246,4 +252,7 @@ def testWriteVariableLengthStringUTF8Array(self):
             Simple((2,)))
         dataset.write(data)
         read = dataset.read()
-        npt.assert_array_equal(read, data)
+        if sys.version_info > (3,):
+            npt.assert_array_equal(read, data)
+        else:
+            npt.assert_array_equal(read, bdata)