Issue 209: fix support for utf8 strings in attribute and field values

.github/workflows/tests.yml

@@ -88,28 +88,3 @@ jobs:
       - name: Stop the docker
         run: docker container stop ndts
 
-  python2_tests:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        os: [debian10]
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-
-      - name: Build the docker
-        env:
-          OS: ${{ matrix.os }}
-        run: docker build -t ndts .ci/${OS}_py2
-
-      - name: Run the docker
-        run: docker run --name  ndts -d -it -v `pwd`:/home/tango  ndts
-
-      - name: install python-pninexus
-        run: .ci/install.sh 2
-
-      - name: run tests
-        run: .ci/run.sh 2
-
-      - name: Stop the docker
-        run: docker container stop ndts

src/pninexus/h5cpp/attribute/__init__.py

@@ -20,15 +20,28 @@ def attribute_write(self, data):
     write_data = data
     if not isinstance(write_data, numpy.ndarray):
         write_data = numpy.array(write_data)
-
     if write_data.dtype.kind == 'U':
-        write_data = write_data.astype("S")
+        try:
+            write_data = write_data.astype("S")
+        except Exception:
+            if isinstance(data, numpy.ndarray) and data.shape:
+                shape = data.shape
+                if len(shape) > 1:
+                    data = data.flatten()
+                write_data = numpy.array(
+                    [bytes(str(dt).encode('utf-8')) for dt in data])
+                if len(shape) > 1:
+                    write_data = write_data.reshape(shape)
+            else:
+                write_data = numpy.array(str(data).encode('utf-8'))
     elif write_data.dtype == 'bool':
         write_data = write_data.astype("int8")
 
+    #  print("DATA", data, write_data)
     try:
         self._write(write_data)
-    except RuntimeError:
+    except RuntimeError as e:
+        print(str(e))
         print(write_data, write_data.dtype)
 
 

src/pninexus/h5cpp/node/__init__.py

@@ -275,8 +275,19 @@ def dataset_write(self, data, selection=None):
     # if the data is a unicode numpy array we have to convert it to a
     # simple string array
     if data.dtype.kind == 'U':
-        data = data.astype('S')
-
+        try:
+            data = data.astype('S')
+        except Exception:
+            if isinstance(data, numpy.ndarray) and data.shape:
+                shape = data.shape
+                if len(shape) > 1:
+                    data = data.flatten()
+                data = numpy.array(
+                    [bytes(str(dt).encode('utf-8')) for dt in data])
+                if len(shape) > 1:
+                    data = data.reshape(shape)
+            else:
+                data = numpy.array(str(data).encode('utf-8'))
     #
     # determine memory datatype and dataspace
     # - if the file type is a variable length string we have to adjust the

test/h5cpp_tests/attribute_tests/attribute_io_test.py

@@ -118,7 +118,7 @@ def testStringScalarFixedLength(self):
         a = self.root.attributes.create("StringScalar", dtype)
         a.write("hello world")
         r = a.read()
-        self.assertEqual(r, "hello world")
+        self.assertEqual(r, data)
 
     def testStringScalarVariableLength(self):
 
@@ -128,6 +128,16 @@ def testStringScalarVariableLength(self):
         r = a.read()
         self.assertEqual(r, data)
 
+    def testStringUTF8ScalarVariableLength(self):
+
+        data = u"µm"
+        dtype = String.variable()
+        dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8
+        a = self.root.attributes.create("StringUTF8ScalarVLength", dtype)
+        a.write(data)
+        r = a.read()
+        self.assertEqual(r, data)
+
     def testStringArray(self):
 
         data = numpy.array([["hello", "world", "this"], ["is", "a", "test"]])
@@ -147,6 +157,17 @@ def testStringArrayVariableLength(self):
         r = a.read()
         npt.assert_array_equal(r, data)
 
+    def testStringUTF8ArrayVariableLength(self):
+
+        data = numpy.array([u"µm", u"µA"])
+        dtype = String.variable()
+        dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8
+        a = self.root.attributes.create(
+            "StringUTF8ArrayVLength", dtype, (2,))
+        a.write(data)
+        r = a.read()
+        npt.assert_array_equal(r, data)
+
     def testIntArray(self):
 
         data = numpy.array([1, 2, 3])

test/h5cpp_tests/node_tests/dataset_io_test.py

@@ -115,6 +115,17 @@ def testWriteVariableLengthScalar(self):
         read = dataset.read()
         self.assertEqual(read, "hello world")
 
+    def testWriteVariableLengthUTF8Scalar(self):
+        data = u"µm"
+        dtype = h5cpp.datatype.String.variable()
+        dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8
+        dataset = Dataset(
+            self.root, h5cpp.Path("VariableLengthStringUTF8Scalar"),
+            dtype, Scalar())
+        dataset.write(data)
+        read = dataset.read()
+        self.assertEqual(read, data)
+
     def testWriteIntegerArray(self):
 
         data = numpy.array([[1, 2, 3, 4], [5, 6, 7, 8]])
@@ -222,3 +233,17 @@ def testWriteVariableLengthStringArray(self):
         dataset.write(data)
         read = dataset.read()
         npt.assert_array_equal(read, data)
+
+    def testWriteVariableLengthStringUTF8Array(self):
+
+        data = numpy.array([u"µm", u"µA"])
+        dtype = h5cpp.datatype.String.variable()
+        dtype.encoding = h5cpp.datatype.CharacterEncoding.UTF8
+        dataset = Dataset(
+            self.root,
+            h5cpp.Path("VariableLengthStringUTF8Array"),
+            dtype,
+            Simple((2,)))
+        dataset.write(data)
+        read = dataset.read()
+        npt.assert_array_equal(read, data)