fix utf8 issues in jpostal

build.gradle

@@ -7,13 +7,20 @@ plugins {
     id 'maven-publish'
 }
 
-// The latest release is 1.1.0, but there are many unreleased changes in the master branch
-version = '1.2.0'
+// The latest libpostal release is 1.1.0, but there are many unreleased changes in the master branch
+version = '1.2.1'
 
 repositories {
     mavenCentral()
 }
 
+tasks.withType(JavaCompile) {
+    options.encoding = 'UTF-8'
+}
+test {
+    systemProperty 'file.encoding', 'UTF-8'
+}
+
 dependencies {
     testImplementation 'junit:junit:4.+'
     implementation 'org.apache.commons:commons-compress:1.26.2'

src/jpostal/c/jpostal_AddressExpander.c

@@ -26,8 +26,17 @@ JNIEXPORT void JNICALL Java_com_mapzen_jpostal_AddressExpander_setupDataDir
 }
 
 JNIEXPORT jobjectArray JNICALL Java_com_mapzen_jpostal_AddressExpander_libpostalExpand
-  (JNIEnv *env, jclass cls, jstring jAddress, jobject jOptions) {
-    const char *address = (*env)->GetStringUTFChars(env, jAddress, 0);
+  (JNIEnv *env, jclass cls, jbyteArray jAddress, jobject jOptions) {
+    jbyte* addressElements = (*env)->GetByteArrayElements(env, jAddress, NULL);
+    jsize size = (*env)->GetArrayLength(env, jAddress);
+    char address[size + 1];
+
+    for (size_t z = 0; z < size; z++) {
+        address[z] = addressElements[z];
+    }
+    (*env) -> ReleaseByteArrayElements(env, jAddress, addressElements, 0);
+
+    address[size] = '\0';
 
     size_t num_expansions = 0;
     libpostal_normalize_options_t options = libpostal_get_default_options();
@@ -210,16 +219,17 @@ JNIEXPORT jobjectArray JNICALL Java_com_mapzen_jpostal_AddressExpander_libpostal
 
     char **expansions = libpostal_expand_address((char *)address, options, &num_expansions);
 
-    (*env)->ReleaseStringUTFChars(env, jAddress, address);
 
     jobjectArray ret = (jobjectArray)(*env)->NewObjectArray(env,
                                                             num_expansions,
-                                                            (*env)->FindClass(env, "java/lang/String"),
+                                                            (*env)->FindClass(env, "[B"),
                                                             (*env)->NewStringUTF(env, ""));
 
     if (num_expansions > 0) {
         for (size_t i = 0; i < num_expansions; i++) {
-            (*env)->SetObjectArrayElement(env, ret, i, (*env)->NewStringUTF(env, expansions[i]));
+            jbyteArray bytes = (*env)->NewByteArray(env,strlen(expansions[i]));
+            (*env)->SetByteArrayRegion(env, bytes, 0, strlen(expansions[i]), (jbyte*) expansions[i]);
+            (*env)->SetObjectArrayElement(env, ret, i, bytes);
         }
 
     }
@@ -409,4 +419,4 @@ JNIEXPORT void JNICALL Java_com_mapzen_jpostal_ExpanderOptions_00024Builder_setD
     }
 
     (*env)->SetBooleanField(env, builder, fid, default_options.roman_numerals);
-}
+}

src/jpostal/c/jpostal_AddressParser.c

@@ -1,5 +1,6 @@
 #include <jni.h>
 #include <libpostal/libpostal.h>
+#include <string.h>
 
 JNIEXPORT void JNICALL Java_com_mapzen_jpostal_AddressParser_setup
   (JNIEnv *env, jclass cls) {
@@ -24,9 +25,19 @@ JNIEXPORT void JNICALL Java_com_mapzen_jpostal_AddressParser_setupDataDir
 }
 
 JNIEXPORT jobjectArray JNICALL Java_com_mapzen_jpostal_AddressParser_libpostalParse
-  (JNIEnv *env, jobject thisObj, jstring jAddress, jobject jOptions) {
+  (JNIEnv *env, jobject thisObj, jbyteArray jAddress, jobject jOptions) {
+
+    jbyte* addressElements = (*env)->GetByteArrayElements(env, jAddress, NULL);
+    jsize size = (*env)->GetArrayLength(env, jAddress);
+    char address[size + 1];
+
+    for (int i = 0; i < size; ++i) {
+        address[i] = addressElements[i];
+    }
+    (*env) -> ReleaseByteArrayElements(env, jAddress, addressElements, 0);
+
+    address[size] = '\0';
 
-    const char *address = (*env)->GetStringUTFChars(env, jAddress, 0);
 
     libpostal_address_parser_options_t options = libpostal_get_address_parser_default_options();
 
@@ -58,8 +69,6 @@ JNIEXPORT jobjectArray JNICALL Java_com_mapzen_jpostal_AddressParser_libpostalPa
 
     libpostal_address_parser_response_t *response = libpostal_parse_address((char *)address, options);
 
-    (*env)->ReleaseStringUTFChars(env, jAddress, address);
-
     if (jLanguage != NULL) {
         (*env)->ReleaseStringUTFChars(env, jLanguage, 0);
     }
@@ -71,7 +80,7 @@ JNIEXPORT jobjectArray JNICALL Java_com_mapzen_jpostal_AddressParser_libpostalPa
     jmethodID mid;
 
     jclass parsedComponentClass = (*env)->FindClass(env, "com/mapzen/jpostal/ParsedComponent");
-    mid = (*env)->GetMethodID(env, parsedComponentClass, "<init>", "(Ljava/lang/String;Ljava/lang/String;)V");
+    mid = (*env)->GetMethodID(env, parsedComponentClass, "<init>", "([BLjava/lang/String;)V");
 
     size_t num_components = response != NULL ? response->num_components : 0;
 
@@ -82,12 +91,16 @@ JNIEXPORT jobjectArray JNICALL Java_com_mapzen_jpostal_AddressParser_libpostalPa
 
     if (num_components > 0) {
         for (size_t i = 0; i < num_components; i++) {
-            jstring jComponent = (*env)->NewStringUTF(env, response->components[i]);
             jstring jLabel = (*env)->NewStringUTF(env, response->labels[i]);
-
-            jobject jParsedComponent = (*env)->NewObject(env, parsedComponentClass, mid, jComponent, jLabel);
-
+            jbyteArray bytes = (*env)->NewByteArray(env,strlen(response->components[i]));
+            (*env)->SetByteArrayRegion(env, bytes, 0, strlen(response->components[i]), (jbyte*) response->components[i]);
+            jobject jParsedComponent = (*env)->NewObject(env, parsedComponentClass, mid, bytes, jLabel);
             (*env)->SetObjectArrayElement(env, ret, i, jParsedComponent);
+
+            // // These might be necessary to help ensure we're not leaking memory in the cluster.
+            // (*env)->DeleteLocalRef(env, bytes);
+            // (*env)->DeleteLocalRef(env, jLabel);
+            // (*env)->DeleteLocalRef(env, jParsedComponent);
         }
     }
 

src/main/java/com/mapzen/jpostal/AddressExpander.java

@@ -2,7 +2,15 @@
 
 import com.mapzen.jpostal.ExpanderOptions;
 
+import java.nio.charset.StandardCharsets;
+
 public class AddressExpander {
+
+    private static native synchronized void setup();
+    private static native synchronized void setupDataDir(String dataDir);
+    private static native synchronized byte[][] libpostalExpand(byte[] address, ExpanderOptions options);
+    private static native synchronized void teardown();
+
     private volatile static AddressExpander instance = null;
 
     private final LibPostal libPostal;
@@ -13,7 +21,7 @@ public static AddressExpander getInstanceDataDir(String dataDir) {
 
     public static AddressExpander getInstanceConfig(Config config) {
         if (instance == null) {
-            synchronized(AddressParser.class) {
+            synchronized(AddressExpander.class) {
                 if (instance == null) {
                     instance = new AddressExpander(LibPostal.getInstance(config));
                 }
@@ -32,11 +40,6 @@ public static boolean isInitialized() {
         return instance != null;
     }
 
-    private static native void setup();
-    private static native void setupDataDir(String dataDir);
-    private static native String[] libpostalExpand(String address, ExpanderOptions options);
-    private static native void teardown();
-
     public String[] expandAddress(String address) {
         return expandAddressWithOptions(address, new ExpanderOptions.Builder().build());
     }
@@ -49,9 +52,12 @@ public String[] expandAddressWithOptions(String address, ExpanderOptions options
             throw new NullPointerException("ExpanderOptions options must not be null");
         }
 
-        synchronized(AddressExpander.class) {
-            return libpostalExpand(address, options);
+        byte[][] expansionBytes = libpostalExpand(address.getBytes(), options);
+        String[] expansions = new String[expansionBytes.length];
+        for (int i = 0; i < expansionBytes.length; i++) {
+            expansions[i] = new String(expansionBytes[i], StandardCharsets.UTF_8);
         }
+        return expansions;
     }
 
     AddressExpander(final LibPostal libPostal) {

src/main/java/com/mapzen/jpostal/AddressParser.java

@@ -4,10 +4,11 @@
 import com.mapzen.jpostal.ParserOptions;
 
 public class AddressParser {
-    private static native void setup();
-    private static native void setupDataDir(String dataDir);
-    private native ParsedComponent[] libpostalParse(String address, ParserOptions options);
-    private static native void teardown();
+
+    private static native synchronized void setup();
+    private static native synchronized void setupDataDir(String dataDir);
+    private native synchronized ParsedComponent[] libpostalParse(byte[] address, ParserOptions options);
+    private static native synchronized void teardown();
 
     private volatile static AddressParser instance = null;
 
@@ -49,10 +50,7 @@ public ParsedComponent[] parseAddressWithOptions(String address, ParserOptions o
         if (options == null) {
             throw new NullPointerException("ParserOptions options must not be null");
         }
-
-        synchronized(AddressParser.class) {
-            return libpostalParse(address, options);
-        }
+        return libpostalParse(address.getBytes(), options);
     } 
 
     AddressParser(final LibPostal libPostal) {

src/main/java/com/mapzen/jpostal/ParsedComponent.java

@@ -1,5 +1,7 @@
 package com.mapzen.jpostal;
 
+import java.nio.charset.StandardCharsets;
+
 public class ParsedComponent {
     private String value;
     private String label;
@@ -24,4 +26,9 @@ public ParsedComponent(String value, String label) {
         this.value = value;
         this.label = label;
     }
+
+    public ParsedComponent(byte[] value, String label) {
+        this.value = new String(value, StandardCharsets.UTF_8);
+        this.label = label;
+    }
 }

src/test/java/com/mapzen/jpostal/TestAddressExpander.java

@@ -93,4 +93,19 @@ public void testConfigMismatchLibraryFile() {
         );
         assertNull(thrown.getCause());
     }
+
+    @Test()
+    public void testNulTerminatedExpansion() {
+        assertTrue(containsExpansion("123 Main St\u0000", "123 main street"));
+    }
+
+    @Test()
+    public void testAltNulTerminatedExpansion() {
+        assertTrue(containsExpansion("123 Main St\0", "123 main street"));
+    }
+
+    @Test()
+    public void test4ByteCharacterExpansion() {
+        assertTrue(containsExpansion("123 Main St, 𠜎𠜱𠝹𠱓, 😀🤠", "123 main street 𠜎𠜱𠝹𠱓 😀🤠"));
+    }
 }

src/test/java/com/mapzen/jpostal/TestAddressParser.java

@@ -56,7 +56,6 @@ public void testParseUSAddress() {
                  );
     }
 
-
     @Test()
     public void testConfigMismatchDataDir() {
         AddressParser.getInstance();
@@ -86,4 +85,31 @@ public void testConfigMismatchLibraryFile() {
         );
         assertNull(thrown.getCause());
     }
+
+    @Test
+    public void testParseNulTerminatedAddress() {
+        testParse("Rue du Médecin-Colonel Calbairac Toulouse France\u0000", 
+                  new ParsedComponent("rue du médecin-colonel calbairac", "road"),
+                  new ParsedComponent("toulouse", "city"),
+                  new ParsedComponent("france", "country")
+                 );
+    }
+
+    @Test
+    public void testParseAltNulTerminatedAddress() {
+        testParse("Rue du Médecin-Colonel Calbairac Toulouse France\0", 
+                  new ParsedComponent("rue du médecin-colonel calbairac", "road"),
+                  new ParsedComponent("toulouse", "city"),
+                  new ParsedComponent("france", "country")
+                 );
+    }
+
+    @Test
+    public void testParse4ByteCharacterAddress() {
+        testParse("𠜎𠜱𠝹𠱓, 😀🤠, London, UK", 
+                  new ParsedComponent("𠜎𠜱𠝹𠱓 😀🤠", "house"),
+                  new ParsedComponent("london", "city"),
+                  new ParsedComponent("uk", "country")
+                 );
+    }
 }