Fix serial monitor output cut short with multibyte (UTF-8) data

app/test/processing/app/SerialTest.java

@@ -0,0 +1,58 @@
+/*
+ * This file is part of Arduino.
+ *
+ * Copyright 2020 Arduino LLC (http://www.arduino.cc/)
+ *
+ * Arduino is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As a special exception, you may use this file as part of a free software
+ * library without restriction.  Specifically, if other files instantiate
+ * templates or use macros or inline functions from this file, or you compile
+ * this file and link it with other files to produce an executable, this
+ * file does not by itself cause the resulting executable to be covered by
+ * the GNU General Public License.  This exception does not however
+ * invalidate any other reasons why the executable file might be covered by
+ * the GNU General Public License.
+ */
+
+package processing.app;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class SerialTest {
+  class NullSerial extends Serial {
+    public NullSerial() throws SerialException {
+      super("none", 0, 'n', 0, 0, false, false);
+    }
+
+    @Override
+    protected void message(char[] chars, int length) {
+      output += new String(chars, 0, length);
+    }
+
+    String output = "";
+  }
+
+  @Test
+  public void testSerialUTF8Decoder() throws Exception {
+    NullSerial s = new NullSerial();
+    // https://github.com/arduino/Arduino/issues/9808
+    String testdata = "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789°0123456789";
+    s.processSerialEvent(testdata.getBytes());
+    assertEquals(s.output, testdata);
+  }
+}

arduino-core/src/processing/app/Serial.java

@@ -116,7 +116,7 @@ public static boolean touchForCDCReset(String iname) throws SerialException {
     }
   }
 
-  private Serial(String iname, int irate, char iparity, int idatabits, float istopbits, boolean setRTS, boolean setDTR) throws SerialException {
+  protected Serial(String iname, int irate, char iparity, int idatabits, float istopbits, boolean setRTS, boolean setDTR) throws SerialException {
     //if (port != null) port.close();
     //this.parent = parent;
     //parent.attach(this);
@@ -131,6 +131,11 @@ private Serial(String iname, int irate, char iparity, int idatabits, float istop
     if (istopbits == 1.5f) stopbits = SerialPort.STOPBITS_1_5;
     if (istopbits == 2) stopbits = SerialPort.STOPBITS_2;
 
+    // This is required for unit-testing
+    if (iname.equals("none")) {
+      return;
+    }
+
     try {
       port = new SerialPort(iname);
       port.openPort();
@@ -175,31 +180,54 @@ public synchronized void serialEvent(SerialPortEvent serialEvent) {
     if (serialEvent.isRXCHAR()) {
       try {
         byte[] buf = port.readBytes(serialEvent.getEventValue());
-        int next = 0;
-        while(next < buf.length) {
-          while(next < buf.length && outToMessage.hasRemaining()) {
-            int spaceInIn = inFromSerial.remaining();
-            int copyNow = buf.length - next < spaceInIn ? buf.length - next : spaceInIn;
-            inFromSerial.put(buf, next, copyNow);
-            next += copyNow;
-            inFromSerial.flip();
-            bytesToStrings.decode(inFromSerial, outToMessage, false);
-            inFromSerial.compact();
-          }
-          outToMessage.flip();
-          if(outToMessage.hasRemaining()) {
-            char[] chars = new char[outToMessage.remaining()];
-            outToMessage.get(chars);
-            message(chars, chars.length);
-          }
-          outToMessage.clear();
-        }
+        processSerialEvent(buf);
       } catch (SerialPortException e) {
         errorMessage("serialEvent", e);
       }
     }
   }
 
+  public void processSerialEvent(byte[] buf) {
+    int next = 0;
+    // This uses a CharsetDecoder to convert from bytes to UTF-8 in
+    // a streaming fashion (i.e. where characters might be split
+    // over multiple reads). This needs the data to be in a
+    // ByteBuffer (inFromSerial, which we also use to store leftover
+    // incomplete characters for the nexst run) and produces a
+    // CharBuffer (outToMessage), which we then convert to char[] to
+    // pass onwards.
+    // Note that these buffers switch from input to output mode
+    // using flip/compact/clear
+    while (next < buf.length || inFromSerial.position() > 0) {
+      do {
+        // This might be 0 when all data was already read from buf
+        // (but then there will be data in inFromSerial left to
+        // decode).
+        int copyNow = Math.min(buf.length - next, inFromSerial.remaining());
+        inFromSerial.put(buf, next, copyNow);
+        next += copyNow;
+
+        inFromSerial.flip();
+        bytesToStrings.decode(inFromSerial, outToMessage, false);
+        inFromSerial.compact();
+
+        // When there are multi-byte characters, outToMessage might
+        // still have room, so add more bytes if we have any.
+      } while (next < buf.length && outToMessage.hasRemaining());
+
+      // If no output was produced, the input only contained
+      // incomplete characters, so we're done processing
+      if (outToMessage.position() == 0)
+        break;
+
+      outToMessage.flip();
+      char[] chars = new char[outToMessage.remaining()];
+      outToMessage.get(chars);
+      message(chars, chars.length);
+      outToMessage.clear();
+    }
+  }
+
   /**
    * This method is intented to be extended to receive messages
    * coming from serial port.