Fix for Bug#45554 (Bug#11754018), Connector/J does not encode binary data if useServerPrepStatements=false.

fjssilva · fjssilva · commit bf0ecd160b43 · 2025-07-31T10:48:44.000+01:00
Change-Id: I9699429d9e7c29d43d2bb1c28cd928c4fe972930
diff --git a/CHANGES b/CHANGES
@@ -3,6 +3,8 @@
 
 Version 9.5.0
 
+  - Fix for Bug#45554 (Bug#11754018), Connector/J does not encode binary data if useServerPrepStatements=false.
+
   - Fix for Bug#114974 (Bug#36614381), the SQL in batch will not clear after statement close.
     Thanks to Chengyi Dong for his contribution.
 
diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/InputStreamValueEncoder.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/InputStreamValueEncoder.java
@@ -61,7 +61,7 @@ public void encodeAsBinary(Message msg, BindValue binding) {
 
     protected byte[] streamToBytes(InputStream in, long length, NativePacketPayload packet) {
         boolean useLength = length == -1 ? false : this.propertySet.getBooleanProperty(PropertyKey.useStreamLengthsInPrepStmts).getValue();
-        in.mark(Integer.MAX_VALUE); // we may need to read this same stream several times, so we need to reset it at the end.
+        in.mark(Integer.MAX_VALUE); // We may need to read this same stream several times, so we need to reset it at the end.
         try {
             if (this.streamConvertBuf == null) {
                 this.streamConvertBuf = new byte[4096];
@@ -71,27 +71,21 @@ protected byte[] streamToBytes(InputStream in, long length, NativePacketPayload
             int lengthLeftToRead = (int) (length - bcnt);
 
             ByteArrayOutputStream bytesOut = null;
-            boolean hexEscape = false;
             if (packet == null) {
                 bytesOut = new ByteArrayOutputStream();
             } else {
-                hexEscape = this.serverSession.isNoBackslashEscapesSet();
-                packet.writeBytes(StringLengthDataType.STRING_FIXED, StringUtils.getBytes(hexEscape ? "x" : "_binary"));
+                packet.writeBytes(StringLengthDataType.STRING_FIXED, StringUtils.getBytes("X"));
                 packet.writeInteger(IntegerDataType.INT1, (byte) '\'');
             }
 
             while (bcnt > 0) {
                 if (packet == null) {
                     bytesOut.write(this.streamConvertBuf, 0, bcnt);
                 } else {
-                    if (hexEscape) {
-                        StringUtils.hexEscapeBlock(this.streamConvertBuf, bcnt, (lowBits, highBits) -> {
-                            packet.writeInteger(IntegerDataType.INT1, lowBits);
-                            packet.writeInteger(IntegerDataType.INT1, highBits);
-                        });
-                    } else {
-                        escapeblockFast(this.streamConvertBuf, packet, bcnt);
-                    }
+                    StringUtils.hexEscapeBlock(this.streamConvertBuf, bcnt, (lowBits, highBits) -> {
+                        packet.writeInteger(IntegerDataType.INT1, lowBits);
+                        packet.writeInteger(IntegerDataType.INT1, highBits);
+                    });
                 }
 
                 if (useLength) {
@@ -127,40 +121,4 @@ protected byte[] streamToBytes(InputStream in, long length, NativePacketPayload
         }
     }
 
-    private final void escapeblockFast(byte[] buf, NativePacketPayload packet, int size) {
-        int lastwritten = 0;
-
-        for (int i = 0; i < size; i++) {
-            byte b = buf[i];
-
-            if (b == '\0') {
-                // write stuff not yet written
-                if (i > lastwritten) {
-                    packet.writeBytes(StringLengthDataType.STRING_FIXED, buf, lastwritten, i - lastwritten);
-                }
-
-                // write escape
-                packet.writeInteger(IntegerDataType.INT1, (byte) '\\');
-                packet.writeInteger(IntegerDataType.INT1, (byte) '0');
-                lastwritten = i + 1;
-            } else {
-                if (b == '\\' || b == '\'') {
-                    // write stuff not yet written
-                    if (i > lastwritten) {
-                        packet.writeBytes(StringLengthDataType.STRING_FIXED, buf, lastwritten, i - lastwritten);
-                    }
-
-                    // write escape
-                    packet.writeInteger(IntegerDataType.INT1, b);
-                    lastwritten = i; // not i+1 as b wasn't written.
-                }
-            }
-        }
-
-        // write out remaining stuff from buffer
-        if (lastwritten < size) {
-            packet.writeBytes(StringLengthDataType.STRING_FIXED, buf, lastwritten, size - lastwritten);
-        }
-    }
-
-}
+}
diff --git a/src/test/java/testsuite/BaseTestCase.java b/src/test/java/testsuite/BaseTestCase.java
@@ -830,9 +830,7 @@ protected String randomString() {
 
     protected void cleanupTempFiles(final File exampleTempFile, final String tempfilePrefix) {
         File tempfilePath = exampleTempFile.getParentFile();
-
         File[] possibleFiles = tempfilePath.listFiles((dir, name) -> name.indexOf(tempfilePrefix) != -1 && !exampleTempFile.getName().equals(name));
-
         if (possibleFiles != null) {
             for (int i = 0; i < possibleFiles.length; i++) {
                 try {
diff --git a/src/test/java/testsuite/regression/StatementRegressionTest.java b/src/test/java/testsuite/regression/StatementRegressionTest.java
@@ -42,6 +42,7 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.PrintStream;
 import java.io.Reader;
 import java.io.StringReader;
@@ -51,6 +52,7 @@
 import java.math.BigInteger;
 import java.net.URL;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.sql.Array;
 import java.sql.BatchUpdateException;
@@ -14402,4 +14404,47 @@ public void testBug114974() throws Exception {
         assertFalse(this.rs.next());
     }
 
+    /**
+     * Tests fix for Bug#45554 (Bug#11754018), Connector/J does not encode binary data if useServerPrepStatements=false.
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testBug45554() throws Exception {
+        final String testString = "€\\\\";
+        final byte[] testStringBytes = testString.getBytes();
+
+        Path filePath = Files.createTempFile("testBug45554", ".dat");
+        OutputStream os = Files.newOutputStream(filePath);
+        os.write(testString.getBytes());
+        os.close();
+
+        boolean useSPS = false;
+        String[] charSets = new String[] { "UTF8", "GBK", "SJIS" };
+
+        do {
+            for (String cs : charSets) {
+                String testCase = String.format("Case [useSPS: %s,  charset: %s]", useSPS ? "Y" : "N", cs);
+
+                Properties props = new Properties();
+                props.setProperty(PropertyKey.useServerPrepStmts.getKeyName(), Boolean.toString(useSPS));
+                props.setProperty(PropertyKey.characterEncoding.getKeyName(), cs);
+
+                try (Connection testConn = getConnectionWithProps(props)) {
+                    this.rs = this.stmt.executeQuery("SELECT '" + testString.replace("\\", "\\\\") + "'");
+                    assertTrue(this.rs.next(), testCase);
+                    assertArrayEquals(testStringBytes, this.rs.getBytes(1), testCase);
+
+                    InputStream is = Files.newInputStream(filePath);
+                    this.pstmt = testConn.prepareStatement("SELECT ?");
+                    this.pstmt.setBinaryStream(1, is);
+                    this.rs = this.pstmt.executeQuery();
+                    assertTrue(this.rs.next(), testCase);
+                    assertArrayEquals(testStringBytes, this.rs.getBytes(1), testCase);
+                    is.close();
+                }
+            }
+        } while (useSPS = !useSPS);
+    }
+
 }
diff --git a/src/test/java/testsuite/simple/ConnectionTest.java b/src/test/java/testsuite/simple/ConnectionTest.java
@@ -26,7 +26,6 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
-import static org.junit.jupiter.api.Assumptions.assumeFalse;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 import java.io.BufferedInputStream;
@@ -849,39 +848,22 @@ public void testUseCompress() throws Exception {
         this.rs = this.stmt.executeQuery("SHOW VARIABLES LIKE 'max_allowed_packet'");
         this.rs.next();
         long defaultMaxAllowedPacket = this.rs.getInt(2);
-        boolean changeMaxAllowedPacket = defaultMaxAllowedPacket < 4 + 1024 * 1024 * 32 - 1;
-
-        if (versionMeetsMinimum(5, 6, 20) && !versionMeetsMinimum(5, 7)) {
-            /*
-             * The 5.6.20 patch for Bug #16963396, Bug #19030353, Bug #69477 limits the size of redo log BLOB writes
-             * to 10% of the redo log file size. The 5.7.5 patch addresses the bug without imposing a limitation.
-             * As a result of the redo log BLOB write limit introduced for MySQL 5.6, innodb_log_file_size should be set to a value
-             * greater than 10 times the largest BLOB data size found in the rows of your tables plus the length of other variable length
-             * fields (VARCHAR, VARBINARY, and TEXT type fields).
-             */
-            this.rs = this.stmt.executeQuery("SHOW VARIABLES LIKE 'innodb_log_file_size'");
-            this.rs.next();
-            assumeFalse(this.rs.getInt(2) < 1024 * 1024 * 32 * 10,
-                    "You need to increase innodb_log_file_size to at least " + 1024 * 1024 * 32 * 10 + " before running this test!");
-        }
+
+        createTable("testUseCompress", "(pos INT PRIMARY KEY AUTO_INCREMENT, blobdata LONGBLOB)");
 
         try {
-            if (changeMaxAllowedPacket) {
-                this.stmt.executeUpdate("SET GLOBAL max_allowed_packet=" + 1024 * 1024 * 33);
-            }
+            this.stmt.executeUpdate("SET GLOBAL max_allowed_packet=" + 1024 * 1024 * 33);
 
-            testCompressionWith("false", 1024 * 1024 * 16 - 2); // no split
-            testCompressionWith("false", 1024 * 1024 * 16 - 1); // split with additional empty packet
-            testCompressionWith("false", 1024 * 1024 * 32);   // big payload
+            testUseCompress("false", 1024 * 1024 * 16 - 2); // No split.
+            testUseCompress("false", 1024 * 1024 * 16 - 1); // Split with additional empty packet.
+            testUseCompress("false", 1024 * 1024 * 32); // Big payload.
 
-            testCompressionWith("true", 1024 * 1024 * 16 - 2 - 3); // no split, one compressed packet
-            testCompressionWith("true", 1024 * 1024 * 16 - 2 - 2); // no split, two compressed packets
-            testCompressionWith("true", 1024 * 1024 * 16 - 1);   // split with additional empty packet, two compressed packets
-            testCompressionWith("true", 1024 * 1024 * 32);     // big payload
+            testUseCompress("true", 1024 * 1024 * 16 - 2 - 3); // No split, one compressed packet.
+            testUseCompress("true", 1024 * 1024 * 16 - 2 - 2); // No split, two compressed packets.
+            testUseCompress("true", 1024 * 1024 * 16 - 1); // Split with additional empty packet, two compressed packets.
+            testUseCompress("true", 1024 * 1024 * 32); // Big payload.
         } finally {
-            if (changeMaxAllowedPacket) {
-                this.stmt.executeUpdate("SET GLOBAL max_allowed_packet=" + defaultMaxAllowedPacket);
-            }
+            this.stmt.executeUpdate("SET GLOBAL max_allowed_packet=" + defaultMaxAllowedPacket);
         }
     }
 
@@ -891,62 +873,59 @@ public void testUseCompress() throws Exception {
      *
      * @throws Exception
      */
-    private void testCompressionWith(String useCompression, int maxPayloadSize) throws Exception {
-        String sqlToSend = "INSERT INTO BLOBTEST(blobdata) VALUES (?)";
-        int requiredSize = maxPayloadSize - sqlToSend.length() - "_binary''".length();
+    private void testUseCompress(String useCompression, int maxPayloadSize) throws Exception {
+        this.stmt.executeUpdate("TRUNCATE TABLE testUseCompress");
 
-        File testBlobFile = File.createTempFile("cmj-testblob", ".dat");
-        testBlobFile.deleteOnExit();
+        String sqlToSend = "INSERT INTO testUseCompress (blobdata) VALUES (?)";
+        int remainingSize = maxPayloadSize - sqlToSend.length() - "X''".length() - 2 /* parameter_count & parameter_set_count */;
+        if (remainingSize % 2 != 0) {
+            sqlToSend += " ";
+            remainingSize--;
+        }
+        int requiredSize = remainingSize / 2; // HEX requires twice the size of the data.
 
-        // TODO: following cleanup doesn't work correctly during concurrent execution of testsuite
-        // cleanupTempFiles(testBlobFile, "cmj-testblob");
+        File testBlobFile = File.createTempFile("testUseCompress", ".dat");
+        testBlobFile.deleteOnExit();
 
         BufferedOutputStream bOut = new BufferedOutputStream(new FileOutputStream(testBlobFile));
-
-        // generate a random sequence of letters. this ensures that no escaped characters cause packet sizes that interfere with bounds tests
+        // Generate a random sequence of letters.
         Random random = new Random();
         for (int i = 0; i < requiredSize; i++) {
             bOut.write((byte) (65 + random.nextInt(26)));
         }
-
         bOut.flush();
         bOut.close();
 
         Properties props = new Properties();
         props.setProperty(PropertyKey.sslMode.getKeyName(), SslMode.DISABLED.name());
         props.setProperty(PropertyKey.allowPublicKeyRetrieval.getKeyName(), "true");
         props.setProperty(PropertyKey.useCompression.getKeyName(), useCompression);
-        Connection conn1 = getConnectionWithProps(props);
-        Statement stmt1 = conn1.createStatement();
+        Connection testConn = getConnectionWithProps(props);
 
-        createTable("BLOBTEST", "(pos int PRIMARY KEY auto_increment, blobdata LONGBLOB)");
+        PreparedStatement testPstmt = testConn.prepareStatement(sqlToSend);
         BufferedInputStream bIn = new BufferedInputStream(new FileInputStream(testBlobFile));
+        testPstmt.setBinaryStream(1, bIn, (int) testBlobFile.length());
+        testPstmt.execute();
+        testPstmt.clearParameters();
+        bIn.close();
 
-        this.pstmt = conn1.prepareStatement(sqlToSend);
-
-        this.pstmt.setBinaryStream(1, bIn, (int) testBlobFile.length());
-        this.pstmt.execute();
-        this.pstmt.clearParameters();
-
-        this.rs = stmt1.executeQuery("SELECT blobdata from BLOBTEST LIMIT 1");
+        Statement testStmt = testConn.createStatement();
+        this.rs = testStmt.executeQuery("SELECT blobdata FROM testUseCompress LIMIT 1");
         this.rs.next();
         InputStream is = this.rs.getBinaryStream(1);
 
-        bIn.close();
         bIn = new BufferedInputStream(new FileInputStream(testBlobFile));
-        int blobbyte = 0;
-        int count = 0;
-        while ((blobbyte = is.read()) > -1) {
-            int filebyte = bIn.read();
-            assertFalse(filebyte < 0 || filebyte != blobbyte, "Blob is not identical to initial data.");
-            count++;
-        }
-        assertEquals(requiredSize, count);
+        int blobByte = 0;
+        int blobSize = 0;
+        while ((blobByte = is.read()) > -1) {
+            int fileByte = bIn.read();
+            assertFalse(fileByte < 0 || fileByte != blobByte, "Blob is not identical to initial data.");
+            blobSize++;
+        }
+        assertEquals(requiredSize, blobSize);
+        bIn.close();
 
         is.close();
-        if (bIn != null) {
-            bIn.close();
-        }
     }
 
     /**
