Skip to content

Commit

Permalink
Workaround Bulk Copy for batch insert operation against specific data…
Browse files Browse the repository at this point in the history 
…types. (#912)

Fix | Workaround Bulk Copy for batch insert operation against specific datatypes. (#912)
  • Loading branch information
@peterbae
peterbae authored Jan 4, 2019
1 parent fc24284 commit d639870
Show file tree
Hide file tree
Showing 5 changed files with 265 additions and 203 deletions.
80 changes: 41 additions & 39 deletions src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkBatchInsertRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,12 +160,20 @@ private Object convertValue(ColumnMetadata cm, Object data) throws SQLServerExce
case Types.VARBINARY:
case Types.LONGVARBINARY:
case Types.BLOB: {
// Strip off 0x if present.
String binData = data.toString().trim();
if (binData.startsWith("0x") || binData.startsWith("0X")) {
return binData.substring(2);
if (data instanceof byte[]) {
/*
* if the binary data comes in as a byte array through setBytes through Bulk Copy for Batch Insert
* API, don't turn the binary array into a string.
*/
return data;
} else {
return binData;
// Strip off 0x if present.
String binData = data.toString().trim();
if (binData.startsWith("0x") || binData.startsWith("0X")) {
return binData.substring(2);
} else {
return binData;
}
}
}

Expand Down Expand Up @@ -229,8 +237,9 @@ public Object[] getRowData() throws SQLServerException {
Object rowData;
int columnListIndex = 0;

// check if the size of the list of values = size of the list of columns
// (which is optional)
/*
* check if the size of the list of values = size of the list of columns (which is optional)
*/
if (null != columnList && columnList.size() != valueList.size()) {
MessageFormat form = new MessageFormat(SQLServerException.getErrString("R_DataSchemaMismatch"));
Object[] msgArgs = {};
Expand All @@ -240,50 +249,43 @@ public Object[] getRowData() throws SQLServerException {
for (Entry<Integer, ColumnMetadata> pair : columnMetadata.entrySet()) {
int index = pair.getKey() - 1;

// To explain what each variable represents:
// columnMetadata = map containing the ENTIRE list of columns in the
// table.
// columnList = the *optional* list of columns the user can provide.
// For example, the (c1, c3) part of this query: INSERT into t1 (c1,
// c3) values (?, ?)
// valueList = the *mandatory* list of columns the user needs
// provide. This is the (?, ?) part of the previous query. The size
// of this valueList will always equal the number of
// the entire columns in the table IF columnList has NOT been
// provided. If columnList HAS been provided, then this valueList
// may be smaller than the list of all columns (which is
// columnMetadata).

// case when the user has not provided the optional list of column
// names.
/*
* To explain what each variable represents: columnMetadata = map containing the ENTIRE list of columns in
* the table. columnList = the *optional* list of columns the user can provide. For example, the (c1, c3)
* part of this query: INSERT into t1 (c1, c3) values (?, ?) valueList = the *mandatory* list of columns the
* user needs provide. This is the (?, ?) part of the previous query. The size of this valueList will always
* equal the number of the entire columns in the table IF columnList has NOT been provided. If columnList
* HAS been provided, then this valueList may be smaller than the list of all columns (which is
* columnMetadata).
*/
// case when the user has not provided the optional list of column names.
if (null == columnList || columnList.size() == 0) {
valueData = valueList.get(index);
// if the user has provided a wildcard for this column, fetch
// the set value from the batchParam.
/*
* if the user has provided a wildcard for this column, fetch the set value from the batchParam.
*/
if (valueData.equalsIgnoreCase("?")) {
rowData = batchParam.get(batchParamIndex)[valueIndex++].getSetterValue();
} else if (valueData.equalsIgnoreCase("null")) {
rowData = null;
}
// if the user has provided a hardcoded value for this column,
// rowData is simply set to the hardcoded value.
/*
* if the user has provided a hardcoded value for this column, rowData is simply set to the hardcoded
* value.
*/
else {
rowData = removeSingleQuote(valueData);
}
}
// case when the user has provided the optional list of column
// names.
// case when the user has provided the optional list of column names.
else {
// columnListIndex is a separate counter we need to keep track
// of for each time we've processed a column
// that the user provided.
// for example, if the user provided an optional columnList of
// (c1, c3, c5, c7) in a table that has 8 columns (c1~c8),
// then the columnListIndex would increment only when we're
// dealing with the four columns inside columnMetadata.
// compare the list of the optional list of column names to the
// table's metadata, and match each other, so we assign the
// correct value to each column.
/*
* columnListIndex is a separate counter we need to keep track of for each time we've processed a column
* that the user provided. for example, if the user provided an optional columnList of (c1, c3, c5, c7)
* in a table that has 8 columns (c1~c8), then the columnListIndex would increment only when we're
* dealing with the four columns inside columnMetadata. compare the list of the optional list of column
* names to the table's metadata, and match each other, so we assign the correct value to each column.
*/
if (columnList.size() > columnListIndex
&& columnList.get(columnListIndex).equalsIgnoreCase(columnMetadata.get(index + 1).columnName)) {
valueData = valueList.get(columnListIndex);
Expand Down
52 changes: 29 additions & 23 deletions src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1936,6 +1936,12 @@ public int[] executeBatch() throws SQLServerException, BatchUpdateException, SQL

try {
if (this.useBulkCopyForBatchInsert && connection.isAzureDW() && isInsert(localUserSQL)) {
if (null == batchParamValues) {
updateCounts = new int[0];
loggerExternal.exiting(getClassNameLogging(), "executeBatch", updateCounts);
return updateCounts;
}

// From the JDBC spec, section 9.1.4 - Making Batch Updates:
// The CallableStatement.executeBatch method (inherited from PreparedStatement) will
// throw a BatchUpdateException if the stored procedure returns anything other than an
Expand All @@ -1955,12 +1961,6 @@ public int[] executeBatch() throws SQLServerException, BatchUpdateException, SQL
}
}

if (batchParamValues == null) {
updateCounts = new int[0];
loggerExternal.exiting(getClassNameLogging(), "executeBatch", updateCounts);
return updateCounts;
}

String tableName = parseUserSQLForTableNameDW(false, false, false, false);
ArrayList<String> columnList = parseUserSQLForColumnListDW();
ArrayList<String> valueList = parseUserSQLForValueListDW(false);
Expand Down Expand Up @@ -2032,7 +2032,7 @@ public int[] executeBatch() throws SQLServerException, BatchUpdateException, SQL
}
}

if (batchParamValues == null)
if (null == batchParamValues)
updateCounts = new int[0];
else
try {
Expand Down Expand Up @@ -2093,6 +2093,12 @@ public long[] executeLargeBatch() throws SQLServerException, BatchUpdateExceptio

try {
if (this.useBulkCopyForBatchInsert && connection.isAzureDW() && isInsert(localUserSQL)) {
if (null == batchParamValues) {
updateCounts = new long[0];
loggerExternal.exiting(getClassNameLogging(), "executeLargeBatch", updateCounts);
return updateCounts;
}

// From the JDBC spec, section 9.1.4 - Making Batch Updates:
// The CallableStatement.executeBatch method (inherited from PreparedStatement) will
// throw a BatchUpdateException if the stored procedure returns anything other than an
Expand All @@ -2112,12 +2118,6 @@ public long[] executeLargeBatch() throws SQLServerException, BatchUpdateExceptio
}
}

if (batchParamValues == null) {
updateCounts = new long[0];
loggerExternal.exiting(getClassNameLogging(), "executeLargeBatch", updateCounts);
return updateCounts;
}

String tableName = parseUserSQLForTableNameDW(false, false, false, false);
ArrayList<String> columnList = parseUserSQLForColumnListDW();
ArrayList<String> valueList = parseUserSQLForValueListDW(false);
Expand Down Expand Up @@ -2189,7 +2189,7 @@ public long[] executeLargeBatch() throws SQLServerException, BatchUpdateExceptio
}
}

if (batchParamValues == null)
if (null == batchParamValues)
updateCounts = new long[0];
else
try {
Expand Down Expand Up @@ -2234,17 +2234,26 @@ public long[] executeLargeBatch() throws SQLServerException, BatchUpdateExceptio

private void checkValidColumns(TypeInfo ti) throws SQLServerException {
int jdbctype = ti.getSSType().getJDBCType().getIntValue();
// currently, we do not support: geometry, geography, datetime and smalldatetime
String typeName;
MessageFormat form;
switch (jdbctype) {
case microsoft.sql.Types.MONEY:
case microsoft.sql.Types.SMALLMONEY:
case java.sql.Types.DATE:
case microsoft.sql.Types.DATETIME:
case microsoft.sql.Types.DATETIMEOFFSET:
case microsoft.sql.Types.SMALLDATETIME:
case java.sql.Types.TIME:
typeName = ti.getSSTypeName();
form = new MessageFormat(SQLServerException.getErrString("R_BulkTypeNotSupportedDW"));
throw new IllegalArgumentException(form.format(new Object[] {typeName}));
case java.sql.Types.INTEGER:
case java.sql.Types.SMALLINT:
case java.sql.Types.BIGINT:
case java.sql.Types.BIT:
case java.sql.Types.TINYINT:
case java.sql.Types.DOUBLE:
case java.sql.Types.REAL:
case microsoft.sql.Types.MONEY:
case microsoft.sql.Types.SMALLMONEY:
case java.sql.Types.DECIMAL:
case java.sql.Types.NUMERIC:
case microsoft.sql.Types.GUID:
Expand All @@ -2258,21 +2267,18 @@ private void checkValidColumns(TypeInfo ti) throws SQLServerException {
case java.sql.Types.LONGVARBINARY:
case java.sql.Types.VARBINARY:
// Spatial datatypes fall under Varbinary, check if the UDT is geometry/geography.
String typeName = ti.getSSTypeName();
typeName = ti.getSSTypeName();
if (typeName.equalsIgnoreCase("geometry") || typeName.equalsIgnoreCase("geography")) {
MessageFormat form = new MessageFormat(SQLServerException.getErrString("R_BulkTypeNotSupported"));
form = new MessageFormat(SQLServerException.getErrString("R_BulkTypeNotSupported"));
throw new IllegalArgumentException(form.format(new Object[] {typeName}));
}
case java.sql.Types.TIMESTAMP:
case java.sql.Types.DATE:
case java.sql.Types.TIME:
case 2013: // java.sql.Types.TIME_WITH_TIMEZONE
case 2014: // java.sql.Types.TIMESTAMP_WITH_TIMEZONE
case microsoft.sql.Types.DATETIMEOFFSET:
case microsoft.sql.Types.SQL_VARIANT:
return;
default: {
MessageFormat form = new MessageFormat(SQLServerException.getErrString("R_BulkTypeNotSupported"));
form = new MessageFormat(SQLServerException.getErrString("R_BulkTypeNotSupported"));
String unsupportedDataType = JDBCType.of(jdbctype).toString();
throw new IllegalArgumentException(form.format(new Object[] {unsupportedDataType}));
}
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/microsoft/sqlserver/jdbc/SQLServerResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ protected Object[][] getContents() {
{"R_unableRetrieveSourceData", "Unable to retrieve data from the source."},
{"R_ParsingError", "Failed to parse data for the {0} type."},
{"R_BulkTypeNotSupported", "Data type {0} is not supported in bulk copy."},
{"R_BulkTypeNotSupportedDW", "Data type {0} is not supported in bulk copy against Azure Data Warehouse."},
{"R_invalidTransactionOption",
"UseInternalTransaction option cannot be set to TRUE when used with a Connection object."},
{"R_invalidNegativeArg", "The {0} argument cannot be negative."},
Expand Down
28 changes: 6 additions & 22 deletions src/test/java/com/microsoft/sqlserver/jdbc/RandomData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@ public class RandomData {
private static Random r = new Random();

public static boolean returnNull = (0 == r.nextInt(5)); // 20% chance of return null
public static boolean returnFullLength = (0 == r.nextInt(2)); // 50% chance of return full length for char/nchar and
// binary types
public static boolean returnMinMax = (0 == r.nextInt(5)); // 20% chance of return Min/Max value
public static boolean returnZero = (0 == r.nextInt(10)); // 10% chance of return zero

Expand Down Expand Up @@ -357,8 +355,8 @@ public static byte[] generateBinaryTypes(String columnLength, boolean nullable,
minimumLength = 1;
}

int length;
if (columnLength.toLowerCase().equals("max")) {
int length;
// 50% chance of return value longer than 8000/4000
if (r.nextBoolean()) {
length = r.nextInt(100000) + maxBound;
Expand All @@ -373,17 +371,9 @@ public static byte[] generateBinaryTypes(String columnLength, boolean nullable,
}
} else {
int columnLengthInt = Integer.parseInt(columnLength);
if (returnFullLength) {
length = columnLengthInt;
byte[] bytes = new byte[length];
r.nextBytes(bytes);
return bytes;
} else {
length = r.nextInt(columnLengthInt - minimumLength) + minimumLength;
byte[] bytes = new byte[length];
r.nextBytes(bytes);
return bytes;
}
byte[] bytes = new byte[columnLengthInt];
r.nextBytes(bytes);
return bytes;
}
}

Expand Down Expand Up @@ -699,8 +689,8 @@ private static String buildCharOrNChar(String columnLength, boolean nullable, bo
minimumLength = 1;
}

int length;
if (columnLength.toLowerCase().equals("max")) {
int length;
// 50% chance of return value longer than 8000/4000
if (r.nextBoolean()) {
length = r.nextInt(100000) + maxBound;
Expand All @@ -711,13 +701,7 @@ private static String buildCharOrNChar(String columnLength, boolean nullable, bo
}
} else {
int columnLengthInt = Integer.parseInt(columnLength);
if (returnFullLength) {
length = columnLengthInt;
return buildRandomString(length, charSet);
} else {
length = r.nextInt(columnLengthInt - minimumLength) + minimumLength;
return buildRandomString(length, charSet);
}
return buildRandomString(columnLengthInt, charSet);
}
}

Expand Down
Loading

0 comments on commit d639870

Please sign in to comment.