Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,9 @@ Values that are wrapped in `%` symbols are treated as varaibles and evaluated at
|
+-- column_mapping struct - maps column names if needed, e.g. "group" -> "group_name"
|
+-- table_transform string - ([""], "lower_case", "upper_case", "camel_to_snake_case")
+-- table_transform string - ([""], "lower_case", "upper_case", "camel_to_snake_case", "dont_change")
|
+-- column_transform string - ([""], "lower_case", "upper_case", "camel_to_snake_case")
+-- column_transform string - ([""], "lower_case", "upper_case", "camel_to_snake_case", "dont_change")
|
+-- ddl
|
Expand All @@ -155,6 +155,8 @@ Values that are wrapped in `%` symbols are treated as varaibles and evaluated at
+-- after_all array of SQL commands to run after data copy
|
+-- recomended ([""], "all") - specifying "all" will execute recommendations
|
+-- implicitConversionTypes array of source column types that should be cast implicitly with postgres (for example "UNIQUEIDENTIFIER")
|
+-- threads (["cores", integer]) - number of concurrent connections
|
Expand Down
8 changes: 8 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,14 @@
</archive>
</configuration>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<fork>true</fork>
<executable>C:\Program Files\Java\jdk1.8.0_202\bin\javac.exe</executable>
</configuration>
</plugin>
</plugins>
</build>

Expand Down
9 changes: 7 additions & 2 deletions src/main/java/net/twentyonesolutions/m2pg/Config.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public class Config {
Map<String, Object> dml, ddl;
Map<String, Object> schemaMapping, tableMapping, columnMapping;
Map<String, String> columnDefaultReplace;
Map<String, String> postgresTypeCasting;

String name, source, target;

Expand Down Expand Up @@ -87,6 +88,7 @@ private void parseDml(){
,"source_column_quote_prefix"
,"source_column_quote_suffix"
,"threads"
,"implicit_conversion_types"
};

// populate result with config value or default of empty string
Expand All @@ -95,7 +97,7 @@ private void parseDml(){
}

// wrap single item String in List
for (String k : new String[]{ "execute.before_all", "execute.after_all" }){
for (String k : new String[]{ "execute.before_all", "execute.after_all", "implicit_conversion_types"}){
Object v = result.get(k);
if (v instanceof String){
result.put(k, new ArrayList<String>(){{ add((String)v); }});
Expand All @@ -104,7 +106,7 @@ private void parseDml(){

mapSrc = (Map)config.get(prefix + "jdbc_type_mapping");
result.put("jdbc_type_mapping", getCaseInsensitiveMap(mapSrc, uppercaseValue));

this.dml = result;
}

Expand Down Expand Up @@ -340,6 +342,9 @@ public static String transform(String input, String type){

if (type.equalsIgnoreCase("camel_to_snake_case"))
return Util.convertCamelToSnakeCase(input);

if (type.equalsIgnoreCase("dont_change"))
return Util.convertDontChangeCase(input);
}

return input;
Expand Down
47 changes: 27 additions & 20 deletions src/main/java/net/twentyonesolutions/m2pg/Schema.java
Original file line number Diff line number Diff line change
Expand Up @@ -147,37 +147,45 @@ public String copyTable(String tableName, IProgress progress) throws IOException

statSrc = conSrc.createStatement();

PreparedStatement statInsert = conTgt.prepareStatement(qInsert);

PreparedStatement statInsert = conTgt.prepareStatement(qInsert);
statSrc.setFetchSize(1000);

rs = statSrc.executeQuery(qSelect);

ResultSetMetaData rsMetaData = rs.getMetaData();

Map<String, String> jdbcTypeMapping = (Map<String, String>) config.dml.get("jdbc_type_mapping");

List<String> implicitConversionTypes = (List<String>) config.dml.getOrDefault("implicit_conversion_types", Collections.EMPTY_LIST);

int columnCount = rsMetaData.getColumnCount();

int[] columnTypes = new int[columnCount];
// SQLType[] sqlTypes = new SQLType[columnCount]; // TODO: use this instead of columnTypes when pgjdbc will support setObject with SQLType

for (int i = 1; i <= columnCount; i++) {

int srcType = rsMetaData.getColumnType(i);
int tgtType = srcType;

// translate unsupported types, e.g. nvarchar to varchar, dml jdbcTypeMapping is based on JDBC types, while ddl jdbcTypeMapping is based on SQL types
// if (jdbcTypeMapping.containsKey(String.valueOf(srcType)))
// tgtType = Integer.parseInt(jdbcTypeMapping.getOrDefault(String.valueOf(srcType), String.valueOf(tgtType)));

String srcTypeName = JDBCType.valueOf(srcType).getName(); // WARN: rsMetaData.getColumnTypeName(i) returns the vendor's name instead of JDBC name, e.g. ntext instead of longnvarchar for MSSQL
if (jdbcTypeMapping.containsKey(srcTypeName)) {
String tgtTypeName = jdbcTypeMapping.get(srcTypeName);
tgtType = JDBCType.valueOf(tgtTypeName).getVendorTypeNumber();
}

columnTypes[i - 1] = tgtType;
String sourceColumnType=new String();
sourceColumnType=table.columns.get(i-1).type.toString();
if(implicitConversionTypes.contains(sourceColumnType)){
columnTypes[i - 1]=java.sql.Types.OTHER;
}else{
int srcType = rsMetaData.getColumnType(i);
int tgtType = srcType;

// translate unsupported types, e.g. nvarchar to varchar, dml jdbcTypeMapping is based on JDBC types, while ddl jdbcTypeMapping is based on SQL types
// if (jdbcTypeMapping.containsKey(String.valueOf(srcType)))
// tgtType = Integer.parseInt(jdbcTypeMapping.getOrDefault(String.valueOf(srcType), String.valueOf(tgtType)));


String srcTypeName = JDBCType.valueOf(srcType).getName();// WARN: rsMetaData.getColumnTypeName(i) returns the vendor's name instead of JDBC name, e.g. ntext instead of longnvarchar for MSSQL


if (jdbcTypeMapping.containsKey(srcTypeName)) {
String tgtTypeName = jdbcTypeMapping.get(srcTypeName);

tgtType = JDBCType.valueOf(tgtTypeName).getVendorTypeNumber();
}
columnTypes[i - 1] = tgtType;
}
}

boolean hasErrors = false;
Expand All @@ -190,8 +198,7 @@ public String copyTable(String tableName, IProgress progress) throws IOException
for (int i = 1; i <= columnCount; i++) {
Object value = rs.getObject(i);
values[i - 1] = value;
statInsert.setObject(i, value, columnTypes[i - 1]);
// statInsert.setObject(i, value, sqlTypes[i - 1]); // throws java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc.PgPreparedStatement.setObject is not yet implemented.
statInsert.setObject(i, value, columnTypes[i - 1]); // throws java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc.PgPreparedStatement.setObject is not yet implemented.
}

try {
Expand Down
10 changes: 10 additions & 0 deletions src/main/java/net/twentyonesolutions/m2pg/Util.java
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,16 @@ public static String convertCamelToSnakeCase(String camelCaseString){
.replaceAll("([^_A-Z0-9])([A-Z0-9])", "$1_$2")
.toLowerCase();
}

/**
* If it is necessary to do not change case, add quote to input string;
*
* @param anyCaseString
* @return
*/
public static String convertDontChangeCase(String anyCaseString){
return "\""+anyCaseString+"\"";
}


public static Map<String, Object> flattenKeys(Map<String, Object> map){
Expand Down