Skip to content

Commit

Permalink
Revert "[SR-4375] Support more multi unescape delimiter (StarRocks#224)…
Browse files Browse the repository at this point in the history 
…" (StarRocks#330)

This reverts commit fb8a204.
  • Loading branch information
@Astralidea
Astralidea authored Sep 22, 2021
1 parent 1ae3467 commit 90da1d6
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 177 deletions.
101 changes: 20 additions & 81 deletions fe/fe-core/src/main/java/com/starrocks/analysis/Delimiter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,90 +31,29 @@ public static String convertDelimiter(String originStr) throws AnalysisException
throw new AnalysisException("Delimiter cannot be empty or null");
}

StringWriter writer = new StringWriter();

for (int i = 0; i < originStr.length(); i++) {
char ch = originStr.charAt(i);
boolean outputOneChar = true;
if (ch == '\\') {
char nextChar = (i == originStr.length() - 1) ? '\\' : originStr.charAt(i + 1);
switch (nextChar) {
case '\\':
ch = '\\';
break;
case 'b':
ch = '\b';
break;
case 'f':
ch = '\f';
break;
case 'n':
ch = '\n';
break;
case 'r':
ch = '\r';
break;
case 't':
ch = '\t';
break;
case 'x':
case 'X': {
outputOneChar = false;
i = parseHexString(originStr, writer, i);
break;
}
case '\"':
ch = '\"';
break;
case '\'':
ch = '\'';
break;
default:
writer.append(ch);
continue;
}
if (outputOneChar) {
writer.append(ch);
i++;
}
// compatible previous 0x / 0X prefix
} else if (ch == '0' && i != originStr.length() - 1
&& (originStr.charAt(i + 1) == 'x' || originStr.charAt(i + 1) == 'X')) {
i = parseHexString(originStr, writer, i);
} else {
writer.append(ch);
if (originStr.toUpperCase().startsWith("\\X") || originStr.toUpperCase().startsWith("0X")) {
String hexStr = originStr.substring(2);
// check hex str
if (hexStr.isEmpty()) {
throw new AnalysisException("Invalid delimiter '" + originStr + ": empty hex string");
}
if (hexStr.length() % 2 != 0) {
throw new AnalysisException("Invalid delimiter '" + originStr + ": hex length must be a even number");
}
for (char hexChar : hexStr.toUpperCase().toCharArray()) {
if (HEX_STRING.indexOf(hexChar) == -1) {
throw new AnalysisException("Invalid delimiter '" + originStr + "': invalid hex format");
}
}
}

return writer.toString();
}

/**
\t Insert a tab
\b Insert a backspace
\n Insert a newline
\r Insert a carriage
\f Insert a formed
\' Insert a single quote character
\" Insert a double quote character
\\ Insert a backslash
\x Insert a hex escape e.g. \x48 represent h
*/
private static int parseHexString(String originStr, StringWriter writer, int offset) throws AnalysisException {
if (offset + 4 > originStr.length()) {
writer.append(originStr, offset, originStr.length());
return originStr.length();
}
String hexStr = originStr.substring(offset + 2, offset + 4);
for (char hexChar : hexStr.toUpperCase().toCharArray()) {
if (HEX_STRING.indexOf(hexChar) == -1) {
throw new AnalysisException("Invalid delimiter '" + originStr + "': invalid hex format");
// transform to delimiter
StringWriter writer = new StringWriter();
for (byte b : hexStrToBytes(hexStr)) {
writer.append((char) b);
}
return writer.toString();
} else {
return originStr;
}
for (byte b : hexStrToBytes(hexStr)) {
writer.append((char) b);
}
return offset + 3;
}

}
52 changes: 12 additions & 40 deletions fe/fe-core/src/test/java/com/starrocks/analysis/ColumnSeparatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,66 +28,33 @@
public class ColumnSeparatorTest {
@Test
public void testNormal() throws AnalysisException {

// \t
ColumnSeparator separator = new ColumnSeparator("\t");
separator.analyze();
Assert.assertEquals("'\t'", separator.toSql());
Assert.assertEquals("\t", separator.getColumnSeparator());

// \x01
separator = new ColumnSeparator("\\x01");
separator.analyze();
Assert.assertEquals("'\\x01'", separator.toSql());
Assert.assertEquals("\1", separator.getColumnSeparator());

separator = new ColumnSeparator("0");
separator.analyze();
Assert.assertEquals("'0'", separator.toSql());
Assert.assertEquals("0", separator.getColumnSeparator());

separator = new ColumnSeparator("0x");
separator.analyze();
Assert.assertEquals("'0x'", separator.toSql());
Assert.assertEquals("0x", separator.getColumnSeparator());

separator = new ColumnSeparator("0x1");
separator.analyze();
Assert.assertEquals("'0x1'", separator.toSql());
Assert.assertEquals("0x1", separator.getColumnSeparator());

separator = new ColumnSeparator("0x1");
separator.analyze();
Assert.assertEquals("'0x1'", separator.toSql());
Assert.assertEquals("0x1", separator.getColumnSeparator());

separator = new ColumnSeparator("\\x");
separator.analyze();
Assert.assertEquals("'\\x'", separator.toSql());
Assert.assertEquals("\\x", separator.getColumnSeparator());

separator = new ColumnSeparator("\\x1");
separator.analyze();
Assert.assertEquals("'\\x1'", separator.toSql());
Assert.assertEquals("\\x1", separator.getColumnSeparator());

// \x00 \x01
separator = new ColumnSeparator("\\x0001");
separator.analyze();
Assert.assertEquals("'\\x0001'", separator.toSql());
Assert.assertEquals("\u000001", separator.getColumnSeparator());
Assert.assertEquals("\0\1", separator.getColumnSeparator());

separator = new ColumnSeparator("\\x011");
separator = new ColumnSeparator("|");
separator.analyze();
Assert.assertEquals("'\\x011'", separator.toSql());
Assert.assertEquals("\u00011", separator.getColumnSeparator());
Assert.assertEquals("'|'", separator.toSql());
Assert.assertEquals("|", separator.getColumnSeparator());

separator = new ColumnSeparator("\\|");
separator.analyze();
Assert.assertEquals("'\\|'", separator.toSql());
Assert.assertEquals("\\|", separator.getColumnSeparator());

separator = new ColumnSeparator("\t\b");
separator.analyze();
Assert.assertEquals("'\t\b'", separator.toSql());
Assert.assertEquals("\t\b", separator.getColumnSeparator());
}

@Test(expected = AnalysisException.class)
Expand All @@ -96,4 +63,9 @@ public void testHexFormatError() throws AnalysisException {
separator.analyze();
}

@Test(expected = AnalysisException.class)
public void testHexLengthError() throws AnalysisException {
ColumnSeparator separator = new ColumnSeparator("\\x011");
separator.analyze();
}
}
85 changes: 29 additions & 56 deletions fe/fe-core/src/test/java/com/starrocks/analysis/RowDelimiterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,71 +28,44 @@
public class RowDelimiterTest {
@Test
public void testNormal() throws AnalysisException {
// \n
RowDelimiter delimiter = new RowDelimiter("\n");
delimiter.analyze();
Assert.assertEquals("'\n'", delimiter.toSql());
Assert.assertEquals("\n", delimiter.getRowDelimiter());

RowDelimiter separator = new RowDelimiter("\t");
separator.analyze();
Assert.assertEquals("'\t'", separator.toSql());
Assert.assertEquals("\t", separator.getRowDelimiter());

separator = new RowDelimiter("\\x01");
separator.analyze();
Assert.assertEquals("'\\x01'", separator.toSql());
Assert.assertEquals("\1", separator.getRowDelimiter());

separator = new RowDelimiter("0");
separator.analyze();
Assert.assertEquals("'0'", separator.toSql());
Assert.assertEquals("0", separator.getRowDelimiter());

separator = new RowDelimiter("0x");
separator.analyze();
Assert.assertEquals("'0x'", separator.toSql());
Assert.assertEquals("0x", separator.getRowDelimiter());

separator = new RowDelimiter("0x1");
separator.analyze();
Assert.assertEquals("'0x1'", separator.toSql());
Assert.assertEquals("0x1", separator.getRowDelimiter());

separator = new RowDelimiter("0x1");
separator.analyze();
Assert.assertEquals("'0x1'", separator.toSql());
Assert.assertEquals("0x1", separator.getRowDelimiter());

separator = new RowDelimiter("\\x");
separator.analyze();
Assert.assertEquals("'\\x'", separator.toSql());
Assert.assertEquals("\\x", separator.getRowDelimiter());

separator = new RowDelimiter("\\x1");
separator.analyze();
Assert.assertEquals("'\\x1'", separator.toSql());
Assert.assertEquals("\\x1", separator.getRowDelimiter());

separator = new RowDelimiter("\\x0001");
separator.analyze();
Assert.assertEquals("'\\x0001'", separator.toSql());
Assert.assertEquals("\u000001", separator.getRowDelimiter());
// \x01
delimiter = new RowDelimiter("\\x01");
delimiter.analyze();
Assert.assertEquals("'\\x01'", delimiter.toSql());
Assert.assertEquals("\1", delimiter.getRowDelimiter());

separator = new RowDelimiter("\\x011");
separator.analyze();
Assert.assertEquals("'\\x011'", separator.toSql());
Assert.assertEquals("\u00011", separator.getRowDelimiter());
// \x00 \x01
delimiter = new RowDelimiter("\\x0001");
delimiter.analyze();
Assert.assertEquals("'\\x0001'", delimiter.toSql());
Assert.assertEquals("\0\1", delimiter.getRowDelimiter());

separator = new RowDelimiter("\\|");
separator.analyze();
Assert.assertEquals("'\\|'", separator.toSql());
Assert.assertEquals("\\|", separator.getRowDelimiter());
delimiter = new RowDelimiter("|");
delimiter.analyze();
Assert.assertEquals("'|'", delimiter.toSql());
Assert.assertEquals("|", delimiter.getRowDelimiter());

separator = new RowDelimiter("\t\b");
separator.analyze();
Assert.assertEquals("'\t\b'", separator.toSql());
Assert.assertEquals("\t\b", separator.getRowDelimiter());
delimiter = new RowDelimiter("\\|");
delimiter.analyze();
Assert.assertEquals("'\\|'", delimiter.toSql());
Assert.assertEquals("\\|", delimiter.getRowDelimiter());
}

@Test(expected = AnalysisException.class)
public void testHexFormatError() throws AnalysisException {
RowDelimiter delimiter = new RowDelimiter("\\x0g");
delimiter.analyze();
}

@Test(expected = AnalysisException.class)
public void testHexLengthError() throws AnalysisException {
RowDelimiter delimiter = new RowDelimiter("\\x011");
delimiter.analyze();
}
}

0 comments on commit 90da1d6

Please sign in to comment.