Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Improve][Connector-V2][Fake] Improve fake connector #3932

Merged
merged 8 commits into from
Jan 16, 2023
Merged

[Improve][Connector-V2][Fake] Improve fake connector #3932

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d9cca75
[Improve][Connector-V2][Fake] Improve fake connector
TyrantLucifer Jan 12, 2023
82ad2ae
[Improve][Connector-V2][Fake] Support setting template and range for …
TyrantLucifer Jan 13, 2023
b26e5ac
[Improve][Connector-V2][Fake] Completed logic
TyrantLucifer Jan 13, 2023
e4506a7
[Improve][Connector-V2][Fake] Update fake e2e
TyrantLucifer Jan 14, 2023
c3f9413
[Improve][Connector-V2][Fake] Update option rule
TyrantLucifer Jan 14, 2023
e9d16a1
[Improve][Connector-V2][Fake] Update option rule
TyrantLucifer Jan 14, 2023
44d7cd3
[Improve][Connector-V2][Fake] Update docs
TyrantLucifer Jan 14, 2023
b8fb29e
[Improve][Connector-V2][Fake] Update docs
TyrantLucifer Jan 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[Improve][Connector-V2][Fake] Completed logic
  • Loading branch information
@TyrantLucifer
TyrantLucifer committed Jan 13, 2023
commit b26e5acb62485f5f62e716736435cfc46784b13d
12 changes: 6 additions & 6 deletions ...-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/config/FakeOption.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ public class FakeOption {

public static final Option<Integer> TINYINT_MIN = Options.key("tinyint.min")
.intType()
.defaultValue((int) Byte.MIN_VALUE)
.defaultValue(0)
.withDescription("The min value of tinyint type data");

public static final Option<Integer> TINYINT_MAX = Options.key("tinyint.max")
Expand All @@ -136,7 +136,7 @@ public class FakeOption {

public static final Option<Integer> SMALLINT_MIN = Options.key("smallint.min")
.intType()
.defaultValue((int) Short.MIN_VALUE)
.defaultValue(0)
.withDescription("The min value of smallint type data");

public static final Option<Integer> SMALLINT_MAX = Options.key("smallint.max")
Expand All @@ -146,7 +146,7 @@ public class FakeOption {

public static final Option<Integer> INT_MIN = Options.key("int.min")
.intType()
.defaultValue(Integer.MIN_VALUE)
.defaultValue(0)
.withDescription("The min value of int type data");

public static final Option<Integer> INT_MAX = Options.key("int.max")
Expand All @@ -156,7 +156,7 @@ public class FakeOption {

public static final Option<Long> BIGINT_MIN = Options.key("bigint.min")
.longType()
.defaultValue(Long.MIN_VALUE)
.defaultValue(0L)
.withDescription("The min value of bigint type data");

public static final Option<Long> BIGINT_MAX = Options.key("bigint.max")
Expand All @@ -166,7 +166,7 @@ public class FakeOption {

public static final Option<Float> FLOAT_MIN = Options.key("float.min")
.floatType()
.defaultValue(Float.MIN_VALUE)
.defaultValue(0F)
.withDescription("The min value of float type data");

public static final Option<Float> FLOAT_MAX = Options.key("float.max")
Expand All @@ -176,7 +176,7 @@ public class FakeOption {

public static final Option<Double> DOUBLE_MIN = Options.key("double.min")
.doubleType()
.defaultValue(Double.MIN_VALUE)
.defaultValue(0D)
.withDescription("The min value of double type data");

public static final Option<Double> DOUBLE_MAX = Options.key("double.max")
Expand Down
43 changes: 13 additions & 30 deletions ...rc/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeDataGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,8 @@
import org.apache.seatunnel.connectors.seatunnel.fake.utils.FakeDataRandomUtils;
import org.apache.seatunnel.format.json.JsonDeserializationSchema;

import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;

import java.io.IOException;
import java.lang.reflect.Array;
import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -123,35 +118,34 @@ private Object randomColumnValue(SeaTunnelDataType<?> fieldType) {
}
return objectMap;
case STRING:
return RandomStringUtils.randomAlphabetic(fakeConfig.getStringLength());
return fakeDataRandomUtils.randomString();
case BOOLEAN:
return RandomUtils.nextInt(0, 2) == 1;
return fakeDataRandomUtils.randomBoolean();
case TINYINT:
return (byte) RandomUtils.nextInt(0, 255);
return fakeDataRandomUtils.randomTinyint();
case SMALLINT:
return (short) RandomUtils.nextInt(Byte.MAX_VALUE, Short.MAX_VALUE);
return fakeDataRandomUtils.randomSmallint();
case INT:
return RandomUtils.nextInt(Short.MAX_VALUE, Integer.MAX_VALUE);
return fakeDataRandomUtils.randomInt();
case BIGINT:
return RandomUtils.nextLong(Integer.MAX_VALUE, Long.MAX_VALUE);
return fakeDataRandomUtils.randomBigint();
case FLOAT:
return RandomUtils.nextFloat(Float.MIN_VALUE, Float.MAX_VALUE);
return fakeDataRandomUtils.randomFloat();
case DOUBLE:
return RandomUtils.nextDouble(Float.MAX_VALUE, Double.MAX_VALUE);
return fakeDataRandomUtils.randomDouble();
case DECIMAL:
DecimalType decimalType = (DecimalType) fieldType;
return new BigDecimal(RandomStringUtils.randomNumeric(decimalType.getPrecision() - decimalType.getScale()) + "." +
RandomStringUtils.randomNumeric(decimalType.getScale()));
return fakeDataRandomUtils.randomBigDecimal(decimalType.getPrecision(), decimalType.getScale());
case NULL:
return null;
case BYTES:
return RandomStringUtils.randomAlphabetic(fakeConfig.getBytesLength()).getBytes();
return fakeDataRandomUtils.randomBytes();
case DATE:
return randomLocalDateTime().toLocalDate();
return fakeDataRandomUtils.randomLocalDate();
case TIME:
return randomLocalDateTime().toLocalTime();
return fakeDataRandomUtils.randomLocalTime();
case TIMESTAMP:
return randomLocalDateTime();
return fakeDataRandomUtils.randomLocalDateTime();
case ROW:
SeaTunnelDataType<?>[] fieldTypes = ((SeaTunnelRowType) fieldType).getFieldTypes();
Object[] objects = new Object[fieldTypes.length];
Expand All @@ -166,15 +160,4 @@ private Object randomColumnValue(SeaTunnelDataType<?> fieldType) {
"SeaTunnel Fake source connector not support this data type");
}
}

@SuppressWarnings("magicnumber")
private LocalDateTime randomLocalDateTime() {
return LocalDateTime.of(
LocalDateTime.now().getYear(),
RandomUtils.nextInt(1, 12),
RandomUtils.nextInt(1, 28),
RandomUtils.nextInt(0, 24),
RandomUtils.nextInt(0, 59)
);
}
}
135 changes: 135 additions & 0 deletions ...c/main/java/org/apache/seatunnel/connectors/seatunnel/fake/utils/FakeDataRandomUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,149 @@

import org.apache.seatunnel.connectors.seatunnel.fake.config.FakeConfig;

import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;

import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.util.List;

public class FakeDataRandomUtils {
private final FakeConfig fakeConfig;

public FakeDataRandomUtils(FakeConfig fakeConfig) {
this.fakeConfig = fakeConfig;
}

private static <T> T randomFromList(List<T> list) {
int index = RandomUtils.nextInt(0, list.size() - 1);
return list.get(index);
}

public Boolean randomBoolean() {
return RandomUtils.nextInt(0, 2) == 1;
}

public BigDecimal randomBigDecimal(int precision, int scale) {
return new BigDecimal(RandomStringUtils.randomNumeric(precision - scale) + "." +
RandomStringUtils.randomNumeric(scale));
}

public byte[] randomBytes() {
return RandomStringUtils.randomAlphabetic(fakeConfig.getBytesLength()).getBytes();
}

public String randomString() {
List<String> stringTemplate = fakeConfig.getStringTemplate();
if (!CollectionUtils.isEmpty(stringTemplate)) {
return randomFromList(stringTemplate);
}
return RandomStringUtils.randomAlphabetic(fakeConfig.getStringLength());
}

public Byte randomTinyint() {
List<Integer> tinyintTemplate = fakeConfig.getTinyintTemplate();
if (!CollectionUtils.isEmpty(tinyintTemplate)) {
return randomFromList(tinyintTemplate).byteValue();
}
return (byte) RandomUtils.nextInt(fakeConfig.getTinyintMin(), fakeConfig.getTinyintMax());
}

public Short randomSmallint() {
List<Integer> smallintTemplate = fakeConfig.getSmallintTemplate();
if (!CollectionUtils.isEmpty(smallintTemplate)) {
return randomFromList(smallintTemplate).shortValue();
}
return (short) RandomUtils.nextInt(fakeConfig.getSmallintMin(), fakeConfig.getSmallintMax());
}

public Integer randomInt() {
List<Integer> intTemplate = fakeConfig.getIntTemplate();
if (!CollectionUtils.isEmpty(intTemplate)) {
return randomFromList(intTemplate);
}
return RandomUtils.nextInt(fakeConfig.getIntMin(), fakeConfig.getIntMax());
}

public Long randomBigint() {
List<Long> bigTemplate = fakeConfig.getBigTemplate();
if (!CollectionUtils.isEmpty(bigTemplate)) {
return randomFromList(bigTemplate);
}
return RandomUtils.nextLong(fakeConfig.getBigintMin(), fakeConfig.getBigintMax());
}

public Float randomFloat() {
List<Double> floatTemplate = fakeConfig.getFloatTemplate();
if (!CollectionUtils.isEmpty(floatTemplate)) {
return randomFromList(floatTemplate).floatValue();
}
return RandomUtils.nextFloat((float) fakeConfig.getFloatMin(), (float) fakeConfig.getFloatMax());
}

public Double randomDouble() {
List<Double> doubleTemplate = fakeConfig.getDoubleTemplate();
if (!CollectionUtils.isEmpty(doubleTemplate)) {
return randomFromList(doubleTemplate);
}
return RandomUtils.nextDouble(fakeConfig.getDoubleMin(), fakeConfig.getDoubleMax());
}

public LocalDate randomLocalDate() {
return randomLocalDateTime().toLocalDate();
}

public LocalTime randomLocalTime() {
return randomLocalDateTime().toLocalTime();
}

@SuppressWarnings("checkstyle:MagicNumber")
public LocalDateTime randomLocalDateTime() {
int year;
int month;
int day;
int hour;
int minute;
int second;
// init year
if (!CollectionUtils.isEmpty(fakeConfig.getDateYearTemplate())) {
year = randomFromList(fakeConfig.getDateYearTemplate());
} else {
year = LocalDateTime.now().getYear();
}
// init month
if (!CollectionUtils.isEmpty(fakeConfig.getDateMonthTemplate())) {
month = randomFromList(fakeConfig.getDateMonthTemplate());
} else {
month = RandomUtils.nextInt(1, 13);
}
// init day
if (!CollectionUtils.isEmpty(fakeConfig.getDateDayTemplate())) {
day = randomFromList(fakeConfig.getDateDayTemplate());
} else {
day = RandomUtils.nextInt(1, 29);
}
// init hour
if (!CollectionUtils.isEmpty(fakeConfig.getTimeHourTemplate())) {
hour = randomFromList(fakeConfig.getTimeHourTemplate());
} else {
hour = RandomUtils.nextInt(0, 24);
}
// init minute
if (!CollectionUtils.isEmpty(fakeConfig.getTimeMinuteTemplate())) {
minute = randomFromList(fakeConfig.getTimeMinuteTemplate());
} else {
minute = RandomUtils.nextInt(0, 60);
}
// init second
if (!CollectionUtils.isEmpty(fakeConfig.getTimeSecondTemplate())) {
second = randomFromList(fakeConfig.getTimeSecondTemplate());
} else {
second = RandomUtils.nextInt(0, 60);
}
return LocalDateTime.of(year, month, day, hour, minute, second);
}
}