Skip to content

Commit

Permalink
Initial work to support core data types in Data Prepper (#4496)
Browse files Browse the repository at this point in the history
Adds an enum to represent core data types in Data Prepper. This is initially created to help connect the convert_entry_type processor with upcoming work for evaluating type information as part of #4478.

Signed-off-by: David Venable <dlv@amazon.com>
  • Loading branch information
dlvenable authored May 3, 2024
1 parent f0f7811 commit 32941ad
Show file tree
Hide file tree
Showing 6 changed files with 163 additions and 14 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.model.event;

import com.fasterxml.jackson.annotation.JsonCreator;

import java.util.Arrays;
import java.util.Map;
import java.util.stream.Collectors;

/**
* Represents data types that are core to Data Prepper.
*
* @since 2.8
*/
public enum DataType {
/**
* Type of <i>string</i>. Compatible with {@link java.lang.String}.
*
* @since 2.8
*/
STRING("string"),
/**
* Type of <i>boolean</i>. Compatible with the Java <b>boolean</b> primitive data type.
*
* @since 2.8
*/
BOOLEAN("boolean"),
/**
* Type of <i>integer</i>. A 32-bit signed integer. Compatible with the Java <b>int</b> primitive data type.
*
* @since 2.8
*/
INTEGER("integer"),
/**
* Type of <i>long</i>. A 64-bit signed integer. Compatible with the Java <b>long</b> primitive data type.
*
* @since 2.8
*/
LONG("long"),
/**
* Type of <i>double</i>. A 64-bit IEEE 754 floating point number. Compatible with the Java <b>double</b> primitive data type.
*
* @since 2.8
*/
DOUBLE("double");

private static final Map<String, DataType> TYPES_MAP = Arrays.stream(DataType.values())
.collect(Collectors.toMap(
value -> value.typeName,
value -> value
));

private final String typeName;

DataType(final String typeName) {
this.typeName = typeName;
}

/**
* Gets the name of the type. This is the name that users of
* Data Prepper use or see.
*
* @return The name of the type.
* @since 2.8
*/
public String getTypeName() {
return typeName;
}

@JsonCreator
static DataType fromTypeName(final String option) {
return TYPES_MAP.get(option);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.model.event;

import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;

import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;

class DataTypeTest {
@ParameterizedTest
@EnumSource(DataType.class)
void fromTypeName_returns_expected_value(final DataType dataType) {
assertThat(DataType.fromTypeName(dataType.getTypeName()), equalTo(dataType));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ private Object convertTargetType(String fieldKey, String fieldValue){
}
try{
if(targetTypeMap.containsKey(fieldKey)){
TypeConverter converter = targetTypeMap.get(fieldKey).getTargetConverter();
final TypeConverter<?> converter = targetTypeMap.get(fieldKey).getTargetConverter();
return converter.convert(fieldValue);
} else {
return fieldValue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
public class ConvertEntryTypeProcessor extends AbstractProcessor<Record<Event>, Record<Event>> {
private static final Logger LOG = LoggerFactory.getLogger(ConvertEntryTypeProcessor.class);
private final List<String> convertEntryKeys;
private final TypeConverter converter;
private final TypeConverter<?> converter;
private final String convertWhen;
private final List<String> nullValues;
private final String type;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.opensearch.dataprepper.plugins.processor.mutateevent;

import com.fasterxml.jackson.annotation.JsonCreator;
import org.opensearch.dataprepper.model.event.DataType;
import org.opensearch.dataprepper.typeconverter.TypeConverter;
import org.opensearch.dataprepper.typeconverter.IntegerConverter;
import org.opensearch.dataprepper.typeconverter.StringConverter;
Expand All @@ -18,30 +19,34 @@
import java.util.stream.Collectors;

public enum TargetType {
INTEGER("integer", new IntegerConverter()),
STRING("string", new StringConverter()),
DOUBLE("double", new DoubleConverter()),
BOOLEAN("boolean", new BooleanConverter()),
LONG("long", new LongConverter());
INTEGER(DataType.INTEGER, new IntegerConverter()),
STRING(DataType.STRING, new StringConverter()),
DOUBLE(DataType.DOUBLE, new DoubleConverter()),
BOOLEAN(DataType.BOOLEAN, new BooleanConverter()),
LONG(DataType.LONG, new LongConverter());

private static final Map<String, TargetType> OPTIONS_MAP = Arrays.stream(TargetType.values())
.collect(Collectors.toMap(
value -> value.option,
value -> value.dataType.getTypeName(),
value -> value
));

private final String option;
private final TypeConverter targetConverter;
private final DataType dataType;
private final TypeConverter<?> targetConverter;

TargetType(final String option, final TypeConverter target) {
this.option = option;
TargetType(final DataType dataType, final TypeConverter<?> target) {
this.dataType = dataType;
this.targetConverter = target;
}

public TypeConverter getTargetConverter() {
public TypeConverter<?> getTargetConverter() {
return targetConverter;
}


DataType getDataType() {
return dataType;
}

@JsonCreator
static TargetType fromOptionValue(final String option) {
return OPTIONS_MAP.get(option.toLowerCase());
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.plugins.processor.mutateevent;

import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.ArgumentsProvider;
import org.junit.jupiter.params.provider.ArgumentsSource;
import org.junit.jupiter.params.provider.EnumSource;
import org.opensearch.dataprepper.model.event.DataType;

import java.util.stream.Stream;

import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.params.provider.Arguments.arguments;

class TargetTypeTest {
@ParameterizedTest
@EnumSource(TargetType.class)
void fromTypeName_returns_expected_value(final TargetType targetType) {
assertThat(TargetType.fromOptionValue(targetType.getDataType().getTypeName()), equalTo(targetType));
}
@ParameterizedTest
@ArgumentsSource(DataTypeToTargetTypeArgumentsProvider.class)
void fromTypeName_returns_expected_value_based_on_DataType(final String typeName, final TargetType targetType) {
assertThat(TargetType.fromOptionValue(typeName), equalTo(targetType));
}

static class DataTypeToTargetTypeArgumentsProvider implements ArgumentsProvider {
@Override
public Stream<? extends Arguments> provideArguments(final ExtensionContext extensionContext) {
return Stream.of(
arguments(DataType.STRING.getTypeName(), TargetType.STRING),
arguments(DataType.BOOLEAN.getTypeName(), TargetType.BOOLEAN),
arguments(DataType.INTEGER.getTypeName(), TargetType.INTEGER),
arguments(DataType.LONG.getTypeName(), TargetType.LONG),
arguments(DataType.DOUBLE.getTypeName(), TargetType.DOUBLE)
);
}
}
}

0 comments on commit 32941ad

Please sign in to comment.