diff --git a/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/MappingsParameterConfig.java b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/MappingsParameterConfig.java new file mode 100644 index 0000000000..25c225188e --- /dev/null +++ b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/MappingsParameterConfig.java @@ -0,0 +1,53 @@ +package org.opensearch.dataprepper.plugins.processor.translate; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotNull; + +import java.util.List; + +public class MappingsParameterConfig { + + @JsonProperty("source") + @NotNull + private Object source; + + @JsonProperty("iterate_on") + private String iterateOn; + + @JsonProperty("targets") + @NotNull + private List targetsParameterConfigs; + + + public Object getSource() { + return source; + } + + public String getIterateOn() { + return iterateOn; + } + + public List getTargetsParameterConfigs() { + return targetsParameterConfigs; + } + + public void parseMappings(){ + for(TargetsParameterConfig targetsParameterConfig: targetsParameterConfigs){ + targetsParameterConfig.parseMappings(); + } + } + + @AssertTrue(message = "source field must be a string or list of strings") + public boolean isSourceFieldValid() { + if (source instanceof String) { + return true; + } + if (source instanceof List) { + List sourceList = (List) source; + return sourceList.stream().allMatch(sourceItem -> sourceItem instanceof String); + } + return false; + } + +} diff --git a/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/MappingsParser.java b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/MappingsParser.java new file mode 100644 index 0000000000..4807e308bf --- /dev/null +++ b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/MappingsParser.java @@ -0,0 +1,101 @@ +package org.opensearch.dataprepper.plugins.processor.translate; + +import org.apache.commons.lang3.Range; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; + +public class MappingsParser { + private final LinkedHashMap, Object> rangeMappings = new LinkedHashMap<>(); + private final Map individualMappings = new HashMap<>(); + private final Map compiledPatterns = new HashMap<>(); + public MappingsParser(TargetsParameterConfig targetConfig){ + RegexParameterConfiguration regexConfig = targetConfig.getRegexParameterConfiguration(); + if (Objects.nonNull(regexConfig)) { + compilePatterns(regexConfig.getPatterns()); + } + processMapField(targetConfig.getMap()); + checkOverlappingKeys(); + } + + public Map fetchIndividualMappings() { return individualMappings; } + + public LinkedHashMap, Object> fetchRangeMappings() { return rangeMappings; } + + public Map fetchCompiledPatterns() { return compiledPatterns; } + + private void compilePatterns(Map mappings) { + for (String pattern : mappings.keySet()) { + Pattern compiledPattern = Pattern.compile(pattern); + compiledPatterns.put(compiledPattern, mappings.get(pattern)); + } + } + + private void processMapField(Map map) { + if (Objects.nonNull(map)) { + for (Map.Entry mapEntry : map.entrySet()) { + parseIndividualKeys(mapEntry); + } + } + } + + private void parseIndividualKeys(Map.Entry mapEntry) { + String[] commaSeparatedKeys = mapEntry.getKey().split(","); + for (String individualKey : commaSeparatedKeys) { + if (individualKey.contains("-")) { + addRangeMapping(Map.entry(individualKey, mapEntry.getValue())); + } else { + addIndividualMapping(individualKey, mapEntry.getValue()); + } + } + } + + private void addRangeMapping(Map.Entry mapEntry) { + String[] rangeKeys = mapEntry.getKey().split("-"); + if (rangeKeys.length != 2 || !StringUtils.isNumericSpace(rangeKeys[0]) || !StringUtils.isNumericSpace(rangeKeys[1])) { + addIndividualMapping(mapEntry.getKey(), mapEntry.getValue()); + } else { + Float lowKey = Float.parseFloat(rangeKeys[0]); + Float highKey = Float.parseFloat(rangeKeys[1]); + Range rangeEntry = Range.between(lowKey, highKey); + if (isRangeOverlapping(rangeEntry)) { + String exceptionMsg = "map option contains key " + mapEntry.getKey() + " that overlaps with other range entries"; + throw new InvalidPluginConfigurationException(exceptionMsg); + } else { + rangeMappings.put(Range.between(lowKey, highKey), mapEntry.getValue()); + } + } + } + + private void addIndividualMapping(final String key, final Object value) { + if (individualMappings.containsKey(key)) { + String exceptionMsg = "map option contains duplicate entries of " + key; + throw new InvalidPluginConfigurationException(exceptionMsg); + } else { + individualMappings.put(key.strip(), value); + } + } + + private boolean isRangeOverlapping(Range rangeEntry) { + return rangeMappings.keySet().stream().anyMatch(range -> range.isOverlappedBy(rangeEntry)); + } + + private void checkOverlappingKeys() { + for (String individualKey : individualMappings.keySet()) { + if (NumberUtils.isParsable(individualKey)) { + Float floatKey = Float.parseFloat(individualKey); + Range range = Range.between(floatKey, floatKey); + if (isRangeOverlapping(range)) { + String exceptionMsg = "map option contains key " + individualKey + " that overlaps with other range entries"; + throw new InvalidPluginConfigurationException(exceptionMsg); + } + } + } + } +} diff --git a/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TargetsParameterConfig.java b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TargetsParameterConfig.java new file mode 100644 index 0000000000..523ae1e606 --- /dev/null +++ b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TargetsParameterConfig.java @@ -0,0 +1,137 @@ +package org.opensearch.dataprepper.plugins.processor.translate; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import org.apache.commons.lang3.Range; +import org.opensearch.dataprepper.plugins.processor.mutateevent.TargetType; +import org.opensearch.dataprepper.typeconverter.TypeConverter; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +public class TargetsParameterConfig { + private final TypeConverter converter; + private final LinkedHashMap, Object> rangeMappings = new LinkedHashMap<>(); + private final Map individualMappings = new HashMap<>(); + private final Map compiledPatterns = new HashMap<>(); + @JsonProperty("target") + @NotNull + @NotEmpty + private String target; + @JsonProperty("map") + private Map map; + @JsonProperty("translate_when") + private String translateWhen; + @JsonProperty("regex") + private RegexParameterConfiguration regexParameterConfig; + @JsonProperty("default") + private String defaultValue; + @JsonProperty("target_type") + private TargetType targetType = TargetType.STRING; + + public TargetsParameterConfig(Map map, String target, RegexParameterConfiguration regexParameterConfig, String translateWhen, String defaultValue, TargetType targetType) { + this.targetType = Optional + .ofNullable(targetType) + .orElse(TargetType.STRING); + this.target = target; + this.map = map; + this.defaultValue = defaultValue; + this.regexParameterConfig = regexParameterConfig; + this.converter = this.targetType.getTargetConverter(); + this.translateWhen = translateWhen; + parseMappings(); + } + + public String getTarget() { + return target; + } + + public Map getMap() { + return map; + } + + public String getDefaultValue() { + return defaultValue; + } + + public String getTranslateWhen() { + return translateWhen; + } + + public TargetType getTargetType() { + return targetType; + } + + public RegexParameterConfiguration getRegexParameterConfiguration() { + return regexParameterConfig; + } + + public Map fetchIndividualMappings() { + return individualMappings; + } + + public LinkedHashMap, Object> fetchRangeMappings() { + return rangeMappings; + } + + public Map fetchCompiledPatterns() { + return compiledPatterns; + } + + public TypeConverter getConverter() { + return converter; + } + + + @AssertTrue(message = "pattern option is mandatory while configuring regex option") + public boolean isPatternPresent() { + return regexParameterConfig == null || regexParameterConfig.getPatterns() != null; + } + + @AssertTrue(message = "Either map or patterns option needs to be configured under targets.") + public boolean hasMappings() { + return Stream.of(map, regexParameterConfig).filter(n -> n != null).count() != 0; + } + + @AssertTrue(message = "The mapped values do not match the target type provided") + public boolean isMapTypeValid() { + return map.keySet().stream().allMatch(key -> checkTargetValueType(map.get(key))); + } + + @AssertTrue(message = "The pattern values do not match the target type provided") + public boolean isPatternTypeValid() { + if (Objects.isNull(regexParameterConfig) || Objects.isNull(regexParameterConfig.getPatterns())) { + return true; + } + Map patterns = regexParameterConfig.getPatterns(); + return patterns.keySet().stream().allMatch(key -> checkTargetValueType(patterns.get(key))); + } + + private boolean checkTargetValueType(Object val) throws NumberFormatException { + if (Objects.isNull(targetType)) { + return true; + } + try { + final TypeConverter converter = targetType.getTargetConverter(); + converter.convert(val); + } catch (Exception ex) { + return false; + } + return true; + } + + public void parseMappings() { + MappingsParser parser = new MappingsParser(this); + individualMappings.putAll(parser.fetchIndividualMappings()); + rangeMappings.putAll(parser.fetchRangeMappings()); + compiledPatterns.putAll(parser.fetchCompiledPatterns()); + } + +} diff --git a/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessor.java b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessor.java index 476bfa807b..bda5df91d4 100644 --- a/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessor.java +++ b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessor.java @@ -6,7 +6,6 @@ package org.opensearch.dataprepper.plugins.processor.translate; import org.apache.commons.lang3.Range; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.PluginMetrics; @@ -23,7 +22,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -38,106 +36,17 @@ @DataPrepperPlugin(name = "translate", pluginType = Processor.class, pluginConfigurationType = TranslateProcessorConfig.class) public class TranslateProcessor extends AbstractProcessor, Record> { - private static final Logger LOG = LoggerFactory.getLogger(TranslateProcessor.class); private final ExpressionEvaluator expressionEvaluator; - private final TranslateProcessorConfig translateProcessorConfig; - private final LinkedHashMap, Object> rangeMappings; - private final Map individualMappings; - private final Map compiledPatterns; - private final TypeConverter converter; + private final List mappingsConfig; @DataPrepperPluginConstructor public TranslateProcessor(PluginMetrics pluginMetrics, final TranslateProcessorConfig translateProcessorConfig, final ExpressionEvaluator expressionEvaluator) { super(pluginMetrics); - this.translateProcessorConfig = translateProcessorConfig; this.expressionEvaluator = expressionEvaluator; - this.converter = translateProcessorConfig.getTargetType().getTargetConverter(); - individualMappings = new HashMap<>(); - rangeMappings = new LinkedHashMap<>(); - compiledPatterns = new HashMap<>(); - if (Objects.nonNull(this.translateProcessorConfig.getRegexParameterConfiguration())) { - compilePatterns(translateProcessorConfig - .getRegexParameterConfiguration() - .getPatterns()); - } - processMapField(translateProcessorConfig.getMap()); + mappingsConfig = translateProcessorConfig.getMappingsParameterConfigs(); + mappingsConfig.forEach(MappingsParameterConfig::parseMappings); parseFile(translateProcessorConfig.getFilePath()); - checkOverlappingKeys(); - } - - private void compilePatterns(Map mappings) { - for (String pattern : mappings.keySet()) { - Pattern compiledPattern = Pattern.compile(pattern); - compiledPatterns.put(compiledPattern, mappings.get(pattern)); - } - } - - private void processMapField(Map map) { - if (Objects.nonNull(map)) { - for (Map.Entry mapEntry : map.entrySet()) { - parseIndividualKeys(mapEntry); - } - } - } - - private void parseIndividualKeys(Map.Entry mapEntry){ - String[] commaSeparatedKeys = mapEntry.getKey().split(","); - for(String individualKey : commaSeparatedKeys){ - if(individualKey.contains("-")){ - addRangeMapping(Map.entry(individualKey, mapEntry.getValue())); - } else { - addIndividualMapping(individualKey, mapEntry.getValue()); - } - } - } - - private void addRangeMapping(Map.Entry mapEntry){ - String[] rangeKeys = mapEntry.getKey().split("-"); - if(rangeKeys.length!=2 || !StringUtils.isNumericSpace(rangeKeys[0]) || !StringUtils.isNumericSpace(rangeKeys[1])){ - addIndividualMapping(mapEntry.getKey(), mapEntry.getValue()); - } else { - Float lowKey = Float.parseFloat(rangeKeys[0]); - Float highKey = Float.parseFloat(rangeKeys[1]); - Range rangeEntry = Range.between(lowKey, highKey); - if (isRangeOverlapping(rangeEntry)) { - String exceptionMsg = "map option contains key "+mapEntry.getKey()+" that overlaps with other range entries"; - throw new InvalidPluginConfigurationException(exceptionMsg); - } else { - rangeMappings.put(Range.between(lowKey, highKey), mapEntry.getValue()); - } - } - } - - private void addIndividualMapping(final String key, final Object value){ - if(individualMappings.containsKey(key)){ - String exceptionMsg = "map option contains duplicate entries of "+key; - throw new InvalidPluginConfigurationException(exceptionMsg); - } else { - individualMappings.put(key.strip(), value); - } - } - - private boolean isRangeOverlapping(Range rangeEntry) { - for (Range range : rangeMappings.keySet()) { - if (range.isOverlappedBy(rangeEntry)) { - return true; - } - } - return false; - } - - private void checkOverlappingKeys() { - for (String individualKey : individualMappings.keySet()) { - if (NumberUtils.isParsable(individualKey)) { - Float floatKey = Float.parseFloat(individualKey); - Range range = Range.between(floatKey, floatKey); - if (isRangeOverlapping(range)) { - String exceptionMsg = "map option contains key " + individualKey + " that overlaps with other range entries"; - throw new InvalidPluginConfigurationException(exceptionMsg); - } - } - } } private void parseFile(String filePath){ @@ -148,23 +57,28 @@ private void parseFile(String filePath){ public Collection> doExecute(Collection> records) { for (final Record record : records) { final Event recordEvent = record.getData(); - if (Objects.nonNull(translateProcessorConfig.getTranslateWhen()) && !expressionEvaluator.evaluateConditional(translateProcessorConfig.getTranslateWhen(), recordEvent)) { - continue; - } - try { - String iterateOn = translateProcessorConfig.getIterateOn(); - if (Objects.nonNull(iterateOn)) { - List> objectsToIterate = recordEvent.get(iterateOn, List.class); - for (Map recordObject : objectsToIterate) { - performMappings(recordObject); + for (MappingsParameterConfig mappingConfig : mappingsConfig) { + try { + String iterateOn = mappingConfig.getIterateOn(); + List targetsConfig = mappingConfig.getTargetsParameterConfigs(); + for (TargetsParameterConfig targetConfig : targetsConfig) { + String translateWhen = targetConfig.getTranslateWhen(); + Object sourceObject = mappingConfig.getSource(); + if (Objects.nonNull(translateWhen) && !expressionEvaluator.evaluateConditional(translateWhen, recordEvent)) { + continue; + } + if (Objects.nonNull(iterateOn)) { + List> objectsToIterate = recordEvent.get(iterateOn, List.class); + objectsToIterate.forEach(recordObject -> performMappings(recordObject, sourceObject, targetConfig)); + recordEvent.put(iterateOn, objectsToIterate); + } else { + performMappings(recordEvent, sourceObject, targetConfig); + } } - recordEvent.put(iterateOn, objectsToIterate); - } else { - performMappings(recordEvent); + } catch (Exception ex) { + LOG.error(EVENT, "Error mapping the source [{}] of entry [{}]", mappingConfig.getSource(), + record.getData(), ex); } - } catch (Exception ex) { - LOG.error(EVENT, "Error mapping the source [{}] of entry [{}]", translateProcessorConfig.getSource(), - record.getData(), ex); } } return records; @@ -178,16 +92,19 @@ private String getSourceValue(Object recordObject, String sourceKey) { } } - private Object getTargetValue(Object sourceObject, List targetValues){ + private Object getTargetValue(Object sourceObject, List targetValues, TargetsParameterConfig targetConfig) { + TypeConverter converter = targetConfig.getConverter(); if(sourceObject instanceof String) { return converter.convert(targetValues.get(0)); } - return targetValues.stream().map(converter::convert).collect(Collectors.toList()); + return targetValues + .stream() + .map(converter::convert) + .collect(Collectors.toList()); } - private void performMappings(Object recordObject) { + private void performMappings(Object recordObject, Object sourceObject, TargetsParameterConfig targetConfig) { List targetValues = new ArrayList<>(); - Object sourceObject = translateProcessorConfig.getSource(); List sourceKeys; if (sourceObject instanceof List) { sourceKeys = (ArrayList) sourceObject; @@ -199,34 +116,38 @@ private void performMappings(Object recordObject) { } for (String sourceKey : sourceKeys) { String sourceValue = getSourceValue(recordObject, sourceKey); - Optional targetValue = getTargetValueForSource(sourceValue); - targetValue.ifPresent(targetValues::add); + if(sourceValue!=null){ + Optional targetValue = getTargetValueForSource(sourceValue, targetConfig); + targetValue.ifPresent(targetValues::add); + } } - addTargetToRecords(sourceObject, targetValues, recordObject); + addTargetToRecords(sourceObject, targetValues, recordObject, targetConfig); } - private Optional getTargetValueForSource(final String sourceValue) { + private Optional getTargetValueForSource(final String sourceValue, TargetsParameterConfig targetConfig) { Optional targetValue = Optional.empty(); targetValue = targetValue - .or(() -> matchesIndividualEntry(sourceValue)) - .or(() -> matchesRangeEntry(sourceValue)) - .or(() -> matchesPatternEntry(sourceValue)) - .or(() -> Optional.ofNullable(translateProcessorConfig.getDefaultValue())); + .or(() -> matchesIndividualEntry(sourceValue, targetConfig)) + .or(() -> matchesRangeEntry(sourceValue, targetConfig)) + .or(() -> matchesPatternEntry(sourceValue, targetConfig)) + .or(() -> Optional.ofNullable(targetConfig.getDefaultValue())); return targetValue; } - private Optional matchesIndividualEntry(final String sourceValue) { + private Optional matchesIndividualEntry(final String sourceValue, TargetsParameterConfig targetConfig) { + Map individualMappings = targetConfig.fetchIndividualMappings(); if (individualMappings.containsKey(sourceValue)) { return Optional.of(individualMappings.get(sourceValue)); } return Optional.empty(); } - private Optional matchesRangeEntry(final String sourceValue) { + private Optional matchesRangeEntry(final String sourceValue, TargetsParameterConfig targetConfig) { if (!NumberUtils.isParsable(sourceValue)) { return Optional.empty(); } Float floatKey = Float.parseFloat(sourceValue); + LinkedHashMap, Object> rangeMappings = targetConfig.fetchRangeMappings(); for (Map.Entry, Object> rangeEntry : rangeMappings.entrySet()) { Range range = rangeEntry.getKey(); if (range.contains(floatKey)) { @@ -236,11 +157,12 @@ private Optional matchesRangeEntry(final String sourceValue) { return Optional.empty(); } - private Optional matchesPatternEntry(final String sourceValue) { + private Optional matchesPatternEntry(final String sourceValue, TargetsParameterConfig targetConfig) { + Map compiledPatterns = targetConfig.fetchCompiledPatterns(); if (compiledPatterns.isEmpty()) { return Optional.empty(); } - final boolean exact = translateProcessorConfig.getRegexParameterConfiguration().getExact(); + final boolean exact = targetConfig.getRegexParameterConfiguration().getExact(); for (Pattern pattern : compiledPatterns.keySet()) { Matcher matcher = pattern.matcher(sourceValue); if (matcher.matches() || (!exact && matcher.find())) { @@ -250,17 +172,17 @@ private Optional matchesPatternEntry(final String sourceValue) { return Optional.empty(); } - private void addTargetToRecords(Object sourceObject, List targetValues, Object recordObject) { + private void addTargetToRecords(Object sourceObject, List targetValues, Object recordObject, TargetsParameterConfig targetMappings) { if (targetValues.isEmpty()) { return; } - final String targetField = translateProcessorConfig.getTarget(); + final String targetField = targetMappings.getTarget(); if (recordObject instanceof Map) { Map recordMap = (Map) recordObject; - recordMap.put(targetField, getTargetValue(sourceObject, targetValues)); + recordMap.put(targetField, getTargetValue(sourceObject, targetValues, targetMappings)); } else if (recordObject instanceof Event) { Event event = (Event) recordObject; - event.put(targetField, getTargetValue(sourceObject, targetValues)); + event.put(targetField, getTargetValue(sourceObject, targetValues, targetMappings)); } } diff --git a/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorConfig.java b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorConfig.java index 845442bc40..1bae063a6d 100644 --- a/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorConfig.java +++ b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorConfig.java @@ -8,116 +8,32 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; -import org.opensearch.dataprepper.plugins.processor.mutateevent.TargetType; -import org.opensearch.dataprepper.typeconverter.TypeConverter; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.stream.Stream; public class TranslateProcessorConfig { - - @JsonProperty("source") - @NotNull - private Object source; - - @JsonProperty("target") - @NotNull - @NotEmpty - private String target; - - @JsonProperty("map") - private Map map; - @JsonProperty("file_path") private String filePath; - @JsonProperty("default") - private String defaultValue; - - @JsonProperty("translate_when") - private String translateWhen; - - @JsonProperty("iterate_on") - private String iterateOn; - - @JsonProperty("regex") - private RegexParameterConfiguration regexParameterConfiguration; - - @JsonProperty("target_type") - private TargetType targetType = TargetType.STRING; - - - public Object getSource() { return source; } - - public String getTarget() { return target; } - - public Map getMap() { return map; } - - public String getDefaultValue() { return defaultValue; } - - public String getFilePath() { return filePath; } - - public String getTranslateWhen() { return translateWhen; } - - public String getIterateOn() { return iterateOn; } - - public TargetType getTargetType() { return targetType; } - - public RegexParameterConfiguration getRegexParameterConfiguration(){ return regexParameterConfiguration; } - - - @AssertTrue(message = "source field must be a string or list of strings") - public boolean isSourceFieldValid(){ - if(source instanceof String){ - return true; - } - if(source instanceof List){ - List sourceList = (List) source; - return sourceList.stream().allMatch(sourceItem -> sourceItem instanceof String); - } - return false; - } - - @AssertTrue(message = "Either of map or patterns or file_path options need to be configured.") - public boolean hasMappings() { - return Stream.of(map, filePath, regexParameterConfiguration).filter(n -> n!=null).count() != 0; - } + @NotNull + @JsonProperty("mappings") + private List mappingsParameterConfigs; - @AssertTrue(message = "pattern option is mandatory while configuring regex option") - public boolean isPatternPresent(){ - return regexParameterConfiguration == null || regexParameterConfiguration.getPatterns() != null; + public String getFilePath() { + return filePath; } - @AssertTrue(message = "The mapped values do not match the target type provided") - public boolean isMapTypeValid() { - return map.keySet().stream().allMatch(key -> checkTargetValueType(map.get(key))); + public List getMappingsParameterConfigs() { + return mappingsParameterConfigs; } - @AssertTrue(message = "The pattern values do not match the target type provided") - public boolean isPatternTypeValid() { - if (Objects.isNull(regexParameterConfiguration) || Objects.isNull(regexParameterConfiguration.getPatterns())) { - return true; - } - Map patterns = regexParameterConfiguration.getPatterns(); - return patterns.keySet().stream().allMatch(key -> checkTargetValueType(patterns.get(key))); + @AssertTrue(message = "Either mappings or file_path option needs to be configured.") + public boolean hasMappings() { + return Stream.of(mappingsParameterConfigs, filePath).filter(n -> n != null).count() != 0; } - private boolean checkTargetValueType(Object val) throws NumberFormatException { - if (Objects.isNull(targetType)) { - return true; - } - try { - final TypeConverter converter = targetType.getTargetConverter(); - converter.convert(val); - } catch (Exception ex) { - return false; - } - return true; - } } diff --git a/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/MappingsParameterConfigTest.java b/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/MappingsParameterConfigTest.java new file mode 100644 index 0000000000..68fdeb9e41 --- /dev/null +++ b/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/MappingsParameterConfigTest.java @@ -0,0 +1,58 @@ +package org.opensearch.dataprepper.plugins.processor.translate; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.core.Is.is; +import static org.opensearch.dataprepper.test.helper.ReflectivelySetField.setField; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.hamcrest.MatcherAssert.assertThat; +import java.util.List; + +class MappingsParameterConfigTest { + + private MappingsParameterConfig mappingsParameterConfig; + + @BeforeEach + void setup() throws NoSuchFieldException, IllegalAccessException{ + mappingsParameterConfig = new MappingsParameterConfig(); + setField(MappingsParameterConfig.class, mappingsParameterConfig, "source", "sourceKey"); + } + + @Test + void test_get_iterate_on() throws NoSuchFieldException, IllegalAccessException{ + assertNull(mappingsParameterConfig.getIterateOn()); + setField(MappingsParameterConfig.class, mappingsParameterConfig, "iterateOn", "iteratorField"); + assertThat(mappingsParameterConfig.getIterateOn(),is("iteratorField")); + } + + @Test + void test_get_source() { + assertThat(mappingsParameterConfig.getSource(),is("sourceKey")); + } + + @Test + void test_source_field_valid_types() throws NoSuchFieldException, IllegalAccessException{ + setField(MappingsParameterConfig.class, mappingsParameterConfig, "source", "key1"); + assertTrue(mappingsParameterConfig.isSourceFieldValid()); + assertThat(mappingsParameterConfig.getSource(), is("key1")); + setField(MappingsParameterConfig.class, mappingsParameterConfig, "source", List.of("key1", "key2", "key3")); + assertTrue(mappingsParameterConfig.isSourceFieldValid()); + assertThat(mappingsParameterConfig.getSource(), is(List.of("key1", "key2", "key3"))); + } + + @Test + void test_source_field_invalid_types() throws NoSuchFieldException, IllegalAccessException{ + setField(MappingsParameterConfig.class, mappingsParameterConfig, "source", 200); + assertFalse(mappingsParameterConfig.isSourceFieldValid()); + setField(MappingsParameterConfig.class, mappingsParameterConfig, "source", false); + assertFalse(mappingsParameterConfig.isSourceFieldValid()); + setField(MappingsParameterConfig.class, mappingsParameterConfig, "source", 20.1); + assertFalse(mappingsParameterConfig.isSourceFieldValid()); + setField(MappingsParameterConfig.class, mappingsParameterConfig, "source", List.of("key1", 200)); + assertFalse(mappingsParameterConfig.isSourceFieldValid()); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TargetsParameterConfigTest.java b/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TargetsParameterConfigTest.java new file mode 100644 index 0000000000..efc209c3c4 --- /dev/null +++ b/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TargetsParameterConfigTest.java @@ -0,0 +1,68 @@ +package org.opensearch.dataprepper.plugins.processor.translate; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.plugins.processor.mutateevent.TargetType; + +import java.util.Collections; + +import static org.hamcrest.core.Is.is; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.dataprepper.test.helper.ReflectivelySetField.setField; +import static org.hamcrest.MatcherAssert.assertThat; + + +class TargetsParameterConfigTest { + private TargetsParameterConfig targetsParameterConfig; + private RegexParameterConfiguration regexParameterConfiguration; + private TargetsParameterConfig createObjectUnderTest() { + return new TargetsParameterConfig(null, null, null, null, null,null); + } + + @BeforeEach + void setup() throws NoSuchFieldException, IllegalAccessException{ + targetsParameterConfig = createObjectUnderTest(); + setField(TargetsParameterConfig.class, targetsParameterConfig, "target", "targetKey"); + } + + @Test + void test_no_map_patterns_filepath_options_present(){ + assertFalse(targetsParameterConfig.hasMappings()); + } + + @Test + void test_only_map_option_present() throws NoSuchFieldException, IllegalAccessException{ + setField(TargetsParameterConfig.class, targetsParameterConfig, "map", Collections.singletonMap("key1", "val1")); + assertTrue(targetsParameterConfig.hasMappings()); + } + + @Test + void test_no_patterns_under_regex() throws NoSuchFieldException, IllegalAccessException{ + regexParameterConfiguration = new RegexParameterConfiguration(); + setField(RegexParameterConfiguration.class, regexParameterConfiguration, "exact", true); + setField(TargetsParameterConfig.class, targetsParameterConfig, "map", Collections.singletonMap("key1", "val1")); + setField(TargetsParameterConfig.class, targetsParameterConfig, "regexParameterConfig", regexParameterConfiguration); + assertFalse(targetsParameterConfig.isPatternPresent()); + } + + @Test + void test_get_default() throws NoSuchFieldException, IllegalAccessException{ + assertNull(targetsParameterConfig.getDefaultValue()); + setField(TargetsParameterConfig.class, targetsParameterConfig, "defaultValue", "No match"); + assertThat(targetsParameterConfig.getDefaultValue(),is("No match")); + } + + @Test + void test_target_type_default(){ + assertThat(targetsParameterConfig.getTargetType(), is(TargetType.STRING)); + } + + @Test + void test_get_target_type() throws NoSuchFieldException, IllegalAccessException{ + setField(TargetsParameterConfig.class, targetsParameterConfig, "targetType", TargetType.INTEGER); + assertThat(targetsParameterConfig.getTargetType(), is(TargetType.INTEGER)); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorConfigTest.java b/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorConfigTest.java index 6af0f325eb..f8b5326392 100644 --- a/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorConfigTest.java +++ b/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorConfigTest.java @@ -2,13 +2,10 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.opensearch.dataprepper.plugins.processor.mutateevent.TargetType; -import java.util.Collections; import java.util.List; import static org.hamcrest.core.Is.is; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.hamcrest.MatcherAssert.assertThat; @@ -17,26 +14,23 @@ class TranslateProcessorConfigTest { private TranslateProcessorConfig translateProcessorConfig; - private RegexParameterConfiguration regexParameterConfiguration; private TranslateProcessorConfig createObjectUnderTest() { return new TranslateProcessorConfig(); } @BeforeEach - void setup() throws NoSuchFieldException, IllegalAccessException{ + void setup(){ translateProcessorConfig = createObjectUnderTest(); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "source", "sourceKey"); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "target", "targetKey"); } @Test - void test_no_map_patterns_filepath_options_present(){ + void test_no_mappings_filepath_options_present(){ assertFalse(translateProcessorConfig.hasMappings()); } @Test - void test_only_map_option_present() throws NoSuchFieldException, IllegalAccessException{ - setField(TranslateProcessorConfig.class, translateProcessorConfig, "map", Collections.singletonMap("key1", "val1")); + void test_only_mappings_option_present() throws NoSuchFieldException, IllegalAccessException{ + setField(TranslateProcessorConfig.class, translateProcessorConfig, "mappingsParameterConfigs", List.of(new MappingsParameterConfig())); assertTrue(translateProcessorConfig.hasMappings()); } @@ -47,66 +41,17 @@ void test_only_filepath_option_present() throws NoSuchFieldException, IllegalAcc } @Test - void test_only_patterns_option_present() throws NoSuchFieldException, IllegalAccessException{ - regexParameterConfiguration = new RegexParameterConfiguration(); - setField(RegexParameterConfiguration.class, regexParameterConfiguration, "patterns", Collections.singletonMap("patternKey1", "patternVal1")); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "regexParameterConfiguration", regexParameterConfiguration); - assertTrue(translateProcessorConfig.hasMappings()); - } - - @Test - void test_no_patterns_under_regex() throws NoSuchFieldException, IllegalAccessException{ - regexParameterConfiguration = new RegexParameterConfiguration(); - setField(RegexParameterConfiguration.class, regexParameterConfiguration, "exact", true); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "map", Collections.singletonMap("key1", "val1")); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "regexParameterConfiguration", regexParameterConfiguration); - assertFalse(translateProcessorConfig.isPatternPresent()); - } - - @Test - void test_source_field_valid_types() throws NoSuchFieldException, IllegalAccessException{ - setField(TranslateProcessorConfig.class, translateProcessorConfig, "source", "key1"); - assertTrue(translateProcessorConfig.isSourceFieldValid()); - assertThat(translateProcessorConfig.getSource(), is("key1")); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "source", List.of("key1", "key2", "key3")); - assertTrue(translateProcessorConfig.isSourceFieldValid()); - assertThat(translateProcessorConfig.getSource(), is(List.of("key1", "key2", "key3"))); - } - - @Test - void test_source_field_invalid_types() throws NoSuchFieldException, IllegalAccessException{ - setField(TranslateProcessorConfig.class, translateProcessorConfig, "source", 200); - assertFalse(translateProcessorConfig.isSourceFieldValid()); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "source", false); - assertFalse(translateProcessorConfig.isSourceFieldValid()); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "source", 20.1); - assertFalse(translateProcessorConfig.isSourceFieldValid()); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "source", List.of("key1", 200)); - assertFalse(translateProcessorConfig.isSourceFieldValid()); + void test_get_file_path() throws NoSuchFieldException, IllegalAccessException{ + String filePath = "/path/to/file.yaml"; + setField(TranslateProcessorConfig.class, translateProcessorConfig, "filePath", filePath); + assertThat(translateProcessorConfig.getFilePath(), is(filePath)); } @Test - void test_get_default() throws NoSuchFieldException, IllegalAccessException{ - assertNull(translateProcessorConfig.getDefaultValue()); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "defaultValue", "No match"); - assertThat(translateProcessorConfig.getDefaultValue(),is("No match")); + void test_get_mappings() throws NoSuchFieldException, IllegalAccessException{ + List mappingsParameterConfigs = List.of(new MappingsParameterConfig()); + setField(TranslateProcessorConfig.class, translateProcessorConfig, "mappingsParameterConfigs", mappingsParameterConfigs); + assertThat(translateProcessorConfig.getMappingsParameterConfigs(), is(mappingsParameterConfigs)); } - @Test - void test_get_iterate_on() throws NoSuchFieldException, IllegalAccessException{ - assertNull(translateProcessorConfig.getIterateOn()); - setField(TranslateProcessorConfig.class, translateProcessorConfig, "iterateOn", "iteratorField"); - assertThat(translateProcessorConfig.getIterateOn(),is("iteratorField")); - } - - @Test - void test_target_type_default(){ - assertThat(translateProcessorConfig.getTargetType(), is(TargetType.STRING)); - } - - @Test - void test_get_target_type() throws NoSuchFieldException, IllegalAccessException{ - setField(TranslateProcessorConfig.class, translateProcessorConfig, "targetType", TargetType.INTEGER); - assertThat(translateProcessorConfig.getTargetType(), is(TargetType.INTEGER)); - } } \ No newline at end of file diff --git a/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorTest.java b/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorTest.java index 394cb11ad8..823f542849 100644 --- a/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorTest.java +++ b/data-prepper-plugins/translate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessorTest.java @@ -21,15 +21,14 @@ import java.util.List; import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.CoreMatchers.is; - @ExtendWith(MockitoExtension.class) class TranslateProcessorTest { @@ -44,17 +43,33 @@ class TranslateProcessorTest { @Mock private ExpressionEvaluator expressionEvaluator; + @Mock + private TargetsParameterConfig targetsParameterConfig; + + @Mock + private MappingsParameterConfig mappingsParameterConfig; + @BeforeEach void setup() { - lenient().when(mockConfig.getSource()).thenReturn("sourceField"); - lenient().when(mockConfig.getTarget()).thenReturn("targetField"); - lenient().when(mockConfig.getTargetType()).thenReturn(TargetType.STRING); - lenient().when(mockRegexConfig.getExact()).thenReturn(mockRegexConfig.DEFAULT_EXACT); + lenient() + .when(mappingsParameterConfig.getSource()) + .thenReturn("sourceField"); + lenient() + .when(targetsParameterConfig.getTargetType()) + .thenReturn(TargetType.STRING); + lenient() + .when(mockRegexConfig.getExact()) + .thenReturn(mockRegexConfig.DEFAULT_EXACT); + lenient() + .when(mockConfig.getMappingsParameterConfigs()) + .thenReturn(List.of(mappingsParameterConfig)); } @Test - void test_string_keys_in_map(){ - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("key1","mappedValue1"))); + void test_string_keys_in_map() { + targetsParameterConfig = new TargetsParameterConfig(createMapEntries(createMapping("key1", "mappedValue1")), + "targetField", null, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("key1"); final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); @@ -65,7 +80,9 @@ void test_string_keys_in_map(){ @Test void test_integer_keys_in_map() { - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("123", "mappedValue1"))); + targetsParameterConfig = new TargetsParameterConfig(createMapEntries(createMapping("123", "mappedValue1")), + "targetField", null, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("123"); final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); @@ -76,10 +93,13 @@ void test_integer_keys_in_map() { @Test void test_integer_range_keys_in_map() { - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("1-10", "mappedValue1"))); + targetsParameterConfig = new TargetsParameterConfig(createMapEntries(createMapping("1-10", "mappedValue1")), + "targetField", null, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("5"); - final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + final List> translatedRecords = (List>) processor.doExecute( + Collections.singletonList(record)); assertTrue(translatedRecords.get(0).getData().containsKey("targetField")); assertThat(translatedRecords.get(0).getData().get("targetField", String.class), is("mappedValue1")); @@ -88,70 +108,89 @@ void test_integer_range_keys_in_map() { @Test void test_comma_separated_keys_in_map() { - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("key1,key2, key3", "mappedValue1"))); + targetsParameterConfig = new TargetsParameterConfig( + createMapEntries(createMapping("key1,key2, key3", "mappedValue1")), "targetField", null, null, null, + null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); for (String key : Arrays.asList("key1", "key2", "key3")) { final Record record = getEvent(key); - final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + final List> translatedRecords = (List>) processor.doExecute( + Collections.singletonList(record)); assertTrue(translatedRecords.get(0).getData().containsKey("targetField")); assertThat(translatedRecords.get(0).getData().get("targetField", String.class), is("mappedValue1")); } final Record failureRecord = getEvent("key4"); - final List> failingTranslatedRecords = (List>) processor.doExecute(Collections.singletonList(failureRecord)); + final List> failingTranslatedRecords = (List>) processor.doExecute( + Collections.singletonList(failureRecord)); assertFalse(failingTranslatedRecords.get(0).getData().containsKey("targetField")); } @Test void test_comma_separated_range_keys_in_map() { - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("1-10,11-20, 21-30", "mappedValue1"))); + targetsParameterConfig = new TargetsParameterConfig( + createMapEntries(createMapping("1-10,11-20, 21-30", "mappedValue1")), "targetField", null, null, null, + null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); for (String key : Arrays.asList("5", "15", "25")) { final Record record = getEvent(key); - final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + final List> translatedRecords = (List>) processor.doExecute( + Collections.singletonList(record)); assertTrue(translatedRecords.get(0).getData().containsKey("targetField")); assertThat(translatedRecords.get(0).getData().get("targetField", String.class), is("mappedValue1")); } final Record failureRecord = getEvent("35"); - final List> failingTranslatedRecords = (List>) processor.doExecute(Collections.singletonList(failureRecord)); + final List> failingTranslatedRecords = (List>) processor.doExecute( + Collections.singletonList(failureRecord)); assertFalse(failingTranslatedRecords.get(0).getData().containsKey("targetField")); } @Test void test_float_source() { - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("1-10,11-20, 21-30", "mappedValue1"))); + targetsParameterConfig = new TargetsParameterConfig( + createMapEntries(createMapping("1-10,11-20, 21-30", "mappedValue1")), "targetField", null, null, null, + null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("11.1"); - final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + final List> translatedRecords = (List>) processor.doExecute( + Collections.singletonList(record)); assertTrue(translatedRecords.get(0).getData().containsKey("targetField")); assertThat(translatedRecords.get(0).getData().get("targetField", String.class), is("mappedValue1")); final Record failureRecord = getEvent("20.5"); - final List> failingTranslatedRecords = (List>) processor.doExecute(Collections.singletonList(failureRecord)); + final List> failingTranslatedRecords = (List>) processor.doExecute( + Collections.singletonList(failureRecord)); assertFalse(failingTranslatedRecords.get(0).getData().containsKey("targetField")); } @Test void test_comma_separated_integer_ranges_and_string_keys() { - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("1-10,key1", "mappedValue1"))); + targetsParameterConfig = new TargetsParameterConfig( + createMapEntries(createMapping("1-10,key1", "mappedValue1")), "targetField", null, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("5.2"); - final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + final List> translatedRecords = (List>) processor.doExecute( + Collections.singletonList(record)); assertTrue(translatedRecords.get(0).getData().containsKey("targetField")); assertThat(translatedRecords.get(0).getData().get("targetField", String.class), is("mappedValue1")); final Record recordStringKey = getEvent("key1"); - final List> translatedStringKeyRecords = (List>) processor.doExecute(Collections.singletonList(recordStringKey)); + final List> translatedStringKeyRecords = (List>) processor.doExecute( + Collections.singletonList(recordStringKey)); assertTrue(translatedStringKeyRecords.get(0).getData().containsKey("targetField")); assertThat(translatedStringKeyRecords.get(0).getData().get("targetField", String.class), is("mappedValue1")); @@ -159,15 +198,19 @@ void test_comma_separated_integer_ranges_and_string_keys() { @Test void test_multiple_dashes_in_keys_should_be_treated_as_string_literal() { - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("1-10-20", "mappedValue1"))); + targetsParameterConfig = new TargetsParameterConfig(createMapEntries(createMapping("1-10-20", "mappedValue1")), + "targetField", null, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record failureRecord = getEvent("1-10-20"); - final List> failingTranslatedRecords = (List>) processor.doExecute(Collections.singletonList(failureRecord)); + final List> failingTranslatedRecords = (List>) processor.doExecute( + Collections.singletonList(failureRecord)); assertTrue(failingTranslatedRecords.get(0).getData().containsKey("targetField")); final Record record = getEvent("10"); - final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + final List> translatedRecords = (List>) processor.doExecute( + Collections.singletonList(record)); assertFalse(translatedRecords.get(0).getData().containsKey("targetField")); @@ -175,26 +218,28 @@ void test_multiple_dashes_in_keys_should_be_treated_as_string_literal() { @Test void test_overlapping_ranges_should_fail_when_overlapping() { - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("1-10", "mappedValue1"), createMapping("10-20", "mappedValue2"))); - - assertThrows(InvalidPluginConfigurationException.class, () -> createObjectUnderTest()); + assertThrows(InvalidPluginConfigurationException.class, () -> new TargetsParameterConfig( + createMapEntries(createMapping("1-10", "mappedValue1"), createMapping("10-20", "mappedValue2")), + "targetField", null, null, null, null)); } @Test void test_overlapping_key_and_range_in_map_option() { - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("1-10", "mappedValue1"), createMapping("5.3", "mappedValue2"))); - - assertThrows(InvalidPluginConfigurationException.class, () -> createObjectUnderTest()); + assertThrows(InvalidPluginConfigurationException.class, () -> new TargetsParameterConfig( + createMapEntries(createMapping("1-10", "mappedValue1"), createMapping("5.3", "mappedValue2")), + "targetField", null, null, null, null)); } @Test void test_string_literal_in_pattern_option() { - when(mockConfig.getRegexParameterConfiguration()).thenReturn(mockRegexConfig); when(mockRegexConfig.getPatterns()).thenReturn(createMapEntries(createMapping("key1", "mappedValue1"))); + targetsParameterConfig = new TargetsParameterConfig(null, "targetField", mockRegexConfig, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("key1"); - final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + final List> translatedRecords = (List>) processor.doExecute( + Collections.singletonList(record)); assertTrue(translatedRecords.get(0).getData().containsKey("targetField")); assertThat(translatedRecords.get(0).getData().get("targetField", String.class), is("mappedValue1")); @@ -207,8 +252,10 @@ void test_string_literal_in_pattern_option() { @Test void test_matching_of_regex_pattern_in_pattern_option() { - when(mockConfig.getRegexParameterConfiguration()).thenReturn(mockRegexConfig); - when(mockRegexConfig.getPatterns()).thenReturn(createMapEntries(createMapping("^(1[0-9]|20)$", "patternValue1"))); //Range between 10-20 + when(mockRegexConfig.getPatterns()).thenReturn( + createMapEntries(createMapping("^(1[0-9]|20)$", "patternValue1"))); + targetsParameterConfig = new TargetsParameterConfig(null, "targetField", mockRegexConfig, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("15"); final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); @@ -224,9 +271,11 @@ void test_matching_of_regex_pattern_in_pattern_option() { @Test void test_pattern_matching_when_no_match_in_map() { - when(mockConfig.getRegexParameterConfiguration()).thenReturn(mockRegexConfig); - when(mockConfig.getMap()).thenReturn((createMapEntries(createMapping("key1", "mappedValue1"), createMapping("key2", "mappedValue2")))); when(mockRegexConfig.getPatterns()).thenReturn(createMapEntries(createMapping("patternKey1", "patternValue1"))); + targetsParameterConfig = new TargetsParameterConfig( + (createMapEntries(createMapping("key1", "mappedValue1"), createMapping("key2", "mappedValue2"))), + "targetField", mockRegexConfig, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("patternKey1"); @@ -244,9 +293,11 @@ void test_pattern_matching_when_no_match_in_map() { @Test void test_map_matching_when_overlapping_ranges_in_map_and_pattern() { - when(mockConfig.getRegexParameterConfiguration()).thenReturn(mockRegexConfig); - when(mockConfig.getMap()).thenReturn((createMapEntries(createMapping("400", "mappedValue1")))); - when(mockRegexConfig.getPatterns()).thenReturn(createMapEntries(createMapping("^(400|404)$", "patternValue1"))); // Matches 400 or 404 + when(mockRegexConfig.getPatterns()).thenReturn(createMapEntries(createMapping("^(400|404)$", "patternValue1"))); + targetsParameterConfig = new TargetsParameterConfig( + (createMapEntries(createMapping("400", "mappedValue1"), createMapping("key2", "mappedValue2"))), + "targetField", mockRegexConfig, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("400"); @@ -264,8 +315,10 @@ void test_map_matching_when_overlapping_ranges_in_map_and_pattern() { @Test void test_source_array_single_key() { - when(mockConfig.getSource()).thenReturn(new ArrayList(List.of("sourceField"))); - when(mockConfig.getMap()).thenReturn((createMapEntries(createMapping("400", "mappedValue1")))); + when(mappingsParameterConfig.getSource()).thenReturn(new ArrayList(List.of("sourceField"))); + targetsParameterConfig = new TargetsParameterConfig(createMapEntries(createMapping("400", "mappedValue1")), + "targetField", null, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("400"); @@ -277,8 +330,11 @@ void test_source_array_single_key() { @Test void test_source_array_multiple_keys() { - when(mockConfig.getSource()).thenReturn(new ArrayList(List.of("sourceField1", "sourceField2"))); - when(mockConfig.getMap()).thenReturn((createMapEntries(createMapping("key1", "mappedValue1"), createMapping("key2", "mappedValue2"), createMapping("key3", "mappedValue3")))); + when(mappingsParameterConfig.getSource()).thenReturn(new ArrayList(List.of("sourceField1", "sourceField2"))); + targetsParameterConfig = new TargetsParameterConfig( + createMapEntries(createMapping("key1", "mappedValue1"), createMapping("key2", "mappedValue2"), + createMapping("key3", "mappedValue3")), "targetField", null, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = buildRecordWithEvent(Map.of("sourceField1", "key1", "sourceField2", "key3")); @@ -290,8 +346,11 @@ void test_source_array_multiple_keys() { @Test void test_source_array_with_partial_match_without_default() { - when(mockConfig.getSource()).thenReturn(new ArrayList(List.of("sourceField1", "sourceField2"))); - when(mockConfig.getMap()).thenReturn((createMapEntries(createMapping("key1", "mappedValue1"), createMapping("key2", "mappedValue2"), createMapping("key3", "mappedValue3")))); + when(mappingsParameterConfig.getSource()).thenReturn(new ArrayList(List.of("sourceField1", "sourceField2"))); + targetsParameterConfig = new TargetsParameterConfig( + createMapEntries(createMapping("key1", "mappedValue1"), createMapping("key2", "mappedValue2"), + createMapping("key3", "mappedValue3")), "targetField", null, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = buildRecordWithEvent(Map.of("sourceField1", "key1", "sourceField2", "key4")); @@ -304,9 +363,11 @@ void test_source_array_with_partial_match_without_default() { @Test void test_source_array_with_partial_match_with_default() { final String defaultValue = "No Match Found"; - when(mockConfig.getSource()).thenReturn(new ArrayList(List.of("sourceField1", "sourceField2"))); - when(mockConfig.getDefaultValue()).thenReturn(defaultValue); - when(mockConfig.getMap()).thenReturn((createMapEntries(createMapping("key1", "mappedValue1"), createMapping("key2", "mappedValue2"), createMapping("key3", "mappedValue3")))); + when(mappingsParameterConfig.getSource()).thenReturn(new ArrayList(List.of("sourceField1", "sourceField2"))); + targetsParameterConfig = new TargetsParameterConfig( + createMapEntries(createMapping("key1", "mappedValue1"), createMapping("key2", "mappedValue2"), + createMapping("key3", "mappedValue3")), "targetField", null, null, defaultValue, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = buildRecordWithEvent(Map.of("sourceField1", "key1", "sourceField2", "key4")); @@ -318,11 +379,12 @@ void test_source_array_with_partial_match_with_default() { @Test void test_non_exact_matching() { - when(mockConfig.getRegexParameterConfiguration()).thenReturn(mockRegexConfig); + when(mockRegexConfig.getExact()).thenReturn(false); when(mockRegexConfig.getPatterns()).thenReturn(createMapEntries( createMapping("^(1[0-9]|20)$", "patternValue1"), createMapping("foo", "bar2"))); - when(mockRegexConfig.getExact()).thenReturn(false); + targetsParameterConfig = new TargetsParameterConfig(null, "targetField", mockRegexConfig, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("footer"); @@ -354,12 +416,13 @@ void test_nested_records_with_default_value() { Map.of("sourceField", "key2", "targetField", "mappedValue2"), Map.of("sourceField", "key3", "targetField", "No Match")); - when(mockConfig.getRegexParameterConfiguration()).thenReturn(mockRegexConfig); when(mockRegexConfig.getPatterns()).thenReturn(createMapEntries( createMapping("key1", "mappedValue1"), createMapping("key2", "mappedValue2"))); - when(mockConfig.getDefaultValue()).thenReturn("No Match"); - when(mockConfig.getIterateOn()).thenReturn("collection"); + when(mappingsParameterConfig.getIterateOn()).thenReturn("collection"); + targetsParameterConfig = new TargetsParameterConfig(null, "targetField", mockRegexConfig, null, "No Match", + null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = buildRecordWithEvent(testJson); @@ -379,11 +442,12 @@ void test_nested_records_without_default_value() { Map.of("sourceField", "key2", "targetField", "mappedValue2"), Map.of("sourceField", "key3")); - when(mockConfig.getRegexParameterConfiguration()).thenReturn(mockRegexConfig); when(mockRegexConfig.getPatterns()).thenReturn(createMapEntries( createMapping("key1", "mappedValue1"), createMapping("key2", "mappedValue2"))); - when(mockConfig.getIterateOn()).thenReturn("collection"); + when(mappingsParameterConfig.getIterateOn()).thenReturn("collection"); + targetsParameterConfig = new TargetsParameterConfig(null, "targetField", mockRegexConfig, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = buildRecordWithEvent(testJson); @@ -403,9 +467,10 @@ void test_nested_records_no_match() { Map.of("sourceField", "key2"), Map.of("sourceField", "key3")); - when(mockConfig.getRegexParameterConfiguration()).thenReturn(mockRegexConfig); when(mockRegexConfig.getPatterns()).thenReturn(createMapEntries(createMapping("key4", "mappedValue1"))); - when(mockConfig.getIterateOn()).thenReturn("collection"); + when(mappingsParameterConfig.getIterateOn()).thenReturn("collection"); + targetsParameterConfig = new TargetsParameterConfig(null, "targetField", mockRegexConfig, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = buildRecordWithEvent(testJson); @@ -415,8 +480,10 @@ void test_nested_records_no_match() { } @Test - void test_target_type_default(){ - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("key1", "200"))); + void test_target_type_default() { + targetsParameterConfig = new TargetsParameterConfig(createMapEntries(createMapping("key1", "200")), + "targetField", null, null, null, null); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("key1"); final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); @@ -426,9 +493,10 @@ void test_target_type_default(){ } @Test - void test_target_type_integer(){ - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("key1", "200"))); - when(mockConfig.getTargetType()).thenReturn(TargetType.INTEGER); + void test_target_type_integer() { + targetsParameterConfig = new TargetsParameterConfig(createMapEntries(createMapping("key1", "200")), + "targetField", null, null, null, TargetType.INTEGER); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("key1"); final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); @@ -438,9 +506,10 @@ void test_target_type_integer(){ } @Test - void test_target_type_boolean(){ - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("key1", "false"))); - when(mockConfig.getTargetType()).thenReturn(TargetType.BOOLEAN); + void test_target_type_boolean() { + targetsParameterConfig = new TargetsParameterConfig(createMapEntries(createMapping("key1", "false")), + "targetField", null, null, null, TargetType.BOOLEAN); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("key1"); final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); @@ -450,9 +519,10 @@ void test_target_type_boolean(){ } @Test - void test_target_type_double(){ - when(mockConfig.getMap()).thenReturn(createMapEntries(createMapping("key1", "20.3"))); - when(mockConfig.getTargetType()).thenReturn(TargetType.DOUBLE); + void test_target_type_double() { + targetsParameterConfig = new TargetsParameterConfig(createMapEntries(createMapping("key1", "20.3")), + "targetField", null, null, null, TargetType.DOUBLE); + when(mappingsParameterConfig.getTargetsParameterConfigs()).thenReturn(List.of(targetsParameterConfig)); final TranslateProcessor processor = createObjectUnderTest(); final Record record = getEvent("key1"); final List> translatedRecords = (List>) processor.doExecute(Collections.singletonList(record)); @@ -461,7 +531,6 @@ void test_target_type_double(){ assertThat(translatedRecords.get(0).getData().get("targetField", Double.class), is(20.3)); } - private TranslateProcessor createObjectUnderTest() { return new TranslateProcessor(pluginMetrics, mockConfig, expressionEvaluator); }