forked from opensearch-project/data-prepper
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Translate Processor - Added file_path functionality for local files (o…
…pensearch-project#3034) * Changed target_type option name to type Signed-off-by: Vishal Boinapalli <vishalboinapalli3@gmail.com> * Added file_path functionality for local file Signed-off-by: Vishal Boinapalli <vishalboinapalli3@gmail.com> * Fixed file parsing issue and changed error msgs Signed-off-by: Vishal Boinapalli <vishalboinapalli3@gmail.com> * Added IOException to log, made testcase change for mappings validation Signed-off-by: Vishal Boinapalli <vishalboinapalli3@gmail.com> --------- Signed-off-by: Vishal Boinapalli <vishalboinapalli3@gmail.com>
- Loading branch information
1 parent
cf47dac
commit 958c15e
Showing
8 changed files
with
339 additions
and
21 deletions.
There are no files selected for viewing
60 changes: 60 additions & 0 deletions
60
.../src/main/java/org/opensearch/dataprepper/plugins/processor/translate/FilePathParser.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
package org.opensearch.dataprepper.plugins.processor.translate; | ||
|
||
import com.fasterxml.jackson.annotation.JsonProperty; | ||
import jakarta.validation.Valid; | ||
|
||
import java.util.ArrayList; | ||
import java.util.List; | ||
import java.util.Objects; | ||
import java.util.Optional; | ||
|
||
public class FilePathParser { | ||
@JsonProperty("mappings") | ||
@Valid | ||
private List<MappingsParameterConfig> fileMappingConfigs; | ||
|
||
public Optional<List<MappingsParameterConfig>> getCombinedMappings(List<MappingsParameterConfig> mappingConfigs) { | ||
if(Objects.isNull(mappingConfigs) || mappingConfigs.isEmpty()){ | ||
return Optional.ofNullable(fileMappingConfigs); | ||
} | ||
try{ | ||
for (MappingsParameterConfig fileMappingConfig : fileMappingConfigs) { | ||
boolean isDuplicateSource = false; | ||
for (MappingsParameterConfig mappingConfig : mappingConfigs) { | ||
if (mappingConfig.getSource().equals(fileMappingConfig.getSource())) { | ||
isDuplicateSource = true; | ||
combineTargets(fileMappingConfig, mappingConfig); | ||
} | ||
} | ||
if (!isDuplicateSource) { | ||
mappingConfigs.add(fileMappingConfig); | ||
} | ||
} | ||
return Optional.of(mappingConfigs); | ||
} catch (Exception ex){ | ||
return Optional.empty(); | ||
} | ||
} | ||
|
||
private void combineTargets(MappingsParameterConfig filePathMapping, MappingsParameterConfig mappingConfig) { | ||
if(Objects.isNull(mappingConfig)){ | ||
return; | ||
} | ||
List<TargetsParameterConfig> fileTargetConfigs = filePathMapping.getTargetsParameterConfigs(); | ||
List<TargetsParameterConfig> mappingsTargetConfigs = mappingConfig.getTargetsParameterConfigs(); | ||
List<TargetsParameterConfig> combinedTargetConfigs = new ArrayList<>(mappingsTargetConfigs); | ||
|
||
for (TargetsParameterConfig fileTargetConfig : fileTargetConfigs) { | ||
if (!isTargetPresent(fileTargetConfig, combinedTargetConfigs)) { | ||
combinedTargetConfigs.add(fileTargetConfig); | ||
} | ||
} | ||
mappingConfig.setTargetsParameterConfigs(combinedTargetConfigs); | ||
} | ||
|
||
private boolean isTargetPresent(TargetsParameterConfig fileTargetConfig, List<TargetsParameterConfig> combinedTargetConfigs){ | ||
String fileTarget = fileTargetConfig.getTarget(); | ||
return combinedTargetConfigs.stream().anyMatch(targetConfig -> fileTarget.equals(targetConfig.getTarget())); | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.