-
Notifications
You must be signed in to change notification settings - Fork 194
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Pipeline Configuration Transformation (#4446)
* Adding templates Signed-off-by: srigovs <srigovs@amazon.com> * Added Dynamic yaml transformer Signed-off-by: srigovs <srigovs@amazon.com> * Added Rule evaluator Signed-off-by: srigovs <srigovs@amazon.com> * Added rule evaluator Signed-off-by: srigovs <srigovs@amazon.com> * Added json walk Signed-off-by: srigovs <srigovs@amazon.com> * Add transformation logic Signed-off-by: srigovs <srigovs@amazon.com> * Add dynamic rule Signed-off-by: srigovs <srigovs@amazon.com> * Almost working Signed-off-by: srigovs <srigovs@amazon.com> * Adding multiple pipelines part1 Signed-off-by: srigovs <srigovs@amazon.com> * Adding multiple pipelines part2-incomplete Signed-off-by: srigovs <srigovs@amazon.com> * Works e2e for 1 pipeline Signed-off-by: srigovs <srigovs@amazon.com> * Added multi pipeline template and pipelinemodel support, works for docDB with big template Signed-off-by: srigovs <srigovs@amazon.com> * added tests for models and fixed beans, one more fix needed for bean Signed-off-by: srigovs <srigovs@amazon.com> * Fixed IT and beans Signed-off-by: srigovs <srigovs@amazon.com> * Update bean to have only pipelineDataModel and not parser Signed-off-by: srigovs <srigovs@amazon.com> * Add banner Signed-off-by: srigovs <srigovs@amazon.com> * Code cleanup and add comments Signed-off-by: srigovs <srigovs@amazon.com> * Support user pipeline configuration dynamic transformation based on templates and rules Signed-off-by: srigovs <srigovs@amazon.com> * Address comments Signed-off-by: srigovs <srigovs@amazon.com> * Added Function Call support in templates Signed-off-by: srigovs <srigovs@amazon.com> * Added Function Call support in templates Signed-off-by: srigovs <srigovs@amazon.com> * Modify documentDB template. Signed-off-by: srigovs <srigovs@amazon.com> * Code clean up Signed-off-by: srigovs <srigovs@amazon.com> * Code clean up Signed-off-by: srigovs <srigovs@amazon.com> --------- Signed-off-by: srigovs <srigovs@amazon.com>
- Loading branch information
1 parent
8bd7fc9
commit e590fde
Showing
47 changed files
with
2,224 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
43 changes: 43 additions & 0 deletions
43
.../java/org/opensearch/dataprepper/pipeline/parser/PipelineTransformationConfiguration.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.dataprepper.pipeline.parser; | ||
|
|
||
| import org.opensearch.dataprepper.pipeline.parser.rule.RuleEvaluator; | ||
| import org.opensearch.dataprepper.pipeline.parser.transformer.TransformersFactory; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Configuration; | ||
|
|
||
| import javax.inject.Named; | ||
|
|
||
| @Configuration | ||
| public class PipelineTransformationConfiguration { | ||
| public static final String TEMPLATES_DIRECTORY_PATH = "TEMPLATES_DIRECTORY_PATH"; | ||
| public static final String RULES_DIRECTORY_PATH = "VALIDATORS_DIRECTORY_PATH"; | ||
|
|
||
| @Bean | ||
| @Named(RULES_DIRECTORY_PATH) | ||
| static String provideRulesDirectoryPath() { | ||
| return "resources/rules"; | ||
| } | ||
|
|
||
| @Bean | ||
| @Named(TEMPLATES_DIRECTORY_PATH) | ||
| static String provideTemplateDirectoryPath() { | ||
| return "resources/templates"; | ||
| } | ||
|
|
||
| @Bean | ||
| TransformersFactory transformersFactory( | ||
| @Named(TEMPLATES_DIRECTORY_PATH) String provideTransformerDirectoryPath, | ||
| @Named(RULES_DIRECTORY_PATH) String provideTemplateDirectoryPath | ||
| ) { | ||
| return new TransformersFactory(RULES_DIRECTORY_PATH, TEMPLATES_DIRECTORY_PATH); | ||
| } | ||
|
|
||
| @Bean | ||
| public RuleEvaluator ruleEvaluator(TransformersFactory transformersFactory) { | ||
| return new RuleEvaluator(transformersFactory); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
118 changes: 118 additions & 0 deletions
118
...e-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.dataprepper.pipeline.parser.rule; | ||
|
|
||
| import com.fasterxml.jackson.core.JsonProcessingException; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; | ||
| import com.jayway.jsonpath.Configuration; | ||
| import com.jayway.jsonpath.JsonPath; | ||
| import com.jayway.jsonpath.Option; | ||
| import com.jayway.jsonpath.ParseContext; | ||
| import com.jayway.jsonpath.PathNotFoundException; | ||
| import com.jayway.jsonpath.ReadContext; | ||
| import com.jayway.jsonpath.spi.json.JacksonJsonProvider; | ||
| import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; | ||
| import org.opensearch.dataprepper.model.configuration.PipelineModel; | ||
| import org.opensearch.dataprepper.model.configuration.PipelinesDataFlowModel; | ||
| import org.opensearch.dataprepper.pipeline.parser.transformer.PipelineTemplateModel; | ||
| import org.opensearch.dataprepper.pipeline.parser.transformer.TransformersFactory; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| public class RuleEvaluator { | ||
|
|
||
| private static final Logger LOG = LoggerFactory.getLogger(RuleEvaluator.class); | ||
| private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); | ||
| private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); | ||
| private final TransformersFactory transformersFactory; | ||
| private String PLUGIN_NAME = null; | ||
|
|
||
| public RuleEvaluator(TransformersFactory transformersFactory) { | ||
| this.transformersFactory = transformersFactory; | ||
| } | ||
|
|
||
| public RuleEvaluatorResult isTransformationNeeded(PipelinesDataFlowModel pipelineModel) { | ||
| return isDocDBSource(pipelineModel); | ||
| } | ||
|
|
||
| /** | ||
| * Evaluates model based on pre defined rules and | ||
| * result contains the name of the pipeline that will need transformation, | ||
| * evaluated boolean result and the corresponding template model | ||
| * Assumption: only one pipeline can have transformation. | ||
| * @param pipelinesModel | ||
| * @return RuleEvaluatorResult | ||
| */ | ||
| private RuleEvaluatorResult isDocDBSource(PipelinesDataFlowModel pipelinesModel) { | ||
| PLUGIN_NAME = "documentdb"; | ||
| String pluginRulesPath = transformersFactory.getPluginRuleFileLocation(PLUGIN_NAME); | ||
| Map<String, PipelineModel> pipelines = pipelinesModel.getPipelines(); | ||
|
|
||
| for (Map.Entry<String, PipelineModel> entry : pipelines.entrySet()) { | ||
| try { | ||
| String pipelineJson = OBJECT_MAPPER.writeValueAsString(entry); | ||
| if (evaluate(pipelineJson, pluginRulesPath)) { | ||
| LOG.debug("Rule path {} is evaluated true for pipelineJson {}",pluginRulesPath, pipelineJson); | ||
|
|
||
| String templateFilePath = transformersFactory.getPluginTemplateFileLocation(PLUGIN_NAME); | ||
| PipelineTemplateModel templateModel = yamlMapper.readValue(new File(templateFilePath), | ||
| PipelineTemplateModel.class); | ||
| LOG.debug("Chosen template file {}",templateFilePath); | ||
|
|
||
| return RuleEvaluatorResult.builder() | ||
| .withEvaluatedResult(true) | ||
| .withPipelineTemplateModel(templateModel) | ||
| .withPipelineName(entry.getKey()) | ||
| .build(); | ||
| } | ||
| } catch (JsonProcessingException e) { | ||
| LOG.error("Error processing json"); | ||
| throw new RuntimeException(e); | ||
| } catch (IOException e) { | ||
| LOG.error("Error reading file"); | ||
| throw new RuntimeException(e); | ||
| } | ||
| } | ||
| return RuleEvaluatorResult.builder() | ||
| .withEvaluatedResult(false) | ||
| .withPipelineName(null) | ||
| .withPipelineTemplateModel(null) | ||
| .build(); | ||
| } | ||
|
|
||
| private Boolean evaluate(String pipelinesJson, | ||
| String rulePath) { | ||
|
|
||
| Configuration parseConfig = Configuration.builder() | ||
| .jsonProvider(new JacksonJsonProvider()) | ||
| .mappingProvider(new JacksonMappingProvider()) | ||
| .options(Option.AS_PATH_LIST) | ||
| .build(); | ||
| ParseContext parseContext = JsonPath.using(parseConfig); | ||
| ReadContext readPathContext = parseContext.parse(pipelinesJson); | ||
|
|
||
| try { | ||
| RuleTransformerModel rulesModel = yamlMapper.readValue(new File(rulePath), RuleTransformerModel.class); | ||
| List<String> rules = rulesModel.getApplyWhen(); | ||
| for (String rule : rules) { | ||
| Object result = readPathContext.read(rule); | ||
| } | ||
| } catch (IOException e) { | ||
| LOG.warn("Error reading file {}", rulePath); | ||
| return false; | ||
| } catch (PathNotFoundException e) { | ||
| LOG.warn("Path not found {}", rulePath); | ||
| return false; | ||
| } | ||
| return true; | ||
| } | ||
| } | ||
|
|
26 changes: 26 additions & 0 deletions
26
...er/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorResult.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.dataprepper.pipeline.parser.rule; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.Builder; | ||
| import lombok.Getter; | ||
| import org.opensearch.dataprepper.pipeline.parser.transformer.PipelineTemplateModel; | ||
|
|
||
| @Builder(setterPrefix = "with") | ||
| @Getter | ||
| @AllArgsConstructor | ||
| public class RuleEvaluatorResult { | ||
|
|
||
| private boolean evaluatedResult; | ||
|
|
||
| private String pipelineName; | ||
|
|
||
| private PipelineTemplateModel pipelineTemplateModel; | ||
|
|
||
| public RuleEvaluatorResult() { | ||
|
|
||
| } | ||
| } |
Oops, something went wrong.