Skip to content

Commit

Permalink
SDC-9924. Validation of ELs fails with some string EL functions
Browse files Browse the repository at this point in the history
Change-Id: I63ee3bf1467bf3d7e428e5a0a7ed76446103795a
Reviewed-on: https://review.streamsets.net/16500
Reviewed-by: Alejandro Abdelnur <tucu@streamsets.com>
Tested-by: StreamSets CI <streamsets-ci-spam@streamsets.com>
  • Loading branch information
tucu00 committed Sep 6, 2018
1 parent 4026886 commit da108f9
Show file tree
Hide file tree
Showing 21 changed files with 99 additions and 222 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -106,30 +106,20 @@ public List<ConfigIssue> init() {
TimeNowEL.setTimeNowInContext(elVars, new Date());

if (partitionTemplate.contains(EL_PREFIX)) {
ELUtils.validateExpression(
partitionEval,
elVars,
partitionTemplate,
ELUtils.validateExpression(partitionTemplate,
getContext(),
Groups.S3.getLabel(),
S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + PARTITION_TEMPLATE,
Errors.S3_03,
String.class,
issues
Errors.S3_03, issues
);
}

if (timeDriverTemplate.contains(EL_PREFIX)) {
ELUtils.validateExpression(
timeDriverEval,
elVars,
timeDriverTemplate,
ELUtils.validateExpression(timeDriverTemplate,
getContext(),
Groups.S3.getLabel(),
S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + TIME_DRIVER_TEMPLATE,
Errors.S3_04,
Date.class,
issues
Errors.S3_04, issues
);
}
if (getContext().getService(DataFormatGeneratorService.class).isWholeFileFormat()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,33 +108,23 @@ protected List<ConfigIssue> init() {
TimeNowEL.setTimeNowInContext(dirPathTemplateVars, new Date());

// Validate Evals
ELUtils.validateExpression(
dirPathTemplateEval,
getContext().createELVars(),
conf.dirPathTemplate,
ELUtils.validateExpression(conf.dirPathTemplate,
getContext(),
Groups.DATALAKE.getLabel(),
DataLakeConfigBean.ADLS_CONFIG_BEAN_PREFIX + "dirPathTemplate",
Errors.ADLS_00,
String.class,
issues
Errors.ADLS_00, issues
);
}

if (conf.timeDriver.startsWith(EL_PREFIX)) {
TimeEL.setCalendarInContext(timeDriverVars, calendar);
TimeNowEL.setTimeNowInContext(timeDriverVars, new Date());

ELUtils.validateExpression(
timeDriverEval,
timeDriverVars,
conf.timeDriver,
ELUtils.validateExpression(conf.timeDriver,
getContext(),
Groups.DATALAKE.getLabel(),
DataLakeConfigBean.ADLS_CONFIG_BEAN_PREFIX + "timeDriver",
Errors.ADLS_01,
Date.class,
issues
Errors.ADLS_01, issues
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,57 +59,41 @@ public List<ConfigIssue> init() {
for (EmailConfig co : conf) {
co.subjectELEval = getContext().createELEval("subject");
co.subjectELVars = getContext().createELVars();
ELUtils.validateExpression(co.subjectELEval,
co.subjectELVars,
co.subject,
ELUtils.validateExpression(co.subject,
getContext(),
Groups.EMAILS.name(),
co.subject,
Errors.EMAIL_01,
String.class,
issues
Errors.EMAIL_01, issues
);

co.bodyELEval = getContext().createELEval("body");
co.bodyELVars = getContext().createELVars();
ELUtils.validateExpression(co.bodyELEval,
co.bodyELVars,
co.body,
ELUtils.validateExpression(co.body,
getContext(),
Groups.EMAILS.name(),
co.body,
Errors.EMAIL_02,
String.class,
issues
Errors.EMAIL_02, issues
);

co.conditionELEval = getContext().createELEval("condition");
co.conditionELVars = getContext().createELVars();
ELUtils.validateExpression(co.conditionELEval,
co.conditionELVars,
co.condition,
ELUtils.validateExpression(co.condition,
getContext(),
Groups.EMAILS.name(),
co.condition,
Errors.EMAIL_06,
String.class,
issues
Errors.EMAIL_06, issues
);

co.emailELEval = new HashMap<>();
co.emailELVars = new HashMap<>();
for (String str : co.email) {
ELEval eval = getContext().createELEval("email");
ELVars vars = getContext().createELVars();
ELUtils.validateExpression(eval,
vars,
str,
ELUtils.validateExpression(str,
getContext(),
Groups.EMAILS.name(),
str,
Errors.EMAIL_07,
String.class,
issues
Errors.EMAIL_07, issues
);
co.emailELEval.put(str, eval);
co.emailELVars.put(str, vars);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -257,16 +257,11 @@ private boolean validateEls(List<ConfigIssue> issues) {
Record validationRecord = getContext().createRecord("recordProcessedAckMessageValidationRecord");
RecordEL.setRecordInContext(vars, validationRecord);

ELUtils.validateExpression(
eval,
vars,
config.recordProcessedAckMessage,
ELUtils.validateExpression(config.recordProcessedAckMessage,
getContext(),
Groups.TCP.name(),
CONF_PREFIX + "recordProcessedAckMessage",
Errors.TCP_30,
String.class,
issues
Errors.TCP_30, issues
);
}

Expand All @@ -280,16 +275,11 @@ private boolean validateEls(List<ConfigIssue> issues) {
Record validationRecord = getContext().createRecord("batchCompletedAckMessageValidationRecord");
RecordEL.setRecordInContext(vars, validationRecord);

ELUtils.validateExpression(
eval,
vars,
config.batchCompletedAckMessage,
ELUtils.validateExpression(config.batchCompletedAckMessage,
getContext(),
Groups.TCP.name(),
CONF_PREFIX + "batchCompletedAckMessage",
Errors.TCP_31,
String.class,
issues
Errors.TCP_31, issues
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,32 +67,26 @@ null, getContext(), Groups.EXPRESSIONS.name(), "constants", Errors.EXPR_01, issu
expressionVars.addContextVariable(StringELConstants.MEMOIZED, memoizedVars);
expressionEval = createExpressionEval(getContext());
for(ExpressionProcessorConfig expressionProcessorConfig : expressionProcessorConfigs) {
ELUtils.validateExpression(expressionEval, expressionVars, expressionProcessorConfig.expression, getContext(),
Groups.EXPRESSIONS.name(), "expressionProcessorConfigs", Errors.EXPR_00,
Object.class, issues);
ELUtils.validateExpression(expressionProcessorConfig.expression, getContext(),
Groups.EXPRESSIONS.name(), "expressionProcessorConfigs", Errors.EXPR_00, issues);
}

if(headerAttributeConfigs != null && !headerAttributeConfigs.isEmpty()) {
headerAttributeEval = createHeaderAttributeEval(getContext());
for (HeaderAttributeConfig headerAttributeConfig : headerAttributeConfigs) {
ELUtils.validateExpression(headerAttributeEval, expressionVars, headerAttributeConfig.headerAttributeExpression,
getContext(), Groups.EXPRESSIONS.name(), "headerAttributeConfigs", Errors.EXPR_00, Object.class, issues);
ELUtils.validateExpression(headerAttributeConfig.headerAttributeExpression,
getContext(), Groups.EXPRESSIONS.name(), "headerAttributeConfigs", Errors.EXPR_00, issues);
}
}

if (fieldAttributeConfigs != null && !fieldAttributeConfigs.isEmpty()) {
fieldAttributeEval = createFieldAttributeEval(getContext());
for (FieldAttributeConfig fieldAttributeConfig : fieldAttributeConfigs) {
ELUtils.validateExpression(
fieldAttributeEval,
expressionVars,
fieldAttributeConfig.fieldAttributeExpression,
ELUtils.validateExpression(fieldAttributeConfig.fieldAttributeExpression,
getContext(),
Groups.EXPRESSIONS.name(),
"fieldAttributeConfigs",
Errors.EXPR_00,
Object.class,
issues
Errors.EXPR_00, issues
);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,8 @@ protected List<ConfigIssue> init() {
issues.add(getContext().createConfigIssue(Groups.CONDITIONS.name(), "lanePredicates", Errors.SELECTOR_08,
predicateLane[0]));
} else {
ELUtils.validateExpression(predicateLanesEval, variables, predicateLane[0], getContext(),
Groups.CONDITIONS.name(), "lanePredicates", Errors.SELECTOR_03,
Boolean.class, issues);
ELUtils.validateExpression(predicateLane[0], getContext(),
Groups.CONDITIONS.name(), "lanePredicates", Errors.SELECTOR_03, issues);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public void testInitWithBadEmail() throws StageException {
EmailConfig conf = new EmailConfig();
conf.email = new ArrayList<>();

conf.email.add("${badEL}admin@example.com");
conf.email.add("${badEL x}admin@example.com");

conf.condition = "${str:contains(time:extractStringFromDate(time:now(), \"YYYY-MM-dd hh:mm:ss\"), \"20\")}";
conf.subject = "subject ${time:extractStringFromDate(time:now(), \"yyyy\")} subject";
Expand All @@ -68,7 +68,7 @@ public void testInitWithBadSubject() throws StageException {
conf.email = new ArrayList<>();
conf.email.add("admin@exa${record:id()}mple.com");

conf.subject = "${thisIsAnError} subject";
conf.subject = "${thisIsAnError x} subject";

conf.condition = "${str:contains(time:extractStringFromDate(time:now(), \"YYYY-MM-dd hh:mm:ss\"), \"20\")}";
conf.body = "body ${time:extractStringFromDate(time:now(), \"yyyy\")} body";
Expand All @@ -91,7 +91,7 @@ public void testInitWithBadBody() throws StageException {
conf.email = new ArrayList<>();
conf.email.add("admin@exa${record:id()}mple.com");

conf.body = "body ${record:hahaha()}";
conf.body = "body ${record:hahaha()x}";

conf.condition = "${str:contains(time:extractStringFromDate(time:now(), \"YYYY-MM-dd hh:mm:ss\"), \"20\")}";
conf.subject = "subject subject";
Expand All @@ -113,7 +113,7 @@ public void testInitWithBadCondition() throws StageException {
EmailConfig conf = new EmailConfig();
conf.email = new ArrayList<>();

conf.condition = "${total_junk}";
conf.condition = "${total_junk x}";
conf.subject = "subject ${time:extractStringFromDate(time:now(), \"yyyy\")} subject";
conf.body = "body ${time:extractStringFromDate(time:now(), \"yyyy\")} body";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,9 @@ public void testValidation() throws Exception {
.addConstants(constants)
.build();
issues = runner.runValidateConfigs();
Assert.assertEquals(3, issues.size());
Assert.assertEquals(2, issues.size());
Assert.assertTrue(issues.get(0).toString().contains("SELECTOR_02"));
Assert.assertTrue(issues.get(1).toString().contains("SELECTOR_08"));
Assert.assertTrue(issues.get(2).toString().contains("SELECTOR_03"));

}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,37 @@ public static ELVars parseConstants(Map<String,?> constants, Stage.Context conte
return variables;
}

public static void validateExpression(ELEval elEvaluator, ELVars variables, String expression,
Stage.Context context, String group, String config, ErrorCode err, Class<?> type, List<Stage.ConfigIssue> issues)
@Deprecated
public static void validateExpression(
ELEval elEvaluator,
ELVars variables,
String expression,
Stage.Context context,
String group,
String config,
ErrorCode err,
Class<?> type,
List<Stage.ConfigIssue> issues
) {
validateExpression(expression, context, group, config, err, issues);
try {
context.parseEL(expression);
} catch (Exception ex) {
issues.add(context.createConfigIssue(group, config, err, expression, ex.toString(), ex));
}
}

public static void validateExpression(
String expression,
Stage.Context context,
String group,
String config,
ErrorCode err,
List<Stage.ConfigIssue> issues
)
{
RecordEL.setRecordInContext(variables, context.createRecord("forValidation"));
try {
context.parseEL(expression);
elEvaluator.eval(variables, expression, type);
} catch (Exception ex) {
issues.add(context.createConfigIssue(group, config, err, expression, ex.toString(), ex));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,42 +116,27 @@ protected List<ConfigIssue> init() {
Errors.HBASE_35
));
} else {
ELUtils.validateExpression(
keyExprEval,
getContext().createELVars(),
lookup.rowExpr,
ELUtils.validateExpression(lookup.rowExpr,
getContext(),
Groups.HBASE.getLabel(),
lookup.rowExpr,
Errors.HBASE_38,
String.class,
issues
Errors.HBASE_38, issues
);
}

// Column and timestamps are optional
ELUtils.validateExpression(
columnExprEval,
getContext().createELVars(),
lookup.columnExpr,
ELUtils.validateExpression(lookup.columnExpr,
getContext(),
Groups.HBASE.getLabel(),
lookup.columnExpr,
Errors.HBASE_38,
String.class,
issues
Errors.HBASE_38, issues
);

ELUtils.validateExpression(
timestampExprEval,
getContext().createELVars(),
lookup.timestampExpr,
ELUtils.validateExpression(lookup.timestampExpr,
getContext(),
Groups.HBASE.getLabel(),
lookup.timestampExpr,
Errors.HBASE_38,
Date.class,
issues
Errors.HBASE_38, issues
);

if(lookup.outputFieldPath.length() < 1) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -182,26 +182,16 @@ public void init(final Stage.Context context, final String prefix, final List<St
String attributeNameExpression = entry.getKey();
String attributeValueExpression = entry.getValue();

ELUtils.validateExpression(
headerAttributeConfigsEL,
context.createELVars(),
attributeNameExpression,
ELUtils.validateExpression(attributeNameExpression,
context,
Groups.ADVANCED.getLabel(),
"headerAttributeConfigs",
Errors.HIVE_39,
Object.class,
issues);
ELUtils.validateExpression(
headerAttributeConfigsEL,
context.createELVars(),
attributeValueExpression,
Errors.HIVE_39, issues);
ELUtils.validateExpression(attributeValueExpression,
context,
Groups.ADVANCED.getLabel(),
"headerAttributeConfigs",
Errors.HIVE_39,
Object.class,
issues);
Errors.HIVE_39, issues);
}
} else {
headersEmpty = true;
Expand Down
Loading

0 comments on commit da108f9

Please sign in to comment.