Merge remote-tracking branch 'apache/master' into METRON-590

Author: nickwallen

metron-platform/metron-common/README.md

@@ -108,6 +108,7 @@ Using parens such as: "foo" : "\<ok\>" requires escaping; "foo": "\'\<ok\>\'"
 | [ `SYSTEM_PROPERTY_GET`](#system_property_get)|
 | [ `TO_DOUBLE`](#to_double)|
 | [ `TO_EPOCH_TIMESTAMP`](#to_epoch_timestamp)|
+| [ `TO_FLOAT`](#to_float)|
 | [ `TO_INTEGER`](#to_integer)|
 | [ `TO_LONG`](#to_long)|
 | [ `TO_LOWER`](#to_lower)|
@@ -436,6 +437,12 @@ MAP_GET`
     * format - DateTime format as a String
     * timezone - Optional timezone in String format
   * Returns: Epoch timestamp
+  
+### `TO_FOAT`
+  * Description: Transforms the first argument to a float
+  * Input:
+    * input - Object of string or numeric type
+  * Returns: Float version of the first argument
 
 ### `TO_INTEGER`
   * Description: Transforms the first argument to an integer
@@ -643,7 +650,7 @@ JOIN, LENGTH, MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP_EXISTS, MAP_GET, MONTH, P
 REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, STATS_COUNT, STATS_GEOMETRIC_MEAN, STATS_INIT, 
 STATS_KURTOSIS, STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN, STATS_PERCENTILE, 
 STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, STATS_SUM, 
-STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, TO_DOUBLE, TO_EPOCH_TIMESTAMP, 
+STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, TO_DOUBLE, TO_EPOCH_TIMESTAMP, TO_FLOAT, 
 TO_INTEGER, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, 
 URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR
 [Stellar]>>> 

metron-platform/metron-common/src/main/antlr4/org/apache/metron/common/stellar/generated/Stellar.g4

@@ -88,8 +88,22 @@ NIN : 'not in'
    ;
 EXISTS : 'exists' | 'EXISTS';
 EXPONENT : ('e' | 'E') ( PLUS|MINUS )? ('0'..'9')+;
-INT_LITERAL     : MINUS? '0'..'9'+ ;
-DOUBLE_LITERAL  : MINUS? '0'..'9'+'.''0'..'9'+EXPONENT? ;
+INT_LITERAL     :
+  MINUS? '0'
+  | MINUS? '1'..'9''0'..'9'*
+  ;
+DOUBLE_LITERAL  :
+  INT_LITERAL '.' '0'..'9'* EXPONENT? ('d'|'D')?
+  | '.' '0'..'9'+ EXPONENT? ('d'|'D')?
+  | INT_LITERAL EXPONENT ('d'|'D')?
+  | INT_LITERAL EXPONENT? ('d'|'D')
+  ;
+FLOAT_LITERAL  :
+  INT_LITERAL'.''0'..'9'* EXPONENT? ('f'|'F')
+  | MINUS? '.''0'..'9'+ EXPONENT? ('f'|'F')
+  | INT_LITERAL EXPONENT? ('f'|'F')
+  ;
+LONG_LITERAL  : INT_LITERAL ('l'|'L') ;
 IDENTIFIER : [a-zA-Z_][a-zA-Z_\.:0-9]* ;
 fragment SCHAR:  ~['"\\\r\n];
 STRING_LITERAL : '"' SCHAR* '"'
@@ -168,6 +182,8 @@ functions : IDENTIFIER func_args #TransformationFunc
 arithmetic_operands : functions #NumericFunctions
                     | DOUBLE_LITERAL #DoubleLiteral
                     | INT_LITERAL #IntLiteral
+                    | LONG_LITERAL #LongLiteral
+                    | FLOAT_LITERAL #FloatLiteral
                     | IDENTIFIER #Variable
                     | LPAREN arithmetic_expr RPAREN #ParenArith
                     | LPAREN conditional_expr RPAREN#condExpr

metron-platform/metron-common/src/main/java/Stellar.tokens

@@ -32,10 +32,12 @@ EXISTS=31
 EXPONENT=32
 INT_LITERAL=33
 DOUBLE_LITERAL=34
-IDENTIFIER=35
-STRING_LITERAL=36
-COMMENT=37
-WS=38
+FLOAT_LITERAL=35
+LONG_LITERAL=36
+IDENTIFIER=37
+STRING_LITERAL=38
+COMMENT=39
+WS=40
 ','=1
 '=='=7
 '!='=8

metron-platform/metron-common/src/main/java/StellarLexer.tokens

@@ -32,10 +32,12 @@ EXISTS=31
 EXPONENT=32
 INT_LITERAL=33
 DOUBLE_LITERAL=34
-IDENTIFIER=35
-STRING_LITERAL=36
-COMMENT=37
-WS=38
+FLOAT_LITERAL=35
+LONG_LITERAL=36
+IDENTIFIER=37
+STRING_LITERAL=38
+COMMENT=39
+WS=40
 ','=1
 '=='=7
 '!='=8

metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/BaseStellarProcessor.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -24,9 +24,12 @@
 
 import java.util.HashSet;
 import java.util.Set;
+import java.util.Stack;
+
 import org.apache.metron.common.dsl.*;
 import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
 import org.apache.metron.common.stellar.evaluators.ArithmeticEvaluator;
+import org.apache.metron.common.stellar.evaluators.NumberLiteralEvaluator;
 import org.apache.metron.common.stellar.generated.StellarBaseListener;
 import org.apache.metron.common.stellar.generated.StellarLexer;
 import org.apache.metron.common.stellar.generated.StellarParser;
@@ -83,7 +86,10 @@ public T parse( String rule
     TokenStream tokens = new CommonTokenStream(lexer);
     StellarParser parser = new StellarParser(tokens);
 
-    StellarCompiler treeBuilder = new StellarCompiler(variableResolver, functionResolver, context, new ArithmeticEvaluator());
+    StellarCompiler treeBuilder = new StellarCompiler(variableResolver, functionResolver, context, new Stack<>(),
+        ArithmeticEvaluator.INSTANCE,
+        NumberLiteralEvaluator.INSTANCE
+    );
     parser.addParseListener(treeBuilder);
     parser.removeErrorListeners();
     parser.addErrorListener(new ErrorListener());

metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -29,6 +29,7 @@
 import org.apache.metron.common.dsl.Token;
 import org.apache.metron.common.dsl.VariableResolver;
 import org.apache.metron.common.stellar.evaluators.ArithmeticEvaluator;
+import org.apache.metron.common.stellar.evaluators.NumberLiteralEvaluator;
 import org.apache.metron.common.stellar.generated.StellarBaseListener;
 import org.apache.metron.common.stellar.generated.StellarParser;
 import org.apache.metron.common.utils.ConversionUtils;
@@ -46,18 +47,26 @@
 
 public class StellarCompiler extends StellarBaseListener {
 
-  private Context context = null;
-  private Stack<Token> tokenStack = new Stack<>();
-  private FunctionResolver functionResolver;
-  private VariableResolver variableResolver;
-  private Throwable actualException = null;
+  private final Context context;
+  private final Stack<Token<?>> tokenStack;
+  private final FunctionResolver functionResolver;
+  private final VariableResolver variableResolver;
+  private Throwable actualException;
   private final ArithmeticEvaluator arithmeticEvaluator;
-
-  public StellarCompiler(VariableResolver variableResolver, FunctionResolver functionResolver, Context context, ArithmeticEvaluator arithmeticEvaluator) {
+  private final NumberLiteralEvaluator numberLiteralEvaluator;
+
+  public StellarCompiler(VariableResolver variableResolver,
+                         FunctionResolver functionResolver,
+                         Context context,
+                         Stack<Token<?>> tokenStack,
+                         ArithmeticEvaluator arithmeticEvaluator,
+                         NumberLiteralEvaluator numberLiteralEvaluator) {
     this.variableResolver = variableResolver;
     this.functionResolver = functionResolver;
     this.context = context;
+    this.tokenStack = tokenStack;
     this.arithmeticEvaluator = arithmeticEvaluator;
+    this.numberLiteralEvaluator = numberLiteralEvaluator;
   }
 
   @Override
@@ -174,12 +183,22 @@ public void exitStringLiteral(StellarParser.StringLiteralContext ctx) {
 
   @Override
   public void exitIntLiteral(StellarParser.IntLiteralContext ctx) {
-    tokenStack.push(new Token<>(Integer.parseInt(ctx.getText()), Integer.class));
+    tokenStack.push(numberLiteralEvaluator.evaluate(ctx));
   }
 
   @Override
   public void exitDoubleLiteral(StellarParser.DoubleLiteralContext ctx) {
-    tokenStack.push(new Token<>(Double.parseDouble(ctx.getText()), Double.class));
+    tokenStack.push(numberLiteralEvaluator.evaluate(ctx));
+  }
+
+  @Override
+  public void exitFloatLiteral(StellarParser.FloatLiteralContext ctx) {
+    tokenStack.push(numberLiteralEvaluator.evaluate(ctx));
+  }
+
+  @Override
+  public void exitLongLiteral(StellarParser.LongLiteralContext ctx) {
+    tokenStack.push(numberLiteralEvaluator.evaluate(ctx));
   }
 
   @Override

metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/evaluators/ArithmeticEvaluator.java

@@ -24,7 +24,8 @@
 
 import java.util.function.BiFunction;
 
-public class ArithmeticEvaluator {
+public enum ArithmeticEvaluator {
+  INSTANCE;
 
   public Token<? extends Number> evaluate(BiFunction<Number, Number, Token<? extends Number>> function,
                                           Pair<Token<? extends Number>, Token<? extends Number>> p) {

metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/evaluators/DoubleLiteralEvaluator.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.stellar.evaluators;
+
+import org.apache.metron.common.dsl.Token;
+import org.apache.metron.common.stellar.generated.StellarParser;
+
+public class DoubleLiteralEvaluator implements NumberEvaluator<StellarParser.DoubleLiteralContext> {
+  @Override
+  public Token<Double> evaluate(StellarParser.DoubleLiteralContext context) {
+    if (context == null) {
+      throw new IllegalArgumentException("Cannot evaluate a context that is null.");
+    }
+
+    return new Token<>(Double.parseDouble(context.getText()), Double.class);
+  }
+}

metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/evaluators/FloatLiteralEvaluator.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.stellar.evaluators;
+
+import org.apache.metron.common.dsl.Token;
+import org.apache.metron.common.stellar.generated.StellarParser;
+
+public class FloatLiteralEvaluator implements NumberEvaluator<StellarParser.FloatLiteralContext> {
+  @Override
+  public Token<Float> evaluate(StellarParser.FloatLiteralContext context) {
+    if (context == null) {
+      throw new IllegalArgumentException("Cannot evaluate a context that is null.");
+    }
+
+    return new Token<>(Float.parseFloat(context.getText()), Float.class);
+  }
+}

metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/evaluators/IntLiteralEvaluator.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.stellar.evaluators;
+
+import org.apache.metron.common.dsl.Token;
+import org.apache.metron.common.stellar.generated.StellarParser;
+
+public class IntLiteralEvaluator implements NumberEvaluator<StellarParser.IntLiteralContext> {
+  @Override
+  public Token<Integer> evaluate(StellarParser.IntLiteralContext context) {
+    if (context == null) {
+      throw new IllegalArgumentException("Cannot evaluate a context that is null.");
+    }
+
+    return new Token<>(Integer.parseInt(context.getText()), Integer.class);
+  }
+}