Supporting nested extracts in projections

pom.xml

@@ -36,7 +36,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
-        <bullet.record.version>0.2.2</bullet.record.version>
+        <bullet.record.version>0.3.0</bullet.record.version>
         <sketches.version>0.9.1</sketches.version>
     </properties>
 

src/main/java/com/yahoo/bullet/common/Utilities.java

@@ -14,8 +14,6 @@
 import java.util.Map;
 
 public class Utilities {
-    public static final String SUB_KEY_SEPERATOR = "\\.";
-
    /**
     * Tries to get the object casted as the target type. If it is generic, the captured types cannot not be
     * validated. Only the base object type is validated.
@@ -123,14 +121,4 @@ public static Number extractFieldAsNumber(String identifier, BulletRecord record
         }
         return (Number) asNumber.getValue();
     }
-
-    /**
-     * Takes a field and returns it split into subfields if necessary.
-     *
-     * @param field The non-null field to get.
-     * @return The field split into field or subfield if it was a map field, or just the field itself.
-     */
-    public static String[] splitField(String field) {
-        return field.split(SUB_KEY_SEPERATOR, 2);
-    }
 }

src/main/java/com/yahoo/bullet/querying/Querier.java

@@ -449,8 +449,8 @@ public void consume(BulletRecord record) {
             return;
         }
 
-        BulletRecord projected = project(record);
         try {
+            BulletRecord projected = project(record);
             window.consume(projected);
             hasNewData = true;
         } catch (RuntimeException e) {

src/main/java/com/yahoo/bullet/querying/operations/ProjectionOperations.java

@@ -13,8 +13,6 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static com.yahoo.bullet.common.Utilities.splitField;
-
 @Slf4j
 public class ProjectionOperations {
     /**
@@ -44,24 +42,10 @@ public static BulletRecord project(BulletRecord record, Projection projection, M
         for (Map.Entry<String, String> e : fields.entrySet()) {
             String field = e.getKey();
             String newName = e.getValue();
-            try {
-                copyInto(projected, newName, record, field);
-            } catch (ClassCastException cce) {
-                log.warn("Skipping copying {} as {} as it is not a field that can be extracted", field, newName);
+            if (field != null) {
+                projected.forceSet(newName, record.extractField(field));
             }
         }
         return projected;
     }
-
-    private static void copyInto(BulletRecord record, String newName, BulletRecord source, String field) throws ClassCastException {
-        if (field == null) {
-            return;
-        }
-        String[] split = splitField(field);
-        if (split.length > 1) {
-            record.set(newName, source, split[0], split[1]);
-        } else {
-            record.set(newName, source, field);
-        }
-    }
 }

src/test/java/com/yahoo/bullet/querying/operations/FilterOperationsTest.java

@@ -426,6 +426,21 @@ public void testComparisonNestedField() {
         Assert.assertTrue(FilterOperations.perform(box.getRecord(), clause));
     }
 
+    @Test
+    public void testComparisonEverMoreNestedField() {
+        FilterClause clause = getFieldFilter("map_of_maps.demog.age", GREATER_THAN, "30");
+
+        // "null" is not > 30
+        Assert.assertFalse(FilterOperations.perform(RecordBox.get().getRecord(), clause));
+        RecordBox box = RecordBox.get();
+        box.addMapOfMaps("map_of_maps", Pair.of("demog", singletonMap("age", 3)), Pair.of("foo", singletonMap("bar", 50)));
+        Assert.assertFalse(FilterOperations.perform(box.getRecord(), clause));
+        box.addMapOfMaps("map_of_maps", Pair.of("demog", singletonMap("age", 30)), Pair.of("foo", singletonMap("bar", 50)));
+        Assert.assertFalse(FilterOperations.perform(box.getRecord(), clause));
+        box.addMapOfMaps("map_of_maps", Pair.of("demog", singletonMap("age", 31)), Pair.of("foo", singletonMap("bar", 50)));
+        Assert.assertTrue(FilterOperations.perform(box.getRecord(), clause));
+    }
+
     @Test
     public void testComparisonBooleanMap() {
         FilterClause clause = getFieldFilter("filter_map.is_fake_event", EQUALS, "true");

src/test/java/com/yahoo/bullet/querying/operations/ProjectionOperationsTest.java

@@ -18,6 +18,9 @@
 import static com.yahoo.bullet.parsing.ProjectionUtils.makeProjection;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
 
 public class ProjectionOperationsTest {
     private static BulletRecordProvider provider = new BulletConfig().getBulletRecordProvider();
@@ -43,17 +46,27 @@ public void testProjection() {
     }
 
     @Test
-    public void testUnsupportedProjection() {
+    public void testNestedProjections() {
         Projection projection = makeProjection(ImmutablePair.of("list_field.1.foo", "bar"),
+                                               ImmutablePair.of("map_field.foo.bar", "baz"),
                                                ImmutablePair.of("field", "foo"));
-        BulletRecord record = RecordBox.get().addListOfMaps("list_field", emptyMap(), singletonMap("foo", "bar"))
+        BulletRecord record = RecordBox.get().addListOfMaps("list_field", emptyMap(), singletonMap("foo", "qux"))
+                                             .addMapOfMaps("map_field", ImmutablePair.of("foo", singletonMap("bar", "foo.bar")))
                                              .add("field", "123")
                                              .getRecord();
         BulletRecord actual = ProjectionOperations.project(record, projection, null, provider);
-        BulletRecord expected = RecordBox.get().add("foo", "123").getRecord();
+        BulletRecord expected = RecordBox.get().add("foo", "123").add("bar", "qux").add("baz", "foo.bar").getRecord();
         Assert.assertEquals(actual, expected);
     }
 
+    @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Testing.*")
+    public void testFailingProjection() {
+        Projection projection = makeProjection(ImmutablePair.of("foo", "bar"));
+        BulletRecord record = spy(RecordBox.get().add("foo", "123").getRecord());
+        doThrow(new RuntimeException("Testing")).when(record).extractField(anyString());
+        ProjectionOperations.project(record, projection, null, provider);
+    }
+
     @Test
     public void testMapList() {
         Projection projection = makeProjection("list_field", "foo");

src/test/java/com/yahoo/bullet/result/RecordBox.java

@@ -81,7 +81,7 @@ public final RecordBox addMap(String name, Pair<String, Object>... entries) {
     @SafeVarargs
     public final RecordBox addMapOfMaps(String name, Pair<String, Map<String, Object>>... entries) {
         if (entries != null && entries.length != 0) {
-            Map<String, Object>[] sampleEntries = (Map<String, Object>[]) Arrays.stream(entries).map(Pair::getRight).toArray();
+            Map<String, Object>[] sampleEntries = (Map<String, Object>[]) Arrays.stream(entries).map(Pair::getRight).toArray(Map[]::new);
             Object value = findObject(sampleEntries);
             if (value instanceof Boolean) {
                 record.setMapOfBooleanMap(name, asMapOfMaps(Boolean.class, entries));