Add metadata edge filter (#92)

* Add metadata edge filter * some modifications to the filter * Fix versioning * style change * Addressing styling comments
twitter · Aug 3, 2017 · d4bb605 · d4bb605
1 parent 2fed841
commit d4bb605
Show file tree

Hide file tree

Showing 3 changed files with 150 additions and 2 deletions.
diff --git a/...n/java/com/twitter/graphjet/algorithms/counting/tweet/TopSecondDegreeByCountForTweet.java b/...n/java/com/twitter/graphjet/algorithms/counting/tweet/TopSecondDegreeByCountForTweet.java
@@ -105,7 +105,6 @@ protected boolean isEdgeUpdateValid(
     );
   }
 
-
   private int[][] collectNodeMetadata(EdgeIterator edgeIterator) {
     int metadataSize = RecommendationType.METADATASIZE.getValue();
     int[][] nodeMetadata = new int[metadataSize][];
@@ -138,7 +137,6 @@ protected void updateNodeInfo(
     NodeInfo nodeInfo;
     if (!super.visitedRightNodes.containsKey(rightNode)) {
       int[][] nodeMetadata = collectNodeMetadata(edgeIterator);
-
       nodeInfo = new NodeInfo(rightNode, nodeMetadata, 0.0, maxSocialProofTypeSize);
       super.visitedRightNodes.put(rightNode, nodeInfo);
     } else {

diff --git a/...-core/src/main/java/com/twitter/graphjet/algorithms/filters/RecentEdgeMetadataFilter.java b/...-core/src/main/java/com/twitter/graphjet/algorithms/filters/RecentEdgeMetadataFilter.java
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2017 Twitter. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.twitter.graphjet.algorithms.filters;
+
+import com.twitter.graphjet.algorithms.RecommendationRequest;
+import com.twitter.graphjet.hashing.SmallArrayBasedLongToDoubleMap;
+import com.twitter.graphjet.stats.StatsReceiver;
+
+/**
+ * This filter assumes the social proofs' edge metadata represent timestamps.
+ * Guarantees that the social proofs for a result node at at least older than a
+ * specified value, ex. at least 3 days old. If any social proof is younger, the node is filtered
+ */
+public class RecentEdgeMetadataFilter extends ResultFilter {
+  private final byte socialProofType;
+  private final long rejectWithInLastXMillis;
+  private long cutoff;
+
+  /**
+   * @param rejectWithInLastXMillis The minimum age of a social proof edge to not have it filtered
+   * @param socialProofType Only check social proof edges of this type
+   * @param statsReceiver
+   */
+  public RecentEdgeMetadataFilter(
+    long rejectWithInLastXMillis,
+    byte socialProofType,
+    StatsReceiver statsReceiver
+  ) {
+    super(statsReceiver);
+    this.rejectWithInLastXMillis = rejectWithInLastXMillis;
+    this.cutoff = System.currentTimeMillis() - rejectWithInLastXMillis;
+    this.socialProofType = socialProofType;
+  }
+
+  @Override
+  public void resetFilter(RecommendationRequest request) {
+    cutoff = System.currentTimeMillis() - rejectWithInLastXMillis;
+  }
+
+  @Override
+  public boolean filterResult(long resultNode, SmallArrayBasedLongToDoubleMap[] socialProofs) {
+    SmallArrayBasedLongToDoubleMap socialProof = socialProofs[socialProofType];
+    if (socialProof == null) {
+      return false;
+    }
+
+    long[] allMetadata = socialProof.metadata();
+    for (long metadataTimestamp: allMetadata) {
+      if (cutoff < metadataTimestamp) {
+        return true; // Too young, filter
+      }
+    }
+    return false;
+  }
+}
diff --git a/...re/src/test/java/com/twitter/graphjet/algorithms/filter/RecentEdgeMetadataFilterTest.java b/...re/src/test/java/com/twitter/graphjet/algorithms/filter/RecentEdgeMetadataFilterTest.java
@@ -0,0 +1,81 @@
+/**
+ * Copyright 2017 Twitter. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.twitter.graphjet.algorithms.filter;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+import com.twitter.graphjet.algorithms.RecommendationRequest;
+import com.twitter.graphjet.algorithms.filters.RecentEdgeMetadataFilter;
+import com.twitter.graphjet.hashing.SmallArrayBasedLongToDoubleMap;
+import com.twitter.graphjet.stats.NullStatsReceiver;
+
+public class RecentEdgeMetadataFilterTest {
+  private long benchmarkTimeInMillis = 1501276549000L; // July 28, 2017 9:15:49 PM GMT
+  private long oneDayInMillis = 1000 * 60 * 60 * 24;
+
+  @Test
+  public void testOneEdgeOldEnough() {
+    long twoDaysPriorBenchmark = benchmarkTimeInMillis - oneDayInMillis * 2;
+    SmallArrayBasedLongToDoubleMap socialProof = new SmallArrayBasedLongToDoubleMap();
+    socialProof.put(100L, 1.0, twoDaysPriorBenchmark);
+    SmallArrayBasedLongToDoubleMap[] socialProofs = {null, null, null, null, socialProof};
+
+    long elapsedTimeInMillis = System.currentTimeMillis() - benchmarkTimeInMillis;
+    RecentEdgeMetadataFilter filter = new RecentEdgeMetadataFilter(
+        elapsedTimeInMillis + oneDayInMillis, // one day before benchmark
+        (byte)RecommendationRequest.AUTHOR_SOCIAL_PROOF_TYPE,
+        new NullStatsReceiver());
+    assertEquals(false, filter.filterResult(1L, socialProofs));
+  }
+
+  @Test
+  public void testOneEdgeTooRecent() {
+
+    long halfDaysPriorBenchmark = benchmarkTimeInMillis - oneDayInMillis / 2;
+    SmallArrayBasedLongToDoubleMap socialProof = new SmallArrayBasedLongToDoubleMap();
+    socialProof.put(100L, 1.0, halfDaysPriorBenchmark);
+    SmallArrayBasedLongToDoubleMap[] socialProofs = {null, null, null, null, socialProof};
+
+    long elapsedTimeInMillis = System.currentTimeMillis() - benchmarkTimeInMillis;
+    RecentEdgeMetadataFilter filter = new RecentEdgeMetadataFilter(
+        elapsedTimeInMillis + oneDayInMillis, // one day before benchmark
+        (byte)RecommendationRequest.AUTHOR_SOCIAL_PROOF_TYPE,
+        new NullStatsReceiver());
+    assertEquals(true, filter.filterResult(1L, socialProofs));
+  }
+
+  @Test
+  public void testTwoEdgeTooRecent() {
+
+    long halfDaysPriorBenchmark = benchmarkTimeInMillis - oneDayInMillis / 2;
+    long twoDaysPriorBenchmark = benchmarkTimeInMillis - oneDayInMillis * 2;
+
+    SmallArrayBasedLongToDoubleMap socialProof = new SmallArrayBasedLongToDoubleMap();
+    socialProof.put(100L, 1.0, halfDaysPriorBenchmark);
+    socialProof.put(101L, 1.0, twoDaysPriorBenchmark);
+    SmallArrayBasedLongToDoubleMap[] socialProofs = {null, null, null, null, socialProof};
+
+    long elapsedTimeInMillis = System.currentTimeMillis() - benchmarkTimeInMillis;
+    RecentEdgeMetadataFilter filter = new RecentEdgeMetadataFilter(
+        elapsedTimeInMillis + oneDayInMillis, // one day before benchmark
+        (byte)RecommendationRequest.AUTHOR_SOCIAL_PROOF_TYPE,
+        new NullStatsReceiver());
+    assertEquals(true, filter.filterResult(1L, socialProofs));
+  }
+}