Modify the query conversion logic for multi valued fields

entity-service-factory/build.gradle.kts

@@ -8,5 +8,5 @@ dependencies {
   implementation(project(":entity-service-attribute-translator"))
   implementation(project(":entity-service-impl"))
   implementation(project(":entity-service-change-event-generator"))
-  implementation("org.hypertrace.core.documentstore:document-store:0.7.23")
+  implementation("org.hypertrace.core.documentstore:document-store:0.7.27")
 }

entity-service-impl/build.gradle.kts

@@ -12,7 +12,7 @@ dependencies {
   annotationProcessor("org.projectlombok:lombok:1.18.22")
   compileOnly("org.projectlombok:lombok:1.18.18")
 
-  implementation("org.hypertrace.core.documentstore:document-store:0.7.23")
+  implementation("org.hypertrace.core.documentstore:document-store:0.7.27")
   implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.11.2")
   implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.11.2")
   implementation("org.hypertrace.core.attribute.service:caching-attribute-service-client:0.14.15")

entity-service-impl/src/main/java/org/hypertrace/entity/query/service/converter/ValueHelper.java

@@ -159,12 +159,7 @@ public SubDocumentValue convertToSubDocumentValue(final Value value) throws Conv
     final ValueType type = value.getValueType();
 
     if (isArray(type)) {
-      final List<Value> values = ARRAY_TO_PRIMITIVE_CONVERTER_MAP.get().get(type).apply(value);
-      final List<Document> documents = new ArrayList<>();
-
-      for (final Value singleValue : values) {
-        documents.add(convertToDocument(singleValue));
-      }
+      final List<Document> documents = getDocumentListFromArrayValue(value);
 
       return SubDocumentValue.of(documents);
     }
@@ -176,6 +171,18 @@ public SubDocumentValue convertToSubDocumentValue(final Value value) throws Conv
     throw new ConversionException(String.format("Unsupported value type: %s", type));
   }
 
+  public List<Document> getDocumentListFromArrayValue(final Value value)
+      throws ConversionException {
+    final List<Value> values =
+        ARRAY_TO_PRIMITIVE_CONVERTER_MAP.get().get(value.getValueType()).apply(value);
+    final List<Document> documents = new ArrayList<>();
+
+    for (final Value singleValue : values) {
+      documents.add(convertToDocument(singleValue));
+    }
+    return documents;
+  }
+
   public ConstantExpression convertToConstantExpression(final Value value, final int index)
       throws ConversionException {
     switch (value.getValueType()) {
@@ -234,7 +241,7 @@ public ValueType getPrimitiveValueType(final String primitiveType) throws Conver
     return arrayType;
   }
 
-  private JSONDocument convertToDocument(final Value value) throws ConversionException {
+  public JSONDocument convertToDocument(final Value value) throws ConversionException {
     try {
       final String dataType = getStringValue(value.getValueType());
       final Object valueObj = extractValueFromPrimitiveValue(value);

entity-service-impl/src/main/java/org/hypertrace/entity/query/service/converter/filter/ContainsFilteringExpressionConverter.java

@@ -0,0 +1,110 @@
+package org.hypertrace.entity.query.service.converter.filter;
+
+import static java.util.Map.entry;
+import static org.hypertrace.core.documentstore.expression.operators.LogicalOperator.AND;
+import static org.hypertrace.core.documentstore.expression.operators.LogicalOperator.OR;
+import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.CONTAINS;
+import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.NOT_CONTAINS;
+import static org.hypertrace.entity.query.service.converter.identifier.IdentifierConverter.getSubDocPathById;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import lombok.AllArgsConstructor;
+import lombok.experimental.Accessors;
+import org.hypertrace.core.documentstore.Document;
+import org.hypertrace.core.documentstore.expression.impl.ConstantExpression;
+import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression;
+import org.hypertrace.core.documentstore.expression.impl.LogicalExpression;
+import org.hypertrace.core.documentstore.expression.impl.RelationalExpression;
+import org.hypertrace.core.documentstore.expression.operators.LogicalOperator;
+import org.hypertrace.core.documentstore.expression.operators.RelationalOperator;
+import org.hypertrace.core.documentstore.expression.type.FilterTypeExpression;
+import org.hypertrace.core.grpcutils.context.RequestContext;
+import org.hypertrace.entity.attribute.translator.EntityAttributeMapping;
+import org.hypertrace.entity.query.service.converter.ConversionException;
+import org.hypertrace.entity.query.service.converter.ValueHelper;
+import org.hypertrace.entity.query.service.converter.identifier.ArrayPathSuffixAddingIdentifierConverter;
+import org.hypertrace.entity.query.service.converter.identifier.IdentifierConversionMetadata;
+import org.hypertrace.entity.query.service.v1.ColumnIdentifier;
+import org.hypertrace.entity.query.service.v1.LiteralConstant;
+import org.hypertrace.entity.query.service.v1.Operator;
+import org.hypertrace.entity.query.service.v1.Value;
+import org.hypertrace.entity.query.service.v1.ValueType;
+
+@Singleton
+@AllArgsConstructor(onConstructor_ = {@Inject})
+public class ContainsFilteringExpressionConverter extends FilteringExpressionConverterBase {
+  private static final Map<Operator, OperatorPair> OPERATOR_MAP =
+      Map.ofEntries(
+          entry(Operator.IN, OperatorPair.of(CONTAINS, OR)),
+          entry(Operator.NOT_IN, OperatorPair.of(NOT_CONTAINS, AND)));
+
+  private final EntityAttributeMapping entityAttributeMapping;
+  private final ArrayPathSuffixAddingIdentifierConverter arrayPathSuffixAddingIdentifierConverter;
+  private final ValueHelper valueHelper;
+
+  @Override
+  public FilterTypeExpression convert(
+      final ColumnIdentifier columnIdentifier,
+      final Operator operator,
+      final LiteralConstant constant,
+      final RequestContext requestContext)
+      throws ConversionException {
+    final String id = columnIdentifier.getColumnName();
+    final String subDocPath = getSubDocPathById(entityAttributeMapping, id, requestContext);
+    final Value value = constant.getValue();
+    final ValueType valueType = value.getValueType();
+
+    final List<Document> list = valueHelper.getDocumentListFromArrayValue(value);
+
+    if (list.isEmpty()) {
+      throw new ConversionException("Conversion of empty-list is unsupported");
+    }
+
+    final OperatorPair operatorPair = OPERATOR_MAP.get(operator);
+    if (operatorPair == null) {
+      throw new ConversionException(
+          String.format("Operator %s is not supported on array column %s", operator, id));
+    }
+
+    final IdentifierConversionMetadata metadata =
+        IdentifierConversionMetadata.builder()
+            .subDocPath(subDocPath)
+            .operator(operator)
+            .valueType(valueType)
+            .build();
+    final String suffixedSubDocPath =
+        arrayPathSuffixAddingIdentifierConverter.convert(metadata, requestContext);
+    final IdentifierExpression lhs = IdentifierExpression.of(suffixedSubDocPath);
+
+    final List<RelationalExpression> expressions = new ArrayList<>();
+
+    for (final Document document : list) {
+      final ConstantExpression rhs = ConstantExpression.of(document);
+      final RelationalExpression expression =
+          RelationalExpression.of(lhs, operatorPair.relationalOperator(), rhs);
+
+      expressions.add(expression);
+    }
+
+    if (expressions.size() == 1) {
+      return expressions.get(0);
+    }
+
+    return LogicalExpression.builder()
+        .operator(operatorPair.logicalOperator())
+        .operands(expressions)
+        .build();
+  }
+
+  @lombok.Value
+  @Accessors(fluent = true)
+  @AllArgsConstructor(staticName = "of")
+  private static class OperatorPair {
+    RelationalOperator relationalOperator;
+    LogicalOperator logicalOperator;
+  }
+}

entity-service-impl/src/main/java/org/hypertrace/entity/query/service/converter/filter/FilteringExpressionConverterFactory.java

@@ -1,10 +1,16 @@
 package org.hypertrace.entity.query.service.converter.filter;
 
+import org.hypertrace.core.grpcutils.context.RequestContext;
 import org.hypertrace.entity.query.service.converter.ConversionException;
+import org.hypertrace.entity.query.service.v1.ColumnIdentifier;
 import org.hypertrace.entity.query.service.v1.Operator;
 import org.hypertrace.entity.query.service.v1.Value;
 
 public interface FilteringExpressionConverterFactory {
-  FilteringExpressionConverter getConverter(final Value value, final Operator operator)
+  FilteringExpressionConverter getConverter(
+      final ColumnIdentifier identifier,
+      final Operator operator,
+      final Value value,
+      final RequestContext context)
       throws ConversionException;
 }

entity-service-impl/src/main/java/org/hypertrace/entity/query/service/converter/filter/FilteringExpressionConverterFactoryImpl.java

@@ -4,8 +4,11 @@
 import com.google.inject.Singleton;
 import java.util.Set;
 import lombok.AllArgsConstructor;
+import org.hypertrace.core.grpcutils.context.RequestContext;
+import org.hypertrace.entity.attribute.translator.EntityAttributeMapping;
 import org.hypertrace.entity.query.service.converter.ConversionException;
 import org.hypertrace.entity.query.service.converter.ValueHelper;
+import org.hypertrace.entity.query.service.v1.ColumnIdentifier;
 import org.hypertrace.entity.query.service.v1.Operator;
 import org.hypertrace.entity.query.service.v1.Value;
 import org.hypertrace.entity.query.service.v1.ValueType;
@@ -18,11 +21,17 @@ public class FilteringExpressionConverterFactoryImpl
   private NullFilteringExpressionConverter nullFilteringExpressionConverter;
   private PrimitiveFilteringExpressionConverter primitiveFilteringExpressionConverter;
   private ArrayFilteringExpressionConverter arrayFilteringExpressionConverter;
+  private ContainsFilteringExpressionConverter containsFilteringExpressionConverter;
   private MapFilteringExpressionConverter mapFilteringExpressionConverter;
   private ValueHelper valueHelper;
+  private EntityAttributeMapping entityAttributeMapping;
 
   @Override
-  public FilteringExpressionConverter getConverter(final Value value, final Operator operator)
+  public FilteringExpressionConverter getConverter(
+      final ColumnIdentifier identifier,
+      final Operator operator,
+      final Value value,
+      final RequestContext context)
       throws ConversionException {
     ValueType valueType = value.getValueType();
 
@@ -32,6 +41,10 @@ public FilteringExpressionConverter getConverter(final Value value, final Operat
     }
 
     if (CONTAINMENT_OPERATORS.contains(operator)) {
+      if (entityAttributeMapping.isArray(context, identifier.getColumnName())) {
+        return containsFilteringExpressionConverter;
+      }
+
       return primitiveFilteringExpressionConverter;
     }
 

entity-service-impl/src/main/java/org/hypertrace/entity/query/service/converter/filter/RelationalExpressionConverter.java

@@ -38,7 +38,8 @@ public FilterTypeExpression convert(final Filter filter, final RequestContext re
         expressionAccessor.access(rhs, rhs.getValueCase(), Set.of(LITERAL));
 
     final FilteringExpressionConverter filteringExpressionConverter =
-        filteringExpressionConverterFactory.getConverter(literal.getValue(), operator);
+        filteringExpressionConverterFactory.getConverter(
+            identifier, operator, literal.getValue(), requestContext);
     return filteringExpressionConverter.convert(identifier, operator, literal, requestContext);
   }
 }

entity-service-impl/src/test/java/org/hypertrace/entity/query/service/converter/filter/RelationalExpressionConverterTest.java

@@ -67,7 +67,11 @@ void setup() throws ConversionException {
     doReturn(constantExpression)
         .when(constantExpressionConverter)
         .convert(literalConstant, requestContext);
-    when(filteringExpressionConverterFactory.getConverter(any(Value.class), any(Operator.class)))
+    when(filteringExpressionConverterFactory.getConverter(
+            any(ColumnIdentifier.class),
+            any(Operator.class),
+            any(Value.class),
+            any(RequestContext.class)))
         .thenReturn(filteringExpressionConverter);
   }
 

entity-service/build.gradle.kts

@@ -23,7 +23,7 @@ dependencies {
   implementation(project(":entity-service-factory"))
 
   implementation("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.49")
-  implementation("org.hypertrace.core.documentstore:document-store:0.7.23")
+  implementation("org.hypertrace.core.documentstore:document-store:0.7.27")
 
   runtimeOnly("io.grpc:grpc-netty:1.50.0")
 

entity-service/src/integrationTest/java/org/hypertrace/entity/service/service/EntityQueryServiceTest.java

@@ -664,6 +664,109 @@ void testExecuteWithCompositeAttributeFilters() {
     assertTrue(list.get(1).getResultSetMetadata().getColumnMetadataCount() > 0);
   }
 
+  @Test
+  void testExecuteWithNotInArrayFilter() {
+    // create and upsert some entities
+    List<String> filterValue1 = List.of(generateRandomUUID(), generateRandomUUID());
+    AttributeValueList.Builder labelsList1 = AttributeValueList.newBuilder();
+
+    filterValue1.stream().map(this::generateAttrValue).forEach(labelsList1::addValues);
+
+    Entity entity1 =
+        Entity.newBuilder()
+            .setTenantId(TENANT_ID)
+            .setEntityType(EntityType.SERVICE.name())
+            .setEntityName("Some Service 1")
+            .putAttributes("labels", AttributeValue.newBuilder().setValueList(labelsList1).build())
+            .putIdentifyingAttributes(
+                EntityConstants.getValue(CommonAttribute.COMMON_ATTRIBUTE_FQN),
+                generateRandomUUIDAttrValue())
+            .build();
+    Entity createdEntity1 = entityDataServiceClient.upsert(entity1);
+    assertNotNull(createdEntity1);
+    assertFalse(createdEntity1.getEntityId().trim().isEmpty());
+
+    Entity entity2 =
+        Entity.newBuilder()
+            .setTenantId(TENANT_ID)
+            .setEntityType(EntityType.SERVICE.name())
+            .setEntityName("Some Service 2")
+            .putAttributes(
+                apiAttributesMap.get(API_DISCOVERY_STATE_ATTR), createAttribute("DISCOVERED"))
+            .putIdentifyingAttributes(
+                EntityConstants.getValue(CommonAttribute.COMMON_ATTRIBUTE_FQN),
+                generateRandomUUIDAttrValue())
+            .build();
+    Entity createdEntity2 = entityDataServiceClient.upsert(entity2);
+    assertNotNull(createdEntity2);
+    assertFalse(createdEntity2.getEntityId().trim().isEmpty());
+
+    List<String> filterValue2 = List.of(generateRandomUUID(), generateRandomUUID());
+    AttributeValueList.Builder labelsList2 = AttributeValueList.newBuilder();
+
+    filterValue2.stream().map(this::generateAttrValue).forEach(labelsList2::addValues);
+    Entity entity3 =
+        Entity.newBuilder()
+            .setTenantId(TENANT_ID)
+            .setEntityType(EntityType.SERVICE.name())
+            .setEntityName("Some Service 3")
+            .putAttributes("labels", AttributeValue.newBuilder().setValueList(labelsList2).build())
+            .putIdentifyingAttributes(
+                EntityConstants.getValue(CommonAttribute.COMMON_ATTRIBUTE_FQN),
+                generateRandomUUIDAttrValue())
+            .build();
+    Entity createdEntity3 = entityDataServiceClient.upsert(entity3);
+    assertNotNull(createdEntity3);
+    assertFalse(createdEntity3.getEntityId().trim().isEmpty());
+
+    EntityQueryRequest queryRequest =
+        EntityQueryRequest.newBuilder()
+            .setEntityType(EntityType.SERVICE.name())
+            .setFilter(
+                Filter.newBuilder()
+                    .setOperator(Operator.NOT_IN)
+                    .setLhs(
+                        Expression.newBuilder()
+                            .setColumnIdentifier(
+                                ColumnIdentifier.newBuilder().setColumnName(API_LABELS_ATTR)))
+                    .setRhs(
+                        Expression.newBuilder()
+                            .setLiteral(
+                                LiteralConstant.newBuilder()
+                                    .setValue(
+                                        org.hypertrace.entity.query.service.v1.Value.newBuilder()
+                                            .setValueType(STRING_ARRAY)
+                                            .addAllStringArray(filterValue1))))
+                    .build())
+            .addSelection(
+                Expression.newBuilder()
+                    .setColumnIdentifier(
+                        ColumnIdentifier.newBuilder().setColumnName("API.id").build())
+                    .build())
+            .addSelection(
+                Expression.newBuilder()
+                    .setColumnIdentifier(
+                        ColumnIdentifier.newBuilder().setColumnName("API.name").build())
+                    .build())
+            .build();
+
+    Iterator<ResultSetChunk> resultSetChunkIterator =
+        GrpcClientRequestContextUtil.executeWithHeadersContext(
+            HEADERS, () -> entityQueryServiceClient.execute(queryRequest));
+    List<ResultSetChunk> list = Lists.newArrayList(resultSetChunkIterator);
+    assertEquals(1, list.size());
+    assertEquals(2, list.get(0).getRowCount());
+    assertEquals(0, list.get(0).getChunkId());
+    assertTrue(list.get(0).getIsLastChunk());
+
+    assertEquals(createdEntity2.getEntityId(), list.get(0).getRow(0).getColumn(0).getString());
+    assertEquals(createdEntity2.getEntityName(), list.get(0).getRow(0).getColumn(1).getString());
+    assertEquals(createdEntity3.getEntityId(), list.get(0).getRow(1).getColumn(0).getString());
+    assertEquals(createdEntity3.getEntityName(), list.get(0).getRow(1).getColumn(1).getString());
+
+    assertTrue(list.get(0).getResultSetMetadata().getColumnMetadataCount() > 0);
+  }
+
   @Test
   void testExecuteWithExistsFilter() {
     // create and upsert some entities