Prototype of NotFilters

src/KernelMemory.ElasticSearch.FunctionalTests/ElasticSearchHelperTests.cs

@@ -7,7 +7,6 @@ public class ElasticSearchHelperTests : BasicElasticTestFixture
 {
     public ElasticSearchHelperTests(IConfiguration cfg, IServiceProvider serviceProvider) : base(cfg, serviceProvider)
     {
-
     }
 
     [Fact]

src/KernelMemory.ElasticSearch.FunctionalTests/ElasticSearchQueryHelperTests.cs

@@ -14,7 +14,7 @@ public ElasticSearchQueryHelperTests(ElasticSearchQueryHelperTestsFixture fixtur
     }
 
     [Fact]
-    public async void Empty_query_return_all()
+    public async Task Empty_query_return_all()
     {
         //need to apply the condition for query
         var queryHelper = _fixture.ElasticSearchHelper.GetQueryHelper();
@@ -24,7 +24,7 @@ public async void Empty_query_return_all()
     }
 
     [Fact]
-    public async void No_query_returns_all()
+    public async Task No_query_returns_all()
     {
         //need to apply the condition for query
         var queryHelper = _fixture.ElasticSearchHelper.GetQueryHelper();
@@ -34,7 +34,7 @@ public async void No_query_returns_all()
     }
 
     [Fact]
-    public async void Verify_basic_query_with_filter()
+    public async Task Verify_basic_query_with_filter()
     {
         var mf = new MemoryFilter();
         mf.ByTag("tag1", "red");
@@ -50,7 +50,7 @@ public async void Verify_basic_query_with_filter()
     [InlineData("RED")]
     [InlineData("Red")]
     [InlineData("rEd")]
-    public async void Verify_basic_query_with_filter_is_case_insensitive(string color)
+    public async Task Verify_basic_query_with_filter_is_case_insensitive(string color)
     {
         var mf = new MemoryFilter();
         mf.ByTag("tag1", color);
@@ -62,8 +62,37 @@ public async void Verify_basic_query_with_filter_is_case_insensitive(string colo
         Assert.Equal(1, results);
     }
 
+    [Theory]
+    [InlineData("RED")]
+    [InlineData("Red")]
+    [InlineData("rEd")]
+    public async Task Verify_not_basic_query_with_filter_is_case_insensitive(string color)
+    {
+        var mf = new ExtendedMemoryFilter();
+        mf.ByNotTag("tag1", color);
+
+        //need to apply the condition for query
+        var queryHelper = _fixture.ElasticSearchHelper.GetQueryHelper();
+        var translatedQuery = ElasticSearchMemoryFilterConverter.CreateQueryDescriptorFromMemoryFilter([mf]);
+        var results = await queryHelper.VerifyQueryAsync(_fixture.IndexName, translatedQuery);
+        Assert.Equal(3, results);
+    }
+
+    [Fact]
+    public async Task Verify_not_with_multiple_value()
+    {
+        var mf = new ExtendedMemoryFilter();
+        mf.ByNotTag("users", "bar");
+
+        //need to apply the condition for query
+        var queryHelper = _fixture.ElasticSearchHelper.GetQueryHelper();
+        var translatedQuery = ElasticSearchMemoryFilterConverter.CreateQueryDescriptorFromMemoryFilter([mf]);
+        var results = await queryHelper.VerifyQueryAsync(_fixture.IndexName, translatedQuery);
+        Assert.Equal(2, results);
+    }
+
     [Fact]
-    public async void Verify_double_conditions()
+    public async Task Verify_double_conditions()
     {
         //this is an and condition, so we need to have both conditions to be true.
         var mf = new MemoryFilter();
@@ -78,7 +107,7 @@ public async void Verify_double_conditions()
     }
 
     [Fact]
-    public async void Verify_or_conditions()
+    public async Task Verify_or_conditions()
     {
         //this is an and condition, so we need to have both conditions to be true.
         var mf = new MemoryFilter();
@@ -121,16 +150,17 @@ public async Task InitializeAsync()
         IndexName = GetIndexTestName();
         await ElasticSearchHelper.EnsureIndexAsync(IndexName, 3, CancellationToken.None);
         //now we can index some data
-        await ElasticSearchHelper.IndexMemoryRecordAsync(IndexName, GenerateAMemoryRecord("red", "nice", [1.0f, 2.0f, 3.0f]), CancellationToken.None);
-        await ElasticSearchHelper.IndexMemoryRecordAsync(IndexName, GenerateAMemoryRecord("blue", "bad", [1.0f, 0.4f, 4.0f]), CancellationToken.None);
-        await ElasticSearchHelper.IndexMemoryRecordAsync(IndexName, GenerateAMemoryRecord("black", "night", [1.0f, 0.4f, 4.0f]), CancellationToken.None);
-        await ElasticSearchHelper.IndexMemoryRecordAsync(IndexName, GenerateAMemoryRecord("black", "day", [1.0f, 0.4f, 4.0f]), CancellationToken.None);
+        await ElasticSearchHelper.IndexMemoryRecordAsync(IndexName, GenerateAMemoryRecord("red", "nice", [1.0f, 2.0f, 3.0f], ["foo"]), CancellationToken.None);
+        await ElasticSearchHelper.IndexMemoryRecordAsync(IndexName, GenerateAMemoryRecord("blue", "bad", [1.0f, 0.4f, 4.0f], ["bar"]), CancellationToken.None);
+        await ElasticSearchHelper.IndexMemoryRecordAsync(IndexName, GenerateAMemoryRecord("black", "night", [1.0f, 0.4f, 4.0f], []), CancellationToken.None);
+        await ElasticSearchHelper.IndexMemoryRecordAsync(IndexName, GenerateAMemoryRecord("black", "day", [1.0f, 0.4f, 4.0f], ["foo", "bar"]), CancellationToken.None);
     }
 
     protected MemoryRecord GenerateAMemoryRecord(
         string tag1,
         string tag2,
-        float[] vector)
+        float[] vector,
+        List<string?> users)
     {
         TotalRecords++;
         return new MemoryRecord()
@@ -144,8 +174,9 @@ protected MemoryRecord GenerateAMemoryRecord(
                 },
             Tags = new Microsoft.KernelMemory.TagCollection()
             {
-                ["tag1"] = new List<string?>() { tag1 },
-                ["tag2"] = new List<string?>() { tag2 },
+                ["tag1"] = [tag1],
+                ["tag2"] = [tag2],
+                ["users"] = users,
             }
         };
     }

src/KernelMemory.ElasticSearch/ElasticSearchMemoryFilterConverter.cs

@@ -12,25 +12,23 @@ internal static class ElasticSearchMemoryFilterConverter
     internal static QueryDescriptor<object> CreateQueryDescriptorFromMemoryFilter(
         IEnumerable<MemoryFilter>? filters)
     {
-        //need to get all filters that have conditions
-        var realFilters = filters?
-            .Where(filters => filters.GetFilters().Any(f => !string.IsNullOrEmpty(f.Value)))?
-            .ToList() ?? [];
+        List<MemoryFilter> allFilters = GetAllRealFilters(filters);
 
-        if (realFilters.Count == 0)
+        //check if we have no filters.
+        if (allFilters.Count == 0)
         {
             return new QueryDescriptor<object>().MatchAll(new MatchAllQuery());
         }
 
         //ok I really have some conditions, we need to build the querydescriptor.
-        if (realFilters.Count == 1)
+        if (allFilters.Count == 1)
         {
-            return ConvertFilterToQueryDescriptor(realFilters[0]);
+            return ConvertFilterToQueryDescriptor(allFilters[0]);
         }
 
         //ok we have really more than one filter, convert all filter to Query object than finally return 
         //a composition with OR
-        var convertedFilters = realFilters
+        var convertedFilters = allFilters
             .Select(ConvertFilterToQuery)
             .ToArray();
 
@@ -43,25 +41,22 @@ internal static QueryDescriptor<object> CreateQueryDescriptorFromMemoryFilter(
     internal static Query CreateQueryFromMemoryFilter(
         IEnumerable<MemoryFilter>? filters)
     {
-        //need to get all filters that have conditions
-        var realFilters = filters?
-            .Where(filters => filters.GetFilters().Any(f => !string.IsNullOrEmpty(f.Value)))?
-            .ToList() ?? [];
+        List<MemoryFilter> allFilters = GetAllRealFilters(filters);
 
-        if (realFilters.Count == 0)
+        if (allFilters.Count == 0)
         {
             return Query.MatchAll(new MatchAllQuery());
         }
 
         //ok I really have some conditions, we need to build the querydescriptor.
-        if (realFilters.Count == 1)
+        if (allFilters.Count == 1)
         {
-            return ConvertFilterToQuery(realFilters[0]);
+            return ConvertFilterToQuery(allFilters[0]);
         }
 
         //ok we have really more than one filter, convert all filter to Query object than finally return 
         //a composition with OR
-        var convertedFilters = realFilters
+        var convertedFilters = allFilters
             .Select(ConvertFilterToQuery)
             .ToArray();
 
@@ -71,9 +66,31 @@ internal static Query CreateQueryFromMemoryFilter(
         });
     }
 
+    private static List<MemoryFilter> GetAllRealFilters(IEnumerable<MemoryFilter>? filters)
+    {
+        //need to get all filters that have conditions
+        var equalFilters = filters?
+            .Where(filters => filters.GetFilters().Any(f => !string.IsNullOrEmpty(f.Value)))?
+            .ToList() ?? [];
+
+        var notFilters = filters?
+            .OfType<ExtendedMemoryFilter>()?
+            .Where(filters => filters.GetNotFilters().Any(f => !string.IsNullOrEmpty(f.Value)))?
+            .ToList() ?? [];
+
+        var allFilters = equalFilters.Union(notFilters).ToList();
+        return allFilters;
+    }
+
+    private record BaseFilter(string Key, string Value);
+
+    private record EqualFilter(string Key, string Value) : BaseFilter(Key, Value);
+
+    private record NotEqualFilter(string Key, string Value) : BaseFilter(Key, Value);
+
     private static QueryDescriptor<object> ConvertFilterToQueryDescriptor(MemoryFilter filter)
     {
-        var innerFilters = filter.GetFilters().Where(f => !string.IsNullOrEmpty(f.Value)).ToArray();
+        BaseFilter[] innerFilters = ConvertToBaseFilterArray(filter);
 
         //lets double check if this filter really has conditions.
         if (innerFilters.Length == 0)
@@ -83,8 +100,30 @@ private static QueryDescriptor<object> ConvertFilterToQueryDescriptor(MemoryFilt
 
         if (innerFilters.Length == 1)
         {
-            var f = innerFilters[0];
-            return new QueryDescriptor<object>().Match(TagMatchQuery(f));
+            //we have a single filter, we can do a simple match query or boolean if we have a not equal filter.
+            var baseFilter = innerFilters[0];
+            if (baseFilter is EqualFilter f)
+            {
+                var mq = new MatchQuery($"tag_{f.Key}")
+                {
+                    Query = f.Value!
+                };
+                return new QueryDescriptor<object>().Match(mq);
+            }
+            else if (baseFilter is NotEqualFilter nf)
+            {
+                var boolQuery = new BoolQuery
+                {
+                    MustNot = new Query[]
+                    {
+                     new MatchQuery($"tag_{nf.Key}")
+                     {
+                         Query = nf.Value!
+                     }
+                     }
+                };
+                return new QueryDescriptor<object>().Bool(boolQuery);
+            }
         }
 
         //ok we have more than one condition, we need to build a bool query.
@@ -93,7 +132,7 @@ private static QueryDescriptor<object> ConvertFilterToQueryDescriptor(MemoryFilt
         {
             if (!String.IsNullOrEmpty(f.Value))
             {
-                convertedFilters.Add(Query.Match(TagMatchQuery(f)));
+                convertedFilters.Add(ConvertToElasticQuery(f));
             }
         }
 
@@ -102,7 +141,7 @@ private static QueryDescriptor<object> ConvertFilterToQueryDescriptor(MemoryFilt
 
     private static Query ConvertFilterToQuery(MemoryFilter filter)
     {
-        var innerFilters = filter.GetFilters().Where(f => !string.IsNullOrEmpty(f.Value)).ToArray();
+        var innerFilters = ConvertToBaseFilterArray(filter);
 
         //lets double check if this filter really has conditions.
         if (innerFilters.Length == 0)
@@ -113,7 +152,7 @@ private static Query ConvertFilterToQuery(MemoryFilter filter)
         if (innerFilters.Length == 1)
         {
             var f = innerFilters[0];
-            return Query.Match(TagMatchQuery(f));
+            return ConvertToElasticQuery(f);
         }
 
         //ok we have more than one condition, we need to build a bool query.
@@ -122,7 +161,7 @@ private static Query ConvertFilterToQuery(MemoryFilter filter)
         {
             if (!String.IsNullOrEmpty(f.Value))
             {
-                convertedFilters.Add(Query.Match(TagMatchQuery(f)));
+                convertedFilters.Add(ConvertToElasticQuery(f));
             }
         }
 
@@ -132,11 +171,57 @@ private static Query ConvertFilterToQuery(MemoryFilter filter)
         });
     }
 
-    private static MatchQuery TagMatchQuery(KeyValuePair<string, string?> f)
+    private static BaseFilter[] ConvertToBaseFilterArray(MemoryFilter filter)
+    {
+        var innerFiltersList = filter
+            .GetFilters()
+            .Where(f => !string.IsNullOrEmpty(f.Value))
+            .Select(f => (BaseFilter)new EqualFilter(f.Key, f.Value!));
+
+        var enhancedFilter = filter as ExtendedMemoryFilter;
+        if (enhancedFilter != null)
+        {
+            innerFiltersList = innerFiltersList.Union(enhancedFilter
+                .GetNotFilters()
+                .Where(f => !string.IsNullOrEmpty(f.Value))
+                .Select(f => (BaseFilter)new NotEqualFilter(f.Key, f.Value!)));
+        }
+
+        var innerFilters = innerFiltersList.ToArray();
+        return innerFilters;
+    }
+
+    //private static MatchQuery TagMatchQuery(BaseFilter baseFilter)
+    //{
+    //    return new MatchQuery($"tag_{baseFilter.Key}")
+    //    {
+    //        Query = baseFilter.Value!
+    //    };
+    //}
+
+    private static Query ConvertToElasticQuery(BaseFilter baseFilter)
     {
-        return new MatchQuery($"tag_{f.Key}")
+        if (baseFilter is EqualFilter f)
         {
-            Query = f.Value!
-        };
+            return new MatchQuery($"tag_{f.Key}")
+            {
+                Query = f.Value!
+            };
+        }
+        else if (baseFilter is NotEqualFilter nf)
+        {
+            return new BoolQuery
+            {
+                MustNot = new Query[]
+                {
+                     new MatchQuery($"tag_{nf.Key}")
+                     {
+                         Query = nf.Value!
+                     }
+                 }
+            };
+        }
+
+        throw new NotSupportedException($"Filter of type {baseFilter.GetType().Name} not supported");
     }
 }

src/KernelMemory.ElasticSearch/ExtendedMemoryFilter.cs

@@ -0,0 +1,28 @@
+using Microsoft.KernelMemory;
+using System.Collections.Generic;
+
+namespace KernelMemory.ElasticSearch;
+
+public class ExtendedMemoryFilter : MemoryFilter
+{
+    /// <summary>
+    /// This collection of tags contains all the tags that are used to
+    /// negatively filter out memory records.
+    /// </summary>
+    private TagCollection _notTags = new();
+
+    public MemoryFilter ByNotTag(string name, string value)
+    {
+        this._notTags.Add(name, value);
+        return this;
+    }
+
+    /// <summary>
+    /// Gets all the filters that needs to be put as not into  the query
+    /// </summary>
+    /// <returns></returns>
+    public IEnumerable<KeyValuePair<string, string?>> GetNotFilters()
+    {
+        return this._notTags.ToKeyValueList();
+    }
+}