Fixing distance functions for Pinecone, Weaviate and Redis and adding…

… integration tests for score testing for these.

dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs

@@ -39,7 +39,7 @@ public static Metric GetSDKMetricAlgorithm(VectorStoreRecordVectorProperty vecto
         {
             DistanceFunction.CosineSimilarity => Metric.Cosine,
             DistanceFunction.DotProductSimilarity => Metric.DotProduct,
-            DistanceFunction.EuclideanDistance => Metric.Euclidean,
+            DistanceFunction.EuclideanSquaredDistance => Metric.Euclidean,
             null => Metric.Cosine,
             _ => throw new InvalidOperationException($"Distance function '{vectorProperty.DistanceFunction}' for {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' is not supported by the Pinecone VectorStore.")
         };

dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStoreCollectionCreateMapping.cs

@@ -179,7 +179,7 @@ public static string GetSDKDistanceAlgorithm(VectorStoreRecordVectorProperty vec
             DistanceFunction.CosineSimilarity => "COSINE",
             DistanceFunction.CosineDistance => "COSINE",
             DistanceFunction.DotProductSimilarity => "IP",
-            DistanceFunction.EuclideanDistance => "L2",
+            DistanceFunction.EuclideanSquaredDistance => "L2",
             _ => throw new InvalidOperationException($"Distance function '{vectorProperty.DistanceFunction}' for {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' is not supported by the Redis VectorStore.")
         };
     }

dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStoreCollectionSearchMapping.cs

@@ -159,7 +159,7 @@ public static string ResolveDistanceFunction(VectorSearchOptions options, IReadO
             DistanceFunction.CosineSimilarity => 1 - redisScore,
             DistanceFunction.CosineDistance => redisScore,
             DistanceFunction.DotProductSimilarity => redisScore,
-            DistanceFunction.EuclideanDistance => redisScore,
+            DistanceFunction.EuclideanSquaredDistance => redisScore,
             _ => throw new InvalidOperationException($"The distance function '{distanceFunction}' is not supported."),
         };
     }

dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs

@@ -112,15 +112,15 @@ private static string MapDistanceFunction(string? distanceFunction, string vecto
         return distanceFunction switch
         {
             DistanceFunction.CosineDistance => Cosine,
-            DistanceFunction.DotProductSimilarity => Dot,
+            DistanceFunction.NegativeDotProduct => Dot,
             DistanceFunction.EuclideanSquaredDistance => EuclideanSquared,
             DistanceFunction.Hamming => Hamming,
             DistanceFunction.ManhattanDistance => Manhattan,
             _ => throw new InvalidOperationException(
                 $"Distance function '{distanceFunction}' on {nameof(VectorStoreRecordVectorProperty)} '{vectorPropertyName}' is not supported by the Weaviate VectorStore. " +
                 $"Supported distance functions: {string.Join(", ",
                     DistanceFunction.CosineDistance,
-                    DistanceFunction.DotProductSimilarity,
+                    DistanceFunction.NegativeDotProduct,
                     DistanceFunction.EuclideanSquaredDistance,
                     DistanceFunction.Hamming,
                     DistanceFunction.ManhattanDistance)}")

dotnet/src/Connectors/VectorData.Abstractions/RecordDefinition/DistanceFunction.cs

@@ -39,10 +39,19 @@ public static class DistanceFunction
     /// Measures both the length and angle between two vectors.
     /// </summary>
     /// <remarks>
-    /// Same as cosine similarity if the vectors are the same length, but more performant.
+    /// The higher the value, the more similar the vectors.
     /// </remarks>
     public const string DotProductSimilarity = nameof(DotProductSimilarity);
 
+    /// <summary>
+    /// Measures both the length and angle between two vectors.
+    /// </summary>
+    /// <remarks>
+    /// The value of NegativeDotProduct = -1 * DotProductSimilarity.
+    /// The higher the value, the greater the distance beteen the vectors and the less similar the vectors.
+    /// </remarks>
+    public const string NegativeDotProduct = nameof(NegativeDotProduct);
+
     /// <summary>
     /// Measures the Euclidean distance between two vectors.
     /// </summary>

dotnet/src/IntegrationTests/Connectors/Memory/BaseVectorStoreRecordCollectionTests.cs

@@ -62,7 +62,7 @@ public async Task VectorSearchShouldReturnExpectedScoresAsync(string distanceFun
         // Arrange
         var definition = CreateKeyWithVectorRecordDefinition(4, distanceFunction);
         var sut = this.GetTargetRecordCollection<KeyWithVectorRecord<TKey>>(
-            $"scorebydistancefunction{distanceFunction}",
+            $"scorebydf{distanceFunction}",
             definition);
 
         await sut.CreateCollectionIfNotExistsAsync();

dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Connectors.Pinecone;
+using SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit;
+using Xunit;
+
+namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone;
+
+/// <summary>
+/// Inherits common integration tests that should pass for any <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/>.
+/// </summary>
+/// <param name="fixture">Pinecone setup and teardown.</param>
+[Collection("PineconeVectorStoreTests")]
+[PineconeApiKeySetCondition]
+public class CommonPineconeVectorStoreRecordCollectionTests(PineconeVectorStoreFixture fixture) : BaseVectorStoreRecordCollectionTests<string>, IClassFixture<PineconeVectorStoreFixture>
+{
+    protected override string Key1 => "1";
+    protected override string Key2 => "2";
+    protected override string Key3 => "3";
+    protected override string Key4 => "4";
+
+    protected override int DelayAfterIndexCreateInMilliseconds => 2000;
+
+    protected override int DelayAfterUploadInMilliseconds => 15000;
+
+    [SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "Pinecone collection names should be lower case.")]
+    protected override IVectorStoreRecordCollection<string, TRecord> GetTargetRecordCollection<TRecord>(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition)
+    {
+        return new PineconeVectorStoreRecordCollection<TRecord>(fixture.Client, recordCollectionName.ToLowerInvariant(), new()
+        {
+            VectorStoreRecordDefinition = vectorStoreRecordDefinition
+        });
+    }
+
+    protected override HashSet<string> GetSupportedDistanceFunctions()
+    {
+        return [DistanceFunction.CosineSimilarity, DistanceFunction.DotProductSimilarity, DistanceFunction.EuclideanSquaredDistance];
+    }
+}

dotnet/src/IntegrationTests/Connectors/Memory/Redis/CommonRedisHashsetVectorStoreRecordCollectionTests.cs

@@ -0,0 +1,40 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Connectors.Redis;
+using Xunit;
+
+namespace SemanticKernel.IntegrationTests.Connectors.Memory.Redis;
+
+// Disable unused class warning, as this class is marked internal to disable the tests in the base class.
+#pragma warning disable CA1812
+#pragma warning disable CA1852
+
+/// <summary>
+/// Inherits common integration tests that should pass for any <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/>.
+/// </summary>
+/// <param name="fixture">Redis setup and teardown.</param>
+[Collection("RedisVectorStoreCollection")]
+internal class CommonRedisHashsetVectorStoreRecordCollectionTests(RedisVectorStoreFixture fixture) : BaseVectorStoreRecordCollectionTests<string>
+{
+    protected override string Key1 => "1";
+    protected override string Key2 => "2";
+    protected override string Key3 => "3";
+    protected override string Key4 => "4";
+
+    protected override IVectorStoreRecordCollection<string, TRecord> GetTargetRecordCollection<TRecord>(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition)
+    {
+        return new RedisHashSetVectorStoreRecordCollection<TRecord>(fixture.Database, recordCollectionName + "hashset", new()
+        {
+            VectorStoreRecordDefinition = vectorStoreRecordDefinition
+        });
+    }
+
+    protected override HashSet<string> GetSupportedDistanceFunctions()
+    {
+        // Excluding DotProductSimilarity from the test even though Redis supports it, because the values that redis returns
+        // are neither DotProductSimilarity nor NegativeDotProduct, but rather 1 - DotProductSimilarity.
+        return [DistanceFunction.CosineDistance, DistanceFunction.CosineSimilarity, DistanceFunction.EuclideanSquaredDistance];
+    }
+}

dotnet/src/IntegrationTests/Connectors/Memory/Redis/CommonRedisJsonVectorStoreRecordCollectionTests.cs

@@ -0,0 +1,40 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Connectors.Redis;
+using Xunit;
+
+namespace SemanticKernel.IntegrationTests.Connectors.Memory.Redis;
+
+// Disable unused class warning, as this class is marked internal to disable the tests in the base class.
+#pragma warning disable CA1812
+#pragma warning disable CA1852
+
+/// <summary>
+/// Inherits common integration tests that should pass for any <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/>.
+/// </summary>
+/// <param name="fixture">Redis setup and teardown.</param>
+[Collection("RedisVectorStoreCollection")]
+internal class CommonRedisJsonVectorStoreRecordCollectionTests(RedisVectorStoreFixture fixture) : BaseVectorStoreRecordCollectionTests<string>
+{
+    protected override string Key1 => "1";
+    protected override string Key2 => "2";
+    protected override string Key3 => "3";
+    protected override string Key4 => "4";
+
+    protected override IVectorStoreRecordCollection<string, TRecord> GetTargetRecordCollection<TRecord>(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition)
+    {
+        return new RedisJsonVectorStoreRecordCollection<TRecord>(fixture.Database, recordCollectionName + "json", new()
+        {
+            VectorStoreRecordDefinition = vectorStoreRecordDefinition
+        });
+    }
+
+    protected override HashSet<string> GetSupportedDistanceFunctions()
+    {
+        // Excluding DotProductSimilarity from the test even though Redis supports it, because the values that redis returns
+        // are neither DotProductSimilarity nor NegativeDotProduct, but rather 1 - DotProductSimilarity.
+        return [DistanceFunction.CosineDistance, DistanceFunction.CosineSimilarity, DistanceFunction.EuclideanSquaredDistance];
+    }
+}

dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/CommonWeaviateVectorStoreRecordCollectionTests.cs

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel.Connectors.Weaviate;
+using Xunit;
+
+namespace SemanticKernel.IntegrationTests.Connectors.Memory.Weaviate;
+
+/// <summary>
+/// Inherits common integration tests that should pass for any <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/>.
+/// </summary>
+/// <param name="fixture">Weaviate setup and teardown.</param>
+[Collection("WeaviateVectorStoreCollection")]
+public class CommonWeaviateVectorStoreRecordCollectionTests(WeaviateVectorStoreFixture fixture) : BaseVectorStoreRecordCollectionTests<Guid>
+{
+    protected override Guid Key1 => new("11111111-1111-1111-1111-111111111111");
+    protected override Guid Key2 => new("22222222-2222-2222-2222-222222222222");
+    protected override Guid Key3 => new("33333333-3333-3333-3333-333333333333");
+    protected override Guid Key4 => new("44444444-4444-4444-4444-444444444444");
+
+    protected override int DelayAfterUploadInMilliseconds => 1000;
+
+    protected override IVectorStoreRecordCollection<Guid, TRecord> GetTargetRecordCollection<TRecord>(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition)
+    {
+        // Weaviate collection names must start with an upper case letter.
+        var recordCollectionNameChars = recordCollectionName.ToCharArray();
+        recordCollectionNameChars[0] = char.ToUpperInvariant(recordCollectionNameChars[0]);
+
+        return new WeaviateVectorStoreRecordCollection<TRecord>(fixture.HttpClient!, new string(recordCollectionNameChars), new()
+        {
+            VectorStoreRecordDefinition = vectorStoreRecordDefinition
+        });
+    }
+
+    protected override HashSet<string> GetSupportedDistanceFunctions()
+    {
+        return [DistanceFunction.CosineDistance, DistanceFunction.NegativeDotProduct, DistanceFunction.EuclideanSquaredDistance, DistanceFunction.Hamming, DistanceFunction.ManhattanDistance];
+    }
+}