feat(transformer): Add MRL-E preprocess Transformer

Signed-off-by: LHT129 <tianlan.lht@antgroup.com>

Author: LHT129

src/impl/transform/mrle_transformer.h

@@ -0,0 +1,61 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "metric_type.h"
+#include "simd/normalize.h"
+#include "vector_transformer.h"
+#include "vsag_exception.h"
+
+namespace vsag {
+struct MRLETMeta : public TransformerMeta {};
+
+template <MetricType metric = MetricType::METRIC_TYPE_L2SQR>
+class MRLETransformer : public VectorTransformer {
+public:
+    explicit MRLETransformer(Allocator* allocator, int64_t input_dim, int64_t output_dim)
+        : VectorTransformer(allocator, input_dim, output_dim) {
+        this->type_ = VectorTransformerType::MRLE;
+    }
+
+    virtual ~MRLETransformer() override = default;
+
+    TransformerMetaPtr
+    Transform(const float* original_vec, float* transformed_vec) const override {
+        auto meta = std::make_shared<MRLETMeta>();
+        memcpy(transformed_vec, original_vec, this->output_dim_ * sizeof(float));
+        if constexpr (metric == MetricType::METRIC_TYPE_COSINE) {
+            Normalize(transformed_vec, transformed_vec, this->output_dim_);
+        }
+        return meta;
+    }
+
+    void
+    InverseTransform(const float* transformed_vec, float* original_vec) const override {
+        throw VsagException(ErrorType::INTERNAL_ERROR, "InverseTransform not implement");
+    }
+
+    void
+    Serialize(StreamWriter& writer) const override{};
+
+    void
+    Deserialize(StreamReader& reader) override{};
+
+    void
+    Train(const float* data, uint64_t count) override{};
+};
+
+}  // namespace vsag

src/impl/transform/transformer_headers.h

@@ -16,6 +16,7 @@
 #pragma once
 
 #include "fht_kac_rotate_transformer.h"
+#include "mrle_transformer.h"
 #include "pca_transformer.h"
 #include "random_orthogonal_transformer.h"
 #include "vector_transformer.h"

src/impl/transform/vector_transformer.h

@@ -26,7 +26,7 @@ class Allocator;
 DEFINE_POINTER(VectorTransformer);
 DEFINE_POINTER(TransformerMeta);
 
-enum class VectorTransformerType { NONE, PCA, RANDOM_ORTHOGONAL, FHT, RESIDUAL, NORMALIZE };
+enum class VectorTransformerType { NONE, PCA, RANDOM_ORTHOGONAL, FHT, RESIDUAL, NORMALIZE, MRLE };
 
 struct TransformerMeta {
     virtual void

src/impl/transform/vector_transformer_parameter.cpp

@@ -28,12 +28,17 @@ VectorTransformerParameter::FromJson(const JsonType& json) {
     if (json.Contains(PCA_DIM_KEY)) {
         pca_dim_ = json[PCA_DIM_KEY].GetInt();
     }
+
+    if (json.Contains(MRLE_DIM_KEY)) {
+        mrle_dim_ = json[MRLE_DIM_KEY].GetInt();
+    }
 }
 
 JsonType
 VectorTransformerParameter::ToJson() const {
     JsonType json;
     json[PCA_DIM_KEY].SetInt(pca_dim_);
+    json[MRLE_DIM_KEY].SetInt(mrle_dim_);
     json[INPUT_DIM_KEY].SetInt(input_dim_);
     return json;
 }
@@ -53,6 +58,9 @@ VectorTransformerParameter::CheckCompatibility(const ParamPtr& other) const {
     if (input_dim_ != param->input_dim_) {
         return false;
     }
+    if (mrle_dim_ != param->mrle_dim_) {
+        return false;
+    }
     return true;
 }
 

src/impl/transform/vector_transformer_parameter.h

@@ -35,8 +35,9 @@ class VectorTransformerParameter : public Parameter {
     CheckCompatibility(const vsag::ParamPtr& other) const override;
 
 public:
-    uint32_t input_dim_;
-    uint32_t pca_dim_;
+    uint32_t input_dim_{0};
+    uint32_t pca_dim_{0};
+    uint32_t mrle_dim_{0};
 };
 
 }  // namespace vsag

src/inner_string_params.h

@@ -82,12 +82,14 @@ const char* const QUANTIZATION_TYPE_VALUE_TQ = "tq";
 const char* const TRANSFORMER_TYPE_VALUE_PCA = "pca";
 const char* const TRANSFORMER_TYPE_VALUE_ROM = "rom";
 const char* const TRANSFORMER_TYPE_VALUE_FHT = "fht";
+const char* const TRANSFORMER_TYPE_VALUE_MRLE = "mrle";
 const char* const TRANSFORMER_TYPE_VALUE_RESIDUAL = "residual";
 const char* const TRANSFORMER_TYPE_VALUE_NORMALIZE = "normalize";
 
 // vector transformer param
 const char* const INPUT_DIM_KEY = "input_dim";
 const char* const PCA_DIM_KEY = "pca_dim";
+const char* const MRLE_DIM_KEY = "mrle_dim";
 const char* const USE_FHT_KEY = "use_fht";
 
 // quantization param

src/quantization/transform_quantization/transform_quantizer.h

@@ -132,6 +132,11 @@ TransformQuantizer<QuantTmpl, metric>::TransformQuantizer(const TransformQuantiz
     transformer_param.input_dim_ = this->dim_;
     for (const auto& transform_str : param->tq_chain_) {
         transform_chain_.emplace_back(MakeTransformerInstance(transform_str, transformer_param));
+        if (transform_chain_.back()->GetType() == VectorTransformerType::MRLE and
+            transform_chain_.size() > 1) {
+            throw VsagException(ErrorType::INVALID_ARGUMENT,
+                                fmt::format("MRLE must be first if exists"));
+        }
         transformer_param.input_dim_ = transform_chain_.back()->GetOutputDim();
     }
 
@@ -186,6 +191,17 @@ TransformQuantizer<QuantTmpl, metric>::MakeTransformerInstance(
         return std::make_shared<RandomOrthogonalMatrix>(this->allocator_, input_dim, output_dim);
     }
 
+    if (transform_str == TRANSFORMER_TYPE_VALUE_MRLE) {
+        if (param.mrle_dim_ != 0) {
+            output_dim = param.mrle_dim_;
+            if (output_dim > input_dim) {
+                throw VsagException(ErrorType::INVALID_ARGUMENT,
+                                    fmt::format("mrle dim must be less than input dim"));
+            }
+        }
+        return std::make_shared<MRLETransformer<metric>>(this->allocator_, input_dim, output_dim);
+    }
+
     throw VsagException(ErrorType::INVALID_ARGUMENT,
                         fmt::format("invalid transformer name {}", transform_str));
 };

src/quantization/transform_quantization/transform_quantizer_test.cpp

@@ -36,10 +36,11 @@ TestComputeMetricTQ(std::string tq_chain, uint64_t dim, int count, float error =
     constexpr static const char* param_template = R"(
         {{
             "tq_chain": "{}",
-            "pca_dim": {}
+            "pca_dim": {},
+            "mrle_dim": {}
         }}
     )";
-    auto param_str = fmt::format(param_template, tq_chain, dim - 1);
+    auto param_str = fmt::format(param_template, tq_chain, dim, dim - 1);
     auto param_json = vsag::JsonType::Parse(param_str);
     param->FromJson(param_json);
 
@@ -66,10 +67,11 @@ TestSerializeDeserializeTQ(std::string tq_chain, uint64_t dim, int count) {
     constexpr static const char* param_template = R"(
                 {{
                     "tq_chain": "{}",
-                    "pca_dim": {}
+                    "pca_dim": {},
+                    "mrle_dim": {}
                 }}
             )";
-    auto param_str = fmt::format(param_template, tq_chain, dim - 2);
+    auto param_str = fmt::format(param_template, tq_chain, dim, dim - 1);
     auto param_json = vsag::JsonType::Parse(param_str);
     param->FromJson(param_json);
 
@@ -92,7 +94,7 @@ TestSerializeDeserializeTQ(std::string tq_chain, uint64_t dim, int count) {
 
 TEST_CASE("TQ Compute", "[ut][TransformQuantizer]") {
     constexpr MetricType metrics[1] = {MetricType::METRIC_TYPE_L2SQR};
-    std::string tq_chain = GENERATE("rom, pca, fp32", "rom, fp32", "fht, fp32");
+    std::string tq_chain = GENERATE("rom, pca, fp32", "rom, fp32", "fht, fp32", "mrle, fp32");
 
     for (auto dim : dims) {
         if (dim < 100) {