support velox engine for segcore(milvus-io#22920)

Signed-off-by: luzhang <luzhang@zilliz.com>
zhagnlu · Mar 22, 2023 · 3d12454 · 3d12454
1 parent ce2c5d1
commit 3d12454
Show file tree

Hide file tree

Showing 37 changed files with 2,326 additions and 139 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -28,6 +28,7 @@ services:
       ETCD_ENDPOINTS: ${ETCD_ENDPOINTS}
       MINIO_ADDRESS: ${MINIO_ADDRESS}
       CONAN_USER_HOME: /home/milvus
+      CPU_TARGET: ${CPU_TARGET}
     volumes: &builder-volumes
       - .:/go/src/github.com/milvus-io/milvus:delegated
       - ${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-ccache:/ccache:delegated

diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 cmake_minimum_required( VERSION 3.18 )
+set(CMAKE_VERBOSE_MAKEFILE ON)
 
 if ( APPLE )
     set( CMAKE_CROSSCOMPILING TRUE )
@@ -42,6 +43,8 @@ else ()
     message(FATAL_ERROR "Unsupported platform!" )
 endif ()
 
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
+
 if (CMAKE_COMPILER_IS_GNUCC)
     if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 11.99)
         # ignore deprecated declarations for gcc>=12
@@ -115,6 +118,7 @@ if (LINUX OR MSYS)
                   "-DELPP_THREAD_SAFE"
                   "-fopenmp"
                   "-Werror"
+                  "-Wp,-w"
                   )
 endif ()
 
@@ -248,6 +252,10 @@ if ( BUILD_DISK_ANN STREQUAL "ON" )
     ADD_DEFINITIONS(-DBUILD_DISK_ANN=${BUILD_DISK_ANN})
 endif ()
 
+if ( USE_VELOX STREQUAL "ON" )
+    ADD_DEFINITIONS(-DUSE_VELOX=${USE_VELOX})
+endif ()
+
 # Warning: add_subdirectory(src) must be after append_flags("-ftest-coverage"),
 # otherwise cpp code coverage tool will miss src folder
 add_subdirectory( thirdparty )

diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py
@@ -36,7 +36,7 @@ class MilvusConan(ConanFile):
         "xsimd/9.0.1",
         "xz_utils/5.4.0",
         "folly/2022.10.31.01@milvus/dev",
-        "velox/2023.02.07.01@milvus/dev",
+        "velox/2023.02.24-1-g6ba27bbc@milvus/dev",
     )
     generators = ("cmake", "cmake_find_package")
     default_options = {

diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt
@@ -22,14 +22,13 @@ set(COMMON_SRC
         )
 
 add_library(milvus_common SHARED ${COMMON_SRC})
-
 target_link_libraries(milvus_common
         milvus_config
         milvus_utils
         milvus_log
         yaml-cpp
         boost_bitset_ext
         ${CONAN_LIBS}
-        )
+)
 
 install(TARGETS milvus_common DESTINATION "${CMAKE_INSTALL_LIBDIR}")
diff --git a/internal/core/src/common/Consts.h b/internal/core/src/common/Consts.h
@@ -44,6 +44,8 @@ const int64_t DEFAULT_THREAD_CORE_COEFFICIENT = 50;
 
 const int64_t DEFAULT_INDEX_FILE_SLICE_SIZE = 4;  // megabytes
 
+const char SEGMENT_CONNECTOR_ID[] = "segment_connector_id";
+
 const int DEFAULT_CPU_NUM = 1;
 
 constexpr const char* RADIUS = knowhere::meta::RADIUS;

diff --git a/internal/core/src/pb/plan.pb.cc b/internal/core/src/pb/plan.pb.cc
diff --git a/internal/core/src/pb/plan.pb.h b/internal/core/src/pb/plan.pb.h