Delete and WAL feature branch merge

CHANGELOG.md

@@ -22,9 +22,11 @@ Please mark all change in change log and use the issue from GitHub
 - \#1075 - improve error message when page size or offset is illegal
 - \#1082 - check page_size or offset value to avoid float
 - \#1115 - http server support load table into memory
+- \#1152 - Error log output continuously after server start
 - \#1211 - Server down caused by searching with index_type: HNSW
 - \#1240 - Update license declaration
 - \#1298 - Unittest failed when on CPU2GPU case
+- \#1359 - Negative distance value returned when searching with HNSW index type
 
 ## Feature
 - \#216 - Add CLI to get server info
@@ -39,6 +41,8 @@ Please mark all change in change log and use the issue from GitHub
 - \#823 - Support binary vector tanimoto/jaccard/hamming metric
 - \#853 - Support HNSW
 - \#910 - Change Milvus c++ standard to c++17
+- \#1204 - Add api to get table data information
+- \#1302 - Get all record IDs in a segment by given a segment id
 
 ## Improvement
 - \#738 - Use Openblas / lapack from apt install
@@ -53,11 +57,14 @@ Please mark all change in change log and use the issue from GitHub
 - \#1002 - Rename minio to s3 in Storage Config section
 - \#1078 - Move 'insert_buffer_size' to Cache Config section
 - \#1105 - Error message is not clear when creating IVFSQ8H index without gpu resources
+- \#1297 - Hide partition_name parameter, avid user directly access partition table
+- \#1310 - Add default partition tag for a table
 - \#740, #849, #878, #972, #1033, #1161, #1173, #1199, #1190, #1223, #1222, #1257, #1264, #1269, #1164, #1303, #1304, #1324, #1388 - Various fixes and improvements for Milvus documentation.
 - \#1234 - Do S3 server validation check when Milvus startup
 - \#1263 - Allow system conf modifiable and some take effect directly
 - \#1320 - Remove debug logging from faiss
 
+
 ## Task
 - \#1327 - Exclude third-party code from codebeat
 - \#1331 - Exclude third-party code from codacy

NOTICE.md

@@ -6,7 +6,6 @@
 
 | Name          | License                                                      |
 | ------------- | ------------------------------------------------------------ |
-| Apache Arrow  | [Apache License 2.0](https://github.com/apache/arrow/blob/master/LICENSE.txt)                                           |
 | Boost         | [Boost Software License](https://github.com/boostorg/boost/blob/master/LICENSE_1_0.txt)                                       |
 | FAISS         | [MIT](https://github.com/facebookresearch/faiss/blob/master/LICENSE)                                                          |
 | Gtest         | [BSD 3-Clause](https://github.com/google/googletest/blob/master/LICENSE)                                                 |

ci/jenkins/step/singleDevNightlyTest.groovy

@@ -1,6 +1,7 @@
 timeout(time: 90, unit: 'MINUTES') {
     dir ("tests/milvus_python_test") {
-        sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
+        // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
+        sh 'python3 -m pip install -r requirements.txt'
         sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
     }
     // mysql database backend test

ci/jenkins/step/singleDevTest.groovy

@@ -1,6 +1,7 @@
 timeout(time: 60, unit: 'MINUTES') {
     dir ("tests/milvus_python_test") {
-        sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
+        // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
+        sh 'python3 -m pip install -r requirements.txt'
         sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
     }
 

core/CMakeLists.txt

@@ -90,7 +90,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL ""
         OR MILVUS_VERSION_MINOR STREQUAL ""
         OR MILVUS_VERSION_PATCH STREQUAL "")
     message(WARNING "Failed to determine Milvus version from git branch name")
-    set(MILVUS_VERSION "0.6.0")
+    set(MILVUS_VERSION "0.7.0")
 endif ()
 
 message(STATUS "Build version = ${MILVUS_VERSION}")

core/conf/demo/server_config.yaml

@@ -106,9 +106,7 @@ metric_config:
 #                      | The sum of 'insert_buffer_size' and 'cpu_cache_capacity'   |            |                 |
 #                      | must be less than system memory size.                      |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
-# cache_insert_data    | Whether to load inserted data into cache immediately for   | Boolean    | false           |
-#                      | hot query. If want to simultaneously insert and query      |            |                 |
-#                      | vectors, it's recommended to enable this config.           |            |                 |
+# cache_insert_data    | Whether to load data to cache for hot query                | Boolean    | false           |
 #----------------------+------------------------------------------------------------+------------+-----------------+
 cache_config:
   cpu_cache_capacity: 4

core/conf/server_cpu_config.template

@@ -46,9 +46,12 @@ server_config:
 #                      | loaded when Milvus server starts up.                       |            |                 |
 #                      | '*' means preload all existing tables.                     |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
+# auto_flush_interval  | Interval of auto flush. Unit is millisecond.               | Integer    | 1000            |
+#----------------------+------------------------------------------------------------+------------+-----------------+
 db_config:
   backend_url: sqlite://:@:/
   preload_table:
+  auto_flush_interval: 1000
 
 #----------------------+------------------------------------------------------------+------------+-----------------+
 # Storage Config       | Description                                                | Type       | Default         |
@@ -106,9 +109,7 @@ metric_config:
 #                      | The sum of 'insert_buffer_size' and 'cpu_cache_capacity'   |            |                 |
 #                      | must be less than system memory size.                      |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
-# cache_insert_data    | Whether to load inserted data into cache immediately for   | Boolean    | false           |
-#                      | hot query. If want to simultaneously insert and query      |            |                 |
-#                      | vectors, it's recommended to enable this config.           |            |                 |
+# cache_insert_data    | Whether to load data to cache for hot query                | Boolean    | false           |
 #----------------------+------------------------------------------------------------+------------+-----------------+
 cache_config:
   cpu_cache_capacity: 4
@@ -167,3 +168,24 @@ gpu_resource_config:
 #----------------------+------------------------------------------------------------+------------+-----------------+
 tracing_config:
   json_config_path:
+
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# Wal Config           | Description                                                | Type       | Default         |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# enable               | Switch of function wal.                                    | Boolean    | false           |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# recovery_error_ignore| Whether ignore the error which happens during wal recovery | Boolean    | true            |
+#                      | stage.                                                     |            |                 |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# buffer_size          | The size of the wal buffer. Unit is MB.                    | Integer    | 256             |
+#                      | It should be in range [64, 4096]. If the value set out of |            |                 |
+#                      | the range, the system will use the boundary value.         |            |                 |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# wal_path             | The root path of wal relative files, include wal meta      | String     | NULL            |
+#                      | files.                                                     |            |                 |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+wal_config:
+  enable: true
+  recovery_error_ignore: true
+  buffer_size: 256   # MB
+  wal_path: /tmp/milvus/wal

core/conf/server_gpu_config.template

@@ -106,9 +106,7 @@ metric_config:
 #                      | The sum of 'insert_buffer_size' and 'cpu_cache_capacity'   |            |                 |
 #                      | must be less than system memory size.                      |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
-# cache_insert_data    | Whether to load inserted data into cache immediately for   | Boolean    | false           |
-#                      | hot query. If want to simultaneously insert and query      |            |                 |
-#                      | vectors, it's recommended to enable this config.           |            |                 |
+# cache_insert_data    | Whether to load data to cache for hot query                | Boolean    | false           |
 #----------------------+------------------------------------------------------------+------------+-----------------+
 cache_config:
   cpu_cache_capacity: 4
@@ -167,3 +165,24 @@ gpu_resource_config:
 #----------------------+------------------------------------------------------------+------------+-----------------+
 tracing_config:
   json_config_path:
+
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# Wal Config           | Description                                                | Type       | Default         |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# enable               | Switch of function wal.                                    | Boolean    | false           |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# recovery_error_ignore| Whether ignore the error which happens during wal recovery | Boolean    | true            |
+#                      | stage.                                                     |            |                 |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# buffer_size          | The size of the wal buffer. Unit is MB.                    | Integer    | 256             |
+#                      | It should be in range [64, 4096]. If the value set out of |            |                 |
+#                      | the range, the system will use the boundary value.         |            |                 |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+# wal_path             | The root path of wal relative files, include wal meta      | String     | NULL            |
+#                      | files.                                                     |            |                 |
+#----------------------+------------------------------------------------------------+------------+-----------------+
+wal_config:
+  enable: true
+  recovery_error_ignore: true
+  buffer_size: 256   # MB
+  wal_path: /tmp/milvus/wal

core/src/CMakeLists.txt

@@ -36,6 +36,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
+aux_source_directory(${MILVUS_ENGINE_SRC}/db/wal db_wal_files)
 
 set(grpc_service_files
         ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc
@@ -65,9 +66,11 @@ set(scheduler_files
 
 aux_source_directory(${MILVUS_THIRDPARTY_SRC}/easyloggingpp thirdparty_easyloggingpp_files)
 aux_source_directory(${MILVUS_THIRDPARTY_SRC}/nlohmann thirdparty_nlohmann_files)
+aux_source_directory(${MILVUS_THIRDPARTY_SRC}/dablooms thirdparty_dablooms_files)
 set(thirdparty_files
         ${thirdparty_easyloggingpp_files}
         ${thirdparty_nlohmann_files}
+        ${thirdparty_dablooms_files}
         )
 
 aux_source_directory(${MILVUS_ENGINE_SRC}/server server_service_files)
@@ -113,22 +116,35 @@ set(storage_files
         )
 
 aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files)
+
 aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files)
 
 aux_source_directory(${MILVUS_ENGINE_SRC}/tracing tracing_files)
 
+aux_source_directory(${MILVUS_ENGINE_SRC}/codecs codecs_files)
+aux_source_directory(${MILVUS_ENGINE_SRC}/codecs/default codecs_default_files)
+
+aux_source_directory(${MILVUS_ENGINE_SRC}/segment segment_files)
+
+aux_source_directory(${MILVUS_ENGINE_SRC}/store store_files)
+
 set(engine_files
         ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
         ${cache_files}
         ${db_main_files}
         ${db_engine_files}
         ${db_insert_files}
         ${db_meta_files}
+        ${db_wal_files}
         ${metrics_files}
         ${storage_files}
         ${thirdparty_files}
         ${utils_files}
         ${wrapper_files}
+        ${codecs_files}
+        ${codecs_default_files}
+        ${segment_files}
+        ${store_files}
         )
 
 if (MILVUS_WITH_PROMETHEUS)

core/src/codecs/AttrsFormat.h

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+namespace milvus {
+namespace codec {
+
+class AttrsFormat {
+    // public:
+    //    virtual Attrs
+    //    read() = 0;
+    //
+    //    virtual void
+    //    write(Attrs attrs) = 0;
+};
+
+}  // namespace codec
+}  // namespace milvus

core/src/codecs/AttrsIndexFormat.h

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+namespace milvus {
+namespace codec {
+
+class AttrsIndexFormat {
+    // public:
+    //    virtual AttrsIndex
+    //    read() = 0;
+    //
+    //    virtual void
+    //    write(AttrsIndex attrs_index) = 0;
+};
+
+}  // namespace codec
+}  // namespace milvus

core/src/codecs/Codec.h

@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "AttrsFormat.h"
+#include "AttrsIndexFormat.h"
+#include "DeletedDocsFormat.h"
+#include "IdBloomFilterFormat.h"
+#include "IdIndexFormat.h"
+#include "VectorsFormat.h"
+#include "VectorsIndexFormat.h"
+
+namespace milvus {
+namespace codec {
+
+class Codec {
+ public:
+    virtual VectorsFormatPtr
+    GetVectorsFormat() = 0;
+
+    virtual DeletedDocsFormatPtr
+    GetDeletedDocsFormat() = 0;
+
+    virtual IdBloomFilterFormatPtr
+    GetIdBloomFilterFormat() = 0;
+
+    // TODO(zhiru)
+    /*
+    virtual AttrsFormat
+    GetAttrsFormat() = 0;
+
+    virtual VectorsIndexFormat
+    GetVectorsIndexFormat() = 0;
+
+    virtual AttrsIndexFormat
+    GetAttrsIndexFormat() = 0;
+
+    virtual IdIndexFormat
+    GetIdIndexFormat() = 0;
+
+    */
+};
+
+}  // namespace codec
+}  // namespace milvus

core/src/codecs/DeletedDocsFormat.h

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+
+#include "segment/DeletedDocs.h"
+#include "store/Directory.h"
+
+namespace milvus {
+namespace codec {
+
+class DeletedDocsFormat {
+ public:
+    virtual void
+    read(const store::DirectoryPtr& directory_ptr, segment::DeletedDocsPtr& deleted_docs) = 0;
+
+    virtual void
+    write(const store::DirectoryPtr& directory_ptr, const segment::DeletedDocsPtr& deleted_docs) = 0;
+};
+
+using DeletedDocsFormatPtr = std::shared_ptr<DeletedDocsFormat>;
+
+}  // namespace codec
+}  // namespace milvus