Merge meilisearch#2244

2244: chore(all): bump milli r=curquiza a=MarinPostma

continues the work initiated by `@psvnlsaikumar` in meilisearch#2228

Co-authored-by: Sai Kumar <psvnlsaikumar@gmail.com>

meilisearch-auth/Cargo.toml

@@ -5,11 +5,11 @@ edition = "2021"
 
 [dependencies]
 enum-iterator = "0.7.0"
-heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
 sha2 = "0.9.6"
 meilisearch-error = { path = "../meilisearch-error" }
 serde_json = { version = "1.0.67", features = ["preserve_order"] }
 time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] }
 rand = "0.8.4"
 serde = { version = "1.0.130", features = ["derive"] }
 thiserror = "1.0.28"
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.24.0" }

meilisearch-auth/src/error.rs

@@ -25,7 +25,7 @@ pub enum AuthControllerError {
 }
 
 internal_error!(
-    AuthControllerError: heed::Error,
+    AuthControllerError: milli::heed::Error,
     std::io::Error,
     serde_json::Error,
     std::str::Utf8Error

meilisearch-auth/src/store.rs

@@ -8,8 +8,8 @@ use std::path::Path;
 use std::str;
 use std::sync::Arc;
 
-use heed::types::{ByteSlice, DecodeIgnore, SerdeJson};
-use heed::{Database, Env, EnvOpenOptions, RwTxn};
+use milli::heed::types::{ByteSlice, DecodeIgnore, SerdeJson};
+use milli::heed::{Database, Env, EnvOpenOptions, RwTxn};
 use time::OffsetDateTime;
 
 use super::error::Result;
@@ -39,7 +39,7 @@ impl Drop for HeedAuthStore {
     }
 }
 
-pub fn open_auth_store_env(path: &Path) -> heed::Result<heed::Env> {
+pub fn open_auth_store_env(path: &Path) -> milli::heed::Result<milli::heed::Env> {
     let mut options = EnvOpenOptions::new();
     options.map_size(AUTH_STORE_SIZE); // 1GB
     options.max_dbs(2);
@@ -203,7 +203,7 @@ impl HeedAuthStore {
 /// optionnally on a spcific index, for a given key.
 pub struct KeyIdActionCodec;
 
-impl<'a> heed::BytesDecode<'a> for KeyIdActionCodec {
+impl<'a> milli::heed::BytesDecode<'a> for KeyIdActionCodec {
     type DItem = (KeyId, Action, Option<&'a [u8]>);
 
     fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
@@ -218,7 +218,7 @@ impl<'a> heed::BytesDecode<'a> for KeyIdActionCodec {
     }
 }
 
-impl<'a> heed::BytesEncode<'a> for KeyIdActionCodec {
+impl<'a> milli::heed::BytesEncode<'a> for KeyIdActionCodec {
     type EItem = (&'a KeyId, &'a Action, Option<&'a [u8]>);
 
     fn bytes_encode((key_id, action, index): &Self::EItem) -> Option<Cow<[u8]>> {

meilisearch-http/Cargo.toml

@@ -39,7 +39,6 @@ flate2 = "1.0.21"
 fst = "0.4.7"
 futures = "0.3.17"
 futures-util = "0.3.17"
-heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
 http = "0.2.4"
 indexmap = { version = "1.7.0", features = ["serde-1"] }
 iso8601-duration = "0.1.0"
@@ -77,6 +76,7 @@ tokio = { version = "1.11.0", features = ["full"] }
 tokio-stream = "0.1.7"
 uuid = { version = "0.8.2", features = ["serde"] }
 walkdir = "2.3.2"
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.24.0" }
 pin-project-lite = "0.2.8"
 
 [dev-dependencies]

meilisearch-http/src/helpers/env.rs

@@ -4,7 +4,7 @@ pub trait EnvSizer {
     fn size(&self) -> u64;
 }
 
-impl EnvSizer for heed::Env {
+impl EnvSizer for milli::heed::Env {
     fn size(&self) -> u64 {
         WalkDir::new(self.path())
             .into_iter()

meilisearch-lib/Cargo.toml

@@ -19,15 +19,14 @@ flate2 = "1.0.21"
 fst = "0.4.7"
 futures = "0.3.17"
 futures-util = "0.3.17"
-heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
 http = "0.2.4"
 indexmap = { version = "1.7.0", features = ["serde-1"] }
 itertools = "0.10.1"
 lazy_static = "1.4.0"
 log = "0.4.14"
 meilisearch-error = { path = "../meilisearch-error" }
 meilisearch-auth = { path = "../meilisearch-auth" }
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.22.2" }
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.24.0" }
 mime = "0.3.16"
 num_cpus = "1.13.0"
 once_cell = "1.8.0"

meilisearch-lib/src/index/dump.rs

@@ -3,9 +3,9 @@ use std::io::{BufReader, Seek, SeekFrom, Write};
 use std::path::Path;
 
 use anyhow::Context;
-use heed::{EnvOpenOptions, RoTxn};
 use indexmap::IndexMap;
 use milli::documents::DocumentBatchReader;
+use milli::heed::{EnvOpenOptions, RoTxn};
 use milli::update::{IndexDocumentsConfig, IndexerConfig};
 use serde::{Deserialize, Serialize};
 

meilisearch-lib/src/index/error.rs

@@ -21,7 +21,7 @@ pub enum IndexError {
 
 internal_error!(
     IndexError: std::io::Error,
-    heed::Error,
+    milli::heed::Error,
     fst::Error,
     serde_json::Error,
     update_file_store::UpdateFileStoreError,

meilisearch-lib/src/index/index.rs

@@ -5,7 +5,7 @@ use std::ops::Deref;
 use std::path::Path;
 use std::sync::Arc;
 
-use heed::{EnvOpenOptions, RoTxn};
+use milli::heed::{EnvOpenOptions, RoTxn};
 use milli::update::{IndexerConfig, Setting};
 use milli::{obkv_to_json, FieldDistribution, FieldId};
 use serde::{Deserialize, Serialize};
@@ -37,7 +37,7 @@ impl IndexMeta {
         Self::new_txn(index, &txn)
     }
 
-    pub fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result<Self> {
+    pub fn new_txn(index: &Index, txn: &milli::heed::RoTxn) -> Result<Self> {
         let created_at = index.created_at(txn)?;
         let updated_at = index.updated_at(txn)?;
         let primary_key = index.primary_key(txn)?.map(String::from);
@@ -250,7 +250,7 @@ impl Index {
 
     fn fields_to_display<S: AsRef<str>>(
         &self,
-        txn: &heed::RoTxn,
+        txn: &milli::heed::RoTxn,
         attributes_to_retrieve: &Option<Vec<S>>,
         fields_ids_map: &milli::FieldsIdsMap,
     ) -> Result<Vec<FieldId>> {
@@ -278,7 +278,7 @@ impl Index {
         let _txn = self.write_txn()?;
         self.inner
             .env
-            .copy_to_path(dst, heed::CompactionOption::Enabled)?;
+            .copy_to_path(dst, milli::heed::CompactionOption::Enabled)?;
         Ok(())
     }
 }

meilisearch-lib/src/index/updates.rs

@@ -176,7 +176,7 @@ pub struct Facets {
 impl Index {
     fn update_primary_key_txn<'a, 'b>(
         &'a self,
-        txn: &mut heed::RwTxn<'a, 'b>,
+        txn: &mut milli::heed::RwTxn<'a, 'b>,
         primary_key: String,
     ) -> Result<IndexMeta> {
         let mut builder = milli::update::Settings::new(txn, self, self.indexer_config.as_ref());

meilisearch-lib/src/index_controller/dump_actor/error.rs

@@ -18,7 +18,7 @@ pub enum DumpActorError {
 }
 
 internal_error!(
-    DumpActorError: heed::Error,
+    DumpActorError: milli::heed::Error,
     std::io::Error,
     tokio::task::JoinError,
     tokio::sync::oneshot::error::RecvError,

meilisearch-lib/src/index_controller/dump_actor/loaders/v4.rs

@@ -1,9 +1,9 @@
 use std::path::Path;
 use std::sync::Arc;
 
-use heed::EnvOpenOptions;
 use log::info;
 use meilisearch_auth::AuthController;
+use milli::heed::EnvOpenOptions;
 
 use crate::analytics;
 use crate::index_controller::dump_actor::Metadata;

meilisearch-lib/src/index_controller/mod.rs

@@ -48,8 +48,8 @@ pub type Payload = Box<
     dyn Stream<Item = std::result::Result<Bytes, PayloadError>> + Send + Sync + 'static + Unpin,
 >;
 
-pub fn open_meta_env(path: &Path, size: usize) -> heed::Result<heed::Env> {
-    let mut options = heed::EnvOpenOptions::new();
+pub fn open_meta_env(path: &Path, size: usize) -> milli::heed::Result<milli::heed::Env> {
+    let mut options = milli::heed::EnvOpenOptions::new();
     options.map_size(size);
     options.max_dbs(20);
     options.open(path)

meilisearch-lib/src/index_resolver/error.rs

@@ -45,7 +45,7 @@ impl From<OneshotRecvError> for IndexResolverError {
 }
 
 internal_error!(
-    IndexResolverError: heed::Error,
+    IndexResolverError: milli::heed::Error,
     uuid::Error,
     std::io::Error,
     tokio::task::JoinError,

meilisearch-lib/src/index_resolver/meta_store.rs

@@ -4,8 +4,8 @@ use std::io::{BufRead, BufReader, Write};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use heed::types::{SerdeBincode, Str};
-use heed::{CompactionOption, Database, Env};
+use milli::heed::types::{SerdeBincode, Str};
+use milli::heed::{CompactionOption, Database, Env};
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
 
@@ -56,7 +56,7 @@ impl Drop for HeedMetaStore {
 }
 
 impl HeedMetaStore {
-    pub fn new(env: Arc<heed::Env>) -> Result<Self> {
+    pub fn new(env: Arc<milli::heed::Env>) -> Result<Self> {
         let db = env.create_database(Some("uuids"))?;
         Ok(Self { env, db })
     }
@@ -153,7 +153,7 @@ impl HeedMetaStore {
         Ok(())
     }
 
-    pub fn load_dump(src: impl AsRef<Path>, env: Arc<heed::Env>) -> Result<()> {
+    pub fn load_dump(src: impl AsRef<Path>, env: Arc<milli::heed::Env>) -> Result<()> {
         let src_indexes = src.as_ref().join(UUIDS_DB_PATH).join("data.jsonl");
         let indexes = File::open(&src_indexes)?;
         let mut indexes = BufReader::new(indexes);

meilisearch-lib/src/index_resolver/mod.rs

@@ -7,10 +7,10 @@ use std::path::Path;
 use std::sync::Arc;
 
 use error::{IndexResolverError, Result};
-use heed::Env;
 use index_store::{IndexStore, MapIndexStore};
 use meilisearch_error::ResponseError;
 use meta_store::{HeedMetaStore, IndexMetaStore};
+use milli::heed::Env;
 use milli::update::{DocumentDeletionResult, IndexerConfig};
 use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
@@ -39,7 +39,7 @@ pub fn create_index_resolver(
     path: impl AsRef<Path>,
     index_size: usize,
     indexer_opts: &IndexerOpts,
-    meta_env: Arc<heed::Env>,
+    meta_env: Arc<milli::heed::Env>,
     file_store: UpdateFileStore,
 ) -> anyhow::Result<HardStateIndexResolver> {
     let uuid_store = HeedMetaStore::new(meta_env)?;

meilisearch-lib/src/lib.rs

@@ -25,7 +25,7 @@ pub trait EnvSizer {
     fn size(&self) -> u64;
 }
 
-impl EnvSizer for heed::Env {
+impl EnvSizer for milli::heed::Env {
     fn size(&self) -> u64 {
         WalkDir::new(self.path())
             .into_iter()

meilisearch-lib/src/snapshot.rs

@@ -149,7 +149,7 @@ impl SnapshotJob {
         let env = open_meta_env(&self.src_path, self.meta_env_size)?;
 
         let dst = path.join("data.mdb");
-        env.copy_to_path(dst, heed::CompactionOption::Enabled)?;
+        env.copy_to_path(dst, milli::heed::CompactionOption::Enabled)?;
 
         Ok(())
     }
@@ -180,12 +180,12 @@ impl SnapshotJob {
 
             let dst = dst.join("data.mdb");
 
-            let mut options = heed::EnvOpenOptions::new();
+            let mut options = milli::heed::EnvOpenOptions::new();
             options.map_size(self.index_size);
             let index = milli::Index::new(options, entry.path())?;
             index
                 .env
-                .copy_to_path(dst, heed::CompactionOption::Enabled)?;
+                .copy_to_path(dst, milli::heed::CompactionOption::Enabled)?;
         }
 
         Ok(())
@@ -198,7 +198,7 @@ impl SnapshotJob {
         let dst = dst.join("data.mdb");
 
         let env = open_auth_store_env(&auth_path)?;
-        env.copy_to_path(dst, heed::CompactionOption::Enabled)?;
+        env.copy_to_path(dst, milli::heed::CompactionOption::Enabled)?;
 
         Ok(())
     }

meilisearch-lib/src/tasks/error.rs

@@ -16,7 +16,7 @@ pub enum TaskError {
 }
 
 internal_error!(
-    TaskError: heed::Error,
+    TaskError: milli::heed::Error,
     JoinError,
     std::io::Error,
     serde_json::Error,

meilisearch-lib/src/tasks/task_store/mod.rs

@@ -5,8 +5,8 @@ use std::io::{BufWriter, Write};
 use std::path::Path;
 use std::sync::Arc;
 
-use heed::{Env, RwTxn};
 use log::debug;
+use milli::heed::{Env, RwTxn};
 use time::OffsetDateTime;
 
 use super::error::TaskError;
@@ -61,7 +61,7 @@ impl Clone for TaskStore {
 }
 
 impl TaskStore {
-    pub fn new(env: Arc<heed::Env>) -> Result<Self> {
+    pub fn new(env: Arc<milli::heed::Env>) -> Result<Self> {
         let store = Arc::new(Store::new(env)?);
         Ok(Self { store })
     }
@@ -248,7 +248,7 @@ pub mod test {
     }
 
     impl MockTaskStore {
-        pub fn new(env: Arc<heed::Env>) -> Result<Self> {
+        pub fn new(env: Arc<milli::heed::Env>) -> Result<Self> {
             Ok(Self::Real(TaskStore::new(env)?))
         }