feat(index): index cache in SQLite3

src/cargo/sources/registry/index/cache.rs

@@ -65,16 +65,22 @@
 //! [`IndexSummary::parse`]: super::IndexSummary::parse
 //! [`RemoteRegistry`]: crate::sources::registry::remote::RemoteRegistry
 
+use std::cell::OnceCell;
 use std::fs;
 use std::io;
 use std::path::PathBuf;
 use std::str;
 
 use anyhow::bail;
 use cargo_util::registry::make_dep_path;
+use rusqlite::params;
+use rusqlite::Connection;
 use semver::Version;
 
 use crate::util::cache_lock::CacheLockMode;
+use crate::util::sqlite;
+use crate::util::sqlite::basic_migration;
+use crate::util::sqlite::Migration;
 use crate::util::Filesystem;
 use crate::CargoResult;
 use crate::GlobalContext;
@@ -242,7 +248,11 @@ impl<'gctx> CacheManager<'gctx> {
     ///
     /// `root` --- The root path where caches are located.
     pub fn new(cache_root: Filesystem, gctx: &'gctx GlobalContext) -> CacheManager<'gctx> {
-        let store = Box::new(LocalFileSystem::new(cache_root, gctx));
+        let store: Box<dyn CacheStore> = if gctx.cli_unstable().index_cache_sqlite {
+            Box::new(LocalDatabase::new(cache_root, gctx))
+        } else {
+            Box::new(LocalFileSystem::new(cache_root, gctx))
+        };
         CacheManager { store }
     }
 
@@ -318,3 +328,88 @@ impl CacheStore for LocalFileSystem<'_> {
         }
     }
 }
+
+/// Stores index caches in a local SQLite database.
+struct LocalDatabase<'gctx> {
+    /// The root path where caches are located.
+    cache_root: Filesystem,
+    /// Connection to the SQLite database.
+    conn: OnceCell<Option<Connection>>,
+    /// [`GlobalContext`] reference for convenience.
+    gctx: &'gctx GlobalContext,
+}
+
+impl LocalDatabase<'_> {
+    /// Creates a new instance of the SQLite index cache store.
+    fn new(cache_root: Filesystem, gctx: &GlobalContext) -> LocalDatabase<'_> {
+        LocalDatabase {
+            cache_root,
+            conn: OnceCell::new(),
+            gctx,
+        }
+    }
+
+    fn conn(&self) -> Option<&Connection> {
+        self.conn
+            .get_or_init(|| {
+                self.conn_init()
+                    .map_err(|e| tracing::debug!("cannot open index cache db: {e}"))
+                    .ok()
+            })
+            .as_ref()
+    }
+
+    fn conn_init(&self) -> CargoResult<Connection> {
+        let _lock = self
+            .gctx
+            .acquire_package_cache_lock(CacheLockMode::DownloadExclusive)
+            .unwrap();
+        let cache_root = self.cache_root.as_path_unlocked();
+        fs::create_dir_all(cache_root)?;
+        let mut conn = Connection::open(cache_root.join("index-cache.db"))?;
+        sqlite::migrate(&mut conn, &migrations())?;
+        Ok(conn)
+    }
+}
+
+impl CacheStore for LocalDatabase<'_> {
+    fn get(&self, key: &str) -> Option<Vec<u8>> {
+        self.conn()?
+            .prepare_cached("SELECT value FROM summaries WHERE name = ? LIMIT 1")
+            .and_then(|mut stmt| stmt.query_row([key], |row| row.get(0)))
+            .map_err(|e| tracing::debug!(key, "cache missing: {e}"))
+            .ok()
+    }
+
+    fn put(&self, key: &str, value: &[u8]) {
+        if let Some(conn) = self.conn() {
+            _ = conn
+                .prepare_cached("INSERT OR REPLACE INTO summaries (name, value) VALUES (?, ?)")
+                .and_then(|mut stmt| stmt.execute(params!(key, value)))
+                .map_err(|e| tracing::info!(key, "failed to write cache: {e}"));
+        }
+    }
+
+    fn invalidate(&self, key: &str) {
+        if let Some(conn) = self.conn() {
+            _ = conn
+                .prepare_cached("DELETE FROM summaries WHERE name = ?")
+                .and_then(|mut stmt| stmt.execute([key]))
+                .map_err(|e| tracing::debug!(key, "failed to remove from cache: {e}"));
+        }
+    }
+}
+
+/// Migrations which initialize the database, and can be used to evolve it over time.
+///
+/// See [`Migration`] for more detail.
+///
+/// **Be sure to not change the order or entries here!**
+fn migrations() -> Vec<Migration> {
+    vec![basic_migration(
+        "CREATE TABLE IF NOT EXISTS summaries (
+            name TEXT PRIMARY KEY NOT NULL,
+            value BLOB NOT NULL
+        )",
+    )]
+}

tests/testsuite/index_cache_sqlite.rs

@@ -0,0 +1,101 @@
+//! Tests for the `-Zindex-cache-sqlite`.
+
+use std::collections::HashSet;
+
+use cargo_test_support::paths;
+use cargo_test_support::project;
+use cargo_test_support::registry;
+use cargo_test_support::registry::Package;
+
+#[cargo_test]
+fn gated() {
+    project()
+        .build()
+        .cargo("fetch")
+        .arg("-Zindex-cache-sqlite")
+        .with_status(101)
+        .with_stderr_contains("[ERROR] the `-Z` flag is only accepted on the nightly channel of Cargo, but this is the `stable` channel")
+        .run();
+}
+
+#[cargo_test]
+fn crates_io() {
+    registry::alt_init();
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                edition = "2015"
+
+                [dependencies]
+                dep2 = "0.0.0"
+            "#,
+        )
+        .file("src/main.rs", "fn main() {}")
+        .build();
+
+    Package::new("dep1", "0.0.0").publish();
+    Package::new("dep2", "0.0.0").dep("dep1", "0.0.0").publish();
+    Package::new("dep3", "0.0.0").publish();
+
+    p.cargo("fetch")
+        .masquerade_as_nightly_cargo(&["index-cache-sqlite"])
+        .arg("-Zindex-cache-sqlite")
+        .with_stderr(
+            "\
+[UPDATING] `dummy-registry` index
+[DOWNLOADING] crates ...
+[DOWNLOADED] dep1 v0.0.0 (registry `dummy-registry`)
+[DOWNLOADED] dep2 v0.0.0 (registry `dummy-registry`)
+",
+        )
+        .run();
+
+    assert_rows_inserted(&["dep1", "dep2"]);
+
+    p.change_file(
+        "Cargo.toml",
+        r#"
+            [package]
+            name = "foo"
+            edition = "2015"
+
+            [dependencies]
+            dep2 = "0.0.0"
+            dep3 = "0.0.0"
+        "#,
+    );
+
+    p.cargo("fetch")
+        .masquerade_as_nightly_cargo(&["index-cache-sqlite"])
+        .arg("-Zindex-cache-sqlite")
+        .with_stderr(
+            "\
+[UPDATING] `dummy-registry` index
+[DOWNLOADING] crates ...
+[DOWNLOADED] dep3 v0.0.0 (registry `dummy-registry`)
+",
+        )
+        .run();
+
+    assert_rows_inserted(&["dep1", "dep2", "dep3"]);
+}
+
+#[track_caller]
+fn assert_rows_inserted(names: &[&str]) {
+    let pattern = paths::home().join(".cargo/registry/index/*/.cache/index-cache.db");
+    let pattern = pattern.to_str().unwrap();
+    let db_path = glob::glob(pattern).unwrap().next().unwrap().unwrap();
+
+    let set: HashSet<String> = rusqlite::Connection::open(&db_path)
+        .unwrap()
+        .prepare("SELECT name FROM summaries")
+        .unwrap()
+        .query_map([], |row| row.get(0))
+        .unwrap()
+        .collect::<Result<_, _>>()
+        .unwrap();
+    assert_eq!(set, HashSet::from_iter(names.iter().map(|n| n.to_string())));
+}

tests/testsuite/main.rs

@@ -103,6 +103,7 @@ mod glob_targets;
 mod global_cache_tracker;
 mod help;
 mod https;
+mod index_cache_sqlite;
 mod inheritable_workspace_fields;
 mod install;
 mod install_upgrade;