feat(project): start saving nix_file in database

In addition to updating the backlink to nix_file, we also update the gc_roots
table in our database to contain the same info.

The next step is to read everything from the database instead of from the
symlinks.

Currently, the gc_roots table is deleted at every start (to make changes to the
schema easier), after everything works through the database, we have to set up a
one-time migration (and write a test for it …) and clean up the symlinks.

src/daemon.rs

@@ -8,6 +8,7 @@ use crate::nix::options::NixOptions;
 use crate::ops::error::ExitError;
 use crate::socket::communicate;
 use crate::socket::path::SocketPath;
+use crate::sqlite::Sqlite;
 use crate::{project, AbsPathBuf, NixFile};
 use crossbeam_channel as chan;
 use slog::debug;
@@ -71,6 +72,7 @@ impl Daemon {
     pub fn serve(
         &mut self,
         socket_path: &SocketPath,
+        sqlite_path: &AbsPathBuf,
         gc_root_dir: &AbsPathBuf,
         cas: crate::cas::ContentAddressable,
         nix_gc_root_user_dir: project::NixGcRootUserDir,
@@ -105,12 +107,15 @@ impl Daemon {
         let tx_build_events = self.tx_build_events.clone();
         let extra_nix_options = self.extra_nix_options.clone();
         let gc_root_dir = gc_root_dir.clone();
+        let sqlite_path = sqlite_path.clone();
         pool.spawn("build-instruction-handler", move || {
+            let mut conn = Sqlite::new_connection(&sqlite_path);
             Self::build_instruction_handler(
                 tx_build_events,
                 extra_nix_options,
                 rx_activity,
                 &gc_root_dir,
+                &mut conn,
                 cas,
                 nix_gc_root_user_dir,
                 &logger3,
@@ -177,6 +182,7 @@ impl Daemon {
         extra_nix_options: NixOptions,
         rx_activity: chan::Receiver<IndicateActivity>,
         gc_root_dir: &AbsPathBuf,
+        conn: &mut Sqlite,
         cas: crate::cas::ContentAddressable,
         nix_gc_root_user_dir: project::NixGcRootUserDir,
         logger: &slog::Logger,
@@ -187,9 +193,10 @@ impl Daemon {
         // For each build instruction, add the corresponding file
         // to the watch list.
         for IndicateActivity { nix_file, rebuild } in rx_activity {
-            let project = crate::project::Project::new_and_gc_nix_files(nix_file, gc_root_dir)
-                // TODO: the project needs to create its gc root dir
-                .unwrap();
+            let project =
+                crate::project::Project::new_and_gc_nix_files(conn, nix_file, gc_root_dir)
+                    // TODO: the project needs to create its gc root dir
+                    .unwrap();
 
             let key = project.nix_file.clone();
             let project_is_watched = handler_threads.get(&key);

src/main.rs

@@ -4,8 +4,8 @@ use lorri::ops;
 use lorri::ops::error::ExitError;
 use lorri::project::Project;
 use lorri::sqlite::Sqlite;
+use lorri::AbsPathBuf;
 use lorri::NixFile;
-use lorri::{constants, AbsPathBuf};
 use slog::{debug, error, o};
 use std::env;
 use std::path::{Path, PathBuf};
@@ -87,27 +87,30 @@ fn find_nix_file(shellfile: &Path) -> Result<NixFile, ExitError> {
     }
 }
 
-fn create_project(paths: &constants::Paths, shell_nix: NixFile) -> Result<Project, ExitError> {
-    Project::new_and_gc_nix_files(shell_nix, paths.gc_root_dir()).map_err(|err| {
-        ExitError::temporary(anyhow::anyhow!(err).context("Could not set up project paths"))
-    })
-}
-
 /// Run the main function of the relevant command.
 fn run_command(orig_logger: &slog::Logger, opts: Arguments) -> Result<(), ExitError> {
     let paths = lorri::ops::get_paths()?;
-    let conn = Sqlite::new_connection(&paths.sqlite_db);
+    let mut conn = Sqlite::new_connection(&paths.sqlite_db);
 
     // TODO: TMP
     conn.migrate_gc_roots(&orig_logger, &paths).unwrap();
 
-    let with_project_resolved =
+    let mut with_project_resolved =
         |nix_file| -> std::result::Result<(Project, slog::Logger), ExitError> {
-            let project = create_project(&lorri::ops::get_paths()?, nix_file)?;
+            let project = {
+                let paths = &lorri::ops::get_paths()?;
+                Project::new_and_gc_nix_files(&mut conn, nix_file, paths.gc_root_dir()).map_err(
+                    |err| {
+                        ExitError::temporary(
+                            anyhow::anyhow!(err).context("Could not set up project paths"),
+                        )
+                    },
+                )
+            }?;
             let logger = orig_logger.new(o!("nix_file" => project.nix_file.clone()));
             Ok((project, logger))
         };
-    let with_project = |nix_file| -> std::result::Result<(Project, slog::Logger), ExitError> {
+    let mut with_project = |nix_file| -> std::result::Result<(Project, slog::Logger), ExitError> {
         with_project_resolved(find_nix_file(nix_file)?)
     };
 

src/ops.rs

@@ -26,6 +26,7 @@ use crate::project::GcRootInfo;
 use crate::project::{NixGcRootUserDir, Project};
 use crate::run_async::Async;
 use crate::socket::path::SocketPath;
+use crate::sqlite::Sqlite;
 use crate::NixFile;
 use crate::VERSION_BUILD_REV;
 use crate::{builder, project};
@@ -93,6 +94,7 @@ pub fn op_daemon(opts: crate::cli::DaemonOptions, logger: &slog::Logger) -> Resu
     let paths = crate::ops::get_paths()?;
     daemon.serve(
         &SocketPath::from(paths.daemon_socket_file().clone()),
+        &paths.sqlite_db,
         paths.gc_root_dir(),
         paths.cas_store().clone(),
         nix_gc_root_user_dir,
@@ -967,6 +969,7 @@ pub fn op_gc(
     opts: crate::cli::GcOptions,
 ) -> Result<(), ExitError> {
     let infos = Project::list_roots(logger, paths)?;
+    let mut conn = Sqlite::new_connection(&paths.sqlite_db);
     match opts.action {
         cli::GcSubcommand::Info => {
             if opts.json {
@@ -1024,7 +1027,7 @@ pub fn op_gc(
                 }
             } else {
                 for (info, project) in to_remove {
-                    match project.remove_project() {
+                    match project.remove_project(&mut conn) {
                         Ok(()) => result.push(Ok(info)),
                         Err(e) => result.push(Err((info, e.to_string()))),
                     }

src/project.rs

@@ -1,11 +1,14 @@
 //! Wrap a nix file and manage corresponding state.
 
+use anyhow::Context;
+use rusqlite::named_params;
 use slog::{debug, warn};
 use thiserror::Error;
 
 use crate::builder::{OutputPath, RootedPath};
 use crate::constants::Paths;
 use crate::ops::error::ExitError;
+use crate::sqlite::Sqlite;
 use crate::{pretty_time_ago, AbsPathBuf, NixFile};
 use std::ffi::{CString, OsString};
 use std::os::unix::ffi::OsStrExt;
@@ -31,38 +34,48 @@ impl Project {
     /// and the base GC root directory
     /// (as returned by `Paths.gc_root_dir()`),
     pub fn new_and_gc_nix_files(
+        conn: &mut Sqlite,
         nix_file: NixFile,
         gc_root_dir: &AbsPathBuf,
-    ) -> std::io::Result<Project> {
+    ) -> anyhow::Result<Project> {
         let p = Self::new_internal(nix_file.clone(), gc_root_dir)?;
 
         // Adjust the nix_file symlink to point to this project’s nix file
-
-        let nix_file_symlink = p.nix_file_backlink();
-        let (remove, create) = match std::fs::read_link(&nix_file_symlink) {
-            Ok(path) => {
-                if path == nix_file.as_absolute_path() {
-                    (false, false)
-                } else {
-                    (true, true)
+        conn.in_transaction(|t| {
+            dbg!("inserting new project into db for {}", p.nix_file.display());
+            t.execute(
+                r#"
+              INSERT INTO gc_roots (nix_file)
+              VALUES (:nix_file)
+              ON CONFLICT (nix_file) DO NOTHING
+            "#,
+                named_params!(":nix_file": p.nix_file.to_sql()),
+            )?;
+            let nix_file_symlink = p.nix_file_backlink();
+            let (remove, create) = match std::fs::read_link(&nix_file_symlink) {
+                Ok(path) => {
+                    if path == nix_file.as_absolute_path() {
+                        (false, false)
+                    } else {
+                        (true, true)
+                    }
                 }
-            }
-            Err(e) => {
-                if e.kind() == std::io::ErrorKind::NotFound {
-                    (false, true)
-                } else {
-                    (true, true)
+                Err(e) => {
+                    if e.kind() == std::io::ErrorKind::NotFound {
+                        (false, true)
+                    } else {
+                        (true, true)
+                    }
                 }
+            };
+            if remove {
+                std::fs::remove_file(&nix_file_symlink)?;
+            };
+            if create {
+                std::os::unix::fs::symlink(nix_file.as_absolute_path(), &nix_file_symlink)?;
             }
-        };
-        if remove {
-            std::fs::remove_file(&nix_file_symlink)?;
-        }
-        if create {
-            std::os::unix::fs::symlink(nix_file.as_absolute_path(), &nix_file_symlink)?;
-        }
-
-        Ok(p)
+            Ok(p)
+        })?
     }
 
     fn new_internal(nix_file: NixFile, gc_root_dir: &AbsPathBuf) -> std::io::Result<Project> {
@@ -188,8 +201,19 @@ where {
     }
 
     /// Removes this project from lorri. Removes the GC root and consumes the project.
-    pub fn remove_project(self) -> std::io::Result<()> {
-        std::fs::remove_dir_all(self.project_root_dir)
+    pub fn remove_project(self, conn: &mut Sqlite) -> anyhow::Result<()> {
+        conn.in_transaction(|t| {
+            std::fs::remove_dir_all(&self.project_root_dir).context(format!(
+                "Unable to remove the project directory from {}",
+                self.project_root_dir.display()
+            ))?;
+
+            t.execute(
+                "DELETE FROM gc_roots WHERE nix_file = :nix_file",
+                named_params!(":nix_file": self.nix_file.to_sql()),
+            )?;
+            Ok(())
+        })?
     }
 
     /// Returns a list of existing gc roots along with some metadata

src/sqlite.rs

@@ -1,13 +1,8 @@
 //! TODO
-use std::{
-    ffi::OsString,
-    os::unix::ffi::OsStringExt,
-    time::{Duration, SystemTime},
-};
+use std::time::SystemTime;
 
 use anyhow::Context;
 use rusqlite as sqlite;
-use slog::info;
 use sqlite::named_params;
 
 use crate::{constants::Paths, ops::error::ExitError, project::Project, AbsPathBuf};
@@ -38,6 +33,8 @@ impl Sqlite {
     pub fn migrate_gc_roots(&self, logger: &slog::Logger, paths: &Paths) -> Result<(), ExitError> {
         let infos = Project::list_roots(logger, paths)?;
 
+        self.conn.execute("DELETE FROM gc_roots", ()).unwrap();
+
         let mut stmt = self
             .conn
             .prepare(
@@ -61,35 +58,35 @@ impl Sqlite {
             .expect("cannot insert");
         }
 
-        let mut stmt = self
-            .conn
-            .prepare("SELECT nix_file, last_updated from gc_roots")
-            .unwrap();
-        let mut res = stmt
-            .query_map((), |row| {
-                let nix_file = row
-                    .get::<_, Option<Vec<u8>>>("nix_file")
-                    .unwrap()
-                    .map(|v: Vec<u8>| OsString::from_vec(v));
-                let t = row.get::<_, Option<u64>>("last_updated").unwrap().map(|u| {
-                    SystemTime::elapsed(&(SystemTime::UNIX_EPOCH + Duration::from_secs(u))).unwrap()
-                });
-                Ok((nix_file, t, t.map(ago)))
-            })
-            .unwrap()
-            .filter_map(|r| match r {
-                Err(_) => None,
-                Ok(r) => r.0.map(|nix| (nix, r.1, r.2)),
-            })
-            .collect::<Vec<_>>();
-        res.sort_by_key(|r| r.1);
-        info!(logger, "We have these nix files: {:#?}", res);
+        // let mut stmt = self
+        //     .conn
+        //     .prepare("SELECT nix_file, last_updated from gc_roots")
+        //     .unwrap();
+        // let mut res = stmt
+        //     .query_map((), |row| {
+        //         let nix_file = row
+        //             .get::<_, Option<Vec<u8>>>("nix_file")
+        //             .unwrap()
+        //             .map(|v: Vec<u8>| OsString::from_vec(v));
+        //         let t = row.get::<_, Option<u64>>("last_updated").unwrap().map(|u| {
+        //             SystemTime::elapsed(&(SystemTime::UNIX_EPOCH + Duration::from_secs(u))).unwrap()
+        //         });
+        //         Ok((nix_file, t, t.map(ago)))
+        //     })
+        //     .unwrap()
+        //     .filter_map(|r| match r {
+        //         Err(_) => None,
+        //         Ok(r) => r.0.map(|nix| (nix, r.1, r.2)),
+        //     })
+        //     .collect::<Vec<_>>();
+        // res.sort_by_key(|r| r.1);
+        // info!(logger, "We have these nix files: {:#?}", res);
 
         Ok(())
     }
 
     /// Run the given code in the context of a transaction, automatically aborting the transaction if the function returns `Err`, comitting if it returns `Ok`.
-    pub fn in_transaction<F, A, E>(mut self, f: F) -> anyhow::Result<Result<A, E>>
+    pub fn in_transaction<F, A, E>(&mut self, f: F) -> anyhow::Result<Result<A, E>>
     where
         F: FnOnce(&sqlite::Transaction) -> Result<A, E>,
     {
@@ -109,22 +106,3 @@ impl Sqlite {
             .context("executing sqlite transaction failed")
     }
 }
-
-fn ago(dur: Duration) -> String {
-    let secs = dur.as_secs();
-    let mins = dur.as_secs() / 60;
-    let hours = dur.as_secs() / (60 * 60);
-    let days = dur.as_secs() / (60 * 60 * 24);
-
-    if days > 0 {
-        return format!("{} days ago", days);
-    }
-    if hours > 0 {
-        return format!("{} hours ago", hours);
-    }
-    if mins > 0 {
-        return format!("{} minutes ago", mins);
-    }
-
-    format!("{} seconds ago", secs)
-}

tests/integration/direnvtestcase.rs

@@ -9,6 +9,7 @@ use lorri::nix::options::NixOptions;
 use lorri::ops;
 use lorri::project;
 use lorri::project::Project;
+use lorri::sqlite::Sqlite;
 use lorri::AbsPathBuf;
 use lorri::NixFile;
 
@@ -41,8 +42,13 @@ impl DirenvTestCase {
         let shell_file = NixFile::from(AbsPathBuf::new(test_root.join("shell.nix")).unwrap());
 
         let cas = ContentAddressable::new(cachedir.join("cas").to_owned()).unwrap();
-        let project =
-            Project::new_and_gc_nix_files(shell_file.clone(), &cachedir.join("gc_roots")).unwrap();
+        let mut conn = Sqlite::new_connection(&cachedir.join("sqlite"));
+        let project = Project::new_and_gc_nix_files(
+            &mut conn,
+            shell_file.clone(),
+            &cachedir.join("gc_roots"),
+        )
+        .unwrap();
 
         DirenvTestCase {
             projectdir,