fix: fixed log dir for aw-sync, misc sync fixes and refactor

aw-server/src/dirs.rs

@@ -53,9 +53,9 @@ pub fn get_cache_dir() -> Result<PathBuf, ()> {
 }
 
 #[cfg(not(target_os = "android"))]
-pub fn get_log_dir() -> Result<PathBuf, ()> {
+pub fn get_log_dir(module: &str) -> Result<PathBuf, ()> {
     let mut dir = appdirs::user_log_dir(Some("activitywatch"), None)?;
-    dir.push("aw-server-rust");
+    dir.push(module);
     fs::create_dir_all(dir.clone()).expect("Unable to create log dir");
     Ok(dir)
 }
@@ -87,7 +87,7 @@ fn test_get_dirs() {
     set_android_data_dir("/test");
 
     get_cache_dir().unwrap();
-    get_log_dir().unwrap();
+    get_log_dir("aw-server-rust").unwrap();
     db_path(true).unwrap();
     db_path(false).unwrap();
 }

aw-server/src/logging.rs

@@ -5,19 +5,17 @@ use fern::colors::{Color, ColoredLevelConfig};
 
 use crate::dirs;
 
-pub fn setup_logger(testing: bool, verbose: bool) -> Result<(), fern::InitError> {
+pub fn setup_logger(module: &str, testing: bool, verbose: bool) -> Result<(), fern::InitError> {
     let mut logfile_path: PathBuf =
-        dirs::get_log_dir().expect("Unable to get log dir to store logs in");
+        dirs::get_log_dir(module).expect("Unable to get log dir to store logs in");
     fs::create_dir_all(logfile_path.clone()).expect("Unable to create folder for logs");
-    logfile_path.push(
-        chrono::Local::now()
-            .format(if !testing {
-                "aw-server_%Y-%m-%dT%H-%M-%S%z.log"
-            } else {
-                "aw-server-testing_%Y-%m-%dT%H-%M-%S%z.log"
-            })
-            .to_string(),
-    );
+    let filename = if !testing {
+        format!("{}_%Y-%m-%dT%H-%M-%S%z.log", module)
+    } else {
+        format!("{}-testing_%Y-%m-%dT%H-%M-%S%z.log", module)
+    };
+
+    logfile_path.push(chrono::Local::now().format(&filename).to_string());
 
     log_panics::init();
 
@@ -93,6 +91,6 @@ mod tests {
     #[ignore]
     #[test]
     fn test_setup_logger() {
-        setup_logger(true, true).unwrap();
+        setup_logger("aw-server-rust", true, true).unwrap();
     }
 }

aw-server/src/main.rs

@@ -70,7 +70,8 @@ async fn main() -> Result<(), rocket::Error> {
         testing = true;
     }
 
-    logging::setup_logger(testing, opts.verbose).expect("Failed to setup logging");
+    logging::setup_logger("aw-server-rust", testing, opts.verbose)
+        .expect("Failed to setup logging");
 
     if testing {
         info!("Running server in Testing mode");

aw-sync/src/main.rs

@@ -107,7 +107,8 @@ fn main() -> Result<(), Box<dyn Error>> {
 
     info!("Started aw-sync...");
 
-    aw_server::logging::setup_logger(true, verbose).expect("Failed to setup logging");
+    aw_server::logging::setup_logger("aw-sync", opts.testing, verbose)
+        .expect("Failed to setup logging");
 
     let port = opts
         .port
@@ -129,6 +130,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                     }
                 }
                 None => {
+                    info!("Pulling from all hosts");
                     sync_wrapper::pull_all(&client)?;
                 }
             }
@@ -147,7 +149,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             sync_db,
         } => {
             let sync_directory = if sync_dir.is_empty() {
-                println!("No sync directory specified, exiting...");
+                error!("No sync directory specified, exiting...");
                 std::process::exit(1);
             } else {
                 Path::new(&sync_dir)

aw-sync/src/sync.rs

@@ -9,7 +9,6 @@ extern crate chrono;
 extern crate reqwest;
 extern crate serde_json;
 
-use std::ffi::OsStr;
 use std::fs;
 use std::path::{Path, PathBuf};
 
@@ -65,7 +64,7 @@ pub fn sync_run(client: &AwClient, sync_spec: &SyncSpec, mode: SyncMode) -> Resu
 
     // FIXME: Bad device_id assumption?
     let ds_localremote = setup_local_remote(sync_spec.path.as_path(), device_id)?;
-    let remote_dbfiles = find_remotes_nonlocal(
+    let remote_dbfiles = crate::util::find_remotes_nonlocal(
         sync_spec.path.as_path(),
         device_id,
         sync_spec.path_db.as_ref(),
@@ -138,7 +137,7 @@ pub fn list_buckets(client: &AwClient) -> Result<(), String> {
     let device_id = info.device_id.as_str();
     let ds_localremote = setup_local_remote(sync_directory, device_id)?;
 
-    let remote_dbfiles = find_remotes_nonlocal(sync_directory, device_id, None);
+    let remote_dbfiles = crate::util::find_remotes_nonlocal(sync_directory, device_id, None);
     info!("Found remotes: {:?}", remote_dbfiles);
 
     // TODO: Check for compatible remote db version before opening
@@ -175,47 +174,6 @@ fn setup_local_remote(path: &Path, device_id: &str) -> Result<Datastore, String>
     Ok(ds_localremote)
 }
 
-/// Returns a list of all remote dbs
-fn find_remotes(sync_directory: &Path) -> std::io::Result<Vec<PathBuf>> {
-    let dbs = fs::read_dir(sync_directory)?
-        .map(|res| res.ok().unwrap().path())
-        .filter(|p| p.is_dir())
-        .flat_map(|d| fs::read_dir(d).unwrap())
-        .map(|res| res.ok().unwrap().path())
-        .filter(|path| path.extension().unwrap_or_else(|| OsStr::new("")) == "db")
-        .collect();
-    Ok(dbs)
-}
-
-/// Returns a list of all remotes, excluding local ones
-fn find_remotes_nonlocal(
-    sync_directory: &Path,
-    device_id: &str,
-    sync_db: Option<&PathBuf>,
-) -> Vec<PathBuf> {
-    let remotes_all = find_remotes(sync_directory).unwrap();
-    remotes_all
-        .into_iter()
-        // Filter out own remote
-        .filter(|path| {
-            !(path
-                .clone()
-                .into_os_string()
-                .into_string()
-                .unwrap()
-                .contains(device_id))
-        })
-        // If sync_db is Some, return only remotes in that path
-        .filter(|path| {
-            if let Some(sync_db) = sync_db {
-                path.starts_with(sync_db)
-            } else {
-                true
-            }
-        })
-        .collect()
-}
-
 pub fn create_datastore(path: &Path) -> Datastore {
     let pathstr = path.as_os_str().to_str().unwrap();
     Datastore::new(pathstr.to_string(), false)

aw-sync/src/sync_wrapper.rs

@@ -15,10 +15,13 @@ pub fn pull_all(client: &AwClient) -> Result<(), Box<dyn Error>> {
 }
 
 pub fn pull(host: &str, client: &AwClient) -> Result<(), Box<dyn Error>> {
-    info!("Pulling data from sync server {}", host);
-
     // Check if server is running
-    if TcpStream::connect(client.baseurl.clone()).is_err() {
+    let parts: Vec<&str> = client.baseurl.split("://").collect();
+    let host_parts: Vec<&str> = parts[1].split(':').collect();
+    let addr = host_parts[0];
+    let port = host_parts[1].parse::<u16>().unwrap();
+
+    if TcpStream::connect((addr, port)).is_err() {
         return Err(format!("Local server {} not running", &client.baseurl).into());
     }
 
@@ -41,14 +44,21 @@ pub fn pull(host: &str, client: &AwClient) -> Result<(), Box<dyn Error>> {
 
     // filter out dbs that are smaller than 50kB (workaround for trying to sync empty database
     // files that are spuriously created somewhere)
-    let dbs = dbs
+    let mut dbs = dbs
         .into_iter()
         .filter(|entry| entry.metadata().map(|m| m.len() > 50_000).unwrap_or(false))
         .collect::<Vec<_>>();
 
-    // if more than one db, error
+    // if more than one db, warn and use the largest one
     if dbs.len() > 1 {
-        return Err("More than one db found in sync folder".into());
+        warn!(
+            "More than one db found in sync folder for host, choosing largest db {:?}",
+            dbs
+        );
+        dbs = vec![dbs
+            .into_iter()
+            .max_by_key(|entry| entry.metadata().map(|m| m.len()).unwrap_or(0))
+            .unwrap()];
     }
     // if no db, error
     if dbs.is_empty() {
@@ -65,7 +75,6 @@ pub fn pull(host: &str, client: &AwClient) -> Result<(), Box<dyn Error>> {
             ]),
             start: None,
         };
-        info!("Pulling data with spec {:?}", sync_spec);
         sync_run(client, &sync_spec, SyncMode::Pull)?;
     }
 

aw-sync/src/util.rs

@@ -1,8 +1,10 @@
 use std::boxed::Box;
 use std::error::Error;
+use std::ffi::OsStr;
 use std::fs;
 use std::fs::File;
 use std::io::Read;
+use std::path::{Path, PathBuf};
 
 pub fn get_hostname() -> Result<String, Box<dyn Error>> {
     let hostname = gethostname::gethostname()
@@ -33,12 +35,42 @@ pub fn get_server_port(testing: bool) -> Result<u16, Box<dyn Error>> {
     Ok(port)
 }
 
+/// Check if a directory contains a .db file
+fn contains_db_file(dir: &std::path::Path) -> bool {
+    fs::read_dir(dir)
+        .ok()
+        .map(|entries| {
+            entries.filter_map(Result::ok).any(|entry| {
+                entry
+                    .path()
+                    .extension()
+                    .map(|ext| ext == "db")
+                    .unwrap_or(false)
+            })
+        })
+        .unwrap_or(false)
+}
+
+/// Check if a directory contains a subdirectory that contains a .db file
+fn contains_subdir_with_db_file(dir: &std::path::Path) -> bool {
+    fs::read_dir(dir)
+        .ok()
+        .map(|entries| {
+            entries
+                .filter_map(Result::ok)
+                .any(|entry| entry.path().is_dir() && contains_db_file(&entry.path()))
+        })
+        .unwrap_or(false)
+}
+
 /// Return all remotes in the sync folder
+/// Only returns folders that match ./{host}/{device_id}/*.db
+// TODO: share logic with find_remotes and find_remotes_nonlocal
 pub fn get_remotes() -> Result<Vec<String>, Box<dyn Error>> {
     let sync_root_dir = crate::dirs::get_sync_dir().map_err(|_| "Could not get sync dir")?;
     let hostnames = fs::read_dir(sync_root_dir)?
         .filter_map(Result::ok)
-        .filter(|entry| entry.path().is_dir())
+        .filter(|entry| entry.path().is_dir() && contains_subdir_with_db_file(&entry.path()))
         .filter_map(|entry| {
             entry
                 .path()
@@ -49,3 +81,44 @@ pub fn get_remotes() -> Result<Vec<String>, Box<dyn Error>> {
     info!("Found remotes: {:?}", hostnames);
     Ok(hostnames)
 }
+
+/// Returns a list of all remote dbs
+fn find_remotes(sync_directory: &Path) -> std::io::Result<Vec<PathBuf>> {
+    let dbs = fs::read_dir(sync_directory)?
+        .map(|res| res.ok().unwrap().path())
+        .filter(|p| p.is_dir())
+        .flat_map(|d| fs::read_dir(d).unwrap())
+        .map(|res| res.ok().unwrap().path())
+        .filter(|path| path.extension().unwrap_or_else(|| OsStr::new("")) == "db")
+        .collect();
+    Ok(dbs)
+}
+
+/// Returns a list of all remotes, excluding local ones
+pub fn find_remotes_nonlocal(
+    sync_directory: &Path,
+    device_id: &str,
+    sync_db: Option<&PathBuf>,
+) -> Vec<PathBuf> {
+    let remotes_all = find_remotes(sync_directory).unwrap();
+    remotes_all
+        .into_iter()
+        // Filter out own remote
+        .filter(|path| {
+            !(path
+                .clone()
+                .into_os_string()
+                .into_string()
+                .unwrap()
+                .contains(device_id))
+        })
+        // If sync_db is Some, return only remotes in that path
+        .filter(|path| {
+            if let Some(sync_db) = sync_db {
+                path.starts_with(sync_db)
+            } else {
+                true
+            }
+        })
+        .collect()
+}