Merge pull request #22 from griffi-gh/save-files

Save files

.gitignore

@@ -1,3 +1,5 @@
+.direnv
+
 # Generated by Cargo
 # will have compiled files and executables
 debug/
@@ -15,6 +17,7 @@ _src
 _visualizer.json
 
 *.kubi
+*.kbi
 
 /*_log*.txt
 /*.log
@@ -37,4 +40,4 @@ _visualizer.json
 
 /mods
 
-.direnv
+

Server.toml

@@ -4,6 +4,7 @@ max_clients = 32
 timeout_ms = 10000
 
 [world]
+file = "world.kubi"
 seed = 0xfeb_face_dead_cafe
 preheat_radius = 8
 

kubi-server/src/client.rs

@@ -50,7 +50,7 @@ pub fn sync_client_positions(
     };
 
     //log movement (annoying duh)
-    log::debug!("dbg: player moved id: {} coords: {} quat: {}", message.client_id, position, direction);
+    // log::debug!("dbg: player moved id: {} coords: {} quat: {}", message.client_id, position, direction);
 
     //Apply position to server-side client
     let mut trans = (&mut transforms).get(message.entity_id).unwrap();

kubi-server/src/config.rs

@@ -1,6 +1,6 @@
 use shipyard::{AllStoragesView, Unique};
 use serde::{Serialize, Deserialize};
-use std::{fs, net::SocketAddr};
+use std::{fs, net::SocketAddr, path::PathBuf};
 
 #[derive(Serialize, Deserialize)]
 pub struct ConfigTableServer {
@@ -12,6 +12,7 @@ pub struct ConfigTableServer {
 
 #[derive(Serialize, Deserialize)]
 pub struct ConfigTableWorld {
+  pub file: Option<PathBuf>,
   pub seed: u64,
   pub preheat_radius: u32,
 }

kubi-server/src/main.rs

@@ -1,5 +1,6 @@
-use shipyard::{IntoWorkload, Workload, WorkloadModificator, World};
+use shipyard::{IntoWorkload, SystemModificator, Workload, WorkloadModificator, World};
 use std::{thread, time::Duration};
+use kubi_shared::fixed_timestamp::{FixedTimestamp, init_fixed_timestamp_storage};
 
 mod util;
 mod config;
@@ -12,10 +13,11 @@ use config::read_config;
 use server::{bind_server, update_server, log_server_errors};
 use client::{init_client_maps, on_client_disconnect, sync_client_positions};
 use auth::authenticate_players;
-use world::{update_world, init_world};
+use world::{init_world, save::save_modified, update_world};
 
 fn initialize() -> Workload {
   (
+    init_fixed_timestamp_storage,
     read_config,
     bind_server,
     init_client_maps,
@@ -32,7 +34,10 @@ fn update() -> Workload {
       update_world,
       sync_client_positions,
       on_client_disconnect,
-    ).into_workload()
+    ).into_workload(),
+    save_modified
+      .into_workload()
+      .make_fixed(10000, 0),
   ).into_sequential_workload()
 }
 

kubi-server/src/world.rs

@@ -24,12 +24,13 @@ use crate::{
 
 pub mod chunk;
 pub mod tasks;
+pub mod save;
 
 use chunk::Chunk;
 
 use self::{
   tasks::{ChunkTaskManager, ChunkTask, ChunkTaskResponse, init_chunk_task_manager},
-  chunk::ChunkState
+  chunk::ChunkState,
 };
 
 #[derive(Unique, Default)]
@@ -106,7 +107,7 @@ fn process_chunk_requests(
       chunk.state = ChunkState::Loading;
       chunk.subscriptions.insert(message.client_id);
       chunk_manager.chunks.insert(chunk_position, chunk);
-      task_manager.spawn_task(ChunkTask::LoadChunk {
+      task_manager.run(ChunkTask::LoadChunk {
         position: chunk_position,
         seed: config.world.seed,
       });
@@ -249,7 +250,11 @@ fn process_block_queue(
     let Some(blocks) = &mut chunk.blocks else {
       return true
     };
-    blocks[block_position.x as usize][block_position.y as usize][block_position.z as usize] = item.block_type;
+    let block = &mut blocks[block_position.x as usize][block_position.y as usize][block_position.z as usize];
+    if item.block_type != *block {
+      *block = item.block_type;
+      chunk.data_modified = true;
+    }
     false
   });
   if initial_len != queue.queue.len() {
@@ -278,7 +283,7 @@ pub fn preheat_world(
         let mut chunk = Chunk::new();
         chunk.state = ChunkState::Loading;
         chunk_manager.chunks.insert(chunk_position, chunk);
-        task_manager.spawn_task(ChunkTask::LoadChunk {
+        task_manager.run(ChunkTask::LoadChunk {
           position: chunk_position,
           seed: config.world.seed,
         });
@@ -292,7 +297,7 @@ pub fn init_world() -> Workload {
     init_chunk_manager_and_block_queue.before_all(preheat_world),
     init_chunk_task_manager.before_all(preheat_world),
     preheat_world,
-  ).into_workload()
+  ).into_sequential_workload()
 }
 
 pub fn update_world() -> Workload {

kubi-server/src/world/chunk.rs

@@ -16,13 +16,16 @@ pub struct Chunk {
   pub state: ChunkState,
   pub blocks: Option<BlockData>,
   pub subscriptions: HashSet<ClientId, BuildNoHashHasher<ClientId>>,
+  pub data_modified: bool,
 }
+
 impl Chunk {
   pub fn new() -> Self {
     Self {
       state: ChunkState::Nothing,
       blocks: None,
       subscriptions: HashSet::with_capacity_and_hasher(4, BuildNoHashHasher::default()),
+      data_modified: false,
     }
   }
 }

kubi-server/src/world/save.rs

@@ -0,0 +1,43 @@
+use kubi_shared::data::{io_thread::IOThreadManager, open_local_save_file};
+use shipyard::{AllStoragesView, UniqueView, UniqueViewMut};
+use crate::config::ConfigTable;
+use super::{
+  tasks::{ChunkTask, ChunkTaskManager},
+  ChunkManager,
+};
+
+pub fn init_save_file(storages: &AllStoragesView) -> Option<IOThreadManager> {
+  let config = storages.borrow::<UniqueView<ConfigTable>>().unwrap();
+  if let Some(file_path) = &config.world.file {
+    log::info!("Initializing save file from {:?}", file_path);
+    let save = open_local_save_file(&file_path).unwrap();
+    Some(IOThreadManager::new(save))
+  } else {
+    log::warn!("No save file specified, world will not be saved");
+    None
+  }
+}
+
+pub fn save_modified(
+  mut chunks: UniqueViewMut<ChunkManager>,
+  ctm: UniqueView<ChunkTaskManager>,
+) {
+  log::info!("Saving...");
+  let mut amount_saved = 0;
+  for (position, chunk) in chunks.chunks.iter_mut() {
+    if chunk.data_modified {
+      let Some(data) = chunk.blocks.clone() else {
+        continue
+      };
+      ctm.run(ChunkTask::SaveChunk {
+        position: *position,
+        data: data,
+      });
+      chunk.data_modified = false;
+      amount_saved += 1;
+    }
+  }
+  if amount_saved > 0 {
+    log::info!("Queued {} chunks for saving", amount_saved);
+  }
+}

kubi-server/src/world/tasks.rs

@@ -4,16 +4,19 @@ use glam::IVec3;
 use rayon::{ThreadPool, ThreadPoolBuilder};
 use anyhow::Result;
 use kubi_shared::{
-  chunk::BlockData, 
-  worldgen::generate_world,
-  queue::QueuedBlock,
+  chunk::BlockData, data::io_thread::{IOCommand, IOResponse, IOThreadManager}, queue::QueuedBlock, worldgen::generate_world
 };
+use super::save::init_save_file;
 
 pub enum ChunkTask {
   LoadChunk {
     position: IVec3,
     seed: u64,
-  }
+  },
+  SaveChunk {
+    position: IVec3,
+    data: BlockData,
+  },
 }
 
 pub enum ChunkTaskResponse {
@@ -28,33 +31,74 @@ pub enum ChunkTaskResponse {
 pub struct ChunkTaskManager {
   channel: (Sender<ChunkTaskResponse>, Receiver<ChunkTaskResponse>),
   pool: ThreadPool,
+  iota: Option<IOThreadManager>,
 }
+
 impl ChunkTaskManager {
-  pub fn new() -> Result<Self> {
+  pub fn new(iota: Option<IOThreadManager>) -> Result<Self> {
     Ok(Self {
       channel: unbounded(),
-      pool: ThreadPoolBuilder::new().build()?
+      pool: ThreadPoolBuilder::new().build()?,
+      iota,
     })
   }
-  pub fn spawn_task(&self, task: ChunkTask) {
-    let sender = self.channel.0.clone();
-    self.pool.spawn(move || {
-      sender.send(match task {
-        ChunkTask::LoadChunk { position: chunk_position, seed } => {
-          //unwrap is fine because abort is not possible
-          let (blocks, queue) = generate_world(chunk_position, seed, None).unwrap();
-          ChunkTaskResponse::ChunkLoaded { chunk_position, blocks, queue }
+
+  pub fn run(&self, task: ChunkTask) {
+    match task {
+      ChunkTask::LoadChunk { position: chunk_position, seed } => {
+        // 1. Check if the chunk exists in the save file
+        if let ChunkTask::LoadChunk { position, .. } = &task {
+          if let Some(iota) = &self.iota {
+            if iota.chunk_exists(*position) {
+              iota.send(IOCommand::LoadChunk { position: *position });
+              return
+            }
+          }
         }
-      }).unwrap()
-    })
+
+        // 2. Generate the chunk if it doesn't exist
+        let sender = self.channel.0.clone();
+        self.pool.spawn(move || {
+          sender.send({
+            //unwrap is fine because abort is not possible
+            let (blocks, queue) = generate_world(chunk_position, seed, None).unwrap();
+            ChunkTaskResponse::ChunkLoaded { chunk_position, blocks, queue }
+          }).unwrap()
+        });
+      },
+      ChunkTask::SaveChunk { position, data } => {
+        // Save the chunk to the save file
+        if let Some(iota) = &self.iota {
+          iota.send(IOCommand::SaveChunk { position, data });
+        }
+      },
+    }
   }
+
   pub fn receive(&self) -> Option<ChunkTaskResponse> {
-    self.channel.1.try_recv().ok()
+    // Try to receive IO results first
+    // If there are none, try to receive worldgen results
+    self.iota.as_ref().map(|iota| {
+      iota.poll_single().map(|response| match response {
+        IOResponse::ChunkLoaded { position, data } => ChunkTaskResponse::ChunkLoaded {
+          chunk_position: position,
+          blocks: data.expect("chunk data exists in the header, but was not loaded"),
+          queue: Vec::with_capacity(0)
+        },
+        _ => panic!("Unexpected response from IO thread"),
+      })
+    }).flatten().or_else(|| {
+      self.channel.1.try_recv().ok()
+    })
   }
 }
 
 pub fn init_chunk_task_manager(
   storages: AllStoragesView
 ) {
-  storages.add_unique(ChunkTaskManager::new().expect("ChunkTaskManager Init failed"));
+  let iota = init_save_file(&storages);
+  storages.add_unique(
+    ChunkTaskManager::new(iota)
+      .expect("ChunkTaskManager Init failed")
+  );
 }

kubi-shared/Cargo.toml

@@ -14,6 +14,7 @@ serde = { version = "1.0", default-features = false, features = ["alloc", "deriv
 serde_with = "3.4"
 bincode = "1.3"
 anyhow = "1.0"
+flume = "0.11"
 fastnoise-lite = { version = "1.1", features = ["std", "f64"] }
 rand = { version = "0.8", default_features = false, features = ["std", "min_const_gen"] }
 rand_xoshiro = "0.6"
@@ -23,6 +24,7 @@ bytemuck = { version = "1.14", features = ["derive"] }
 static_assertions = "1.1"
 nz = "0.4"
 atomic = "0.6"
+log = "0.4"
 
 [features]
 default = []