Merge #1: Improved documentation

Approved-by: Qqwy
Priority: Normal
Auto-deploy: false

Author: OpsBotPrime

opsqueue/src/common/chunk.rs

@@ -1,3 +1,8 @@
+//! Dealing with `Chunk``s, the most fine-grained datatype the queue itself works with.
+//!
+//! While the smallest datatype is the 'Operation', the Opsqueue queue binary itself
+//! only deals with _chunks_ of them. The operations inside of them are hidden and only read/written
+//! from/to object store all the way in the client.
 use chrono::{DateTime, Utc};
 
 use serde::{Deserialize, Serialize};

opsqueue/src/common/errors.rs

@@ -1,3 +1,11 @@
+//! Common error types
+//!
+//! You might notice in many places in Opsqueue that we use very fine-grained error types,
+//! and combine them together using the `E` helper.
+//!
+//! This is a conscious choice: While it makes some function signatures more complex,
+//! it allows us to be super precise in what kind of errors can and cannot occur
+//! in certain API calls.
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 

opsqueue/src/common/mod.rs

@@ -1,3 +1,4 @@
+//! Common datatypes and errors shared across all parts of Opsqueue
 use rustc_hash::FxHashMap;
 
 pub mod chunk;

opsqueue/src/common/submission.rs

@@ -1,3 +1,4 @@
+//! Dealing with `Submission`s: Collections of (`Chunks`s of) operations.
 use std::fmt::Display;
 use std::time::Duration;
 

opsqueue/src/config.rs

@@ -1,3 +1,7 @@
+//! Defines the source of truth for configuring the Opsqueue queue
+//!
+//! We make use of the excellent `clap` crate to make customizing the configuration
+//! with command-line args easier.
 use std::num::NonZero;
 
 use clap::Parser;

opsqueue/src/consumer/mod.rs

@@ -1,3 +1,4 @@
+//! The Consumer side: Interface to reserve, work on, and complete/fail individual Chunks
 pub mod common;
 pub mod strategy;
 

opsqueue/src/db/conn.rs

@@ -17,7 +17,7 @@ use super::{magic::*, Connection};
 /// You can get a `WriterPool` from [`DBPools::writer_pool`], and a [`ReaderPool`] from
 /// [`DBPools::reader_pool`].
 ///
-/// [`Pool`]: super::pool::Pool
+/// [`Pool`]: super::Pool
 /// [`ReaderPool`]: super::ReaderPool
 /// [`WriterPool`]: super::WriterPool
 /// [`DBPools::reader_pool`]: super::DBPools::reader_pool

opsqueue/src/db/mod.rs

@@ -276,7 +276,7 @@ impl WriterPool {
     ///
     /// See [`DBPools`] for further explanation about readers and writers.
     ///
-    /// [`DBPools`]: super::DBPools
+    /// [`DBPools`]: DBPools
     pub async fn writer_conn(&self) -> sqlx::Result<Writer<NoTransaction>> {
         self.acquire().await
     }
@@ -287,7 +287,7 @@ impl ReaderPool {
     ///
     /// See [`DBPools`] for further explanation about readers and writers.
     ///
-    /// [`DBPools`]: super::DBPools
+    /// [`DBPools`]: DBPools
     pub async fn reader_conn(&self) -> sqlx::Result<Reader<NoTransaction>> {
         self.acquire().await
     }

opsqueue/src/lib.rs

@@ -1,3 +1,24 @@
+//! Opsqueue: A lightweight batch processing queue for heavy loads
+//!
+//! Simple 'getting started' instructions can be found [in the repository README](https://github.com/channable/opsqueue/).
+//!
+//! The Rust codebase defines both the 'server', which makes up the bulk of the Opsqueue binary itself,
+//! and the 'client', which is the common part of functionality that clients written in different other programming languages
+//! can all use.
+//!
+//! Many datatypes are shared between this server and client, and therefore their code lives together in the same crate.
+//! Instead, we use feature-flags (`client-logic` and `server-logic`) to decide what concrete parts to include when building.
+//! The set of dependencies is based on these same feature-flags.
+//! Most interestingly, in the test suite we enable both feature-flags so we're able to do a bunch of round-trip testing
+//! immediately in Rust code.
+//!
+//! # Module setup
+//! - The basic logic is divided in the `producer` and `consumer` modules. These both have their own `db` submodule.
+//! - Common functionality and datatypes exists in the `common` module
+//! - Common database helpers live in the `db` module.
+//! - Reading/writing to object stores like GCS or S3 is abstracted in the `object_store` module.
+//! - Finally, extra modules to have a single source of truth for configuration of the queue, and to nicely do tracing and expose metrics exist.
+
 pub mod common;
 pub mod consumer;
 pub mod producer;

opsqueue/src/producer/mod.rs

@@ -1,3 +1,4 @@
+//! The Producer side: Interface to create and read submissions
 #[cfg(feature = "client-logic")]
 pub mod client;
 pub mod common;