Add documentation generation for datafusion tables.

To test it: `cargo xtask generate-table-docs`

crates/storage-query-datafusion/Cargo.toml

@@ -10,6 +10,7 @@ publish = false
 [features]
 default = []
 options_schema = ["dep:schemars"]
+table_docs = []
 
 [dependencies]
 restate-core = { workspace = true }

crates/storage-query-datafusion/src/deployment/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/deployment/schema.rs

@@ -15,8 +15,8 @@ use crate::table_macro::*;
 use datafusion::arrow::datatypes::DataType;
 
 define_table!(deployment(
-    id: DataType::LargeUtf8,
-    ty: DataType::LargeUtf8,
-    endpoint: DataType::LargeUtf8,
-    created_at: DataType::Date64,
+    id "The ID of the service deployment.": DataType::LargeUtf8,
+    ty "The type of the endpoint. Either `http` or `lambda`.": DataType::LargeUtf8,
+    endpoint "The address of the endpoint. Either HTTP URL or Lambda ARN.": DataType::LargeUtf8,
+    created_at "Timestamp indicating the deployment registration time.": DataType::Date64,
 ));

crates/storage-query-datafusion/src/idempotency/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/idempotency/schema.rs

@@ -15,12 +15,12 @@ use crate::table_macro::*;
 use datafusion::arrow::datatypes::DataType;
 
 define_table!(idempotency(
-    partition_key: DataType::UInt64,
+    partition_key "Internal column that is used for partitioning the services invocations. Can be ignored.": DataType::UInt64,
 
-    service_name: DataType::LargeUtf8,
-    service_key: DataType::LargeUtf8,
-    service_handler: DataType::LargeUtf8,
-    idempotency_key: DataType::LargeUtf8,
+    service_name "The name for the invoked service.": DataType::LargeUtf8,
+    service_key "The key of the Virtual Object or of the Workflow. Null for regular services.": DataType::LargeUtf8,
+    service_handler "The invoked handler.": DataType::LargeUtf8,
+    idempotency_key "The user provided idempotency key.": DataType::LargeUtf8,
 
-    invocation_id: DataType::LargeUtf8
+    invocation_id "[Invocation ID](/operate/invocation#invocation-identifier).": DataType::LargeUtf8
 ));

crates/storage-query-datafusion/src/inbox/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/inbox/schema.rs

@@ -15,14 +15,14 @@ use crate::table_macro::*;
 use datafusion::arrow::datatypes::DataType;
 
 define_table!(inbox(
-    partition_key: DataType::UInt64,
+    partition_key "Internal column that is used for partitioning the services invocations. Can be ignored.": DataType::UInt64,
 
-    service_name: DataType::LargeUtf8,
-    service_key: DataType::LargeUtf8,
+    service_name "The name for the invoked virtual object/workflow.": DataType::LargeUtf8,
+    service_key "The key of the virtual object/workflow.": DataType::LargeUtf8,
 
-    id: DataType::LargeUtf8,
+    id "[Invocation ID](/operate/invocation#invocation-identifier).": DataType::LargeUtf8,
 
-    sequence_number: DataType::UInt64,
+    sequence_number "Sequence number in the inbox.": DataType::UInt64,
 
-    created_at: DataType::Date64,
+    created_at "Timestamp indicating the start of this invocation.": DataType::Date64,
 ));

crates/storage-query-datafusion/src/invocation_state/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/invocation_state/schema.rs

@@ -15,23 +15,23 @@ use crate::table_macro::*;
 use datafusion::arrow::datatypes::DataType;
 
 define_table!(state(
-    partition_key: DataType::UInt64,
-    id: DataType::LargeUtf8,
+    partition_key "Internal column that is used for partitioning the services invocations. Can be ignored.": DataType::UInt64,
+    id "[Invocation ID](/operate/invocation#invocation-identifier).": DataType::LargeUtf8,
 
-    in_flight: DataType::Boolean,
-    retry_count: DataType::UInt64,
-    last_start_at: DataType::Date64,
+    in_flight "If true, the invocation is currently in-flight": DataType::Boolean,
+    retry_count "The number of attempts since the last successful attempt of this invocation. Increments on start, so 2 or more means a failure occurred.": DataType::UInt64,
+    last_start_at "Timestamp indicating the start of the most recent attempt of this invocation.": DataType::Date64,
 
     // The deployment that was selected in the last invocation attempt. This is
     // guaranteed to be set unlike in `sys_status` table which require that the
     // deployment to be committed before it is set.
-    last_attempt_deployment_id: DataType::LargeUtf8,
-    last_attempt_server: DataType::LargeUtf8,
-    next_retry_at: DataType::Date64,
+    last_attempt_deployment_id "The opaque service deployment ID that was used in the most recent attempt of this invocation; this will be set before a journal entry is stored, but can change later.": DataType::LargeUtf8,
+    last_attempt_server "Server/SDK version, e.g. `restate-sdk-java/1.0.1`": DataType::LargeUtf8,
+    next_retry_at "Timestamp indicating the start of the next attempt of this invocation.": DataType::Date64,
 
-    last_failure: DataType::LargeUtf8,
-    last_failure_error_code: DataType::LargeUtf8,
-    last_failure_related_entry_index: DataType::UInt64,
-    last_failure_related_entry_name: DataType::LargeUtf8,
-    last_failure_related_entry_type: DataType::LargeUtf8,
+    last_failure "An error message describing the most recent failed attempt of this invocation, if any.": DataType::LargeUtf8,
+    last_failure_error_code "The error code of the most recent failed attempt of this invocation, if any.": DataType::LargeUtf8,
+    last_failure_related_entry_index "The index of the journal entry that caused the failure, if any. It may be out-of-bound of the currently stored entries in `sys_journal`.": DataType::UInt64,
+    last_failure_related_entry_name "The name of the journal entry that caused the failure, if any.": DataType::LargeUtf8,
+    last_failure_related_entry_type "The type of the journal entry that caused the failure, if any. You can check all the available entry types in [`entries.rs`](https://github.com/restatedev/restate/blob/main/crates/types/src/journal/entries.rs).": DataType::LargeUtf8,
 ));

crates/storage-query-datafusion/src/invocation_status/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/invocation_status/schema.rs

@@ -15,25 +15,25 @@ use crate::table_macro::*;
 use datafusion::arrow::datatypes::DataType;
 
 define_table!(invocation_status(
-    partition_key: DataType::UInt64,
-    id: DataType::LargeUtf8,
-
-    status: DataType::LargeUtf8,
-
-    target: DataType::LargeUtf8,
-    target_service_name: DataType::LargeUtf8,
-    target_service_key: DataType::LargeUtf8,
-    target_handler_name: DataType::LargeUtf8,
-    target_service_ty: DataType::LargeUtf8,
-
-    invoked_by: DataType::LargeUtf8,
-    invoked_by_service_name: DataType::LargeUtf8,
-    invoked_by_id: DataType::LargeUtf8,
-    invoked_by_target: DataType::LargeUtf8,
-
-    pinned_deployment_id: DataType::LargeUtf8,
-    trace_id: DataType::LargeUtf8,
-    journal_size: DataType::UInt32,
-    created_at: DataType::Date64,
-    modified_at: DataType::Date64,
+    partition_key "Internal column that is used for partitioning the services invocations. Can be ignored.": DataType::UInt64,
+    id "[Invocation ID](/operate/invocation#invocation-identifier).": DataType::LargeUtf8,
+
+    status "Either `inboxed` or `invoked` or `suspended` or `completed`": DataType::LargeUtf8,
+
+    target "Invocation Target. Format for plain services: `ServiceName/HandlerName`, e.g. `Greeter/greet`. Format for Virtual Objects/Workflows: `VirtualObjectName/Key/HandlerName`, e.g. `Greeter/Francesco/greet`.": DataType::LargeUtf8,
+    target_service_name "The name for the invoked service.": DataType::LargeUtf8,
+    target_service_key "The key of the Virtual Object or of the Workflow. Null for regular services.": DataType::LargeUtf8,
+    target_handler_name "The invoked handler.": DataType::LargeUtf8,
+    target_service_ty "The service type. Either `service` or `virtual_object` or `workflow`.": DataType::LargeUtf8,
+
+    invoked_by "Either `ingress` if the service was invoked externally or `service` if the service was invoked by another Restate service.": DataType::LargeUtf8,
+    invoked_by_service_name "The name of the invoking service. Or `null` if invoked externally.": DataType::LargeUtf8,
+    invoked_by_id "The caller [Invocation ID](/operate/invocation#invocation-identifier) if the service was invoked by another Restate service. Or `null` if invoked externally.": DataType::LargeUtf8,
+    invoked_by_target "The caller invocation target if the service was invoked by another Restate service. Or `null` if invoked externally.": DataType::LargeUtf8,
+
+    pinned_deployment_id "The opaque service deployment ID that has been committed for this invocation; this is set after the first journal entry is stored for this invocation.": DataType::LargeUtf8,
+    trace_id "The ID of the trace that is assigned to this invocation. Only relevant when tracing is enabled.": DataType::LargeUtf8,
+    journal_size "The number of journal entries durably logged for this invocation.": DataType::UInt32,
+    created_at "Timestamp indicating the start of this invocation.": DataType::Date64,
+    modified_at "Timestamp indicating the last state transition. For example, last time the status changed from `invoked` to `suspended`.": DataType::Date64,
 ));

crates/storage-query-datafusion/src/journal/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/journal/schema.rs

@@ -15,19 +15,19 @@ use crate::table_macro::*;
 use datafusion::arrow::datatypes::DataType;
 
 define_table!(journal(
-    partition_key: DataType::UInt64,
-    id: DataType::LargeUtf8,
+    partition_key "Internal column that is used for partitioning the services invocations. Can be ignored.": DataType::UInt64,
+    id "[Invocation ID](/operate/invocation#invocation-identifier).": DataType::LargeUtf8,
 
-    index: DataType::UInt32,
-    entry_type: DataType::LargeUtf8,
-    name: DataType::LargeUtf8,
+    index "The index of this journal entry.": DataType::UInt32,
+    entry_type "The entry type. You can check all the available entry types in [`entries.rs`](https://github.com/restatedev/restate/blob/main/crates/types/src/journal/entries.rs).": DataType::LargeUtf8,
+    name "The name of the entry supplied by the user, if any.": DataType::LargeUtf8,
 
-    completed: DataType::Boolean,
+    completed "Indicates whether this journal entry has been completed; this is only valid for some entry types.": DataType::Boolean,
 
-    invoked_id: DataType::LargeUtf8,
-    invoked_target: DataType::LargeUtf8,
+    invoked_id "If this entry represents an outbound invocation, indicates the ID of that invocation.": DataType::LargeUtf8,
+    invoked_target "If this entry represents an outbound invocation, indicates the invocation Target. Format for plain services: `ServiceName/HandlerName`, e.g. `Greeter/greet`. Format for Virtual Objects/Workflows: `VirtualObjectName/Key/HandlerName`, e.g. `Greeter/Francesco/greet`.": DataType::LargeUtf8,
 
-    sleep_wakeup_at: DataType::Date64,
+    sleep_wakeup_at "If this entry represents a sleep, indicates wakeup time.": DataType::Date64,
 
-    raw: DataType::LargeBinary,
+    raw "Raw binary representation of the entry. Check the [service protocol](https://github.com/restatedev/service-protocol) for more details to decode it.": DataType::LargeBinary,
 ));

crates/storage-query-datafusion/src/keyed_service_status/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/keyed_service_status/schema.rs

@@ -15,10 +15,10 @@ use crate::table_macro::*;
 use datafusion::arrow::datatypes::DataType;
 
 define_table!(keyed_service_status(
-    partition_key: DataType::UInt64,
+    partition_key "Internal column that is used for partitioning the services invocations. Can be ignored.": DataType::UInt64,
 
-    service_name: DataType::LargeUtf8,
-    service_key: DataType::LargeUtf8,
+    service_name "The name for the invoked virtual object/workflow.": DataType::LargeUtf8,
+    service_key "The key of the virtual object/workflow.": DataType::LargeUtf8,
 
-    invocation_id: DataType::LargeUtf8,
+    invocation_id "[Invocation ID](/operate/invocation#invocation-identifier).": DataType::LargeUtf8,
 ));

crates/storage-query-datafusion/src/lib.rs

@@ -28,6 +28,74 @@ mod table_util;
 
 pub use context::BuildError;
 
+#[cfg(feature = "table_docs")]
+pub fn table_docs(table: &str) -> Vec<(&str, &str, &str)> {
+    match table {
+        "sys_journal" => Vec::from(journal::schema::TABLE_DOCS),
+        "state" => Vec::from(state::schema::TABLE_DOCS),
+        "sys_keyed_service_status" => Vec::from(keyed_service_status::schema::TABLE_DOCS),
+        "sys_inbox" => Vec::from(inbox::schema::TABLE_DOCS),
+        "sys_deployment" => Vec::from(deployment::schema::TABLE_DOCS),
+        "sys_service" => Vec::from(service::schema::TABLE_DOCS),
+        "sys_idempotency" => Vec::from(idempotency::schema::TABLE_DOCS),
+        "sys_promise" => Vec::from(promise::schema::TABLE_DOCS),
+        "sys_invocation" => {
+            // We need to compile this manually, due to the fact that it's a view.
+            use std::collections::HashMap;
+            let mut sys_invocation_state: HashMap<&'static str, (&'static str, &'static str)> =
+                invocation_state::schema::TABLE_DOCS
+                    .iter()
+                    .map(|(name, ty, desc)| (*name, (*ty, *desc)))
+                    .collect();
+            let mut sys_invocation_status: HashMap<&'static str, (&'static str, &'static str)> =
+                invocation_status::schema::TABLE_DOCS
+                    .iter()
+                    .map(|(name, ty, desc)| (*name, (*ty, *desc)))
+                    .collect();
+
+            fn copy_desc<'a>(
+                map: &mut HashMap<&'a str, (&'a str, &'a str)>,
+                name: &str,
+            ) -> (&'a str, &'a str, &'a str) {
+                let (name, (ty, desc)) = map.remove_entry(name).unwrap();
+                (name, ty, desc)
+            }
+
+            vec![
+                copy_desc(&mut sys_invocation_status, "id"),
+                copy_desc(&mut sys_invocation_status, "target"),
+                copy_desc(&mut sys_invocation_status, "target_service_name"),
+                copy_desc(&mut sys_invocation_status, "target_service_key"),
+                copy_desc(&mut sys_invocation_status, "target_handler_name"),
+                copy_desc(&mut sys_invocation_status, "target_service_ty"),
+                copy_desc(&mut sys_invocation_status, "invoked_by"),
+                copy_desc(&mut sys_invocation_status, "invoked_by_service_name"),
+                copy_desc(&mut sys_invocation_status, "invoked_by_id"),
+                copy_desc(&mut sys_invocation_status, "invoked_by_target"),
+                copy_desc(&mut sys_invocation_status, "pinned_deployment_id"),
+                copy_desc(&mut sys_invocation_status, "trace_id"),
+                copy_desc(&mut sys_invocation_status, "journal_size"),
+                copy_desc(&mut sys_invocation_status, "created_at"),
+                copy_desc(&mut sys_invocation_status, "modified_at"),
+
+                copy_desc(&mut sys_invocation_state, "retry_count"),
+                copy_desc(&mut sys_invocation_state, "last_start_at"),
+                copy_desc(&mut sys_invocation_state, "next_retry_at"),
+                copy_desc(&mut sys_invocation_state, "last_attempt_deployment_id"),
+                copy_desc(&mut sys_invocation_state, "last_attempt_server"),
+                copy_desc(&mut sys_invocation_state, "last_failure"),
+                copy_desc(&mut sys_invocation_state, "last_failure_error_code"),
+                copy_desc(&mut sys_invocation_state, "last_failure_related_entry_index"),
+                copy_desc(&mut sys_invocation_state, "last_failure_related_entry_name"),
+                copy_desc(&mut sys_invocation_state, "last_failure_related_entry_type"),
+
+                ("status", "Utf8", "Either `pending` or `ready` or `running` or `backing-off` or `suspended` or `completed`.")
+            ]
+        }
+        _ => panic!("Unknown table '{table}'"),
+    }
+}
+
 #[cfg(test)]
 pub(crate) mod mocks;
 

crates/storage-query-datafusion/src/promise/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/promise/schema.rs

@@ -15,15 +15,15 @@ use crate::table_macro::*;
 use datafusion::arrow::datatypes::DataType;
 
 define_table!(promise(
-    partition_key: DataType::UInt64,
+    partition_key "Internal column that is used for partitioning the services invocations. Can be ignored.": DataType::UInt64,
 
-    service_name: DataType::LargeUtf8,
-    service_key: DataType::LargeUtf8,
+    service_name "The name for the workflow.": DataType::LargeUtf8,
+    service_key "The key of the virtual workflow.": DataType::LargeUtf8,
 
-    key: DataType::LargeUtf8,
-    completed: DataType::Boolean,
+    key "The promise key.": DataType::LargeUtf8,
+    completed "True if the promise was completed.": DataType::Boolean,
 
-    completion_success_value: DataType::LargeBinary,
-    completion_success_value_utf8: DataType::LargeUtf8,
-    completion_failure: DataType::LargeUtf8
+    completion_success_value "The completion success, if any.": DataType::LargeBinary,
+    completion_success_value_utf8 "The completion success as UTF-8 string, if any.": DataType::LargeUtf8,
+    completion_failure "The completion failure, if any.": DataType::LargeUtf8,
 ));

crates/storage-query-datafusion/src/service/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/service/schema.rs

@@ -15,11 +15,11 @@ use crate::table_macro::*;
 use datafusion::arrow::datatypes::DataType;
 
 define_table!(service(
-    name: DataType::LargeUtf8,
-    revision: DataType::UInt64,
+    name "The name of the registered user service.": DataType::LargeUtf8,
+    revision "The latest deployed revision.": DataType::UInt64,
 
-    public: DataType::Boolean,
+    public "Whether the service is accessible through the ingress endpoint or not.": DataType::Boolean,
 
-    ty: DataType::LargeUtf8,
-    deployment_id: DataType::LargeUtf8,
+    ty "The service type. Either `service` or `virtual_object` or `workflow`.": DataType::LargeUtf8,
+    deployment_id "The ID of the latest deployment": DataType::LargeUtf8,
 ));