Introduce basic rustdoc infrastructure (#980)

This commit adds a new `docs` command to the flake which can be used to
build rustdocs. The setup is compatible with Cargo and Bazel.

Docs may also be tested via `bazel test docs`.

This commit doesn't publish the docs and doesn't fully surface all
configuration. It provides some basic infrastructure as a starting point
for future enhancments to autogenerated documentation.

Co-authored-by: Blake Hatch <blakewihatch@gmail.com>

Co-Authored-By: Blake Hatch <blakewihatch@gmail.com>

BUILD.bazel

@@ -41,6 +41,36 @@ rust_binary(
     ],
 )
 
+filegroup(
+    name = "docs",
+    srcs = [
+        "//nativelink-config:docs",
+        "//nativelink-error:docs",
+        "//nativelink-macro:docs",
+        "//nativelink-proto:docs",
+        "//nativelink-scheduler:docs",
+        "//nativelink-service:docs",
+        "//nativelink-store:docs",
+        "//nativelink-util:docs",
+        "//nativelink-worker:docs",
+    ],
+)
+
+test_suite(
+    name = "doctests",
+    tests = [
+        "//nativelink-config:doc_test",
+        "//nativelink-error:doc_test",
+        "//nativelink-macro:doc_test",
+        "//nativelink-proto:doc_test",
+        "//nativelink-scheduler:doc_test",
+        "//nativelink-service:doc_test",
+        "//nativelink-store:doc_test",
+        "//nativelink-util:doc_test",
+        "//nativelink-worker:doc_test",
+    ],
+)
+
 genrule(
     name = "dummy_test_sh",
     outs = ["dummy_test.sh"],

CONTRIBUTING.md

@@ -300,6 +300,32 @@ This will automatically apply some fixes like automated line fixes and format
 changes. Note that changed files aren't automatically staged. Use `git add` to
 add the changed files manually to the staging area.
 
+## Generating documentation
+
+Automatically generated documentation is still under construction. To view the
+documentation for the `nativelink-*` crates, run the `docs` command in the nix
+flake:
+
+```bash
+docs
+```
+
+To build individual crate-level docs:
+
+```bash
+# All docs
+bazel build docs
+
+# A single crate
+bazel build nativelink-config:docs
+```
+
+To run documentation tests with Bazel:
+
+```bash
+bazel test doctests
+```
+
 ## Writing documentation
 
 NativeLink largely follows the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/welcome/).

flake.nix

@@ -96,7 +96,7 @@
         src = pkgs.lib.cleanSourceWith {
           src = craneLib.path ./.;
           filter = path: type:
-            (builtins.match "^.+/data/SekienAkashita\\.jpg" path != null)
+            (builtins.match "^.*(data/SekienSkashita\.jpg|nativelink-config/README\.md)" path != null)
             || (craneLib.filterCargoSources path type);
         };
 
@@ -142,6 +142,8 @@
 
         native-cli = import ./native-cli/default.nix {inherit pkgs;};
 
+        docs = pkgs.callPackage ./tools/docs.nix {rust = stable-rust.default;};
+
         inherit (nix2container.packages.${system}.nix2container) pullImage;
         inherit (nix2container.packages.${system}.nix2container) buildImage;
 
@@ -275,6 +277,7 @@
               generate-toolchains
               customClang
               native-cli
+              docs
             ]
             ++ maybeDarwinDeps;
           shellHook = ''

nativelink-config/BUILD.bazel

@@ -15,6 +15,9 @@ rust_library(
         "src/serde_utils.rs",
         "src/stores.rs",
     ],
+    compile_data = [
+        "README.md",
+    ],
     visibility = ["//visibility:public"],
     deps = [
         "@crates//:byte-unit",
@@ -44,6 +47,7 @@ rust_test_suite(
 rust_doc(
     name = "docs",
     crate = ":nativelink-config",
+    visibility = ["//visibility:public"],
 )
 
 rust_doc_test(

nativelink-config/src/lib.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![doc = include_str!("../README.md")]
+
 pub mod cas_server;
 pub mod schedulers;
 pub mod serde_utils;

nativelink-config/src/serde_utils.rs

@@ -59,7 +59,7 @@ where
     deserializer.deserialize_any(USizeVisitor::<T>(PhantomData::<T> {}))
 }
 
-/// Same as convert_numeric_with_shellexpand, but supports Option<T>.
+/// Same as convert_numeric_with_shellexpand, but supports `Option<T>`.
 pub fn convert_optional_numeric_with_shellexpand<'de, D, T, E>(
     deserializer: D,
 ) -> Result<Option<T>, D::Error>
@@ -113,7 +113,7 @@ pub fn convert_string_with_shellexpand<'de, D: Deserializer<'de>>(
     Ok((*(shellexpand::env(&value).map_err(de::Error::custom)?)).to_string())
 }
 
-/// Same as convert_string_with_shellexpand, but supports Vec<String>.
+/// Same as convert_string_with_shellexpand, but supports `Vec<String>`.
 pub fn convert_vec_string_with_shellexpand<'de, D: Deserializer<'de>>(
     deserializer: D,
 ) -> Result<Vec<String>, D::Error> {
@@ -127,7 +127,7 @@ pub fn convert_vec_string_with_shellexpand<'de, D: Deserializer<'de>>(
         .collect()
 }
 
-/// Same as convert_string_with_shellexpand, but supports Option<String>.
+/// Same as convert_string_with_shellexpand, but supports `Option<String>`.
 pub fn convert_optional_string_with_shellexpand<'de, D: Deserializer<'de>>(
     deserializer: D,
 ) -> Result<Option<String>, D::Error> {

nativelink-error/BUILD.bazel

@@ -1,4 +1,9 @@
-load("@rules_rust//rust:defs.bzl", "rust_library")
+load(
+    "@rules_rust//rust:defs.bzl",
+    "rust_doc",
+    "rust_doc_test",
+    "rust_library",
+)
 
 rust_library(
     name = "nativelink-error",
@@ -16,3 +21,15 @@ rust_library(
         "@crates//:tonic",
     ],
 )
+
+rust_doc(
+    name = "docs",
+    crate = ":nativelink-error",
+    visibility = ["//visibility:public"],
+)
+
+rust_doc_test(
+    name = "doc_test",
+    timeout = "short",
+    crate = ":nativelink-error",
+)

nativelink-macro/BUILD.bazel

@@ -1,6 +1,7 @@
 load(
     "@rules_rust//rust:defs.bzl",
     "rust_doc",
+    "rust_doc_test",
     "rust_proc_macro",
 )
 
@@ -20,4 +21,11 @@ rust_proc_macro(
 rust_doc(
     name = "docs",
     crate = ":nativelink-macro",
+    visibility = ["//visibility:public"],
+)
+
+rust_doc_test(
+    name = "doc_test",
+    timeout = "short",
+    crate = ":nativelink-macro",
 )

nativelink-macro/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.4.0"
 edition = "2021"
 
 [lib]
-crate_type = ["proc-macro"]
+proc-macro = true
 
 [dependencies]
 # TODO(allada) We currently need to pin these to specific version.

nativelink-proto/BUILD.bazel

@@ -1,4 +1,10 @@
-load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library")
+load(
+    "@rules_rust//rust:defs.bzl",
+    "rust_binary",
+    "rust_doc",
+    "rust_doc_test",
+    "rust_library",
+)
 
 PROTO_NAMES = [
     "build.bazel.remote.execution.v2",
@@ -108,3 +114,15 @@ py_test(
     ],
     main = "update_protos.py",
 )
+
+rust_doc(
+    name = "docs",
+    crate = ":nativelink-proto",
+    visibility = ["//visibility:public"],
+)
+
+rust_doc_test(
+    name = "doc_test",
+    timeout = "short",
+    crate = ":nativelink-proto",
+)

nativelink-proto/build/bazel/remote/execution/v2/remote_execution.proto

@@ -1869,11 +1869,13 @@ message DigestFunction {
     //   3. A single invocation is made to the SHA-256 block cipher with
     //      the following parameters:
     //
-    //          M = Hash(left) || Hash(right)
-    //          H = {
-    //              0xcbbb9d5d, 0x629a292a, 0x9159015a, 0x152fecd8,
-    //              0x67332667, 0x8eb44a87, 0xdb0c2e0d, 0x47b5481d,
-    //          }
+    //      ```text
+    //      M = Hash(left) || Hash(right)
+    //      H = {
+    //          0xcbbb9d5d, 0x629a292a, 0x9159015a, 0x152fecd8,
+    //          0x67332667, 0x8eb44a87, 0xdb0c2e0d, 0x47b5481d,
+    //      }
+    //      ```
     //
     //      The values of H are the leading fractional parts of the
     //      square roots of the 9th to the 16th prime number (23 to 53).
@@ -1884,7 +1886,9 @@ message DigestFunction {
     //   4. The hash of the full blob can then be obtained by
     //      concatenating the outputs of the block cipher:
     //
-    //          Hash(blob) = a || b || c || d || e || f || g || h
+    //      ```text
+    //      Hash(blob) = a || b || c || d || e || f || g || h
+    //      ```
     //
     //      Addition of the original values of H, as normally done
     //      through the use of the Davies-Meyer structure, is not

nativelink-proto/genproto/build.bazel.remote.execution.v2.pb.rs

@@ -1566,11 +1566,13 @@ pub mod digest_function {
         ///    3. A single invocation is made to the SHA-256 block cipher with
         ///       the following parameters:
         ///
-        ///           M = Hash(left) || Hash(right)
-        ///           H = {
-        ///               0xcbbb9d5d, 0x629a292a, 0x9159015a, 0x152fecd8,
-        ///               0x67332667, 0x8eb44a87, 0xdb0c2e0d, 0x47b5481d,
-        ///           }
+        ///       ```text
+        ///       M = Hash(left) || Hash(right)
+        ///       H = {
+        ///           0xcbbb9d5d, 0x629a292a, 0x9159015a, 0x152fecd8,
+        ///           0x67332667, 0x8eb44a87, 0xdb0c2e0d, 0x47b5481d,
+        ///       }
+        ///       ```
         ///
         ///       The values of H are the leading fractional parts of the
         ///       square roots of the 9th to the 16th prime number (23 to 53).
@@ -1581,7 +1583,9 @@ pub mod digest_function {
         ///    4. The hash of the full blob can then be obtained by
         ///       concatenating the outputs of the block cipher:
         ///
-        ///           Hash(blob) = a || b || c || d || e || f || g || h
+        ///       ```text
+        ///       Hash(blob) = a || b || c || d || e || f || g || h
+        ///       ```
         ///
         ///       Addition of the original values of H, as normally done
         ///       through the use of the Davies-Meyer structure, is not