feat(sdk): initial SDK (#149)

.github/labels.yaml

@@ -64,4 +64,8 @@
 
 - name: "security 🔒"
   color: "d93f0b"
-  description: "Security-related changes or fixes"
+  description: "Security-related changes or fixes"
+
+- name: "sdk 👷"
+  color: "#fb00ff"
+  description: "Adding to or modifying SDK"

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members  = ["crates/learner", "crates/learnerd"]
+members  = ["crates/learner", "crates/learnerd", "crates/sdk"]
 resolver = "2"
 
 [workspace.package]

README.md

@@ -16,14 +16,15 @@
 
 </div>
 
-[Features](#features)
-[Installation](#installation)
-[Usage](#usage)
-[Configuration](#configuration)
-[Roadmap](#roadmap)
-[Contributing](#contributing)
-[Development](#development)
-[License](#license)
+[Features](#features) | 
+[Installation](#installation) | 
+[Usage](#usage) | 
+[Configuration](#configuration) | 
+[Roadmap](#roadmap) | 
+[Contributing](#contributing) | 
+[Development](#development) | 
+[SDK](#sdk) | 
+[License](#license) | 
 [Acknowledgements](#acknowledgements)
 
 ---
@@ -320,6 +321,23 @@ just build-all  # build all targets
 > [!TIP]
 > Running `just setup` and `just ci` locally is a quick way to get up to speed and see that the repo is working on your system!
 
+## SDK
+This repository now supplies a very basic SDK for validating a `Retriever` and a `Resource` TOML configurations. 
+To work with this SDK, use:
+```
+# Setup
+just setup-sdk
+
+# Validations
+just validate-retriever <PATH> <OPTIONAL_INPUT> # optionally supply url/identifer
+just validate-resource <PATH>
+
+# Examples
+just validate-retriever crates/learner/config/retrievers/arxiv.toml 2301.07041
+just validate-resource crates/learner/config/resources/thesis.toml
+```
+
+
 ## License
 
 This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

crates/learner/src/resource/mod.rs

@@ -177,7 +177,7 @@ mod tests {
 
   #[test]
   fn test_thesis_from_toml() -> Result<()> {
-    let toml_str = include_str!("../../config/resource/thesis.toml");
+    let toml_str = include_str!("../../config/resources/thesis.toml");
     let config: ResourceConfig = toml::from_str(toml_str)?;
     dbg!(&config);
 

crates/sdk/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+edition = "2021"
+name    = "learner-sdk"
+version = "0.1.0"
+
+[dependencies]
+clap               = { workspace = true }
+learner            = { workspace = true }
+tempfile           = { workspace = true }
+tokio              = { workspace = true }
+toml               = { workspace = true }
+tracing            = { workspace = true }
+tracing-subscriber = { workspace = true }

crates/sdk/src/main.rs

@@ -0,0 +1,65 @@
+mod validate;
+
+use std::path::PathBuf;
+
+use clap::{Parser, Subcommand};
+use learner::prelude::*;
+use tracing::{debug, error, info, warn};
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+struct LearnerSdk {
+  #[command(subcommand)]
+  command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+  /// Validate a resource config
+  ValidateResource {
+    /// Path to the configuration file
+    path: PathBuf,
+  },
+  /// Validate a retriever config for an optional given input
+  ValidateRetriever {
+    /// Path to the configuration file
+    path: PathBuf,
+
+    /// Identifier or URL
+    input: Option<String>,
+  },
+}
+
+#[tokio::main]
+async fn main() {
+  tracing_subscriber::fmt()
+    .without_time()
+    .with_file(false)
+    .with_line_number(false)
+    .with_target(false)
+    .with_max_level(tracing::Level::TRACE)
+    .init();
+
+  let cli = LearnerSdk::parse();
+
+  match &cli.command {
+    Commands::ValidateRetriever { path, input } => {
+      info!("Validating retriever...");
+      if !path.exists() {
+        error!("Path to retriever config was invalid.\nPath used: {path:?}");
+        return;
+      }
+      debug!("Validating retriever config at {:?}", path);
+      validate::validate_retriever(path, input).await;
+    },
+    Commands::ValidateResource { path } => {
+      info!("Validating resource...");
+      if !path.exists() {
+        error!("Path to resource config was invalid.\nPath used: {path:?}");
+        return;
+      }
+      debug!("Validating resource config at {:?}", path);
+      validate::validate_resource(path);
+    },
+  }
+}

crates/sdk/src/validate.rs

@@ -0,0 +1,114 @@
+use std::fs::read_to_string;
+
+use learner::{
+  resource::ResourceConfig,
+  retriever::{ResponseFormat, RetrieverConfig},
+};
+
+use super::*;
+
+pub fn validate_resource(path: &PathBuf) {
+  let config_str = match read_to_string(path) {
+    Ok(str) => str,
+    Err(e) => {
+      error!("Failed to read config to string due to: {e:?}");
+      return;
+    },
+  };
+
+  let resource: ResourceConfig = match toml::from_str(&config_str) {
+    Ok(config) => config,
+    Err(e) => {
+      error!("Failed to parse config to string due to: {e:?}");
+      return;
+    },
+  };
+
+  info!("Resource type: {}", resource.type_name);
+
+  // Check all required fields are present
+  debug!("All config fields are:\n{:#?}", resource.fields());
+}
+
+pub async fn validate_retriever(path: &PathBuf, input: &Option<String>) {
+  let config_str = match read_to_string(path) {
+    Ok(str) => str,
+    Err(e) => {
+      error!("Failed to read config to string due to: {e:?}");
+      return;
+    },
+  };
+
+  let retriever: RetrieverConfig = match toml::from_str(&config_str) {
+    Ok(config) => config,
+    Err(e) => {
+      error!("Failed to parse config to string due to: {e:?}");
+      return;
+    },
+  };
+
+  match &retriever.response_format {
+    ResponseFormat::Xml(config) => {
+      debug!("Retriever is configured for: XML\n{config:#?}")
+    },
+    ResponseFormat::Json(config) => {
+      debug!("Retriever is configured for: JSON\n{config:#?}")
+    },
+  }
+
+  if let Some(input) = input {
+    info!("Attempting to match against pattern...");
+    match retriever.extract_identifier(input) {
+      Ok(identifier) => info!("Retriever extracted input into: {identifier}"),
+      Err(e) => {
+        error!("Retriever failed to extract input due to: {e:?}");
+        return;
+      },
+    }
+
+    info!("Attempting to fetch paper using retriever...");
+    let paper = match retriever.retrieve_paper(input).await {
+      Ok(paper) => {
+        info!("Paper retrieved!\n{paper:#?}");
+        paper
+      },
+      Err(e) => {
+        error!("Retriever failed to retriever paper due to: {e:?}");
+        return;
+      },
+    };
+
+    if paper.pdf_url.is_some() {
+      info!("Attempting to download associated pdf");
+      let tempdir = tempfile::tempdir().unwrap();
+      match paper.download_pdf(tempdir.path()).await {
+        Ok(filename) => {
+          let pdf_filepath = tempdir.path().join(filename);
+          if pdf_filepath.exists() {
+            let bytes = std::fs::read(path).unwrap();
+            if bytes.is_empty() {
+              error!("PDF download was empty.");
+            } else {
+              info!("Non-empty PDF downloaded successfully.");
+            }
+          } else {
+            error!("PDF path did not end up getting written.")
+          }
+        },
+        Err(e) => {
+          error!("PDF was unable to be downloaded due to: {e:?}")
+        },
+      }
+    } else {
+      warn!(
+        "PDF URL was not determined. Please check your configuration against the server response."
+      );
+    }
+  } else {
+    warn!(
+      "No input string provided to further debug your `RetrieverConfig`. If you want to test \
+       identifier pattern matching and online fetching, please pass in an input string with an \
+       additional input, e.g., `2301.07041`."
+    );
+  }
+}

justfile

@@ -150,6 +150,20 @@ debug:
     @just header "Installing learnerd in debug mode"
     cargo install --path crates/learnerd --features tui --debug
 
+# Setup SDK
+setup-sdk:
+    cargo install --path crates/sdk --debug
+
+# Validate a retriever config
+validate-retriever path input="":
+    @just header "Validating retriever config"
+    learner-sdk validate-retriever {{path}} {{input}} 
+
+# Validate a resource config
+validate-resource path:
+    @just header "Validating resource config"
+    learner-sdk validate-resource {{path}}
+
 # Show your relevant environment information
 info:
     @just header "Environment Information"
@@ -211,4 +225,6 @@ _ci-summary-success:
 _ci-summary-failure:
     @printf "\n{{bold}}CI Summary:{{reset}}\n"
     @printf "{{error}}{{bold}}Some checks failed. See output above for details.{{reset}}\n"
-    @exit 1
+    @exit 1
+
+