Enhance CI workflow, implement async features, and refactor code

.github/workflows/ci.yml

@@ -8,6 +8,7 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
 
 jobs:
   check:
@@ -17,7 +18,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - run: cargo check
+      - run: cargo check --all-features
 
   fmt:
     name: Format
@@ -27,7 +28,7 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt
-      - run: cargo fmt -- --check
+      - run: cargo fmt --all -- --check
 
   clippy:
     name: Clippy
@@ -38,7 +39,7 @@ jobs:
         with:
           components: clippy
       - uses: Swatinem/rust-cache@v2
-      - run: cargo clippy -- -W clippy::all
+      - run: cargo clippy --all-targets --all-features -- -D warnings
 
   test:
     name: Test
@@ -47,4 +48,42 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - run: cargo test --lib
+      - run: cargo test --lib --all-features
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+  docs:
+    name: Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - name: Check documentation build
+        run: cargo doc --no-deps --all-features
+        env:
+          RUSTDOCFLAGS: -D warnings
+
+  doctest:
+    name: Doc Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - name: Run documentation tests
+        run: cargo test --doc --all-features
+        continue-on-error: true  # Allow doctest failures initially
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+  # Release build check
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    needs: [check, fmt, clippy, test, docs]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo build --release

examples/basic.rs

@@ -11,7 +11,7 @@
 //! cargo run --example basic
 //! ```
 
-use vectorless::Engine;
+use vectorless::{Engine, IndexContext};
 
 #[tokio::main]
 async fn main() -> vectorless::Result<()> {
@@ -21,17 +21,18 @@ async fn main() -> vectorless::Result<()> {
     let client = Engine::builder()
         .with_workspace("./workspace")
         .build()
-        .map_err(|e| vectorless::Error::Config(e.to_string()))?;
+        .await
+        .map_err(|e: vectorless::BuildError| vectorless::Error::Config(e.to_string()))?;
 
     println!("✓ Client created\n");
 
     // 2. Index a document
-    let doc_id = client.index("./README.md").await?;
+    let doc_id = client.index(IndexContext::from_path("./README.md")).await?;
     println!("✓ Indexed: {}\n", doc_id);
 
     // 3. List documents
     println!("Documents:");
-    for doc in client.list_documents() {
+    for doc in client.list_documents().await? {
         println!("  - {} ({})", doc.name, doc.id);
     }
     println!();
@@ -55,7 +56,7 @@ async fn main() -> vectorless::Result<()> {
     println!("✓ Client cloned for concurrent use\n");
 
     // 6. Cleanup
-    client.remove(&doc_id)?;
+    client.remove(&doc_id).await?;
     println!("✓ Removed: {}", doc_id);
 
     println!("\n=== Done ===");

examples/batch_processing.rs

@@ -12,7 +12,7 @@
 //! cargo run --example batch_processing
 //! ```
 
-use vectorless::client::EngineBuilder;
+use vectorless::client::{EngineBuilder, IndexContext};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -23,9 +23,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let engine = EngineBuilder::new()
         .with_workspace("./workspace_batch_example")
         .build()
-        .map_err(|e| vectorless::Error::Config(e.to_string()))?;
+        .await
+        .map_err(|e: vectorless::BuildError| vectorless::Error::Config(e.to_string()))?;
 
-    let session = engine.session();
+    let session = engine.session().await;
     println!("  ✓ Session created: {}\n", session.id());
 
     // 2. Create sample documents
@@ -1058,7 +1059,7 @@ Implement authentication for production.
 
     for (name, _) in &documents {
         let path = temp_dir.path().join(name);
-        match session.index(&path).await {
+        match session.index(IndexContext::from_path(&path)).await {
             Ok(doc_id) => {
                 doc_ids.push(doc_id);
             }
@@ -1146,7 +1147,7 @@ Implement authentication for production.
     // 7. Cleanup
     println!("Step 7: Cleanup...");
     for doc_id in &doc_ids {
-        engine.remove(doc_id)?;
+        engine.remove(doc_id).await?;
     }
     println!("  ✓ Removed {} documents\n", doc_ids.len());
 

examples/events.rs

@@ -17,7 +17,7 @@
 use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
-use vectorless::client::{EngineBuilder, EventEmitter, IndexEvent, QueryEvent};
+use vectorless::client::{EngineBuilder, EventEmitter, IndexContext, IndexEvent, QueryEvent};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -95,7 +95,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .with_workspace("./workspace_events_example")
         .with_events(events)
         .build()
-        .map_err(|e| vectorless::Error::Config(e.to_string()))?;
+        .await
+        .map_err(|e: vectorless::BuildError| vectorless::Error::Config(e.to_string()))?;
     println!("  ✓ Engine created\n");
 
     // 3. Index a document (events will fire)
@@ -122,7 +123,7 @@ The event system uses handlers that can be attached to the engine builder.
     let doc_path = temp_dir.path().join("example.md");
     tokio::fs::write(&doc_path, doc_content).await?;
 
-    let doc_id = engine.index(&doc_path).await?;
+    let doc_id = engine.index(IndexContext::from_path(&doc_path)).await?;
     println!();
 
     // 4. Query the document (events will fire)
@@ -161,7 +162,7 @@ The event system uses handlers that can be attached to the engine builder.
 
     // 7. Cleanup
     println!("Step 7: Cleanup...");
-    engine.remove(&doc_id)?;
+    engine.remove(&doc_id).await?;
     println!("  ✓ Document removed\n");
 
     println!("=== Example Complete ===");

examples/markdownflow.rs

@@ -20,7 +20,7 @@
 //! ```
 
 use vectorless::Engine;
-use vectorless::client::IndexOptions;
+use vectorless::client::{IndexContext, IndexOptions};
 
 /// Sample markdown content for demonstration.
 const SAMPLE_MARKDOWN: &str = r#"
@@ -46,7 +46,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let client = Engine::builder()
         .with_workspace("./workspace")
         .build()
-        .map_err(|e| vectorless::Error::Config(e.to_string()))?;
+        .await
+        .map_err(|e: vectorless::BuildError| vectorless::Error::Config(e.to_string()))?;
 
     println!("  - Client created successfully");
     println!();
@@ -61,8 +62,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // Check if we should generate summaries (requires API key)
     println!("  - API key detected, generating summaries...");
-    let options = IndexOptions::new().with_summaries();
-    let doc_id = client.index_with_options(&md_path, options).await?;
+    let doc_id = client
+        .index(IndexContext::from_path(&md_path).with_options(IndexOptions::new().with_summaries()))
+        .await?;
 
     println!("  - Document indexed successfully");
     println!("  - Document ID: {}", doc_id);
@@ -72,7 +74,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     println!("Step 3: Document structure (JSON):");
     println!();
 
-    match client.get_structure(&doc_id) {
+    match client.get_structure(&doc_id).await {
         Ok(tree) => {
             // Export to JSON format (PageIndex compatible)
             let structure = tree.to_structure_json("sample.md");
@@ -121,7 +123,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Step 5: Cleanup
     println!("Step 5: Cleanup...");
 
-    client.remove(&doc_id)?;
+    client.remove(&doc_id).await?;
     println!("  - Document removed");
 
     println!("\n=== Example Complete ===");

examples/session.rs

@@ -15,7 +15,7 @@
 //! cargo run --example session
 //! ```
 
-use vectorless::client::EngineBuilder;
+use vectorless::client::{EngineBuilder, IndexContext};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -26,12 +26,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let engine = EngineBuilder::new()
         .with_workspace("./workspace_session_example")
         .build()
-        .map_err(|e| vectorless::Error::Config(e.to_string()))?;
+        .await
+        .map_err(|e: vectorless::BuildError| vectorless::Error::Config(e.to_string()))?;
     println!("  ✓ Engine created\n");
 
     // 2. Create a session for multi-document operations
     println!("Step 2: Creating session...");
-    let session = engine.session();
+    let session = engine.session().await;
     println!("  ✓ Session ID: {}\n", session.id());
 
     // 3. Index multiple documents into the session
@@ -118,13 +119,13 @@ token_budget = 4000
     tokio::fs::write(&doc3_path, doc3_content).await?;
 
     // Index into session
-    let doc1_id = session.index(&doc1_path).await?;
+    let doc1_id = session.index(IndexContext::from_path(&doc1_path)).await?;
     println!("  ✓ Indexed: architecture.md -> {}", &doc1_id[..8]);
 
-    let doc2_id = session.index(&doc2_path).await?;
+    let doc2_id = session.index(IndexContext::from_path(&doc2_path)).await?;
     println!("  ✓ Indexed: api.md -> {}", &doc2_id[..8]);
 
-    let doc3_id = session.index(&doc3_path).await?;
+    let doc3_id = session.index(IndexContext::from_path(&doc3_path)).await?;
     println!("  ✓ Indexed: config.md -> {}", &doc3_id[..8]);
     println!();
 
@@ -194,9 +195,9 @@ token_budget = 4000
 
     // 9. Cleanup
     println!("Step 9: Cleanup...");
-    engine.remove(&doc1_id)?;
-    engine.remove(&doc2_id)?;
-    engine.remove(&doc3_id)?;
+    engine.remove(&doc1_id).await?;
+    engine.remove(&doc2_id).await?;
+    engine.remove(&doc3_id).await?;
     println!("  ✓ Documents removed\n");
 
     println!("=== Example Complete ===");