feat: add typescript support (#7)

.gitignore

@@ -130,4 +130,9 @@ dist
 .pnp.*
 
 .ideas.md
-.todos.md
+.todos.md
+
+# Custom
+dist
+types
+build

embedchain/chunkers/base_chunker.js → embedchain/chunkers/BaseChunker.ts

@@ -1,15 +1,20 @@
-const { createHash } = require("crypto");
+import { createHash } from "crypto";
+import { BaseLoader, } from "../loaders";
+import { Input, LoaderResult } from "../models";
+import { ChunkResult } from "../models/ChunkResult";
 
 class BaseChunker {
-  constructor(text_splitter) {
+  text_splitter: any;  // the type of text_splitter is not specified in your code
+
+  constructor(text_splitter: any) {
     this.text_splitter = text_splitter;
   }
 
-  async create_chunks(loader, url) {
-    const documents = [];
-    const ids = [];
-    const datas = await loader.load_data(url);
-    const metadatas = [];
+  async create_chunks(loader: BaseLoader, url: Input): Promise<ChunkResult> {
+    const documents: ChunkResult['documents'] = [];
+    const ids: ChunkResult['ids'] = [];
+    const datas: LoaderResult = await loader.load_data(url);
+    const metadatas: ChunkResult['metadatas'] = [];
     for (const data of datas) {
       const content = data["content"];
       const meta_data = data["meta_data"];
@@ -32,4 +37,4 @@ class BaseChunker {
   }
 }
 
-module.exports = { BaseChunker };
+export { BaseChunker };

embedchain/chunkers/PdfFile.ts

@@ -0,0 +1,25 @@
+import { BaseChunker } from "./BaseChunker";
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
+
+interface TextSplitterChunkParams {
+  chunkSize: number;
+  chunkOverlap: number;
+  keepSeparator: boolean;
+}
+
+const TEXT_SPLITTER_CHUNK_PARAMS: TextSplitterChunkParams = {
+  chunkSize: 1000,
+  chunkOverlap: 0,
+  keepSeparator: false,
+};
+
+class PdfFileChunker extends BaseChunker {
+  constructor() {
+    const text_splitter = new RecursiveCharacterTextSplitter(
+      TEXT_SPLITTER_CHUNK_PARAMS
+    );
+    super(text_splitter);
+  }
+}
+
+export { PdfFileChunker };

embedchain/chunkers/QnaPair.ts

@@ -0,0 +1,25 @@
+import { BaseChunker } from "./BaseChunker";
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
+
+interface TextSplitterChunkParams {
+  chunkSize: number;
+  chunkOverlap: number;
+  keepSeparator: boolean;
+}
+
+const TEXT_SPLITTER_CHUNK_PARAMS: TextSplitterChunkParams = {
+  chunkSize: 300,
+  chunkOverlap: 0,
+  keepSeparator: false,
+};
+
+class QnaPairChunker extends BaseChunker {
+  constructor() {
+    const text_splitter = new RecursiveCharacterTextSplitter(
+      TEXT_SPLITTER_CHUNK_PARAMS
+    );
+    super(text_splitter);
+  }
+}
+
+export { QnaPairChunker };

embedchain/chunkers/WebPage.ts

@@ -0,0 +1,25 @@
+import { BaseChunker } from "./BaseChunker";
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
+
+interface TextSplitterChunkParams {
+  chunkSize: number;
+  chunkOverlap: number;
+  keepSeparator: boolean;
+}
+
+const TEXT_SPLITTER_CHUNK_PARAMS: TextSplitterChunkParams = {
+  chunkSize: 500,
+  chunkOverlap: 0,
+  keepSeparator: false,
+};
+
+class WebPageChunker extends BaseChunker {
+  constructor() {
+    const text_splitter = new RecursiveCharacterTextSplitter(
+      TEXT_SPLITTER_CHUNK_PARAMS
+    );
+    super(text_splitter);
+  }
+}
+
+export { WebPageChunker };

embedchain/chunkers/index.ts

@@ -0,0 +1,6 @@
+import { BaseChunker } from "./BaseChunker";
+import { PdfFileChunker } from "./PdfFile";
+import { QnaPairChunker } from "./QnaPair";
+import { WebPageChunker } from "./WebPage";
+
+export { BaseChunker, PdfFileChunker, QnaPairChunker, WebPageChunker }