From c496199373745c066d82875700fcedd1e476ac3e Mon Sep 17 00:00:00 2001 From: mohamed yahia Date: Wed, 20 Dec 2023 01:00:41 +0200 Subject: [PATCH] Add configuration with include/exclude options (#97) * Add configuration with include/exclude options * Create changeset * Make config file path optional --- .changeset/rude-pants-add.md | 5 ++++ markdowndb.config.js | 8 +++++ package-lock.json | 20 +++++++++++-- package.json | 2 ++ src/bin/index.js | 25 +++++++++++----- src/lib/CustomConfig.ts | 2 ++ src/lib/indexFolder.ts | 57 +++++++++++++++++++++++++++++++----- src/lib/loadConfig.ts | 18 ++++++++++++ src/lib/markdowndb.ts | 19 +++++++++--- 9 files changed, 134 insertions(+), 22 deletions(-) create mode 100644 .changeset/rude-pants-add.md create mode 100644 markdowndb.config.js create mode 100644 src/lib/loadConfig.ts diff --git a/.changeset/rude-pants-add.md b/.changeset/rude-pants-add.md new file mode 100644 index 0000000..70c0b8d --- /dev/null +++ b/.changeset/rude-pants-add.md @@ -0,0 +1,5 @@ +--- +"mddb": minor +--- + +- Add configuration with include/exclude options diff --git a/markdowndb.config.js b/markdowndb.config.js new file mode 100644 index 0000000..cd8dfb2 --- /dev/null +++ b/markdowndb.config.js @@ -0,0 +1,8 @@ +export default { + computedFields: [], // Array of functions to computed fields + schemas: { + // Add zod schemas + }, + include: [], // Pattern for files to be included + exclude: [] // Patten for files to be excluded +}; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 285faf0..b8da53b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,18 +1,20 @@ { "name": "mddb", - "version": "0.7.0", + "version": "0.8.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mddb", - "version": "0.7.0", + "version": "0.8.0", "license": "MIT", "dependencies": { "@portaljs/remark-wiki-link": "^1.0.4", + "@types/micromatch": "^4.0.6", "chokidar": "^3.5.3", "gray-matter": "^4.0.3", "knex": "^2.4.2", + "micromatch": "^4.0.5", "react-markdown": "^9.0.1", "remark-gfm": "^3.0.1", "remark-parse": "^10.0.1", @@ -2156,6 +2158,11 @@ "@babel/types": "^7.3.0" } }, + "node_modules/@types/braces": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.4.tgz", + "integrity": "sha512-0WR3b8eaISjEW7RpZnclONaLFDf7buaowRHdqLp4vLj54AsSAYWfh3DRbfiYJY9XDxMgx1B4sE1Afw2PGpuHOA==" + }, "node_modules/@types/debug": { "version": "4.1.7", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz", @@ -2238,6 +2245,14 @@ "@types/unist": "*" } }, + "node_modules/@types/micromatch": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/micromatch/-/micromatch-4.0.6.tgz", + "integrity": "sha512-2eulCHWqjEpk9/vyic4tBhI8a9qQEl6DaK2n/sF7TweX9YESlypgKyhXMDGt4DAOy/jhLPvVrZc8pTDAMsplJA==", + "dependencies": { + "@types/braces": "*" + } + }, "node_modules/@types/minimist": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", @@ -7685,7 +7700,6 @@ "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "dev": true, "dependencies": { "braces": "^3.0.2", "picomatch": "^2.3.1" diff --git a/package.json b/package.json index 2881a47..549272c 100644 --- a/package.json +++ b/package.json @@ -45,9 +45,11 @@ "types": "./dist/src/index.d.ts", "dependencies": { "@portaljs/remark-wiki-link": "^1.0.4", + "@types/micromatch": "^4.0.6", "chokidar": "^3.5.3", "gray-matter": "^4.0.3", "knex": "^2.4.2", + "micromatch": "^4.0.5", "react-markdown": "^9.0.1", "remark-gfm": "^3.0.1", "remark-parse": "^10.0.1", diff --git a/src/bin/index.js b/src/bin/index.js index 6e71d78..238214f 100755 --- a/src/bin/index.js +++ b/src/bin/index.js @@ -1,17 +1,27 @@ #!/usr/bin/env node - import { MarkdownDB } from "../lib/markdowndb.js"; // TODO get these from markdowndb.config.js or something const dbPath = "markdown.db"; const ignorePatterns = [/Excalidraw/, /\.obsidian/, /DS_Store/]; -const [contentPath, watchFlag] = process.argv.slice(2); -if (!contentPath) { - throw new Error("Invalid/Missing path to markdown content folder"); +let watchFlag; +const args = process.argv.slice(2); + +// Check for the watch flag and its position +const watchIndex = args.indexOf('--watch'); +if (watchIndex !== -1) { + watchFlag = args[watchIndex]; + args.splice(watchIndex, 1); // Remove the watch flag from the array } -const watchEnabled = watchFlag && watchFlag === "--watch"; +// Assign values to contentPath and configFilePath based on their positions +const [contentPath, configFilePath] = args; + +if (!contentPath) { + console.error('Invalid/Missing path to markdown content folder'); + process.exit(1); +} const client = new MarkdownDB({ client: "sqlite3", @@ -25,9 +35,10 @@ await client.init(); await client.indexFolder({ folderPath: contentPath, ignorePatterns: ignorePatterns, - watch: watchEnabled, + watch: watchFlag, + configFilePath: configFilePath }); -if (!watchEnabled) { +if (!watchFlag) { process.exit(); } diff --git a/src/lib/CustomConfig.ts b/src/lib/CustomConfig.ts index 32ca29d..eab3cf3 100644 --- a/src/lib/CustomConfig.ts +++ b/src/lib/CustomConfig.ts @@ -8,4 +8,6 @@ type Schemas = { [index: string]: ZodObject }; export interface CustomConfig { computedFields?: ComputedFields; schemas?: Schemas; + include?: string[]; + exclude?: string[]; } diff --git a/src/lib/indexFolder.ts b/src/lib/indexFolder.ts index c017df8..a909d58 100644 --- a/src/lib/indexFolder.ts +++ b/src/lib/indexFolder.ts @@ -2,6 +2,7 @@ import { ZodError } from "zod"; import { CustomConfig } from "./CustomConfig.js"; import { FileInfo, processFile } from "./process.js"; import { recursiveWalkDir } from "./recursiveWalkDir.js"; +import micromatch from "micromatch"; export function indexFolder( folderPath: string, @@ -11,7 +12,12 @@ export function indexFolder( ) { const filePathsToIndex = recursiveWalkDir(folderPath); const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) => - shouldIncludeFile(filePath, ignorePatterns) + shouldIncludeFile({ + filePath, + ignorePatterns, + includeGlob: config.include, + excludeGlob: config.exclude, + }) ); const files: FileInfo[] = []; const computedFields = config.computedFields || []; @@ -56,11 +62,46 @@ export function indexFolder( return files; } -export function shouldIncludeFile( - filePath: string, - ignorePatterns?: RegExp[] -): boolean { - return !( - ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath)) - ); +export function shouldIncludeFile({ + filePath, + ignorePatterns, + includeGlob, + excludeGlob, +}: { + filePath: string; + ignorePatterns?: RegExp[]; + includeGlob?: string[]; + excludeGlob?: string[]; +}): boolean { + const normalizedFilePath = filePath.replace(/\\/g, "/"); + + if ( + ignorePatterns && + ignorePatterns.some((pattern) => pattern.test(normalizedFilePath)) + ) { + return false; + } + + // Check if the file should be included based on includeGlob + if ( + includeGlob && + includeGlob.length > 0 && + !includeGlob.some((pattern) => + micromatch.isMatch(normalizedFilePath, pattern) + ) + ) { + return false; + } + + // Check if the file should be excluded based on excludeGlob + if ( + excludeGlob && + excludeGlob.some((pattern) => + micromatch.isMatch(normalizedFilePath, pattern) + ) + ) { + return false; + } + + return true; } diff --git a/src/lib/loadConfig.ts b/src/lib/loadConfig.ts new file mode 100644 index 0000000..a084bd9 --- /dev/null +++ b/src/lib/loadConfig.ts @@ -0,0 +1,18 @@ +import * as path from "path"; + +export async function loadConfig(configFilePath?: string) { + const normalizedPath = path.resolve(configFilePath || "markdowndb.config.js"); + const fileUrl = new URL(`file://${normalizedPath}`); + + try { + // Import the module using the file URL + const configModule = await import(fileUrl.href); + return configModule.default; + } catch (error) { + if (configFilePath) { + throw new Error( + `Error loading configuration file from ${normalizedPath}` + ); + } + } +} diff --git a/src/lib/markdowndb.ts b/src/lib/markdowndb.ts index 1ec25ec..c017426 100644 --- a/src/lib/markdowndb.ts +++ b/src/lib/markdowndb.ts @@ -17,6 +17,7 @@ import { CustomConfig } from "./CustomConfig.js"; import { FileInfo, processFile } from "./process.js"; import chokidar from "chokidar"; import { recursiveWalkDir } from "./recursiveWalkDir.js"; +import { loadConfig } from "./loadConfig.js"; const defaultFilePathToUrl = (filePath: string) => { let url = filePath @@ -77,19 +78,22 @@ export class MarkdownDB { // TODO support glob patterns ignorePatterns = [], pathToUrlResolver = defaultFilePathToUrl, - customConfig = {}, + customConfig, watch = false, + configFilePath, }: { folderPath: string; ignorePatterns?: RegExp[]; pathToUrlResolver?: (filePath: string) => string; customConfig?: CustomConfig; watch?: boolean; + configFilePath?: string; }) { + const config = customConfig || (await loadConfig(configFilePath)) || {}; const fileObjects = indexFolder( folderPath, pathToUrlResolver, - customConfig, + config, ignorePatterns ); await this.saveDataToDisk(fileObjects); @@ -100,10 +104,17 @@ export class MarkdownDB { }); const filePathsToIndex = recursiveWalkDir(folderPath); - const computedFields = customConfig.computedFields || []; + const computedFields = config.computedFields || []; const handleFileEvent = (event: string, filePath: string) => { - if (!shouldIncludeFile(filePath, ignorePatterns)) { + if ( + !shouldIncludeFile({ + filePath, + ignorePatterns, + includeGlob: config.include, + excludeGlob: config.exclude, + }) + ) { return; }