Merge branch 'main' of https://github.com/datopian/markdowndb

__mocks__/content/taskmetadata.md

@@ -0,0 +1,6 @@
+---
+title: Task metadata fixture
+---
+
+- [ ] Task without metadata
+- [x] Task with metadata #tag1 #tag2 [person:: Athena Person] [due:: 2024-10-01] #tag3

src/lib/databaseUtils.ts

@@ -1,10 +1,10 @@
 import { Knex } from "knex";
-import { MddbFile, MddbTag, MddbLink, MddbFileTag, File } from "./schema.js";
+import { MddbFile, MddbTag, MddbTask, MddbLink, MddbFileTag, File } from "./schema.js";
 import path from "path";
 import { WikiLink } from "./parseFile.js";
 
 export async function resetDatabaseTables(db: Knex) {
-  const tableNames = [MddbTag, MddbFileTag, MddbLink];
+  const tableNames = [MddbTag, MddbFileTag, MddbLink, MddbTask];
   // Drop and Create tables
   for (const table of tableNames) {
     await table.deleteTable(db);
@@ -81,3 +81,19 @@ export function getUniqueProperties(objects: any[]): string[] {
 
   return uniqueProperties;
 }
+
+export function mapTasksToInsert(file: any) {
+  return file.tasks.map((task: any) => {
+    return {
+      file: file._id,
+      description: task.description,
+      checked: task.checked,
+      metadata: JSON.stringify(task.metadata),
+      created: task.created,
+      due: task.due,
+      completion: task.completion,
+      start: task.start,
+      scheduled: task.scheduled,
+    };
+  });
+}

src/lib/markdowndb.ts

@@ -1,7 +1,7 @@
 import path from "path";
 import knex, { Knex } from "knex";
 
-import { MddbFile, MddbTag, MddbLink, MddbFileTag } from "./schema.js";
+import { MddbFile, MddbTag, MddbLink, MddbFileTag, MddbTask } from "./schema.js";
 import { indexFolder, shouldIncludeFile } from "./indexFolder.js";
 import {
   resetDatabaseTables,
@@ -11,6 +11,7 @@ import {
   mapFileTagsToInsert,
   getUniqueValues,
   getUniqueProperties,
+  mapTasksToInsert,
 } from "./databaseUtils.js";
 import fs from "fs";
 import { CustomConfig } from "./CustomConfig.js";
@@ -178,11 +179,14 @@ export class MarkdownDB {
       .filter(isLinkToDefined);
     const fileTagsToInsert = fileObjects.flatMap(mapFileTagsToInsert);
 
+    const tasksToInsert = fileObjects.flatMap(mapTasksToInsert);
+
     writeJsonToFile(".markdowndb/files.json", fileObjects);
     await MddbFile.batchInsert(this.db, filesToInsert);
     await MddbTag.batchInsert(this.db, tagsToInsert);
     await MddbFileTag.batchInsert(this.db, fileTagsToInsert);
     await MddbLink.batchInsert(this.db, getUniqueValues(linksToInsert));
+    await MddbTask.batchInsert(this.db, tasksToInsert);
   }
 
   /**

src/lib/parseFile.ts

@@ -6,6 +6,7 @@ import * as path from "path";
 import gfm from "remark-gfm";
 import remarkWikiLink from "@portaljs/remark-wiki-link";
 import { Root } from "remark-parse/lib";
+import { MetaData, Task } from "./schema";
 
 export function parseFile(source: string, options?: ParsingOptions) {
   // Metadata
@@ -189,22 +190,30 @@ export const extractWikiLinks = (ast: Root, options?: ParsingOptions) => {
   return wikiLinks;
 };
 
-export interface Task {
-  description: string;
-  checked: boolean;
-}
-
 export const extractTasks = (ast: Root) => {
   const nodes = selectAll("*", ast);
   const tasks: Task[] = [];
   nodes.map((node: any) => {
     if (node.type === "listItem") {
       const description = recursivelyExtractText(node).trim();
-      const checked = node.checked;
-      if (checked !== null && checked !== undefined) {
+      const metadata = extractAllTaskMetadata(description);
+      const checked = node.checked !== null && node.checked !== undefined ? node.checked : null;
+      const created = metadata.created !== null && metadata.created !== undefined ? metadata.created : null;
+      const due = metadata.due !== null && metadata.due !== undefined ? metadata.due : null;
+      const completion = metadata.completion !== null && metadata.completion !== undefined ? metadata.completion : null;
+      const scheduled = metadata.scheduled !== null && metadata.scheduled !== undefined ? metadata.scheduled : null;
+      const start = metadata.start !== null && metadata.start !== undefined ? metadata.start : null;
+
+      if (checked !== null) {
         tasks.push({
           description,
           checked,
+          created,
+          due,
+          completion,
+          scheduled,
+          start,
+          metadata: metadata,
         });
       }
     }
@@ -221,6 +230,34 @@ function recursivelyExtractText(node: any) {
   } else {
     return "";
   }
+};
+
+export function extractAllTaskMetadata(description: string) : MetaData  {
+  // Extract metadata fields from the description with the form [field:: value]
+  // where field is the name of the metadata without spaces and value is the value of the metadata
+  // There can be multiple metadata fields in the description
+  const metadataRegex = /\[(.*?)::(.*?)\]/g;
+  const matches = description.match(metadataRegex);
+  if (matches) {
+    const metadata: MetaData = {};
+    matches.forEach((match) => {
+      // extract field and value from groups in the match
+      const allMatches = match.matchAll(metadataRegex).next().value;
+      const field = allMatches[1].trim();
+      const value = allMatches[2].trim();
+      metadata[field] = value;
+    }); // Add closing parenthesis here
+    const tags = extractTags(description);
+    metadata["tags"] = tags;
+    return metadata;
+  } else {
+    return {};
+  } 
+
+
+
+
+
 }
 
 // links = extractWikiLinks({

src/lib/process.ts

@@ -2,13 +2,14 @@ import crypto from "crypto";
 import fs from "fs";
 import path from "path";
 
-import { File } from "./schema.js";
+import { File, Task } from "./schema.js";
 import { WikiLink, parseFile } from "./parseFile.js";
 import { Root } from "remark-parse/lib/index.js";
 
 export interface FileInfo extends File {
   tags: string[];
   links: WikiLink[];
+  tasks: Task[];
 }
 
 // this file is an extraction of the file info parsing from markdowndb.ts without any sql stuff
@@ -39,6 +40,7 @@ export function processFile(
     metadata: {},
     tags: [],
     links: [],
+    tasks: [],
   };
 
   // if not a file type we can parse exit here ...
@@ -72,5 +74,7 @@ export function processFile(
     customFieldFunction(fileInfo, ast);
   }
 
+  fileInfo.tasks = metadata?.tasks || [];
+
   return fileInfo;
 }

src/lib/schema.ts

@@ -9,6 +9,7 @@ export enum Table {
   Tags = "tags",
   FileTags = "file_tags",
   Links = "links",
+  Tasks = "tasks",
 }
 
 type MetaData = {
@@ -308,4 +309,74 @@ class MddbFileTag {
   }
 }
 
-export { File, MddbFile, Link, MddbLink, Tag, MddbTag, FileTag, MddbFileTag };
+interface Task {
+  description: string;
+  checked: boolean;
+  due: string | null;
+  completion: string | null;
+  created: string;
+  start: string | null;
+  scheduled: string | null;
+  metadata: MetaData | null;
+
+}
+
+class MddbTask {
+  static table = Table.Tasks;
+  description: string;
+  checked: boolean;
+  due: string | null;
+  completion: string | null;
+  created: string;
+  start: string | null;
+  scheduled: string | null;
+  metadata: MetaData | null;
+
+  constructor(task: Task) {
+    this.description = task.description;
+    this.checked = task.checked;
+    this.due = task.due;
+    this.completion = task.completion;
+    this.created = task.created;
+    this.start = task.start;
+    this.scheduled = task.scheduled;
+    this.metadata = task.metadata;
+  }
+
+  static async createTable(db: Knex) {
+    const creator = (table: Knex.TableBuilder) => {
+      table.string("description").notNullable();
+      table.boolean("checked").notNullable();
+      table.string("file").notNullable();
+      table.string("due");
+      table.string("completion");
+      table.string("created");
+      table.string("start");
+      table.string("scheduled");
+      table.string("metadata");
+    };
+    const tableExists = await db.schema.hasTable(this.table);
+
+    if (!tableExists) {
+      await db.schema.createTable(this.table, creator);
+    }
+  }
+
+  static async deleteTable(db: Knex) {
+    await db.schema.dropTableIfExists(this.table);
+  }
+
+  static batchInsert(db: Knex, tasks: Task[]) {
+    if (tasks.length >= 500) {
+      const promises = [];
+      for (let i = 0; i < tasks.length; i += 500) {
+        promises.push(db.batchInsert(Table.Tasks, tasks.slice(i, i + 500)));
+      }
+      return Promise.all(promises);
+    } else {
+      return db.batchInsert(Table.Tasks, tasks);
+    }
+  }
+}
+
+export { MetaData, File, MddbFile, Link, MddbLink, Tag, MddbTag, FileTag, MddbFileTag, Task, MddbTask };

src/tests/computedField.spec.ts

@@ -60,14 +60,32 @@ describe("Can parse a file and get file info", () => {
                 {
                     checked: false,
                     description: "uncompleted task 2",
+                    metadata: {},
+                    created: null,
+                    due: null,
+                    completion: null,
+                    start: null,
+                    scheduled: null,
                 },
                 {
                     checked: true,
                     description: "completed task 1",
+                    metadata: {},
+                    created: null,
+                    due: null,
+                    completion: null,
+                    start: null,
+                    scheduled: null,
                 },
                 {
                     checked: true,
                     description: "completed task 2",
+                    metadata: {},
+                    created: null,
+                    due: null,
+                    completion: null,
+                    start: null,
+                    scheduled: null,
                 },
             ],
         });
@@ -195,14 +213,32 @@ describe("Can parse a file and get file info", () => {
                 {
                     checked: false,
                     description: "uncompleted task 2",
+                    metadata: {},
+                    created: null,
+                    due: null,
+                    completion: null,
+                    start: null,
+                    scheduled: null,
                 },
                 {
                     checked: true,
                     description: "completed task 1",
+                    metadata: {},
+                    created: null,
+                    due: null,
+                    completion: null,
+                    start: null,
+                    scheduled: null,
                 },
                 {
                     checked: true,
                     description: "completed task 2",
+                    metadata: {},
+                    created: null,
+                    due: null,
+                    completion: null,
+                    start: null,
+                    scheduled: null,
                 },
             ],
         });
@@ -261,14 +297,33 @@ describe("Can parse a file and get file info", () => {
                 {
                     checked: false,
                     description: "uncompleted task 2",
+                    metadata: {},
+                    created: null,
+                    due: null,
+                    completion: null,
+                    start: null,
+                    scheduled: null,
+
                 },
                 {
                     checked: true,
                     description: "completed task 1",
+                    metadata: {},
+                    created: null,
+                    due: null,
+                    completion: null,
+                    start: null,
+                    scheduled: null,
                 },
                 {
                     checked: true,
                     description: "completed task 2",
+                    metadata: {},
+                    created: null,
+                    due: null,
+                    completion: null,
+                    start: null,
+                    scheduled: null,
                 },
             ],
         });