Rewrite indexing; a few UI changes

src/common.ts

@@ -14,7 +14,7 @@ type Message = SearchMessage | GotoMessage;
 type SearchResult = {
     id: string,
     title: string,
-    notes: NoteRef[]
+    note: NoteRef
 };
 
 type NoteRef = {

src/plugin/DbUtils.ts

@@ -0,0 +1,27 @@
+import { Database } from "sqlite3";
+
+async function query(db: Database, query, ...params): Promise<any[]> {
+	return new Promise((resolve, reject) => {
+		db.all(query, params, (err, rows) => {
+			if (!!err) {
+				reject(err);
+			} else {
+				resolve(rows);
+			}
+		});
+	});
+}
+
+async function run(db: Database, query, ...params): Promise<boolean> {
+    return new Promise(function(resolve, reject) {
+        db.run(query, params, function(err)  {
+            if(err) {
+                reject(err.message);
+            } else {
+                resolve(true);
+            }
+        });
+    });
+}
+
+export { query, run };

src/plugin/ResourceIndex.ts

@@ -0,0 +1,119 @@
+import JoplinData from "api/JoplinData";
+import { Database } from "sqlite3";
+import { pdfToText } from "../index/pdf";
+import { query, run } from "./DbUtils";
+
+const SCHEMA_VERSION = 1;
+
+// resource descriptor as returned by Joplin's data API
+type JoplinResource = {
+    id: string,
+    title: string,
+    mime: string,
+    updated_time: number
+};
+
+type Resource = {
+    id: string,
+    title: string
+}
+
+class ResourceIndex {
+
+    private db: Database;
+    private data: JoplinData;
+    private resourceDir: string;
+
+    public static async init(db: Database, data: JoplinData, resourceDir: string): Promise<ResourceIndex> {
+        // init the database
+        await run(db, 'CREATE TABLE IF NOT EXISTS settings (name TEXT PRIMARY KEY, value TEXT)');
+        const versionResult = await query(db, 'SELECT value FROM settings WHERE name = ?', 'version');
+        const version = !!versionResult ? Number(versionResult[0].value) : -1;
+        console.log(`plugin schema version: ${SCHEMA_VERSION}, db schema version: ${version}`);
+
+        if (version !== SCHEMA_VERSION) {
+            console.log('Schema version mismatch - rebuilding the database');
+            await initDatabase(db);
+        }
+        return new ResourceIndex(db, data, resourceDir);
+    }
+
+    constructor(db: Database, data: JoplinData, resourceDir: string) {
+        this.db = db;
+        this.data = data;
+        this.resourceDir = resourceDir;
+    }
+
+    public async update() {
+        console.log('Updating index');
+        let page = 1;
+        let response = await this.data.get(['resources'], { page: page, fields: ['id', 'title', 'mime', 'updated_time']});
+        response.items.forEach(async (r: JoplinResource) => await this.indexResource(r));
+        while (!!response.has_more) {
+            page += 1;
+            response = await this.data.get(['resources'], { page: page, fields: ['id', 'title', 'mime', 'updated_time']});
+            response.items.forEach(async (r: JoplinResource) => await this.indexResource(r));
+        }
+    }
+
+    private async indexResource(resource: JoplinResource) {
+        try {
+            console.log(`Indexing ${JSON.stringify(resource)}`);
+
+            const lastIndexed = await this.getLastIndexTime(resource.id);
+            if (!!lastIndexed && lastIndexed > resource.updated_time) {
+                console.log(`Skip indexing ${resource.id}/${resource.title}`);
+                return;
+            }
+
+            const isSupportedType = resource.mime === 'application/pdf';
+            if (isSupportedType) {
+                const text = await pdfToText(`${this.resourceDir}/${resource.id}.pdf`);
+                console.log(`extracted text from ${resource.id}/${resource.title}: ${text.substring(0, 100)}`);
+                await run(this.db, 'INSERT INTO resources_fts VALUES(?, ?, ?)', resource.id, resource.title, text);
+            } else {
+                console.log(`Skip indexing ${resource.id} - MIME type not supported`);
+            }
+
+            await this.updateLastIndexTime(resource.id);
+        } catch (e) {
+            console.log(`error indexing ${JSON.stringify(resource)}: ${e}`);
+        }
+    }
+
+    private async updateLastIndexTime(id: string) {
+        await run(this.db, 'INSERT INTO index_time VALUES(?, ?) ON CONFLICT(id) DO UPDATE SET index_time = ?',
+            id, Date.now(), Date.now());
+    }
+
+    private async getLastIndexTime(id: string): Promise<number> {
+        const result = await query(this.db, 'SELECT index_time FROM index_time WHERE id = ?', id);
+		return !!result && Array.isArray(result) && result.length > 0 ? Number(result[0].index_time) : 0;
+    }
+
+    public async rebuild() {
+        console.log('Rebuilding the database');
+
+        await initDatabase(this.db);
+        await this.update();
+    }
+
+    public async query(text: string): Promise<Resource[]> {
+        return await query(this.db, 'SELECT id, title FROM resources_fts WHERE text MATCH ?', text) as Resource[];
+    }
+}
+
+async function initDatabase(db: Database) {
+    // rebuild index
+    await run(db, 'DROP TABLE IF EXISTS resources_fts');
+    await run(db, 'CREATE VIRTUAL TABLE IF NOT EXISTS resources_fts USING fts5(id, title, text)');
+
+    // create or clean index_time
+    await run(db, 'CREATE TABLE IF NOT EXISTS index_time (id TEXT PRIMARY KEY, index_time INTEGER)');
+    await run(db, 'DELETE FROM index_time');
+
+    await run(db, 'VACUUM');
+    await run(db, `INSERT INTO settings VALUES('version', ${SCHEMA_VERSION}) ON CONFLICT(name) DO UPDATE SET value=${SCHEMA_VERSION}`);
+}
+
+export { Resource, ResourceIndex };

src/plugin/index.ts

@@ -1,64 +1,63 @@
 import joplin from 'api';
-import JoplinData from 'api/JoplinData';
 import { MenuItemLocation } from 'api/types';
-import { pdfToText} from '../index/pdf';
-import { addToIndex, initDb, query } from './database';
-import { Database } from 'sqlite3';
 import { Message, SearchResult, NoteRef } from '../common';
+import { Resource, ResourceIndex } from './ResourceIndex';
+import { Database, sqlite3 } from 'sqlite3';
 
-async function indexResources(api: JoplinData, resourceDir: string, db: Database) {
-	let page = 0;
-	let response = await api.get(['resources'], { page: page, fields: ['id', 'title', 'mime', 'updated_time']});
-	console.log(`response: ${JSON.stringify(response)}`);
-	response.items.forEach(r => indexResource(r, resourceDir, db));
-	while (!!response.has_more) {
-		page += 1;
-		response = await api.get(['resources'], { page: page, fields: ['id', 'title', 'mime', 'updated_time']});
-		console.log(`response: ${JSON.stringify(response)}`);
-		response.items.forEach(r => indexResource(r, resourceDir, db));
-	}
-}
 
-async function indexResource(resource: any, resourceDir: string, db: Database) {
-	console.log(`index ${JSON.stringify(resource)}`);
-	const lastIndexed = await query(db, 'SELECT index_time FROM index_time WHERE id = ?', resource.id);
-	console.log(`indexed=${lastIndexed}, updated=${resource.updated_time}`);
-	if (lastIndexed	 > resource.updated_time) {
-		console.log(`Skip indexing ${resource.id}/${resource.title}`);
-		return;
-	}
-	if (resource.mime === 'application/pdf') {
-		const text = await pdfToText(`${resourceDir}/${resource.id}.pdf`);
-		console.log(`extracted text from ${resource.title}: ${text.substring(0, 100)}`);
-		addToIndex(db, resource.title, resource.id, text);
-	}
-}
-async function transformResult(searchResult: any[]): Promise<SearchResult[]> {
+async function transformResult(searchResult: Resource[]): Promise<SearchResult[]> {
 	const res: SearchResult[] = [];
 	console.log(`result: ${JSON.stringify(searchResult)}`);
 	for (let i = 0; i < searchResult.length; i++) {
 		const resource = searchResult[i];
 		// TODO collect promises and await all
 		const notes: NoteRef[] = (await joplin.data.get(['resources', resource.id, 'notes'], { fields: ['id', 'title']})).items;
-		res.push({
+		notes.forEach(n => res.push({
 			id: resource.id,
 			title: resource.title,
-			notes: notes,
-		});
+			note: n
+		}));
 	}
 	return res;
 }
 
 joplin.plugins.register({
 	onStart: async function() {
+		const resourceDir = await joplin.settings.globalValue('resourceDir');
+
 		const dbPath = await joplin.plugins.dataDir();
-		const db = await initDb(dbPath, joplin.plugins.require('sqlite3'));
+		const sqlite3: sqlite3 = joplin.plugins.require('sqlite3');
+		const db: Database = new sqlite3.Database(`${dbPath}/resources.sqlite`);
 
-		const resourceDir = await joplin.settings.globalValue('resourceDir');
-		await indexResources(joplin.data, resourceDir, db);
+		const data = joplin.data;
+		const index: ResourceIndex = await ResourceIndex.init(db, data, resourceDir);
+
+		index.update();
+
+		// TODO get interval from settings
+		const indexRefreshInterval = 60 * 1000; // TODO reivew
+		setInterval(() => index.update(), indexRefreshInterval);
+
+		await joplin.commands.register({
+			name: 'indexResources',
+			label: 'Index Now',
+			execute: () => index.update(),
+		});
+		await joplin.commands.register({
+			name: 'rebuildIndex',
+			label: 'Wipe and Rebuild Index',
+			execute: () => index.rebuild(),
+		});
+		await joplin.views.menus.create('resourceSearchTools', 'Resource Search', [
+			{label: 'Index Now', commandName: 'indexResources' },
+			{label: 'Wipe and Rebuild Index', commandName: 'rebuildIndex' },
+		], MenuItemLocation.Tools);
+
+		const resourceSearch = await joplin.views.dialogs.create('resourceSearchDialog');
+
+		// joplin.views.dialogs.setButtons(resourceSearch, []); TODO uncomment when there's a way to close dialog
+		joplin.views.dialogs.setButtons(resourceSearch, [ { id: 'ok' }]);
 
-		const resourceSearch = await joplin.views.dialogs.create('resourceSearch');
-		joplin.views.dialogs.setButtons(resourceSearch, []);
 		joplin.views.dialogs.addScript(resourceSearch, './resource-search-view.js')
 		joplin.views.dialogs.addScript(resourceSearch, './resource-search-view.css')
 
@@ -73,7 +72,7 @@ joplin.plugins.register({
 			console.log(`on message: ${JSON.stringify(msg)}`);
 			switch (msg.type) {
 				case 'search':
-					const result: any[] = await query(db, 'SELECT id,title FROM resources_fts WHERE text MATCH ?', msg.query);
+					const result: any[] = await index.query(msg.query);
 					console.log(`results: ${JSON.stringify(result)}`);
 					return await transformResult(result);
 				case 'goto':
@@ -86,11 +85,11 @@ joplin.plugins.register({
 
 		await joplin.commands.register({
 			name: 'searchAttachments',
-			label: 'Search in attachments',
+			label: 'Search in Attachments',
 			execute: async () => {
 				await joplin.views.dialogs.open(resourceSearch);
 			},
-		})
+		});
 		await joplin.views.menuItems.create('Search in attachments', 'searchAttachments', MenuItemLocation.Edit);
 	},
 });

src/webview/index.ts

@@ -3,8 +3,8 @@ import { GotoMessage, SearchResult } from "src/common";
 declare const webviewApi: any;
 
 function debounce(func: Function, timeout = 300) {
-    let timer;
-    return (...args) => {
+    let timer: any;
+    return (...args: any[]) => {
         clearTimeout(timer);
         timer = setTimeout(() => { func.apply(this, args); }, timeout);
     };
@@ -70,7 +70,7 @@ class SearchDialog {
             webviewApi.postMessage({
                 type: 'goto',
                 resourceId: result.id,
-                noteId: result.notes[0].id //TODO
+                noteId: result.note.id
             } as GotoMessage);
         }
     }
@@ -95,17 +95,8 @@ class SearchDialog {
 
                 const includedIn = document.createElement('div');
                 includedIn.setAttribute('class', 'referencing-notes-cell');
+                includedIn.innerText = `In: ${searchResult.note.title}`;
                 row.appendChild(includedIn);
-
-                const referencingNotesList = document.createElement('div');
-                referencingNotesList.setAttribute('class', 'referencing-notes-list')
-                includedIn.appendChild(referencingNotesList);
-
-                searchResult.notes.forEach(n => {
-                    const noteLink = document.createElement('div');
-                    noteLink.innerText = `In: ${n.title}`;
-                    referencingNotesList.appendChild(noteLink);
-                });
             }
         }
     }

src/webview/resource-search-view.css

@@ -16,9 +16,8 @@
 #query-input {
     width: 100%;
     box-sizing: border-box;
-    border-radius: 8px;
     font-size: var(--joplin-font-size);
-	font-family: var(--joplin-font-family);
+    font-family: var(--joplin-font-family);
 }
 
 #search-results {