Use KV to store notify patterns (#1)

* Add script to store notify-patterns in KV.

* Store notify-patterns in KV on bootstrap.

* Read notify-patterns from KV when executing.

.gitignore

@@ -171,3 +171,7 @@ dist
 .dev.vars
 wrangler.toml
 .wrangler/
+
+# notify patterns
+
+.notify-patterns.json

README.md

@@ -36,16 +36,31 @@ In order to send notifications, we currently only support Discord Webhooks.
     NORDIGEN_SECRET_ID=<secret>
     NORDIGEN_SECRET_KEY=<secret>
     NORDIGEN_ACCOUNT_ID=<account_id>
-    NOTIFY_PATTERN=<pattern>
     DISCORD_URL=<discord-webhook-url>
     ```
 
-3. Install the dependencies:
+3. Create a `.notify-patterns.json` file with the transaction patterns to notify:
+    ```json
+    [
+        {
+            "name": "salary",
+            "pattern": "Weyland-Yutani"
+        },
+        {
+            "name": "mortgage",
+            "pattern": "Goliath National Bank"
+        }
+    ]
+    ```
+    Patterns are tested against the transaction description.
+    If you need to update them later, edit the file above and run `./scripts/put-patterns.sh`.
+
+4. Install the dependencies:
     ```shell
     npm i
     ```
 
-4. Run the bootstrap script to create all the resources and deploy:
+5. Run the bootstrap script to create all the resources and deploy:
     ```shell
     npm run bootstrap
     ```

scripts/bootstrap.sh

@@ -2,37 +2,7 @@
 
 set -euf -o pipefail
 
-NO_D1_WARNING=true
-export NO_D1_WARNING
-
-export RED="\033[1;31m"
-export GREEN="\033[1;32m"
-export YELLOW="\033[1;33m"
-export BLUE="\033[1;34m"
-export PURPLE="\033[1;35m"
-export CYAN="\033[1;36m"
-export GREY="\033[0;37m"
-export RESET="\033[m"
-
-check_command() {
-    if ! command -v "$1" &> /dev/null
-    then
-        echo "$1 could not be found."
-        echo "Exiting."
-        exit 1
-    fi
-}
-
-print_error() {
-    echo -e "${RED}$1${RESET}"
-}
-
-print_info() {
-    echo -e "${CYAN}$1${RESET}"
-}
-print_success() {
-    echo -e "${GREEN}$1${RESET}"
-}
+source ./scripts/common.sh
 
 check_command jq
 check_command npx
@@ -57,6 +27,9 @@ npx wrangler kv:namespace create "KV"
 KV_ID=$(npx wrangler kv:namespace list | jq -r '.[] | select(.title == "bankman-KV") | .id')
 export KV_ID
 
+print_info "> Storing notify patterns in KV"
+npx wrangler kv:key --namespace-id $KV_ID put 'transaction-matchers' "$(cat .notify-patterns.json | jq -c .)"
+
 print_info "> Creating D1 database"
 npx wrangler d1 create bankmandb
 # shellcheck disable=SC2034

scripts/common.sh

@@ -0,0 +1,31 @@
+NO_D1_WARNING=true
+export NO_D1_WARNING
+
+export RED="\033[1;31m"
+export GREEN="\033[1;32m"
+export YELLOW="\033[1;33m"
+export BLUE="\033[1;34m"
+export PURPLE="\033[1;35m"
+export CYAN="\033[1;36m"
+export GREY="\033[0;37m"
+export RESET="\033[m"
+
+check_command() {
+    if ! command -v "$1" &> /dev/null
+    then
+        echo "$1 could not be found."
+        echo "Exiting."
+        exit 1
+    fi
+}
+
+print_error() {
+    echo -e "${RED}$1${RESET}"
+}
+
+print_info() {
+    echo -e "${CYAN}$1${RESET}"
+}
+print_success() {
+    echo -e "${GREEN}$1${RESET}"
+}

scripts/put-patterns.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -euf -o pipefail
+
+source scripts/common.sh
+
+check_command jq
+check_command npx
+
+if ! npm exec --no -- wrangler --version &> /dev/null
+then
+    # shellcheck disable=SC2016
+    print_error "wrangler could not be found. Did you run \`npm install\` ?"
+    print_error "Exiting."
+    exit 1
+fi
+
+# shellcheck disable=SC2034
+KV_ID=$(npx wrangler kv:namespace list | jq -r '.[] | select(.title == "bankman-KV") | .id')
+
+npx wrangler kv:key --namespace-id $KV_ID put 'transaction-matchers' "$(cat .notify-patterns.json | jq -c .)"

src/index.ts

@@ -15,7 +15,6 @@ export interface Env {
 	NORDIGEN_SECRET_KEY: string;
 	NORDIGEN_ACCOUNT_ID: string;
 	NORDIGEN_AGREEMENT_ID: string;
-	NOTIFY_PATTERN: string;
 	DISCORD_URL: string;
 	KV: KVNamespace;
 	DB: D1Database;
@@ -29,6 +28,8 @@ const DAYS_TO_FETCH = 2;
 const NOTIFY_EXPIRATION_DAYS = 7;
 // key in KV to store date when we last notified agreement expiration
 const NOTIFY_EXPIRATION_KEY = "agreement-expiration-notified";
+// key in KV to store match transaction patterns
+const TRANSACTION_MATCHERS_KEY = "transaction-matchers";
 // Nordigen API host
 const NORDIGEN_HOST = "https://ob.nordigen.com"
 
@@ -63,6 +64,11 @@ type NordigenTransactions = {
 	}
 }
 
+type TransactionMatcher = {
+	name: string;
+	pattern: string;
+}
+
 async function fetchNordigenToken(
 	secretId: string,
 	secretKey: string,
@@ -223,11 +229,26 @@ async function notifyDiscord(
 	});
 }
 
+async function fetchTransactionMatchers(kv: KVNamespace): Promise<TransactionMatcher[]> {
+	const matchersRaw = await kv.get(TRANSACTION_MATCHERS_KEY);
+	if (matchersRaw === null) {
+		return [] as TransactionMatcher[];
+	}
+
+	return JSON.parse(matchersRaw) as TransactionMatcher[];
+}
+
 async function execute(env: Env) {
 	try {
 		await doExecute(env);
-	} catch (e) {
-		console.error("error", e);
+	} catch (e: any) {
+		const stringed = JSON.stringify(e);
+		console.error("stringed", stringed);
+		console.error("type of", typeof e);
+		console.error("properties", e.message, e.lineNumber, e.fileName. e.stack);
+		for(var property in e){
+			console.log("error props", e[property]);
+		}
 	}
 }
 
@@ -257,23 +278,33 @@ async function doExecute(env: Env) {
 	// store transactions in DB
 	await storeBankTransactions(env.DB, results.transactions.booked);
 
-	const re = RegExp(env.NOTIFY_PATTERN);
+	// read transaction matchers from KV
+	const transactionMatchers = await fetchTransactionMatchers(env.KV);
 
-	// list transactions with matching pattern
-	const matched = results.transactions.booked.filter(
-		(transaction) => re.test(transaction.remittanceInformationUnstructured)
-	);
+	// create map of matching transactions
+	const matched = new Map<string, BankTransaction>();
 
-	if (matched.length === 0) {
+	transactionMatchers.forEach(matcher => {
+		const re = RegExp(matcher.pattern);
+		results.transactions.booked.forEach(transaction => {
+			if (re.test(transaction.remittanceInformationUnstructured)) {
+				matched.set(matcher.name, transaction);
+			}
+		})
+	})
+
+	if (matched.size === 0) {
 		return;
 	}
 
-	console.log("matched", matched);
+	const matchedList = Array.from(matched.values());
+
+	console.log("matched", matchedList);
 
 	// list transactions which haven't yet been notified
-	const checkNotifiedResults = await checkNotifiedTransactions(env.DB, matched);
+	const checkNotifiedResults = await checkNotifiedTransactions(env.DB, matchedList);
 
-	const toNotify = matched.filter((_, index) => {
+	const toNotify = matchedList.filter((_, index) => {
 		const results = checkNotifiedResults[index].results;
 		return !results || results.length === 0
 	});
@@ -285,10 +316,10 @@ async function doExecute(env: Env) {
 	console.log("to notify", toNotify);
 
 	// notify transactions
-	matched.forEach(async tx => {
+	matched.forEach(async (tx, name) => {
 		await notifyDiscord(
 			env.DISCORD_URL,
-			generateTransactionNotification(env.NOTIFY_PATTERN, tx),
+			generateTransactionNotification(name, tx),
 		);
 	});