From 700ad94925f244bd71e8897047d7f61b01c58a19 Mon Sep 17 00:00:00 2001 From: Pedro Calado Date: Sat, 14 Jan 2023 14:21:50 +0000 Subject: [PATCH] Use KV to store notify patterns (#1) * Add script to store notify-patterns in KV. * Store notify-patterns in KV on bootstrap. * Read notify-patterns from KV when executing. --- .gitignore | 4 +++ README.md | 21 ++++++++++++--- scripts/bootstrap.sh | 35 +++--------------------- scripts/common.sh | 31 ++++++++++++++++++++++ scripts/put-patterns.sh | 21 +++++++++++++++ src/index.ts | 59 +++++++++++++++++++++++++++++++---------- 6 files changed, 123 insertions(+), 48 deletions(-) create mode 100644 scripts/common.sh create mode 100755 scripts/put-patterns.sh diff --git a/.gitignore b/.gitignore index fa218db..fccc121 100644 --- a/.gitignore +++ b/.gitignore @@ -171,3 +171,7 @@ dist .dev.vars wrangler.toml .wrangler/ + +# notify patterns + +.notify-patterns.json diff --git a/README.md b/README.md index a528bf6..97d4046 100644 --- a/README.md +++ b/README.md @@ -36,16 +36,31 @@ In order to send notifications, we currently only support Discord Webhooks. NORDIGEN_SECRET_ID= NORDIGEN_SECRET_KEY= NORDIGEN_ACCOUNT_ID= - NOTIFY_PATTERN= DISCORD_URL= ``` -3. Install the dependencies: +3. Create a `.notify-patterns.json` file with the transaction patterns to notify: + ```json + [ + { + "name": "salary", + "pattern": "Weyland-Yutani" + }, + { + "name": "mortgage", + "pattern": "Goliath National Bank" + } + ] + ``` + Patterns are tested against the transaction description. + If you need to update them later, edit the file above and run `./scripts/put-patterns.sh`. + +4. Install the dependencies: ```shell npm i ``` -4. Run the bootstrap script to create all the resources and deploy: +5. Run the bootstrap script to create all the resources and deploy: ```shell npm run bootstrap ``` diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 30b410f..0eb2065 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -2,37 +2,7 @@ set -euf -o pipefail -NO_D1_WARNING=true -export NO_D1_WARNING - -export RED="\033[1;31m" -export GREEN="\033[1;32m" -export YELLOW="\033[1;33m" -export BLUE="\033[1;34m" -export PURPLE="\033[1;35m" -export CYAN="\033[1;36m" -export GREY="\033[0;37m" -export RESET="\033[m" - -check_command() { - if ! command -v "$1" &> /dev/null - then - echo "$1 could not be found." - echo "Exiting." - exit 1 - fi -} - -print_error() { - echo -e "${RED}$1${RESET}" -} - -print_info() { - echo -e "${CYAN}$1${RESET}" -} -print_success() { - echo -e "${GREEN}$1${RESET}" -} +source ./scripts/common.sh check_command jq check_command npx @@ -57,6 +27,9 @@ npx wrangler kv:namespace create "KV" KV_ID=$(npx wrangler kv:namespace list | jq -r '.[] | select(.title == "bankman-KV") | .id') export KV_ID +print_info "> Storing notify patterns in KV" +npx wrangler kv:key --namespace-id $KV_ID put 'transaction-matchers' "$(cat .notify-patterns.json | jq -c .)" + print_info "> Creating D1 database" npx wrangler d1 create bankmandb # shellcheck disable=SC2034 diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100644 index 0000000..82f7483 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,31 @@ +NO_D1_WARNING=true +export NO_D1_WARNING + +export RED="\033[1;31m" +export GREEN="\033[1;32m" +export YELLOW="\033[1;33m" +export BLUE="\033[1;34m" +export PURPLE="\033[1;35m" +export CYAN="\033[1;36m" +export GREY="\033[0;37m" +export RESET="\033[m" + +check_command() { + if ! command -v "$1" &> /dev/null + then + echo "$1 could not be found." + echo "Exiting." + exit 1 + fi +} + +print_error() { + echo -e "${RED}$1${RESET}" +} + +print_info() { + echo -e "${CYAN}$1${RESET}" +} +print_success() { + echo -e "${GREEN}$1${RESET}" +} diff --git a/scripts/put-patterns.sh b/scripts/put-patterns.sh new file mode 100755 index 0000000..767ded8 --- /dev/null +++ b/scripts/put-patterns.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -euf -o pipefail + +source scripts/common.sh + +check_command jq +check_command npx + +if ! npm exec --no -- wrangler --version &> /dev/null +then + # shellcheck disable=SC2016 + print_error "wrangler could not be found. Did you run \`npm install\` ?" + print_error "Exiting." + exit 1 +fi + +# shellcheck disable=SC2034 +KV_ID=$(npx wrangler kv:namespace list | jq -r '.[] | select(.title == "bankman-KV") | .id') + +npx wrangler kv:key --namespace-id $KV_ID put 'transaction-matchers' "$(cat .notify-patterns.json | jq -c .)" \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 34a1d0c..0cfb0ff 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,7 +15,6 @@ export interface Env { NORDIGEN_SECRET_KEY: string; NORDIGEN_ACCOUNT_ID: string; NORDIGEN_AGREEMENT_ID: string; - NOTIFY_PATTERN: string; DISCORD_URL: string; KV: KVNamespace; DB: D1Database; @@ -29,6 +28,8 @@ const DAYS_TO_FETCH = 2; const NOTIFY_EXPIRATION_DAYS = 7; // key in KV to store date when we last notified agreement expiration const NOTIFY_EXPIRATION_KEY = "agreement-expiration-notified"; +// key in KV to store match transaction patterns +const TRANSACTION_MATCHERS_KEY = "transaction-matchers"; // Nordigen API host const NORDIGEN_HOST = "https://ob.nordigen.com" @@ -63,6 +64,11 @@ type NordigenTransactions = { } } +type TransactionMatcher = { + name: string; + pattern: string; +} + async function fetchNordigenToken( secretId: string, secretKey: string, @@ -223,11 +229,26 @@ async function notifyDiscord( }); } +async function fetchTransactionMatchers(kv: KVNamespace): Promise { + const matchersRaw = await kv.get(TRANSACTION_MATCHERS_KEY); + if (matchersRaw === null) { + return [] as TransactionMatcher[]; + } + + return JSON.parse(matchersRaw) as TransactionMatcher[]; +} + async function execute(env: Env) { try { await doExecute(env); - } catch (e) { - console.error("error", e); + } catch (e: any) { + const stringed = JSON.stringify(e); + console.error("stringed", stringed); + console.error("type of", typeof e); + console.error("properties", e.message, e.lineNumber, e.fileName. e.stack); + for(var property in e){ + console.log("error props", e[property]); + } } } @@ -257,23 +278,33 @@ async function doExecute(env: Env) { // store transactions in DB await storeBankTransactions(env.DB, results.transactions.booked); - const re = RegExp(env.NOTIFY_PATTERN); + // read transaction matchers from KV + const transactionMatchers = await fetchTransactionMatchers(env.KV); - // list transactions with matching pattern - const matched = results.transactions.booked.filter( - (transaction) => re.test(transaction.remittanceInformationUnstructured) - ); + // create map of matching transactions + const matched = new Map(); - if (matched.length === 0) { + transactionMatchers.forEach(matcher => { + const re = RegExp(matcher.pattern); + results.transactions.booked.forEach(transaction => { + if (re.test(transaction.remittanceInformationUnstructured)) { + matched.set(matcher.name, transaction); + } + }) + }) + + if (matched.size === 0) { return; } - console.log("matched", matched); + const matchedList = Array.from(matched.values()); + + console.log("matched", matchedList); // list transactions which haven't yet been notified - const checkNotifiedResults = await checkNotifiedTransactions(env.DB, matched); + const checkNotifiedResults = await checkNotifiedTransactions(env.DB, matchedList); - const toNotify = matched.filter((_, index) => { + const toNotify = matchedList.filter((_, index) => { const results = checkNotifiedResults[index].results; return !results || results.length === 0 }); @@ -285,10 +316,10 @@ async function doExecute(env: Env) { console.log("to notify", toNotify); // notify transactions - matched.forEach(async tx => { + matched.forEach(async (tx, name) => { await notifyDiscord( env.DISCORD_URL, - generateTransactionNotification(env.NOTIFY_PATTERN, tx), + generateTransactionNotification(name, tx), ); });