From 4bba2645b5647356878ce0d8c2219043d755084a Mon Sep 17 00:00:00 2001 From: Aaron Dixon Date: Fri, 23 Dec 2022 22:06:43 -0600 Subject: [PATCH] Integrate HikariCP db cxn pool and metrics --- deps.edn | 3 +- src/me/untethr/nostr/app.clj | 19 ++++++++--- src/me/untethr/nostr/store.clj | 61 +++++++++++++++++++++++++++++++--- 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/deps.edn b/deps.edn index fd0c098..6b8992f 100644 --- a/deps.edn +++ b/deps.edn @@ -7,6 +7,7 @@ org.xerial/sqlite-jdbc {:mvn/version "3.40.0.0"} com.github.seancorfield/next.jdbc {:mvn/version "1.3.847"} + com.zaxxer/HikariCP {:mvn/version "5.0.1"} fr.acinq.secp256k1/secp256k1-kmp {:mvn/version "0.7.0"} fr.acinq.secp256k1/secp256k1-kmp-jni-jvm {:mvn/version "0.7.0"} @@ -20,7 +21,7 @@ metrics-clojure/metrics-clojure {:mvn/version "2.10.0"} metrics-clojure-jvm/metrics-clojure-jvm {:mvn/version "2.10.0"} - io.dropwizard.metrics/metrics-json {:mvn/version "4.2.14"} + io.dropwizard.metrics/metrics-json {:mvn/version "4.2.15"} org.clojure/tools.logging {:mvn/version "1.2.4"} ch.qos.logback/logback-classic {:mvn/version "1.4.5"}} diff --git a/src/me/untethr/nostr/app.clj b/src/me/untethr/nostr/app.clj index 79a4db1..51c0a62 100644 --- a/src/me/untethr/nostr/app.clj +++ b/src/me/untethr/nostr/app.clj @@ -20,7 +20,8 @@ [org.httpkit.server :as hk] [ring.middleware.params] [reitit.ring :as ring]) - (:import (java.nio.charset StandardCharsets) + (:import (com.codahale.metrics MetricRegistry) + (java.nio.charset StandardCharsets) (java.util UUID) (java.io File) (javax.sql DataSource) @@ -582,8 +583,7 @@ (defn go! "Start the server and sleep forever, blocking the main process thread." [^Conf conf nip05-json nip11-json] - (let [^DataSource db (store/init! (:sqlite-file conf)) - ;; This is our subscription registry, stored in an atom for concurrent + (let [;; This is our subscription registry, stored in an atom for concurrent ;; access. All updates to this atom will be implementd in the subscribe ;; namespace. subs-atom (atom (subscribe/create-empty-subs)) @@ -599,12 +599,21 @@ ;; any corresponding ongoing fulfillment for that id; or if the websocket ;; is closed, we kill any associated fulfillments). fulfill-atom (atom (fulfill/create-empty-registry)) + ;; We have a cyclic dependency between our db/datasource and metric + ;; registry. (Mainly we want the metric registry to be able to query + ;; the db for max-rowid. But we also want our datasource to be able + ;; to report metrics.) So we use a holder volatile so the metrics + ;; registry can use it after it's been instantiated. + db-holder (volatile! nil) ;; We report various server metrics and expose them via http for ;; observability (we use https://metrics.dropwizard.io/ for this): metrics (metrics/create-metrics - #(store/max-event-rowid db) + #(if @db-holder + (store/max-event-rowid @db-holder) -1) #(subscribe/num-subscriptions subs-atom) - #(subscribe/num-firehose-filters subs-atom))] + #(subscribe/num-firehose-filters subs-atom)) + ^DataSource db (store/init! (:sqlite-file conf) ^MetricRegistry (:codahale-registry metrics)) + _ (vreset! db-holder db)] ;; Run our ring-compatible httpkit server on the configured port. :max-ws ;; here is the max websocket message size. (hk/run-server diff --git a/src/me/untethr/nostr/store.clj b/src/me/untethr/nostr/store.clj index 4a00e8d..870967e 100644 --- a/src/me/untethr/nostr/store.clj +++ b/src/me/untethr/nostr/store.clj @@ -4,13 +4,63 @@ [clojure.tools.logging :as log] [next.jdbc :as jdbc] [next.jdbc.result-set :as rs]) - (:import (javax.sql DataSource) + (:import (com.codahale.metrics MetricRegistry) + (com.zaxxer.hikari HikariConfig HikariDataSource) + (com.zaxxer.hikari.metrics.dropwizard CodahaleMetricsTrackerFactory) + (javax.sql DataSource) (org.sqlite SQLiteException))) -(def get-datasource* +(defn- create-hikari-datasource + ^HikariDataSource [jdbc-url] + (HikariDataSource. + (doto (HikariConfig.) + (.setJdbcUrl jdbc-url) + ;; note: jdbc.next with-transaction disables and re-enables auto-commit + ;; before/after running the transaction. + (.setAutoCommit true) + ;; how long for a connection request to wait before throwing a SQLException + ;; from DataSource/getConnection (See also maximumPoolSize below) + (.setConnectionTimeout (* 15 1000)) ;; 15s. + ;; Setting 0 here means we never remove idle connections from the pool. + (.setIdleTimeout 0) + ;; See docs @ https://github.com/brettwooldridge/HikariCP + (.setKeepaliveTime (* 5 60 1000)) ;; 5 minutes. + ;; And at some point read, + ;; https://github.com/brettwooldridge/HikariCP/wiki/About-Pool-Sizing + (.setMaximumPoolSize 10) + ;; note: we leave .setMinimumIdle alone per docs recommendation. + ;; Setting 0 here means no maximum lifetime to a connection in the pool. + (.setMaxLifetime 0) + ;; note: we will leave .setValidationTimeout as default. it must be + ;; less than conn timeout. when HikariCP takes a connection from pool + ;; this is how long it's allowed to validate it before returning it + ;; from getConnection. + ;; Trying 15s for now for leakDetectionThreshold. + (.setLeakDetectionThreshold (* 15 1000)) + ))) + +(defn- create-connection-pool + "Create a connection pool. Our configuration here has all of our connections + living forever in a fixed-sized pool. While the pool may or may not give us + enormous benefit over a file-sys based db like sqlite, it's integration with + metrics gives us a ton of observability with what's happening with the db." + (^DataSource [jdbc-url] + (create-connection-pool jdbc-url nil)) + (^DataSource [jdbc-url ^MetricRegistry metric-registry] + (let [the-pool (create-hikari-datasource jdbc-url)] + (when (some? metric-registry) + (.setMetricsTrackerFactory the-pool + (CodahaleMetricsTrackerFactory. metric-registry))) + the-pool))) + +(def get-unpooled-datasource* (memoize #(jdbc/get-datasource (str "jdbc:sqlite:" %)))) +(def get-datasource* + (memoize + #(create-connection-pool (str "jdbc:sqlite:" %1) %2))) + (defn- comment-line? [line] (str/starts-with? line "--")) @@ -27,6 +77,7 @@ acc))))) (defn apply-schema! [db] + {:pre [(some? db)]} (doseq [statement (parse-schema)] (try (jdbc/execute-one! db [statement]) @@ -38,12 +89,12 @@ (throw e)))))) (defn init! - ^DataSource [path] - (doto (get-datasource* path) + ^DataSource [path ^MetricRegistry metric-registry] + (doto (get-datasource* path metric-registry) apply-schema!)) (comment - (init! "./n.db")) + (init! "./n.db" nil)) ;; --