Add script to run tests / add to CI job; add optional config params f…

…or queue and cache redis DSNs

.github/workflows/server-ci.yml

@@ -69,18 +69,11 @@ jobs:
         args: --manifest-path server/Cargo.toml -- migrate
 
     - name: Run tests
-      uses: actions-rs/cargo@v1
-      env:
-        DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/postgres"
-        SVIX_REDIS_DSN: "redis://localhost:6379"
-        SVIX_JWT_SECRET: "test value"
-      with:
-        command: test
-        args: --manifest-path server/Cargo.toml  --all --all-features --all-targets
+      working-directory: ./server
+      run: ./run-tests.sh
 
     - name: Stop dependencies
       run: docker-compose -f "server/docker-compose.yml" down
-
 #  deny-check:
 #    name: cargo-deny check
 #    runs-on: ubuntu-latest

server/run-tests.sh

@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+# Run tests with various configurations:
+
+TEST_COMMAND="cargo test --all --all-features --all-targets -- --test-threads 1"
+
+# Common variables:
+export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+export SVIX_JWT_SECRET="test value"
+
+SVIX_QUEUE_TYPE="redis" \
+SVIX_CACHE_TYPE="redis" \
+SVIX_REDIS_DSN="redis://localhost:6379" \
+${TEST_COMMAND}
+
+SVIX_QUEUE_TYPE="rediscluster" \
+SVIX_CACHE_TYPE="rediscluster"  \
+SVIX_QUEUE_REDIS_DSN="redis://localhost:6380" \
+SVIX_CACHE_REDIS_DSN="redis://localhost:6380" \
+${TEST_COMMAND}
+
+

server/svix-server/config.example.toml

@@ -17,11 +17,17 @@ retry_schedule = "5,300,1800,7200,18000,36000,36000"
 # The DSN for the database. Only postgres is currently supported.
 db_dsn = "postgresql://postgres:postgres@pgbouncer/postgres"
 
-# The DSN for redis (can be left empty if not using redis)
+# The DSN for redis (can be left empty if not using redis). This will be default for both redis-based caching and queuing unless overridden
 redis_dsn = "redis://redis:6379"
 
+# Optional DSN for redis-based queue, if diffferent than that for caching. If left empty, `redis_dsn` will be used
+queue_redis_dsn = "redis://redis:6379"
+
+# Optional DSN for redis-based caching, if diffferent than that for queuing. If left empty, `redis_dsn` will be used
+cache_redis_dsn = "redis://redis:6379"
+
 # What kind of message queue to use. Supported: memory, redis (must have redis_dsn configured), rediscluster (add multiple hosts to redis_dsn separated by commas).
-queue_type: "redis"
+queue_type = "redis"
 
 # If true, headers are prefixed with `Webhook-`, otherwise with `Svix-` (default).
 whitelabel_headers = false

server/svix-server/src/cfg.rs

@@ -74,12 +74,21 @@ retry_schedule = "5,300,1800,7200,18000,36000,36000"
 # The DSN for the database. Only postgres is currently supported.
 # db_dsn = "postgresql://postgres:postgres@pgbouncer/postgres"
 
-# The DSN for redis (can be left empty if not using redis)
+# The DSN for redis (can be left empty if not using redis). This will be default for both redis-based caching and queuing unless overridden
 # redis_dsn = "redis://redis:6379"
 
+# Optional DSN for redis-based queue, if diffferent than that for caching. If left empty, `redis_dsn` will be used
+# queue_redis_dsn = "redis://redis:6379"
+
+# Optional DSN for redis-based caching, if diffferent than that for queuing. If left empty, `redis_dsn` will be used
+# cache_redis_dsn = "redis://redis:6379"
+
 # What kind of message queue to use. Supported: memory, redis (must have redis_dsn configured), rediscluster (add multiple hosts to redis_dsn separated by commas).
 queue_type = "redis"
 
+# What kind of cache to use. Supported: redis (must have redis_dsn configured), rediscluster (add multiple hosts to redis_dsn separated by commas).
+cache_type = "redis"
+
 # If true, headers are prefixed with `Webhook-`, otherwise with `Svix-` (default).
 whitelabel_headers = false
 
@@ -111,10 +120,18 @@ pub struct ConfigurationInner {
     /// The DSN for the database. Only postgres is currently supported.
     pub db_dsn: String,
 
-    /// The DSN for redis (can be left empty if not using redis)
-    #[serde(deserialize_with = "deserialize_redis_dsn")]
+    /// The DSN for redis (can be left empty if not using redis). This will be default for both redis-based caching and queuing unless overridden
+    #[serde(deserialize_with = "deserialize_redis_dsn", default)]
     pub redis_dsn: Option<Vec<String>>,
 
+    /// Optional DSN for redis-based queue, if diffferent than that for caching. If left empty, `redis_dsn` will be used
+    #[serde(deserialize_with = "deserialize_redis_dsn", default)]
+    pub queue_redis_dsn: Option<Vec<String>>,
+
+    /// Optional DSN for redis-based caching, if diffferent than that for queuing. If left empty, `redis_dsn` will be used
+    #[serde(deserialize_with = "deserialize_redis_dsn", default)]
+    pub cache_redis_dsn: Option<Vec<String>>,
+
     /// What kind of message queue to use. Supported: memory, redis (must have redis_dsn configured), rediscluster (add multiple hosts to redis_dsn separated by commas).
     pub queue_type: QueueType,
 

server/svix-server/src/core/cache.rs

@@ -136,8 +136,8 @@ impl RedisCache {
 #[cfg(test)]
 mod tests {
 
-    use crate::cfg::CacheType;
     use super::*;
+    use crate::cfg::CacheType;
     use serde::Deserialize;
 
     // Test structures

server/svix-server/src/lib.rs

@@ -35,44 +35,42 @@ pub async fn run(cfg: Configuration, listener: Option<TcpListener>) {
     let pool = init_db(&cfg).await;
 
     tracing::debug!("Cache type: {:?}", cfg.cache_type);
-    let redis_cache = match cfg.cache_type {
-        CacheType::Redis => {
-            let mgr = crate::redis::create_redis_pool(
-                cfg.redis_dsn.as_ref().expect("Redis DSN not found"),
-                false,
-            )
-            .await;
-            Some(RedisCache::new(mgr))
-        }
-        CacheType::RedisCluster => {
-            let mgr = crate::redis::create_redis_pool(
-                cfg.redis_dsn.as_ref().expect("Redis DSN not found"),
-                true,
-            )
-            .await;
-            Some(RedisCache::new(mgr))
+    let redis_cache = {
+        let redis_dsn = cfg
+            .cache_redis_dsn
+            .as_ref()
+            .or(cfg.redis_dsn.as_ref())
+            .expect("Redis DSN not found");
+        match cfg.cache_type {
+            CacheType::Redis => {
+                let mgr = crate::redis::create_redis_pool(redis_dsn, false).await;
+                Some(RedisCache::new(mgr))
+            }
+            CacheType::RedisCluster => {
+                let mgr = crate::redis::create_redis_pool(redis_dsn, true).await;
+                Some(RedisCache::new(mgr))
+            }
         }
     };
 
     tracing::debug!("Queue type: {:?}", cfg.queue_type);
-    let (queue_tx, queue_rx) = match cfg.queue_type {
-        QueueType::Redis => {
-            let pool = crate::redis::create_redis_pool(
-                cfg.redis_dsn.as_ref().expect("Redis DSN not found"),
-                false,
-            )
-            .await;
-            queue::redis::new_pair(pool).await
-        }
-        QueueType::RedisCluster => {
-            let pool = crate::redis::create_redis_pool(
-                cfg.redis_dsn.as_ref().expect("Redis DSN not found"),
-                true,
-            )
-            .await;
-            queue::redis::new_pair(pool).await
+    let (queue_tx, queue_rx) = {
+        let redis_dsn = cfg
+            .queue_redis_dsn
+            .as_ref()
+            .or(cfg.redis_dsn.as_ref())
+            .expect("Redis DSN not found");
+        match cfg.queue_type {
+            QueueType::Redis => {
+                let pool = crate::redis::create_redis_pool(redis_dsn, false).await;
+                queue::redis::new_pair(pool).await
+            }
+            QueueType::RedisCluster => {
+                let pool = crate::redis::create_redis_pool(redis_dsn, true).await;
+                queue::redis::new_pair(pool).await
+            }
+            QueueType::Memory => queue::memory::new_pair().await,
         }
-        QueueType::Memory => queue::memory::new_pair().await,
     };
 
     // build our application with a route

server/svix-server/src/redis/mod.rs

@@ -116,8 +116,8 @@ pub async fn create_redis_pool(redis_dsn: &[String], clustered: bool) -> SvixRed
 #[cfg(test)]
 mod tests {
 
-    use crate::cfg::CacheType;
     use super::*;
+    use crate::cfg::CacheType;
 
     // Ensure basic set/get works -- should test sharding as well:
     #[tokio::test]
@@ -149,11 +149,15 @@ mod tests {
 
         for (val, key) in "abcdefghijklmnopqrstuvwxyz".chars().enumerate() {
             let key = key.to_string();
-            pool.query_async::<()>(redis::Cmd::set::<String, usize>(key.clone(), val)).await.unwrap();
+            pool.query_async::<()>(redis::Cmd::set::<String, usize>(key.clone(), val))
+                .await
+                .unwrap();
             assert_eq!(
-                pool.query_async::<usize>(redis::Cmd::get(&key)).await.unwrap(),
+                pool.query_async::<usize>(redis::Cmd::get(&key))
+                    .await
+                    .unwrap(),
                 val
             );
         }
     }
-}
+}