From 94679bbbadcb365300f36e7bad4f648d790a8082 Mon Sep 17 00:00:00 2001 From: James Lucas Date: Fri, 22 Apr 2022 16:25:12 -0500 Subject: [PATCH] Add script to run tests / add to CI job; add optional config params for queue and cache redis DSNs --- .github/workflows/server-ci.yml | 11 +---- server/run-tests.sh | 22 +++++++++ server/svix-server/config.example.toml | 10 +++- server/svix-server/src/cfg.rs | 23 +++++++-- server/svix-server/src/core/cache.rs | 2 +- server/svix-server/src/lib.rs | 64 +++++++++++++------------- server/svix-server/src/redis/mod.rs | 12 +++-- 7 files changed, 92 insertions(+), 52 deletions(-) create mode 100755 server/run-tests.sh diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index e07d9a6dc..398adfddf 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -69,18 +69,11 @@ jobs: args: --manifest-path server/Cargo.toml -- migrate - name: Run tests - uses: actions-rs/cargo@v1 - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/postgres" - SVIX_REDIS_DSN: "redis://localhost:6379" - SVIX_JWT_SECRET: "test value" - with: - command: test - args: --manifest-path server/Cargo.toml --all --all-features --all-targets + working-directory: ./server + run: ./run-tests.sh - name: Stop dependencies run: docker-compose -f "server/docker-compose.yml" down - # deny-check: # name: cargo-deny check # runs-on: ubuntu-latest diff --git a/server/run-tests.sh b/server/run-tests.sh new file mode 100755 index 000000000..28ba77302 --- /dev/null +++ b/server/run-tests.sh @@ -0,0 +1,22 @@ +#!/bin/sh -e + +# Run tests with various configurations: + +TEST_COMMAND="cargo test --all --all-features --all-targets -- --test-threads 1" + +# Common variables: +export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres" +export SVIX_JWT_SECRET="test value" + +SVIX_QUEUE_TYPE="redis" \ +SVIX_CACHE_TYPE="redis" \ +SVIX_REDIS_DSN="redis://localhost:6379" \ +${TEST_COMMAND} + +SVIX_QUEUE_TYPE="rediscluster" \ +SVIX_CACHE_TYPE="rediscluster" \ +SVIX_QUEUE_REDIS_DSN="redis://localhost:6380" \ +SVIX_CACHE_REDIS_DSN="redis://localhost:6380" \ +${TEST_COMMAND} + + diff --git a/server/svix-server/config.example.toml b/server/svix-server/config.example.toml index cf48b7054..1efdd7a84 100644 --- a/server/svix-server/config.example.toml +++ b/server/svix-server/config.example.toml @@ -17,11 +17,17 @@ retry_schedule = "5,300,1800,7200,18000,36000,36000" # The DSN for the database. Only postgres is currently supported. db_dsn = "postgresql://postgres:postgres@pgbouncer/postgres" -# The DSN for redis (can be left empty if not using redis) +# The DSN for redis (can be left empty if not using redis). This will be default for both redis-based caching and queuing unless overridden redis_dsn = "redis://redis:6379" +# Optional DSN for redis-based queue, if diffferent than that for caching. If left empty, `redis_dsn` will be used +queue_redis_dsn = "redis://redis:6379" + +# Optional DSN for redis-based caching, if diffferent than that for queuing. If left empty, `redis_dsn` will be used +cache_redis_dsn = "redis://redis:6379" + # What kind of message queue to use. Supported: memory, redis (must have redis_dsn configured), rediscluster (add multiple hosts to redis_dsn separated by commas). -queue_type: "redis" +queue_type = "redis" # If true, headers are prefixed with `Webhook-`, otherwise with `Svix-` (default). whitelabel_headers = false diff --git a/server/svix-server/src/cfg.rs b/server/svix-server/src/cfg.rs index 70f4d88cb..694b9d2f5 100644 --- a/server/svix-server/src/cfg.rs +++ b/server/svix-server/src/cfg.rs @@ -74,12 +74,21 @@ retry_schedule = "5,300,1800,7200,18000,36000,36000" # The DSN for the database. Only postgres is currently supported. # db_dsn = "postgresql://postgres:postgres@pgbouncer/postgres" -# The DSN for redis (can be left empty if not using redis) +# The DSN for redis (can be left empty if not using redis). This will be default for both redis-based caching and queuing unless overridden # redis_dsn = "redis://redis:6379" +# Optional DSN for redis-based queue, if diffferent than that for caching. If left empty, `redis_dsn` will be used +# queue_redis_dsn = "redis://redis:6379" + +# Optional DSN for redis-based caching, if diffferent than that for queuing. If left empty, `redis_dsn` will be used +# cache_redis_dsn = "redis://redis:6379" + # What kind of message queue to use. Supported: memory, redis (must have redis_dsn configured), rediscluster (add multiple hosts to redis_dsn separated by commas). queue_type = "redis" +# What kind of cache to use. Supported: redis (must have redis_dsn configured), rediscluster (add multiple hosts to redis_dsn separated by commas). +cache_type = "redis" + # If true, headers are prefixed with `Webhook-`, otherwise with `Svix-` (default). whitelabel_headers = false @@ -111,10 +120,18 @@ pub struct ConfigurationInner { /// The DSN for the database. Only postgres is currently supported. pub db_dsn: String, - /// The DSN for redis (can be left empty if not using redis) - #[serde(deserialize_with = "deserialize_redis_dsn")] + /// The DSN for redis (can be left empty if not using redis). This will be default for both redis-based caching and queuing unless overridden + #[serde(deserialize_with = "deserialize_redis_dsn", default)] pub redis_dsn: Option>, + /// Optional DSN for redis-based queue, if diffferent than that for caching. If left empty, `redis_dsn` will be used + #[serde(deserialize_with = "deserialize_redis_dsn", default)] + pub queue_redis_dsn: Option>, + + /// Optional DSN for redis-based caching, if diffferent than that for queuing. If left empty, `redis_dsn` will be used + #[serde(deserialize_with = "deserialize_redis_dsn", default)] + pub cache_redis_dsn: Option>, + /// What kind of message queue to use. Supported: memory, redis (must have redis_dsn configured), rediscluster (add multiple hosts to redis_dsn separated by commas). pub queue_type: QueueType, diff --git a/server/svix-server/src/core/cache.rs b/server/svix-server/src/core/cache.rs index 187613c1a..1aa571dba 100644 --- a/server/svix-server/src/core/cache.rs +++ b/server/svix-server/src/core/cache.rs @@ -136,8 +136,8 @@ impl RedisCache { #[cfg(test)] mod tests { - use crate::cfg::CacheType; use super::*; + use crate::cfg::CacheType; use serde::Deserialize; // Test structures diff --git a/server/svix-server/src/lib.rs b/server/svix-server/src/lib.rs index 385728bdc..5fd2c6c91 100644 --- a/server/svix-server/src/lib.rs +++ b/server/svix-server/src/lib.rs @@ -35,44 +35,42 @@ pub async fn run(cfg: Configuration, listener: Option) { let pool = init_db(&cfg).await; tracing::debug!("Cache type: {:?}", cfg.cache_type); - let redis_cache = match cfg.cache_type { - CacheType::Redis => { - let mgr = crate::redis::create_redis_pool( - cfg.redis_dsn.as_ref().expect("Redis DSN not found"), - false, - ) - .await; - Some(RedisCache::new(mgr)) - } - CacheType::RedisCluster => { - let mgr = crate::redis::create_redis_pool( - cfg.redis_dsn.as_ref().expect("Redis DSN not found"), - true, - ) - .await; - Some(RedisCache::new(mgr)) + let redis_cache = { + let redis_dsn = cfg + .cache_redis_dsn + .as_ref() + .or(cfg.redis_dsn.as_ref()) + .expect("Redis DSN not found"); + match cfg.cache_type { + CacheType::Redis => { + let mgr = crate::redis::create_redis_pool(redis_dsn, false).await; + Some(RedisCache::new(mgr)) + } + CacheType::RedisCluster => { + let mgr = crate::redis::create_redis_pool(redis_dsn, true).await; + Some(RedisCache::new(mgr)) + } } }; tracing::debug!("Queue type: {:?}", cfg.queue_type); - let (queue_tx, queue_rx) = match cfg.queue_type { - QueueType::Redis => { - let pool = crate::redis::create_redis_pool( - cfg.redis_dsn.as_ref().expect("Redis DSN not found"), - false, - ) - .await; - queue::redis::new_pair(pool).await - } - QueueType::RedisCluster => { - let pool = crate::redis::create_redis_pool( - cfg.redis_dsn.as_ref().expect("Redis DSN not found"), - true, - ) - .await; - queue::redis::new_pair(pool).await + let (queue_tx, queue_rx) = { + let redis_dsn = cfg + .queue_redis_dsn + .as_ref() + .or(cfg.redis_dsn.as_ref()) + .expect("Redis DSN not found"); + match cfg.queue_type { + QueueType::Redis => { + let pool = crate::redis::create_redis_pool(redis_dsn, false).await; + queue::redis::new_pair(pool).await + } + QueueType::RedisCluster => { + let pool = crate::redis::create_redis_pool(redis_dsn, true).await; + queue::redis::new_pair(pool).await + } + QueueType::Memory => queue::memory::new_pair().await, } - QueueType::Memory => queue::memory::new_pair().await, }; // build our application with a route diff --git a/server/svix-server/src/redis/mod.rs b/server/svix-server/src/redis/mod.rs index 94ba481f9..2f5e4f664 100644 --- a/server/svix-server/src/redis/mod.rs +++ b/server/svix-server/src/redis/mod.rs @@ -116,8 +116,8 @@ pub async fn create_redis_pool(redis_dsn: &[String], clustered: bool) -> SvixRed #[cfg(test)] mod tests { - use crate::cfg::CacheType; use super::*; + use crate::cfg::CacheType; // Ensure basic set/get works -- should test sharding as well: #[tokio::test] @@ -149,11 +149,15 @@ mod tests { for (val, key) in "abcdefghijklmnopqrstuvwxyz".chars().enumerate() { let key = key.to_string(); - pool.query_async::<()>(redis::Cmd::set::(key.clone(), val)).await.unwrap(); + pool.query_async::<()>(redis::Cmd::set::(key.clone(), val)) + .await + .unwrap(); assert_eq!( - pool.query_async::(redis::Cmd::get(&key)).await.unwrap(), + pool.query_async::(redis::Cmd::get(&key)) + .await + .unwrap(), val ); } } -} \ No newline at end of file +}