From 2706be0d870584c3b5ab772a12f209a40cec6680 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Mon, 11 Mar 2024 19:14:54 +0100 Subject: [PATCH 1/4] server: Reduce redundant test runs The redis tests are now run once for the redis backend, and once for the redis cluster backend in run-tests.sh. Previously, all six runs would execute these tests, most of the time with the redis backend. Since these tests are relatively low-level, they are not affected by other configuration changes. It really only makes sense to run e2e tests with all the different configuration combinations. --- server/run-tests.sh | 55 +++++++++++-------- server/svix-server/src/core/cache/redis.rs | 9 ++- server/svix-server/src/queue/redis.rs | 15 ++++- server/svix-server/src/redis/mod.rs | 7 ++- server/svix-server/tests/it/main.rs | 2 +- .../tests/it/{queue.rs => redis_queue.rs} | 6 +- 6 files changed, 66 insertions(+), 28 deletions(-) rename server/svix-server/tests/it/{queue.rs => redis_queue.rs} (94%) diff --git a/server/run-tests.sh b/server/run-tests.sh index a2b3eba8e..faa349c76 100755 --- a/server/run-tests.sh +++ b/server/run-tests.sh @@ -2,8 +2,6 @@ # Run tests with various configurations: -TEST_COMMAND="cargo test --all --all-features --all-targets $*" - # Common variables: export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres" export SVIX_JWT_SECRET="test value" @@ -13,34 +11,45 @@ export SVIX_DB_POOL_MAX_SIZE="500" export SVIX_REDIS_POOL_MAX_SIZE="10000" echo "*********** RUN 1 ***********" -SVIX_QUEUE_TYPE="redis" \ -SVIX_CACHE_TYPE="redis" \ -SVIX_REDIS_DSN="redis://localhost:6379" \ -${TEST_COMMAND} +( + export SVIX_QUEUE_TYPE="redis" + export SVIX_CACHE_TYPE="redis" + export SVIX_REDIS_DSN="redis://localhost:6379" + cargo test + cargo test -- --ignored redis +) echo "*********** RUN 2 ***********" -SVIX_QUEUE_TYPE="redis" \ -SVIX_CACHE_TYPE="memory" \ -SVIX_REDIS_DSN="redis://localhost:6379" \ -${TEST_COMMAND} +( + export SVIX_QUEUE_TYPE="redis" + export SVIX_CACHE_TYPE="memory" + export SVIX_REDIS_DSN="redis://localhost:6379" + cargo test +) echo "*********** RUN 3 ***********" -SVIX_QUEUE_TYPE="redis" \ -SVIX_CACHE_TYPE="none" \ -SVIX_REDIS_DSN="redis://localhost:6379" \ -${TEST_COMMAND} +( + export SVIX_QUEUE_TYPE="redis" + export SVIX_CACHE_TYPE="none" + export SVIX_REDIS_DSN="redis://localhost:6379" + cargo test +) echo "*********** RUN 4 ***********" -SVIX_QUEUE_TYPE="rediscluster" \ -SVIX_CACHE_TYPE="rediscluster" \ -SVIX_REDIS_DSN="redis://localhost:6380" \ -${TEST_COMMAND} +( + export SVIX_QUEUE_TYPE="rediscluster" + export SVIX_CACHE_TYPE="rediscluster" + export SVIX_REDIS_DSN="redis://localhost:6380" + cargo test + cargo test -- --ignored redis +) echo "*********** RUN 5 ***********" -SVIX_QUEUE_TYPE="memory" \ -SVIX_CACHE_TYPE="none" \ -SVIX_REDIS_DSN="redis://localhost:6379" \ -${TEST_COMMAND} +( + export SVIX_QUEUE_TYPE="memory" + export SVIX_CACHE_TYPE="none" + cargo test +) echo "*********** RUN 6 ***********" ( @@ -48,6 +57,6 @@ echo "*********** RUN 6 ***********" export SVIX_CACHE_TYPE="redis" export SVIX_REDIS_DSN="redis://localhost:6379" export SVIX_RABBIT_DSN="amqp://xivs:xivs@localhost:5672/%2f" - ${TEST_COMMAND} + cargo test cargo test -- --ignored rabbitmq ) diff --git a/server/svix-server/src/core/cache/redis.rs b/server/svix-server/src/core/cache/redis.rs index 1d84a3d44..99a55fe77 100644 --- a/server/svix-server/src/core/cache/redis.rs +++ b/server/svix-server/src/core/cache/redis.rs @@ -116,11 +116,16 @@ mod tests { async fn get_pool(redis_dsn: &str, cfg: &crate::cfg::Configuration) -> RedisPool { match cfg.cache_type { CacheType::RedisCluster => crate::redis::new_redis_pool_clustered(redis_dsn, cfg).await, - _ => crate::redis::new_redis_pool(redis_dsn, cfg).await, + CacheType::Redis => crate::redis::new_redis_pool(redis_dsn, cfg).await, + _ => panic!( + "This test should only be run when redis is configured as the cache provider" + ), } } #[tokio::test] + // run with `cargo test -- --ignored redis` only when redis is up and configured + #[ignore] async fn test_cache_crud_no_ttl() { dotenv::dotenv().ok(); let cfg = crate::cfg::load().unwrap(); @@ -197,6 +202,7 @@ mod tests { } #[tokio::test] + #[ignore] async fn test_cache_ttl() { dotenv::dotenv().ok(); let cfg = crate::cfg::load().unwrap(); @@ -215,6 +221,7 @@ mod tests { } #[tokio::test] + #[ignore] async fn test_cache_nx_status() { dotenv::dotenv().ok(); let cfg = crate::cfg::load().unwrap(); diff --git a/server/svix-server/src/queue/redis.rs b/server/svix-server/src/queue/redis.rs index 4814627e3..1d5073749 100644 --- a/server/svix-server/src/queue/redis.rs +++ b/server/svix-server/src/queue/redis.rs @@ -362,11 +362,18 @@ pub mod tests { CacheType::RedisCluster => { crate::redis::new_redis_pool_clustered(cfg.redis_dsn.as_deref().unwrap(), cfg).await } - _ => crate::redis::new_redis_pool(cfg.redis_dsn.as_deref().unwrap(), cfg).await, + CacheType::Redis => { + crate::redis::new_redis_pool(cfg.redis_dsn.as_deref().unwrap(), cfg).await + } + _ => { + panic!("This test should only be run when redis is configured as the queue backend") + } } } #[tokio::test] + // run with `cargo test -- --ignored redis` only when redis is up and configured + #[ignore] async fn test_migrate_list() { let cfg = crate::cfg::load().unwrap(); let pool = get_pool(&cfg).await; @@ -399,6 +406,7 @@ pub mod tests { } #[tokio::test] + #[ignore] async fn test_migrate_sset() { let cfg = crate::cfg::load().unwrap(); let pool = get_pool(&cfg).await; @@ -448,6 +456,7 @@ pub mod tests { } #[tokio::test] + #[ignore] async fn test_idle_period() { let cfg = crate::cfg::load().unwrap(); let pool = get_pool(&cfg).await; @@ -513,6 +522,7 @@ pub mod tests { } #[tokio::test] + #[ignore] async fn test_ack() { let cfg = crate::cfg::load().unwrap(); let pool = get_pool(&cfg).await; @@ -572,6 +582,7 @@ pub mod tests { } #[tokio::test] + #[ignore] async fn test_nack() { let cfg = crate::cfg::load().unwrap(); @@ -614,6 +625,7 @@ pub mod tests { } #[tokio::test] + #[ignore] async fn test_delay() { let cfg = crate::cfg::load().unwrap(); @@ -665,6 +677,7 @@ pub mod tests { } #[tokio::test] + #[ignore] async fn test_migrations() { let cfg = crate::cfg::load().unwrap(); let pool = get_pool(&cfg).await; diff --git a/server/svix-server/src/redis/mod.rs b/server/svix-server/src/redis/mod.rs index 0f6d03a44..d00dbd7f5 100644 --- a/server/svix-server/src/redis/mod.rs +++ b/server/svix-server/src/redis/mod.rs @@ -168,12 +168,17 @@ mod tests { async fn get_pool(redis_dsn: &str, cfg: &Configuration) -> RedisPool { match cfg.cache_type { CacheType::RedisCluster => crate::redis::new_redis_pool_clustered(redis_dsn, cfg).await, - _ => crate::redis::new_redis_pool(redis_dsn, cfg).await, + CacheType::Redis => crate::redis::new_redis_pool(redis_dsn, cfg).await, + _ => panic!( + "This test should only be run when redis is configured as the cache provider" + ), } } // Ensure basic set/get works -- should test sharding as well: #[tokio::test] + // run with `cargo test -- --ignored redis` only when redis is up and configured + #[ignore] async fn test_set_read_random_keys() { dotenv::dotenv().ok(); let cfg = crate::cfg::load().unwrap(); diff --git a/server/svix-server/tests/it/main.rs b/server/svix-server/tests/it/main.rs index c6e5d80de..f8ce1ce31 100644 --- a/server/svix-server/tests/it/main.rs +++ b/server/svix-server/tests/it/main.rs @@ -8,6 +8,6 @@ mod e2e_message; mod e2e_operational_webhooks; mod integ_webhook_http_client; mod message_app; -mod queue; +mod redis_queue; mod utils; mod worker; diff --git a/server/svix-server/tests/it/queue.rs b/server/svix-server/tests/it/redis_queue.rs similarity index 94% rename from server/svix-server/tests/it/queue.rs rename to server/svix-server/tests/it/redis_queue.rs index 32cf79feb..3bf364b41 100644 --- a/server/svix-server/tests/it/queue.rs +++ b/server/svix-server/tests/it/redis_queue.rs @@ -22,7 +22,8 @@ pub async fn get_pool(cfg: Configuration) -> RedisPool { CacheType::RedisCluster => { new_redis_pool_clustered(cfg.redis_dsn.as_ref().unwrap().as_str(), &cfg).await } - _ => new_redis_pool(cfg.redis_dsn.as_ref().unwrap().as_str(), &cfg).await, + CacheType::Redis => new_redis_pool(cfg.redis_dsn.as_ref().unwrap().as_str(), &cfg).await, + _ => panic!("This test should only be run when redis is configured as the cache provider"), } } @@ -137,6 +138,8 @@ async fn test_many_queue_consumers_inner(prefix: &str, delay: Option) // Without the `multi_thread` and `worker_threads` directive, the `block_on` call will never return // and the test will hang. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] +// run with `cargo test -- --ignored redis` only when redis is up and configured +#[ignore] async fn test_many_queue_consumers() { test_many_queue_consumers_inner("test_many_queue_consumers_", None).await; } @@ -144,6 +147,7 @@ async fn test_many_queue_consumers() { // Without the `multi_thread` and `worker_threads` directive, the `block_on` call will never return // and the test will hang. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore] async fn test_many_queue_consumers_delayed() { test_many_queue_consumers_inner( "test_many_queue_consumers_delayed_", From 6ac33d894b1a8212c036d89cbc180152e3eb25a4 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Mon, 11 Mar 2024 19:34:09 +0100 Subject: [PATCH 2/4] ci: Disable annoying linter --- .github/workflows/other-lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/other-lint.yml b/.github/workflows/other-lint.yml index 04faf385d..5a5e25889 100644 --- a/.github/workflows/other-lint.yml +++ b/.github/workflows/other-lint.yml @@ -25,6 +25,7 @@ jobs: VALIDATE_ALL_CODEBASE: false DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VALIDATE_BASH: false VALIDATE_CHECKOV: false VALIDATE_PYTHON: false VALIDATE_GOOGLE_JAVA_FORMAT: false # Only use checkstyle From 0caadd4638122e0fff6e377f6a72c604155132e6 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Mon, 11 Mar 2024 19:39:50 +0100 Subject: [PATCH 3/4] server: Disable doctests that don't actually work --- server/svix-server/src/core/types/mod.rs | 4 ++-- server/svix-server_derive/src/lib.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/svix-server/src/core/types/mod.rs b/server/svix-server/src/core/types/mod.rs index 57e454a73..b57e57416 100644 --- a/server/svix-server/src/core/types/mod.rs +++ b/server/svix-server/src/core/types/mod.rs @@ -1234,7 +1234,7 @@ macro_rules! repr_enum { /// 3. The string description to be used in the docs. /// Remaining arguments must be the variants in order. For example: /// -/// ``` +/// ```ignore /// #[derive(IntoPrimitive)] /// #[repr(u8)] /// enum MyEnum { @@ -1243,7 +1243,7 @@ macro_rules! repr_enum { /// Qux = 5, /// } /// -/// jsonschema_for_repr_enum{ +/// jsonschema_for_repr_enum! { /// MyEnum, /// u8, /// "My nice little enum", diff --git a/server/svix-server_derive/src/lib.rs b/server/svix-server_derive/src/lib.rs index 0980ec0a8..b734ab266 100644 --- a/server/svix-server_derive/src/lib.rs +++ b/server/svix-server_derive/src/lib.rs @@ -72,7 +72,7 @@ pub fn derive_model_out(input: proc_macro::TokenStream) -> proc_macro::TokenStre /// ID, summary and description. /// /// # Example -/// ``` +/// ```ignore /// /// This is foo! /// #[aide_annotate] /// fn foo() { From f927e6771820e38fe17c8c878b27d8b5c22b66b6 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Mon, 11 Mar 2024 19:50:52 +0100 Subject: [PATCH 4/4] server: Fix missing feature flag on syn dependency Without this, `cargo check` on just `svix-server_derive` was failing. --- server/svix-server_derive/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/svix-server_derive/Cargo.toml b/server/svix-server_derive/Cargo.toml index ca56d29fe..49d5163f8 100644 --- a/server/svix-server_derive/Cargo.toml +++ b/server/svix-server_derive/Cargo.toml @@ -11,7 +11,7 @@ proc-macro = true [dependencies] proc-macro2 = "1.0.78" quote = "1.0.15" -syn = "2.0.48" +syn = { version = "2.0.48", features = ["full"] } [lints] workspace = true