Automatically downsample transaction when system is under load (backpressure) (#3072)

Co-authored-by: Sentry Github Bot <bot+github-bot@sentry.io>
Co-authored-by: Roman Zavarnitsyn <rom4ek93@gmail.com>
Co-authored-by: Markus Hintersteiner <markus.hintersteiner@sentry.io>
Co-authored-by: Lukas Bloder <lukas.bloder@gmail.com>
Co-authored-by: Stefano <stefano.siano@sentry.io>
Co-authored-by: getsentry-bot <bot@sentry.io>
Co-authored-by: getsentry-bot <bot@getsentry.com>
Co-authored-by: Richard Harrah <1672786+ToppleTheNun@users.noreply.github.com>

Author: 8 people

CHANGELOG.md

@@ -5,6 +5,11 @@
 ### Features
 
 - Support multiple debug-metadata.properties ([#3024](https://github.com/getsentry/sentry-java/pull/3024))
+- Automatically downsample transactions when the system is under load ([#3072](https://github.com/getsentry/sentry-java/pull/3072))
+  - You can opt into this behaviour by setting `enable-backpressure-handling=true`.
+  - We're happy to receive feedback, e.g. [in this GitHub issue](https://github.com/getsentry/sentry-java/issues/2829)
+  - When the system is under load we start reducing the `tracesSampleRate` automatically.
+  - Once the system goes back to healthy, we reset the `tracesSampleRate` to its original value.
 - (Android) Experimental: Provide more detailed cold app start information ([#3057](https://github.com/getsentry/sentry-java/pull/3057))
   - Attaches spans for Application, ContentProvider, and Activities to app-start timings
   - Uses Process.startUptimeMillis to calculate app-start timings

sentry-samples/sentry-samples-spring-boot-jakarta/src/main/resources/application.properties

@@ -13,6 +13,7 @@ sentry.enable-tracing=true
 sentry.ignored-checkins=ignored_monitor_slug_1,ignored_monitor_slug_2
 sentry.debug=true
 sentry.graphql.ignored-error-types=SOME_ERROR,ANOTHER_ERROR
+sentry.enable-backpressure-handling=true
 in-app-includes="io.sentry.samples"
 
 # Uncomment and set to true to enable aot compatibility

sentry-samples/sentry-samples-spring-boot-webflux-jakarta/src/main/resources/application.properties

@@ -9,3 +9,4 @@ sentry.logging.minimum-event-level=info
 sentry.logging.minimum-breadcrumb-level=debug
 sentry.reactive.thread-local-accessor-enabled=true
 sentry.enable-tracing=true
+sentry.enable-backpressure-handling=true

sentry-samples/sentry-samples-spring-boot-webflux/src/main/resources/application.properties

@@ -11,3 +11,4 @@ sentry.enable-tracing=true
 spring.graphql.graphiql.enabled=true
 spring.graphql.websocket.path=/graphql
 spring.graphql.schema.printer.enabled=true
+sentry.enable-backpressure-handling=true

sentry-samples/sentry-samples-spring-boot/src/main/resources/application.properties

@@ -13,6 +13,7 @@ sentry.enable-tracing=true
 sentry.ignored-checkins=ignored_monitor_slug_1,ignored_monitor_slug_2
 sentry.debug=true
 sentry.graphql.ignored-error-types=SOME_ERROR,ANOTHER_ERROR
+sentry.enable-backpressure-handling=true
 in-app-includes="io.sentry.samples"
 
 # Database configuration

sentry-spring-boot-jakarta/src/test/kotlin/io/sentry/spring/boot/jakarta/SentryAutoConfigurationTest.kt

@@ -162,7 +162,8 @@ class SentryAutoConfigurationTest {
             "sentry.trace-propagation-targets=localhost,^(http|https)://api\\..*\$",
             "sentry.enabled=false",
             "sentry.send-modules=false",
-            "sentry.ignored-checkins=slug1,slugB"
+            "sentry.ignored-checkins=slug1,slugB",
+            "sentry.enable-backpressure-handling=true"
         ).run {
             val options = it.getBean(SentryProperties::class.java)
             assertThat(options.readTimeoutMillis).isEqualTo(10)
@@ -194,6 +195,7 @@ class SentryAutoConfigurationTest {
             assertThat(options.isEnabled).isEqualTo(false)
             assertThat(options.isSendModules).isEqualTo(false)
             assertThat(options.ignoredCheckIns).containsOnly("slug1", "slugB")
+            assertThat(options.isEnableBackpressureHandling).isEqualTo(true)
         }
     }
 

sentry-spring-boot-jakarta/src/test/kotlin/io/sentry/spring/boot/jakarta/it/SentrySpringIntegrationTest.kt

@@ -210,7 +210,9 @@ class SentrySpringIntegrationTest {
 
 @SpringBootApplication
 open class App {
-    private val transport = mock<ITransport>()
+    private val transport = mock<ITransport>().also {
+        whenever(it.isHealthy).thenReturn(true)
+    }
 
     @Bean
     open fun mockTransportFactory(): ITransportFactory {

sentry-spring-boot/src/test/kotlin/io/sentry/spring/boot/SentryAutoConfigurationTest.kt

@@ -162,7 +162,8 @@ class SentryAutoConfigurationTest {
             "sentry.trace-propagation-targets=localhost,^(http|https)://api\\..*\$",
             "sentry.enabled=false",
             "sentry.send-modules=false",
-            "sentry.ignored-checkins=slug1,slugB"
+            "sentry.ignored-checkins=slug1,slugB",
+            "sentry.enable-backpressure-handling=true"
         ).run {
             val options = it.getBean(SentryProperties::class.java)
             assertThat(options.readTimeoutMillis).isEqualTo(10)
@@ -194,6 +195,7 @@ class SentryAutoConfigurationTest {
             assertThat(options.isEnabled).isEqualTo(false)
             assertThat(options.isSendModules).isEqualTo(false)
             assertThat(options.ignoredCheckIns).containsOnly("slug1", "slugB")
+            assertThat(options.isEnableBackpressureHandling).isEqualTo(true)
         }
     }
 

sentry-spring-boot/src/test/kotlin/io/sentry/spring/boot/it/SentrySpringIntegrationTest.kt

@@ -210,7 +210,9 @@ class SentrySpringIntegrationTest {
 
 @SpringBootApplication
 open class App {
-    private val transport = mock<ITransport>()
+    private val transport = mock<ITransport>().also {
+        whenever(it.isHealthy).thenReturn(true)
+    }
 
     @Bean
     open fun mockTransportFactory(): ITransportFactory {

sentry-spring-jakarta/src/test/kotlin/io/sentry/spring/jakarta/EnableSentryTest.kt

@@ -6,8 +6,11 @@ import io.sentry.ITransportFactory
 import io.sentry.Integration
 import io.sentry.Sentry
 import io.sentry.SentryOptions
+import io.sentry.transport.ITransport
 import org.assertj.core.api.Assertions.assertThat
+import org.mockito.kotlin.any
 import org.mockito.kotlin.mock
+import org.mockito.kotlin.whenever
 import org.springframework.boot.context.annotation.UserConfigurations
 import org.springframework.boot.test.context.runner.ApplicationContextRunner
 import org.springframework.context.annotation.Bean
@@ -185,7 +188,9 @@ class EnableSentryTest {
     class AppConfigWithCustomTransportFactory {
 
         @Bean
-        fun transport() = mock<ITransportFactory>()
+        fun transport() = mock<ITransportFactory>().also {
+            whenever(it.create(any(), any())).thenReturn(mock<ITransport>())
+        }
     }
 
     @EnableSentry(dsn = "http://key@localhost/proj")