[AWS] update max_number_of_messages parameter description (#10480)

This PR is to update the documentation for max_number_of_messages to inform users to be cautious when using a large value for this var.
elastic · Aug 6, 2024 · 31ad9a6 · 31ad9a6
1 parent f75096f
commit 31ad9a6
Show file tree

Hide file tree

Showing 17 changed files with 110 additions and 22 deletions.
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.22.1"
+  changes:
+    - description: Update max_number_of_messages parameter description
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10480
 - version: "2.22.0"
   changes:
     - description: Add global dataset filter for dashboards to improve performance.

diff --git a/packages/aws/data_stream/apigateway_logs/manifest.yml b/packages/aws/data_stream/apigateway_logs/manifest.yml
@@ -85,7 +85,13 @@ streams:
       - name: max_number_of_messages
         type: integer
         title: "[SQS] Maximum Concurrent SQS Messages"
-        description: The maximum number of SQS messages that can be inflight at any time.
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
         default: 5
         required: false
         show_user: false

diff --git a/packages/aws/data_stream/cloudfront_logs/manifest.yml b/packages/aws/data_stream/cloudfront_logs/manifest.yml
@@ -122,8 +122,14 @@ streams:
         default: false
       - name: max_number_of_messages
         type: integer
-        title: Maximum Concurrent SQS Messages
-        description: The maximum number of SQS messages that can be inflight at any time.
+        title: "[SQS] Maximum Concurrent SQS Messages"
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
         default: 5
         required: false
         show_user: false

diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml
@@ -144,7 +144,13 @@ streams:
       - name: max_number_of_messages
         type: integer
         title: "[SQS] Maximum Concurrent SQS Messages"
-        description: The maximum number of SQS messages that can be inflight at any time.
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
         default: 5
         required: false
       - name: custom

diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml
@@ -118,8 +118,14 @@ streams:
         default: false
       - name: max_number_of_messages
         type: integer
-        title: Maximum Concurrent SQS Messages
-        description: The maximum number of SQS messages that can be inflight at any time.
+        title: "[SQS] Maximum Concurrent SQS Messages"
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
         default: 5
         required: false
         show_user: false

diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml
@@ -117,8 +117,14 @@ streams:
         default: false
       - name: max_number_of_messages
         type: integer
-        title: Maximum Concurrent SQS Messages
-        description: The maximum number of SQS messages that can be inflight at any time.
+        title: "[SQS] Maximum Concurrent SQS Messages"
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
         default: 5
         required: false
         show_user: false

diff --git a/packages/aws/data_stream/emr_logs/manifest.yml b/packages/aws/data_stream/emr_logs/manifest.yml
@@ -86,7 +86,13 @@ streams:
       - name: max_number_of_messages
         type: integer
         title: "[SQS] Maximum Concurrent SQS Messages"
-        description: The maximum number of SQS messages that can be inflight at any time.
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
         default: 5
         required: false
         show_user: false

diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml
@@ -117,8 +117,14 @@ streams:
         default: false
       - name: max_number_of_messages
         type: integer
-        title: Maximum Concurrent SQS Messages
-        description: The maximum number of SQS messages that can be inflight at any time.
+        title: "[SQS] Maximum Concurrent SQS Messages"
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
         default: 5
         required: false
         show_user: false

diff --git a/packages/aws/data_stream/guardduty/manifest.yml b/packages/aws/data_stream/guardduty/manifest.yml
@@ -189,7 +189,13 @@ streams:
         required: false
         show_user: true
         default: 5
-        description: The maximum number of SQS messages that can be inflight at any time.
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
       - name: file_selectors
         type: yaml
         title: "[SQS] File Selectors"

diff --git a/packages/aws/data_stream/route53_resolver_logs/manifest.yml b/packages/aws/data_stream/route53_resolver_logs/manifest.yml
@@ -233,11 +233,17 @@ streams:
         default: false
       - name: max_number_of_messages
         type: integer
-        title: Maximum Concurrent SQS Messages
-        description: The maximum number of SQS messages that can be inflight at any time.
+        title: "[SQS] Maximum Concurrent SQS Messages"
         default: 5
         required: false
         show_user: false
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
       - name: custom
         type: yaml
         title: Custom configurations

diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml
@@ -117,11 +117,17 @@ streams:
         default: false
       - name: max_number_of_messages
         type: integer
-        title: Maximum Concurrent SQS Messages
-        description: The maximum number of SQS messages that can be inflight at any time.
+        title: "[SQS] Maximum Concurrent SQS Messages"
         default: 5
         required: false
         show_user: false
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
       - name: custom
         type: yaml
         title: Custom configurations

diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml
@@ -69,7 +69,13 @@ streams:
       - name: max_number_of_messages
         type: integer
         title: "[SQS] Maximum Concurrent SQS Messages"
-        description: The maximum number of SQS messages that can be inflight at any time.
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
         default: 5
         required: false
         show_user: false

diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml
@@ -117,11 +117,17 @@ streams:
         default: false
       - name: max_number_of_messages
         type: integer
-        title: Maximum Concurrent SQS Messages
-        description: The maximum number of SQS messages that can be inflight at any time.
+        title: "[SQS] Maximum Concurrent SQS Messages"
         default: 5
         required: false
         show_user: false
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
       - name: custom
         type: yaml
         title: Custom configurations

diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: aws
 title: AWS
-version: 2.22.0
+version: 2.22.1
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/aws_logs/changelog.yml b/packages/aws_logs/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.3.1"
+  changes:
+    - description: Update max_number_of_messages parameter description
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10480
 - version: "1.3.0"
   changes:
     - description: Improve clarity and validity of system testing.

diff --git a/packages/aws_logs/data_stream/generic/manifest.yml b/packages/aws_logs/data_stream/generic/manifest.yml
@@ -255,7 +255,13 @@ streams:
       - name: max_number_of_messages
         type: integer
         title: Maximum Concurrent SQS Messages
-        description: The maximum number of SQS messages that can be inflight at any time.
+        description: >
+          The maximum number of SQS messages that can be inflight at any time. Defaults to 5. When processing large 
+          amount of large size S3 objects and each object has large amount of events, if this parameter sets too high,
+          it can cause the input to process too many messages concurrently, overload the agent and cause ingest failure.
+          We recommend to keep the default value 5 and use the [preset](https://www.elastic.co/guide/en/fleet/current/es-output-settings.html#es-output-settings-performance-tuning-settings)
+          option to tune your Elastic Agent performance. You can optimize for throughput, scale, latency, or you can
+          choose a balanced (the default) set of performance specifications.
         default: 5
         required: false
         show_user: false

diff --git a/packages/aws_logs/manifest.yml b/packages/aws_logs/manifest.yml
@@ -3,7 +3,7 @@ name: aws_logs
 title: Custom AWS Logs
 description: Collect raw logs from AWS S3 or CloudWatch with Elastic Agent.
 type: integration
-version: "1.3.0"
+version: "1.3.1"
 categories:
   - observability
   - custom