[Metricbeat] Add collecting AuroraDB metrics in rds metricset (#16004)

* Add collecting AuroraDB metrics in rds metricset

* update changelog

* Rename to use iops as unit

* run mage fmt update

CHANGELOG.next.asciidoc

@@ -132,6 +132,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `up` metric to prometheus metrics collected from host {pull}15948[15948]
 - Add citadel metricset for Istio Metricbeat module {pull}15990[15990]
 - Add support for processors in light modules. {issue}14740[14740] {pull}15923[15923]
+- Add collecting AuroraDB metrics in rds metricset. {issue}14142[14142] {pull}16004[16004]
 - Reuse connections in SQL module. {pull}16001[16001]
 
 *Packetbeat*

metricbeat/docs/fields.asciidoc

@@ -2121,7 +2121,7 @@ type: long
 
 --
 
-*`aws.rds.read_io.ops_per_sec`*::
+*`aws.rds.read.iops`*::
 +
 --
 The average number of disk read I/O operations per second.
@@ -2165,7 +2165,7 @@ type: long
 
 --
 
-*`aws.rds.write_io.ops_per_sec`*::
+*`aws.rds.write.iops`*::
 +
 --
 The average number of disk write I/O operations per second.
@@ -2201,6 +2201,30 @@ type: long
 The amount of storage used by your Aurora DB instance, in bytes.
 
 
+type: long
+
+format: bytes
+
+--
+
+*`aws.rds.volume.read.iops`*::
++
+--
+The number of billed read I/O operations from a cluster volume, reported at 5-minute intervals.
+
+
+type: long
+
+format: bytes
+
+--
+
+*`aws.rds.volume.write.iops`*::
++
+--
+The number of write disk I/O operations to the cluster volume, reported at 5-minute intervals.
+
+
 type: long
 
 format: bytes
@@ -2439,6 +2463,18 @@ format: duration
 The average amount of time taken per disk I/O operation.
 
 
+type: float
+
+format: duration
+
+--
+
+*`aws.rds.latency.delete`*::
++
+--
+The amount of latency for delete queries, in milliseconds.
+
+
 type: float
 
 format: duration
@@ -2527,6 +2563,196 @@ type: keyword
 
 --
 
+*`aws.rds.aurora_bin_log_replica_lag`*::
++
+--
+The amount of time a replica DB cluster running on Aurora with MySQL compatibility lags behind the source DB cluster.
+
+
+type: long
+
+--
+
+*`aws.rds.aurora_global_db.replicated_write_io.bytes`*::
++
+--
+In an Aurora Global Database, the number of write I/O operations replicated from the primary AWS Region to the cluster volume in a secondary AWS Region.
+
+
+type: long
+
+--
+
+*`aws.rds.aurora_global_db.data_transfer.bytes`*::
++
+--
+In an Aurora Global Database, the amount of redo log data transferred from the master AWS Region to a secondary AWS Region.
+
+
+type: long
+
+--
+
+*`aws.rds.aurora_global_db.replication_lag.ms`*::
++
+--
+For an Aurora Global Database, the amount of lag when replicating updates from the primary AWS Region, in milliseconds.
+
+
+type: long
+
+--
+
+*`aws.rds.aurora_replica.lag.ms`*::
++
+--
+For an Aurora Replica, the amount of lag when replicating updates from the primary instance, in milliseconds.
+
+
+type: long
+
+--
+
+*`aws.rds.aurora_replica.lag_max.ms`*::
++
+--
+The maximum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds.
+
+
+type: long
+
+--
+
+*`aws.rds.aurora_replica.lag_min.ms`*::
++
+--
+The minimum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds.
+
+
+type: long
+
+--
+
+*`aws.rds.backtrack_change_records.creation_rate`*::
++
+--
+The number of backtrack change records created over five minutes for your DB cluster.
+
+
+type: long
+
+--
+
+*`aws.rds.backtrack_change_records.stored`*::
++
+--
+The actual number of backtrack change records used by your DB cluster.
+
+
+type: long
+
+--
+
+*`aws.rds.backtrack_window.actual`*::
++
+--
+The difference between the target backtrack window and the actual backtrack window.
+
+
+type: long
+
+--
+
+*`aws.rds.backtrack_window.alert`*::
++
+--
+The number of times that the actual backtrack window is smaller than the target backtrack window for a given period of time.
+
+
+type: long
+
+--
+
+*`aws.rds.storage_used.backup_retention_period.bytes`*::
++
+--
+The total amount of backup storage in bytes used to support the point-in-time restore feature within the Aurora DB cluster's backup retention window.
+
+
+type: long
+
+--
+
+*`aws.rds.storage_used.snapshot.bytes`*::
++
+--
+The total amount of backup storage in bytes consumed by all Aurora snapshots for an Aurora DB cluster outside its backup retention window.
+
+
+type: long
+
+--
+
+*`aws.rds.cache_hit_ratio.buffer`*::
++
+--
+The percentage of requests that are served by the buffer cache.
+
+
+type: long
+
+--
+
+*`aws.rds.cache_hit_ratio.result_set`*::
++
+--
+The percentage of requests that are served by the Resultset cache.
+
+
+type: long
+
+--
+
+*`aws.rds.engine_uptime.sec`*::
++
+--
+The amount of time that the instance has been running, in seconds.
+
+
+type: long
+
+--
+
+*`aws.rds.rds_to_aurora_postgresql_replica_lag.sec`*::
++
+--
+The amount of lag in seconds when replicating updates from the primary RDS PostgreSQL instance to other nodes in the cluster.
+
+
+type: long
+
+--
+
+*`aws.rds.backup_storage_billed_total.bytes`*::
++
+--
+The total amount of backup storage in bytes for which you are billed for a given Aurora DB cluster.
+
+
+type: long
+
+--
+
+*`aws.rds.aurora_volume_left_total.bytes`*::
++
+--
+The remaining available space for the cluster volume, measured in bytes.
+
+
+type: long
+
+--
+
 [float]
 === s3_daily_storage
 

x-pack/metricbeat/module/aws/rds/_meta/data.json

@@ -2,48 +2,62 @@
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "aws": {
         "rds": {
+            "aurora_bin_log_replica_lag": 0,
+            "aurora_replica.lag.ms": 19.683,
+            "aurora_replica.lag_max.ms": 19.651500701904297,
+            "aurora_replica.lag_min.ms": 19.651500701904297,
+            "cache_hit_ratio.buffer": 100,
+            "cache_hit_ratio.result_set": 0,
             "cpu": {
                 "total": {
-                    "pct": 0.999999999997575
+                    "pct": 0.035
                 }
             },
             "database_connections": 0,
-            "db_instance": {
-                "arn": "arn:aws:rds:ap-southeast-1:627959692251:db:database-1",
-                "class": "db.t2.micro",
-                "identifier": "database-1",
-                "status": "available"
-            },
-            "disk_queue_depth": 0.00006666888896296543,
+            "db_instance.class": "db.r5.large",
+            "deadlocks": 0,
             "disk_usage": {
-                "bin_log.bytes": 2752
+                "bin_log.bytes": 0
             },
-            "free_storage.bytes": 20402475008,
-            "freeable_memory.bytes": 441335808,
+            "engine_uptime.sec": 278598.25,
+            "free_local_storage.bytes": 33112842240,
+            "freeable_memory.bytes": 4879514624,
             "latency": {
-                "read": 0,
-                "write": 0.0002909090909090909
+                "commit": 2.9322999999999997,
+                "ddl": 0,
+                "delete": 0,
+                "dml": 0.08325833333333334,
+                "insert": 0.08325833333333334,
+                "select": 0.20890321021571023,
+                "update": 0
             },
-            "read_io.ops_per_sec": 0,
-            "swap_usage.bytes": 1167360,
+            "login_failures": 0,
+            "queries": 7.807070323085375,
             "throughput": {
-                "network_receive": 570.7618714269047,
-                "network_transmit": 3228.0614030701536,
-                "read": 0,
-                "write": 3754.7918263942133
+                "commit": 0.2500020925349523,
+                "ddl": 0,
+                "delete": 0,
+                "dml": 0.2500020925349523,
+                "insert": 0.2500020925349523,
+                "network": 1.7498192019524124,
+                "network_receive": 0.8749096009762062,
+                "network_transmit": 0.8749096009762062,
+                "select": 2.8751975752392616,
+                "update": 0
             },
-            "transactions": {},
-            "write_io.ops_per_sec": 0.40002000100005003
+            "transactions": {
+                "active": 0,
+                "blocked": 0
+            }
         }
     },
     "cloud": {
         "account": {
-            "id": "627959692251",
-            "name": "elastic-test"
+            "id": "428152502467",
+            "name": "elastic-beats"
         },
-        "availability_zone": "ap-southeast-1a",
         "provider": "aws",
-        "region": "ap-southeast-1"
+        "region": "eu-west-1"
     },
     "event": {
         "dataset": "aws.rds",