Failure on X-Pack integration due to some ES API change

[andsel]

In integration test for x-pack (./gradlew :logstash-xpack:rubyIntegrationTests) executing the code in no_ssl_create_monitoring_indexes_spec.rb The test fail with records metrics on es (FAILED - 2).
Looking a the HTTP flow against the ES I saw that the following request fail:

curl -X POST \
  'http://localhost:9200/_monitoring/bulk?system_id=logstash&system_api_version=7&interval=1s' \
  -H 'Accept: */*' \
  -H 'Accept-Encoding: gzip, deflate' \
  -H 'Cache-Control: no-cache' \
  -H 'Connection: keep-alive' \
  -H 'Content-Length: 3814' \
  -H 'Content-Type: application/json' \
  -H 'Host: localhost:9200' \
  -H 'Postman-Token: eabcc560-476b-46f9-b307-2112f0ea4279,0c31af74-92cb-4074-8c3f-626864338ef9' \
  -H 'User-Agent: PostmanRuntime/7.19.0' \
  -H 'cache-control: no-cache' \
  -d '{"index":{"_id":null,"_index":"","routing":null}}
{"events":{"out":100,"filtered":100,"in":100,"duration_in_millis":9},"pipelines":[{"vertices":[{"events_out":100,"id":"0088ad1832aae0bcb100ed9fa10613fb4c04ec7511213f9f00ab239c0341ed6e","pipeline_ephemeral_id":"4e451f98-c8d2-4d50-ab32-7b3ed9e772f8","queue_push_duration_in_millis":45},{"events_out":0,"id":"8b7254226d429280d72538ac989cf33a0d7eea47c4d95573aa03838e281fd2fb","pipeline_ephemeral_id":"4e451f98-c8d2-4d50-ab32-7b3ed9e772f8","queue_push_duration_in_millis":0},{"duration_in_millis":3,"events_in":100,"id":"f54c6abeacd1e1efe51b89ef216c8160d3b2cb3bd91faefe10c596f5fc8e1a81","pipeline_ephemeral_id":"4e451f98-c8d2-4d50-ab32-7b3ed9e772f8","events_out":100}],"events":{"out":100,"filtered":100,"in":100,"queue_push_duration_in_millis":45,"duration_in_millis":9},"id":"main","reloads":{"successes":0,"failures":0},"queue":{"type":"persisted","events_count":0,"max_queue_size_in_bytes":1073741824,"queue_size_in_bytes":28301},"ephemeral_id":"4e451f98-c8d2-4d50-ab32-7b3ed9e772f8","hash":"99c3329ae006e7e7e9fb706acdacfcd7a99a970e657ae27b2e0c8721fde898aa"}],"os":{"cpu":{"load_average":{"1m":3.86,"15m":0.88,"5m":1.39}},"cgroup":{"cpuacct":{"control_group":"/user.slice","usage_nanos":18184048518761},"cpu":{"control_group":"/user.slice","stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"timestamp":"2019-11-18T10:35:51.651Z","reloads":{"successes":0,"failures":0},"process":{"max_file_descriptors":1048576,"cpu":{"percent":28},"open_file_descriptors":154},"jvm":{"gc":{"collectors":{"old":{"collection_count":4,"collection_time_in_millis":1463},"young":{"collection_count":5,"collection_time_in_millis":279}}},"uptime_in_millis":18012,"mem":{"heap_used_in_bytes":236008832,"heap_used_percent":22,"heap_max_in_bytes":1037959168}},"queue":{"events_count":0},"logstash":{"version":"8.0.0","snapshot":null,"name":"kalispera","uuid":"a6dc637f-0b74-4084-87ba-ab1168f6ffe6","http_address":"127.0.0.1:9600","status":"green","pipeline":{"workers":1,"batch_size":125},"host":"kalispera","ephemeral_id":"0d0e62ee-7b02-4c3c-99c4-22a1330c7f16"}}
{"index":{"_id":null,"_index":"","routing":null}}
{"pipeline":{"workers":1,"id":"main","batch_size":125,"ephemeral_id":"4e451f98-c8d2-4d50-ab32-7b3ed9e772f8","hash":"99c3329ae006e7e7e9fb706acdacfcd7a99a970e657ae27b2e0c8721fde898aa","representation":{"version":"0.0.0","type":"lir","graph":{"edges":[{"type":"plain","id":"06dd4ecce32a89aa9c777cb5b153d746599730de525f481d3f32ef30ad59ace9","from":"0088ad1832aae0bcb100ed9fa10613fb4c04ec7511213f9f00ab239c0341ed6e","to":"__QUEUE__"},{"type":"plain","id":"9686d6711bbdf3301e0d673403c68f22e21e37fc0381c54f3e7533ec6c7cd7fd","from":"8b7254226d429280d72538ac989cf33a0d7eea47c4d95573aa03838e281fd2fb","to":"__QUEUE__"},{"type":"plain","id":"e56f86cc244ee133c3c18143d8bfc9b43571081c897bb39816c1bc9f262dcff8","from":"__QUEUE__","to":"f54c6abeacd1e1efe51b89ef216c8160d3b2cb3bd91faefe10c596f5fc8e1a81"}],"vertices":[{"meta":{"source":{"line":1,"column":9,"protocol":"str","id":"pipeline"}},"id":"0088ad1832aae0bcb100ed9fa10613fb4c04ec7511213f9f00ab239c0341ed6e","explicit_id":false,"type":"plugin","config_name":"generator","plugin_type":"input"},{"meta":{"source":{"line":1,"column":36,"protocol":"str","id":"pipeline"}},"id":"8b7254226d429280d72538ac989cf33a0d7eea47c4d95573aa03838e281fd2fb","explicit_id":false,"type":"plugin","config_name":"tcp","plugin_type":"input"},{"meta":null,"explicit_id":false,"type":"queue","id":"__QUEUE__"},{"meta":{"source":{"line":1,"column":68,"protocol":"str","id":"pipeline"}},"id":"f54c6abeacd1e1efe51b89ef216c8160d3b2cb3bd91faefe10c596f5fc8e1a81","explicit_id":false,"type":"plugin","config_name":"null","plugin_type":"output"}]},"hash":"99c3329ae006e7e7e9fb706acdacfcd7a99a970e657ae27b2e0c8721fde898aa"}}}'

On ES logs I can see the error:

{"type": "server", "timestamp": "2019-11-18T10:46:33,476Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "docker-cluster", "node.name": "33537e8e82f8", "message": "path: /_monitoring/bulk, params: {system_id=logstash, system_api_version=7,  nterval=1s}", "cluster.uuid": "BqqvzG6DQzCxK7Jzu5QdBQ", "node.id": "ojbhUFw2ShKzsWPWYkWt_w"}

with stack trace:

java.lang.NullPointerException: null,
 at java.util.Objects.requireNonNull(Objects.java:222) ~[?:?],
 at org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkDoc.<init>(MonitoringBulkDoc.java:40) ~[?:?],
 at org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkRequest.lambda$add$0(MonitoringBulkRequest.java:96) ~[?:?],
 at org.elasticsearch.action.bulk.BulkRequestParser.parse(BulkRequestParser.java:238) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkRequest.add(MonitoringBulkRequest.java:82) ~[?:?],
 at org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkRequestBuilder.add(MonitoringBulkRequestBuilder.java:33) ~[?:?],
 at org.elasticsearch.xpack.monitoring.rest.action.RestMonitoringBulkAction.prepareRequest(RestMonitoringBulkAction.java:93) ~[?:?],
 at org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:81) ~[elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at org.elasticsearch.xpack.security.rest.SecurityRestFilter.handleRequest(SecurityRestFilter.java:69) ~[?:?],
 at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:221) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:303) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:165) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at org.elasticsearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:322) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at org.elasticsearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:372) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at org.elasticsearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:301) [elasticsearch-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:69) [transport-netty4-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:31) [transport-netty4-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:58) [transport-netty4-8.0.0-SNAPSHOT.jar:8.0.0-SNAPSHOT],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:326) [netty-codec-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:300) [netty-codec-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:287) [netty-handler-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) [netty-transport-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) [netty-common-4.1.43.Final.jar:4.1.43.Final],
 at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.43.Final.jar:4.1.43.Final],
 at java.lang.Thread.run(Thread.java:830) [?:?]

• Version: 8.0.0-SNAPSHOT
• Operating System: Linux
• Steps to Reproduce:

1. pull lastes ES docker image
   docker pull docker.elastic.co/elasticsearch/elasticsearch:8.0.0-SNAPSHOT
2. start it
   docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:8.0.0-SNAPSHOT
3. use the above cURL and check ES logs

[jsvd]

Full request/ response from curl below:

❯ curl -vv -X POST \
  'http://localhost:9200/_monitoring/bulk?system_id=logstash&system_api_version=7&interval=1s' \
  -H 'Accept: */*' \
  -H 'Accept-Encoding: gzip, deflate' \
  -H 'Cache-Control: no-cache' \
  -H 'Connection: keep-alive' \
  -H 'Content-Length: 3814' \
  -H 'Content-Type: application/json' \
  -H 'Host: localhost:9200' \
  -H 'Postman-Token: eabcc560-476b-46f9-b307-2112f0ea4279,0c31af74-92cb-4074-8c3f-626864338ef9' \
  -H 'User-Agent: PostmanRuntime/7.19.0' \
  -H 'cache-control: no-cache' \
  -d '{"index":{"_id":null,"_index":"","routing":null}}
{"events":{"out":100,"filtered":100,"in":100,"duration_in_millis":9},"pipelines":[{"vertices":[{"events_out":100,"id":"0088ad1832aae0bcb100ed9fa10613fb4c04ec7511213f9f00ab239c0341ed6e","pipeline_ephemeral_id":"4e451f98-c8d2-4d50-ab32-7b3ed9e772f8","queue_push_duration_in_millis":45},{"events_out":0,"id":"8b7254226d429280d72538ac989cf33a0d7eea47c4d95573aa03838e281fd2fb","pipeline_ephemeral_id":"4e451f98-c8d2-4d50-ab32-7b3ed9e772f8","queue_push_duration_in_millis":0},{"duration_in_millis":3,"events_in":100,"id":"f54c6abeacd1e1efe51b89ef216c8160d3b2cb3bd91faefe10c596f5fc8e1a81","pipeline_ephemeral_id":"4e451f98-c8d2-4d50-ab32-7b3ed9e772f8","events_out":100}],"events":{"out":100,"filtered":100,"in":100,"queue_push_duration_in_millis":45,"duration_in_millis":9},"id":"main","reloads":{"successes":0,"failures":0},"queue":{"type":"persisted","events_count":0,"max_queue_size_in_bytes":1073741824,"queue_size_in_bytes":28301},"ephemeral_id":"4e451f98-c8d2-4d50-ab32-7b3ed9e772f8","hash":"99c3329ae006e7e7e9fb706acdacfcd7a99a970e657ae27b2e0c8721fde898aa"}],"os":{"cpu":{"load_average":{"1m":3.86,"15m":0.88,"5m":1.39}},"cgroup":{"cpuacct":{"control_group":"/user.slice","usage_nanos":18184048518761},"cpu":{"control_group":"/user.slice","stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"timestamp":"2019-11-18T10:35:51.651Z","reloads":{"successes":0,"failures":0},"process":{"max_file_descriptors":1048576,"cpu":{"percent":28},"open_file_descriptors":154},"jvm":{"gc":{"collectors":{"old":{"collection_count":4,"collection_time_in_millis":1463},"young":{"collection_count":5,"collection_time_in_millis":279}}},"uptime_in_millis":18012,"mem":{"heap_used_in_bytes":236008832,"heap_used_percent":22,"heap_max_in_bytes":1037959168}},"queue":{"events_count":0},"logstash":{"version":"8.0.0","snapshot":null,"name":"kalispera","uuid":"a6dc637f-0b74-4084-87ba-ab1168f6ffe6","http_address":"127.0.0.1:9600","status":"green","pipeline":{"workers":1,"batch_size":125},"host":"kalispera","ephemeral_id":"0d0e62ee-7b02-4c3c-99c4-22a1330c7f16"}}
{"index":{"_id":null,"_index":"","routing":null}}
{"pipeline":{"workers":1,"id":"main","batch_size":125,"ephemeral_id":"4e451f98-c8d2-4d50-ab32-7b3ed9e772f8","hash":"99c3329ae006e7e7e9fb706acdacfcd7a99a970e657ae27b2e0c8721fde898aa","representation":{"version":"0.0.0","type":"lir","graph":{"edges":[{"type":"plain","id":"06dd4ecce32a89aa9c777cb5b153d746599730de525f481d3f32ef30ad59ace9","from":"0088ad1832aae0bcb100ed9fa10613fb4c04ec7511213f9f00ab239c0341ed6e","to":"__QUEUE__"},{"type":"plain","id":"9686d6711bbdf3301e0d673403c68f22e21e37fc0381c54f3e7533ec6c7cd7fd","from":"8b7254226d429280d72538ac989cf33a0d7eea47c4d95573aa03838e281fd2fb","to":"__QUEUE__"},{"type":"plain","id":"e56f86cc244ee133c3c18143d8bfc9b43571081c897bb39816c1bc9f262dcff8","from":"__QUEUE__","to":"f54c6abeacd1e1efe51b89ef216c8160d3b2cb3bd91faefe10c596f5fc8e1a81"}],"vertices":[{"meta":{"source":{"line":1,"column":9,"protocol":"str","id":"pipeline"}},"id":"0088ad1832aae0bcb100ed9fa10613fb4c04ec7511213f9f00ab239c0341ed6e","explicit_id":false,"type":"plugin","config_name":"generator","plugin_type":"input"},{"meta":{"source":{"line":1,"column":36,"protocol":"str","id":"pipeline"}},"id":"8b7254226d429280d72538ac989cf33a0d7eea47c4d95573aa03838e281fd2fb","explicit_id":false,"type":"plugin","config_name":"tcp","plugin_type":"input"},{"meta":null,"explicit_id":false,"type":"queue","id":"__QUEUE__"},{"meta":{"source":{"line":1,"column":68,"protocol":"str","id":"pipeline"}},"id":"f54c6abeacd1e1efe51b89ef216c8160d3b2cb3bd91faefe10c596f5fc8e1a81","explicit_id":false,"type":"plugin","config_name":"null","plugin_type":"output"}]},"hash":"99c3329ae006e7e7e9fb706acdacfcd7a99a970e657ae27b2e0c8721fde898aa"}}}' | gunzip -
Note: Unnecessary use of -X or --request, POST is already inferred.
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying ::1...
* TCP_NODELAY set
* Connected to localhost (::1) port 9200 (#0)
> POST /_monitoring/bulk?system_id=logstash&system_api_version=7&interval=1s HTTP/1.1
> Host: localhost:9200
> Accept: */*
> Accept-Encoding: gzip, deflate
> Cache-Control: no-cache
> Connection: keep-alive
> Content-Length: 3814
> Content-Type: application/json
> Postman-Token: eabcc560-476b-46f9-b307-2112f0ea4279,0c31af74-92cb-4074-8c3f-626864338ef9
> User-Agent: PostmanRuntime/7.19.0
> cache-control: no-cache
> Expect: 100-continue
> 
< HTTP/1.1 100 Continue
} [3814 bytes data]
* We are completely uploaded and fine
< HTTP/1.1 500 Internal Server Error
< content-type: application/json; charset=UTF-8
< content-encoding: gzip
< content-length: 107
< 
{ [107 bytes data]
100  3921  100   107  100  3814   5446   189k --:--:-- --:--:-- --:--:--  196k
* Connection #0 to host localhost left intact
{"error":{"root_cause":[{"type":"null_pointer_exception","reason":null}],"type":"null_pointer_exception","reason":null},"status":500}

[jsvd]

hi @elastic/stack-monitoring, any idea on why this would happen?

[andsel]

The failure comes from https://github.com/elastic/elasticsearch/blob/3517cf8ede4a8cd3e90ceb89b562b0b6d608f86e/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkDoc.java#L40 and the type is null. But the types wasn't removed?

[chrisronline]

ccing @jakelandis as this looks like an ES-specific thing

[jakelandis]

Yup, this is due to a recent change in ES to completely remove types for 8.0. I think the fix for this is easy, but will need to some help to confirm that the fix doesn't accidentally break any parts of monitoring of Logstash. There are some inconsistencies in how the stack monitoring uses a _type or type field or HTTP parameter, and it appears that Logstash uses none of the above. Which is just fine, but could result in breaking a UI if we get it wrong.

I'll have a PR up for the fix shortly.

[jakelandis]

There are some inconsistencies in how the stack monitoring uses a _type or type field or HTTP parameter, and it appears that Logstash uses none of the above.

Actually, ^^ is wrong. It appears that LS does indeed (or has in the past) use the _type field. It appears to have been set (at least) to "_type":"logstash_stats" and "_type":"logstash_state", but it appears that in the lastest master of Logstash it is no longer sending those values. While it is true that we are removing types entirely from Elasticsearch, there is one exception, which is the _monitoring/bulk endpoint. You can and should continue to send the _type in the bulk request for the monitoring data. That value is used internally to re-shape the document before indexing.

I think this issue is with LS and missing the _type in monitoring payload.

7.4 example payload:

{"index":{"_id":null,"_index":"","_type":"logstash_stats","routing":null}}
{"jvm":{"gc":{"collectors":{"young":{"collection_count":7,"collection_time_in_millis":241},"old":{"collection_count":4,"collection_time_in_millis":828}}},"uptime_in_millis":23431,"mem":{"heap_max_in_bytes":1037959168,"heap_used_percent":31,"heap_used_in_bytes":324971968}},"timestamp":"2019-11-18T22:28:51.550Z","os":{"cpu":{"load_average":{"1m":3.892578125}}},"pipelines":[{"hash":"055b227018dd27d2258224073fe6b0dec96781c47f04cf92a13d1ca7a2579fa2","ephemeral_id":"d9192179-b3ec-42c5-ad36-5d28fe45de11","queue":{"type":"memory","queue_size_in_bytes":0,"max_queue_size_in_bytes":0,"events_count":0},"vertices":[{"id":"cbd587c887e899ff8a25bc72246f4f5fe0a338788cce0b307f79b8b2e33dc35f","pipeline_ephemeral_id":"d9192179-b3ec-42c5-ad36-5d28fe45de11","events_out":0,"queue_push_duration_in_millis":0},{"id":"667bc09607e6d7cda594099c10b030257b28c6f2ca8bd72a05817b6d8779889b","pipeline_ephemeral_id":"d9192179-b3ec-42c5-ad36-5d28fe45de11","duration_in_millis":113,"events_out":0,"events_in":0}],"id":"main","reloads":{"failures":0,"successes":0},"events":{"filtered":0,"queue_push_duration_in_millis":0,"out":0,"duration_in_millis":0,"in":0}}],"reloads":{"failures":0,"successes":0},"events":{"filtered":0,"out":0,"duration_in_millis":0,"in":0},"queue":{"events_count":0},"logstash":{"uuid":"7c4beec6-a8e9-4f47-bf39-5e37cd32acb2","host":"Jakes-MBP","snapshot":false,"ephemeral_id":"131d0b29-9960-4b83-bce3-0fb36b7b4263","status":"green","pipeline":{"batch_size":125,"workers":16},"version":"7.4.0","name":"Jakes-MBP","http_address":"127.0.0.1:9600"},"process":{"cpu":{"percent":0},"open_file_descriptors":98,"max_file_descriptors":10240}}
{"index":{"_id":null,"_index":"","_type":"logstash_state","routing":null}}
{"pipeline":{"hash":"055b227018dd27d2258224073fe6b0dec96781c47f04cf92a13d1ca7a2579fa2","ephemeral_id":"d9192179-b3ec-42c5-ad36-5d28fe45de11","workers":16,"batch_size":125,"id":"main","representation":{"hash":"055b227018dd27d2258224073fe6b0dec96781c47f04cf92a13d1ca7a2579fa2","type":"lir","version":"0.0.0","graph":{"edges":[{"id":"afe12729d1f34a6bf29e75fcc764eaf47c06213693ccbc1e46e1c7c460ff31b0","from":"cbd587c887e899ff8a25bc72246f4f5fe0a338788cce0b307f79b8b2e33dc35f","type":"plain","to":"__QUEUE__"},{"id":"58bd1e48d79424f409b32a0e2ca91eb58b31d96a7db3cd5af9915a369c565ebb","from":"__QUEUE__","type":"plain","to":"667bc09607e6d7cda594099c10b030257b28c6f2ca8bd72a05817b6d8779889b"}],"vertices":[{"type":"plugin","meta":{"source":{"id":"pipeline","line":2,"column":9,"protocol":"str"}},"explicit_id":false,"plugin_type":"input","id":"cbd587c887e899ff8a25bc72246f4f5fe0a338788cce0b307f79b8b2e33dc35f","config_name":"stdin"},{"id":"__QUEUE__","type":"queue","meta":null,"explicit_id":false},{"type":"plugin","meta":{"source":{"id":"pipeline","line":1,"column":10,"protocol":"str"}},"explicit_id":false,"plugin_type":"output","id":"667bc09607e6d7cda594099c10b030257b28c6f2ca8bd72a05817b6d8779889b","config_name":"stdout"}]}}}}

Master example payload:

{"index":{"_id":null,"_index":"","routing":null}}
{"reloads":{"failures":0,"successes":0},"os":{"cpu":{"load_average":{"1m":5.26318359375}}},"queue":{"events_count":0},"logstash":{"status":"green","version":"8.0.0","uuid":"3e52e932-67a0-41f6-97be-4ed4da28e844","host":"Jakes-MBP","name":"Jakes-MBP","ephemeral_id":"cf305e78-7310-48dd-81f3-9a52b6d34051","pipeline":{"workers":16,"batch_size":125},"snapshot":null,"http_address":"127.0.0.1:9600"},"events":{"filtered":0,"out":0,"duration_in_millis":0,"in":0},"pipelines":[{"reloads":{"failures":0,"successes":0},"id":"main","queue":{"events_count":0,"queue_size_in_bytes":0,"type":"memory","max_queue_size_in_bytes":0},"hash":"0c3bc049f8c3e43036e52382516b6bb9f94fc952b2928b2245309048de0c8ab9","ephemeral_id":"69ea1e8e-9a41-4266-aade-dace64637b43","vertices":[{"id":"090adeded1f683f517fe89a36a45516ef205da8068bb289f966fe3abf580d007","queue_push_duration_in_millis":0,"events_out":0,"pipeline_ephemeral_id":"69ea1e8e-9a41-4266-aade-dace64637b43"},{"events_in":0,"id":"fe9b592fffc2e5eb06e50082db36c2bbc365972d0dc3df59199efbad600ed2ad","events_out":0,"duration_in_millis":199,"pipeline_ephemeral_id":"69ea1e8e-9a41-4266-aade-dace64637b43"}],"events":{"filtered":0,"queue_push_duration_in_millis":0,"out":0,"duration_in_millis":0,"in":0}}],"process":{"max_file_descriptors":10240,"open_file_descriptors":80,"cpu":{"percent":0}},"jvm":{"uptime_in_millis":20738,"mem":{"heap_max_in_bytes":1037959168,"heap_used_percent":37,"heap_used_in_bytes":385951928},"gc":{"collectors":{"young":{"collection_time_in_millis":184,"collection_count":6},"old":{"collection_time_in_millis":832,"collection_count":4}}}},"timestamp":"2019-11-18T22:41:24.948Z"}
{"index":{"_id":null,"_index":"","routing":null}}
{"pipeline":{"id":"main","workers":16,"representation":{"graph":{"edges":[{"to":"__QUEUE__","id":"68448b72c36685c5cb52d6f1e0ac46172198468e1956862c48ffe406e827e1d8","from":"090adeded1f683f517fe89a36a45516ef205da8068bb289f966fe3abf580d007","type":"plain"},{"to":"fe9b592fffc2e5eb06e50082db36c2bbc365972d0dc3df59199efbad600ed2ad","id":"e220db17a4374f8dc677610393b18ca3d83da9723a829226cf07c9b6d6906ef2","from":"__QUEUE__","type":"plain"}],"vertices":[{"id":"090adeded1f683f517fe89a36a45516ef205da8068bb289f966fe3abf580d007","config_name":"stdin","plugin_type":"input","explicit_id":false,"type":"plugin","meta":{"source":{"line":2,"id":"pipeline","column":8,"protocol":"str"}}},{"explicit_id":false,"id":"__QUEUE__","type":"queue","meta":null},{"id":"fe9b592fffc2e5eb06e50082db36c2bbc365972d0dc3df59199efbad600ed2ad","config_name":"stdout","plugin_type":"output","explicit_id":false,"type":"plugin","meta":{"source":{"line":1,"id":"pipeline","column":10,"protocol":"str"}}}]},"version":"0.0.0","type":"lir","hash":"0c3bc049f8c3e43036e52382516b6bb9f94fc952b2928b2245309048de0c8ab9"},"hash":"0c3bc049f8c3e43036e52382516b6bb9f94fc952b2928b2245309048de0c8ab9","ephemeral_id":"69ea1e8e-9a41-4266-aade-dace64637b43","batch_size":125}}

[andsel]

The erb template file /x-pack/lib/template.cfg.erb which is used to create the metrics pipeline contains the definition document_type => "%{[@metadata][document_type]}" but having removed the passing of type with PR logstash-plugins/logstash-output-elasticsearch#892 we are in the situation where we use the standard elasticsearch output plugin to index the metrics against an ES 8, the plugin correctly doesn't set the document_type but ES 8 needs the document_type for metrics tracking at the _monitoring endpoint.

proposed solution:

• change the output plugin to use a function to determine if type needs to be added
• in LS's x-pack customize that function to insert the type for monitoring endpoint