Add missing traces ilm policy for OTel traces data streams (elastic#119449) (elastic#119823)

Author: felixbarny

docs/changelog/119449.yaml

@@ -0,0 +1,5 @@
+pr: 119449
+summary: Add missing traces ilm policy for OTel traces data streams
+area: Data streams
+type: bug
+issues: []

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -802,6 +802,7 @@ protected Set<String> preserveILMPolicyIds() {
             "profiling-60-days@lifecycle",
             "synthetics",
             "synthetics@lifecycle",
+            "traces@lifecycle",
             "7-days-default",
             "7-days@lifecycle",
             "30-days-default",

x-pack/plugin/core/template-resources/src/main/resources/traces@lifecycle.json

@@ -0,0 +1,17 @@
+{
+  "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_primary_shard_size": "50gb",
+          "max_age": "30d"
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default policy for the traces index template installed by x-pack",
+    "managed": true
+  },
+  "deprecated": ${xpack.stack.template.deprecated}
+}

x-pack/plugin/core/template-resources/src/main/resources/traces@settings.json

@@ -2,6 +2,9 @@
   "template": {
     "settings": {
       "index": {
+        "lifecycle": {
+          "name": "traces@lifecycle"
+        },
         "codec": "best_compression",
         "mapping": {
           "ignore_malformed": true

x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml

@@ -120,3 +120,23 @@ Conflicting attribute types:
   - is_true: $datastream-backing-index
   - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" }
   - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" }
+
+---
+traces@lifecycle:
+  - do:
+      bulk:
+        index: traces-generic.otel-default
+        refresh: true
+        body:
+          - create: {}
+          - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "span_id":"1"}'
+  - is_false: errors
+  - do:
+      indices.get_data_stream:
+        name: traces-generic.otel-default
+  - set: { data_streams.0.indices.0.index_name: datastream-backing-index }
+  - do:
+      indices.get_settings:
+        index: $datastream-backing-index
+  - is_true: $datastream-backing-index
+  - match: { .$datastream-backing-index.settings.index.lifecycle.name: "traces@lifecycle" }

x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java

@@ -101,6 +101,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
     //////////////////////////////////////////////////////////
     public static final String TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME = "traces@mappings";
     public static final String TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME = "traces@settings";
+    public static final String TRACES_ILM_POLICY_NAME = "traces@lifecycle";
 
     //////////////////////////////////////////////////////////
     // Synthetics components (for matching synthetics-*-* indices)
@@ -254,6 +255,7 @@ private void updateEnabledSetting(boolean newValue) {
         new LifecyclePolicyConfig(LOGS_ILM_POLICY_NAME, "/logs@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(METRICS_ILM_POLICY_NAME, "/metrics@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(SYNTHETICS_ILM_POLICY_NAME, "/synthetics@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
+        new LifecyclePolicyConfig(TRACES_ILM_POLICY_NAME, "/traces@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(ILM_7_DAYS_POLICY_NAME, "/7-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(ILM_30_DAYS_POLICY_NAME, "/30-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),
         new LifecyclePolicyConfig(ILM_90_DAYS_POLICY_NAME, "/90-days@lifecycle.json", ADDITIONAL_TEMPLATE_VARIABLES),

x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java

@@ -189,6 +189,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {
                         equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME),
                         equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME),
                         equalTo(StackTemplateRegistry.SYNTHETICS_ILM_POLICY_NAME),
+                        equalTo(StackTemplateRegistry.TRACES_ILM_POLICY_NAME),
                         equalTo(StackTemplateRegistry.ILM_7_DAYS_POLICY_NAME),
                         equalTo(StackTemplateRegistry.ILM_30_DAYS_POLICY_NAME),
                         equalTo(StackTemplateRegistry.ILM_90_DAYS_POLICY_NAME),
@@ -212,7 +213,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {
 
         ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
         registry.clusterChanged(event);
-        assertBusy(() -> assertThat(calledTimes.get(), equalTo(8)));
+        assertBusy(() -> assertThat(calledTimes.get(), equalTo(9)));
     }
 
     public void testPolicyAlreadyExists() {
@@ -221,7 +222,7 @@ public void testPolicyAlreadyExists() {
 
         Map<String, LifecyclePolicy> policyMap = new HashMap<>();
         List<LifecyclePolicy> policies = registry.getLifecyclePolicies();
-        assertThat(policies, hasSize(8));
+        assertThat(policies, hasSize(9));
         policies.forEach(p -> policyMap.put(p.getName(), p));
 
         client.setVerifier((action, request, listener) -> {
@@ -292,7 +293,7 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException {
         Map<String, LifecyclePolicy> policyMap = new HashMap<>();
         String policyStr = "{\"phases\":{\"delete\":{\"min_age\":\"1m\",\"actions\":{\"delete\":{}}}}}";
         List<LifecyclePolicy> policies = registry.getLifecyclePolicies();
-        assertThat(policies, hasSize(8));
+        assertThat(policies, hasSize(9));
         policies.forEach(p -> policyMap.put(p.getName(), p));
 
         client.setVerifier((action, request, listener) -> {