Merge remote-tracking branch 'upstream/master' into task/ingest-60677-add-rest-api-response-types

# Conflicts:
#	x-pack/plugins/endpoint/public/applications/endpoint/store/policy_list/middleware.ts

Author: paul-tavares

docs/images/tutorial-ilm-custom-policy.png

@@ -1,23 +1,179 @@
 [role="xpack"]
+
 [[example-using-index-lifecycle-policy]]
-=== Example of using an index lifecycle policy
+=== Tutorial: Use {ilm-init} to manage {filebeat} time-based indices
+
+With {ilm} ({ilm-init}), you can create policies that perform actions automatically 
+on indices as they age and grow. {ilm-init} policies help you to manage
+performance, resilience, and retention of your data during its lifecycle. This tutorial shows
+you how to use {kib}’s *Index Lifecycle Policies* to modify and create {ilm-init}
+policies. You can learn more about all of the actions, benefits, and lifecycle
+phases in the {ref}/overview-index-lifecycle-management.html[{ilm-init} overview].
+
+
+[discrete]
+[[example-using-index-lifecycle-policy-scenario]]
+==== Scenario
+
+You’re tasked with sending syslog files to an {es} cluster. This
+log data has the following data retention guidelines:
+
+* Keep logs on hot data nodes for 30 days
+* Roll over to a new index if the size reaches 50GB
+* After 30 days:
+** Move the logs to warm data nodes
+** Set {ref}/glossary.html#glossary-replica-shard[replica shards] to 1
+** {ref}/indices-forcemerge.html[Force merge] multiple index segments to free up the space used by deleted documents
+* Delete logs after 90 days
+
+
+[discrete]
+[[example-using-index-lifecycle-policy-prerequisites]]
+==== Prerequisites
+
+To complete this tutorial, you'll need:
+
+* An {es} cluster with hot and warm nodes configured for shard allocation
+awareness. If you’re using {cloud}/ec-getting-started-templates-hot-warm.html[{ess}],
+choose the hot-warm architecture deployment template.
+
++
+For a self-managed cluster, add node attributes as described for {ref}/shard-allocation-filtering.html[shard allocation filtering]
+to label data nodes as hot or warm. This step is required to migrate shards between
+nodes configured with specific hardware for the hot or warm phases.
++
+For example, you can set this in your `elasticsearch.yml` for each data node:
++
+[source,yaml]
+--------------------------------------------------------------------------------
+node.attr.data: "warm"
+--------------------------------------------------------------------------------
+
+* A server with {filebeat} installed and configured to send logs to the `elasticsearch`
+output as described in {filebeat-ref}/filebeat-getting-started.html[Getting Started with {filebeat}].
+
+[discrete]
+[[example-using-index-lifecycle-policy-view-fb-ilm-policy]]
+==== View the {filebeat} {ilm-init} policy
+
+{filebeat} includes a default {ilm-init} policy that enables rollover. {ilm-init}
+is enabled automatically if you’re using the default `filebeat.yml` and index template.
+
+To view the default policy in {kib}, go to *Management > Index Lifecycle Policies*,
+search for _filebeat_, and choose the _filebeat-version_ policy.
+
+This policy initiates the rollover action when the index size reaches 50GB or
+becomes 30 days old.
+
+[role="screenshot"]
+image::images/tutorial-ilm-hotphaserollover-default.png["Default policy"]
+
+
+[float]
+==== Modify the policy
+
+The default policy is enough to prevent the creation of many tiny daily indices.
+You can modify the policy to meet more complex requirements.
+
+. Activate the warm phase.
+
++
+. Set either of the following options to control when the index moves to the warm phase:
+
+** Provide a value for *Timing for warm phase*. Setting this to *15* keeps the
+indices on hot nodes for a range of  15-45 days, depending on when the initial
+rollover occurred.
+
+** Enable *Move to warm phase on rollover*. The index might move to the warm phase
+more quickly than intended if it reaches the *Maximum index size* before the
+the *Maximum age*.
+
+. In the *Select a node attribute to control shard allocation* dropdown, select
+*data:warm(2)* to migrate shards to warm data nodes.
+
+. Change *Number of replicas* to *1*.
+
+. Enable *Force merge data* and set *Number of segments* to *1*.
++
+NOTE:  When rollover is enabled in the hot phase, action timing in the other phases
+is based on the rollover date.
+
++
+[role="screenshot"]
+image::images/tutorial-ilm-modify-default-warm-phase-rollover.png["Modify to add warm phase"]
+
+. Activate the delete phase and set *Timing for delete phase* to *90* days.
++
+[role="screenshot"]
+image::images/tutorial-ilm-delete-rollover.png["Add a delete phase"]
+
+[float]
+==== Create a custom policy
+
+If meeting a specific retention time period is most important, you can create a
+custom policy.  For this option, you will use {filebeat} daily indices without
+rollover.
+
+. Create a custom policy in {kib}, go to *Management > Index Lifecycle Policies >
+Create Policy*.
+
+. Activate the warm phase and configure it as follows:
++
+|===
+|*Setting* |*Value*
+
+|Timing for warm phase
+|30 days from index creation
+
+|Node attribute
+|`data:warm`
+
+|Number of replicas
+|1
+
+|Force merge data
+|enable
+
+|Number of segments
+|1
+|===
+
++
+[role="screenshot"]
+image::images/tutorial-ilm-custom-policy.png["Modify the custom policy to add a warm phase"]
+
 
-A common use case for managing index lifecycle policies is when you’re using 
-{beats-ref}/beats-reference.html[Beats] to continually send time-series data, 
-such as metrics and log data, to {es}.  When you create the Beats packages, an 
-index template is installed.  The template includes a default policy to apply 
-when new indices are created.  
++
+. Activate the delete phase and set the timing.
++
+|===
+|*Setting* |*Value*
+|Timing for delete phase
+|90
+|===
 
-You can edit the policy in {kib}'s *Index Lifecycle Policies*.  For example, you might:
++
+[role="screenshot"]
+image::images/tutorial-ilm-delete-phase-creation.png["Delete phase"]
 
-* Rollover the index when it reaches 50 GB in size or is 30 days old.  These 
-settings are the default for the Beats lifecycle policy. This avoids 
-having 1000s of tiny indices. When a rollover occurs, a new “hot” index is 
-created and added to the index alias.
+. Configure the index to use the new policy in *{kib} > Management > Index Lifecycle
+Policies*
 
-* Move the index into the warm phase, shrink the index down to a single shard, 
-and force merge to a single segment.
+.. Find your {ilm-init} policy.
+.. Click the *Actions* link next to your policy name.
+.. Choose *Add policy to index template*.
+.. Select your {filebeat} index template name from the *Index template* list. For example, `filebeat-7.5.x`.
+.. Click *Add Policy* to save the changes.
 
-* After 60 days, move the index into the cold phase and onto less expensive hardware.
++
+NOTE: If you initially used the default {filebeat} {ilm-init} policy, you will
+see a notice that the template already has a policy associated with it. Confirm
+that you want to overwrite that configuration.
 
-* Delete the index after 90 days.
++
++
+TIP: When you change the policy associated with the index template, the active
+index will continue to use the policy it was associated with at index creation
+unless you manually update it. The next new index will use the updated policy.
+For more reasons that your {ilm-init} policy changes might be delayed, see
+{ref}/update-lifecycle-policy.html#update-lifecycle-policy[Update Lifecycle Policy].

docs/images/tutorial-ilm-delete-phase-creation.png

@@ -5,8 +5,6 @@
  */
 
 import { ExpressionFunctionDefinition } from 'src/plugins/expressions/common';
-import { TimeRange, Filter as DataFilter } from 'src/plugins/data/public';
-import { EmbeddableInput } from 'src/plugins/embeddable/public';
 import { getQueryFilters } from '../../../public/lib/build_embeddable_filters';
 import { Filter, MapCenter, TimeRange as TimeRangeArg } from '../../../types';
 import {
@@ -15,6 +13,7 @@ import {
   EmbeddableExpression,
 } from '../../expression_types';
 import { getFunctionHelp } from '../../../i18n';
+import { MapEmbeddableInput } from '../../../../../plugins/maps/public';
 
 interface Arguments {
   id: string;
@@ -24,32 +23,12 @@ interface Arguments {
   timerange: TimeRangeArg | null;
 }
 
-// Map embeddable is missing proper typings, so type is just to document what we
-// are expecting to pass to the embeddable
-export type SavedMapInput = EmbeddableInput & {
-  id: string;
-  isLayerTOCOpen: boolean;
-  timeRange?: TimeRange;
-  refreshConfig: {
-    isPaused: boolean;
-    interval: number;
-  };
-  hideFilterActions: true;
-  filters: DataFilter[];
-  mapCenter?: {
-    lat: number;
-    lon: number;
-    zoom: number;
-  };
-  hiddenLayers?: string[];
-};
-
 const defaultTimeRange = {
   from: 'now-15m',
   to: 'now',
 };
 
-type Output = EmbeddableExpression<SavedMapInput>;
+type Output = EmbeddableExpression<MapEmbeddableInput>;
 
 export function savedMap(): ExpressionFunctionDefinition<
   'savedMap',
@@ -108,8 +87,8 @@ export function savedMap(): ExpressionFunctionDefinition<
           filters: getQueryFilters(filters),
           timeRange: args.timerange || defaultTimeRange,
           refreshConfig: {
-            isPaused: false,
-            interval: 0,
+            pause: false,
+            value: 0,
           },
 
           mapCenter: center,

docs/images/tutorial-ilm-delete-rollover.png

@@ -6,11 +6,14 @@
 
 import { CoreSetup, CoreStart, Plugin } from 'src/core/public';
 import { CanvasSetup } from '../public';
+import { EmbeddableStart } from '../../../../../src/plugins/embeddable/public';
+import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public';
+import { Start as InspectorStart } from '../../../../../src/plugins/inspector/public';
 
 import { functions } from './functions/browser';
 import { typeFunctions } from './expression_types';
 // @ts-ignore: untyped local
-import { renderFunctions } from './renderers';
+import { renderFunctions, renderFunctionFactories } from './renderers';
 
 import { elementSpecs } from './elements';
 // @ts-ignore Untyped Local
@@ -30,13 +33,26 @@ interface SetupDeps {
   canvas: CanvasSetup;
 }
 
+export interface StartDeps {
+  embeddable: EmbeddableStart;
+  uiActions: UiActionsStart;
+  inspector: InspectorStart;
+}
+
 /** @internal */
-export class CanvasSrcPlugin implements Plugin<{}, {}, SetupDeps, {}> {
-  public setup(core: CoreSetup, plugins: SetupDeps) {
+export class CanvasSrcPlugin implements Plugin<void, void, SetupDeps, StartDeps> {
+  public setup(core: CoreSetup<StartDeps>, plugins: SetupDeps) {
     plugins.canvas.addFunctions(functions);
     plugins.canvas.addTypes(typeFunctions);
+
     plugins.canvas.addRenderers(renderFunctions);
 
+    core.getStartServices().then(([coreStart, depsStart]) => {
+      plugins.canvas.addRenderers(
+        renderFunctionFactories.map((factory: any) => factory(coreStart, depsStart))
+      );
+    });
+
     plugins.canvas.addElements(elementSpecs);
     plugins.canvas.addDatasourceUIs(datasourceSpecs);
     plugins.canvas.addModelUIs(modelSpecs);
@@ -45,11 +61,7 @@ export class CanvasSrcPlugin implements Plugin<{}, {}, SetupDeps, {}> {
     plugins.canvas.addTagUIs(tagSpecs);
     plugins.canvas.addTemplates(templateSpecs);
     plugins.canvas.addTransformUIs(transformSpecs);
-
-    return {};
   }
 
-  public start(core: CoreStart, plugins: {}) {
-    return {};
-  }
+  public start(core: CoreStart, plugins: StartDeps) {}
 }