Merge branch 'master' into test_spaces_permissions

Author: elasticmachine

docs/settings/dev-settings.asciidoc

@@ -14,7 +14,7 @@ They are enabled by default.
 
 [cols="2*<"]
 |===
-| `xpack.grokdebugger.enabled`
+| `xpack.grokdebugger.enabled` {ess-icon}
   | Set to `true` to enable the <<xpack-grokdebugger,Grok Debugger>>. Defaults to `true`.
 
 |===

docs/settings/monitoring-settings.asciidoc

@@ -37,6 +37,11 @@ For more information, see
   monitoring back-end does not run and {kib} stats are not sent to the monitoring
   cluster.
 
+a|`monitoring.cluster_alerts.`
+`email_notifications.email_address` {ess-icon}
+    | Specifies the email address where you want to receive cluster alerts.
+    See <<cluster-alert-email-notifications, email notifications>> for details.
+
 | `monitoring.ui.elasticsearch.hosts`
   | Specifies the location of the {es} cluster where your monitoring data is stored.
   By default, this is the same as `elasticsearch.hosts`. This setting enables
@@ -85,7 +90,7 @@ These settings control how data is collected from {kib}.
   | Set to `true` (default) to enable data collection from the {kib} NodeJS server
   for {kib} dashboards to be featured in *{stack-monitor-app}*.
 
-| `monitoring.kibana.collection.interval`
+| `monitoring.kibana.collection.interval` {ess-icon}
   | Specifies the number of milliseconds to wait in between data sampling on the
   {kib} NodeJS server for the metrics that are displayed in the {kib} dashboards.
   Defaults to `10000` (10 seconds).
@@ -111,7 +116,7 @@ about configuring {kib}, see
   | Set to `false` to hide *{stack-monitor-app}*. The monitoring back-end
     continues to run as an agent for sending {kib} stats to the monitoring
     cluster. Defaults to `true`.
-  
+
 | `monitoring.ui.logs.index`
   | Specifies the name of the indices that are shown on the
   <<logs-monitor-page,*Logs*>> page in *{stack-monitor-app}*. The default value
@@ -124,7 +129,7 @@ about configuring {kib}, see
   {ref}/search-aggregations-bucket-terms-aggregation.html#search-aggregations-bucket-terms-aggregation-size[Terms Aggregation].
   Defaults to `10000`.
 
-| `monitoring.ui.min_interval_seconds`
+| `monitoring.ui.min_interval_seconds` {ess-icon}
   | Specifies the minimum number of seconds that a time bucket in a chart can
   represent. Defaults to 10. If you modify the
   `monitoring.ui.collection.interval` in `elasticsearch.yml`, use the same
@@ -143,7 +148,7 @@ container, then Cgroup statistics are not useful.
 
 [cols="2*<"]
 |===
-| `monitoring.ui.container.elasticsearch.enabled`
+| `monitoring.ui.container.elasticsearch.enabled` {ess-icon}
   | For {es} clusters that are running in containers, this setting changes the
   *Node Listing* to display the CPU utilization based on the reported Cgroup
   statistics. It also adds the calculated Cgroup CPU utilization to the

src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts

@@ -481,7 +481,7 @@ export class IndexPattern implements IIndexPattern {
       fields: this.mapping.fields._serialize!(this.fields),
       fieldFormatMap: this.mapping.fieldFormatMap._serialize!(this.fieldFormatMap),
       type: this.type,
-      typeMeta: this.mapping.typeMeta._serialize!(this.mapping),
+      typeMeta: this.mapping.typeMeta._serialize!(this.typeMeta),
     };
   }
 

test/functional/page_objects/settings_page.ts

@@ -282,11 +282,15 @@ export function SettingsPageProvider({ getService, getPageObjects }: FtrProvider
       await PageObjects.header.waitUntilLoadingHasFinished();
     }
 
-    async clickIndexPatternLogstash() {
-      const indexLink = await find.byXPath(`//a[descendant::*[text()='logstash-*']]`);
+    async clickIndexPatternByName(name: string) {
+      const indexLink = await find.byXPath(`//a[descendant::*[text()='${name}']]`);
       await indexLink.click();
     }
 
+    async clickIndexPatternLogstash() {
+      await this.clickIndexPatternByName('logstash-*');
+    }
+
     async getIndexPatternList() {
       await testSubjects.existOrFail('indexPatternTable', { timeout: 5000 });
       return await find.allByCssSelector(

x-pack/plugins/ingest_manager/common/constants/agent.ts

@@ -12,6 +12,7 @@ export const AGENT_TYPE_PERMANENT = 'PERMANENT';
 export const AGENT_TYPE_EPHEMERAL = 'EPHEMERAL';
 export const AGENT_TYPE_TEMPORARY = 'TEMPORARY';
 
+export const AGENT_POLLING_REQUEST_TIMEOUT_MS = 300000; // 5 minutes
 export const AGENT_POLLING_THRESHOLD_MS = 30000;
 export const AGENT_POLLING_INTERVAL = 1000;
 export const AGENT_UPDATE_LAST_CHECKIN_INTERVAL_MS = 30000;

x-pack/plugins/ingest_manager/server/errors.ts

@@ -17,15 +17,14 @@ export const getHTTPResponseCode = (error: IngestManagerError): number => {
     return 502; // Bad Gateway
   }
   if (error instanceof PackageNotFoundError) {
-    return 404;
-  }
-  if (error instanceof PackageOutdatedError) {
-    return 400;
-  } else {
-    return 400; // Bad Request
+    return 404; // Not Found
   }
+
+  return 400; // Bad Request
 };
 
 export class RegistryError extends IngestManagerError {}
+export class RegistryConnectionError extends RegistryError {}
+export class RegistryResponseError extends RegistryError {}
 export class PackageNotFoundError extends IngestManagerError {}
 export class PackageOutdatedError extends IngestManagerError {}

x-pack/plugins/ingest_manager/server/index.ts

@@ -9,6 +9,7 @@ import { IngestManagerPlugin } from './plugin';
 import {
   AGENT_POLICY_ROLLOUT_RATE_LIMIT_INTERVAL_MS,
   AGENT_POLICY_ROLLOUT_RATE_LIMIT_REQUEST_PER_INTERVAL,
+  AGENT_POLLING_REQUEST_TIMEOUT_MS,
 } from '../common';
 export { AgentService, ESIndexPatternService, getRegistryUrl } from './services';
 export {
@@ -29,7 +30,10 @@ export const config = {
     fleet: schema.object({
       enabled: schema.boolean({ defaultValue: true }),
       tlsCheckDisabled: schema.boolean({ defaultValue: false }),
-      pollingRequestTimeout: schema.number({ defaultValue: 60000 }),
+      pollingRequestTimeout: schema.number({
+        defaultValue: AGENT_POLLING_REQUEST_TIMEOUT_MS,
+        min: 5000,
+      }),
       maxConcurrentConnections: schema.number({ defaultValue: 0 }),
       kibana: schema.object({
         host: schema.maybe(

x-pack/plugins/ingest_manager/server/plugin.ts

@@ -238,7 +238,7 @@ export class IngestManagerPlugin
           // we currently only use this global interceptor if fleet is enabled
           // since it would run this func on *every* req (other plugins, CSS, etc)
           registerLimitedConcurrencyRoutes(core, config);
-          registerAgentRoutes(router);
+          registerAgentRoutes(router, config);
           registerEnrollmentApiKeyRoutes(router);
           registerInstallScriptRoutes({
             router,

x-pack/plugins/ingest_manager/server/routes/agent/index.ts

@@ -41,8 +41,9 @@ import * as AgentService from '../../services/agents';
 import { postNewAgentActionHandlerBuilder } from './actions_handlers';
 import { appContextService } from '../../services';
 import { postAgentsUnenrollHandler } from './unenroll_handler';
+import { IngestManagerConfigType } from '../..';
 
-export const registerRoutes = (router: IRouter) => {
+export const registerRoutes = (router: IRouter, config: IngestManagerConfigType) => {
   // Get one
   router.get(
     {
@@ -80,12 +81,22 @@ export const registerRoutes = (router: IRouter) => {
     getAgentsHandler
   );
 
+  const pollingRequestTimeout = config.fleet.pollingRequestTimeout;
   // Agent checkin
   router.post(
     {
       path: AGENT_API_ROUTES.CHECKIN_PATTERN,
       validate: PostAgentCheckinRequestSchema,
-      options: { tags: [] },
+      options: {
+        tags: [],
+        ...(pollingRequestTimeout
+          ? {
+              timeout: {
+                idleSocket: pollingRequestTimeout,
+              },
+            }
+          : {}),
+      },
     },
     postAgentCheckinHandler
   );

x-pack/plugins/ingest_manager/server/services/agents/checkin/state_new_actions.ts

@@ -170,7 +170,10 @@ export function agentCheckinStateNewActionsFactory() {
     }
 
     const stream$ = agentPolicy$.pipe(
-      timeout(appContextService.getConfig()?.fleet.pollingRequestTimeout || 0),
+      timeout(
+        // Set a timeout 3s before the real timeout to have a chance to respond an empty response before socket timeout
+        Math.max((appContextService.getConfig()?.fleet.pollingRequestTimeout ?? 0) - 3000, 3000)
+      ),
       filter((agentPolicy) => shouldCreateAgentPolicyAction(agent, agentPolicy)),
       rateLimiter(),
       mergeMap((agentPolicy) => createAgentActionFromAgentPolicy(soClient, agent, agentPolicy)),