[Ingest Manager] install previous version of package if update fails (#76694) (#76972)

* add InstallType and getInstallType helper

* rollback to previous version if update fails or unisntall if not an update

* fix pipelines from failed update not removing if update fails after installing pipelines and rolling back to previous version

* change type from enum to union, improve getInstallType, add ts-config comment

* throw error if no install type, update logging errors

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Author: neptunian

x-pack/plugins/ingest_manager/common/types/models/epm.ts

@@ -19,6 +19,8 @@ export enum InstallStatus {
   uninstalling = 'uninstalling',
 }
 
+export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install';
+
 export type EpmPackageInstallStatus = 'installed' | 'installing';
 
 export type DetailViewPanelName = 'overview' | 'usages' | 'settings';

x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts

@@ -34,6 +34,7 @@ import {
 } from '../../services/epm/packages';
 import { IngestManagerError, defaultIngestErrorHandler } from '../../errors';
 import { splitPkgKey } from '../../services/epm/registry';
+import { getInstallType } from '../../services/epm/packages/install';
 
 export const getCategoriesHandler: RequestHandler<
   undefined,
@@ -138,6 +139,8 @@ export const installPackageHandler: RequestHandler<
   const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser;
   const { pkgkey } = request.params;
   const { pkgName, pkgVersion } = splitPkgKey(pkgkey);
+  const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
+  const installType = getInstallType({ pkgVersion, installedPkg });
   try {
     const res = await installPackage({
       savedObjectsClient,
@@ -156,15 +159,25 @@ export const installPackageHandler: RequestHandler<
     if (e instanceof IngestManagerError) {
       return defaultResult;
     }
-    // if there is an unknown server error, uninstall any package assets
+
+    // if there is an unknown server error, uninstall any package assets or reinstall the previous version if update
     try {
-      const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
-      const isUpdate = installedPkg && installedPkg.attributes.version < pkgVersion ? true : false;
-      if (!isUpdate) {
+      if (installType === 'install' || installType === 'reinstall') {
+        logger.error(`uninstalling ${pkgkey} after error installing`);
         await removeInstallation({ savedObjectsClient, pkgkey, callCluster });
       }
+      if (installType === 'update') {
+        // @ts-ignore installType conditions already check for existence of installedPkg
+        const prevVersion = `${pkgName}-${installedPkg.attributes.version}`;
+        logger.error(`rolling back to ${prevVersion} after error installing ${pkgkey}`);
+        await installPackage({
+          savedObjectsClient,
+          pkgkey: prevVersion,
+          callCluster,
+        });
+      }
     } catch (error) {
-      logger.error(`could not remove failed installation ${error}`);
+      logger.error(`failed to uninstall or rollback package after installation error ${error}`);
     }
     return defaultResult;
   }

x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ElasticsearchAssetType, Installation, KibanaAssetType } from '../../../types';
+import { SavedObject } from 'src/core/server';
+import { getInstallType } from './install';
+
+const mockInstallation: SavedObject<Installation> = {
+  id: 'test-pkg',
+  references: [],
+  type: 'epm-packages',
+  attributes: {
+    id: 'test-pkg',
+    installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }],
+    installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
+    es_index_patterns: { pattern: 'pattern-name' },
+    name: 'test packagek',
+    version: '1.0.0',
+    install_status: 'installed',
+    install_version: '1.0.0',
+    install_started_at: new Date().toISOString(),
+  },
+};
+const mockInstallationUpdateFail: SavedObject<Installation> = {
+  id: 'test-pkg',
+  references: [],
+  type: 'epm-packages',
+  attributes: {
+    id: 'test-pkg',
+    installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }],
+    installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
+    es_index_patterns: { pattern: 'pattern-name' },
+    name: 'test packagek',
+    version: '1.0.0',
+    install_status: 'installing',
+    install_version: '1.0.1',
+    install_started_at: new Date().toISOString(),
+  },
+};
+describe('install', () => {
+  describe('getInstallType', () => {
+    it('should return correct type when installing and no other version is currently installed', () => {});
+    const installTypeInstall = getInstallType({ pkgVersion: '1.0.0', installedPkg: undefined });
+    expect(installTypeInstall).toBe('install');
+
+    it('should return correct type when installing the same version', () => {});
+    const installTypeReinstall = getInstallType({
+      pkgVersion: '1.0.0',
+      installedPkg: mockInstallation,
+    });
+    expect(installTypeReinstall).toBe('reinstall');
+
+    it('should return correct type when moving from one version to another', () => {});
+    const installTypeUpdate = getInstallType({
+      pkgVersion: '1.0.1',
+      installedPkg: mockInstallation,
+    });
+    expect(installTypeUpdate).toBe('update');
+
+    it('should return correct type when update fails and trys again', () => {});
+    const installTypeReupdate = getInstallType({
+      pkgVersion: '1.0.1',
+      installedPkg: mockInstallationUpdateFail,
+    });
+    expect(installTypeReupdate).toBe('reupdate');
+
+    it('should return correct type when attempting to rollback from a failed update', () => {});
+    const installTypeRollback = getInstallType({
+      pkgVersion: '1.0.0',
+      installedPkg: mockInstallationUpdateFail,
+    });
+    expect(installTypeRollback).toBe('rollback');
+  });
+});

x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { SavedObjectsClientContract } from 'src/core/server';
+import { SavedObject, SavedObjectsClientContract } from 'src/core/server';
 import semver from 'semver';
 import { PACKAGES_SAVED_OBJECT_TYPE, MAX_TIME_COMPLETE_INSTALL } from '../../../constants';
 import {
@@ -16,6 +16,7 @@ import {
   KibanaAssetReference,
   EsAssetReference,
   ElasticsearchAssetType,
+  InstallType,
 } from '../../../types';
 import { installIndexPatterns } from '../kibana/index_pattern/install';
 import * as Registry from '../registry';
@@ -110,11 +111,13 @@ export async function installPackage({
   const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
   // get the currently installed package
   const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
-  const reinstall = pkgVersion === installedPkg?.attributes.version;
-  const reupdate = pkgVersion === installedPkg?.attributes.install_version;
 
-  // let the user install if using the force flag or this is a reinstall or reupdate due to intallation interruption
-  if (semver.lt(pkgVersion, latestPackage.version) && !force && !reinstall && !reupdate) {
+  const installType = getInstallType({ pkgVersion, installedPkg });
+
+  // let the user install if using the force flag or needing to reinstall or install a previous version due to failed update
+  const installOutOfDateVersionOk =
+    installType === 'reinstall' || installType === 'reupdate' || installType === 'rollback';
+  if (semver.lt(pkgVersion, latestPackage.version) && !force && !installOutOfDateVersionOk) {
     throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`);
   }
   const paths = await Registry.getArchiveInfo(pkgName, pkgVersion);
@@ -188,16 +191,26 @@ export async function installPackage({
   // update current backing indices of each data stream
   await updateCurrentWriteIndices(callCluster, installedTemplates);
 
-  // if this is an update, delete the previous version's pipelines
-  if (installedPkg && !reinstall) {
+  // if this is an update or retrying an update, delete the previous version's pipelines
+  if (installType === 'update' || installType === 'reupdate') {
     await deletePreviousPipelines(
       callCluster,
       savedObjectsClient,
       pkgName,
+      // @ts-ignore installType conditions already check for existence of installedPkg
       installedPkg.attributes.version
     );
   }
-
+  // pipelines from a different version may have installed during a failed update
+  if (installType === 'rollback') {
+    await deletePreviousPipelines(
+      callCluster,
+      savedObjectsClient,
+      pkgName,
+      // @ts-ignore installType conditions already check for existence of installedPkg
+      installedPkg.attributes.install_version
+    );
+  }
   const installedTemplateRefs = installedTemplates.map((template) => ({
     id: template.templateName,
     type: ElasticsearchAssetType.indexTemplate,
@@ -326,3 +339,23 @@ export async function ensurePackagesCompletedInstall(
   await Promise.all(installingPromises);
   return installingPackages;
 }
+
+export function getInstallType({
+  pkgVersion,
+  installedPkg,
+}: {
+  pkgVersion: string;
+  installedPkg: SavedObject<Installation> | undefined;
+}): InstallType {
+  const isInstalledPkg = !!installedPkg;
+  const currentPkgVersion = installedPkg?.attributes.version;
+  const lastStartedInstallVersion = installedPkg?.attributes.install_version;
+  if (!isInstalledPkg) return 'install';
+  if (pkgVersion === currentPkgVersion && pkgVersion !== lastStartedInstallVersion)
+    return 'rollback';
+  if (pkgVersion === currentPkgVersion) return 'reinstall';
+  if (pkgVersion === lastStartedInstallVersion && pkgVersion !== currentPkgVersion)
+    return 'reupdate';
+  if (pkgVersion !== lastStartedInstallVersion && pkgVersion !== currentPkgVersion) return 'update';
+  throw new Error('unknown install type');
+}

x-pack/plugins/ingest_manager/server/types/index.tsx

@@ -63,6 +63,7 @@ export {
   IndexTemplateMappings,
   Settings,
   SettingsSOAttributes,
+  InstallType,
   // Agent Request types
   PostAgentEnrollRequest,
   PostAgentCheckinRequest,