[Fleet] handle ESO errors in message signing key pair generation

x-pack/plugins/fleet/server/plugin.ts

@@ -473,6 +473,7 @@ export class FleetPlugin
 
   public start(core: CoreStart, plugins: FleetStartDeps): FleetStartContract {
     const messageSigningService = new MessageSigningService(
+      this.initializerContext.logger,
       plugins.encryptedSavedObjects.getClient({
         includedHiddenTypes: [MESSAGE_SIGNING_KEYS_SAVED_OBJECT_TYPE],
       })

x-pack/plugins/fleet/server/services/security/message_signing_service.test.ts

@@ -11,6 +11,7 @@ import type { KibanaRequest } from '@kbn/core-http-server';
 import type { SavedObjectsClientContract } from '@kbn/core/server';
 import type { EncryptedSavedObjectsClient } from '@kbn/encrypted-saved-objects-plugin/server';
 import { encryptedSavedObjectsMock } from '@kbn/encrypted-saved-objects-plugin/server/mocks';
+import { loggingSystemMock } from '@kbn/core/server/mocks';
 
 import { MESSAGE_SIGNING_KEYS_SAVED_OBJECT_TYPE } from '../../constants';
 import { createAppContextStartContractMock } from '../../mocks';
@@ -64,7 +65,7 @@ describe('MessageSigningService', () => {
       .getSavedObjects()
       .getScopedClient({} as unknown as KibanaRequest) as jest.Mocked<SavedObjectsClientContract>;
 
-    messageSigningService = new MessageSigningService(esoClientMock);
+    messageSigningService = new MessageSigningService(loggingSystemMock.create(), esoClientMock);
   }
 
   describe('with encryption key configured', () => {
@@ -208,6 +209,33 @@ describe('MessageSigningService', () => {
       expect(isVerified).toBe(true);
       expect(data).toBe(message);
     });
+
+    it('will retry getting keypair if ESO error', async () => {
+      esoClientMock.createPointInTimeFinderDecryptedAsInternalUser = jest
+        .fn()
+        .mockRejectedValueOnce(new Error('some error'))
+        .mockRejectedValueOnce(new Error('another error'))
+        .mockResolvedValueOnce({
+          close: jest.fn(),
+          find: function* asyncGenerator() {
+            yield { saved_objects: [] };
+          },
+        });
+
+      const generateKeyPairResponse = await messageSigningService.generateKeyPair();
+      expect(esoClientMock.createPointInTimeFinderDecryptedAsInternalUser).toBeCalledTimes(3);
+      expect(soClientMock.create).toHaveBeenLastCalledWith(MESSAGE_SIGNING_KEYS_SAVED_OBJECT_TYPE, {
+        private_key: expect.any(String),
+        public_key: expect.any(String),
+        passphrase: expect.any(String),
+      });
+
+      expect(generateKeyPairResponse).toEqual({
+        passphrase: expect.any(String),
+        privateKey: expect.any(String),
+        publicKey: expect.any(String),
+      });
+    });
   });
 
   describe('with NO encryption key configured', () => {

x-pack/plugins/fleet/server/services/security/message_signing_service.ts

@@ -5,8 +5,11 @@
  * 2.0.
  */
 
+import { backOff } from 'exponential-backoff';
+
 import { generateKeyPairSync, createSign, randomBytes } from 'crypto';
 
+import type { LoggerFactory, Logger } from '@kbn/core/server';
 import type { KibanaRequest } from '@kbn/core-http-server';
 import type {
   SavedObjectsClientContract,
@@ -39,8 +42,11 @@ export interface MessageSigningServiceInterface {
 
 export class MessageSigningService implements MessageSigningServiceInterface {
   private _soClient: SavedObjectsClientContract | undefined;
+  private logger: Logger;
 
-  constructor(private esoClient: EncryptedSavedObjectsClient) {}
+  constructor(loggerFactory: LoggerFactory, private esoClient: EncryptedSavedObjectsClient) {
+    this.logger = loggerFactory.get('messageSigningService');
+  }
 
   public get isEncryptionAvailable(): MessageSigningServiceInterface['isEncryptionAvailable'] {
     return appContextService.getEncryptedSavedObjectsSetup()?.canEncrypt ?? false;
@@ -188,6 +194,30 @@ export class MessageSigningService implements MessageSigningServiceInterface {
     return this._soClient;
   }
 
+  private async getCurrentKeyPairObjWithRetry() {
+    let soDoc: SavedObjectsFindResult<MessageSigningKeys> | undefined;
+
+    await backOff(
+      async () => {
+        soDoc = await this.getCurrentKeyPairObj();
+      },
+      {
+        maxDelay: 60 * 60 * 1000, // 1 hour in milliseconds
+        startingDelay: 1000, // 1 second
+        jitter: 'full',
+        numOfAttempts: Infinity,
+        retry: (_err: Error, attempt: number) => {
+          // not logging the error since we don't control what's in the error and it might contain sensitive data
+          // ESO already logs specific caught errors before passing the error along
+          this.logger.warn(`failed to get message signing key pair. retrying attempt: ${attempt}`);
+          return true;
+        },
+      }
+    );
+
+    return soDoc;
+  }
+
   private async getCurrentKeyPairObj(): Promise<
     SavedObjectsFindResult<MessageSigningKeys> | undefined
   > {
@@ -205,6 +235,10 @@ export class MessageSigningService implements MessageSigningServiceInterface {
     }
     finder.close();
 
+    if (soDoc?.error) {
+      throw soDoc.error;
+    }
+
     return soDoc;
   }
 
@@ -216,7 +250,7 @@ export class MessageSigningService implements MessageSigningServiceInterface {
       }
     | undefined
   > {
-    const currentKeyPair = await this.getCurrentKeyPairObj();
+    const currentKeyPair = await this.getCurrentKeyPairObjWithRetry();
     if (!currentKeyPair) {
       return;
     }