From 0ee5413f1ea27e6994324538ed495aeae927e1a8 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 12 Oct 2022 16:46:46 -0700 Subject: [PATCH] [MD] Add data source signing support (#2510) (#2547) * Add data source signing support * Optimize error handling and logging * Update wording on error message and readme Signed-off-by: Louis Chu (cherry picked from commit e3bbdefb34895b58079f36380f35067cbceba198) Co-authored-by: Louis Chu --- CHANGELOG.md | 1 + src/plugins/data_source/README.md | 41 +++-- src/plugins/data_source/config.ts | 2 +- .../server/client/configure_client.test.ts | 49 +++++- .../server/client/configure_client.ts | 37 +++-- .../cryptography/cryptography_client.test.ts | 117 ------------- .../cryptography/cryptography_client.ts | 70 -------- .../data_source/server/cryptography/index.ts | 6 - .../server/cryptography_service.mocks.ts | 14 ++ .../server/cryptography_service.test.ts | 57 +++++++ .../server/cryptography_service.ts | 87 ++++++++++ .../legacy/configure_legacy_client.test.ts | 56 ++++++- .../server/legacy/configure_legacy_client.ts | 16 +- src/plugins/data_source/server/plugin.ts | 30 ++-- ...ata_source_saved_objects_client_wrapper.ts | 157 +++++++++++++++--- src/plugins/data_source/server/types.ts | 5 +- 16 files changed, 465 insertions(+), 280 deletions(-) delete mode 100644 src/plugins/data_source/server/cryptography/cryptography_client.test.ts delete mode 100644 src/plugins/data_source/server/cryptography/cryptography_client.ts delete mode 100644 src/plugins/data_source/server/cryptography/index.ts create mode 100644 src/plugins/data_source/server/cryptography_service.mocks.ts create mode 100644 src/plugins/data_source/server/cryptography_service.test.ts create mode 100644 src/plugins/data_source/server/cryptography_service.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 67711160ea71..c0339170df21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### ๐Ÿ“ˆ Features/Enhancements * [MD] Support legacy client for data source ([#2204](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2204)) +* [MD] Add data source signing support ([#2510](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2510)) * [Plugin Helpers] Facilitate version changes ([#2398](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2398)) * [MD] Display error toast for create index pattern with data source ([#2506](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2506)) * [Multi DataSource] UX enhancement on index pattern management stack ([#2505](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2505)) diff --git a/src/plugins/data_source/README.md b/src/plugins/data_source/README.md index cfda79a2908f..fdf6baab783f 100755 --- a/src/plugins/data_source/README.md +++ b/src/plugins/data_source/README.md @@ -5,15 +5,17 @@ An OpenSearch Dashboards plugin This plugin introduces support for multiple data sources into OpenSearch Dashboards and provides related functions to connect to OpenSearch data sources. ## Configuration + Update the following configuration in the `opensearch_dashboards.yml` file to apply changes. Refer to the schema [here](https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/src/plugins/data_source/config.ts) for supported configurations. 1. The dataSource plugin is disabled by default; to enable it: -`data_source.enabled: true` + `data_source.enabled: true` 2. The audit trail is enabled by default for logging the access to data source; to disable it: -`data_source.audit.enabled: false` + `data_source.audit.enabled: false` + +- Current auditor configuration: - - Current auditor configuration: ``` data_source.audit.appender.kind: 'file' data_source.audit.appender.layout.kind: 'pattern' @@ -21,34 +23,43 @@ data_source.audit.appender.path: '/tmp/opensearch-dashboards-data-source-audit.l ``` 3. The default encryption-related configuration parameters are: + ``` data_source.encryption.wrappingKeyName: 'changeme' data_source.encryption.wrappingKeyNamespace: 'changeme' data_source.encryption.wrappingKey: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] ``` + Note that if any of the encryption keyring configuration values change (wrappingKeyName/wrappingKeyNamespace/wrappingKey), none of the previously-encrypted credentials can be decrypted; therefore, credentials of previously created data sources must be updated to continue use. **What are the best practices for generating a secure wrapping key?** WrappingKey is an array of 32 random numbers. Read [more](https://en.wikipedia.org/wiki/Cryptographically_secure_pseudorandom_number_generator) about best practices for generating a secure wrapping key. ## Public + The public plugin is used to enable and disable the features related to multi data source available in other plugins. e.g. data_source_management, index_pattern_management - Add as a required dependency for whole plugin on/off switch - Add as opitional dependency for partial flow changes control ## Server + The provided data source client is integrated with default search strategy in data plugin. When data source id presented in IOpenSearchSearchRequest, data source client will be used. ### Data Source Service -The data source service will provide a data source client given a data source id and optional client configurations. + +The data source service will provide a data source client given a data source id and optional client configurations. Currently supported client config is: + - `data_source.clientPool.size` Data source service uses LRU cache to cache the root client to improve client pool usage. + #### Example usage: + In the RequestHandler, get an instance of the client using: + ```ts client: OpenSearchClient = await context.dataSource.opensearch.getClient(dataSourceId); @@ -57,17 +68,21 @@ apiCaller: LegacyAPICaller = context.dataSource.opensearch.legacy.getClient(data ``` ### Data Source Client Wrapper + The data source saved object client wrapper overrides the write related action for data source object in order to perform validation and encryption actions of the authentication information inside data source. -### Cryptography Client -The research for choosing a suitable stack can be found in: [#1756](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1756) -#### Example usage: -```ts -//Encrypt -const encryptedPassword = await this.cryptographyClient.encryptAndEncode(password); -//Decrypt -const decodedPassword = await this.cryptographyClient.decodeAndDecrypt(password); -``` +### Cryptography service + +The cryptography service encrypts and decrypts data source credentials (support no_auth and username_password credential types). Highlight the best security practices listed below: + +a. Envelope encryption - provides strong protection on data keys. Read more details [here](https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/concepts.html#envelope-encryption) + +b. Key derivation with HMAC - KDF with SHA-384 protects against accidental reuse of a data encryption keys and reduces the risk of overusing data keys. + +c. Signature algorithm - ECDSA with P-384 and SHA-384. Under multiple data source case, data source documents stored on OpenSearch can be modified / replaced by attacker. With ECDSA signature, ciphertext decryption will fail if itโ€™s getting pullted. No one will be able to create another signature that verifies with the public key because the private key has been dropped. + +Please check https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1756 for more details. + --- ## Development diff --git a/src/plugins/data_source/config.ts b/src/plugins/data_source/config.ts index d7579026c6e2..f2fd79fade9a 100644 --- a/src/plugins/data_source/config.ts +++ b/src/plugins/data_source/config.ts @@ -8,7 +8,7 @@ import { fileAppenderSchema } from './audit_config'; const KEY_NAME_MIN_LENGTH: number = 1; const KEY_NAME_MAX_LENGTH: number = 100; -// Wrapping key size shoule be 32 bytes, as used in envelope encryption algorithms. +// Wrapping key size should be 32 bytes, as used in envelope encryption algorithms. const WRAPPING_KEY_SIZE: number = 32; export const configSchema = schema.object({ diff --git a/src/plugins/data_source/server/client/configure_client.test.ts b/src/plugins/data_source/server/client/configure_client.test.ts index 696c7ce2daf6..fa4044163610 100644 --- a/src/plugins/data_source/server/client/configure_client.test.ts +++ b/src/plugins/data_source/server/client/configure_client.test.ts @@ -14,17 +14,18 @@ import { configureClient } from './configure_client'; import { ClientOptions } from '@opensearch-project/opensearch'; // eslint-disable-next-line @osd/eslint/no-restricted-paths import { opensearchClientMock } from '../../../../core/server/opensearch/client/mocks'; -import { CryptographyClient } from '../cryptography'; +import { cryptographyServiceSetupMock } from '../cryptography_service.mocks'; +import { CryptographyServiceSetup } from '../cryptography_service'; import { DataSourceClientParams } from '../types'; const DATA_SOURCE_ID = 'a54b76ec86771ee865a0f74a305dfff8'; -const cryptoClient = new CryptographyClient('test', 'test', new Array(32).fill(0)); // TODO: improve UT describe('configureClient', () => { let logger: ReturnType; let config: DataSourcePluginConfigType; let savedObjectsMock: jest.Mocked; + let cryptographyMock: jest.Mocked; let clientPoolSetup: OpenSearchClientPoolSetup; let clientOptions: ClientOptions; let dataSourceAttr: DataSourceAttributes; @@ -35,6 +36,8 @@ describe('configureClient', () => { dsClient = opensearchClientMock.createInternalClient(); logger = loggingSystemMock.createLogger(); savedObjectsMock = savedObjectsClientMock.create(); + cryptographyMock = cryptographyServiceSetupMock.create(); + config = { enabled: true, clientPool: { @@ -75,7 +78,7 @@ describe('configureClient', () => { dataSourceClientParams = { dataSourceId: DATA_SOURCE_ID, savedObjects: savedObjectsMock, - cryptographyClient: cryptoClient, + cryptography: cryptographyMock, }; ClientMock.mockImplementation(() => dsClient); @@ -109,14 +112,48 @@ describe('configureClient', () => { expect(client).toBe(dsClient.child.mock.results[0].value); }); - test('configure client with auth.type == username_password, will first call decrypt()', async () => { - const spy = jest.spyOn(cryptoClient, 'decodeAndDecrypt').mockResolvedValue('password'); + test('configure client with auth.type == username_password, will first call decodeAndDecrypt()', async () => { + const decodeAndDecryptSpy = jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({ + decryptedText: 'password', + encryptionContext: { endpoint: 'http://localhost' }, + }); const client = await configureClient(dataSourceClientParams, clientPoolSetup, config, logger); expect(ClientMock).toHaveBeenCalledTimes(1); expect(savedObjectsMock.get).toHaveBeenCalledTimes(1); - expect(spy).toHaveBeenCalledTimes(1); + expect(decodeAndDecryptSpy).toHaveBeenCalledTimes(1); expect(client).toBe(dsClient.child.mock.results[0].value); }); + + test('configure client with auth.type == username_password and password contaminated', async () => { + const decodeAndDecryptSpy = jest + .spyOn(cryptographyMock, 'decodeAndDecrypt') + .mockImplementation(() => { + throw new Error(); + }); + + await expect( + configureClient(dataSourceClientParams, clientPoolSetup, config, logger) + ).rejects.toThrowError(); + + expect(ClientMock).toHaveBeenCalledTimes(1); + expect(savedObjectsMock.get).toHaveBeenCalledTimes(1); + expect(decodeAndDecryptSpy).toHaveBeenCalledTimes(1); + }); + + test('configure client with auth.type == username_password and endpoint contaminated', async () => { + const decodeAndDecryptSpy = jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({ + decryptedText: 'password', + encryptionContext: { endpoint: 'http://dummy.com' }, + }); + + await expect( + configureClient(dataSourceClientParams, clientPoolSetup, config, logger) + ).rejects.toThrowError(); + + expect(ClientMock).toHaveBeenCalledTimes(1); + expect(savedObjectsMock.get).toHaveBeenCalledTimes(1); + expect(decodeAndDecryptSpy).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/plugins/data_source/server/client/configure_client.ts b/src/plugins/data_source/server/client/configure_client.ts index febac85070ea..0e153f89b0f9 100644 --- a/src/plugins/data_source/server/client/configure_client.ts +++ b/src/plugins/data_source/server/client/configure_client.ts @@ -12,14 +12,14 @@ import { UsernamePasswordTypedContent, } from '../../common/data_sources'; import { DataSourcePluginConfigType } from '../../config'; -import { CryptographyClient } from '../cryptography'; +import { CryptographyServiceSetup } from '../cryptography_service'; import { DataSourceConfigError } from '../lib/error'; import { DataSourceClientParams } from '../types'; import { parseClientOptions } from './client_config'; import { OpenSearchClientPoolSetup } from './client_pool'; export const configureClient = async ( - { dataSourceId, savedObjects, cryptographyClient }: DataSourceClientParams, + { dataSourceId, savedObjects, cryptography }: DataSourceClientParams, openSearchClientPoolSetup: OpenSearchClientPoolSetup, config: DataSourcePluginConfigType, logger: Logger @@ -28,12 +28,12 @@ export const configureClient = async ( const dataSource = await getDataSource(dataSourceId, savedObjects); const rootClient = getRootClient(dataSource.attributes, config, openSearchClientPoolSetup); - return await getQueryClient(rootClient, dataSource, cryptographyClient); + return await getQueryClient(rootClient, dataSource, cryptography); } catch (error: any) { - logger.error(`Fail to get data source client for dataSourceId: [${dataSourceId}]`); + logger.error(`Failed to get data source client for dataSourceId: [${dataSourceId}]`); logger.error(error); // Re-throw as DataSourceConfigError - throw new DataSourceConfigError('Fail to get data source client: ', error); + throw new DataSourceConfigError('Failed to get data source client: ', error); } }; @@ -50,13 +50,28 @@ export const getDataSource = async ( export const getCredential = async ( dataSource: SavedObject, - cryptographyClient: CryptographyClient + cryptography: CryptographyServiceSetup ): Promise => { + const { endpoint } = dataSource.attributes!; + const { username, password } = dataSource.attributes.auth.credentials!; - const decodedPassword = await cryptographyClient.decodeAndDecrypt(password); + + const { decryptedText, encryptionContext } = await cryptography + .decodeAndDecrypt(password) + .catch((err: any) => { + // Re-throw as DataSourceConfigError + throw new DataSourceConfigError('Unable to decrypt "auth.credentials.password".', err); + }); + + if (encryptionContext!.endpoint !== endpoint) { + throw new Error( + 'Data source "endpoint" contaminated. Please delete and create another data source.' + ); + } + const credential = { username, - password: decodedPassword, + password: decryptedText, }; return credential; @@ -67,13 +82,13 @@ export const getCredential = async ( * * @param rootClient root client for the connection with given data source endpoint. * @param dataSource data source saved object - * @param cryptographyClient cryptography client for password encryption / decryption + * @param cryptography cryptography service for password encryption / decryption * @returns child client. */ const getQueryClient = async ( rootClient: Client, dataSource: SavedObject, - cryptographyClient: CryptographyClient + cryptography: CryptographyServiceSetup ): Promise => { const authType = dataSource.attributes.auth.type; @@ -82,7 +97,7 @@ const getQueryClient = async ( return rootClient.child(); case AuthType.UsernamePasswordType: - const credential = await getCredential(dataSource, cryptographyClient); + const credential = await getCredential(dataSource, cryptography); return getBasicAuthClient(rootClient, credential); default: diff --git a/src/plugins/data_source/server/cryptography/cryptography_client.test.ts b/src/plugins/data_source/server/cryptography/cryptography_client.test.ts deleted file mode 100644 index 1f8d2596a3c4..000000000000 --- a/src/plugins/data_source/server/cryptography/cryptography_client.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { CryptographyClient } from './cryptography_client'; -import { randomBytes } from 'crypto'; - -const dummyWrappingKeyName = 'dummy_wrapping_key_name'; -const dummyWrappingKeyNamespace = 'dummy_wrapping_key_namespace'; - -test('Invalid wrapping key size throws error', () => { - const dummyRandomBytes = [...randomBytes(31)]; - const expectedErrorMsg = `Wrapping key size shoule be 32 bytes, as used in envelope encryption. Current wrapping key size: '${dummyRandomBytes.length}' bytes`; - expect(() => { - new CryptographyClient(dummyWrappingKeyName, dummyWrappingKeyNamespace, dummyRandomBytes); - }).toThrowError(new Error(expectedErrorMsg)); -}); - -describe('Test encrpyt and decrypt module', () => { - const dummyPlainText = 'dummy'; - const dummyNumArray1 = [...randomBytes(32)]; - const dummyNumArray2 = [...randomBytes(32)]; - - describe('Positive test cases', () => { - test('Encrypt and Decrypt with same in memory keyring', async () => { - const cryptographyClient = new CryptographyClient( - dummyWrappingKeyName, - dummyWrappingKeyNamespace, - dummyNumArray1 - ); - const encrypted = await cryptographyClient.encryptAndEncode(dummyPlainText); - const outputText = await cryptographyClient.decodeAndDecrypt(encrypted); - expect(outputText).toBe(dummyPlainText); - }); - test('Encrypt and Decrypt with two different keyrings with exact same identifiers', async () => { - const cryptographyClient1 = new CryptographyClient( - dummyWrappingKeyName, - dummyWrappingKeyNamespace, - dummyNumArray1 - ); - const encrypted = await cryptographyClient1.encryptAndEncode(dummyPlainText); - - const cryptographyClient2 = new CryptographyClient( - dummyWrappingKeyName, - dummyWrappingKeyNamespace, - dummyNumArray1 - ); - const outputText = await cryptographyClient2.decodeAndDecrypt(encrypted); - expect(cryptographyClient1 === cryptographyClient2).toBeFalsy(); - expect(outputText).toBe(dummyPlainText); - }); - }); - - describe('Negative test cases', () => { - const defaultWrappingKeyName = 'changeme'; - const defaultWrappingKeyNamespace = 'changeme'; - const expectedErrorMsg = 'unencryptedDataKey has not been set'; - test('Encrypt and Decrypt with different key names', async () => { - const cryptographyClient1 = new CryptographyClient( - dummyWrappingKeyName, - dummyWrappingKeyNamespace, - dummyNumArray1 - ); - const encrypted = await cryptographyClient1.encryptAndEncode(dummyPlainText); - - const cryptographyClient2 = new CryptographyClient( - defaultWrappingKeyName, - dummyWrappingKeyNamespace, - dummyNumArray1 - ); - try { - await cryptographyClient2.decodeAndDecrypt(encrypted); - } catch (error) { - expect(error.message).toMatch(expectedErrorMsg); - } - }); - test('Encrypt and Decrypt with different key namespaces', async () => { - const cryptographyClient1 = new CryptographyClient( - dummyWrappingKeyName, - dummyWrappingKeyNamespace, - dummyNumArray1 - ); - const encrypted = await cryptographyClient1.encryptAndEncode(dummyPlainText); - - const cryptographyClient2 = new CryptographyClient( - dummyWrappingKeyName, - defaultWrappingKeyNamespace, - dummyNumArray1 - ); - try { - await cryptographyClient2.decodeAndDecrypt(encrypted); - } catch (error) { - expect(error.message).toMatch(expectedErrorMsg); - } - }); - test('Encrypt and Decrypt with different wrapping keys', async () => { - const cryptographyClient1 = new CryptographyClient( - dummyWrappingKeyName, - dummyWrappingKeyNamespace, - dummyNumArray1 - ); - const encrypted = await cryptographyClient1.encryptAndEncode(dummyPlainText); - - const cryptographyClient2 = new CryptographyClient( - dummyWrappingKeyName, - dummyWrappingKeyNamespace, - dummyNumArray2 - ); - try { - await cryptographyClient2.decodeAndDecrypt(encrypted); - } catch (error) { - expect(error.message).toMatch(expectedErrorMsg); - } - }); - }); -}); diff --git a/src/plugins/data_source/server/cryptography/cryptography_client.ts b/src/plugins/data_source/server/cryptography/cryptography_client.ts deleted file mode 100644 index f5968ae13adb..000000000000 --- a/src/plugins/data_source/server/cryptography/cryptography_client.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { - buildClient, - CommitmentPolicy, - RawAesKeyringNode, - RawAesWrappingSuiteIdentifier, -} from '@aws-crypto/client-node'; - -export const ENCODING_STRATEGY: BufferEncoding = 'base64'; -export const WRAPPING_KEY_SIZE: number = 32; - -export class CryptographyClient { - private readonly commitmentPolicy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT; - private readonly wrappingSuite = RawAesWrappingSuiteIdentifier.AES256_GCM_IV12_TAG16_NO_PADDING; - - private keyring: RawAesKeyringNode; - - private readonly encrypt: Function; - private readonly decrypt: Function; - - /** - * @param {string} wrappingKeyName name value to identify the AES key in a keyring - * @param {string} wrappingKeyNamespace namespace value to identify the AES key in a keyring, - * @param {number[]} wrappingKey 32 Bytes raw wrapping key used to perform envelope encryption - */ - constructor(wrappingKeyName: string, wrappingKeyNamespace: string, wrappingKey: number[]) { - if (wrappingKey.length !== WRAPPING_KEY_SIZE) { - const wrappingKeySizeMismatchMsg = `Wrapping key size shoule be 32 bytes, as used in envelope encryption. Current wrapping key size: '${wrappingKey.length}' bytes`; - throw new Error(wrappingKeySizeMismatchMsg); - } - - // Create raw AES keyring - this.keyring = new RawAesKeyringNode({ - keyName: wrappingKeyName, - keyNamespace: wrappingKeyNamespace, - unencryptedMasterKey: new Uint8Array(wrappingKey), - wrappingSuite: this.wrappingSuite, - }); - - // Destructuring encrypt and decrypt functions from client - const { encrypt, decrypt } = buildClient(this.commitmentPolicy); - - this.encrypt = encrypt; - this.decrypt = decrypt; - } - - /** - * Input text content and output encrypted string encoded with ENCODING_STRATEGY - * @param {string} plainText - * @returns {Promise} - */ - public async encryptAndEncode(plainText: string): Promise { - const result = await this.encrypt(this.keyring, plainText); - return result.result.toString(ENCODING_STRATEGY); - } - - /** - * Input encrypted content and output decrypted string - * @param {string} encrypted - * @returns {Promise} - */ - public async decodeAndDecrypt(encrypted: string): Promise { - const result = await this.decrypt(this.keyring, Buffer.from(encrypted, ENCODING_STRATEGY)); - return result.plaintext.toString(); - } -} diff --git a/src/plugins/data_source/server/cryptography/index.ts b/src/plugins/data_source/server/cryptography/index.ts deleted file mode 100644 index 857fa691bddf..000000000000 --- a/src/plugins/data_source/server/cryptography/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export { CryptographyClient } from './cryptography_client'; diff --git a/src/plugins/data_source/server/cryptography_service.mocks.ts b/src/plugins/data_source/server/cryptography_service.mocks.ts new file mode 100644 index 000000000000..d74912dfda9e --- /dev/null +++ b/src/plugins/data_source/server/cryptography_service.mocks.ts @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CryptographyServiceSetup } from './cryptography_service'; + +const create = () => + (({ + encryptAndEncode: jest.fn(), + decodeAndDecrypt: jest.fn(), + } as unknown) as jest.Mocked); + +export const cryptographyServiceSetupMock = { create }; diff --git a/src/plugins/data_source/server/cryptography_service.test.ts b/src/plugins/data_source/server/cryptography_service.test.ts new file mode 100644 index 000000000000..783a4f951393 --- /dev/null +++ b/src/plugins/data_source/server/cryptography_service.test.ts @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import { loggingSystemMock } from '../../../core/server/mocks'; +import { DataSourcePluginConfigType } from '../config'; +import { CryptographyService } from './cryptography_service'; + +const logger = loggingSystemMock.create(); + +describe('Cryptography Service', () => { + let service: CryptographyService; + + beforeEach(() => { + const mockLogger = logger.get('cryptography-service-test'); + service = new CryptographyService(mockLogger); + }); + + afterEach(() => { + service.stop(); + jest.clearAllMocks(); + }); + + // TODO: Add more UTs after Jest issue resolved https://github.com/facebook/jest/issues/13349 + describe('setup()', () => { + test('invalid wrapping key size throws error', () => { + const config = { + enabled: true, + encryption: { + wrappingKeyName: 'dummy', + wrappingKeyNamespace: 'dummy', + wrappingKey: new Array(31).fill(0), + }, + } as DataSourcePluginConfigType; + + const expectedErrorMsg = `Wrapping key size should be 32 bytes, as used in envelope encryption. Current wrapping key size: '${config.encryption.wrappingKey.length}' bytes`; + + expect(() => { + service.setup(config); + }).toThrowError(new Error(expectedErrorMsg)); + }); + + test('exposes proper contract', () => { + const config = { + enabled: true, + encryption: { + wrappingKeyName: 'dummy', + wrappingKeyNamespace: 'dummy', + wrappingKey: new Array(32).fill(0), + }, + } as DataSourcePluginConfigType; + const setup = service.setup(config); + expect(setup).toHaveProperty('encryptAndEncode'); + expect(setup).toHaveProperty('decodeAndDecrypt'); + }); + }); +}); diff --git a/src/plugins/data_source/server/cryptography_service.ts b/src/plugins/data_source/server/cryptography_service.ts new file mode 100644 index 000000000000..a2fcc0dfe301 --- /dev/null +++ b/src/plugins/data_source/server/cryptography_service.ts @@ -0,0 +1,87 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + buildClient, + CommitmentPolicy, + RawAesKeyringNode, + RawAesWrappingSuiteIdentifier, +} from '@aws-crypto/client-node'; +import { Logger } from '../../../../src/core/server'; +import { DataSourcePluginConfigType } from '../config'; + +export const ENCODING_STRATEGY: BufferEncoding = 'base64'; +export const WRAPPING_KEY_SIZE: number = 32; + +export interface EncryptionContext { + endpoint?: string; +} + +export interface RefinedDecryptOutPut { + decryptedText: string; + encryptionContext: EncryptionContext; +} + +export interface CryptographyServiceSetup { + encryptAndEncode: (plainText: string, encryptionContext: EncryptionContext) => Promise; + decodeAndDecrypt: (encrypted: string) => Promise; +} + +export class CryptographyService { + // commitment policy to enable data key derivation and ECDSA signature + private readonly commitmentPolicy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT; + // algorithm suite identifier to adopt AES-GCM + private readonly wrappingSuite = RawAesWrappingSuiteIdentifier.AES256_GCM_IV12_TAG16_NO_PADDING; + + constructor(private logger: Logger) {} + + setup(config: DataSourcePluginConfigType): CryptographyServiceSetup { + // Fetch configs used to create credential saved objects client wrapper + const { wrappingKeyName, wrappingKeyNamespace, wrappingKey } = config.encryption; + + if (wrappingKey.length !== WRAPPING_KEY_SIZE) { + const wrappingKeySizeMismatchMsg = `Wrapping key size should be 32 bytes, as used in envelope encryption. Current wrapping key size: '${wrappingKey.length}' bytes`; + this.logger.error(wrappingKeySizeMismatchMsg); + throw new Error(wrappingKeySizeMismatchMsg); + } + + // Create raw AES keyring + const keyring = new RawAesKeyringNode({ + keyName: wrappingKeyName, + keyNamespace: wrappingKeyNamespace, + unencryptedMasterKey: new Uint8Array(wrappingKey), + wrappingSuite: this.wrappingSuite, + }); + + // Destructuring encrypt and decrypt functions from client + const { encrypt, decrypt } = buildClient(this.commitmentPolicy); + + const encryptAndEncode = async (plainText: string, encryptionContext = {}): Promise => { + const result = await encrypt(keyring, plainText, { + encryptionContext, + }); + return result.result.toString(ENCODING_STRATEGY); + }; + + const decodeAndDecrypt = async (encrypted: string): Promise => { + const { plaintext, messageHeader } = await decrypt( + keyring, + Buffer.from(encrypted, ENCODING_STRATEGY) + ); + return { + decryptedText: plaintext.toString(), + encryptionContext: { + endpoint: messageHeader.encryptionContext.endpoint, + }, + }; + }; + + return { encryptAndEncode, decodeAndDecrypt }; + } + + start() {} + + stop() {} +} diff --git a/src/plugins/data_source/server/legacy/configure_legacy_client.test.ts b/src/plugins/data_source/server/legacy/configure_legacy_client.test.ts index 752487741f10..bfdf0ce585f0 100644 --- a/src/plugins/data_source/server/legacy/configure_legacy_client.test.ts +++ b/src/plugins/data_source/server/legacy/configure_legacy_client.test.ts @@ -8,7 +8,8 @@ import { loggingSystemMock, savedObjectsClientMock } from '../../../../core/serv import { DATA_SOURCE_SAVED_OBJECT_TYPE } from '../../common'; import { AuthType, DataSourceAttributes } from '../../common/data_sources'; import { DataSourcePluginConfigType } from '../../config'; -import { CryptographyClient } from '../cryptography'; +import { cryptographyServiceSetupMock } from '../cryptography_service.mocks'; +import { CryptographyServiceSetup } from '../cryptography_service'; import { DataSourceClientParams, LegacyClientCallAPIParams } from '../types'; import { OpenSearchClientPoolSetup } from '../client'; import { ConfigOptions } from 'elasticsearch'; @@ -16,13 +17,13 @@ import { ClientMock, parseClientOptionsMock } from './configure_legacy_client.te import { configureLegacyClient } from './configure_legacy_client'; const DATA_SOURCE_ID = 'a54b76ec86771ee865a0f74a305dfff8'; -const cryptographyClient = new CryptographyClient('test', 'test', new Array(32).fill(0)); // TODO: improve UT describe('configureLegacyClient', () => { let logger: ReturnType; let config: DataSourcePluginConfigType; let savedObjectsMock: jest.Mocked; + let cryptographyMock: jest.Mocked; let clientPoolSetup: OpenSearchClientPoolSetup; let configOptions: ConfigOptions; let dataSourceAttr: DataSourceAttributes; @@ -33,7 +34,6 @@ describe('configureLegacyClient', () => { }; let dataSourceClientParams: DataSourceClientParams; let callApiParams: LegacyClientCallAPIParams; - let decodeAndDecryptSpy: jest.SpyInstance, [encrypted: string]>; const mockResponse = { data: 'ping' }; @@ -44,6 +44,7 @@ describe('configureLegacyClient', () => { }; logger = loggingSystemMock.createLogger(); savedObjectsMock = savedObjectsClientMock.create(); + cryptographyMock = cryptographyServiceSetupMock.create(); config = { enabled: true, clientPool: { @@ -89,7 +90,7 @@ describe('configureLegacyClient', () => { dataSourceClientParams = { dataSourceId: DATA_SOURCE_ID, savedObjects: savedObjectsMock, - cryptographyClient, + cryptography: cryptographyMock, }; ClientMock.mockImplementation(() => mockOpenSearchClientInstance); @@ -100,10 +101,6 @@ describe('configureLegacyClient', () => { response: mockResponse, }); }); - - decodeAndDecryptSpy = jest - .spyOn(cryptographyClient, 'decodeAndDecrypt') - .mockResolvedValue('password'); }); afterEach(() => { @@ -140,7 +137,12 @@ describe('configureLegacyClient', () => { expect(savedObjectsMock.get).toHaveBeenCalledTimes(1); }); - test('configure client with auth.type == no_auth, will first call decrypt()', async () => { + test('configure client with auth.type == username_password, will first call decrypt()', async () => { + const decodeAndDecryptSpy = jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({ + decryptedText: 'password', + encryptionContext: { endpoint: 'http://localhost' }, + }); + const mockResult = await configureLegacyClient( dataSourceClientParams, callApiParams, @@ -155,7 +157,43 @@ describe('configureLegacyClient', () => { expect(mockResult).toBeDefined(); }); + test('configure client with auth.type == username_password and password contaminated', async () => { + const decodeAndDecryptSpy = jest + .spyOn(cryptographyMock, 'decodeAndDecrypt') + .mockImplementation(() => { + throw new Error(); + }); + + await expect( + configureLegacyClient(dataSourceClientParams, callApiParams, clientPoolSetup, config, logger) + ).rejects.toThrowError(); + + expect(ClientMock).toHaveBeenCalledTimes(1); + expect(savedObjectsMock.get).toHaveBeenCalledTimes(1); + expect(decodeAndDecryptSpy).toHaveBeenCalledTimes(1); + }); + + test('configure client with auth.type == username_password and endpoint contaminated', async () => { + const decodeAndDecryptSpy = jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({ + decryptedText: 'password', + encryptionContext: { endpoint: 'http://dummy.com' }, + }); + + await expect( + configureLegacyClient(dataSourceClientParams, callApiParams, clientPoolSetup, config, logger) + ).rejects.toThrowError(); + + expect(ClientMock).toHaveBeenCalledTimes(1); + expect(savedObjectsMock.get).toHaveBeenCalledTimes(1); + expect(decodeAndDecryptSpy).toHaveBeenCalledTimes(1); + }); + test('correctly called with endpoint and params', async () => { + jest.spyOn(cryptographyMock, 'decodeAndDecrypt').mockResolvedValue({ + decryptedText: 'password', + encryptionContext: { endpoint: 'http://localhost' }, + }); + const mockParams = { param: 'ping' }; const mockResult = await configureLegacyClient( dataSourceClientParams, diff --git a/src/plugins/data_source/server/legacy/configure_legacy_client.ts b/src/plugins/data_source/server/legacy/configure_legacy_client.ts index 0f2a63287e14..2921e0cad48e 100644 --- a/src/plugins/data_source/server/legacy/configure_legacy_client.ts +++ b/src/plugins/data_source/server/legacy/configure_legacy_client.ts @@ -19,14 +19,14 @@ import { UsernamePasswordTypedContent, } from '../../common/data_sources'; import { DataSourcePluginConfigType } from '../../config'; -import { CryptographyClient } from '../cryptography'; +import { CryptographyServiceSetup } from '../cryptography_service'; import { DataSourceClientParams, LegacyClientCallAPIParams } from '../types'; import { OpenSearchClientPoolSetup, getCredential, getDataSource } from '../client'; import { parseClientOptions } from './client_config'; import { DataSourceConfigError } from '../lib/error'; export const configureLegacyClient = async ( - { dataSourceId, savedObjects, cryptographyClient }: DataSourceClientParams, + { dataSourceId, savedObjects, cryptography }: DataSourceClientParams, callApiParams: LegacyClientCallAPIParams, openSearchClientPoolSetup: OpenSearchClientPoolSetup, config: DataSourcePluginConfigType, @@ -36,12 +36,12 @@ export const configureLegacyClient = async ( const dataSource = await getDataSource(dataSourceId, savedObjects); const rootClient = getRootClient(dataSource.attributes, config, openSearchClientPoolSetup); - return await getQueryClient(rootClient, dataSource, cryptographyClient, callApiParams); + return await getQueryClient(rootClient, dataSource, cryptography, callApiParams); } catch (error: any) { - logger.error(`Fail to get data source client for dataSourceId: [${dataSourceId}]`); + logger.error(`Failed to get data source client for dataSourceId: [${dataSourceId}]`); logger.error(error); // Re-throw as DataSourceConfigError - throw new DataSourceConfigError('Fail to get data source client: ', error); + throw new DataSourceConfigError('Failed to get data source client: ', error); } }; @@ -50,13 +50,13 @@ export const configureLegacyClient = async ( * * @param rootClient root client for the connection with given data source endpoint. * @param dataSource data source saved object - * @param cryptographyClient cryptography client for password encryption / decryption + * @param cryptography cryptography service for password encryption / decryption * @returns child client. */ const getQueryClient = async ( rootClient: Client, dataSource: SavedObject, - cryptographyClient: CryptographyClient, + cryptography: CryptographyServiceSetup, { endpoint, clientParams, options }: LegacyClientCallAPIParams ) => { const authType = dataSource.attributes.auth.type; @@ -69,7 +69,7 @@ const getQueryClient = async ( options ); case AuthType.UsernamePasswordType: - const credential = await getCredential(dataSource, cryptographyClient); + const credential = await getCredential(dataSource, cryptography); return getBasicAuthClient(rootClient, { endpoint, clientParams, options }, credential); default: diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts index c166fb736768..2a3d02be6838 100644 --- a/src/plugins/data_source/server/plugin.ts +++ b/src/plugins/data_source/server/plugin.ts @@ -20,7 +20,7 @@ import { } from '../../../../src/core/server'; import { DataSourcePluginConfigType } from '../config'; import { LoggingAuditor } from './audit/logging_auditor'; -import { CryptographyClient } from './cryptography'; +import { CryptographyService, CryptographyServiceSetup } from './cryptography_service'; import { DataSourceService, DataSourceServiceSetup } from './data_source_service'; import { DataSourceSavedObjectsClientWrapper, dataSource } from './saved_objects'; import { DataSourcePluginSetup, DataSourcePluginStart } from './types'; @@ -30,11 +30,13 @@ import { DATA_SOURCE_SAVED_OBJECT_TYPE } from '../common'; import { ensureRawRequest } from '../../../../src/core/server/http/router'; export class DataSourcePlugin implements Plugin { private readonly logger: Logger; + private readonly cryptographyService: CryptographyService; private readonly dataSourceService: DataSourceService; private readonly config$: Observable; constructor(private initializerContext: PluginInitializerContext) { this.logger = this.initializerContext.logger.get(); + this.cryptographyService = new CryptographyService(this.logger.get('cryptography-service')); this.dataSourceService = new DataSourceService(this.logger.get('data-source-service')); this.config$ = this.initializerContext.config.create(); } @@ -47,17 +49,13 @@ export class DataSourcePlugin implements Plugin( map((dataSourceConfig) => ({ @@ -94,12 +90,13 @@ export class DataSourcePlugin implements Plugin coreStart.auditTrail); + const dataSourceService: DataSourceServiceSetup = await this.dataSourceService.setup(config); // Register data source plugin context to route handler context core.http.registerRouteHandlerContext( 'dataSource', this.createDataSourceRouteHandlerContext( dataSourceService, - cryptographyClient, + cryptographyServiceSetup, this.logger, auditTrailPromise ) @@ -120,7 +117,7 @@ export class DataSourcePlugin implements Plugin ): IContextProvider, 'dataSource'> => { @@ -129,12 +126,13 @@ export class DataSourcePlugin implements Plugin { const auditor = auditTrailPromise.then((auditTrail) => auditTrail.asScoped(req)); + this.logAuditMessage(auditor, dataSourceId, req); return dataSourceService.getDataSourceClient({ dataSourceId, savedObjects: context.core.savedObjects.client, - cryptographyClient, + cryptography, }); }, legacy: { @@ -142,7 +140,7 @@ export class DataSourcePlugin implements Plugin = await this.validateAndUpdatePartialAttributes( - attributes + wrapperOptions, + id, + attributes, + options ); return await wrapperOptions.client.update(type, id, encryptedAttributes, options); @@ -92,14 +93,17 @@ export class DataSourceSavedObjectsClientWrapper { ): Promise> => { objects = await Promise.all( objects.map(async (object) => { - const { type, attributes } = object; + const { id, type, attributes } = object; if (DATA_SOURCE_SAVED_OBJECT_TYPE !== type) { return object; } const encryptedAttributes: Partial = await this.validateAndUpdatePartialAttributes( - attributes + wrapperOptions, + id, + attributes, + options ); return { @@ -141,26 +145,39 @@ export class DataSourceSavedObjectsClientWrapper { private async validateAndEncryptAttributes(attributes: T) { this.validateAttributes(attributes); - const { auth } = attributes; + const { endpoint, auth } = attributes; switch (auth.type) { case AuthType.NoAuth: return { ...attributes, // Drop the credentials attribute for no_auth - credentials: undefined, + auth: { + type: auth.type, + credentials: undefined, + }, }; case AuthType.UsernamePasswordType: + // Signing the data source with endpoint + const encryptionContext = { + endpoint, + }; + return { ...attributes, - auth: await this.encryptCredentials(auth), + auth: await this.encryptCredentials(auth, encryptionContext), }; default: - throw SavedObjectsErrorHelpers.createBadRequestError(`Invalid auth type: '${type}'`); + throw SavedObjectsErrorHelpers.createBadRequestError(`Invalid auth type: '${auth.type}'`); } } - private async validateAndUpdatePartialAttributes(attributes: T) { + private async validateAndUpdatePartialAttributes( + wrapperOptions: SavedObjectsClientWrapperOptions, + id: string, + attributes: Partial, + options: SavedObjectsUpdateOptions = {} + ) { const { auth, endpoint } = attributes; if (endpoint) { @@ -169,7 +186,7 @@ export class DataSourceSavedObjectsClientWrapper { ); } - if (auth === undefined) { + if (!auth) { return attributes; } @@ -180,13 +197,23 @@ export class DataSourceSavedObjectsClientWrapper { return { ...attributes, // Drop the credentials attribute for no_auth - credentials: undefined, + auth: { + type: auth.type, + credentials: null, + }, }; case AuthType.UsernamePasswordType: if (credentials?.password) { + // Fetch and validate existing signature + const encryptionContext = await this.validateEncryptionContext( + wrapperOptions, + id, + options + ); + return { ...attributes, - auth: await this.encryptCredentials(auth), + auth: await this.encryptCredentials(auth, encryptionContext), }; } else { return attributes; @@ -208,7 +235,7 @@ export class DataSourceSavedObjectsClientWrapper { throw SavedObjectsErrorHelpers.createBadRequestError('"endpoint" attribute is not valid'); } - if (auth === undefined) { + if (!auth) { throw SavedObjectsErrorHelpers.createBadRequestError('"auth" attribute is required'); } @@ -226,7 +253,7 @@ export class DataSourceSavedObjectsClientWrapper { case AuthType.NoAuth: break; case AuthType.UsernamePasswordType: - if (credentials === undefined) { + if (!credentials) { throw SavedObjectsErrorHelpers.createBadRequestError( '"auth.credentials" attribute is required' ); @@ -252,7 +279,95 @@ export class DataSourceSavedObjectsClientWrapper { } } - private async encryptCredentials(auth: T) { + private async validateEncryptionContext( + wrapperOptions: SavedObjectsClientWrapperOptions, + id: string, + options: SavedObjectsUpdateOptions = {} + ) { + let attributes; + + try { + // Fetch existing data source by id + const savedObject = await wrapperOptions.client.get(DATA_SOURCE_SAVED_OBJECT_TYPE, id, { + namespace: options.namespace, + }); + attributes = savedObject.attributes; + } catch (err: any) { + const errMsg = `Failed to fetch existing data source for dataSourceId [${id}]`; + this.logger.error(errMsg); + this.logger.error(err); + throw SavedObjectsErrorHelpers.decorateBadRequestError(err, errMsg); + } + + if (!attributes) { + throw SavedObjectsErrorHelpers.createBadRequestError( + 'Update failed due to deprecated data source: "attributes" missing. Please delete and create another data source.' + ); + } + + const { endpoint, auth } = attributes; + + if (!endpoint) { + throw SavedObjectsErrorHelpers.createBadRequestError( + 'Update failed due to deprecated data source: "endpoint" missing. Please delete and create another data source.' + ); + } + + if (!auth) { + throw SavedObjectsErrorHelpers.createBadRequestError( + 'Update failed due to deprecated data source: "auth" missing. Please delete and create another data source.' + ); + } + + switch (auth.type) { + case AuthType.NoAuth: + // Signing the data source with exsiting endpoint + return { + endpoint, + }; + case AuthType.UsernamePasswordType: + const { credentials } = auth; + if (!credentials) { + throw SavedObjectsErrorHelpers.createBadRequestError( + 'Update failed due to deprecated data source: "credentials" missing. Please delete and create another data source.' + ); + } + + const { username, password } = credentials; + + if (!username) { + throw SavedObjectsErrorHelpers.createBadRequestError( + 'Update failed due to deprecated data source: "auth.credentials.username" missing. Please delete and create another data source.' + ); + } + + if (!password) { + throw SavedObjectsErrorHelpers.createBadRequestError( + 'Update failed due to deprecated data source: "auth.credentials.username" missing. Please delete and create another data source.' + ); + } + + const { encryptionContext } = await this.cryptography + .decodeAndDecrypt(password) + .catch((err: any) => { + const errMsg = `Failed to update existing data source for dataSourceId [${id}]: unable to decrypt "auth.credentials.password"`; + this.logger.error(errMsg); + this.logger.error(err); + throw SavedObjectsErrorHelpers.decorateBadRequestError(err, errMsg); + }); + + if (encryptionContext.endpoint !== endpoint) { + throw SavedObjectsErrorHelpers.createBadRequestError( + 'Update failed due to deprecated data source: "endpoint" contaminated. Please delete and create another data source.' + ); + } + return encryptionContext; + default: + throw SavedObjectsErrorHelpers.createBadRequestError(`Invalid auth type: '${type}'`); + } + } + + private async encryptCredentials(auth: T, encryptionContext: EncryptionContext) { const { credentials: { username, password }, } = auth; @@ -261,7 +376,7 @@ export class DataSourceSavedObjectsClientWrapper { ...auth, credentials: { username, - password: await this.cryptographyClient.encryptAndEncode(password), + password: await this.cryptography.encryptAndEncode(password, encryptionContext), }, }; } diff --git a/src/plugins/data_source/server/types.ts b/src/plugins/data_source/server/types.ts index 2f20363b4b2d..b5cc61a772bd 100644 --- a/src/plugins/data_source/server/types.ts +++ b/src/plugins/data_source/server/types.ts @@ -8,7 +8,8 @@ import { OpenSearchClient, SavedObjectsClientContract, } from 'src/core/server'; -import { CryptographyClient } from './cryptography'; + +import { CryptographyServiceSetup } from './cryptography_service'; export interface LegacyClientCallAPIParams { endpoint: string; @@ -20,7 +21,7 @@ export interface DataSourceClientParams { dataSourceId: string; // this saved objects client is used to fetch data source on behalf of users, caller should pass scoped saved objects client savedObjects: SavedObjectsClientContract; - cryptographyClient: CryptographyClient; + cryptography: CryptographyServiceSetup; } export interface DataSourcePluginRequestContext {