Add Crypto Handler abstractions for encryption/decryption and Crypto …

…Key Provider Plugin to provide keys (opensearch-project#8466)

Signed-off-by: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com>
Signed-off-by: Gaurav Bafna <gbbafna@amazon.com>

CHANGELOG.md

@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Rate limiter integration for remote store uploads and downloads([#9448](https://github.com/opensearch-project/OpenSearch/pull/9448/))
 - Add support for extensions to search responses using SearchExtBuilder ([#9379](https://github.com/opensearch-project/OpenSearch/pull/9379))
 - [Remote State] Create service to publish cluster state to remote store ([#9160](https://github.com/opensearch-project/OpenSearch/pull/9160))
+- Core crypto library to perform encryption and decryption of source content ([#8466](https://github.com/opensearch-project/OpenSearch/pull/8466))
 
 ### Dependencies
 - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307))

buildSrc/version.properties

@@ -35,7 +35,7 @@ httpcore          = 4.4.16
 httpasyncclient   = 4.1.5
 commonslogging    = 1.2
 commonscodec      = 1.15
-
+commonslang       = 3.13.0
 # plugin dependencies
 aws               = 2.20.55
 reactivestreams   = 1.0.4

libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java

@@ -0,0 +1,114 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.crypto;
+
+import org.opensearch.common.io.InputStreamContainer;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Crypto provider abstractions for encryption and decryption of data. Allows registering multiple providers
+ * for defining different ways of encrypting or decrypting data.
+ *
+ * T - Encryption Metadata / CryptoContext
+ * U - Parsed Encryption Metadata / CryptoContext
+ */
+public interface CryptoHandler<T, U> {
+
+    /**
+     * To initialise or create a new crypto metadata to be used in encryption. This is needed to set the context before
+     * beginning encryption.
+     *
+     * @return crypto metadata instance
+     */
+    T initEncryptionMetadata();
+
+    /**
+     * To load crypto metadata to be used in encryption from content header.
+     * Note that underlying information in the loaded metadata object is same as present in the object created during
+     * encryption but object type may differ.
+     *
+     * @return crypto metadata instance used in decryption.
+     */
+    U loadEncryptionMetadata(EncryptedHeaderContentSupplier encryptedHeaderContentSupplier) throws IOException;
+
+    /**
+     * Few encryption algorithms have certain conditions on the unit of content to be encrypted. This requires the
+     * content size to be re adjusted in order to fulfil these conditions for partial writes. If write requests for
+     * encryption of a part of content do not fulfil these conditions then encryption fails or can result in corrupted
+     * content depending on the algorithm used. This method exposes a means to re-adjust sizes of such writes.
+     *
+     * @param cryptoContext crypto metadata instance
+     * @param contentSize Size of the raw content
+     * @return Adjusted size of the content.
+     */
+    long adjustContentSizeForPartialEncryption(T cryptoContext, long contentSize);
+
+    /**
+     * Estimate length of the encrypted content. It should only be used to determine length of entire content after
+     * encryption.
+     *
+     * @param cryptoContext crypto metadata instance consisting of encryption metadata used in encryption.
+     * @param contentLength Size of the raw content
+     * @return Calculated size of the encrypted content.
+     */
+    long estimateEncryptedLengthOfEntireContent(T cryptoContext, long contentLength);
+
+    /**
+     * For given encrypted content length, estimate the length of the decrypted content.
+     * @param cryptoContext crypto metadata instance consisting of encryption metadata used in encryption.
+     * @param contentLength Size of the encrypted content
+     * @return Calculated size of the decrypted content.
+     */
+    long estimateDecryptedLength(U cryptoContext, long contentLength);
+
+    /**
+     * Wraps a raw InputStream with encrypting stream
+     *
+     * @param encryptionMetadata created earlier to set the crypto metadata.
+     * @param stream Raw InputStream to encrypt
+     * @return encrypting stream wrapped around raw InputStream.
+     */
+    InputStreamContainer createEncryptingStream(T encryptionMetadata, InputStreamContainer stream);
+
+    /**
+     * Provides encrypted stream for a raw stream emitted for a part of content.
+     *
+     * @param cryptoContext crypto metadata instance.
+     * @param stream raw stream for which encrypted stream has to be created.
+     * @param totalStreams Number of streams being used for the entire content.
+     * @param streamIdx Index of the current stream.
+     * @return Encrypted stream for the provided raw stream.
+     */
+    InputStreamContainer createEncryptingStreamOfPart(T cryptoContext, InputStreamContainer stream, int totalStreams, int streamIdx);
+
+    /**
+     * This method accepts an encrypted stream and provides a decrypting wrapper.
+     * @param encryptingStream to be decrypted.
+     * @return Decrypting wrapper stream
+     */
+    InputStream createDecryptingStream(InputStream encryptingStream);
+
+    /**
+     * This method creates a {@link DecryptedRangedStreamProvider} which provides a wrapped stream to decrypt the
+     * underlying stream. This also provides adjusted range against the actual range which should be used for fetching
+     * and supplying the encrypted content for decryption. Extra content outside the range is trimmed down and returned
+     * by the decrypted stream.
+     * For partial reads of encrypted content, few algorithms require the range of content to be adjusted for
+     * successful decryption. Adjusted range may or may not be same as the provided range. If range is adjusted then
+     * starting offset of resultant range can be lesser than the starting offset of provided range and end
+     * offset can be greater than the ending offset of the provided range.
+     *
+     * @param cryptoContext crypto metadata instance.
+     * @param startPosOfRawContent starting position in the raw/decrypted content
+     * @param endPosOfRawContent ending position in the raw/decrypted content
+     */
+    DecryptedRangedStreamProvider createDecryptingStreamOfRange(U cryptoContext, long startPosOfRawContent, long endPosOfRawContent);
+}

libs/common/src/main/java/org/opensearch/common/crypto/DataKeyPair.java

@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.common.crypto;
+
+/**
+ * Key pair generated by {@link MasterKeyProvider}
+ */
+public class DataKeyPair {
+    private final byte[] rawKey;
+    private final byte[] encryptedKey;
+
+    /**
+     * Constructor to initialize key-pair values
+     * @param rawKey Unencrypted data key used for encryption and decryption
+     * @param encryptedKey Encrypted version of rawKey
+     */
+    public DataKeyPair(byte[] rawKey, byte[] encryptedKey) {
+        this.rawKey = rawKey;
+        this.encryptedKey = encryptedKey;
+    }
+
+    /**
+     * Returns raw key
+     * @return raw/decrypted key
+     */
+    public byte[] getRawKey() {
+        return rawKey;
+    }
+
+    /**
+     * Returns encrypted key
+     * @return encrypted key
+     */
+    public byte[] getEncryptedKey() {
+        return encryptedKey;
+    }
+}

libs/common/src/main/java/org/opensearch/common/crypto/DecryptedRangedStreamProvider.java

@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.crypto;
+
+import java.io.InputStream;
+import java.util.function.UnaryOperator;
+
+/**
+ * Contains adjusted range of partial encrypted content which needs to be used for decryption.
+ */
+public class DecryptedRangedStreamProvider {
+
+    private final long[] adjustedRange;
+    private final UnaryOperator<InputStream> decryptedStreamProvider;
+
+    /**
+     * To construct adjusted encrypted range.
+     * @param adjustedRange range of partial encrypted content which needs to be used for decryption.
+     * @param decryptedStreamProvider stream provider for decryption and range re-adjustment.
+     */
+    public DecryptedRangedStreamProvider(long[] adjustedRange, UnaryOperator<InputStream> decryptedStreamProvider) {
+        this.adjustedRange = adjustedRange;
+        this.decryptedStreamProvider = decryptedStreamProvider;
+    }
+
+    /**
+     * Adjusted range of partial encrypted content which needs to be used for decryption.
+     * @return adjusted range
+     */
+    public long[] getAdjustedRange() {
+        return adjustedRange;
+    }
+
+    /**
+     * A utility stream provider which supplies the stream responsible for decrypting the content and reading the
+     * desired range of decrypted content by skipping extra content which got decrypted as a result of range adjustment.
+     * @return stream provider for decryption and supplying the desired range of content.
+     */
+    public UnaryOperator<InputStream> getDecryptedStreamProvider() {
+        return decryptedStreamProvider;
+    }
+
+}

libs/common/src/main/java/org/opensearch/common/crypto/EncryptedHeaderContentSupplier.java

@@ -0,0 +1,25 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.common.crypto;
+
+import java.io.IOException;
+
+/**
+ * This is used in partial decryption. Header information is required for decryption of actual encrypted content.
+ * Implementation of this supplier only requires first few bytes of encrypted content to be supplied.
+ */
+public interface EncryptedHeaderContentSupplier {
+
+    /**
+     * @param start Start position of the encrypted content (Generally supplied as 0 during usage)
+     * @param end End position of the header.
+     * @return Encrypted header content (May contain additional content which is later discarded)
+     * @throws IOException In case content fetch fails.
+     */
+    byte[] supply(long start, long end) throws IOException;
+}

libs/common/src/main/java/org/opensearch/common/crypto/MasterKeyProvider.java

@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.common.crypto;
+
+import java.io.Closeable;
+import java.util.Map;
+
+/**
+ * Master key provider responsible for management of master keys.
+ */
+public interface MasterKeyProvider extends Closeable {
+
+    /**
+     * Returns data key pair
+     * @return data key pair generated by master key.
+     */
+    DataKeyPair generateDataPair();
+
+    /**
+     * Returns decrpted key against the encrypted key.
+     * @param encryptedKey Key to decrypt
+     * @return Decrypted version of key.
+     */
+    byte[] decryptKey(byte[] encryptedKey);
+
+    /**
+     * Returns key id.
+     * @return key id
+     */
+    String getKeyId();
+
+    /**
+     * @return encryption context associated with this master key.
+     */
+    Map<String, String> getEncryptionContext();
+}

libs/common/src/main/java/org/opensearch/common/crypto/package-info.java

@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Common crypto utilities used across opensearch. */
+package org.opensearch.common.crypto;

libs/encryption-sdk/build.gradle

@@ -0,0 +1,52 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+apply plugin: 'opensearch.build'
+apply plugin: 'opensearch.publish'
+
+forbiddenApis.ignoreFailures = false
+
+thirdPartyAudit.enabled = false
+forbiddenApisTest.ignoreFailures = true
+testingConventions.enabled = false
+
+dependencies {
+  // Common crypto classes
+  api project(':libs:opensearch-common')
+
+  // Logger
+  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
+  implementation 'commons-logging:commons-logging:1.2'
+
+  // Encryption
+  implementation "com.amazonaws:aws-encryption-sdk-java:2.4.0"
+  implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
+  implementation "org.apache.commons:commons-lang3:${versions.commonslang}"
+
+  //Tests
+  testImplementation "junit:junit:${versions.junit}"
+  testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.opensearch', module: 'opensearch-encryption-sdk'
+  }
+
+  compileOnly 'com.google.code.findbugs:annotations:3.0.1'
+}
+
+tasks.named('forbiddenApisMain').configure {
+  // Only enable limited check because AD code has too many violations.
+  replaceSignatureFiles 'jdk-signatures'
+  signaturesFiles += files('src/forbidden/crypto-signatures.txt')
+}
+
+// Encryption SDK files have missing java docs so disabling for the lib.
+tasks.named('missingJavadoc').configure {
+  enabled = false
+}
+
+forbiddenApisTest.setSignaturesFiles(files('src/forbidden/crypto-test-signatures.txt'))

libs/encryption-sdk/licenses/aws-encryption-sdk-java-2.4.0.jar.sha1

@@ -0,0 +1 @@
+98943eda1dc05bb01f4f5405e115b08dc541afbf