Skip to content

Commit

Permalink
Remove keystore v1 and v2 formats (elastic#87893)
Browse files Browse the repository at this point in the history 
The keystore format has been changed a few times since it was first
introduced. Part of Elasticsearch startup automatically upgrades the
format. Since Elasticsearch has fixed bounds of supported versions for
upgrades, there are also fixed bounds on the keystore formats we might
need to read.

The v3 keystore format was introduced in Elasticsearch 6.3.0. Since
current Elasticsearch master branch is 8.x, and 8.x only supports
offline upgrades from 7.x, it is therefore impossible to need to read
v1 or v2 formats. This commit removes support for those formats.
  • Loading branch information
@rjernst
rjernst authored Jun 22, 2022
1 parent f627bdd commit fc09896
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 229 deletions.
94 changes: 0 additions & 94 deletions ...tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,10 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.MessageDigest;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.Locale;
import java.util.Set;
Expand Down Expand Up @@ -375,98 +373,6 @@ public void testIllegalSettingName() throws Exception {
assertTrue(e.getMessage().contains("does not match the allowed setting name pattern"));
}

public void testBackcompatV1() throws Exception {
assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
Path configDir = env.configFile();
try (
Directory directory = newFSDirectory(configDir);
IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT);
) {
CodecUtil.writeHeader(output, "elasticsearch.keystore", 1);
output.writeByte((byte) 0); // hasPassword = false
output.writeString("PKCS12");
output.writeString("PBE");

SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE");
KeyStore keystore = KeyStore.getInstance("PKCS12");
keystore.load(null, null);
SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);

ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
keystore.store(keystoreBytesStream, new char[0]);
byte[] keystoreBytes = keystoreBytesStream.toByteArray();
output.writeInt(keystoreBytes.length);
output.writeBytes(keystoreBytes, keystoreBytes.length);
CodecUtil.writeFooter(output);
}

KeyStoreWrapper keystore = KeyStoreWrapper.load(configDir);
keystore.decrypt(new char[0]);
SecureString testValue = keystore.getString("string_setting");
assertThat(testValue.toString(), equalTo("stringSecretValue"));
}

public void testBackcompatV2() throws Exception {
assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
Path configDir = env.configFile();
byte[] fileBytes = new byte[20];
random().nextBytes(fileBytes);
try (
Directory directory = newFSDirectory(configDir);
IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT);
) {
CodecUtil.writeHeader(output, "elasticsearch.keystore", KeyStoreWrapper.V2_VERSION);
output.writeByte((byte) 0); // hasPassword = false
output.writeString("PKCS12");
output.writeString("PBE"); // string algo
output.writeString("PBE"); // file algo

output.writeVInt(2); // num settings
output.writeString("string_setting");
output.writeString("STRING");
output.writeString("file_setting");
output.writeString("FILE");

SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE");
KeyStore keystore = KeyStore.getInstance("PKCS12");
keystore.load(null, null);
SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);

byte[] base64Bytes = Base64.getEncoder().encode(fileBytes);
char[] chars = new char[base64Bytes.length];
for (int i = 0; i < chars.length; ++i) {
chars[i] = (char) base64Bytes[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
}
secretKey = secretFactory.generateSecret(new PBEKeySpec(chars));
keystore.setEntry("file_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);

ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
keystore.store(keystoreBytesStream, new char[0]);
byte[] keystoreBytes = keystoreBytesStream.toByteArray();
output.writeInt(keystoreBytes.length);
output.writeBytes(keystoreBytes, keystoreBytes.length);
CodecUtil.writeFooter(output);
}

KeyStoreWrapper keystore = KeyStoreWrapper.load(configDir);
keystore.decrypt(new char[0]);
SecureString testValue = keystore.getString("string_setting");
assertThat(testValue.toString(), equalTo("stringSecretValue"));

try (InputStream fileInput = keystore.getFile("file_setting")) {
byte[] readBytes = new byte[20];
assertEquals(20, fileInput.read(readBytes));
for (int i = 0; i < fileBytes.length; ++i) {
assertThat("byte " + i, readBytes[i], equalTo(fileBytes[i]));
}
assertEquals(-1, fileInput.read());
}
}

public void testBackcompatV4() throws Exception {
assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
Path configDir = env.configFile();
Expand Down
142 changes: 7 additions & 135 deletions server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,9 @@
import java.nio.file.attribute.PosixFileAttributeView;
import java.nio.file.attribute.PosixFilePermissions;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Base64;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -109,7 +105,7 @@ private static class Entry {
public static final String KEYSTORE_FILENAME = "elasticsearch.keystore";

/** The oldest metadata format version that can be read. */
private static final int MIN_FORMAT_VERSION = 1;
private static final int MIN_FORMAT_VERSION = 3;
/** Legacy versions of the metadata written before the keystore data. */
public static final int V2_VERSION = 2;
public static final int V3_VERSION = 3;
Expand Down Expand Up @@ -268,59 +264,15 @@ public static KeyStoreWrapper load(Path configDir) throws IOException {
throw new IllegalStateException("hasPassword boolean is corrupt: " + String.format(Locale.ROOT, "%02x", hasPasswordByte));
}

if (formatVersion <= V2_VERSION) {
String type = input.readString();
if (type.equals("PKCS12") == false) {
throw new IllegalStateException("Corrupted legacy keystore string encryption algorithm");
}

final String stringKeyAlgo = input.readString();
if (stringKeyAlgo.equals("PBE") == false) {
throw new IllegalStateException("Corrupted legacy keystore string encryption algorithm");
}
if (formatVersion == V2_VERSION) {
final String fileKeyAlgo = input.readString();
if (fileKeyAlgo.equals("PBE") == false) {
throw new IllegalStateException("Corrupted legacy keystore file encryption algorithm");
}
}
}

final byte[] dataBytes;
if (formatVersion == V2_VERSION) {
// For v2 we had a map of strings containing the types for each setting. In v3 this map is now
// part of the encrypted bytes. Unfortunately we cannot seek backwards with checksum input, so
// we cannot just read the map and find out how long it is. So instead we read the map and
// store it back using java's builtin DataOutput in a byte array, along with the actual keystore bytes
Map<String, String> settingTypes = input.readMapOfStrings();
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try (DataOutputStream output = new DataOutputStream(bytes)) {
output.writeInt(settingTypes.size());
for (Map.Entry<String, String> entry : settingTypes.entrySet()) {
output.writeUTF(entry.getKey());
output.writeUTF(entry.getValue());
}
final int keystoreLen;
if (formatVersion < LE_VERSION) {
keystoreLen = Integer.reverseBytes(input.readInt());
} else {
keystoreLen = input.readInt();
}
byte[] keystoreBytes = new byte[keystoreLen];
input.readBytes(keystoreBytes, 0, keystoreLen);
output.write(keystoreBytes);
}
dataBytes = bytes.toByteArray();
int dataBytesLen;
if (formatVersion < LE_VERSION) {
dataBytesLen = Integer.reverseBytes(input.readInt());
} else {
int dataBytesLen;
if (formatVersion < LE_VERSION) {
dataBytesLen = Integer.reverseBytes(input.readInt());
} else {
dataBytesLen = input.readInt();
}
dataBytes = new byte[dataBytesLen];
input.readBytes(dataBytes, 0, dataBytesLen);
dataBytesLen = input.readInt();
}
dataBytes = new byte[dataBytesLen];
input.readBytes(dataBytes, 0, dataBytesLen);

CodecUtil.checkFooter(input);
return new KeyStoreWrapper(formatVersion, hasPassword, dataBytes);
Expand Down Expand Up @@ -378,13 +330,6 @@ public void decrypt(char[] password) throws GeneralSecurityException, IOExceptio
if (entries.get() != null) {
throw new IllegalStateException("Keystore has already been decrypted");
}
if (formatVersion <= V2_VERSION) {
decryptLegacyEntries();
if (password.length != 0) {
throw new IllegalArgumentException("Keystore format does not accept non-empty passwords");
}
return;
}

final byte[] salt;
final byte[] iv;
Expand Down Expand Up @@ -462,79 +407,6 @@ private byte[] encrypt(char[] password, byte[] salt, byte[] iv) throws GeneralSe
return bytes.toByteArray();
}

private void decryptLegacyEntries() throws GeneralSecurityException, IOException {
// v1 and v2 keystores never had passwords actually used, so we always use an empty password
KeyStore keystore = KeyStore.getInstance("PKCS12");
Map<String, EntryType> settingTypes = new HashMap<>();
ByteArrayInputStream inputBytes = new ByteArrayInputStream(dataBytes);
try (DataInputStream input = new DataInputStream(inputBytes)) {
// first read the setting types map
if (formatVersion == V2_VERSION) {
int numSettings = input.readInt();
for (int i = 0; i < numSettings; ++i) {
String key = input.readUTF();
String value = input.readUTF();
settingTypes.put(key, EntryType.valueOf(value));
}
}
// then read the actual keystore
keystore.load(input, "".toCharArray());
}

// verify the settings metadata matches the keystore entries
Enumeration<String> aliases = keystore.aliases();
if (formatVersion == MIN_FORMAT_VERSION) {
while (aliases.hasMoreElements()) {
settingTypes.put(aliases.nextElement(), EntryType.STRING);
}
} else {
// verify integrity: keys in keystore match what the metadata thinks exist
Set<String> expectedSettings = new HashSet<>(settingTypes.keySet());
while (aliases.hasMoreElements()) {
String settingName = aliases.nextElement();
if (expectedSettings.remove(settingName) == false) {
throw new SecurityException("Keystore has been corrupted or tampered with");
}
}
if (expectedSettings.isEmpty() == false) {
throw new SecurityException("Keystore has been corrupted or tampered with");
}
}

// fill in the entries now that we know all the types to expect
this.entries.set(new HashMap<>());
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("PBE");
KeyStore.PasswordProtection password = new KeyStore.PasswordProtection("".toCharArray());

for (Map.Entry<String, EntryType> settingEntry : settingTypes.entrySet()) {
String setting = settingEntry.getKey();
EntryType settingType = settingEntry.getValue();
KeyStore.SecretKeyEntry keystoreEntry = (KeyStore.SecretKeyEntry) keystore.getEntry(setting, password);
PBEKeySpec keySpec = (PBEKeySpec) keyFactory.getKeySpec(keystoreEntry.getSecretKey(), PBEKeySpec.class);
char[] chars = keySpec.getPassword();
keySpec.clearPassword();

final byte[] bytes;
if (settingType == EntryType.STRING) {
ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(CharBuffer.wrap(chars));
bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
Arrays.fill(byteBuffer.array(), (byte) 0);
} else {
assert settingType == EntryType.FILE;
// The PBE keyspec gives us chars, we convert to bytes
byte[] tmpBytes = new byte[chars.length];
for (int i = 0; i < tmpBytes.length; ++i) {
tmpBytes[i] = (byte) chars[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
}
bytes = Base64.getDecoder().decode(tmpBytes);
Arrays.fill(tmpBytes, (byte) 0);
}
Arrays.fill(chars, '\0');

entries.get().put(setting, new Entry(bytes));
}
}

/** Write the keystore to the given config directory. */
public synchronized void save(Path configDir, char[] password) throws Exception {
save(configDir, password, true);
Expand Down

0 comments on commit fc09896

Please sign in to comment.