Skip to content

Commit a630b31

Browse files
K0K0V0KK0K0V0K
authored
HADOOP-19639. SecretManager configuration at runtime (apache#7827) Contributed by Bence Kosztolnik.
* HADOOP-19639. SecretManager configuration at runtime - static configuration of SecretManager is required because it has some static method what use the selected algorithm - in case if class path not contains the config values (for example TEZ DAGAppMaster run) the default values will be loaded at runtime - the default values can cause failers in modern environments (they are not FIPS compliant) - new SecretManagerConfig created to be able to modify the SecretManager config without core-site.xml present on class path Signed-off-by: Shilun Fan <slfan1989@apache.org>
1 parent f79d6ad commit a630b31

File tree

2 files changed

+210
-50
lines changed

2 files changed

+210
-50
lines changed

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/SecretManager.java

Lines changed: 101 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,61 @@
4747
public abstract class SecretManager<T extends TokenIdentifier> {
4848

4949
public static final Logger LOG = LoggerFactory.getLogger(SecretManager.class);
50+
51+
private static String selectedAlgorithm;
52+
private static int selectedLength;
53+
54+
/**
55+
* Key generator to use.
56+
*/
57+
private static boolean keygenInitialized;
58+
private final Object keyGenLock = new Object();
59+
private volatile KeyGenerator keyGen;
60+
61+
/**
62+
* A thread local store for the Macs.
63+
*/
64+
private static boolean macInitialized;
65+
private static final ThreadLocal<Mac> threadLocalMac =
66+
ThreadLocal.withInitial(SecretManager::createMac);
67+
68+
private static boolean secretKeyInitialized;
69+
70+
static {
71+
update(new Configuration());
72+
}
73+
74+
private static final String UPDATE_LOG_TEMPLATE =
75+
"{} was already initialized with older config, those will not be updated." +
76+
"Hint: If you turn on debug log you can see when it is happening. Thread: {}";
77+
/**
78+
* Updates the selected cryptographic algorithm and key length using the provided
79+
* Hadoop {@link Configuration}. This method reads the values for
80+
* {@code HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY} and
81+
* {@code HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY}, or uses default values if not set.
82+
*
83+
* @param conf the configuration object containing cryptographic settings
84+
*/
85+
public static synchronized void update(Configuration conf) {
86+
if (keygenInitialized) {
87+
LOG.warn(UPDATE_LOG_TEMPLATE, "KeyGenerator", Thread.currentThread());
88+
}
89+
if (macInitialized) {
90+
LOG.warn(UPDATE_LOG_TEMPLATE, "Mac", Thread.currentThread());
91+
}
92+
if (secretKeyInitialized) {
93+
LOG.warn(UPDATE_LOG_TEMPLATE, "SecretKey", Thread.currentThread());
94+
}
95+
selectedAlgorithm = conf.get(
96+
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY,
97+
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT);
98+
LOG.debug("Selected hash algorithm: {}", selectedAlgorithm);
99+
selectedLength = conf.getInt(
100+
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY,
101+
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_DEFAULT);
102+
LOG.debug("Selected hash key length: {}", selectedLength);
103+
}
104+
50105
/**
51106
* The token was invalid and the message explains why.
52107
*/
@@ -115,61 +170,17 @@ public void checkAvailableForRead() throws StandbyException {
115170
// Default to being available for read.
116171
}
117172

118-
private static final String SELECTED_ALGORITHM;
119-
private static final int SELECTED_LENGTH;
120-
121-
static {
122-
Configuration conf = new Configuration();
123-
String algorithm = conf.get(
124-
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY,
125-
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT);
126-
LOG.debug("Selected hash algorithm: {}", algorithm);
127-
SELECTED_ALGORITHM = algorithm;
128-
int length = conf.getInt(
129-
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY,
130-
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_DEFAULT);
131-
LOG.debug("Selected hash key length:{}", length);
132-
SELECTED_LENGTH = length;
133-
}
134-
135-
/**
136-
* A thread local store for the Macs.
137-
*/
138-
private static final ThreadLocal<Mac> threadLocalMac =
139-
new ThreadLocal<Mac>(){
140-
@Override
141-
protected Mac initialValue() {
142-
try {
143-
return Mac.getInstance(SELECTED_ALGORITHM);
144-
} catch (NoSuchAlgorithmException nsa) {
145-
throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM, nsa);
146-
}
147-
}
148-
};
149-
150-
/**
151-
* Key generator to use.
152-
*/
153-
private final KeyGenerator keyGen;
154-
{
155-
try {
156-
keyGen = KeyGenerator.getInstance(SELECTED_ALGORITHM);
157-
keyGen.init(SELECTED_LENGTH);
158-
} catch (NoSuchAlgorithmException nsa) {
159-
throw new IllegalArgumentException("Can't find " + SELECTED_ALGORITHM, nsa);
160-
}
161-
}
162-
163173
/**
164174
* Generate a new random secret key.
165175
* @return the new key
166176
*/
167177
protected SecretKey generateSecret() {
168-
SecretKey key;
169-
synchronized (keyGen) {
170-
key = keyGen.generateKey();
178+
synchronized (keyGenLock) {
179+
if (keyGen == null) {
180+
keyGen = createKeyGenerator();
181+
}
182+
return keyGen.generateKey();
171183
}
172-
return key;
173184
}
174185

175186
/**
@@ -197,6 +208,46 @@ public static byte[] createPassword(byte[] identifier,
197208
* @return the secret key
198209
*/
199210
protected static SecretKey createSecretKey(byte[] key) {
200-
return new SecretKeySpec(key, SELECTED_ALGORITHM);
211+
LOG.debug("Creating secretKey with algorithm {} with thread {}",
212+
selectedAlgorithm, Thread.currentThread());
213+
secretKeyInitialized = true;
214+
return new SecretKeySpec(key, selectedAlgorithm);
215+
}
216+
217+
/**
218+
* Creates a new {@link KeyGenerator} instance configured with the currently selected
219+
* algorithm and key length.
220+
*
221+
* @return a new {@code KeyGenerator} instance
222+
* @throws IllegalArgumentException if the specified algorithm is not available
223+
*/
224+
private static synchronized KeyGenerator createKeyGenerator() {
225+
LOG.debug("Creating key generator instance {} - {} bit with thread {}",
226+
selectedAlgorithm, selectedLength, Thread.currentThread());
227+
try {
228+
KeyGenerator keyGen = KeyGenerator.getInstance(selectedAlgorithm);
229+
keyGen.init(selectedLength);
230+
keygenInitialized = true;
231+
return keyGen;
232+
} catch (NoSuchAlgorithmException nsa) {
233+
throw new IllegalArgumentException("Can't find " + selectedAlgorithm, nsa);
234+
}
235+
}
236+
237+
/**
238+
* Creates a new {@link Mac} instance using the currently selected algorithm.
239+
*
240+
* @return a new {@code Mac} instance
241+
* @throws IllegalArgumentException if the specified algorithm is not available
242+
*/
243+
private static synchronized Mac createMac() {
244+
LOG.debug("Creating mac instance {} with thread {}", selectedAlgorithm, Thread.currentThread());
245+
try {
246+
Mac mac = Mac.getInstance(selectedAlgorithm);
247+
macInitialized = true;
248+
return mac;
249+
} catch (NoSuchAlgorithmException nsa) {
250+
throw new IllegalArgumentException("Can't find " + selectedAlgorithm, nsa);
251+
}
201252
}
202253
}

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestSecretManager.java

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.security.token;
19+
20+
import org.apache.hadoop.conf.Configuration;
21+
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
22+
23+
import org.junit.jupiter.api.AfterEach;
24+
import org.junit.jupiter.api.BeforeEach;
25+
import org.junit.jupiter.api.Test;
26+
27+
import javax.crypto.SecretKey;
28+
29+
import static org.junit.jupiter.api.Assertions.assertEquals;
30+
import static org.junit.jupiter.api.Assertions.assertThrows;
31+
32+
public class TestSecretManager {
33+
34+
private final String defaultAlgorithm =
35+
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_DEFAULT;
36+
private final int defaultLength =
37+
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_DEFAULT;
38+
private final String strongAlgorithm = "HmacSHA256";
39+
private final int strongLength = 256;
40+
private SecretManager<TokenIdentifier> secretManager;
41+
42+
@Test
43+
public void testDefaults() {
44+
assertKey(secretManager.generateSecret(), defaultAlgorithm, defaultLength);
45+
}
46+
47+
@Test
48+
public void testUpdate() {
49+
SecretManager.update(createConfiguration(strongAlgorithm, strongLength));
50+
assertKey(secretManager.generateSecret(), strongAlgorithm, strongLength);
51+
}
52+
53+
@Test
54+
public void testUnknownAlgorithm() {
55+
SecretManager.update(createConfiguration("testUnknownAlgorithm_NO_ALG", strongLength));
56+
assertThrows(IllegalArgumentException.class, secretManager::generateSecret);
57+
}
58+
59+
@Test
60+
public void testUpdateAfterInitialisation() {
61+
SecretKey oldSecretKey = secretManager.generateSecret();
62+
SecretManager.update(createConfiguration(strongAlgorithm, strongLength));
63+
SecretKey newSecretKey = secretManager.generateSecret();
64+
assertKey(oldSecretKey, defaultAlgorithm, defaultLength);
65+
assertKey(newSecretKey, defaultAlgorithm, defaultLength);
66+
}
67+
68+
@BeforeEach
69+
public void setUp() {
70+
secretManager = new SecretManager<TokenIdentifier>() {
71+
@Override
72+
protected byte[] createPassword(TokenIdentifier identifier) {
73+
return new byte[0];
74+
}
75+
76+
@Override
77+
public byte[] retrievePassword(TokenIdentifier identifier) throws InvalidToken {
78+
return new byte[0];
79+
}
80+
81+
@Override
82+
public TokenIdentifier createIdentifier() {
83+
return null;
84+
}
85+
};
86+
}
87+
88+
@AfterEach
89+
public void tearDown() {
90+
SecretManager.update(createConfiguration(defaultAlgorithm, defaultLength));
91+
}
92+
93+
private void assertKey(SecretKey secretKey, String algorithm, int length) {
94+
assertEquals(algorithm, secretKey.getAlgorithm(),
95+
"Algorithm of created key is not as expected.");
96+
assertEquals(length, secretKey.getEncoded().length * 8,
97+
"Length of created key is not as expected.");
98+
}
99+
100+
private Configuration createConfiguration(String algorithm, int length) {
101+
Configuration conf = new Configuration();
102+
conf.set(
103+
CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_GENERATOR_ALGORITHM_KEY,
104+
algorithm);
105+
conf.setInt(CommonConfigurationKeysPublic.HADOOP_SECURITY_SECRET_MANAGER_KEY_LENGTH_KEY,
106+
length);
107+
return conf;
108+
}
109+
}

0 commit comments

Comments
 (0)