Reload secure settings with password

build.gradle

@@ -160,8 +160,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/43197"
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java

@@ -19,22 +19,95 @@
 
 package org.elasticsearch.action.admin.cluster.node.reload;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
+import org.elasticsearch.common.CharArrays;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.SecureString;
+
+import java.io.IOException;
+import java.util.Arrays;
 
 /**
- * Request for a reload secure settings action.
+ * Request for a reload secure settings action
  */
 public class NodesReloadSecureSettingsRequest extends BaseNodesRequest<NodesReloadSecureSettingsRequest> {
 
+    /**
+     * The password is used to re-read and decrypt the contents
+     * of the node's keystore (backing the implementation of
+     * {@code SecureSettings}).
+     */
+    @Nullable
+    private SecureString secureSettingsPassword;
+
     public NodesReloadSecureSettingsRequest() {
     }
 
     /**
-     * Reload secure settings only on certain nodes, based on the nodes IDs specified. If none are passed, secure settings will be reloaded
-     * on all the nodes.
+     * Reload secure settings only on certain nodes, based on the nodes ids
+     * specified. If none are passed, secure settings will be reloaded on all the
+     * nodes.
      */
-    public NodesReloadSecureSettingsRequest(final String... nodesIds) {
+    public NodesReloadSecureSettingsRequest(String... nodesIds) {
         super(nodesIds);
     }
 
+    @Nullable
+    public SecureString getSecureSettingsPassword() {
+        return secureSettingsPassword;
+    }
+
+    public void setSecureStorePassword(SecureString secureStorePassword) {
+        this.secureSettingsPassword = secureStorePassword;
+    }
+
+    public void closePassword() {
+        if (this.secureSettingsPassword != null) {
+            this.secureSettingsPassword.close();
+        }
+    }
+
+    boolean hasPassword() {
+        return this.secureSettingsPassword != null && this.secureSettingsPassword.length() > 0;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        if (in.getVersion().onOrAfter(Version.V_7_4_0)) {
+            final BytesReference bytesRef = in.readOptionalBytesReference();
+            if (bytesRef != null) {
+                byte[] bytes = BytesReference.toBytes(bytesRef);
+                try {
+                    this.secureSettingsPassword = new SecureString(CharArrays.utf8BytesToChars(bytes));
+                } finally {
+                    Arrays.fill(bytes, (byte) 0);
+                }
+            } else {
+                this.secureSettingsPassword = null;
+            }
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_7_4_0)) {
+            if (this.secureSettingsPassword == null) {
+                out.writeOptionalBytesReference(null);
+            } else {
+                final byte[] passwordBytes = CharArrays.toUtf8Bytes(this.secureSettingsPassword.getChars());
+                try {
+                    out.writeOptionalBytesReference(new BytesArray(passwordBytes));
+                } finally {
+                    Arrays.fill(passwordBytes, (byte) 0);
+                }
+            }
+        }
+    }
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequestBuilder.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.SecureString;
 
 /**
  * Builder for the reload secure settings nodes request
@@ -32,4 +33,9 @@ public NodesReloadSecureSettingsRequestBuilder(ElasticsearchClient client, Nodes
         super(client, action, new NodesReloadSecureSettingsRequest());
     }
 
+    public NodesReloadSecureSettingsRequestBuilder setSecureStorePassword(SecureString secureStorePassword) {
+        request.setSecureStorePassword(secureStorePassword);
+        return this;
+    }
+
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java

@@ -21,16 +21,20 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.PluginsService;
@@ -78,15 +82,39 @@ protected NodesReloadSecureSettingsResponse.NodeResponse newNodeResponse() {
         return new NodesReloadSecureSettingsResponse.NodeResponse();
     }
 
+    @Override
+    protected void doExecute(Task task, NodesReloadSecureSettingsRequest request,
+                             ActionListener<NodesReloadSecureSettingsResponse> listener) {
+        if (request.hasPassword() && isNodeLocal(request) == false && isNodeTransportTLSEnabled() == false) {
+            request.closePassword();
+            listener.onFailure(
+                new ElasticsearchException("Secure settings cannot be updated cluster wide when TLS for the transport layer" +
+                " is not enabled. Enable TLS or use the API with a `_local` filter on each node."));
+        } else {
+            super.doExecute(task, request, ActionListener.wrap(response -> {
+                request.closePassword();
+                listener.onResponse(response);
+            }, e -> {
+                request.closePassword();
+                listener.onFailure(e);
+            }));
+        }
+    }
+
     @Override
     protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation(NodeRequest nodeReloadRequest, Task task) {
+        final NodesReloadSecureSettingsRequest request = nodeReloadRequest.request;
+        // We default to using an empty string as the keystore password so that we mimic pre 7.3 API behavior
+        final SecureString secureSettingsPassword = request.hasPassword() ? request.getSecureSettingsPassword() :
+            new SecureString(new char[0]);
         try (KeyStoreWrapper keystore = KeyStoreWrapper.load(environment.configFile())) {
             // reread keystore from config file
             if (keystore == null) {
                 return new NodesReloadSecureSettingsResponse.NodeResponse(clusterService.localNode(),
                         new IllegalStateException("Keystore is missing"));
             }
-            keystore.decrypt(new char[0]);
+            // decrypt the keystore using the password from the request
+            keystore.decrypt(secureSettingsPassword.getChars());
             // add the keystore to the original node settings object
             final Settings settingsWithKeystore = Settings.builder()
                     .put(environment.settings(), false)
@@ -107,6 +135,8 @@ protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation(NodeReque
             return new NodesReloadSecureSettingsResponse.NodeResponse(clusterService.localNode(), null);
         } catch (final Exception e) {
             return new NodesReloadSecureSettingsResponse.NodeResponse(clusterService.localNode(), e);
+        } finally {
+            secureSettingsPassword.close();
         }
     }
 
@@ -134,4 +164,20 @@ public void writeTo(StreamOutput out) throws IOException {
             request.writeTo(out);
         }
     }
+
+    /**
+     * Returns true if the node is configured for TLS on the transport layer
+     */
+    private boolean isNodeTransportTLSEnabled() {
+        return transportService.isTransportSecure();
+    }
+
+    private boolean isNodeLocal(NodesReloadSecureSettingsRequest request) {
+        if (null == request.concreteNodes()) {
+            resolveRequest(request, clusterService.state());
+            assert request.concreteNodes() != null;
+        }
+        final DiscoveryNode[] nodes = request.concreteNodes();
+        return nodes.length == 1 && nodes[0].getId().equals(clusterService.localNode().getId());
+    }
 }

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java

@@ -23,8 +23,11 @@
 import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsRequestBuilder;
 import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsResponse;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
@@ -41,6 +44,14 @@
 
 public final class RestReloadSecureSettingsAction extends BaseRestHandler {
 
+    static final ObjectParser<NodesReloadSecureSettingsRequest, String> PARSER =
+        new ObjectParser<>("reload_secure_settings", NodesReloadSecureSettingsRequest::new);
+
+    static {
+        PARSER.declareString((request, value) -> request.setSecureStorePassword(new SecureString(value.toCharArray())),
+            new ParseField("secure_settings_password"));
+    }
+
     public RestReloadSecureSettingsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_nodes/reload_secure_settings", this);
@@ -56,23 +67,28 @@ public String getName() {
     public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         final String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
         final NodesReloadSecureSettingsRequestBuilder nodesRequestBuilder = client.admin()
-                .cluster()
-                .prepareReloadSecureSettings()
-                .setTimeout(request.param("timeout"))
-                .setNodesIds(nodesIds);
-        final NodesReloadSecureSettingsRequest nodesRequest = nodesRequestBuilder.request();
+            .cluster()
+            .prepareReloadSecureSettings()
+            .setTimeout(request.param("timeout"))
+            .setNodesIds(nodesIds);
+        request.withContentOrSourceParamParserOrNull(parser -> {
+            if (parser != null) {
+                final NodesReloadSecureSettingsRequest nodesRequest = PARSER.parse(parser, null);
+                nodesRequestBuilder.setSecureStorePassword(nodesRequest.getSecureSettingsPassword());
+            }
+        });
+
         return channel -> nodesRequestBuilder
                 .execute(new RestBuilderListener<NodesReloadSecureSettingsResponse>(channel) {
                     @Override
                     public RestResponse buildResponse(NodesReloadSecureSettingsResponse response, XContentBuilder builder)
-                            throws Exception {
+                        throws Exception {
                         builder.startObject();
-                        {
-                            RestActions.buildNodesHeader(builder, channel.request(), response);
-                            builder.field("cluster_name", response.getClusterName().value());
-                            response.toXContent(builder, channel.request());
-                        }
+                        RestActions.buildNodesHeader(builder, channel.request(), response);
+                        builder.field("cluster_name", response.getClusterName().value());
+                        response.toXContent(builder, channel.request());
                         builder.endObject();
+                        nodesRequestBuilder.request().closePassword();
                         return new BytesRestResponse(RestStatus.OK, builder);
                     }
                 });

server/src/main/java/org/elasticsearch/transport/Transport.java

@@ -53,6 +53,10 @@ public interface Transport extends LifecycleComponent {
 
     void setMessageListener(TransportMessageListener listener);
 
+    default boolean isSecure() {
+        return false;
+    }
+
     /**
      * The address the transport is bound on.
      */

server/src/main/java/org/elasticsearch/transport/TransportService.java

@@ -301,6 +301,10 @@ public TransportStats stats() {
         return transport.getStats();
     }
 
+    public boolean isTransportSecure() {
+        return transport.isSecure();
+    }
+
     public BoundTransportAddress boundAddress() {
         return transport.boundAddress();
     }