Decentralize plugin security

* Add ability for plugins to declare additional permissions with a custom plugin-security.policy file and corresponding AccessController logic. See the plugin author's guide for more information.
* Add warning messages to users for extra plugin permissions in bin/plugin.
* When bin/plugin is run interactively (stdin is a controlling terminal and -b/--batch not supplied), require user confirmation.
* Improve unit test and IDE support for plugins with additional permissions by exposing plugin's metadata as a maven test resource.

Closes #14108

Squashed commit of the following:

commit cf8ace6
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 13:36:05 2015 -0400

    fix new unit test from master merge

commit 9be3c5a
Merge: 2f168b8 7368231
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 12:58:31 2015 -0400

    Merge branch 'master' into off_my_back

commit 2f168b8
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 12:56:04 2015 -0400

    improve plugin author documentation

commit 6e6c2bf
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 12:52:14 2015 -0400

    move security confirmation after 'plugin already installed' check, to prevent user from answering unnecessary questions.

commit 08233a2
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 05:36:42 2015 -0400

    Add documentation and pluginmanager support

commit 05dad86
Author: Robert Muir <rmuir@apache.org>
Date:   Wed Oct 14 02:22:24 2015 -0400

    Decentralize plugin permissions (modulo docs and pluginmanager work)

Author: rmuir

core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java

@@ -29,6 +29,7 @@
 import java.security.Policy;
 import java.security.ProtectionDomain;
 import java.security.URIParameter;
+import java.util.Map;
 
 /** custom policy for union of static and dynamic permissions */
 final class ESPolicy extends Policy {
@@ -41,13 +42,15 @@ final class ESPolicy extends Policy {
     final Policy template;
     final Policy untrusted;
     final PermissionCollection dynamic;
+    final Map<String,PermissionCollection> plugins;
 
-    public ESPolicy(PermissionCollection dynamic) throws Exception {
+    public ESPolicy(PermissionCollection dynamic, Map<String,PermissionCollection> plugins) throws Exception {
         URI policyUri = getClass().getResource(POLICY_RESOURCE).toURI();
         URI untrustedUri = getClass().getResource(UNTRUSTED_RESOURCE).toURI();
         this.template = Policy.getInstance("JavaPolicy", new URIParameter(policyUri));
         this.untrusted = Policy.getInstance("JavaPolicy", new URIParameter(untrustedUri));
         this.dynamic = dynamic;
+        this.plugins = plugins;
     }
 
     @Override @SuppressForbidden(reason = "fast equals check is desired")
@@ -66,6 +69,11 @@ public boolean implies(ProtectionDomain domain, Permission permission) {
             if (BootstrapInfo.UNTRUSTED_CODEBASE.equals(location.getFile())) {
                 return untrusted.implies(domain, permission);
             }
+            // check for an additional plugin permission
+            PermissionCollection plugin = plugins.get(location.getFile());
+            if (plugin != null && plugin.implies(permission)) {
+                return true;
+            }
         }
 
         // Special handling for broken AWS code which destroys all SSL security

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.PluginInfo;
 
 import java.io.*;
 import java.net.URL;
@@ -30,8 +31,11 @@
 import java.nio.file.Files;
 import java.nio.file.NotDirectoryException;
 import java.nio.file.Path;
+import java.security.NoSuchAlgorithmException;
+import java.security.PermissionCollection;
 import java.security.Permissions;
 import java.security.Policy;
+import java.security.URIParameter;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
@@ -65,7 +69,7 @@
  * when they are so dangerous that general code should not be granted the
  * permission, but there are extenuating circumstances.
  * <p>
- * Groovy scripts are assigned no permissions. This does not provide adequate
+ * Scripts (groovy, javascript, python) are assigned minimal permissions. This does not provide adequate
  * sandboxing, as these scripts still have access to ES classes, and could
  * modify members, etc that would cause bad things to happen later on their
  * behalf (no package protections are yet in place, this would need some
@@ -81,7 +85,7 @@
  * <h1>Debugging Security</h1>
  * A good place to start when there is a problem is to turn on security debugging:
  * <pre>
- * JAVA_OPTS="-Djava.security.debug=access:failure" bin/elasticsearch
+ * JAVA_OPTS="-Djava.security.debug=access,failure" bin/elasticsearch
  * </pre>
  * See <a href="https://docs.oracle.com/javase/7/docs/technotes/guides/security/troubleshooting-security.html">
  * Troubleshooting Security</a> for information.
@@ -97,11 +101,9 @@ private Security() {}
     static void configure(Environment environment) throws Exception {
         // set properties for jar locations
         setCodebaseProperties();
-        // set properties for problematic plugins
-        setPluginCodebaseProperties(environment);
 
-        // enable security policy: union of template and environment-based paths.
-        Policy.setPolicy(new ESPolicy(createPermissions(environment)));
+        // enable security policy: union of template and environment-based paths, and possibly plugin permissions
+        Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment)));
 
         // enable security manager
         System.setSecurityManager(new SecurityManager() {
@@ -157,70 +159,39 @@ static void setCodebaseProperties() {
         }
     }
 
-    // mapping of plugins to plugin class name. see getPluginClass why we need this.
-    // plugin codebase property is always implicit (es.security.plugin.foobar)
-    // note that this is only read once, when policy is parsed.
-    static final Map<String,String> SPECIAL_PLUGINS;
-    static {
-        Map<String,String> m = new HashMap<>();
-        m.put("repository-s3",       "org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin");
-        m.put("discovery-ec2",       "org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin");
-        m.put("discovery-gce",       "org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin");
-        m.put("lang-expression",     "org.elasticsearch.script.expression.ExpressionPlugin");
-        m.put("lang-groovy",         "org.elasticsearch.script.groovy.GroovyPlugin");
-        m.put("lang-javascript",     "org.elasticsearch.plugin.javascript.JavaScriptPlugin");
-        m.put("lang-python",         "org.elasticsearch.plugin.python.PythonPlugin");
-        SPECIAL_PLUGINS = Collections.unmodifiableMap(m);
-    }
-
-    /**
-     * Returns policy property for plugin, if it has special permissions.
-     * otherwise returns null.
-     */
-    static String getPluginProperty(String pluginName) {
-        if (SPECIAL_PLUGINS.containsKey(pluginName)) {
-            return "es.security.plugin." + pluginName;
-        } else {
-            return null;
-        }
-    }
-
-    /**
-     * Returns plugin class name, if it has special permissions.
-     * otherwise returns null.
-     */
-    // this is only here to support the intellij IDE
-    // it sucks to duplicate information, but its worth the tradeoff: sanity
-    // if it gets out of sync, tests will fail.
-    static String getPluginClass(String pluginName) {
-        return SPECIAL_PLUGINS.get(pluginName);
-    }
-
     /**
      * Sets properties (codebase URLs) for policy files.
      * we look for matching plugins and set URLs to fit
      */
     @SuppressForbidden(reason = "proper use of URL")
-    static void setPluginCodebaseProperties(Environment environment) throws IOException {
+    static Map<String,PermissionCollection> getPluginPermissions(Environment environment) throws IOException, NoSuchAlgorithmException {
+        Map<String,PermissionCollection> map = new HashMap<>();
         if (Files.exists(environment.pluginsFile())) {
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.pluginsFile())) {
                 for (Path plugin : stream) {
-                    String prop = getPluginProperty(plugin.getFileName().toString());
-                    if (prop != null) {
-                        if (System.getProperty(prop) != null) {
-                            throw new IllegalStateException("property: " + prop + " is unexpectedly set: " + System.getProperty(prop));
+                    Path policyFile = plugin.resolve(PluginInfo.ES_PLUGIN_POLICY);
+                    if (Files.exists(policyFile)) {
+                        // parse the plugin's policy file into a set of permissions
+                        Policy policy = Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toUri()));
+                        PermissionCollection permissions = policy.getPermissions(Security.class.getProtectionDomain());
+                        // this method is supported with the specific implementation we use, but just check for safety.
+                        if (permissions == Policy.UNSUPPORTED_EMPTY_COLLECTION) {
+                            throw new UnsupportedOperationException("JavaPolicy implementation does not support retrieving permissions");
+                        }
+                        // grant the permissions to each jar in the plugin
+                        try (DirectoryStream<Path> jarStream = Files.newDirectoryStream(plugin, "*.jar")) {
+                            for (Path jar : jarStream) {
+                                if (map.put(jar.toUri().toURL().getFile(), permissions) != null) {
+                                    // just be paranoid ok?
+                                    throw new IllegalStateException("per-plugin permissions already granted for jar file: " + jar);
+                                }
+                            }
                         }
-                        System.setProperty(prop, plugin.toUri().toURL().toString() + "*");
                     }
                 }
             }
         }
-        for (String plugin : SPECIAL_PLUGINS.keySet()) {
-            String prop = getPluginProperty(plugin);
-            if (System.getProperty(prop) == null) {
-                System.setProperty(prop, "file:/dev/null"); // no chance to be interpreted as "all"
-            }
-        }
+        return Collections.unmodifiableMap(map);
     }
 
     /** returns dynamic Permissions to configured paths */

core/src/main/java/org/elasticsearch/plugins/PluginInfo.java

@@ -36,6 +36,7 @@
 public class PluginInfo implements Streamable, ToXContent {
 
     public static final String ES_PLUGIN_PROPERTIES = "plugin-descriptor.properties";
+    public static final String ES_PLUGIN_POLICY = "plugin-security.policy";
 
     static final class Fields {
         static final XContentBuilderString NAME = new XContentBuilderString("name");

core/src/main/java/org/elasticsearch/plugins/PluginManager.java

@@ -100,7 +100,7 @@ public PluginManager(Environment environment, URL url, OutputMode outputMode, Ti
         this.timeout = timeout;
     }
 
-    public void downloadAndExtract(String name, Terminal terminal) throws IOException {
+    public void downloadAndExtract(String name, Terminal terminal, boolean batch) throws IOException {
         if (name == null && url == null) {
             throw new IllegalArgumentException("plugin name or url must be supplied with install.");
         }
@@ -124,7 +124,7 @@ public void downloadAndExtract(String name, Terminal terminal) throws IOExceptio
         }
 
         Path pluginFile = download(pluginHandle, terminal);
-        extract(pluginHandle, terminal, pluginFile);
+        extract(pluginHandle, terminal, pluginFile, batch);
     }
 
     private Path download(PluginHandle pluginHandle, Terminal terminal) throws IOException {
@@ -207,7 +207,7 @@ private Path download(PluginHandle pluginHandle, Terminal terminal) throws IOExc
         return pluginFile;
     }
 
-    private void extract(PluginHandle pluginHandle, Terminal terminal, Path pluginFile) throws IOException {
+    private void extract(PluginHandle pluginHandle, Terminal terminal, Path pluginFile, boolean batch) throws IOException {
         // unzip plugin to a staging temp dir, named for the plugin
         Path tmp = Files.createTempDirectory(environment.tmpFile(), null);
         Path root = tmp.resolve(pluginHandle.name);
@@ -232,6 +232,13 @@ private void extract(PluginHandle pluginHandle, Terminal terminal, Path pluginFi
             throw new IOException("plugin directory " + extractLocation.toAbsolutePath() + " already exists. To update the plugin, uninstall it first using 'remove " + pluginHandle.name + "' command");
         }
 
+        // read optional security policy (extra permissions)
+        // if it exists, confirm or warn the user
+        Path policy = root.resolve(PluginInfo.ES_PLUGIN_POLICY);
+        if (Files.exists(policy)) {
+            PluginSecurity.readPolicy(policy, terminal, environment, batch);
+        }
+
         // install plugin
         FileSystemUtils.copyDirectoryRecursively(root, extractLocation);
         terminal.println("Installed %s into %s", pluginHandle.name, extractLocation.toAbsolutePath());
@@ -335,7 +342,7 @@ private static void setPosixFileAttributes(Path path, UserPrincipal owner, Group
         fileAttributeView.setPermissions(permissions);
     }
 
-    private void tryToDeletePath(Terminal terminal, Path ... paths) {
+    static void tryToDeletePath(Terminal terminal, Path ... paths) {
         for (Path path : paths) {
             try {
                 IOUtils.rm(path);

core/src/main/java/org/elasticsearch/plugins/PluginManagerCliParser.java

@@ -180,6 +180,7 @@ static class Install extends Command {
 
         private static final CliToolConfig.Cmd CMD = cmd(NAME, Install.class)
                 .options(option("t", "timeout").required(false).hasArg(false))
+                .options(option("b", "batch").required(false))
                 .build();
 
         static Command parse(Terminal terminal, CommandLine cli) {
@@ -210,21 +211,28 @@ static Command parse(Terminal terminal, CommandLine cli) {
             if (cli.hasOption("v")) {
                 outputMode = OutputMode.VERBOSE;
             }
+            
+            boolean batch = System.console() == null;
+            if (cli.hasOption("b")) {
+                batch = true;
+            }
 
-            return new Install(terminal, name, outputMode, optionalPluginUrl, timeout);
+            return new Install(terminal, name, outputMode, optionalPluginUrl, timeout, batch);
         }
 
         final String name;
         private OutputMode outputMode;
         final URL url;
         final TimeValue timeout;
+        final boolean batch;
 
-        Install(Terminal terminal, String name, OutputMode outputMode, URL url, TimeValue timeout) {
+        Install(Terminal terminal, String name, OutputMode outputMode, URL url, TimeValue timeout, boolean batch) {
             super(terminal);
             this.name = name;
             this.outputMode = outputMode;
             this.url = url;
             this.timeout = timeout;
+            this.batch = batch;
         }
 
         @Override
@@ -235,7 +243,7 @@ public ExitStatus execute(Settings settings, Environment env) throws Exception {
             } else {
                 terminal.println("-> Installing from " + URLDecoder.decode(url.toString(), "UTF-8") + "...");
             }
-            pluginManager.downloadAndExtract(name, terminal);
+            pluginManager.downloadAndExtract(name, terminal, batch);
             return ExitStatus.OK;
         }
     }