Merge branch 'zen2' into zen2_storage_fix

Author: Andrey Ershov

.ci/java-versions.properties

@@ -4,5 +4,7 @@
 # build and test Elasticsearch for this branch. Valid Java versions
 # are 'java' or 'openjdk' followed by the major release number.
 
-ES_BUILD_JAVA=java10
+ES_BUILD_JAVA=java11
 ES_RUNTIME_JAVA=java8
+GRADLE_TASK=build
+

.ci/matrix-build-javas.yml

@@ -6,5 +6,4 @@
 # or 'openjdk' followed by the major release number.
 
 ES_BUILD_JAVA:
-  - java10
   - java11

.ci/matrix-runtime-javas.yml

@@ -8,5 +8,4 @@
 ES_RUNTIME_JAVA:
   - java8
   - java8fips
-  - java10
   - java11

CONTRIBUTING.md

@@ -161,7 +161,7 @@ Please follow these formatting guidelines:
 * Line width is 140 characters
 * The rest is left to Java coding standards
 * Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do.
-* Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. Please attempt to tame your IDE so it doesn't make them and please send a PR against this document with instructions for your IDE if it doesn't contain them.
+* Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. This can be done automatically by your IDE:
  * Eclipse: `Preferences->Java->Code Style->Organize Imports`. There are two boxes labeled "`Number of (static )? imports needed for .*`". Set their values to 99999 or some other absurdly high value.
  * IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value.
 * Don't worry too much about import order. Try not to change it but don't worry about fighting your IDE to stop it from doing so.
@@ -320,7 +320,7 @@ have to test Elasticsearch.
 #### Configurations
 
 Gradle organizes dependencies and build artifacts into "configurations" and
-allows you to use these configurations arbitrarilly. Here are some of the most
+allows you to use these configurations arbitrarily. Here are some of the most
 common configurations in our build and how we use them:
 
 <dl>

TESTING.asciidoc

@@ -1,4 +1,4 @@
-[[Testing Framework Cheatsheet]]
+[[TestingFrameworkCheatsheet]]
 = Testing
 
 [partintro]
@@ -250,7 +250,7 @@ Pass arbitrary jvm arguments.
 
 Running backwards compatibility tests is disabled by default since it
 requires a release version of elasticsearch to be present on the test system.
-To run backwards compatibilty tests untar or unzip a release and run the tests
+To run backwards compatibility tests untar or unzip a release and run the tests
 with the following command:
 
 ---------------------------------------------------------------------------
@@ -350,26 +350,26 @@ and running elasticsearch distributions works correctly on supported operating s
 These tests should really only be run in vagrant vms because they're destructive.
 
 . Install Virtual Box and Vagrant.
-
++
 . (Optional) Install https://github.com/fgrehm/vagrant-cachier[vagrant-cachier] to squeeze
 a bit more performance out of the process:
-
++
 --------------------------------------
 vagrant plugin install vagrant-cachier
 --------------------------------------
-
++
 . Validate your installed dependencies:
-
++
 -------------------------------------
 ./gradlew :qa:vagrant:vagrantCheckVersion
 -------------------------------------
-
++
 . Download and smoke test the VMs with `./gradlew vagrantSmokeTest` or
 `./gradlew -Pvagrant.boxes=all vagrantSmokeTest`. The first time you run this it will
 download the base images and provision the boxes and immediately quit. Downloading all
 the images may take a long time. After the images are already on your machine, they won't
 be downloaded again unless they have been updated to a new version.
-
++
 . Run the tests with `./gradlew packagingTest`. This will cause Gradle to build
 the tar, zip, and deb packages and all the plugins. It will then run the tests
 on ubuntu-1404 and centos-7. We chose those two distributions as the default
@@ -402,6 +402,7 @@ These are the linux flavors supported, all of which we provide images for
 
 * ubuntu-1404 aka trusty
 * ubuntu-1604 aka xenial
+* ubuntu-1804 aka bionic beaver
 * debian-8 aka jessie
 * debian-9 aka stretch, the current debian stable distribution
 * centos-6

Vagrantfile

@@ -61,6 +61,15 @@ Vagrant.configure(2) do |config|
       SHELL
     end
   end
+  'ubuntu-1804'.tap do |box|
+    config.vm.define box, define_opts do |config|
+      config.vm.box = 'elastic/ubuntu-18.04-x86_64'
+      deb_common config, box, extra: <<-SHELL
+       # Install Jayatana so we can work around it being present.
+       [ -f /usr/share/java/jayatanaag.jar ] || install jayatana
+      SHELL
+    end
+  end
   # Wheezy's backports don't contain Openjdk 8 and the backflips
   # required to get the sun jdk on there just aren't worth it. We have
   # jessie and stretch for testing debian and it works fine.

benchmarks/build.gradle

@@ -44,6 +44,8 @@ compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-u
 // needs to be added separately otherwise Gradle will quote it and javac will fail
 compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
 
+run.executable = new File(project.runtimeJavaHome, 'bin/java')
+
 // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
 forbiddenApisMain.enabled = false
 

benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark.fs;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(3)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Benchmark)
+public class AvailableIndexFoldersBenchmark {
+
+    private NodeEnvironment.NodePath nodePath;
+    private NodeEnvironment nodeEnv;
+    private Set<String> excludedDirs;
+
+    @Setup
+    public void setup() throws IOException {
+        Path path = Files.createTempDirectory("test");
+        String[] paths = new String[] {path.toString()};
+        nodePath = new NodeEnvironment.NodePath(path);
+
+        LogConfigurator.setNodeName("test");
+        Settings settings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), path)
+            .putList(Environment.PATH_DATA_SETTING.getKey(), paths).build();
+        nodeEnv = new NodeEnvironment(settings, new Environment(settings, null));
+
+        Files.createDirectories(nodePath.indicesPath);
+        excludedDirs = new HashSet<>();
+        int numIndices = 5000;
+        for (int i = 0; i < numIndices; i++) {
+            String dirName = "dir" + i;
+            Files.createDirectory(nodePath.indicesPath.resolve(dirName));
+            excludedDirs.add(dirName);
+        }
+        if (nodeEnv.availableIndexFoldersForPath(nodePath).size() != numIndices) {
+            throw new IllegalStateException("bad size");
+        }
+        if (nodeEnv.availableIndexFoldersForPath(nodePath, excludedDirs::contains).size() != 0) {
+            throw new IllegalStateException("bad size");
+        }
+    }
+
+
+    @Benchmark
+    public Set<String> availableIndexFolderNaive() throws IOException {
+        return nodeEnv.availableIndexFoldersForPath(nodePath);
+    }
+
+    @Benchmark
+    public Set<String> availableIndexFolderOptimized() throws IOException {
+        return nodeEnv.availableIndexFoldersForPath(nodePath, excludedDirs::contains);
+    }
+
+}

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy

@@ -264,7 +264,11 @@ class RandomizedTestingTask extends DefaultTask {
                         throw new InvalidUserDataException('Seed should be ' +
                             'set on the project instead of a system property')
                     }
-                    sysproperty key: prop.getKey(), value: prop.getValue().toString()
+                    if (prop.getValue() instanceof Closure) {
+                        sysproperty key: prop.getKey(), value: (prop.getValue() as Closure).call().toString()
+                    } else {
+                        sysproperty key: prop.getKey(), value: prop.getValue().toString()
+                    }
                 }
                 systemProperty 'tests.seed', project.testSeed
                 for (Map.Entry<String, Object> envvar : environmentVariables) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy

@@ -122,7 +122,7 @@ class VersionCollection {
                         if (isReleased(version) == false) {
                             // caveat 1 - This should only ever contain 2 non released branches in flight. An example is 6.x is frozen,
                             // and 6.2 is cut but not yet released there is some simple logic to make sure that in the case of more than 2,
-                            // it will bail. The order is that the minor snapshot is fufilled first, and then the staged minor snapshot
+                            // it will bail. The order is that the minor snapshot is fulfilled first, and then the staged minor snapshot
                             if (nextMinorSnapshot == null) {
                                 // it has not been set yet
                                 nextMinorSnapshot = replaceAsSnapshot(version)