Merge remote-tracking branch 'remotes/origin/master' into allow_auto_create_index_param

Author: Vladimir Dolzhenko

README.textile

@@ -48,21 +48,21 @@ h3. Indexing
 Let's try and index some twitter like information. First, let's index some tweets (the @twitter@ index will be created automatically):
 
 <pre>
-curl -XPUT 'http://localhost:9200/twitter/doc/1?pretty' -H 'Content-Type: application/json' -d '
+curl -XPUT 'http://localhost:9200/twitter/_doc/1?pretty' -H 'Content-Type: application/json' -d '
 {
     "user": "kimchy",
     "post_date": "2009-11-15T13:12:00",
     "message": "Trying out Elasticsearch, so far so good?"
 }'
 
-curl -XPUT 'http://localhost:9200/twitter/doc/2?pretty' -H 'Content-Type: application/json' -d '
+curl -XPUT 'http://localhost:9200/twitter/_doc/2?pretty' -H 'Content-Type: application/json' -d '
 {
     "user": "kimchy",
     "post_date": "2009-11-15T14:12:12",
     "message": "Another tweet, will it be indexed?"
 }'
 
-curl -XPUT 'http://localhost:9200/twitter/doc/3?pretty' -H 'Content-Type: application/json' -d '
+curl -XPUT 'http://localhost:9200/twitter/_doc/3?pretty' -H 'Content-Type: application/json' -d '
 {
     "user": "elastic",
     "post_date": "2010-01-15T01:46:38",
@@ -73,9 +73,9 @@ curl -XPUT 'http://localhost:9200/twitter/doc/3?pretty' -H 'Content-Type: applic
 Now, let's see if the information was added by GETting it:
 
 <pre>
-curl -XGET 'http://localhost:9200/twitter/doc/1?pretty=true'
-curl -XGET 'http://localhost:9200/twitter/doc/2?pretty=true'
-curl -XGET 'http://localhost:9200/twitter/doc/3?pretty=true'
+curl -XGET 'http://localhost:9200/twitter/_doc/1?pretty=true'
+curl -XGET 'http://localhost:9200/twitter/_doc/2?pretty=true'
+curl -XGET 'http://localhost:9200/twitter/_doc/3?pretty=true'
 </pre>
 
 h3. Searching
@@ -133,14 +133,14 @@ Elasticsearch supports multiple indices. In the previous example we used an inde
 Another way to define our simple twitter system is to have a different index per user (note, though that each index has an overhead). Here is the indexing curl's in this case:
 
 <pre>
-curl -XPUT 'http://localhost:9200/kimchy/doc/1?pretty' -H 'Content-Type: application/json' -d '
+curl -XPUT 'http://localhost:9200/kimchy/_doc/1?pretty' -H 'Content-Type: application/json' -d '
 {
     "user": "kimchy",
     "post_date": "2009-11-15T13:12:00",
     "message": "Trying out Elasticsearch, so far so good?"
 }'
 
-curl -XPUT 'http://localhost:9200/kimchy/doc/2?pretty' -H 'Content-Type: application/json' -d '
+curl -XPUT 'http://localhost:9200/kimchy/_doc/2?pretty' -H 'Content-Type: application/json' -d '
 {
     "user": "kimchy",
     "post_date": "2009-11-15T14:12:12",

TESTING.asciidoc

@@ -77,39 +77,24 @@ Run a single test case (variants)
 ./gradlew test "-Dtests.class=*.ClassName"
 ----------------------------------------------------------
 
-Run all tests in a package and sub-packages
+Run all tests in a package and its sub-packages
 
 ----------------------------------------------------
 ./gradlew test "-Dtests.class=org.elasticsearch.package.*"
 ----------------------------------------------------
 
-Run any test methods that contain 'esi' (like: ...r*esi*ze...).
+Run any test methods that contain 'esi' (like: ...r*esi*ze...)
 
 -------------------------------
 ./gradlew test "-Dtests.method=*esi*"
 -------------------------------
 
-You can also filter tests by certain annotations ie:
-
-  * `@Nightly` - tests that only run in nightly builds (disabled by default)
-  * `@Backwards` - backwards compatibility tests (disabled by default)
-  * `@AwaitsFix` - tests that are waiting for a bugfix (disabled by default)
-  * `@BadApple` - tests that are known to fail randomly (disabled by default)
-
-Those annotation names can be combined into a filter expression like:
+Run all tests that are waiting for a bugfix (disabled by default)
 
 ------------------------------------------------
-./gradlew test -Dtests.filter="@nightly and not @backwards"
+./gradlew test -Dtests.filter=@awaitsfix
 ------------------------------------------------
 
-to run all nightly test but not the ones that are backwards tests. `tests.filter` supports
-the boolean operators `and, or, not` and grouping ie:
-
-
----------------------------------------------------------------
-./gradlew test -Dtests.filter="@nightly and not(@badapple or @backwards)"
----------------------------------------------------------------
-
 === Seed and repetitions.
 
 Run with a given seed (seed is a hex-encoded long).
@@ -160,8 +145,6 @@ Test groups can be enabled or disabled (true/false).
 Default value provided below in [brackets].
 
 ------------------------------------------------------------------
-./gradlew test -Dtests.nightly=[false]   - nightly test group (@Nightly)
-./gradlew test -Dtests.weekly=[false]    - weekly tests (@Weekly)
 ./gradlew test -Dtests.awaitsfix=[false] - known issue (@AwaitsFix)
 ------------------------------------------------------------------
 

build.gradle

@@ -170,8 +170,8 @@ task verifyVersions {
  * the enabled state of every bwc task. It should be set back to true
  * after the backport of the backcompat code is complete.
  */
-final boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+final boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/34724" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/build.gradle

@@ -106,6 +106,7 @@ dependencies {
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
   compile 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
+  compile 'de.thetaphi:forbiddenapis:2.6'
   testCompile "junit:junit:${props.getProperty('junit')}"
 }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -193,7 +193,12 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                         "$snippet: Use `js` instead of `${snippet.language}`.")
             }
             if (snippet.testSetup) {
-                setup(snippet)
+                testSetup(snippet)
+                previousTest = snippet
+                return
+            }
+            if (snippet.testTearDown) {
+                testTearDown(snippet)
                 previousTest = snippet
                 return
             }
@@ -223,6 +228,10 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                     throw new InvalidUserDataException("// TEST[continued] " +
                         "cannot immediately follow // TESTSETUP: $test")
                 }
+                if (previousTest != null && previousTest.testTearDown) {
+                    throw new InvalidUserDataException("// TEST[continued] " +
+                        "cannot immediately follow // TEARDOWN: $test")
+                }
             } else {
                 current.println('---')
                 current.println("\"line_$test.start\":")
@@ -259,20 +268,23 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 current.println("      reason: $test.skip")
             }
             if (test.setup != null) {
-                // Insert a setup defined outside of the docs
-                for (String setupName : test.setup.split(',')) {
-                    String setup = setups[setupName]
-                    if (setup == null) {
-                        throw new InvalidUserDataException("Couldn't find setup "
-                                + "for $test")
-                    }
-                    current.println(setup)
-                }
+                setup(test)
             }
 
             body(test, false)
         }
 
+        private void setup(final Snippet snippet) {
+            // insert a setup defined outside of the docs
+            for (final String setupName : snippet.setup.split(',')) {
+                final String setup = setups[setupName]
+                if (setup == null) {
+                    throw new InvalidUserDataException("Couldn't find setup for $snippet")
+                }
+                current.println(setup)
+            }
+        }
+
         private void response(Snippet response) {
             if (null == response.skip) {
                 current.println("  - match: ")
@@ -339,14 +351,27 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             }
         }
 
-        private void setup(Snippet setup) {
-            if (lastDocsPath == setup.path) {
-                throw new InvalidUserDataException("$setup: wasn't first")
+        private void testSetup(Snippet snippet) {
+            if (lastDocsPath == snippet.path) {
+                throw new InvalidUserDataException("$snippet: wasn't first")
             }
-            setupCurrent(setup)
+            setupCurrent(snippet)
             current.println('---')
             current.println("setup:")
-            body(setup, true)
+            if (snippet.setup != null) {
+                setup(snippet)
+            }
+            body(snippet, true)
+        }
+
+        private void testTearDown(Snippet snippet) {
+            if (previousTest.testSetup == false && lastDocsPath == snippet.path) {
+                throw new InvalidUserDataException("$snippet must follow test setup or be first")
+            }
+            setupCurrent(snippet)
+            current.println('---')
+            current.println('teardown:')
+            body(snippet, true)
         }
 
         private void body(Snippet snippet, boolean inSetup) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -273,6 +273,10 @@ public class SnippetsTask extends DefaultTask {
                     snippet.testSetup = true
                     return
                 }
+                if (line ==~ /\/\/\s*TEARDOWN\s*/) {
+                    snippet.testTearDown = true
+                    return
+                }
                 if (snippet == null) {
                     // Outside
                     return
@@ -317,6 +321,7 @@ public class SnippetsTask extends DefaultTask {
         boolean test = false
         boolean testResponse = false
         boolean testSetup = false
+        boolean testTearDown = false
         String skip = null
         boolean continued = false
         String language = null

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -19,7 +19,10 @@
 package org.elasticsearch.gradle.precommit
 
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
+import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
+import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
+import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.plugins.JavaBasePlugin
@@ -33,7 +36,7 @@ class PrecommitTasks {
     public static Task create(Project project, boolean includeDependencyLicenses) {
         project.configurations.create("forbiddenApisCliJar")
         project.dependencies {
-            forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.5')
+            forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.6')
         }
 
         List<Task> precommitTasks = [
@@ -109,47 +112,43 @@ class PrecommitTasks {
     }
 
     private static Task configureForbiddenApisCli(Project project) {
-        Task forbiddenApisCli = project.tasks.create('forbiddenApis')
-        project.sourceSets.all { sourceSet ->
-            forbiddenApisCli.dependsOn(
-                project.tasks.create(sourceSet.getTaskName('forbiddenApis', null), ForbiddenApisCliTask) {
-                    ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
-                    dependsOn(buildResources)
-                    it.sourceSet = sourceSet
-                    javaHome = project.runtimeJavaHome
-                    targetCompatibility = project.compilerJavaVersion
-                    bundledSignatures = [
-                       "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"
-                    ]
-                    signaturesFiles = project.files(
-                            buildResources.copy("forbidden/jdk-signatures.txt"),
-                            buildResources.copy("forbidden/es-all-signatures.txt")
-                    )
-                    suppressAnnotations = ['**.SuppressForbidden']
-                    if (sourceSet.name == 'test') {
-                        signaturesFiles += project.files(
-                                buildResources.copy("forbidden/es-test-signatures.txt"),
-                                buildResources.copy("forbidden/http-signatures.txt")
-                        )
-                    } else {
-                        signaturesFiles += project.files(buildResources.copy("forbidden/es-server-signatures.txt"))
-                    }
-                    dependsOn sourceSet.classesTaskName
-                    classesDirs = sourceSet.output.classesDirs
-                    ext.replaceSignatureFiles = { String... names ->
-                        signaturesFiles = project.files(
-                                names.collect { buildResources.copy("forbidden/${it}.txt") }
-                        )
-                    }
-                    ext.addSignatureFiles = { String... names ->
-                        signaturesFiles += project.files(
-                                names.collect { buildResources.copy("forbidden/${it}.txt") }
-                        )
-                    }
-                }
+        project.pluginManager.apply(ForbiddenApisPlugin)
+        ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
+        project.tasks.withType(CheckForbiddenApis) {
+            dependsOn(buildResources)
+            targetCompatibility = project.runtimeJavaVersion >= JavaVersion.VERSION_1_9 ?
+                    project.runtimeJavaVersion.getMajorVersion() :
+                    project.runtimeJavaVersion
+            bundledSignatures = [
+                    "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"
+            ]
+            signaturesFiles = project.files(
+                    buildResources.copy("forbidden/jdk-signatures.txt"),
+                    buildResources.copy("forbidden/es-all-signatures.txt")
             )
+            suppressAnnotations = ['**.SuppressForbidden']
+            if (name.endsWith('Test')) {
+                signaturesFiles += project.files(
+                        buildResources.copy("forbidden/es-test-signatures.txt"),
+                        buildResources.copy("forbidden/http-signatures.txt")
+                )
+            } else {
+                signaturesFiles += project.files(buildResources.copy("forbidden/es-server-signatures.txt"))
+            }
+            ext.replaceSignatureFiles = { String... names ->
+                signaturesFiles = project.files(
+                        names.collect { buildResources.copy("forbidden/${it}.txt") }
+                )
+            }
+            ext.addSignatureFiles = { String... names ->
+                signaturesFiles += project.files(
+                        names.collect { buildResources.copy("forbidden/${it}.txt") }
+                )
+            }
         }
-        return forbiddenApisCli
+        Task forbiddenApis =  project.tasks.getByName("forbiddenApis")
+        forbiddenApis.group = ""
+        return forbiddenApis
     }
 
     private static Task configureCheckstyle(Project project) {