Generate beta and experimental APIs, guarded by Cargo features

.ci/DockerFile

@@ -28,4 +28,4 @@ COPY elasticsearch/src ./elasticsearch/src
 COPY elasticsearch/build.rs ./elasticsearch/build.rs
 COPY yaml_test_runner ./yaml_test_runner
 
-RUN cargo build --tests
+RUN cargo build --tests

.ci/functions/imports.sh

@@ -18,7 +18,7 @@ require_stack_version
 if [[ -z $es_node_name ]]; then
   # only set these once
   set -euo pipefail
-  export TEST_SUITE=${TEST_SUITE-oss}
+  export TEST_SUITE=${TEST_SUITE-free}
   export RUNSCRIPTS=${RUNSCRIPTS-}
   export DETACH=${DETACH-false}
   export CLEANUP=${CLEANUP-false}
@@ -27,8 +27,7 @@ if [[ -z $es_node_name ]]; then
   export elastic_password=changeme
   export elasticsearch_image=elasticsearch
   export elasticsearch_url=https://elastic:${elastic_password}@${es_node_name}:9200
-  if [[ $TEST_SUITE != "xpack" ]]; then
-    export elasticsearch_image=elasticsearch-${TEST_SUITE}
+  if [[ $TEST_SUITE != "platinum" ]]; then
     export elasticsearch_url=http://${es_node_name}:9200
   fi
   export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}

.ci/run-elasticsearch.sh

@@ -4,7 +4,7 @@
 # to form a cluster suitable for running the REST API tests.
 #
 # Export the STACK_VERSION variable, eg. '8.0.0-SNAPSHOT'.
-# Export the TEST_SUITE variable, eg. 'oss' or 'xpack' defaults to 'oss'.
+# Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'.
 # Export the NUMBER_OF_NODES variable to start more than 1 node
 
 # Version 1.1.0
@@ -38,7 +38,7 @@ environment=($(cat <<-END
   --env repositories.url.allowed_urls=http://snapshot.test*
 END
 ))
-if [[ "$TEST_SUITE" == "xpack" ]]; then
+if [[ "$TEST_SUITE" == "platinum" ]]; then
   environment+=($(cat <<-END
     --env ELASTIC_PASSWORD=$elastic_password
     --env xpack.license.self_generated.type=trial
@@ -63,7 +63,7 @@ END
 fi
 
 cert_validation_flags=""
-if [[ "$TEST_SUITE" == "xpack" ]]; then
+if [[ "$TEST_SUITE" == "platinum" ]]; then
   cert_validation_flags="--insecure --cacert /usr/share/elasticsearch/config/certs/ca.crt --resolve ${es_node_name}:443:127.0.0.1"
 fi
 

.ci/run-repository.sh

@@ -3,7 +3,7 @@
 
 
 # STACK_VERSION -- version e.g Major.Minor.Patch(-Prelease)
-# TEST_SUITE -- which test suite to run: oss or xpack
+# TEST_SUITE -- which test suite to run: free or platinum
 # ELASTICSEARCH_URL -- The url at which elasticsearch is reachable, a default is composed based on STACK_VERSION and TEST_SUITE
 # RUST_TOOLCHAIN -- Rust toolchain version to compile and run tests
 script_path=$(dirname $(realpath -s $0))

.ci/test-matrix.yml

@@ -4,8 +4,8 @@ STACK_VERSION:
   - 7.x-SNAPSHOT
 
 TEST_SUITE:
-  - oss
-  - xpack
+  - free
+  - platinum
 
 RUST_TOOLCHAIN:
   - latest

.dockerignore

@@ -6,8 +6,9 @@ target
 docs
 .vscode
 .idea
+*.iml
 
 api_generator/src/bin
 
 yaml_test_runner/tests
-!yaml_test_runner/tests/common
+!yaml_test_runner/tests/common

CONTRIBUTING.md

@@ -79,10 +79,10 @@ The `Elasticsearch` category of steps are specifically defined for this project
 
   - `STACK_VERSION`: Elasticsearch version like `7.9.0` or can be
   a snapshot release like `7.x-SNAPSHOT`
-  - `TEST_SUITE`: Elasticsearch distribution of `oss` or `xpack`
+  - `TEST_SUITE`: Elasticsearch distribution of `free` or `platinum`
 
   ```sh
-  cargo make test-yaml --env STACK_VERSION=7.9.0 --env TEST_SUITE=oss
+  cargo make test-yaml --env STACK_VERSION=7.9.0 --env TEST_SUITE=free
   ```
 
 ### Packages
@@ -198,4 +198,4 @@ if wishing to use the MSVC debugger with Rust in VS code, which may be preferred
     }
     ```
 
-3. Add `"debug.allowBreakpointsEverywhere": true` to VS code settings.json
+3. Add `"debug.allowBreakpointsEverywhere": true` to VS code settings.json

Makefile.toml

@@ -3,24 +3,24 @@ default_to_workspace = false
 
 [env]
 # Determines the version of Elasticsearch docker container used
-STACK_VERSION = "7.x-SNAPSHOT"
-# Determines the distribution of docker container used. Either xpack or oss
-TEST_SUITE = "xpack"
+STACK_VERSION = { value = "7.x-SNAPSHOT", condition = { env_not_set = ["STACK_VERSION"] }}
+# Determines the distribution of docker container used. Either platinum or free
+TEST_SUITE = { value = "free", condition = { env_not_set = ["TEST_SUITE"] }}
 # Set publish flags to dry-run by default, to force user to explicitly define for publishing
 CARGO_MAKE_CARGO_PUBLISH_FLAGS = "--dry-run"
 
-[tasks.set-oss-env]
+[tasks.set-free-env]
 category = "Elasticsearch"
-description = "Sets ELASTICSEARCH_URL environment variable if not already set for later tasks when oss test suite used"
+description = "Sets ELASTICSEARCH_URL environment variable if not already set for later tasks when free test suite used"
 private = true
-condition = { env = { "TEST_SUITE" = "oss" }, env_not_set = ["ELASTICSEARCH_URL"] }
+condition = { env = { "TEST_SUITE" = "free" }, env_not_set = ["ELASTICSEARCH_URL"] }
 env = { "ELASTICSEARCH_URL" = "http://localhost:9200" }
 
-[tasks.set-xpack-env]
+[tasks.set-platinum-env]
 category = "Elasticsearch"
-description = "Sets ELASTICSEARCH_URL environment variable if not already set for later tasks when xpack test suite used"
+description = "Sets ELASTICSEARCH_URL environment variable if not already set for later tasks when platinum test suite used"
 private = true
-condition = { env = { "TEST_SUITE" = "xpack" }, env_not_set = ["ELASTICSEARCH_URL"] }
+condition = { env = { "TEST_SUITE" = "platinum" }, env_not_set = ["ELASTICSEARCH_URL"] }
 env = { "ELASTICSEARCH_URL" = "https://elastic:changeme@localhost:9200" }
 
 [tasks.run-yaml-test-runner]
@@ -52,7 +52,7 @@ dependencies = ["generate-yaml-tests"]
 [tasks.test-elasticsearch]
 category = "Elasticsearch"
 private = true
-condition = { env_set = [ "ELASTICSEARCH_URL" ], env = { "TEST_SUITE" = "xpack" } }
+condition = { env_set = [ "ELASTICSEARCH_URL" ], env = { "TEST_SUITE" = "platinum" } }
 command = "cargo"
 args = ["test", "-p", "elasticsearch"]
 dependencies = ["start-elasticsearch"]
@@ -85,7 +85,7 @@ dependencies = ["install-cargo2junit"]
 category = "Elasticsearch"
 private = true
 condition = { env_set = [ "STACK_VERSION", "TEST_SUITE" ], env_false = ["CARGO_MAKE_CI"] }
-dependencies = ["set-oss-env", "set-xpack-env"]
+dependencies = ["set-free-env", "set-platinum-env"]
 
 [tasks.run-elasticsearch.linux]
 command = "./.ci/run-elasticsearch.sh"
@@ -120,17 +120,19 @@ exec cargo publish %{CARGO_MAKE_CARGO_PUBLISH_FLAGS}
 
 [tasks.start-elasticsearch]
 extend = "run-elasticsearch"
+private = false
 description = "Starts Elasticsearch docker container with the given version and distribution"
 env = { "CLEANUP" = false, "DETACH" = true }
 
 [tasks.stop-elasticsearch]
 extend = "run-elasticsearch"
+private = false
 description = "Stops Elasticsearch docker container, if running"
 env = { "CLEANUP" = true, "DETACH" = false }
 
 [tasks.test-yaml]
 category = "Elasticsearch"
-description = "Generates and runs yaml_test_runner package xpack/oss tests against a given Elasticsearch version"
+description = "Generates and runs yaml_test_runner package platinum/free tests against a given Elasticsearch version"
 condition = { env_set = [ "STACK_VERSION", "TEST_SUITE" ] }
 dependencies = ["generate-yaml-tests", "create-test-results-dir", "test-yaml-test-runner", "test-yaml-test-runner-ci", "convert-test-results-junit"]
 run_task = "stop-elasticsearch"
@@ -146,7 +148,7 @@ args = ["test", "-p", "api_generator"]
 category = "Elasticsearch"
 clear = true
 description = "Runs elasticsearch package tests against a given Elasticsearch version"
-env = { "TEST_SUITE" = { value = "xpack", condition = { env_set = ["TEST_SUITE"] } } }
+env = { "TEST_SUITE" = { value = "platinum", condition = { env_set = ["TEST_SUITE"] } } }
 dependencies = ["test-elasticsearch"]
 run_task = "stop-elasticsearch"
 
@@ -167,7 +169,7 @@ description = "Generate Elasticsearch client documentation and opens in browser"
 clear = true
 category = "Elasticsearch"
 command = "cargo"
-args = ["doc", "-p", "elasticsearch", "--no-deps", "--open"]
+args = ["doc", "-p", "elasticsearch", "--no-deps", "--open", "--all-features"]
 
 [tasks.generate-release-notes]
 category = "Elasticsearch"
@@ -217,7 +219,7 @@ script = ['''
     echo - start-elasticsearch: Starts Elasticsearch docker container with the given version and distribution
     echo - stop-elasticsearch: Stops Elasticsearch docker container, if running
     echo
-    echo - test-yaml: Generates and runs yaml_test_runner package xpack/oss tests against a given Elasticsearch version
+    echo - test-yaml: Generates and runs yaml_test_runner package platinum/free tests against a given Elasticsearch version
     echo - test-generator: Generates and runs api_generator package tests
     echo - test: Runs elasticsearch package tests against a given Elasticsearch version
     echo
@@ -230,7 +232,7 @@ script = ['''
     echo
     echo Most tasks use these environment variables:
     echo - STACK_VERSION (default '${STACK_VERSION}'): the version of Elasticsearch
-    echo - TEST_SUITE ('oss' or 'xpack', default '${TEST_SUITE}'): the distribution of Elasticsearch
+    echo - TEST_SUITE ('free' or 'platinum', default '${TEST_SUITE}'): the distribution of Elasticsearch
     echo - CI (default not set): set when running on CI to determine whether to start Elasticsearch and format test output as JSON
     echo
     echo Run 'cargo make --list-all-steps' for a complete list of available tasks.

README.md

@@ -46,6 +46,11 @@ The following are a list of Cargo features that can be enabled or disabled:
 
 - **native-tls** *(enabled by default)*: Enables TLS functionality provided by `native-tls`.
 - **rustls-tls**: Enables TLS functionality provided by `rustls`.
+- **beta-apis**: Enables beta APIs. Beta APIs are on track to become stable and permanent features. Use them with 
+   caution because it is possible that breaking changes are made to these APIs in a minor version.
+- **experimental-apis**: Enables experimental APIs. Experimental APIs are just that - an experiment. An experimental
+  API might have breaking changes in any future version, or it might even be removed entirely. This feature also
+  enables `beta-apis`.
 
 ## Getting started
 

api_generator/Cargo.toml

@@ -16,7 +16,9 @@ flate2 = "~1"
 globset = "~0.4"
 Inflector = "0.11.4"
 indicatif = "0.12.0"
+itertools = "0.10.0"
 lazy_static = "1.4.0"
+log = "0.4.8"
 path-slash = "0.1.3"
 quote = "~0.3"
 reduce = "0.1.2"
@@ -26,6 +28,7 @@ semver = "0.9.0"
 serde = "~1"
 serde_json = "~1"
 serde_derive = "~1"
+simple_logger = "1.9.0"
 syn = { version = "~0.11", features = ["full"] }
 tar = "~0.4"
 toml = "0.5.6"

api_generator/docs/namespaces/autoscaling.md

@@ -0,0 +1,5 @@
+Autoscaling APIs
+
+The [autoscaling feature](https://www.elastic.co/guide/en/elasticsearch/reference/master/xpack-autoscaling.html)
+enables an operator to configure tiers of nodes that self-monitor whether or not they need to scale based on an
+operator-defined policy.

api_generator/docs/namespaces/eql.md

@@ -0,0 +1,6 @@
+EQL APIs
+
+[Event Query Language (EQL)](https://www.elastic.co/guide/en/elasticsearch/reference/master/eql.html) is a query
+language for event-based time series data, such as logs, metrics, and traces.
+
+

api_generator/docs/namespaces/logstash.md

@@ -0,0 +1,4 @@
+Logstash APIs
+
+The [Logstash APIs](https://www.elastic.co/guide/en/elasticsearch/reference/master/logstash-apis.html) are used to
+manage pipelines used by Logstash Central Management.

api_generator/docs/namespaces/monitoring.md

@@ -0,0 +1,4 @@
+Monitoring APIs
+
+The Elastic Stack [monitoring features](https://www.elastic.co/guide/en/elasticsearch/reference/master/monitor-elasticsearch-cluster.html)
+provide a way to keep a pulse on thehealth and performance of your Elasticsearch cluster.

api_generator/docs/namespaces/rollup.md

@@ -0,0 +1,5 @@
+Rollup APIs
+
+The Elastic Stack [data rollup features](https://www.elastic.co/guide/en/elasticsearch/reference/master/xpack-rollup.html)
+provide a means to summarize and store historical data so that it can still be used for analysis, but at a fraction of
+the storage cost of raw data.

api_generator/docs/namespaces/searchable_snapshots.md

@@ -0,0 +1,6 @@
+Searchable snapshot APIs
+
+[Searchable snapshots](https://www.elastic.co/guide/en/elasticsearch/reference/master/searchable-snapshots.html) let
+you reduce your operating costs by using snapshots for resiliency rather than maintaining replica shards within a
+cluster. This can result in significant cost savings for less frequently searched data.
+

api_generator/docs/namespaces/text_structure.md

@@ -0,0 +1,4 @@
+Text structure APIs
+
+Determines the structure of text and other information that will be useful to import its contents to an Elasticsearch
+index.

api_generator/src/bin/run.rs

@@ -28,6 +28,11 @@ use std::{
 };
 
 fn main() -> Result<(), failure::Error> {
+    simple_logger::SimpleLogger::new()
+        .with_level(log::LevelFilter::Info)
+        .init()
+        .unwrap();
+
     // This must be run from the repo root directory, with cargo make generate-api
     let download_dir = fs::canonicalize(PathBuf::from("./api_generator/rest_specs"))?;
     let generated_dir = fs::canonicalize(PathBuf::from("./elasticsearch/src"))?;

api_generator/src/generator/code_gen/mod.rs

@@ -22,7 +22,7 @@ pub mod request;
 pub mod root;
 pub mod url;
 
-use crate::generator::TypeKind;
+use crate::generator::{Stability, TypeKind};
 use inflector::Inflector;
 use quote::Tokens;
 use std::str;
@@ -72,6 +72,22 @@ fn doc<I: Into<String>>(comment: I) -> syn::Attribute {
     }
 }
 
+fn stability_doc(stability: Stability) -> Option<syn::Attribute> {
+    match stability {
+        Stability::Experimental => Some(doc(r#"&nbsp;
+# Optional, experimental
+This requires the `experimental-apis` feature. Can have breaking changes in future
+versions or might even be removed entirely.
+        "#)),
+        Stability::Beta => Some(doc(r#"&nbsp;
+# Optional, beta
+This requires the `beta-apis` feature. On track to become stable but breaking changes can
+happen in minor versions.
+        "#)),
+        Stability::Stable => None,
+    }
+}
+
 /// AST for an expression parsed from quoted tokens
 pub fn parse_expr(input: quote::Tokens) -> syn::Expr {
     syn::parse_expr(input.to_string().as_ref()).unwrap()