Simplify generated code module structure (#138)

Remove the `generated` directory and the various re-exports.

- namespace docs have been extracted from the respective `mod.rs` to
  `.md` files in api_generator/docs, and are inserted in the generated
  source file.
- namespace sources are generated directly in the `src` directory
- the generator can now insert generated sections in regular source
  files based on `GENERATE-BEGIN` and `GENERATE-END` markers. This is
  used to generate the namespace module list in the main `lib.rs` and
  merge generated with manual code in `params.rs`
- generated and merged source files are listed in `src/.generated.toml`
  so that the generator can cleanup on start.

Before considering merging generated code with special markers we
experimented with the `include!()` macro, but this didn't work well:
- rustfmt will not reformat include files
- `mod` statement look for files relative to the source file, so
  include files have to be located in the same directory as their
  includer, while locating them in a dedicated directory would have
  provided a nicer organization.

Merging generated sections provides a more natural source code layout.

(cherry picked from commit 9d9a4fb)

Author: swallez

api_generator/Cargo.toml

@@ -16,6 +16,7 @@ globset = "~0.4"
 Inflector = "0.11.4"
 indicatif = "0.12.0"
 lazy_static = "1.4.0"
+path-slash = "0.1.3"
 quote = "~0.3"
 reduce = "0.1.2"
 regex = "1.3.1"
@@ -26,5 +27,9 @@ serde_json = "~1"
 serde_derive = "~1"
 syn = { version = "~0.11", features = ["full"] }
 tar = "~0.4"
+toml = "0.5.6"
 url = "2.1.1"
-void = "1.0.2"
+void = "1.0.2"
+
+[dev-dependencies]
+tempfile = "3.1.0"

elasticsearch/docs/Elasticsearch.fn.scroll.md renamed to api_generator/docs/functions/Elasticsearch.scroll.md

@@ -0,0 +1,5 @@
+Async Search APIs
+
+[Async search APIs](https://www.elastic.co/guide/en/elasticsearch/reference/master/async-search.html)
+let you asynchronously execute a search request, monitor its progress, and retrieve
+partial results as they become available.

elasticsearch/docs/Indices.fn.create.md renamed to api_generator/docs/functions/Indices.create.md

@@ -0,0 +1,94 @@
+Cat APIs
+
+The [Cat APIs](https://www.elastic.co/guide/en/elasticsearch/reference/master/cat.html) aim to
+meet the needs of humans when looking at data returned from Elasticsearch,
+formatting it as compact, column aligned text, making it easier on human eyes.
+
+# Plain text responses
+
+By default, all Cat APIs are configured to send requests with `text/plain` content-type
+and accept headers, returning plain text responses
+
+```rust,no_run
+# use elasticsearch::{Elasticsearch, Error, SearchParts};
+# use url::Url;
+# use elasticsearch::auth::Credentials;
+# use serde_json::{json, Value};
+# async fn doc() -> Result<(), Box<dyn std::error::Error>> {
+# let client = Elasticsearch::default();
+let response = client
+    .cat()
+    .nodes()
+    .send()
+    .await?;
+
+let response_body = response.text().await?;
+# Ok(())
+# }
+```
+
+# JSON responses
+
+JSON responses can be returned from Cat APIs either by using `.format("json")`
+
+```rust,no_run
+# use elasticsearch::{Elasticsearch, Error, SearchParts};
+# use url::Url;
+# use elasticsearch::auth::Credentials;
+# use serde_json::{json, Value};
+# async fn doc() -> Result<(), Box<dyn std::error::Error>> {
+# let client = Elasticsearch::default();
+let response = client
+    .cat()
+    .nodes()
+    .format("json")
+    .send()
+    .await?;
+
+let response_body = response.json::<Value>().await?;
+# Ok(())
+# }
+```
+
+Or by setting an accept header using `.headers()`
+
+```rust,no_run
+# use elasticsearch::{Elasticsearch, Error, SearchParts, http::headers::{HeaderValue, DEFAULT_ACCEPT, ACCEPT}};
+# use url::Url;
+# use serde_json::{json, Value};
+# async fn doc() -> Result<(), Box<dyn std::error::Error>> {
+# let client = Elasticsearch::default();
+let response = client
+    .cat()
+    .nodes()
+    .header(ACCEPT, HeaderValue::from_static(DEFAULT_ACCEPT))
+    .send()
+    .await?;
+
+let response_body = response.json::<Value>().await?;
+# Ok(())
+# }
+```
+
+# Column Headers
+
+The column headers to return can be controlled with `.h()`
+
+```rust,no_run
+# use elasticsearch::{Elasticsearch, Error, SearchParts};
+# use url::Url;
+# use serde_json::{json, Value};
+# async fn doc() -> Result<(), Box<dyn std::error::Error>> {
+# let client = Elasticsearch::default();
+let response = client
+    .cat()
+    .nodes()
+    .h(&["ip", "port", "heapPercent", "name"])
+    .send()
+    .await?;
+
+let response_body = response.json::<String>().await?;
+# Ok(())
+# }
+```
+

elasticsearch/docs/Indices.fn.put_mapping.md renamed to api_generator/docs/functions/Indices.put_mapping.md

@@ -0,0 +1,7 @@
+Cross-cluster Replication APIs
+
+[Enable replication of indices in remote clusters to a local cluster](https://www.elastic.co/guide/en/elasticsearch/reference/master/xpack-ccr.html).
+This functionality can be used in some common production use cases:
+
+- Disaster recovery in case a primary cluster fails. A secondary cluster can serve as a hot backup
+- Geo-proximity so that reads can be served locally

api_generator/docs/namespaces/async_search.md

@@ -0,0 +1,4 @@
+Cluster APIs
+
+[Manage settings](https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster.html),
+perform operations, and retrieve information about an Elasticsearch cluster.

api_generator/docs/namespaces/cat.md

@@ -0,0 +1,8 @@
+Dangling Index APIs
+
+If Elasticsearch encounters index data that is absent from the current cluster state,
+those indices are considered to be _dangling_. For example, this can happen if you delete
+more than `cluster.indices.tombstones.size` number of indices while an Elasticsearch node
+is offline.
+
+The dangling indices APIs can list, import and delete dangling indices.

api_generator/docs/namespaces/ccr.md

@@ -0,0 +1,6 @@
+Enrich APIs
+
+Manage [enrich policies](https://www.elastic.co/guide/en/elasticsearch/reference/master/ingest-enriching-data.html#enrich-policy)
+that can be used by the [enrich processor](https://www.elastic.co/guide/en/elasticsearch/reference/master/enrich-processor.html)
+as part of an [ingest pipeline](../ingest/index.html), to add data from your existing indices
+to incoming documents during ingest.

api_generator/docs/namespaces/cluster.md

@@ -0,0 +1,5 @@
+Graph APIs
+
+Enable extraction and summarization of information about documents and terms in Elasticsearch
+indices, [inferring relationships across documents](https://www.elastic.co/what-is/elasticsearch-graph),
+and allowing the [exploration of such relationships](https://www.elastic.co/guide/en/elasticsearch/reference/master/graph-explore-api.html).

api_generator/docs/namespaces/dangling_indices.md

@@ -0,0 +1,9 @@
+Index Lifecycle Management APIs
+
+Automate how [indices are managed over time](https://www.elastic.co/guide/en/elasticsearch/reference/master/index-lifecycle-management.html).
+Rather than simply performing management actions on indices on a set schedule, actions can be based
+on other factors such as shard size and performance requirements.
+
+Control how indices are handled as they age by attaching a lifecycle policy to the index
+template used to create them. Update the policy to modify the lifecycle of both new
+and existing indices.

api_generator/docs/namespaces/enrich.md

@@ -0,0 +1,4 @@
+Index APIs
+
+[Manage individual indices](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices.html),
+index settings, aliases, mappings, and index templates.

api_generator/docs/namespaces/graph.md

@@ -0,0 +1,17 @@
+Ingest APIs
+
+[Manage ingest pipelines](https://www.elastic.co/guide/en/elasticsearch/reference/master/ingest-apis.html).
+Ingest pipelines can be used on a node with the `ingest` role to
+pre-process documents before indexing, to apply transformations and enrich data. Transformations are performed
+by [processors](https://www.elastic.co/guide/en/elasticsearch/reference/master/ingest-processors.html)
+in the pipeline, and can include such operations as
+
+- add, remove and append fields within the document
+- point documents to the right time-based index based on a timestamp within the document
+- extract details from fields with known formats and add new fields with extracted data
+
+and many more.
+
+All nodes enable ingest by default, so any node can handle ingest tasks. Ingest pipelines can
+be conditionally executed, and failures within pipelines can be explicitly handled by defining
+processors to execute in the event of failure.

api_generator/docs/namespaces/ilm.md

@@ -0,0 +1,8 @@
+Licensing APIs
+
+[Manage Elastic Stack license](https://www.elastic.co/guide/en/elasticsearch/reference/master/licensing-apis.html), including
+
+- Retrieve, update or delete a license
+- Start a 30 day trial of the Platinum license features
+- Start indefinite use of the Basic license features
+- Get the status of trial and basic license features

api_generator/docs/namespaces/indices.md

@@ -0,0 +1,3 @@
+Migration APIs
+
+[Simplify upgrading X-Pack indices from one version to another](https://www.elastic.co/guide/en/elasticsearch/reference/master/migration-api.html).

api_generator/docs/namespaces/ingest.md

@@ -0,0 +1,3 @@
+Machine Learning APIs
+
+[Perform machine learning anomaly detection activities](https://www.elastic.co/guide/en/elasticsearch/reference/master/ml-apis.html).

api_generator/docs/namespaces/license.md

@@ -0,0 +1,4 @@
+Node APIs
+
+Manage settings, perform operations, and retrieve information about the
+[nodes in an Elasticsearch cluster](https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster.html).

api_generator/docs/namespaces/migration.md

@@ -0,0 +1,12 @@
+Security APIs
+
+[Perform security related activities](https://www.elastic.co/guide/en/elasticsearch/reference/master/security-api.html), including
+
+- Manage users
+- Manage application privileges
+- Manage roles
+- Manage role mappings
+- Manage API keys
+- Manage Bearer tokens
+- Authenticate users against an OpenID Connect or SAML authentication realm when using a
+custom web application other than Kibana

api_generator/docs/namespaces/ml.md

@@ -0,0 +1,8 @@
+Snapshot Lifecycle Management APIs
+
+[Manage policies for the time and frequency of automatic snapshots](https://www.elastic.co/guide/en/elasticsearch/reference/master/snapshot-lifecycle-management-api.html).
+Snapshot Lifecycle Management (SLM)
+is related to [Index Lifecycle Management](../ilm/index.html), however, instead of managing a lifecycle of
+actions that are performed on a single index, SLM allows configuring policies spanning multiple
+indices. Snapshot Lifecycle Management can also perform deletion of older snapshots based on a
+configurable retention policy.

api_generator/docs/namespaces/nodes.md

@@ -0,0 +1,6 @@
+Snapshot APIs
+
+[Manage snapshots taken from a running Elasticsearch cluster](https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html).
+A snapshot is a backup of individual
+indices or the entire cluster, stored in a repository on a shared filesystem or a remote repository
+on S3, HDFS, Azure, Google Cloud storage, and more.

api_generator/docs/namespaces/security.md

@@ -0,0 +1,3 @@
+SQL APIs
+
+[Execute SQL queries against Elasticsearch indices and return results in tabular format](https://www.elastic.co/guide/en/elasticsearch/reference/master/xpack-sql.html).

api_generator/docs/namespaces/slm.md

@@ -0,0 +1,3 @@
+SSL APIs
+
+[Retrieve information about the X.509 certificates used to encrypt communications in the cluster](https://www.elastic.co/guide/en/elasticsearch/reference/master/security-api-ssl.html).

api_generator/docs/namespaces/snapshot.md

@@ -0,0 +1,3 @@
+Task Management APIs
+
+[Manage tasks currently executing on one or more nodes in the cluster](https://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html).

api_generator/docs/namespaces/sql.md

@@ -0,0 +1,5 @@
+Transform APIs
+
+[Transforms ](https://www.elastic.co/guide/en/elasticsearch/reference/master/transform-apis.html)
+can be used to copy data from source indices, transforms it, and persists it into an
+entity-centric destination index.

api_generator/docs/namespaces/ssl.md

@@ -0,0 +1,4 @@
+Watcher (Alerting) APIs
+
+Enable [watching for changes or anomalies in data and perform the necessary actions in response](https://www.elastic.co/guide/en/elasticsearch/reference/master/xpack-alerting.html),
+by creating and managing watches that take action based on a met condition.

api_generator/docs/namespaces/tasks.md

@@ -0,0 +1,3 @@
+X-Pack APIs
+
+Provide general information about the installed X-Pack features and their usage.

api_generator/docs/namespaces/transform.md

@@ -28,9 +28,9 @@ use std::{
 };
 
 fn main() -> Result<(), failure::Error> {
-    // This must be run from the src root directory, with cargo run -p api_generator
+    // This must be run from the repo root directory, with cargo make generate-api
     let download_dir = fs::canonicalize(PathBuf::from("./api_generator/rest_specs"))?;
-    let generated_dir = fs::canonicalize(PathBuf::from("./elasticsearch/src/generated"))?;
+    let generated_dir = fs::canonicalize(PathBuf::from("./elasticsearch/src"))?;
     let last_downloaded_version =
         PathBuf::from("./api_generator/rest_specs/last_downloaded_version");
 
@@ -96,12 +96,22 @@ fn main() -> Result<(), failure::Error> {
         }
 
         if generate_code {
-            // delete existing generated files if the exist
-            if generated_dir.exists() {
-                fs::remove_dir_all(&generated_dir)?;
+            // Delete previously generated files
+            let mut generated = generated_dir.clone();
+            generated.push(generator::GENERATED_TOML);
+
+            if generated.exists() {
+                let files =
+                    toml::from_str::<generator::GeneratedFiles>(&fs::read_to_string(generated)?)?;
+
+                for f in files.written {
+                    let mut generated_file = generated_dir.clone();
+                    generated_file.push(f);
+                    let _ = fs::remove_file(generated_file); // ignore missing files
+                }
             }
 
-            fs::create_dir_all(&generated_dir)?;
+            // and generate!
             generator::generate(&branch, &download_dir, &generated_dir)?;
         }
     }

api_generator/docs/namespaces/watcher.md

@@ -687,7 +687,8 @@ impl<'a> RequestBuilder<'a> {
 
         let markdown_doc = {
             let mut path = docs_dir.clone();
-            path.push(format!("{}.fn.{}.md", namespace_name, name));
+            path.push("functions");
+            path.push(format!("{}.{}.md", namespace_name, name));
             if path.exists() {
                 let mut s = fs::read_to_string(&path)
                     .unwrap_or_else(|_| panic!("Could not read file at {:?}", &path));