Backport x-codeSamples changes to 8.19 (#4603)

* Add request details to examples (#4445)

* add request heading line to request examples

* add request heading line to response examples

* changes to the compiler to include the request heading

* add example requests for endpoints that do not use bodies (#4489)

* add a few example requests for endpoints that do not use bodies

* Remove x-codeSamples from overlay

* restructure existing examples w/o bodies as request examples

* skip examples w/o bodies when generating openapi examples

* add a lot of missing examples

* remove duplicate test

* more examples missed in previous commit

---------

Co-authored-by: lcawl <lcawley@elastic.co>

* add language client examples to YAML files (#4514)

* expand model to include language versions of examples

* add language client examples to YAML files

* use $ELASTICSEARCH_URL in curl examples

* remove unnecessary generate-language-examples task from the openapi generation

* upgrade request converter to latest

* Only add x-codeSamples extensions to docs builds of openapi (#4582)

* address example errors

---------

Co-authored-by: lcawl <lcawley@elastic.co>

Author: miguelgrinberg

Makefile

@@ -40,6 +40,7 @@ setup:	## Install dependencies for contrib target
 	@npm install --prefix compiler
 	@npm install --prefix typescript-generator
 	@npm install @redocly/cli
+	@npm install --prefix docs/examples
 
 clean-dep:	## Clean npm dependencies
 	@rm -rf compiler/node_modules
@@ -53,7 +54,9 @@ transform-to-openapi: ## Generate the OpenAPI definition from the compiled schem
 	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --output output/openapi/elasticsearch-serverless-openapi.json
 
 transform-to-openapi-for-docs: ## Generate the OpenAPI definition tailored for API docs generation
-	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --output output/openapi/elasticsearch-openapi-docs.json
+	@make generate-language-examples
+	@make generate
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --include-language-examples --output output/openapi/elasticsearch-openapi-docs.json
 
 filter-for-serverless: ## Generate the serverless version from the compiled schema
 	@npm run --prefix compiler filter-by-availability -- --serverless --visibility=public --input ../output/schema/schema.json --output ../output/output/openapi/elasticsearch-serverless-openapi.json
@@ -67,6 +70,10 @@ overlay-docs: ## Apply overlays to OpenAPI documents
 	@npx @redocly/cli bundle output/openapi/elasticsearch-openapi.tmp2.json --ext json -o output/openapi/elasticsearch-openapi.examples.json
 	rm output/openapi/elasticsearch-openapi.tmp*.json
 
+generate-language-examples:
+	@node docs/examples/generate-language-examples.js
+	@npm run format:fix-examples --prefix compiler
+
 lint-docs: ## Lint the OpenAPI documents after overlays
 	@npx @redocly/cli lint "output/openapi/elasticsearch-*.json" --config "docs/linters/redocly.yaml" --format stylish --max-problems 500
 

compiler-rs/clients_schema/src/lib.rs

@@ -484,19 +484,28 @@ impl TypeDefinition {
 
 /// The Example type is used for both requests and responses.
 ///
-/// This type definition is taken from the OpenAPI spec
+/// This type definition is based on the OpenAPI spec
 ///     https://spec.openapis.org/oas/v3.1.0#example-object
-/// with the exception of using String as the 'value' type.
+/// with the exception of using String as the 'value' type,
+/// and some custom additions.
 ///
 /// The OpenAPI v3 spec also defines the 'Example' type, so
 /// to distinguish them, this type is called SchemaExample.
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ExampleAlternative {
+    pub language: String,
+    pub code: String,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SchemaExample {
     pub summary: Option<String>,
+    pub method_request: Option<String>,
     pub description: Option<String>,
     pub value: Option<String>,
     pub external_value: Option<String>,
+    pub alternatives: Option<Vec<ExampleAlternative>>,
 }
 
 /// Common attributes for all type definitions

compiler-rs/clients_schema_to_openapi/src/cli.rs

@@ -35,6 +35,10 @@ pub struct Cli {
     /// output a redirection map when merging multipath endpoints
     #[argh(switch)]
     pub multipath_redirects: bool,
+
+    /// include the x-codeSamples extension with language examples for all endpoints
+    #[argh(switch)]
+    pub include_language_examples: bool,
 }
 
 impl Cli {
@@ -74,6 +78,7 @@ impl From<Cli> for Configuration {
             lift_enum_descriptions: cli.lift_enum_descriptions,
             merge_multipath_endpoints: cli.merge_multipath_endpoints,
             multipath_redirects: cli.multipath_redirects,
+            include_language_examples: cli.include_language_examples,
             namespaces: if cli.namespace.is_empty() {
                 None
             } else {

compiler-rs/clients_schema_to_openapi/src/lib.rs

@@ -44,6 +44,9 @@ pub struct Configuration {
 
     /// Should we output a redirect map when merging multipath endpoints?
     pub multipath_redirects: bool,
+
+    /// include the x-codeSamples extension with language examples for all endpoints
+    pub include_language_examples: bool,
 }
 
 pub struct OpenApiConversion {

compiler-rs/clients_schema_to_openapi/src/paths.rs

@@ -148,14 +148,16 @@ pub fn add_endpoint(
                 // }
             };
 
-            let openapi_example = Example {
-                value: example,
-                description: schema_example.description.clone(),
-                summary: schema_example.summary.clone(),
-                external_value: None,
-                extensions: Default::default(),
-            };
-            openapi_examples.insert(name.clone(), ReferenceOr::Item(openapi_example));
+            if example.is_some() {
+                let openapi_example = Example {
+                    value: example,
+                    description: schema_example.description.clone(),
+                    summary: schema_example.summary.clone(),
+                    external_value: None,
+                    extensions: Default::default(),
+                };
+                openapi_examples.insert(name.clone(), ReferenceOr::Item(openapi_example));
+            }
         }
         openapi_examples
     }
@@ -202,7 +204,7 @@ pub fn add_endpoint(
     // If this endpoint response has examples in schema.json, convert them to the
     // OpenAPI format and add them to the endpoint response in the OpenAPI document.
     let response_examples = if let Some(examples) = &response_def.examples {
-         get_openapi_examples(examples)
+        get_openapi_examples(examples)
     } else {
         IndexMap::new()
     };
@@ -343,8 +345,34 @@ pub fn add_endpoint(
 
         // add the x-state extension for availability
         let mut extensions = crate::availability_as_extensions(&endpoint.availability, &tac.config.flavor);
-        let mut ext_availability = crate::availability_as_extensions(&endpoint.availability, &tac.config.flavor);
-        extensions.append(&mut ext_availability);
+
+        if tac.config.include_language_examples {
+            // add the x-codeSamples extension
+            let mut code_samples = vec![];
+            if let Some(examples) = request.examples.clone() {
+                if let Some((_, example)) = examples.first() {
+                    let request_line = example.method_request.clone().unwrap_or(String::from(""));
+                    let request_body = example.value.clone().unwrap_or(String::from(""));
+                    if !request_line.is_empty() {
+                        code_samples.push(serde_json::json!({
+                            "lang": "Console",
+                            "source": request_line + "\n" + request_body.as_str(),
+                        }));
+                    }
+                    if let Some(alternatives) = example.alternatives.clone() {
+                        for alternative in alternatives.iter() {
+                            code_samples.push(serde_json::json!({
+                                "lang": alternative.language,
+                                "source": alternative.code.as_str(),
+                            }));
+                        }
+                    }
+                }
+            }
+            if !code_samples.is_empty() {
+                extensions.insert("x-codeSamples".to_string(), serde_json::json!(code_samples));
+            }
+        }
 
         // Create the operation, it will be repeated if we have several methods
         let operation = openapiv3::Operation {
@@ -369,7 +397,7 @@ pub fn add_endpoint(
             deprecated: endpoint.deprecation.is_some(),
             security: None,
             servers: vec![],
-            extensions
+            extensions,
         };
 
 

compiler-rs/compiler-wasm-lib/pkg/compiler_wasm_lib_bg.wasm

@@ -8,6 +8,7 @@
     "lint:fix": "ts-standard --fix src",
     "format:check": "prettier --config .prettierrc.json --loglevel warn --check ../specification/",
     "format:fix": "prettier --config .prettierrc.json --loglevel warn --write ../specification/",
+    "format:fix-examples": "prettier --config .prettierrc.json --loglevel warn --write ../specification/**/*.yaml",
     "generate-schema": "ts-node src/index.ts",
     "transform-expand-generics": "ts-node src/transform/expand-generics.ts",
     "transform-to-openapi": "ts-node src/transform/schema-to-openapi.ts",

compiler/package.json

@@ -260,6 +260,14 @@ export class Interface extends BaseType {
   variants?: Container
 }
 
+/**
+ * An alternative of an example, coded in a given language.
+ */
+export class ExampleAlternative {
+  language: string
+  code: string
+}
+
 /**
  * The Example type is used for both requests and responses
  * This type definition is taken from the OpenAPI spec
@@ -271,10 +279,14 @@ export class Example {
   summary?: string
   /** Long description. */
   description?: string
+  /** request method and URL */
+  method_request?: string
   /** Embedded literal example. Mutually exclusive with `external_value` */
   value?: string
   /** A URI that points to the literal example */
   external_value?: string
+  /** An array of alternatives for this example in other languages */
+  alternatives?: ExampleAlternative[]
 }
 
 /**

compiler/src/model/metamodel.ts

@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { parseDocument: yamlParseDocument } = require('yaml');
+const { convertRequests, loadSchema } = require('@elastic/request-converter');
+
+const LANGUAGES = ['Python', 'JavaScript', 'Ruby', 'PHP', 'curl'];
+
+async function generateLanguages(example) {
+  const doc = yamlParseDocument(await fs.promises.readFile(example, 'utf8'));
+  const data = doc.toJS();
+  let request = data.method_request;
+  if (data.value) {
+    if (typeof data.value === 'string') {
+      request += '\n' + data.value;
+    }
+    else {
+      request += '\n' + JSON.stringify(data.value);
+    }
+  }
+  const alternatives = [];
+  for (const lang of LANGUAGES) {
+    alternatives.push({
+      language: lang,
+      code: (await convertRequests(request, lang, {})).trim(),
+    });
+  }
+  data.alternatives = alternatives.concat((data.alternatives ?? []).filter(pair => !LANGUAGES.includes(pair.language)));
+  doc.delete('alternatives');
+  doc.add(doc.createPair('alternatives', data.alternatives));
+  await fs.promises.writeFile(example, doc.toString({lineWidth: 132}));
+}
+
+async function* walkExamples(dir) {
+  for await (const d of await fs.promises.opendir(dir)) {
+    const entry = path.join(dir, d.name);
+    if (d.isDirectory()) {
+      yield* walkExamples(entry);
+    }
+    else if (d.isFile() && entry.includes('/examples/request/') && entry.endsWith('.yaml')) {
+      yield entry;
+    }
+  }
+}
+
+async function main() {
+  let count = 0;
+  let errors = 0;
+  await loadSchema('output/schema/schema.json');
+  for await (const example of walkExamples('./specification/')) {
+    try {
+      await generateLanguages(example);
+    }
+    catch (err) {
+      console.log(`${example}: ${err}`);
+      errors += 1;
+    }
+    count += 1;
+  }
+  console.log(`${count} examples processed, ${errors} errors.`);
+}
+
+main();