use prebuild

.gitignore

@@ -4,7 +4,6 @@ data/service_checks/
 integrations_data
 
 # Ignore generated examples
-/examples
 content/en/api/**/*.go
 
 # Ignoring all automated content

.gitlab-ci.yml

@@ -41,7 +41,6 @@ build_preview:
   script:
     - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
     - version_static_assets
-    - make examples/go
     - sync_integration_descriptions
     #- sync_integration_descriptions_cached
     - placehold_translations

Makefile

@@ -34,7 +34,6 @@ clean-all: stop  ## Clean everything.
 	make clean-auto-doc
 	make clean-node
 	make clean-virt
-	make clean-examples
 
 clean-build:  ## Remove build artifacts.
 	@if [ -d public ]; then rm -r public; fi
@@ -86,6 +85,10 @@ clean-integrations:  ## Remove built integrations files.
 	@find ./content/en/security_monitoring/default_rules -type f -maxdepth 1 \
 		-a -not -name '_index.md' \
 		-exec rm -rf {} \;
+	@find ./content/en/api/v1 -type f -name '*.go' -maxdepth 2 \
+		-exec rm -rf {} \;
+	@find ./content/en/api/v2 -type f -name '*.go' -maxdepth 2 \
+		-exec rm -rf {} \;
 
 clean-auto-doc: ##Remove all doc automatically created
 	@if [ -d content/en/developers/integrations ]; then \
@@ -128,7 +131,7 @@ source-helpers: hugpython  ## Source the helper functions used in build, test, d
 	@find ${LOCALBIN}/*  -type f -exec cp {} ${EXEDIR} \;
 	@cp -r local/githooks/* .git/hooks
 
-start: clean source-helpers examples/go ## Build the documentation with all external content.
+start: clean source-helpers ## Build the documentation with all external content.
 	@echo "\033[35m\033[1m\nBuilding the documentation with ALL external content:\033[0m"
 	@if [ ${PY3} != "false" ]; then \
 		source ${VIRENV}/bin/activate;  \
@@ -152,23 +155,3 @@ stop:  ## Stop wepack watch/hugo server.
 	@echo "stopping previous..."
 	@pkill -x webpack || true
 	@pkill -x hugo server --renderToDisk || true
-
-clean-examples:
-	@rm -rf examples
-	@git clean -df content/en/api/
-
-examples/datadog-api-client-go:
-	@git clone https://github.com/DataDog/datadog-api-client-go.git examples/datadog-api-client-go
-
-examples/datadog-api-client-java:
-	@git clone https://github.com/DataDog/datadog-api-client-java.git examples/datadog-api-client-java
-
-examples/go: examples/datadog-api-client-go
-	@ls examples/datadog-api-client-go/api/v1/datadog/docs/*Api.md | xargs -n1 local/bin/awk/extract-code-blocks-go.awk -v output=examples/content/en/api/v1
-	@ls examples/datadog-api-client-go/api/v2/datadog/docs/*Api.md | xargs -n1 local/bin/awk/extract-code-blocks-go.awk -v output=examples/content/en/api/v2
-
-	# for f in examples/content/en/api/v*/*/*.go ; do \
-	# 	echo gofmt -w $$f || rm $f; \
-	# done;
-
-	cp -R examples/content ./

local/bin/py/build/actions/go_examples.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+import glob
+import re
+from itertools import chain
+from os.path import basename, dirname
+from os import makedirs
+import logging
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.WARNING)
+
+
+def slug(string):
+    string = string.strip().replace("Api.md", "")
+    return re.sub(r"[A-Z][a-z]+", lambda matched: '-' + matched.group(0), string).strip('-').lower()
+
+
+def get_operation_id(string):
+    RE = re.compile(r'(^|\n)(?P<level>#{2})(?P<header>.*?)#*(\n|$)')
+    result = RE.search(string)
+    if result:
+        return result.group().replace('##', '').strip()
+    else:
+        return ""
+
+
+def go_examples(content, content_dir):
+    """
+    Takes the content from a file from a github repo and
+    pushed it to the doc
+    See https://github.com/DataDog/documentation/wiki/Documentation-Build#pull-and-push-files to learn more
+    :param content: object with a file_name, a file_path, and options to apply
+    :param content_dir: The directory where content should be put
+    """
+    logger.info("Starting go examples...")
+    regex_examples = re.compile(r"(.*)(\`\`\`go\s*)(.*)(\`\`\`)(.*)", re.DOTALL)
+    for file_name in chain.from_iterable(glob.glob(pattern, recursive=True) for pattern in content["globs"]):
+        with open(file_name, mode='r+') as f:
+            result = f.read()
+            op_id = get_operation_id(result)
+            result = re.sub(regex_examples, "\\3", result, 0)
+            version = "v1" if "v1" in file_name else "v2"
+            slugged = slug(basename(file_name))
+            dest = f"{content_dir}api/{version}/{slugged}/{op_id}.go"
+            makedirs(dirname(dest), exist_ok=True)
+            with open(dest, mode='w', encoding='utf-8') as out_file:
+                out_file.write(result)

local/bin/py/build/configurations/pull_config.yaml

@@ -242,3 +242,13 @@
         - "*.json"
       options:
         dest_path: '/security_monitoring/default_rules/'
+
+  - repo_name: datadog-api-client-go
+    contents:
+    - action: go-examples
+      branch: master
+      globs:
+        - "api/v1/datadog/docs/*Api.md"
+        - "api/v2/datadog/docs/*Api.md"
+      options:
+        dest_path: '/api/v3/'

local/bin/py/build/configurations/pull_config_preview.yaml

@@ -243,3 +243,12 @@
       options:
         dest_path: '/security_monitoring/default_rules/'
 
+  - repo_name: datadog-api-client-go
+    contents:
+    - action: go-examples
+      branch: master
+      globs:
+        - "api/v1/datadog/docs/*Api.md"
+        - "api/v2/datadog/docs/*Api.md"
+      options:
+        dest_path: '/api/v3/'

local/bin/py/build/content_manager.py

@@ -140,7 +140,7 @@ def extract_config(configuration):
                 ]
                 content_temp["globs"] = content["globs"]
 
-                if content["action"] in ("pull-and-push-folder", "pull-and-push-file", "security-rules"):
+                if content["action"] in ("pull-and-push-folder", "pull-and-push-file", "security-rules", "go-examples"):
                     content_temp["options"] = content["options"]
 
                 list_of_contents.append(

local/bin/py/build/update_pre_build.py

@@ -8,6 +8,7 @@
 from content_manager import prepare_content
 from integrations import Integrations
 from security_rules import security_rules
+from go_examples import go_examples
 
 from collections import OrderedDict
 from optparse import OptionParser
@@ -71,6 +72,8 @@ def build_documentation(self, list_of_contents):
                     pull_and_push_file(content, self.content_dir)
                 elif content["action"] == "security-rules":
                     security_rules(content, self.content_dir)
+                elif content["action"] == "go-examples":
+                    go_examples(content, self.content_dir)
                 elif content["action"] == "Not Available":
                     if getenv("LOCAL") == 'True':
                         print("\x1b[33mWARNING\x1b[0m: Processing of {} canceled, since content is not available. Documentation is in degraded mode".format(
@@ -79,10 +82,10 @@ def build_documentation(self, list_of_contents):
                     print(
                         "\x1b[31mERROR\x1b[0m: Action {} unknown for {}".format(content["action"], content))
                     raise ValueError
-            except:
+            except Exception as e:
                 if getenv("LOCAL") == 'True':
                     print(
-                        "\x1b[33mWARNING\x1b[0m: Unsuccessful processing of {}".format(content))
+                        "\x1b[33mWARNING\x1b[0m: Unsuccessful processing of {} {}".format(content, e))
                 else:
                     print(
                         "\x1b[31mERROR\x1b[0m: Unsuccessful processing of {}".format(content))