From 6ba1ddbb5595c5c190a7aaa95cca9bd7dfa0d843 Mon Sep 17 00:00:00 2001 From: "david.jones" Date: Wed, 13 May 2020 19:16:39 +0100 Subject: [PATCH] use prebuild --- .gitignore | 1 - .gitlab-ci.yml | 1 - Makefile | 27 ++--------- local/bin/py/build/actions/go_examples.py | 48 +++++++++++++++++++ .../py/build/configurations/pull_config.yaml | 10 ++++ .../configurations/pull_config_preview.yaml | 9 ++++ local/bin/py/build/content_manager.py | 2 +- local/bin/py/build/update_pre_build.py | 7 ++- 8 files changed, 78 insertions(+), 27 deletions(-) create mode 100644 local/bin/py/build/actions/go_examples.py diff --git a/.gitignore b/.gitignore index d7e5d2fe5a368..4741aa3e6c5ed 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ data/service_checks/ integrations_data # Ignore generated examples -/examples content/en/api/**/*.go # Ignoring all automated content diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b266592bd165d..30cbfba286bd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,6 @@ build_preview: script: - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info" - version_static_assets - - make examples/go - sync_integration_descriptions #- sync_integration_descriptions_cached - placehold_translations diff --git a/Makefile b/Makefile index 6190ed90d2353..d65130efba59d 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,6 @@ clean-all: stop ## Clean everything. make clean-auto-doc make clean-node make clean-virt - make clean-examples clean-build: ## Remove build artifacts. @if [ -d public ]; then rm -r public; fi @@ -86,6 +85,10 @@ clean-integrations: ## Remove built integrations files. @find ./content/en/security_monitoring/default_rules -type f -maxdepth 1 \ -a -not -name '_index.md' \ -exec rm -rf {} \; + @find ./content/en/api/v1 -type f -name '*.go' -maxdepth 2 \ + -exec rm -rf {} \; + @find ./content/en/api/v2 -type f -name '*.go' -maxdepth 2 \ + -exec rm -rf {} \; clean-auto-doc: ##Remove all doc automatically created @if [ -d content/en/developers/integrations ]; then \ @@ -128,7 +131,7 @@ source-helpers: hugpython ## Source the helper functions used in build, test, d @find ${LOCALBIN}/* -type f -exec cp {} ${EXEDIR} \; @cp -r local/githooks/* .git/hooks -start: clean source-helpers examples/go ## Build the documentation with all external content. +start: clean source-helpers ## Build the documentation with all external content. @echo "\033[35m\033[1m\nBuilding the documentation with ALL external content:\033[0m" @if [ ${PY3} != "false" ]; then \ source ${VIRENV}/bin/activate; \ @@ -152,23 +155,3 @@ stop: ## Stop wepack watch/hugo server. @echo "stopping previous..." @pkill -x webpack || true @pkill -x hugo server --renderToDisk || true - -clean-examples: - @rm -rf examples - @git clean -df content/en/api/ - -examples/datadog-api-client-go: - @git clone https://github.com/DataDog/datadog-api-client-go.git examples/datadog-api-client-go - -examples/datadog-api-client-java: - @git clone https://github.com/DataDog/datadog-api-client-java.git examples/datadog-api-client-java - -examples/go: examples/datadog-api-client-go - @ls examples/datadog-api-client-go/api/v1/datadog/docs/*Api.md | xargs -n1 local/bin/awk/extract-code-blocks-go.awk -v output=examples/content/en/api/v1 - @ls examples/datadog-api-client-go/api/v2/datadog/docs/*Api.md | xargs -n1 local/bin/awk/extract-code-blocks-go.awk -v output=examples/content/en/api/v2 - - # for f in examples/content/en/api/v*/*/*.go ; do \ - # echo gofmt -w $$f || rm $f; \ - # done; - - cp -R examples/content ./ diff --git a/local/bin/py/build/actions/go_examples.py b/local/bin/py/build/actions/go_examples.py new file mode 100644 index 0000000000000..bd1f44f11f882 --- /dev/null +++ b/local/bin/py/build/actions/go_examples.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +import glob +import re +from itertools import chain +from os.path import basename, dirname +from os import makedirs +import logging + + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.WARNING) + + +def slug(string): + string = string.strip().replace("Api.md", "") + return re.sub(r"[A-Z][a-z]+", lambda matched: '-' + matched.group(0), string).strip('-').lower() + + +def get_operation_id(string): + RE = re.compile(r'(^|\n)(?P#{2})(?P
.*?)#*(\n|$)') + result = RE.search(string) + if result: + return result.group().replace('##', '').strip() + else: + return "" + + +def go_examples(content, content_dir): + """ + Takes the content from a file from a github repo and + pushed it to the doc + See https://github.com/DataDog/documentation/wiki/Documentation-Build#pull-and-push-files to learn more + :param content: object with a file_name, a file_path, and options to apply + :param content_dir: The directory where content should be put + """ + logger.info("Starting go examples...") + regex_examples = re.compile(r"(.*)(\`\`\`go\s*)(.*)(\`\`\`)(.*)", re.DOTALL) + for file_name in chain.from_iterable(glob.glob(pattern, recursive=True) for pattern in content["globs"]): + with open(file_name, mode='r+') as f: + result = f.read() + op_id = get_operation_id(result) + result = re.sub(regex_examples, "\\3", result, 0) + version = "v1" if "v1" in file_name else "v2" + slugged = slug(basename(file_name)) + dest = f"{content_dir}api/{version}/{slugged}/{op_id}.go" + makedirs(dirname(dest), exist_ok=True) + with open(dest, mode='w', encoding='utf-8') as out_file: + out_file.write(result) diff --git a/local/bin/py/build/configurations/pull_config.yaml b/local/bin/py/build/configurations/pull_config.yaml index cbb6025e236ae..01588706eabf8 100644 --- a/local/bin/py/build/configurations/pull_config.yaml +++ b/local/bin/py/build/configurations/pull_config.yaml @@ -242,3 +242,13 @@ - "*.json" options: dest_path: '/security_monitoring/default_rules/' + + - repo_name: datadog-api-client-go + contents: + - action: go-examples + branch: master + globs: + - "api/v1/datadog/docs/*Api.md" + - "api/v2/datadog/docs/*Api.md" + options: + dest_path: '/api/v3/' diff --git a/local/bin/py/build/configurations/pull_config_preview.yaml b/local/bin/py/build/configurations/pull_config_preview.yaml index 897a6f5920619..01588706eabf8 100644 --- a/local/bin/py/build/configurations/pull_config_preview.yaml +++ b/local/bin/py/build/configurations/pull_config_preview.yaml @@ -243,3 +243,12 @@ options: dest_path: '/security_monitoring/default_rules/' + - repo_name: datadog-api-client-go + contents: + - action: go-examples + branch: master + globs: + - "api/v1/datadog/docs/*Api.md" + - "api/v2/datadog/docs/*Api.md" + options: + dest_path: '/api/v3/' diff --git a/local/bin/py/build/content_manager.py b/local/bin/py/build/content_manager.py index 662bfeba01446..42ba7c699cff7 100644 --- a/local/bin/py/build/content_manager.py +++ b/local/bin/py/build/content_manager.py @@ -140,7 +140,7 @@ def extract_config(configuration): ] content_temp["globs"] = content["globs"] - if content["action"] in ("pull-and-push-folder", "pull-and-push-file", "security-rules"): + if content["action"] in ("pull-and-push-folder", "pull-and-push-file", "security-rules", "go-examples"): content_temp["options"] = content["options"] list_of_contents.append( diff --git a/local/bin/py/build/update_pre_build.py b/local/bin/py/build/update_pre_build.py index 7cfee00fc182e..120c35933517e 100755 --- a/local/bin/py/build/update_pre_build.py +++ b/local/bin/py/build/update_pre_build.py @@ -8,6 +8,7 @@ from content_manager import prepare_content from integrations import Integrations from security_rules import security_rules +from go_examples import go_examples from collections import OrderedDict from optparse import OptionParser @@ -71,6 +72,8 @@ def build_documentation(self, list_of_contents): pull_and_push_file(content, self.content_dir) elif content["action"] == "security-rules": security_rules(content, self.content_dir) + elif content["action"] == "go-examples": + go_examples(content, self.content_dir) elif content["action"] == "Not Available": if getenv("LOCAL") == 'True': print("\x1b[33mWARNING\x1b[0m: Processing of {} canceled, since content is not available. Documentation is in degraded mode".format( @@ -79,10 +82,10 @@ def build_documentation(self, list_of_contents): print( "\x1b[31mERROR\x1b[0m: Action {} unknown for {}".format(content["action"], content)) raise ValueError - except: + except Exception as e: if getenv("LOCAL") == 'True': print( - "\x1b[33mWARNING\x1b[0m: Unsuccessful processing of {}".format(content)) + "\x1b[33mWARNING\x1b[0m: Unsuccessful processing of {} {}".format(content, e)) else: print( "\x1b[31mERROR\x1b[0m: Unsuccessful processing of {}".format(content))