Merge branch 'master' into nsollecito/lang-redirects

Author: nsollecito

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,7 +1,6 @@
 ---
 name: Feature request
-about: Suggest an idea for the Datadog documentation. To request a product feature, please [file a ticket to Datadog support team directly](https://docs.datadoghq.com/help/).
-
+about: Suggest an idea for the Datadog documentation. To request a product feature, please file a ticket to Datadog's support team directly https://docs.datadoghq.com/help/.
 ---
 
 **Is your feature request related to a problem? Please describe.**

.gitignore

@@ -4,59 +4,41 @@ data/integrations/
 data/service_checks/
 integrations_data
 
-# Ignoring all integrations file
-content/integrations/*.md
-!content/integrations/_index.md
-!content/integrations/cloudability.md
-!content/integrations/cloudcheckr.md
-!content/integrations/carbon_black.md
-!content/integrations/jenkins.md
-!content/integrations/kubernetes.md
-!content/integrations/rss.md
-!content/integrations/system.md
-!content/integrations/tcprtt.md
-!content/integrations/adobe_experience_manager.md
-!content/integrations/pivotal_platform.md
-!content/integrations/alcide.md
-!content/integrations/amazon_eks_fargate.md
-!content/integrations/iam_access_analyzer.md
-
 # Ignoring all automated content
 # Developers/integrations
-content/developers/integrations/*.md
-content/agent/basic_agent_usage/heroku.md
+content/en/developers/integrations/*.md
+content/en/developers/amazon_cloudformation.md
+
+# Agent section
+content/en/agent/basic_agent_usage/heroku.md
+
+# Tracing
 content/en/tracing/setup/ruby.md
+content/en/logs/log_collection/android.md
 
 # Generated by build script
-content/integrations/faq/list-of-api-source-attribute-value.md
 package-lock.json
 
-# new content subdir version
+# Ignoring all integrations file
 content/en/integrations/*.md
 !content/en/integrations/_index.md
 !content/en/integrations/cloudability.md
 !content/en/integrations/cloudcheckr.md
-!content/en/integrations/jenkins.md
 !content/en/integrations/kubernetes.md
 !content/en/integrations/rss.md
 !content/en/integrations/system.md
 !content/en/integrations/tcprtt.md
 !content/en/integrations/adobe_experience_manager.md
 !content/en/integrations/pivotal_platform.md
 !content/en/integrations/alcide.md
-!content/en/integrations/amazon_eks_fargate.md
 !content/en/integrations/iam_access_analyzer.md
-content/en/developers/integrations/*.md
-content/en/agent/basic_agent_usage/heroku.md
-content/en/integrations/faq/list-of-api-source-attribute-value.md
-content/en/developers/amazon_cloudformation.md
+!content/en/integrations/kubernetes_audit_logs.md
 
 # Logs
 /logs/*
 *.log
 npm-debug.log*
 
-
 # Runtime data
 pids
 *.pid

.gitlab-ci.yml

@@ -35,7 +35,8 @@ build_preview:
     URL: ${PREVIEW_DOMAIN}
     CONFIG: ${PREVIEW_CONFIG}
     MESSAGE: ":gift_heart: Your preview site is available!\nNow running tests..."
-    CONFIGURATION_FILE: "./local/etc/pull_config_preview.yaml"
+    CONFIGURATION_FILE: "./local/bin/py/build/configurations/pull_config_preview.yaml"
+    LOCAL: "False"
   script:
     - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
     - version_static_assets
@@ -106,7 +107,8 @@ build_live:
     CONFIG: ${LIVE_CONFIG}
     URL: ${LIVE_DOMAIN}
     UNTRACKED_EXTRAS: "data,content/en/integrations,content/en/agent/basic_agent_usage/heroku.md,content/en/developers/integrations,content/en/developers/amazon_cloudformation.md"
-    CONFIGURATION_FILE: "./local/etc/pull_config.yaml"
+    CONFIGURATION_FILE: "./local/bin/py/build/configurations/pull_config.yaml"
+    LOCAL: "False"
   script:
     - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
     - notify_slack "<https://github.com/DataDog/documentation/commit/${CI_COMMIT_SHA}|${CI_COMMIT_SHA:0:8}> sent to gitlab for production deployment. <${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}|details>" "#FFD700"

Makefile

@@ -21,27 +21,27 @@ include $(CONFIG_FILE)
 help:
 	@perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
 
-clean: stop  ## clean all make installs.
+clean: stop  ## Clean all make installs.
 	@echo "cleaning up..."
 	make clean-build
 	make clean-integrations
 	make clean-auto-doc
 
-clean-all: stop  ## clean everything.
+clean-all: stop  ## Clean everything.
 	make clean-build
 	make clean-exe
 	make clean-integrations
 	make clean-auto-doc
 	make clean-node
 	make clean-virt
 
-clean-build:  ## remove build artifacts.
+clean-build:  ## Remove build artifacts.
 	@if [ -d public ]; then rm -r public; fi
 
-clean-exe:  ## remove execs.
+clean-exe:  ## Remove execs.
 	@rm -rf ${EXE_LIST}
 
-clean-integrations:  ## remove built integrations files.
+clean-integrations:  ## Remove built integrations files.
 	@rm -rf ./integrations_data/
 	@if [ -d data/integrations ]; then \
 		find ./data/integrations -type f -maxdepth 1 \
@@ -54,10 +54,9 @@ clean-integrations:  ## remove built integrations files.
 	@find ./content/en/integrations -type f -maxdepth 1 \
 	    -a -not -name '_index.md' \
 		-a -not -name 'adobe_experience_manager.md' \
+		-a -not -name 'kubernetes_audit_logs.md' \
 		-a -not -name 'alcide.md' \
 	    -a -not -name 'amazon_guardduty.md' \
-	    -a -not -name 'amazon_vpc.md' \
-			-a -not -name 'amazon_eks_fargate.md' \
 	    -a -not -name 'amazon_cloudhsm.md' \
 	    -a -not -name 'pivotal_platform.md' \
 			-a -not -name 'carbon_black.md' \
@@ -66,10 +65,8 @@ clean-integrations:  ## remove built integrations files.
 		  -a -not -name 'fluentbit.md' \
 			-a -not -name 'iam_access_analyzer.md' \
 	    -a -not -name 'integration_sdk.md' \
-	    -a -not -name 'jenkins.md' \
 	    -a -not -name 'journald.md' \
 	    -a -not -name 'kubernetes.md' \
-			-a -not -name 'lacework.md' \
 	    -a -not -name 'marklogic.md' \
 	    -a -not -name 'nxlog.md' \
 	    -a -not -name 'rss.md' \
@@ -84,7 +81,7 @@ clean-integrations:  ## remove built integrations files.
 	    -a -not -name '*.fr.md' \
 	    -exec rm -rf {} \;
 
-clean-auto-doc: ##remove all doc automatically created
+clean-auto-doc: ##Remove all doc automatically created
 	@if [ -d content/en/developers/integrations ]; then \
 	find ./content/en/developers/integrations -type f -maxdepth 1 -exec rm -rf {} \; ;fi
 	@if [ content/en/agent/basic_agent_usage/heroku.md ]; then \
@@ -93,43 +90,49 @@ clean-auto-doc: ##remove all doc automatically created
 	rm -f content/en/tracing/setup/ruby.md ;fi
 	@if [ content/en/developers/amazon_cloudformation.md ]; then \
 	rm -f content/en/developers/amazon_cloudformation.md ;fi
+	@if [ content/en/logs/log_collection/android.md ]; then \
+	rm -f content/en/logs/log_collection/android.md ;fi
 
-clean-node:  ## remove node_modules.
+clean-node:  ## Remove node_modules.
 	@if [ -d node_modules ]; then rm -r node_modules; fi
 
-clean-virt:  ## remove python virtual env.
+clean-virt:  ## Remove python virtual env.
 	@if [ -d ${VIRENV} ]; then rm -rf $(VIRENV); fi
 
-hugpython: hugpython/bin/activate  ## build virtualenv used for tests.
+hugpython: hugpython/bin/activate  ## Build virtualenv used for tests.
 
-hugpython/bin/activate: local/etc/requirements3.txt  ## start python virtual environment.
+hugpython/bin/activate: local/etc/requirements3.txt  ## Start python virtual environment.
 	@if [ ${PY3} != "false" ]; then \
 		test -x ${VIRENV}/bin/pip || ${PY3} -m venv --clear ${VIRENV}; \
 		$(VIRENV)/bin/pip install -r local/etc/requirements3.txt; \
 	else printf "\e[93mPython 3 is required to fetch integrations and run tests.\033[0m Try https://github.com/pyenv/pyenv.\n"; fi
 
-source-helpers: hugpython  ## source the helper functions used in build, test, deploy.
+source-helpers: hugpython  ## Source the helper functions used in build, test, deploy.
 	@mkdir -p ${EXEDIR}
 	@find ${LOCALBIN}/*  -type f -exec cp {} ${EXEDIR} \;
 	@cp -r local/githooks/* .git/hooks
 
-start: clean source-helpers ## start the webpack/hugo server.
-	@echo "starting up..."
+start: clean source-helpers ## Build the documentation with all external content.
+	@echo "\033[35m\033[1m\nBuilding the documentation with ALL external content:\033[0m"
 	@if [ ${PY3} != "false" ]; then \
 		source ${VIRENV}/bin/activate;  \
-		FETCH_INTEGRATIONS=${FETCH_INTEGRATIONS} \
 		GITHUB_TOKEN=${GITHUB_TOKEN} \
 		RUN_SERVER=${RUN_SERVER} \
 		CREATE_I18N_PLACEHOLDERS=${CREATE_I18N_PLACEHOLDERS} \
 		CONFIGURATION_FILE=${CONFIGURATION_FILE} \
+		LOCAL=${LOCAL}\
 		run-site.sh; \
-	else \
-		FETCH_INTEGRATIONS="false" \
-		CREATE_I18N_PLACEHOLDERS="false" \
+	else @echo "\033[31m\033[1mPython 3 must be available to Build the documentation.\033[0m" ; fi
+
+start-no-pre-build: clean source-helpers ## Build the documentation without automatically pulled content.
+	@echo "\033[35m\033[1m\nBuilding the documentation with NO external content:\033[0m"
+	@if [ ${PY3} != "false" ]; then \
+		source ${VIRENV}/bin/activate;  \
 		RUN_SERVER=${RUN_SERVER} \
-		run-site.sh; fi
+		run-site-no-pre-build.sh; \
+	else @echo "\033[31m\033[1mPython 3 must be available to Build the documentation.\033[0m" ; fi
 
-stop:  ## stop wepack watch/hugo server.
+stop:  ## Stop wepack watch/hugo server.
 	@echo "stopping previous..."
 	@pkill -x webpack || true
 	@pkill -x hugo server --renderToDisk || true

Makefile.config.example

@@ -1,15 +1,16 @@
 # Makefile.config
 # Create a new file called Makefile.config using this file as a template for your local config
 
-CREATE_I18N_PLACEHOLDERS := false
-FETCH_INTEGRATIONS := true
 GITHUB_TOKEN := ${github_personal_token}
+CREATE_I18N_PLACEHOLDERS:= false
 RUN_SERVER := true
 RUN_WEBPACK := true
-CONFIGURATION_FILE := "./local/etc/pull_config_preview.yaml"
+CONFIGURATION_FILE := "./local/bin/py/build/configurations/pull_config_preview.yaml"
 
+LOCAL := True
 LOCALBIN := local/bin
 LOCALETC := local/etc
+
 EXEDIR := /usr/local/bin
 EXE_LIST := `find local/bin -type f -exec sh -c "echo ${EXEDIR}/{} | sed s@${LOCALBIN}/@@" \;`
 VIRENV=hugpython

README.md

@@ -3,27 +3,31 @@
 Built with [hugo][1], a static website generation tool.
 
 ## Setup
+
 ### Installation
 
 1. [Install npm][2]
 
 2. [Install Python3][3] (you can also use [pyenv][4])
 
-3. Install yarn: ```npm install -g yarn```
+3. Install yarn: `npm install -g yarn`
 
-4. Download the documentation repo ```git clone https://github.com/DataDog/documentation.git```
+4. Download the documentation repo `git clone https://github.com/DataDog/documentation.git`
 
 ### Run the server
 
 Inside `documentation/` folder, create a `Makefile.config` file from the [Makefile.config.example][5]
 
 If you are a Datadog employee, add your [Github personal token][6]
 
-To run the site and perform administrative tasks (compile metrics, create i18n placeholders, etc), just execute:
+To run the documentation site locally, execute:
 
-`make start`
+| Command                   | Description                                                                                                                                                                                                                             |
+|---------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `make start-no-pre-build` | Build the lightweight version of the documentation with no extra content                                                                                                                                                                |
+| `make start`              | Build the full documentation with all extra content (integrations, extra pulled files, localised content...). Only useful if you have a Github personal token setup in your `Makefile.config` or the extra content is available localy. |
 
-Documentation is available at `http://localhost:1313`
+**Documentation is then available at `http://localhost:1313`**
 
 To learn more about how the documentation is build refer to the [Documentation Build Wiki][7].
 
@@ -33,17 +37,19 @@ To use the Makefile, create a Makefile.config. See the instructions at the top o
 
 After you have a config file you can run `make help` to see options:
 
-```
-clean-build               remove build artifacts.
-clean-exe                 remove execs.
-clean-integrations        remove built integrations files.
-clean-node                remove node_modules.
-clean-virt                remove python virtual env.
-clean                     clean all make installs.
-hugpython                 build virtualenv used for tests.
-source-helpers            source the helper functions used in build, test, deploy.
-start                     start the gulp/hugo server.
-stop                      stop the gulp/hugo server.
+```text
+clean-all                 Clean everything.
+clean-build               Remove build artifacts.
+clean-exe                 Remove execs.
+clean-integrations        Remove built integrations files.
+clean-node                Remove node_modules.
+clean-virt                Remove python virtual env.
+clean                     Clean all make installs.
+hugpython                 Build virtualenv used for tests.
+source-helpers            Source the helper functions used in build, test, deploy.
+start-no-pre-build        Build the documentation without automatically pulled content.
+start                     Build the documentation with all external content.
+stop                      Stop wepack watch/hugo server.
 ```
 
 ## Working on Docs
@@ -63,16 +69,14 @@ stop                      stop the gulp/hugo server.
 
 ### A note about markdown
 
-This site uses Blackfriday for markdown. To learn about the syntax, [see this site][9].
+This site uses [Goldmark][9] for markdown which is compliant with [CommonMark 0.29][10].
 
-If you include ANY Markdown in a file, give it an .md extension.
+If you include ANY Markdown in a file, give it an `.md` extension.
 
 Make sure all files are lowercase. Macs are case insensitive when creating links to images and pages, but the server is not so tests may be fine locally but the site will fail in production.
 
 ## Releasing
 
-If you receive an error regarding `There was a problem getting GitHub Metrics`, please see the [Github personal access token][10].
-
 Within 5 minutes of merging to master, it deploys automatically.
 
 ## How to add a new integration
@@ -87,6 +91,6 @@ Within 5 minutes of merging to master, it deploys automatically.
 [6]: https://github.com/DataDog/documentation/wiki/Github-personal-token
 [7]: https://github.com/DataDog/documentation/wiki/Documentation-Build
 [8]: https://github.com/DataDog/documentation/blob/master/CONTRIBUTING.md
-[9]: https://github.com/russross/blackfriday
-[10]: https://github.com/DataDog/documentation/wiki/Github-personal-token
+[9]: https://github.com/yuin/goldmark
+[10]: https://spec.commonmark.org/0.29/
 [11]: https://docs.datadoghq.com/developers/integrations

config/_default/config.yaml

@@ -9,12 +9,21 @@ pygmentsCodeFences: true
 # taxonomies
 taxonomies:
   tag: "tags"
-  video: "videos"
-preserveTaxonomyNames: true
+  category: "categories"
+  "video-category": "video-categories"
 
 # language
 defaultContentLanguage: "en"
 defaultContentLanguageInSubdir: false
 
 # set this as a default otherwise we get "Error: found overlapping content dirs"
 contentDir: "content/en"
+
+# gitinfo vars
+enableGitInfo: true
+
+# Allow to render HTML in markdown files
+markup:
+  defaultMarkdownHandler: blackfriday
+  highlight:
+    guessSyntax: true