Add build docs script (elastic#1873)

To build the docs, `make docs` can be used. To preview the docs, `make docs-preview` can be used.

* Building the docs is now part of make testsuite
* Make local docs build and preview easy by adding it to the makefile
* Update CONTRIBUTING.md with new commands

The generators are currently not part of the build as they require glide to be installed.

CONTRIBUTING.md

@@ -101,6 +101,15 @@ Running the testsuite has the following requirements:
 * Docker >=1.10.0
 * Docker-compose >= 1.7.0
 
+
+## Documentation
+
+The documentation for each beat can be found in {beatname}/docs and are based on asciidoc. After every change of the
+docs it should be verified that the docs are still building to not break the automated docs build. To build the docs
+run `make docs`. In case you want to preview the docs for a beat, run `make docs-preview` inside the beats folder.
+This will automatically open your browser with the docs for preview.
+
+
 ## Dependencies
 
 To manage the `vendor/` folder we use

Makefile

@@ -3,11 +3,12 @@ COVERAGE_DIR=${BUILD_DIR}/coverage
 BEATS=packetbeat topbeat filebeat winlogbeat metricbeat
 PROJECTS=libbeat ${BEATS}
 
-# Runs complete testsuites (unit, system, integration) for all beats,
-# with coverage and race detection.
+# Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection.
+# Also it builds the docs and the generators
 .PHONY: testsuite
 testsuite:
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) testsuite || exit 1;)
+	#$(MAKE) -C generate test
 
 # Runs unit and system tests without coverage and race detection.
 .PHONY: test
@@ -38,9 +39,9 @@ update:
 
 .PHONY: clean
 clean:
+	rm -rf build
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) clean || exit 1;)
-	$(MAKE) -C generate/beat clean
-	$(MAKE) -C generate/metricbeat/metricset clean
+	$(MAKE) -C generate clean
 
 .PHONY: check
 check:
@@ -60,3 +61,7 @@ beats-dashboards:
 	mkdir -p build
 	$(foreach var,$(PROJECTS),cp -r $(var)/etc/kibana/ build/dashboards  || exit 1;)
 
+# Builds the documents for each beat
+.PHONY: docs
+docs:
+	sh libbeat/scripts/build_docs.sh ${PROJECTS}

generate/Makefile

@@ -0,0 +1,12 @@
+
+# Runs test build for the count metricset
+.PHONY: test
+test:
+	$(MAKE) -C beat test
+	$(MAKE) -C metricbeat/metricset test
+
+# Cleans up environment
+.PHONY: clean
+clean:
+	$(MAKE) -C beat clean
+	$(MAKE) -C metricbeat/metricset clean

libbeat/scripts/Makefile

@@ -234,6 +234,18 @@ update: python-env
 	. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_template.py $(PWD) ${BEATNAME} ${ES_BEATS}
 	. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_template.py --es2x $(PWD) ${BEATNAME} ${ES_BEATS}
 
+# Builds the documents for the beat
+.PHONY: docs
+docs:
+	sh ${ES_BEATS}/libbeat/scripts/build_docs.sh ${BEATNAME}
+
+# Preview the documents for the beat in the browser
+.PHONY: docs-preview
+docs-preview:
+	if [ ! -d "build/docs" ]; then $(MAKE) docs; fi;
+	${BUILD_DIR}/docs/build_docs.pl --chunk=1 --web -open chunk=1 -open --doc ${GOPATH}/src/github.com/elastic/beats/${BEATNAME}/docs/index.asciidoc -out ${BUILD_DIR}/html_docs
+
+
 ### KIBANA FILES HANDLING ###
 ES_URL?=http://localhost:9200
 
@@ -271,7 +283,6 @@ write-environment:
 	echo "ES_USER=beats" >> ${BUILD_DIR}/test.env
 	echo "ES_PASS=testing" >> ${BUILD_DIR}/test.env
 
-
 ### Packaging targets ####
 
 # Installs the files that need to get to the home path on installations

libbeat/scripts/build_docs.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Checks if docs clone already exists
+if [ ! -d "build/docs" ]; then
+    # Only head is cloned
+    git clone --depth=1 https://github.com/elastic/docs.git build/docs
+else
+    echo "build/docs already exists. Not cloning."
+fi
+
+# beatnames must be passed as parameters. Example: packetbeat filebeat
+for name in $*
+do
+  index="$GOPATH/src/github.com/elastic/beats/${name}/docs/index.asciidoc"
+  echo $index
+  if [ -f "$index" ]; then
+    echo "Building docs for ${name}..."
+    dest_dir="build/html_docs/${name}"
+    mkdir -p "$dest_dir"
+    ./build/docs/build_docs.pl --doc "$index" -out "$dest_dir"
+  fi
+done

metricbeat/Makefile

@@ -30,8 +30,8 @@ fields:
 	. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/metricbeat/scripts/fields_collector.py >> etc/fields.yml
 
 # Collects all module docs
-.PHONY: docs
-docs: python-env
+.PHONY: collect-docs
+collect-docs: python-env
 	-rm -r docs/modules
 	mkdir -p docs/modules
 	. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/metricbeat/scripts/docs_collector.py
@@ -54,7 +54,7 @@ before-build:
 
 # Runs all collection steps and updates afterwards
 .PHONY: collect
-collect: fields docs configs update imports kibana
+collect: fields collect-docs configs update imports kibana
 
 # Creates a new metricset. Requires the params MODULE and METRICSET
 .PHONY: create-metricset