Unify BEAT_DIR and BEAT_PATH, rename BEATNAME to BEAT_NAME (#3339)

BEAT_PATH is not consistent between community beats and official beats. Both include the beatname in the path. BEAT_PATH and BEAT_DIR were both used with the same meaning. All BEAT_DIR were not changed to BEAT_PATH. This PR breaks with existing community beats if they depend on master.

* Rename all BEAT_DIR to BEAT_PATH
* Remove notes about community beats paths as part of help now
* Fix beat URL for official beats. For filebeat it was github.com/elastic/beats/filebeat which does not work. Now links point to product page and community beats will point to Github page if not changed.
* Rename BEATNAME to BEAT_NAME for consistency with other variables

dev-tools/packer/Makefile

@@ -7,32 +7,32 @@ BEATS_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 
 makefile_abspath:=$(abspath $(lastword $(MAKEFILE_LIST)))
 packer_absdir=$(shell dirname ${makefile_abspath})
-beat_abspath=${BEATS_GOPATH}/src/${BEAT_DIR}
+beat_abspath=${BEATS_GOPATH}/src/${BEAT_PATH}
 
 
 %/deb: ${BUILD_DIR}/god-linux-386 ${BUILD_DIR}/god-linux-amd64 fpm-image
 	echo Creating DEB packages for $(@D)
-	ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_DIR=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/debian/build.sh
-	ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_DIR=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/debian/build.sh
+	ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/debian/build.sh
+	ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/debian/build.sh
 
 %/rpm: ${BUILD_DIR}/god-linux-386 ${BUILD_DIR}/god-linux-amd64 fpm-image
 	echo Creating RPM packages for $(@D)
-	ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_DIR=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/centos/build.sh
-	ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_DIR=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/centos/build.sh
+	ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/centos/build.sh
+	ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/centos/build.sh
 
 %/darwin:
 	echo Creating Darwin packages for $(@D)
-	ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_DIR=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/darwin/build.sh
+	ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/darwin/build.sh
 
 %/win:
 	echo Creating Darwin packages for $(@D)
-	ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_DIR=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/windows/build.sh
-	ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_DIR=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/windows/build.sh
+	ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/windows/build.sh
+	ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/windows/build.sh
 
 %/bin:
 	echo Creating Linux packages for $(@D)
-	ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_DIR=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/binary/build.sh
-	ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_DIR=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/binary/build.sh
+	ARCH=386 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/binary/build.sh
+	ARCH=amd64 BEAT=$(@D) BUILD_DIR=${BUILD_DIR} BEAT_PATH=$(beat_abspath) BUILDID=$(BUILDID) SNAPSHOT=$(SNAPSHOT) $(packer_absdir)/platforms/binary/build.sh
 
 .PHONY: package-dashboards
 package-dashboards:

dev-tools/packer/platforms/dashboards/build.sh

@@ -12,7 +12,7 @@ gotpl ${BASEDIR}/run.sh.j2 < ${BUILD_DIR}/settings-$runid.yml > ${BUILD_DIR}/run
 chmod +x ${BUILD_DIR}/run-$runid.sh
 
 docker run --rm -v ${BUILD_DIR}:/build \
-    -e BUILDID=$BUILDID -e SNAPSHOT=$SNAPSHOT -e RUNID=$runid -e BEATNAME=$BEATNAME \
+    -e BUILDID=$BUILDID -e SNAPSHOT=$SNAPSHOT -e RUNID=$runid -e BEAT_NAME=$BEAT_NAME \
     tudorg/fpm /build/run-$runid.sh
 
 rm ${BUILD_DIR}/settings-$runid.yml ${BUILD_DIR}/run-$runid.sh

dev-tools/packer/platforms/dashboards/run.sh.j2

@@ -10,14 +10,14 @@ if [ "$SNAPSHOT" = "yes" ]; then
     VERSION="${VERSION}-SNAPSHOT"
 fi
 
-mkdir /${BEATNAME:-beats}-dashboards-${VERSION}
-cp -a dashboards/. /${BEATNAME:-beats}-dashboards-${VERSION}/
-echo "$BUILDID" > /${BEATNAME:-beats}-dashboards-${VERSION}/.build_hash.txt
+mkdir /${BEAT_NAME:-beats}-dashboards-${VERSION}
+cp -a dashboards/. /${BEAT_NAME:-beats}-dashboards-${VERSION}/
+echo "$BUILDID" > /${BEAT_NAME:-beats}-dashboards-${VERSION}/.build_hash.txt
 
 mkdir -p upload
-zip -r upload/${BEATNAME:-beats}-dashboards-${VERSION}.zip /${BEATNAME:-beats}-dashboards-${VERSION}
-echo "Created upload/${BEATNAME:-beats}-dashboards-${VERSION}.zip"
+zip -r upload/${BEAT_NAME:-beats}-dashboards-${VERSION}.zip /${BEAT_NAME:-beats}-dashboards-${VERSION}
+echo "Created upload/${BEAT_NAME:-beats}-dashboards-${VERSION}.zip"
 
 cd upload
-sha1sum ${BEATNAME:-beats}-dashboards-${VERSION}.zip | awk '{print $1;}' > ${BEATNAME:-beats}-dashboards-${VERSION}.zip.sha1
-echo "Created upload/${BEATNAME:-beats}-dashboards-${VERSION}.zip.sha1"
+sha1sum ${BEAT_NAME:-beats}-dashboards-${VERSION}.zip | awk '{print $1;}' > ${BEAT_NAME:-beats}-dashboards-${VERSION}.zip.sha1
+echo "Created upload/${BEAT_NAME:-beats}-dashboards-${VERSION}.zip.sha1"

dev-tools/packer/xgo-scripts/before_build.sh

@@ -3,10 +3,10 @@
 set -e
 
 BEAT_PATH=/go/src/${1}
-# BEATNAME is in the $PACK variable
-BEATNAME=$PACK
+# BEAT_NAME is in the $PACK variable
+BEAT_NAME=$PACK
 
-if [ $BEATNAME = "packetbeat" ]; then
+if [ $BEAT_NAME = "packetbeat" ]; then
 	patch -p1 < /gopacket_pcap.patch
 fi
 
@@ -35,7 +35,7 @@ for TARGET in $TARGETS; do
 	XGOOS=`echo $TARGET | cut -d '/' -f 1`
 	XGOARCH=`echo $TARGET | cut -d '/' -f 2`
 
-	GOOS=$XGOOS GOARCH=$XGOARCH go build -ldflags "-X main.beat=${BEATNAME}" -o $PREFIX/import_dashboards-$XGOOS-$XGOARCH $LIBBEAT_PATH/dashboards/import_dashboards.go
+	GOOS=$XGOOS GOARCH=$XGOARCH go build -ldflags "-X main.beat=${BEAT_NAME}" -o $PREFIX/import_dashboards-$XGOOS-$XGOARCH $LIBBEAT_PATH/dashboards/import_dashboards.go
 done
 
 if [ -n "BUILDID" ]; then
@@ -57,20 +57,20 @@ sed -i -e 's/:doc-branch/doc_branch/g' ${PREFIX}/package.yml
 /go/bin/gotpl /templates/readme.md.j2 < ${PREFIX}/package.yml > ${PREFIX}/homedir/README.md
 
 # Copy template
-cp $BEATNAME.template.json $PREFIX/$BEATNAME.template.json
-cp $BEATNAME.template-es2x.json $PREFIX/$BEATNAME.template-es2x.json
+cp $BEAT_NAME.template.json $PREFIX/$BEAT_NAME.template.json
+cp $BEAT_NAME.template-es2x.json $PREFIX/$BEAT_NAME.template-es2x.json
 
 # linux
-cp $BEATNAME.yml $PREFIX/$BEATNAME-linux.yml
-cp $BEATNAME.full.yml $PREFIX/$BEATNAME-linux.full.yml
+cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-linux.yml
+cp $BEAT_NAME.full.yml $PREFIX/$BEAT_NAME-linux.full.yml
 
 # darwin
-cp $BEATNAME.yml $PREFIX/$BEATNAME-darwin.yml
-cp $BEATNAME.full.yml $PREFIX/$BEATNAME-darwin.full.yml
+cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-darwin.yml
+cp $BEAT_NAME.full.yml $PREFIX/$BEAT_NAME-darwin.full.yml
 
 # win
-cp $BEATNAME.yml $PREFIX/$BEATNAME-win.yml
-cp $BEATNAME.full.yml $PREFIX/$BEATNAME-win.full.yml
+cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-win.yml
+cp $BEAT_NAME.full.yml $PREFIX/$BEAT_NAME-win.full.yml
 
 # Contains beat specific adjustments. As it is platform specific knowledge, it should be in packer not the beats itself
 PREFIX=$PREFIX make before-build

filebeat/Makefile

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-BEATNAME?=filebeat
+BEAT_NAME?=filebeat
 BEAT_DESCRIPTION?=Filebeat sends log files to Logstash or directly to Elasticsearch.
 SYSTEM_TESTS=true
 TEST_ENVIRONMENT?=true

generate/beat/CHANGELOG.md

@@ -15,6 +15,10 @@ All notable changes to this project will be documented in this file based on the
 ### Backward Compatibility Breaks
 - Remove dependency on cookiecutter
 
+- Renamed BEAT_DIR to BEAT_PATH.
+- Renamed BEATNAME to BEAT_NAME.
+- Community beats now required BEAT_URL for packaging.
+
 ### Bugfixes
 
 ### Deprecated

generate/beat/{beat}/Makefile

@@ -1,6 +1,7 @@
-BEATNAME={beat}
-BEAT_DIR={beat_path}/{beat}
+BEAT_NAME={beat}
+BEAT_PATH={beat_path}/{beat}
 BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
+BEAT_URL=https://${BEAT_PATH}
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
 ES_BEATS?=./vendor/github.com/elastic/beats

generate/metricbeat/metricset/CHANGELOG.md

@@ -10,6 +10,10 @@ All notable changes to this project will be documented in this file based on the
 
 ### Backward Compatibility Breaks
 
+* Renamed BEAT_DIR to BEAT_PATH.
+* Renamed BEAT_NAME to BEAT_NAME.
+* Community beats now required BEAT_URL for packaging.
+
 ### Bugfixes
 
 ### Deprecated

generate/metricbeat/metricset/{{cookiecutter.beat}}/Makefile

@@ -1,5 +1,6 @@
-BEATNAME={{cookiecutter.beat}}
-BEAT_DIR={{cookiecutter.beat_path}}/{{cookiecutter.beat}}
+BEAT_NAME={{cookiecutter.beat}}
+BEAT_PATH={{cookiecutter.beat_path}}/{{cookiecutter.beat}}
+BEAT_URL=https://${BEAT_PATH}
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
 ES_BEATS?=./vendor/github.com/elastic/beats

heartbeat/Makefile

@@ -1,4 +1,4 @@
-BEATNAME=heartbeat
+BEAT_NAME=heartbeat
 BEAT_DESCRIPTION?=Ping remote services for availability and log results to Elasticsearch or send to Logstash.
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false

libbeat/Makefile

@@ -1,5 +1,5 @@
 
-BEATNAME=libbeat
+BEAT_NAME=libbeat
 TEST_ENVIRONMENT=true
 SYSTEM_TESTS=true