Fix make update

dev-tools/mage/check.go

@@ -0,0 +1,180 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/magefile/mage/mg"
+	"github.com/magefile/mage/sh"
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/processors/dissect"
+)
+
+// Check looks for created/modified/deleted/renamed files and returns an error
+// if it finds any modifications. If executed in in verbose mode it will write
+// the results of 'git diff' to stdout to indicate what changes have been made.
+//
+// It also checks the file permissions of nosetests test cases and YAML files.
+func Check() error {
+	fmt.Println(">> check: Checking for modified files or incorrect permissions")
+
+	mg.Deps(CheckNosetestsNotExecutable, CheckYAMLNotExecutable)
+
+	changes, err := GitDiffIndex()
+	if err != nil {
+		return errors.Wrap(err, "failed to diff the git index")
+	}
+
+	if len(changes) > 0 {
+		if mg.Verbose() {
+			GitDiff()
+		}
+
+		return errors.Errorf("some files are not up-to-date. "+
+			"Run 'mage fmt update' then review and commit the changes. "+
+			"Modified: %v", changes)
+	}
+	return nil
+}
+
+// GitDiffIndex returns a list of files that differ from what is committed.
+// These could file that were created, deleted, modified, or moved.
+func GitDiffIndex() ([]string, error) {
+	// Ensure the index is updated so that diff-index gives accurate results.
+	if err := sh.Run("git", "update-index", "-q", "--refresh"); err != nil {
+		return nil, err
+	}
+
+	// git diff-index provides a list of modified files.
+	// https://www.git-scm.com/docs/git-diff-index
+	out, err := sh.Output("git", "diff-index", "HEAD", "--", ".")
+	if err != nil {
+		return nil, err
+	}
+
+	// Example formats.
+	// :100644 100644 bcd1234... 0123456... M file0
+	// :100644 100644 abcd123... 1234567... R86 file1 file3
+	d, err := dissect.New(":%{src_mode} %{dst_mode} %{src_sha1} %{dst_sha1} %{status}\t%{paths}")
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse lines.
+	var modified []string
+	s := bufio.NewScanner(bytes.NewBufferString(out))
+	for s.Scan() {
+		m, err := d.Dissect(s.Text())
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to dissect git diff-index output")
+		}
+
+		paths := strings.Split(m["paths"], "\t")
+		if len(paths) > 1 {
+			modified = append(modified, paths[1])
+		} else {
+			modified = append(modified, paths[0])
+		}
+	}
+	if err = s.Err(); err != nil {
+		return nil, err
+	}
+
+	return modified, nil
+}
+
+// GitDiff runs 'git diff' and writes the output to stdout.
+func GitDiff() error {
+	c := exec.Command("git", "--no-pager", "diff", "--minimal")
+	c.Stdin = nil
+	c.Stdout = os.Stdout
+	c.Stderr = os.Stderr
+	log.Println("exec:", strings.Join(c.Args, " "))
+	err := c.Run()
+	return err
+}
+
+// CheckNosetestsNotExecutable checks that none of the nosetests files are
+// executable. Nosetests silently skips executable .py files and we don't want
+// this to happen.
+func CheckNosetestsNotExecutable() error {
+	if runtime.GOOS == "windows" {
+		// Skip windows because it doesn't have POSIX permissions.
+		return nil
+	}
+
+	tests, err := FindFiles(nosetestsTestFiles...)
+	if err != nil {
+		return err
+	}
+
+	var executableTestFiles []string
+	for _, file := range tests {
+		info, err := os.Stat(file)
+		if err != nil {
+			return err
+		}
+
+		if info.Mode().Perm()&0111 > 0 {
+			executableTestFiles = append(executableTestFiles, file)
+		}
+	}
+
+	if len(executableTestFiles) > 0 {
+		return errors.Errorf("nosetests files cannot be executable because "+
+			"they will be skipped. Fix permissions of %v", executableTestFiles)
+	}
+	return nil
+}
+
+// CheckYAMLNotExecutable checks that no .yml or .yaml files are executable.
+func CheckYAMLNotExecutable() error {
+	if runtime.GOOS == "windows" {
+		// Skip windows because it doesn't have POSIX permissions.
+		return nil
+	}
+
+	executableYAMLFiles, err := FindFilesRecursive(func(path string, info os.FileInfo) bool {
+		switch filepath.Ext(path) {
+		default:
+			return false
+		case ".yml", ".yaml":
+			return info.Mode().Perm()&0111 > 0
+		}
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed search for YAML files")
+	}
+
+	if len(executableYAMLFiles) > 0 {
+		return errors.Errorf("YAML files cannot be executable. Fix "+
+			"permissions of %v", executableYAMLFiles)
+
+	}
+	return nil
+}

dev-tools/mage/common.go

@@ -147,12 +147,12 @@ func expandFile(src, dst string, args ...map[string]interface{}) error {
 }
 
 // CWD return the current working directory.
-func CWD() string {
+func CWD(elem ...string) string {
 	wd, err := os.Getwd()
 	if err != nil {
 		panic(errors.Wrap(err, "failed to get the CWD"))
 	}
-	return wd
+	return filepath.Join(append([]string{wd}, elem...)...)
 }
 
 // EnvOr returns the value of the specified environment variable if it is
@@ -215,6 +215,13 @@ func dockerInfo() (*DockerInfo, error) {
 	return &info, nil
 }
 
+// HaveDockerCompose returns an error if docker-compose is not found on the
+// PATH.
+func HaveDockerCompose() error {
+	_, err := exec.LookPath("docker-compose")
+	return errors.Wrap(err, "docker-compose was not found on the PATH")
+}
+
 // FindReplace reads a file, performs a find/replace operation, then writes the
 // output to the same file path.
 func FindReplace(file string, re *regexp.Regexp, repl string) error {
@@ -513,6 +520,29 @@ func FindFiles(globs ...string) ([]string, error) {
 	return configFiles, nil
 }
 
+// FindFilesRecursive recursively traverses from the CWD and invokes the given
+// match function on each regular file to determine if the given path should be
+// returned as a match.
+func FindFilesRecursive(match func(path string, info os.FileInfo) bool) ([]string, error) {
+	var matches []string
+	err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !info.Mode().IsRegular() {
+			// continue
+			return nil
+		}
+
+		if match(filepath.ToSlash(path), info) {
+			matches = append(matches, path)
+		}
+		return nil
+	})
+	return matches, err
+}
+
 // FileConcat concatenates files and writes the output to out.
 func FileConcat(out string, perm os.FileMode, files ...string) error {
 	f, err := os.OpenFile(createDir(out), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm)
@@ -646,23 +676,6 @@ func IsUpToDate(dst string, sources ...string) bool {
 	return err == nil && !execute
 }
 
-// OSSBeatDir returns the OSS beat directory. You can pass paths and they will
-// be joined and appended to the OSS beat dir.
-func OSSBeatDir(path ...string) string {
-	ossDir := CWD()
-
-	// Check if we need to correct ossDir because it's in x-pack.
-	if parentDir := filepath.Base(filepath.Dir(ossDir)); parentDir == "x-pack" {
-		// If the OSS version of the beat exists.
-		tmp := filepath.Join(ossDir, "../..", BeatName)
-		if _, err := os.Stat(tmp); !os.IsNotExist(err) {
-			ossDir = tmp
-		}
-	}
-
-	return filepath.Join(append([]string{ossDir}, path...)...)
-}
-
 // LibbeatDir returns the libbeat directory. You can pass paths and
 // they will be joined and appended to the libbeat dir.
 func LibbeatDir(path ...string) string {

dev-tools/mage/dashboard.go

@@ -0,0 +1,53 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/magefile/mage/sh"
+)
+
+// ExportDashboard exports a dashboard from Kibana and writes it into the given module.
+func ExportDashboard() error {
+	module := EnvOr("MODULE", "")
+	if module == "" {
+		return fmt.Errorf("MODULE must be specified")
+	}
+
+	id := EnvOr("ID", "")
+	if id == "" {
+		return fmt.Errorf("Dashboad ID must be specified")
+	}
+
+	beatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		return err
+	}
+
+	// TODO: This is currently hardcoded for KB 6, we need to figure out what we do for KB 7
+	file := CWD("module", module, "_meta/kibana/6/dashboard", id+".json")
+
+	dashboardCmd := sh.RunCmd("go", "run",
+		filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go"),
+		"-output", file, "-dashboard", id,
+	)
+
+	return dashboardCmd()
+}

dev-tools/mage/gotest.go

@@ -325,3 +325,16 @@ func (s *GoTestSummary) String() string {
 
 	return strings.TrimRight(b.String(), "\n")
 }
+
+// BuildSystemTestBinary build a binary for testing that is instrumented for
+// testing and measuring code coverage. The binary is only instrumented for
+// coverage when TEST_COVERAGE=true (default is false).
+func BuildSystemTestBinary() error {
+	args := []string{
+		"test", "-c",
+	}
+	if TestCoverage {
+		args = append(args, "-coverpkg", "./...")
+	}
+	return sh.RunV("go", args...)
+}

dev-tools/mage/pkgspecs.go

@@ -18,6 +18,7 @@
 package mage
 
 import (
+	"bytes"
 	"io/ioutil"
 	"log"
 	"path/filepath"
@@ -88,10 +89,24 @@ func MustUsePackaging(specName, specFile string) {
 	}
 }
 
+// LoadLocalNamedSpec loads the named package spec from the packages.yml in the
+// current directory.
+func LoadLocalNamedSpec(name string) {
+	beatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		panic(err)
+	}
+
+	err = LoadNamedSpec(name, filepath.Join(beatsDir, packageSpecFile), "packages.yml")
+	if err != nil {
+		panic(err)
+	}
+}
+
 // LoadNamedSpec loads a packaging specification with the given name from the
 // specified YAML file. name should be a sub-key of 'specs'.
-func LoadNamedSpec(name, file string) error {
-	specs, err := LoadSpecs(file)
+func LoadNamedSpec(name string, files ...string) error {
+	specs, err := LoadSpecs(files...)
 	if err != nil {
 		return errors.Wrap(err, "failed to load spec file")
 	}
@@ -122,7 +137,7 @@ func LoadSpecs(files ...string) (map[string][]OSPackageArgs, error) {
 	}
 
 	var packages PackageYAML
-	if err = yaml.Unmarshal(data, &packages); err != nil {
+	if err := yaml.Unmarshal(bytes.Join(data, []byte{'\n'}), &packages); err != nil {
 		return nil, errors.Wrap(err, "failed to unmarshal spec data")
 	}