[Filebeat] move create-[module,fileset,fields] to mage (#15836)

- move create-[module,fileset,fields] to mage
- make mage create commands available in x-pack/filebeat
- change Makefile to use mage for create commands

CHANGELOG.next.asciidoc

@@ -109,6 +109,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dashboard for AWS vpcflow fileset. {pull}16007[16007]
 - Add ECS tls fields to zeek:smtp,rdp,ssl and aws:s3access,elb {issue}15757[15757] {pull}15935[15936]
 - Add custom string mapping to CEF module to support Forcepoint NGFW {issue}14663[14663] {pull}15910[15910]
+- move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836]
 
 *Heartbeat*
 

filebeat/Makefile

@@ -14,15 +14,15 @@ update: mage
 
 # Creates a new module. Requires the params MODULE.
 .PHONY: create-module
-create-module:
-	@go run ${ES_BEATS}/filebeat/scripts/generator/module/main.go --path=$(PWD) --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE)
+create-module: mage
+	mage generate:module
 
 # Creates a new fileset. Requires the params MODULE and FILESET.
 .PHONY: create-fileset
-create-fileset:
-	@go run ${ES_BEATS}/filebeat/scripts/generator/fileset/main.go --path=$(PWD) --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) --fileset=$(FILESET)
+create-fileset: mage
+	mage generate:fileset
 
 # Creates a fields.yml based on a pipeline.json file. Requires the params MODULE and FILESET.
 .PHONY: create-fields
-create-fields:
-	@go run ${ES_BEATS}/filebeat/scripts/generator/fields/main.go --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) --fileset=$(FILESET)
+create-fields: mage
+	mage generate:fields

filebeat/magefile.go

@@ -31,6 +31,8 @@ import (
 
 	// mage:import
 	"github.com/elastic/beats/dev-tools/mage/target/common"
+	// mage:import generate
+	_ "github.com/elastic/beats/filebeat/scripts/mage/generate"
 )
 
 func init() {

filebeat/scripts/generator/module/main.go → filebeat/scripts/mage/generate/fields.go

@@ -15,32 +15,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package main
+package generate
 
 import (
-	"flag"
 	"fmt"
 	"os"
 
-	"github.com/elastic/beats/filebeat/generator/module"
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	genfields "github.com/elastic/beats/filebeat/generator/fields"
 )
 
-func main() {
-	moduleName := flag.String("module", "", "Name of the module")
-	modulePath := flag.String("path", ".", "Path to the generated fileset")
-	beatsPath := flag.String("beats_path", ".", "Path to elastic/beats")
-	flag.Parse()
+// Fields creates a new fields.yml for an existing Filebeat fileset.
+// Use MODULE=module to specify the name of the existing module
+// Use FILESET=fileset to specify the name of the existing fileset
+func Fields() error {
+	targetModule := os.Getenv("MODULE")
+	targetFileset := os.Getenv("FILESET")
 
-	if *moduleName == "" {
-		fmt.Println("Missing parameter: module")
-		os.Exit(1)
+	if targetModule == "" || targetFileset == "" {
+		return fmt.Errorf("you must specify the module and fileset: MODULE=module FILESET=fileset mage generate:fields")
 	}
 
-	err := module.Generate(*moduleName, *modulePath, *beatsPath)
-	if err != nil {
-		fmt.Printf("Cannot generate module: %v\n", err)
-		os.Exit(2)
-	}
+	curDir := devtools.CWD()
 
-	fmt.Println("New module was generated, now you can start creating filesets by create-fileset command.")
+	return genfields.Generate(curDir, targetModule, targetFileset, false)
 }

filebeat/scripts/mage/generate/fileset.go

@@ -0,0 +1,50 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package generate
+
+import (
+	"fmt"
+	"os"
+
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	genfileset "github.com/elastic/beats/filebeat/generator/fileset"
+)
+
+// Fileset creates a new fileset for an existing Filebeat module.
+// Use MODULE=module to specify the name of the existing module
+// Use FILESET=fileset to specify the name of the new fileset
+func Fileset() error {
+	targetModule := os.Getenv("MODULE")
+	targetFileset := os.Getenv("FILESET")
+
+	if targetModule == "" || targetFileset == "" {
+		return fmt.Errorf("you must specify the module and fileset: MODULE=module FILESET=fileset createFileset")
+	}
+
+	ossDir := devtools.OSSBeatDir()
+	xPackDir := devtools.XPackBeatDir()
+
+	switch devtools.CWD() {
+	case ossDir:
+		return genfileset.Generate(targetModule, targetFileset, ossDir, ossDir)
+	case xPackDir:
+		return genfileset.Generate(targetModule, targetFileset, xPackDir, ossDir)
+	default:
+		return fmt.Errorf("you must be in a filebeat directory")
+	}
+}

filebeat/scripts/generator/fields/main.go → filebeat/scripts/mage/generate/module.go

@@ -15,38 +15,33 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package main
+package generate
 
 import (
-	"flag"
 	"fmt"
 	"os"
 
-	"github.com/elastic/beats/filebeat/generator/fields"
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	genmod "github.com/elastic/beats/filebeat/generator/module"
 )
 
-func main() {
-	module := flag.String("module", "", "Name of the module")
-	fileset := flag.String("fileset", "", "Name of the fileset")
-	beatsPath := flag.String("beats_path", ".", "Path to elastic/beats")
-	noDoc := flag.Bool("nodoc", false, "Generate documentation for fields")
-	flag.Parse()
-
-	if *module == "" {
-		fmt.Println("Missing parameter: module")
-		os.Exit(1)
+// Module creates a new Filebeat module.
+// Use MODULE=module to specify the name of the new module
+func Module() error {
+	targetModule := os.Getenv("MODULE")
+	if targetModule == "" {
+		return fmt.Errorf("you must specify the module: MODULE=name mage generate:module")
 	}
 
-	if *fileset == "" {
-		fmt.Println("Missing parameter: fileset")
-		os.Exit(1)
-	}
+	ossDir := devtools.OSSBeatDir()
+	xPackDir := devtools.XPackBeatDir()
 
-	err := fields.Generate(*beatsPath, *module, *fileset, *noDoc)
-	if err != nil {
-		fmt.Printf("Error while generating fields.yml: %v\n", err)
-		os.Exit(2)
+	switch devtools.CWD() {
+	case ossDir:
+		return genmod.Generate(targetModule, ossDir, ossDir)
+	case xPackDir:
+		return genmod.Generate(targetModule, xPackDir, ossDir)
+	default:
+		return fmt.Errorf("you must be in a filebeat directory")
 	}
-
-	fmt.Printf("Fields.yml generated for %s/%s\n", *module, *fileset)
 }

x-pack/filebeat/magefile.go

@@ -18,6 +18,8 @@ import (
 
 	// mage:import
 	"github.com/elastic/beats/dev-tools/mage/target/common"
+	// mage:import generate
+	_ "github.com/elastic/beats/filebeat/scripts/mage/generate"
 )
 
 func init() {