(2) Implement datasources (hashicorp#10440)

cmd/packer-plugin-check/main.go

@@ -12,7 +12,6 @@ import (
 	"strings"
 
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
-	"github.com/hashicorp/packer/packer"
 	"github.com/hashicorp/packer/packer/plugin"
 )
 
@@ -130,47 +129,57 @@ func discoverAndLoad() error {
 	}
 
 	// TODO: validate correctness of plugins loaded by checking them against the output of the `describe` command.
-	builders, provisioners, postProcessors := config.GetPlugins()
-	if len(builders) == 0 &&
-		len(provisioners) == 0 &&
-		len(postProcessors) == 0 {
-		return fmt.Errorf("couldn't load any Builder/Provisioner/Post-Processor from the plugin binary")
+	plugins := config.GetPlugins()
+	if len(plugins.Builders) == 0 &&
+		len(plugins.Provisioners) == 0 &&
+		len(plugins.PostProcessors) == 0 &&
+		len(plugins.DataSources) == 0 {
+		return fmt.Errorf("couldn't load any Builder/Provisioner/Post-Processor/Datasource from the plugin binary")
 	}
 
-	return checkHCL2ConfigSpec(builders, provisioners, postProcessors)
+	return checkHCL2ConfigSpec(plugins)
 }
 
 // checkHCL2ConfigSpec checks if the hcl2spec config is present for the given plugins by validating that ConfigSpec() does not
 // return an empty map of specs.
-func checkHCL2ConfigSpec(builders packer.BuilderStore, provisioners packer.ProvisionerStore, postProcessors packer.PostProcessorStore) error {
+func checkHCL2ConfigSpec(plugins plugin.Plugins) error {
 	var errs *packersdk.MultiError
-	for _, b := range builders.List() {
-		builder, err := builders.Start(b)
+	for _, b := range plugins.Builders.List() {
+		builder, err := plugins.Builders.Start(b)
 		if err != nil {
 			return packersdk.MultiErrorAppend(err, errs)
 		}
 		if len(builder.ConfigSpec()) == 0 {
 			errs = packersdk.MultiErrorAppend(fmt.Errorf("builder %q does not contain the required hcl2spec configuration", b), errs)
 		}
 	}
-	for _, p := range provisioners.List() {
-		provisioner, err := provisioners.Start(p)
+	for _, p := range plugins.Provisioners.List() {
+		provisioner, err := plugins.Provisioners.Start(p)
 		if err != nil {
 			return packersdk.MultiErrorAppend(err, errs)
 		}
 		if len(provisioner.ConfigSpec()) == 0 {
 			errs = packersdk.MultiErrorAppend(fmt.Errorf("provisioner %q does not contain the required hcl2spec configuration", p), errs)
 		}
 	}
-	for _, pp := range postProcessors.List() {
-		postProcessor, err := postProcessors.Start(pp)
+	for _, pp := range plugins.PostProcessors.List() {
+		postProcessor, err := plugins.PostProcessors.Start(pp)
 		if err != nil {
 			return packersdk.MultiErrorAppend(err, errs)
 		}
 		if len(postProcessor.ConfigSpec()) == 0 {
 			errs = packersdk.MultiErrorAppend(fmt.Errorf("post-processor %q does not contain the required hcl2spec configuration", pp), errs)
 		}
 	}
+	for _, d := range plugins.DataSources.List() {
+		datasource, err := plugins.DataSources.Start(d)
+		if err != nil {
+			return packersdk.MultiErrorAppend(err, errs)
+		}
+		if len(datasource.ConfigSpec()) == 0 {
+			errs = packersdk.MultiErrorAppend(fmt.Errorf("datasource %q does not contain the required hcl2spec configuration", d), errs)
+		}
+	}
 	if errs != nil && len(errs.Errors) > 0 {
 		return errs
 	}

command/build.go

@@ -70,6 +70,7 @@ func (m *Meta) GetConfigFromHCL(cla *MetaArgs) (*hcl2template.PackerConfig, int)
 		BuilderSchemas:          m.CoreConfig.Components.BuilderStore,
 		ProvisionersSchemas:     m.CoreConfig.Components.ProvisionerStore,
 		PostProcessorsSchemas:   m.CoreConfig.Components.PostProcessorStore,
+		DatasourceSchemas:       m.CoreConfig.Components.DatasourceStore,
 	}
 	cfg, diags := parser.Parse(cla.Path, cla.VarFiles, cla.Vars)
 	return cfg, writeDiags(m.Ui, parser.Files(), diags)
@@ -147,7 +148,7 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
 	if ret != 0 {
 		return ret
 	}
-	diags := packerStarter.Initialize()
+	diags := packerStarter.Initialize(packer.InitializeOptions{})
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
 		return ret

command/build_test.go

@@ -371,6 +371,17 @@ func TestBuild(t *testing.T) {
 			},
 			expectedCode: 1,
 		},
+		{
+			name: "hcl - execute and use datasource",
+			args: []string{
+				testFixture("hcl", "datasource.pkr.hcl"),
+			},
+			fileCheck: fileCheck{
+				expectedContent: map[string]string{
+					"chocolate.txt": "chocolate",
+				},
+			},
+		},
 	}
 
 	for _, tt := range tc {
@@ -849,6 +860,9 @@ func testCoreConfigBuilder(t *testing.T) *packer.CoreConfig {
 			"shell-local": func() (packersdk.PostProcessor, error) { return &shell_local_pp.PostProcessor{}, nil },
 			"manifest":    func() (packersdk.PostProcessor, error) { return &manifest.PostProcessor{}, nil },
 		},
+		DatasourceStore: packersdk.MapOfDatasource{
+			"mock": func() (packersdk.Datasource, error) { return &packersdk.MockDatasource{}, nil },
+		},
 	}
 	return &packer.CoreConfig{
 		Components: components,

command/console.go

@@ -60,7 +60,7 @@ func (c *ConsoleCommand) RunContext(ctx context.Context, cla *ConsoleArgs) int {
 		return ret
 	}
 
-	_ = packerStarter.Initialize()
+	_ = packerStarter.Initialize(packer.InitializeOptions{})
 
 	// Determine if stdin is a pipe. If so, we evaluate directly.
 	if c.StdinPiped() {

command/core_wrapper.go

@@ -11,7 +11,7 @@ type CoreWrapper struct {
 	*packer.Core
 }
 
-func (c *CoreWrapper) Initialize() hcl.Diagnostics {
+func (c *CoreWrapper) Initialize(_ packer.InitializeOptions) hcl.Diagnostics {
 	err := c.Core.Initialize()
 	if err != nil {
 		return hcl.Diagnostics{

command/hcl2_upgrade.go

@@ -13,8 +13,8 @@ import (
 	texttemplate "text/template"
 
 	"github.com/hashicorp/hcl/v2/hclwrite"
+	hcl2shim "github.com/hashicorp/packer-plugin-sdk/hcl2helper"
 	"github.com/hashicorp/packer-plugin-sdk/template"
-	hcl2shim "github.com/hashicorp/packer/hcl2template/shim"
 	"github.com/mitchellh/mapstructure"
 	"github.com/posener/complete"
 	"github.com/zclconf/go-cty/cty"

command/inspect.go

@@ -46,7 +46,7 @@ func (c *InspectCommand) RunContext(ctx context.Context, cla *InspectArgs) int {
 	}
 
 	// here we ignore init diags to allow unknown variables to be used
-	_ = packerStarter.Initialize()
+	_ = packerStarter.Initialize(packer.InitializeOptions{})
 
 	return packerStarter.InspectConfig(packer.InspectConfigOptions{
 		Ui: c.Ui,

command/plugin.go

@@ -212,7 +212,9 @@ var PostProcessors = map[string]packersdk.PostProcessor{
 	"yandex-import":        new(yandeximportpostprocessor.PostProcessor),
 }
 
-var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)")
+var Datasources = map[string]packersdk.Datasource{}
+
+var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner|datasource)-(.+)")
 
 func (c *PluginCommand) Run(args []string) int {
 	// This is an internal call (users should not call this directly) so we're
@@ -261,6 +263,13 @@ func (c *PluginCommand) Run(args []string) int {
 			return 1
 		}
 		server.RegisterPostProcessor(postProcessor)
+	case "datasource":
+		datasource, found := Datasources[pluginName]
+		if !found {
+			c.Ui.Error(fmt.Sprintf("Could not load datasource: %s", pluginName))
+			return 1
+		}
+		server.RegisterDatasource(datasource)
 	}
 
 	server.Serve()

command/test-fixtures/hcl/datasource.pkr.hcl

@@ -0,0 +1,14 @@
+data "mock" "content" {
+  foo = "chocolate"
+}
+
+source "file" "chocolate" {
+  content = data.mock.content.foo
+  target = "chocolate.txt"
+}
+
+build {
+  sources = [
+    "sources.file.chocolate",
+  ]
+}

command/test-fixtures/validate/datasource.pkr.hcl

@@ -0,0 +1,16 @@
+packer {
+  required_version = ">= v1.0.0"
+}
+
+data "mock" "content" {
+  foo = "content"
+}
+
+source "file" "chocolate" {
+  target = "chocolate.txt"
+  content = data.mock.content.foo
+}
+
+build {
+  sources = ["source.file.chocolate"]
+}

command/validate.go

@@ -56,7 +56,9 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int
 		return 0
 	}
 
-	diags := packerStarter.Initialize()
+	diags := packerStarter.Initialize(packer.InitializeOptions{
+		SkipDatasourcesExecution: true,
+	})
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
 		return ret

command/validate_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 )
 
 func TestValidateCommand(t *testing.T) {
@@ -45,6 +46,29 @@ func TestValidateCommand(t *testing.T) {
 	}
 }
 
+func TestValidateCommand_SkipDatasourceExecution(t *testing.T) {
+	datasourceMock := &packersdk.MockDatasource{}
+	meta := testMetaFile(t)
+	meta.CoreConfig.Components.DatasourceStore = packersdk.MapOfDatasource{
+		"mock": func() (packersdk.Datasource, error) {
+			return datasourceMock, nil
+		},
+	}
+	c := &ValidateCommand{
+		Meta: meta,
+	}
+	args := []string{filepath.Join(testFixture("validate"), "datasource.pkr.hcl")}
+	if code := c.Run(args); code != 0 {
+		fatalCommand(t, c.Meta)
+	}
+	if datasourceMock.ExecuteCalled {
+		t.Fatalf("Datasource should not be executed on validation")
+	}
+	if !datasourceMock.OutputSpecCalled {
+		t.Fatalf("Datasource OutPutSpec should be called on validation")
+	}
+}
+
 func TestValidateCommand_SyntaxOnly(t *testing.T) {
 	tt := []struct {
 		path     string

config.go

@@ -30,6 +30,7 @@ type config struct {
 	Builders                   packer.MapOfBuilder       `json:"-"`
 	Provisioners               packer.MapOfProvisioner   `json:"-"`
 	PostProcessors             packer.MapOfPostProcessor `json:"-"`
+	Datasources                packer.MapOfDatasource    `json:"-"`
 	Plugins                    plugin.Config
 }
 
@@ -186,5 +187,17 @@ func (c *config) discoverInternalComponents() error {
 		}
 	}
 
+	for dataSource := range command.Datasources {
+		dataSource := dataSource
+		_, found := (c.Datasources)[dataSource]
+		if !found {
+			c.Datasources[dataSource] = func() (packersdk.Datasource, error) {
+				bin := fmt.Sprintf("%s%splugin%spacker-datasource-%s",
+					packerPath, PACKERSPACE, PACKERSPACE, dataSource)
+				return c.Plugins.Client(bin).Datasource()
+			}
+		}
+	}
+
 	return nil
 }

go.mod

@@ -49,7 +49,7 @@ require (
 	github.com/hashicorp/go-uuid v1.0.2
 	github.com/hashicorp/go-version v1.2.0
 	github.com/hashicorp/hcl/v2 v2.8.0
-	github.com/hashicorp/packer-plugin-sdk v0.0.7-0.20210111224258-fd30ebb797f0
+	github.com/hashicorp/packer-plugin-sdk v0.0.7
 	github.com/hashicorp/vault/api v1.0.4
 	github.com/hetznercloud/hcloud-go v1.15.1
 	github.com/hyperonecom/h1-client-go v0.0.0-20191203060043-b46280e4c4a4