feat: add Extism plugin system next to native binary exec option (#156)

* feat: support Extism plugins

* chore: update go modules

* ci: use go 1.16 in build, supports embed. new minimum version supported

* chore: update go.mod required versions

* ci: update to go 1.20

* ci: remove test case, narrowing

* ci: add test case, expand

* ci: add test case, expand

* ci: rearrange shell commands

* ci: more arrangement

* ci: more test debugging

* ci: try without pipefail, as this is supposed to fail

* ci: try with go run

* ci: try with file output

* ci: never fail the status check before testing grep

* ci: remove combined test

.github/workflows/test.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set Up Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.12'
+          go-version: '1.20'
       - name: fetch depenencies, test code
         run: |
           go get -v -d ./...
@@ -47,11 +47,17 @@ jobs:
           if [ "$WARNINGS" != 2 ]; then
             exit 1
           fi
-      - name: check output using multiple plugins, with one error expected
+      - name: check output using plugin-sample-wasm
         run: |
           set +o pipefail
 
-          protolock status --plugins=plugin-sample,plugin-sample-error | grep "some error"
+          protolock status --plugins=plugin-samples/plugin-sample-wasm/status.wasm | grep "Extism plugin ran"
+      - name: check output using multiple plugins, with one error expected
+        run: |
+          protolock status \
+            --plugins=plugin-sample,plugin-sample-error,plugin-samples/plugin-sample-wasm/status.wasm \
+            | grep "some error"
+
           protolock status --plugins=plugin-sample-error,plugin-sample | grep "some error"
       - name: check output using multiple plugins with errors
         run: |

cmd/protolock/plugins.go

@@ -2,13 +2,15 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"os/exec"
 	"strings"
 	"sync"
 
+	extism "github.com/extism/go-sdk"
 	"github.com/nilslice/protolock"
 	"github.com/nilslice/protolock/extend"
 )
@@ -68,28 +70,61 @@ func runPlugins(
 		// and updated Protolock structs from the `protolock status` call
 		go func(name string) {
 			defer wg.Done()
+			// output is populated either by the execution of an Extism plugin or a native binary
+			var output []byte
 			name = strings.TrimSpace(name)
-			path, err := exec.LookPath(name)
-			if err != nil {
-				if path == "" {
-					path = name
-				}
-				fmt.Println(logPrefix, name, "plugin exec error:", err)
-				return
-			}
+			path := name
 
-			// initialize the executable to be called from protolock using the
-			// absolute path and copy of the input data
-			plugin := &exec.Cmd{
-				Path:  path,
-				Stdin: pluginInputData,
+			if debug {
+				fmt.Println(logPrefix, name, "running plugin")
 			}
 
-			// execute the plugin and capture the output
-			output, err := plugin.CombinedOutput()
-			if err != nil {
-				pluginErrsChan <- wrapPluginErr(name, path, err, output)
-				return
+			if strings.HasSuffix(name, ".wasm") {
+				// do extism call
+				manifest := extism.Manifest{
+					Wasm: []extism.Wasm{extism.WasmFile{Path: name}},
+					// TODO: consider enabling external configuration to add hosts and paths
+					// AllowedHosts: []string{},
+					// AllowedPaths: map[string]string{},
+				}
+
+				plugin, err := extism.NewPlugin(context.Background(), manifest, extism.PluginConfig{EnableWasi: true}, nil)
+				if err != nil {
+					fmt.Println(logPrefix, name, "failed to create extism plugin:", err)
+					return
+				}
+
+				var exitCode uint32
+				exitCode, output, err = plugin.Call("status", inputData.Bytes())
+				if err != nil {
+					fmt.Println(logPrefix, name, "plugin exec error: ", err, "code:", exitCode)
+					pluginErrsChan <- wrapPluginErr(name, path, err, output)
+					return
+				}
+
+			} else {
+				path, err = exec.LookPath(name)
+				if err != nil {
+					if path == "" {
+						path = name
+					}
+					fmt.Println(logPrefix, name, "plugin exec error:", err)
+					return
+				}
+
+				// initialize the executable to be called from protolock using the
+				// absolute path and copy of the input data
+				plugin := &exec.Cmd{
+					Path:  path,
+					Stdin: pluginInputData,
+				}
+
+				// execute the plugin and capture the output
+				output, err = plugin.CombinedOutput()
+				if err != nil {
+					pluginErrsChan <- wrapPluginErr(name, path, err, output)
+					return
+				}
 			}
 
 			pluginData := &extend.Data{}

go.mod

@@ -1,8 +1,17 @@
 module github.com/nilslice/protolock
 
+go 1.21
+
 require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/proto v1.9.1
+	github.com/extism/go-sdk v1.0.0
+	github.com/stretchr/testify v1.8.4
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/stretchr/testify v1.2.2
+	github.com/tetratelabs/wazero v1.3.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -2,7 +2,17 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emicklei/proto v1.9.1 h1:MUgjFo5xlMwYv72TnF5xmmdKZ04u+dVbv6wdARv16D8=
 github.com/emicklei/proto v1.9.1/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
+github.com/extism/go-sdk v1.0.0 h1://UAyiQGok1ihrlzpkfF6UTY5TwJs6hKJBXnQ0sui20=
+github.com/extism/go-sdk v1.0.0/go.mod h1:xUfKSEQndAvHBc1Ohdre0e+UdnRzUpVfbA8QLcx4fbY=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/tetratelabs/wazero v1.3.0 h1:nqw7zCldxE06B8zSZAY0ACrR9OH5QCcPwYmYlwtcwtE=
+github.com/tetratelabs/wazero v1.3.0/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

plugin-samples/plugin-sample-wasm/go.mod

@@ -0,0 +1,9 @@
+module github.com/nilslice/protolock/plugin-samples/plugin-sample-wasm
+
+go 1.21
+
+require (
+	github.com/emicklei/proto v1.9.1 // indirect
+	github.com/extism/go-pdk v1.0.0 // indirect
+	github.com/nilslice/protolock v0.17.0 // indirect
+)

plugin-samples/plugin-sample-wasm/go.sum

@@ -0,0 +1,9 @@
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emicklei/proto v1.9.1 h1:MUgjFo5xlMwYv72TnF5xmmdKZ04u+dVbv6wdARv16D8=
+github.com/emicklei/proto v1.9.1/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
+github.com/extism/go-pdk v1.0.0 h1:/VlFLDnpYfooMl+VW94VHrbdruDyKkpa47yYJ7YcCAE=
+github.com/extism/go-pdk v1.0.0/go.mod h1:Gz+LIU/YCKnKXhgge8yo5Yu1F/lbv7KtKFkiCSzW/P4=
+github.com/nilslice/protolock v0.17.0 h1:sYvcukABl62tZX77H6NuV+jtlwTIfQbn0ln0ixTqr4A=
+github.com/nilslice/protolock v0.17.0/go.mod h1:DYFqop7QlHjmBCaJKfcVO1Mw5b8JejJZgMvmFng/N9Y=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

plugin-samples/plugin-sample-wasm/main.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"encoding/json"
+
+	pdk "github.com/extism/go-pdk"
+	"github.com/nilslice/protolock"
+	"github.com/nilslice/protolock/extend"
+)
+
+// an Extism plugin uses a 'PDK' to communicate data input and output from its host system, in
+// this case, the `protolock` command.
+
+// see https://extism.org and https://github.com/extism/extism for more information.
+
+// In order to satisfy the current usage, an Extism Protolock plugin must export a single function
+// "status" with the following signature:
+
+//export status
+func status() int32 {
+	// rather than taking input from stdin, like native Protolock plugins, Extism plugins take data
+	// from their host, using the `pdk.Input()` function, returning bytes from protolock.
+	var data extend.Data
+	err := json.Unmarshal(pdk.Input(), &data.Current)
+	if err != nil {
+		pdk.SetError(err)
+		return 1
+	}
+
+	// with the `extend.Data` available, you would do some checks on the current and updated set of
+	// `proto.lock` representations. Here we are adding a warning to demonstrate that the plugin
+	// works with some known data output to verify.
+	warning := protolock.Warning{
+		Filepath: "fake.proto",
+		Message:  "An Extism plugin ran and checked the status of the proto.lock files",
+		RuleName: "RuleNameXYZ",
+	}
+	data.PluginWarnings = append(data.PluginWarnings, warning)
+
+	b, err := json.Marshal(data)
+	if err != nil {
+		pdk.SetError(err)
+		return 1
+	}
+
+	// tather than writing data to stdout, like native Protolock plugins, Extism plugins provide
+	// data back to their host, using the `pdk.Output()` function, returning bytes to protolock.
+	pdk.Output(b)
+
+	// non-zero return code here will result in Extism detecting an error.
+	return 0
+}
+
+// this Go code is compiled to WebAssembly, and current compilers expect some entrypoint, even if
+// this function isn't called.
+func main() {}