feat(plugins SDK): implement go-plugin framework (#207)

Signed-off-by: Miguel Martinez Trivino <miguel@chainloop.dev>

Author: migmartri

.goreleaser.yml

@@ -28,6 +28,16 @@ builds:
       - darwin_arm64
       - linux_amd64
       - linux_arm64
+  # Plugins build
+  # NOTE: On the event of a new plugin added to the project you need to
+  # 1 - Add the plugins binary to be built in this section
+  # 2 - Add the plugin ID to the allow-list in the dockers.IDs section
+  # 3 - Update the Dockerfile.goreleaser
+  - binary: chainloop-plugin-discord-webhook
+    id: chainloop-plugin-discord-webhook
+    main: ./app/controlplane/plugins/core/discord-webhook/v1/cmd
+    targets:
+      - linux_amd64
 archives:
   - builds:
       - cli
@@ -62,6 +72,7 @@ dockers:
   - dockerfile: app/controlplane/Dockerfile.goreleaser
     ids:
       - control-plane
+      - chainloop-plugin-discord-webhook
     image_templates:
       - "ghcr.io/chainloop-dev/chainloop/control-plane:{{ .Tag }}"
       - "ghcr.io/chainloop-dev/chainloop/control-plane:latest"

Makefile

@@ -42,3 +42,12 @@ lint:
 # All tests, both unit and integration
 test: 
 	go test ./... 
+
+.PHONY: build_devel
+# build for development testing
+build_devel:
+	goreleaser build --snapshot --clean --single-target
+.PHONY: build_devel_container_mages
+# build container images for development testing
+build_devel_container_mages:
+	goreleaser release --clean --snapshot --skip-sign --skip-sbom

app/controlplane/Dockerfile.goreleaser

@@ -2,7 +2,14 @@ FROM golang:1.20 AS builder
 
 FROM scratch
 
-COPY ./control-plane /
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 
+COPY ./control-plane /
+# Add plugins here
+# NOTE: they are built by go-releaser in the builds section
+# Make sure to update it acordingly if you add more plugins
+COPY ./chainloop-plugin-discord-webhook /plugins/
+# tmp is required for the plugins to run
+COPY --from=builder /tmp /tmp
+
 ENTRYPOINT [ "/control-plane", "--conf", "/data/conf"]

app/controlplane/cmd/main.go

@@ -26,6 +26,8 @@ import (
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/biz"
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/conf"
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/server"
+	"github.com/chainloop-dev/chainloop/app/controlplane/plugins"
+	"github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1"
 	credsConfig "github.com/chainloop-dev/chainloop/internal/credentials/api/credentials/v1"
 	"github.com/chainloop-dev/chainloop/internal/servicelogger"
 
@@ -54,7 +56,7 @@ func init() {
 	flag.StringVar(&flagconf, "conf", "../configs", "config path, eg: -conf config.yaml")
 }
 
-func newApp(logger log.Logger, gs *grpc.Server, hs *http.Server, ms *server.HTTPMetricsServer, expirer *biz.WorkflowRunExpirerUseCase) *app {
+func newApp(logger log.Logger, gs *grpc.Server, hs *http.Server, ms *server.HTTPMetricsServer, expirer *biz.WorkflowRunExpirerUseCase, plugins sdk.AvailablePlugins) *app {
 	return &app{
 		kratos.New(
 			kratos.ID(id),
@@ -63,7 +65,7 @@ func newApp(logger log.Logger, gs *grpc.Server, hs *http.Server, ms *server.HTTP
 			kratos.Metadata(map[string]string{}),
 			kratos.Logger(logger),
 			kratos.Server(gs, hs, ms),
-		), expirer}
+		), expirer, plugins}
 }
 
 func main() {
@@ -72,7 +74,7 @@ func main() {
 		config.WithSource(
 			file.NewSource(flagconf),
 			// Load environments variables prefixed with CP_
-			// NOTE: They get resolved withouth the prefix, i.e CP_DB_HOST -> DB_HOST
+			// NOTE: They get resolved without the prefix, i.e CP_DB_HOST -> DB_HOST
 			env.NewSource("CP_"),
 		),
 	)
@@ -105,7 +107,15 @@ func main() {
 		panic(err)
 	}
 
-	app, cleanup, err := wireApp(&bc, credsWriter, logger)
+	// Load plugins
+	availablePlugins, err := plugins.Load(bc.GetPluginsDir(), logger)
+	if err != nil {
+		panic(err)
+	}
+	// Kill plugins processes on exit
+	defer availablePlugins.Cleanup()
+
+	app, cleanup, err := wireApp(&bc, credsWriter, logger, availablePlugins)
 	if err != nil {
 		panic(err)
 	}
@@ -127,7 +137,8 @@ func main() {
 type app struct {
 	*kratos.App
 	// Periodic job that expires unfinished attestation processes older than a given threshold
-	runsExpirer *biz.WorkflowRunExpirerUseCase
+	runsExpirer      *biz.WorkflowRunExpirerUseCase
+	availablePlugins sdk.AvailablePlugins
 }
 
 func filterSensitiveArgs(_ log.Level, keyvals ...interface{}) bool {

app/controlplane/cmd/wire.go

@@ -27,15 +27,15 @@ import (
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/dispatcher"
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/server"
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/service"
-	"github.com/chainloop-dev/chainloop/app/controlplane/plugins"
+	"github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1"
 	backend "github.com/chainloop-dev/chainloop/internal/blobmanager"
 	"github.com/chainloop-dev/chainloop/internal/blobmanager/oci"
 	"github.com/chainloop-dev/chainloop/internal/credentials"
 	"github.com/go-kratos/kratos/v2/log"
 	"github.com/google/wire"
 )
 
-func wireApp(*conf.Bootstrap, credentials.ReaderWriter, log.Logger) (*app, func(), error) {
+func wireApp(*conf.Bootstrap, credentials.ReaderWriter, log.Logger, sdk.AvailablePlugins) (*app, func(), error) {
 	panic(
 		wire.Build(
 			wire.Bind(new(credentials.Reader), new(credentials.ReaderWriter)),
@@ -49,7 +49,6 @@ func wireApp(*conf.Bootstrap, credentials.ReaderWriter, log.Logger) (*app, func(
 			serviceOpts,
 			wire.Value([]biz.CASClientOpts{}),
 			wire.FieldsOf(new(*conf.Bootstrap), "Server", "Auth", "Data", "CasServer"),
-			plugins.Load,
 			dispatcher.New,
 			newApp,
 		),

app/controlplane/cmd/wire_gen.go

@@ -13,6 +13,9 @@ server:
     # We have some slow operations such as verifying an OCI registry
     timeout: 10s
 
+# Directory where the plugins are located
+# NOTE: plugins have the form of chainloop-plugin-<name>
+plugins_dir: "./plugins/bin"
 # Local CAS server for development
 cas_server:
   grpc:

app/controlplane/configs/config.devel.yaml

@@ -79,9 +79,9 @@ func WithCredsReaderWriter(rw credentials.ReaderWriter) NewTestingUCOpt {
 func WithRegisteredIntegration(i sdk.FanOut) NewTestingUCOpt {
 	return func(tu *newTestingOpts) {
 		if tu.integrations == nil {
-			tu.integrations = []sdk.FanOut{i}
+			tu.integrations = []*sdk.FanOutP{{FanOut: i}}
 		} else {
-			tu.integrations = append(tu.integrations, i)
+			tu.integrations = append(tu.integrations, &sdk.FanOutP{FanOut: i})
 		}
 	}
 }