From 7c59693d2fa2b973a78286b5f9797cb7386c5aca Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 3 Oct 2024 18:02:50 -0400 Subject: [PATCH] wip --- go.mod | 4 +- go.sum | 8 +- private/buf/bufcli/cache.go | 50 ++ private/buf/bufcli/controller.go | 7 + private/buf/bufcli/errors.go | 12 + private/buf/bufcli/flags_args.go | 16 + private/buf/bufcli/plugin_key_provider.go | 36 + private/buf/bufctl/controller.go | 17 + private/buf/bufctl/image_with_config.go | 29 +- private/buf/bufmigrate/migrator.go | 5 +- private/buf/bufprint/bufprint.go | 27 + .../buf/bufworkspace/workspace_dep_manager.go | 50 +- private/buf/cmd/buf/buf.go | 30 +- private/buf/cmd/buf/buf_test.go | 3 +- private/buf/cmd/buf/command/beta/lsp/lsp.go | 9 +- .../betaplugindelete.go} | 2 +- .../plugin/betaplugindelete/usage.gen.go | 19 + .../betapluginpush.go} | 2 +- .../plugin/betapluginpush/usage.gen.go | 19 + .../buf/cmd/buf/command/breaking/breaking.go | 9 +- .../buf/command/config/internal/internal.go | 9 +- .../buf/command/dep/depupdate/depupdate.go | 18 +- .../cmd/buf/command/dep/internal/internal.go | 21 +- private/buf/cmd/buf/command/lint/lint.go | 7 +- .../cmd/buf/command/mod/internal/internal.go | 8 +- .../command/plugin/pluginpush/pluginpush.go | 614 ++++++++++++++++++ .../plugin/pluginpush/usage.gen.go | 0 .../plugin/plugincreate/plugincreate.go | 193 ++++++ .../registry/plugin/plugincreate/usage.gen.go | 19 + .../plugin/plugindelete/plugindelete.go | 114 ++++ .../registry/plugin/plugindelete/usage.gen.go | 0 .../registry/plugin/plugininfo/plugininfo.go | 121 ++++ .../registry/plugin/plugininfo/usage.gen.go | 19 + .../plugin/pluginupdate/pluginupdate.go | 161 +++++ .../registry/plugin/pluginupdate/usage.gen.go | 19 + .../cmd/protoc-gen-buf-breaking/breaking.go | 8 +- private/buf/cmd/protoc-gen-buf-lint/lint.go | 8 +- private/bufpkg/bufapi/bufapi.go | 91 +++ private/bufpkg/bufcheck/breaking_test.go | 11 +- private/bufpkg/bufcheck/bufcheck.go | 16 +- private/bufpkg/bufcheck/lint_test.go | 8 +- private/bufpkg/bufcheck/multi_client_test.go | 15 +- private/bufpkg/bufcheck/runner_provider.go | 81 ++- private/bufpkg/bufconfig/buf_lock_file.go | 67 +- private/bufpkg/bufconfig/plugin_config.go | 33 +- private/bufpkg/bufmodule/module_ref.go | 2 +- private/bufpkg/bufparse/bufparse.go | 15 + private/bufpkg/bufparse/errors.go | 85 +++ private/bufpkg/bufparse/parse.go | 83 +++ private/bufpkg/bufparse/usage.gen.go | 19 + private/bufpkg/bufplugin/bufplugin.go | 16 + .../bufplugin/bufpluginapi/bufpluginapi.go | 15 + .../bufpkg/bufplugin/bufpluginapi/convert.go | 53 ++ .../bufpluginapi/plugin_data_provider.go | 174 +++++ .../bufpluginapi/plugin_key_provider.go | 167 +++++ .../bufplugincache/bufplugincache.go | 15 + .../bufplugincache/plugin_data_provider.go | 111 ++++ .../bufplugin/bufplugincache/usage.gen.go | 19 + .../bufpluginstore/bufpluginstore.go | 15 + .../bufpluginstore/module_data_store.go | 167 +++++ .../bufplugin/bufpluginstore/usage.gen.go | 19 + private/bufpkg/bufplugin/commit.go | 72 ++ private/bufpkg/bufplugin/digest.go | 211 ++++++ private/bufpkg/bufplugin/errors.go | 69 ++ private/bufpkg/bufplugin/plugin_data.go | 113 ++++ .../bufpkg/bufplugin/plugin_data_provider.go | 57 ++ private/bufpkg/bufplugin/plugin_full_name.go | 124 ++++ private/bufpkg/bufplugin/plugin_key.go | 106 +++ .../bufpkg/bufplugin/plugin_key_provider.go | 43 ++ private/bufpkg/bufplugin/plugin_ref.go | 98 +++ private/bufpkg/bufplugin/usage.gen.go | 19 + 71 files changed, 3848 insertions(+), 54 deletions(-) create mode 100644 private/buf/bufcli/plugin_key_provider.go rename private/buf/cmd/buf/command/beta/registry/plugin/{plugindelete/plugindelete.go => betaplugindelete/betaplugindelete.go} (99%) create mode 100644 private/buf/cmd/buf/command/beta/registry/plugin/betaplugindelete/usage.gen.go rename private/buf/cmd/buf/command/beta/registry/plugin/{pluginpush/pluginpush.go => betapluginpush/betapluginpush.go} (99%) create mode 100644 private/buf/cmd/buf/command/beta/registry/plugin/betapluginpush/usage.gen.go create mode 100644 private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go rename private/buf/cmd/buf/command/{beta/registry => }/plugin/pluginpush/usage.gen.go (100%) create mode 100644 private/buf/cmd/buf/command/registry/plugin/plugincreate/plugincreate.go create mode 100644 private/buf/cmd/buf/command/registry/plugin/plugincreate/usage.gen.go create mode 100644 private/buf/cmd/buf/command/registry/plugin/plugindelete/plugindelete.go rename private/buf/cmd/buf/command/{beta => }/registry/plugin/plugindelete/usage.gen.go (100%) create mode 100644 private/buf/cmd/buf/command/registry/plugin/plugininfo/plugininfo.go create mode 100644 private/buf/cmd/buf/command/registry/plugin/plugininfo/usage.gen.go create mode 100644 private/buf/cmd/buf/command/registry/plugin/pluginupdate/pluginupdate.go create mode 100644 private/buf/cmd/buf/command/registry/plugin/pluginupdate/usage.gen.go create mode 100644 private/bufpkg/bufparse/bufparse.go create mode 100644 private/bufpkg/bufparse/errors.go create mode 100644 private/bufpkg/bufparse/parse.go create mode 100644 private/bufpkg/bufparse/usage.gen.go create mode 100644 private/bufpkg/bufplugin/bufplugin.go create mode 100644 private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go create mode 100644 private/bufpkg/bufplugin/bufpluginapi/convert.go create mode 100644 private/bufpkg/bufplugin/bufpluginapi/plugin_data_provider.go create mode 100644 private/bufpkg/bufplugin/bufpluginapi/plugin_key_provider.go create mode 100644 private/bufpkg/bufplugin/bufplugincache/bufplugincache.go create mode 100644 private/bufpkg/bufplugin/bufplugincache/plugin_data_provider.go create mode 100644 private/bufpkg/bufplugin/bufplugincache/usage.gen.go create mode 100644 private/bufpkg/bufplugin/bufpluginstore/bufpluginstore.go create mode 100644 private/bufpkg/bufplugin/bufpluginstore/module_data_store.go create mode 100644 private/bufpkg/bufplugin/bufpluginstore/usage.gen.go create mode 100644 private/bufpkg/bufplugin/commit.go create mode 100644 private/bufpkg/bufplugin/digest.go create mode 100644 private/bufpkg/bufplugin/errors.go create mode 100644 private/bufpkg/bufplugin/plugin_data.go create mode 100644 private/bufpkg/bufplugin/plugin_data_provider.go create mode 100644 private/bufpkg/bufplugin/plugin_full_name.go create mode 100644 private/bufpkg/bufplugin/plugin_key.go create mode 100644 private/bufpkg/bufplugin/plugin_key_provider.go create mode 100644 private/bufpkg/bufplugin/plugin_ref.go create mode 100644 private/bufpkg/bufplugin/usage.gen.go diff --git a/go.mod b/go.mod index c48026d0dd..9a08b68f9a 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ toolchain go1.23.2 require ( buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.34.2-20240928190436-5e8abcfd7a7e.2 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240920164238-5a7b106cbb87.2 - buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20240925012807-1610ffa05635.1 - buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.2-20240925012807-1610ffa05635.2 + buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-00000000000000-f384556653e6.1 + buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.2-00000000000000-f384556653e6.2 buf.build/go/bufplugin v0.5.0 buf.build/go/protoyaml v0.2.0 buf.build/go/spdx v0.2.0 diff --git a/go.sum b/go.sum index 25d83cca46..dc61cfd5ad 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,10 @@ buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.34.2-20240928190436-5e buf.build/gen/go/bufbuild/bufplugin/protocolbuffers/go v1.34.2-20240928190436-5e8abcfd7a7e.2/go.mod h1:B+9TKHRYqoAUW57pLjhkLOnBCu0DQYMV+f7imQ9nXwI= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240920164238-5a7b106cbb87.2 h1:hl0FrmGlNpQZIGvU1/jDz0lsPDd0BhCE0QDRwPfLZcA= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240920164238-5a7b106cbb87.2/go.mod h1:ylS4c28ACSI59oJrOdW4pHS4n0Hw4TgSPHn8rpHl4Yw= -buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20240925012807-1610ffa05635.1 h1:p4A9QnhBrKjCquBt1mKqfO37QseLwgWqQp+Wb9ZjasE= -buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-20240925012807-1610ffa05635.1/go.mod h1:7WtU+waNF+dyxDsuNaqmG3d0w3y2poNju8cvun1/jLs= -buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.2-20240925012807-1610ffa05635.2 h1:3sSS9z8k6zVe7rNNt9R6DN2fOFBVClEflmICIjbXwms= -buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.2-20240925012807-1610ffa05635.2/go.mod h1:psseUmlKRo9v5LZJtR/aTpdTLuyp9o3X7rnLT87SZEo= +buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-00000000000000-f384556653e6.1 h1:Rh6uADIMohx6myydsQ+VDRovAEs8wxO7CWuggBON0Vo= +buf.build/gen/go/bufbuild/registry/connectrpc/go v1.17.0-00000000000000-f384556653e6.1/go.mod h1:LM8iUDMDNerdWwvuUeujsuKbYEM9Tcy9V41/NNQ6n6Y= +buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.2-00000000000000-f384556653e6.2 h1:u6D0MOYus2seFpswpKRteA6viAWK6rYSATuxDVg7iyQ= +buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.2-00000000000000-f384556653e6.2/go.mod h1:psseUmlKRo9v5LZJtR/aTpdTLuyp9o3X7rnLT87SZEo= buf.build/gen/go/pluginrpc/pluginrpc/protocolbuffers/go v1.34.2-20240828222655-5345c0a56177.2 h1:oSi+Adw4xvIjXrW8eY8QGR3sBdfWeY5HN/RefnRt52M= buf.build/gen/go/pluginrpc/pluginrpc/protocolbuffers/go v1.34.2-20240828222655-5345c0a56177.2/go.mod h1:GjH0gjlY/ns16X8d6eaXV2W+6IFwsO5Ly9WVnzyd1E0= buf.build/go/bufplugin v0.5.0 h1:pmK1AloAMp+4woH5hEisK9qVmDdLySzIKexUUVZLJ2Q= diff --git a/private/buf/bufcli/cache.go b/private/buf/bufcli/cache.go index 568e1104f0..0b9b1eab90 100644 --- a/private/buf/bufcli/cache.go +++ b/private/buf/bufcli/cache.go @@ -27,6 +27,10 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulecache" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulestore" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufplugincache" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginstore" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/filelock" @@ -103,6 +107,10 @@ var ( // // Normalized. v3CacheModuleLockRelDirPath = normalpath.Join("v3", "modulelocks") + // v3CachePluginRelDirPath is the relative path to the files cache directory in its newest iteration. + // + // Normalized. + v3CachePluginRelDirPath = normalpath.Join("v3", "plugins") // v3CacheWasmRuntimeRelDirPath is the relative path to the Wasm runtime cache directory in its newest iteration. // This directory is used to store the Wasm runtime cache. This is an implementation specific cache and opaque outside of the runtime. // @@ -140,6 +148,21 @@ func NewCommitProvider(container appext.Container) (bufmodule.CommitProvider, er ) } +// NewPluginDataProvider returns a new PluginDataProvider while creating the +// required cache directories. +func NewPluginDataProvider(container appext.Container) (bufplugin.PluginDataProvider, error) { + clientConfig, err := NewConnectClientConfig(container) + if err != nil { + return nil, err + } + return newPluginDataProvider( + container, + bufapi.NewClientProvider( + clientConfig, + ), + ) +} + // CreateWasmRuntimeCacheDir creates the cache directory for the Wasm runtime. // // This is used by the Wasm runtime to cache compiled Wasm plugins. This is an @@ -209,6 +232,33 @@ func newModuleDataProvider( ), nil } +func newPluginDataProvider( + container appext.Container, + clientProvider bufapi.ClientProvider, +) (bufplugin.PluginDataProvider, error) { + if err := createCacheDir(container.CacheDirPath(), v3CachePluginRelDirPath); err != nil { + return nil, err + } + fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CachePluginRelDirPath) + storageosProvider := storageos.NewProvider() // No symlinks. + cacheBucket, err := storageosProvider.NewReadWriteBucket(fullCacheDirPath) + if err != nil { + return nil, err + } + delegateModuleDataProvider := bufpluginapi.NewPluingDataProvider( + container.Logger(), + clientProvider, + ) + return bufplugincache.NewPluginDataProvider( + container.Logger(), + delegateModuleDataProvider, + bufpluginstore.NewPluginDataStore( + container.Logger(), + cacheBucket, + ), + ), nil +} + func newCommitProvider( container appext.Container, clientProvider bufapi.ClientProvider, diff --git a/private/buf/bufcli/controller.go b/private/buf/bufcli/controller.go index 05ae2048f2..e33d23f70a 100644 --- a/private/buf/bufcli/controller.go +++ b/private/buf/bufcli/controller.go @@ -18,6 +18,7 @@ import ( "github.com/bufbuild/buf/private/buf/bufctl" "github.com/bufbuild/buf/private/bufpkg/bufapi" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi" "github.com/bufbuild/buf/private/pkg/app/appext" ) @@ -49,6 +50,10 @@ func NewController( if err != nil { return nil, err } + pluginDataProvider, err := newPluginDataProvider(container, clientProvider) + if err != nil { + return nil, err + } return bufctl.NewController( container.Logger(), container, @@ -56,6 +61,8 @@ func NewController( bufmoduleapi.NewModuleKeyProvider(container.Logger(), clientProvider), moduleDataProvider, commitProvider, + bufpluginapi.NewPluginKeyProvider(container.Logger(), clientProvider), + pluginDataProvider, wktStore, // TODO FUTURE: Delete defaultHTTPClient and use the one from newConfig defaultHTTPClient, diff --git a/private/buf/bufcli/errors.go b/private/buf/bufcli/errors.go index 7d57f4863a..5978cd6b9e 100644 --- a/private/buf/bufcli/errors.go +++ b/private/buf/bufcli/errors.go @@ -54,6 +54,12 @@ func NewLabelNameAlreadyExistsError(name string) error { return fmt.Errorf("a label named %q already exists", name) } +// NewPluginNameAlreadyExistsError informs the user that a plugin +// with that name already exists. +func NewPluginNameAlreadyExistsError(name string) error { + return fmt.Errorf("a plugin named %q already exists", name) +} + // NewOrganizationNotFoundError informs the user that an organization with // that name does not exist. func NewOrganizationNotFoundError(name string) error { @@ -88,6 +94,12 @@ func NewTokenNotFoundError(tokenID string) error { return fmt.Errorf("a token with ID %q does not exist", tokenID) } +// NewPluginNotFoundError informs the user that a plugin with +// that name does not exist. +func NewPluginNotFoundError(name string) error { + return fmt.Errorf("a plugin named %q does not exist", name) +} + // NewInvalidRemoteError informs the user that the given remote is invalid. func NewInvalidRemoteError(err error, remote string, moduleFullName string) error { var connectErr *connect.Error diff --git a/private/buf/bufcli/flags_args.go b/private/buf/bufcli/flags_args.go index a2238f7c67..e8d2ca56f7 100644 --- a/private/buf/bufcli/flags_args.go +++ b/private/buf/bufcli/flags_args.go @@ -19,6 +19,7 @@ import ( "fmt" modulev1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/module/v1" + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" "github.com/bufbuild/buf/private/buf/buffetch" "github.com/bufbuild/buf/private/pkg/app" "github.com/bufbuild/buf/private/pkg/app/appcmd" @@ -302,6 +303,21 @@ func VisibilityFlagToVisibilityAllowUnspecified(visibility string) (modulev1.Mod } } +// VisibilityFlagToPluginVisibilityAllowUnspecified parses the given string as a pluginv1.PluginVisibility +// where an empty string will be parsed as unspecified. +func VisibilityFlagToPluginVisibilityAllowUnspecified(visibility string) (pluginv1beta1.PluginVisibility, error) { + switch visibility { + case publicVisibility: + return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_PUBLIC, nil + case privateVisibility: + return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_PRIVATE, nil + case "": + return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_UNSPECIFIED, nil + default: + return 0, fmt.Errorf("invalid visibility: %s", visibility) + } +} + // ArchiveStatusFlagToArchiveStatusFilter parses the given string as a modulev1.ListLabelsRequest_ArchiveFilter. func ArchiveStatusFlagToArchiveStatusFilter(archiveStatus string) (modulev1.ListLabelsRequest_ArchiveFilter, error) { switch archiveStatus { diff --git a/private/buf/bufcli/plugin_key_provider.go b/private/buf/bufcli/plugin_key_provider.go new file mode 100644 index 0000000000..f1f2577286 --- /dev/null +++ b/private/buf/bufcli/plugin_key_provider.go @@ -0,0 +1,36 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufcli + +import ( + "github.com/bufbuild/buf/private/bufpkg/bufapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi" + "github.com/bufbuild/buf/private/pkg/app/appext" +) + +// NewPluginKeyProvider returns a new PluginKeyProvider. +func NewPluginKeyProvider(container appext.Container) (bufplugin.PluginKeyProvider, error) { + clientConfig, err := NewConnectClientConfig(container) + if err != nil { + return nil, err + } + return bufpluginapi.NewPluginKeyProvider( + container.Logger(), + bufapi.NewClientProvider( + clientConfig, + ), + ), nil +} diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index 0e1322348a..752f286df5 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -33,6 +33,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/bufpkg/bufreflect" "github.com/bufbuild/buf/private/gen/data/datawkt" imagev1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/image/v1" @@ -59,6 +60,8 @@ type ImageWithConfig interface { LintConfig() bufconfig.LintConfig BreakingConfig() bufconfig.BreakingConfig PluginConfigs() []bufconfig.PluginConfig + PluginKeyProvider() bufplugin.PluginKeyProvider + PluginDataProvider() bufplugin.PluginDataProvider isImageWithConfig() } @@ -137,6 +140,8 @@ func NewController( moduleKeyProvider bufmodule.ModuleKeyProvider, moduleDataProvider bufmodule.ModuleDataProvider, commitProvider bufmodule.CommitProvider, + pluginKeyProvider bufplugin.PluginKeyProvider, + pluginDataProvider bufplugin.PluginDataProvider, wktStore bufwktstore.Store, httpClient *http.Client, httpauthAuthenticator httpauth.Authenticator, @@ -150,6 +155,8 @@ func NewController( moduleKeyProvider, moduleDataProvider, commitProvider, + pluginKeyProvider, + pluginDataProvider, wktStore, httpClient, httpauthAuthenticator, @@ -171,6 +178,8 @@ type controller struct { moduleDataProvider bufmodule.ModuleDataProvider graphProvider bufmodule.GraphProvider commitProvider bufmodule.CommitProvider + pluginKeyProvider bufplugin.PluginKeyProvider + pluginDataProvider bufplugin.PluginDataProvider wktStore bufwktstore.Store disableSymlinks bool @@ -194,6 +203,8 @@ func newController( moduleKeyProvider bufmodule.ModuleKeyProvider, moduleDataProvider bufmodule.ModuleDataProvider, commitProvider bufmodule.CommitProvider, + pluginKeyProvider bufplugin.PluginKeyProvider, + pluginDataProvider bufplugin.PluginDataProvider, wktStore bufwktstore.Store, httpClient *http.Client, httpauthAuthenticator httpauth.Authenticator, @@ -206,6 +217,7 @@ func newController( graphProvider: graphProvider, moduleDataProvider: moduleDataProvider, commitProvider: commitProvider, + pluginKeyProvider: pluginKeyProvider, wktStore: wktStore, } for _, option := range options { @@ -414,6 +426,8 @@ func (c *controller) GetTargetImageWithConfigs( lintConfig, breakingConfig, pluginConfigs, + c.pluginKeyProvider, + c.pluginDataProvider, ), }, nil default: @@ -1064,6 +1078,9 @@ func (c *controller) buildTargetImageWithConfigs( workspace.GetLintConfigForOpaqueID(module.OpaqueID()), workspace.GetBreakingConfigForOpaqueID(module.OpaqueID()), workspace.PluginConfigs(), + // TODO(ed): plugin key provider and data provider should come from the workspace. + c.pluginKeyProvider, // KeyProvider should read directly from the lockfile. + c.pluginDataProvider, ), ) } diff --git a/private/buf/bufctl/image_with_config.go b/private/buf/bufctl/image_with_config.go index 01fd747cad..0b2cf475dc 100644 --- a/private/buf/bufctl/image_with_config.go +++ b/private/buf/bufctl/image_with_config.go @@ -17,14 +17,17 @@ package bufctl import ( "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" ) type imageWithConfig struct { bufimage.Image - lintConfig bufconfig.LintConfig - breakingConfig bufconfig.BreakingConfig - pluginConfigs []bufconfig.PluginConfig + lintConfig bufconfig.LintConfig + breakingConfig bufconfig.BreakingConfig + pluginConfigs []bufconfig.PluginConfig + pluginKeyProvider bufplugin.PluginKeyProvider + pluginDataProvider bufplugin.PluginDataProvider } func newImageWithConfig( @@ -32,12 +35,16 @@ func newImageWithConfig( lintConfig bufconfig.LintConfig, breakingConfig bufconfig.BreakingConfig, pluginConfigs []bufconfig.PluginConfig, + pluginKeyProvider bufplugin.PluginKeyProvider, + pluginDataProvider bufplugin.PluginDataProvider, ) *imageWithConfig { return &imageWithConfig{ - Image: image, - lintConfig: lintConfig, - breakingConfig: breakingConfig, - pluginConfigs: pluginConfigs, + Image: image, + lintConfig: lintConfig, + breakingConfig: breakingConfig, + pluginConfigs: pluginConfigs, + pluginKeyProvider: pluginKeyProvider, + pluginDataProvider: pluginDataProvider, } } @@ -53,4 +60,12 @@ func (i *imageWithConfig) PluginConfigs() []bufconfig.PluginConfig { return i.pluginConfigs } +func (i *imageWithConfig) PluginKeyProvider() bufplugin.PluginKeyProvider { + return i.pluginKeyProvider +} + +func (i *imageWithConfig) PluginDataProvider() bufplugin.PluginDataProvider { + return i.pluginDataProvider +} + func (*imageWithConfig) isImageWithConfig() {} diff --git a/private/buf/bufmigrate/migrator.go b/private/buf/bufmigrate/migrator.go index 37d38bf5e9..ec07fb37e6 100644 --- a/private/buf/bufmigrate/migrator.go +++ b/private/buf/bufmigrate/migrator.go @@ -28,6 +28,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/normalpath" "github.com/bufbuild/buf/private/pkg/slicesext" @@ -406,6 +407,7 @@ func (m *migrator) buildBufYAMLAndBufLockFiles( bufLock, err = bufconfig.NewBufLockFile( bufconfig.FileVersionV2, resolvedLockEntries, + nil, ) if err != nil { return nil, nil, err @@ -450,6 +452,7 @@ func (m *migrator) buildBufYAMLAndBufLockFiles( bufLock, err = bufconfig.NewBufLockFile( bufconfig.FileVersionV2, resolvedDepModuleKeys, + nil, // Plugins not supported in v1. ) if err != nil { return nil, nil, err @@ -704,7 +707,7 @@ func equivalentCheckConfigInV2( ) (bufconfig.CheckConfig, error) { // No need for custom lint/breaking plugins since there's no plugins to migrate from <=v1. // TODO: If we ever need v3, then we will have to deal with this. - client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(runner, wasm.UnimplementedRuntime)) + client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(runner, wasm.UnimplementedRuntime, bufplugin.NopPluginKeyProvider, bufplugin.NopPluginDataProvider)) if err != nil { return nil, err } diff --git a/private/buf/bufprint/bufprint.go b/private/buf/bufprint/bufprint.go index 1647a41f93..7ac632ff2c 100644 --- a/private/buf/bufprint/bufprint.go +++ b/private/buf/bufprint/bufprint.go @@ -26,7 +26,9 @@ import ( modulev1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/module/v1" ownerv1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/owner/v1" + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" registryv1alpha1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/registry/v1alpha1" "github.com/bufbuild/buf/private/pkg/protoencoding" "github.com/bufbuild/buf/private/pkg/protostat" @@ -248,6 +250,18 @@ func NewOrganizationEntity(organization *ownerv1.Organization, remote string) En } } +// NewPluginEntity returns a new plugin entity to print. +func NewPluginEntity(plugin *pluginv1beta1.Plugin, pluginFullName bufplugin.PluginFullName) Entity { + return outputPlugin{ + ID: plugin.Id, + Remote: pluginFullName.Registry(), + Owner: pluginFullName.Owner(), + Name: pluginFullName.Name(), + FullName: pluginFullName.String(), + CreateTime: plugin.CreateTime.AsTime(), + } +} + // CuratedPluginPrinter is a printer for curated plugins. type CuratedPluginPrinter interface { PrintCuratedPlugin(ctx context.Context, format Format, plugin *registryv1alpha1.CuratedPlugin) error @@ -456,3 +470,16 @@ type outputOrganization struct { func (o outputOrganization) fullName() string { return o.FullName } + +type outputPlugin struct { + ID string `json:"id,omitempty"` + Remote string `json:"remote,omitempty"` + Owner string `json:"owner,omitempty"` + Name string `json:"name,omitempty"` + FullName string `json:"-" bufprint:"Name"` + CreateTime time.Time `json:"create_time,omitempty" bufprint:"Create Time"` +} + +func (m outputPlugin) fullName() string { + return m.FullName +} diff --git a/private/buf/bufworkspace/workspace_dep_manager.go b/private/buf/bufworkspace/workspace_dep_manager.go index ddf6901c2a..4758a73af4 100644 --- a/private/buf/bufworkspace/workspace_dep_manager.go +++ b/private/buf/bufworkspace/workspace_dep_manager.go @@ -21,6 +21,8 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/syserror" ) @@ -42,7 +44,7 @@ type WorkspaceDepManager interface { // the given ModuleKeys. // // If a buf.lock does not exist, one will be created. - UpdateBufLockFile(ctx context.Context, depModuleKeys []bufmodule.ModuleKey) error + UpdateBufLockFile(ctx context.Context, depModuleKeys []bufmodule.ModuleKey, depPluginKeys []bufplugin.PluginKey) error // ConfiguredDepModuleRefs returns the configured dependencies of the Workspace as ModuleRefs. // // These come from buf.yaml files. @@ -55,6 +57,8 @@ type WorkspaceDepManager interface { // // Sorted. ConfiguredDepModuleRefs(ctx context.Context) ([]bufmodule.ModuleRef, error) + // TODO(ed) + ConfiguredRemotePluginRefs(ctx context.Context) ([]bufplugin.PluginRef, error) isWorkspaceDepManager() } @@ -117,6 +121,41 @@ func (w *workspaceDepManager) ConfiguredDepModuleRefs(ctx context.Context) ([]bu return bufYAMLFile.ConfiguredDepModuleRefs(), nil } +func (w *workspaceDepManager) ConfiguredRemotePluginRefs(ctx context.Context) ([]bufplugin.PluginRef, error) { + bufYAMLFile, err := bufconfig.GetBufYAMLFileForPrefix(ctx, w.bucket, w.targetSubDirPath) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + } + if bufYAMLFile == nil { + return nil, nil + } + switch fileVersion := bufYAMLFile.FileVersion(); fileVersion { + case bufconfig.FileVersionV1Beta1, bufconfig.FileVersionV1: + if w.isV2 { + return nil, syserror.Newf("buf.yaml at %q did had version %v but expected v1beta1, v1", w.targetSubDirPath, fileVersion) + } + case bufconfig.FileVersionV2: + if !w.isV2 { + return nil, syserror.Newf("buf.yaml at %q did had version %v but expected v12", w.targetSubDirPath, fileVersion) + } + default: + return nil, syserror.Newf("unknown FileVersion: %v", fileVersion) + } + return slicesext.Filter( + slicesext.Map( + bufYAMLFile.PluginConfigs(), + func(value bufconfig.PluginConfig) bufplugin.PluginRef { + return value.PluginRef() + }, + ), + func(value bufplugin.PluginRef) bool { + return value != nil + }, + ), nil +} + func (w *workspaceDepManager) BufLockFileDigestType() bufmodule.DigestType { if w.isV2 { return bufmodule.DigestTypeB5 @@ -135,11 +174,11 @@ func (w *workspaceDepManager) ExistingBufLockFileDepModuleKeys(ctx context.Conte return bufLockFile.DepModuleKeys(), nil } -func (w *workspaceDepManager) UpdateBufLockFile(ctx context.Context, depModuleKeys []bufmodule.ModuleKey) error { +func (w *workspaceDepManager) UpdateBufLockFile(ctx context.Context, depModuleKeys []bufmodule.ModuleKey, depPluginKeys []bufplugin.PluginKey) error { var bufLockFile bufconfig.BufLockFile var err error if w.isV2 { - bufLockFile, err = bufconfig.NewBufLockFile(bufconfig.FileVersionV2, depModuleKeys) + bufLockFile, err = bufconfig.NewBufLockFile(bufconfig.FileVersionV2, depModuleKeys, depPluginKeys) if err != nil { return err } @@ -153,7 +192,10 @@ func (w *workspaceDepManager) UpdateBufLockFile(ctx context.Context, depModuleKe } else { fileVersion = existingBufYAMLFile.FileVersion() } - bufLockFile, err = bufconfig.NewBufLockFile(fileVersion, depModuleKeys) + if len(depPluginKeys) > 0 { + return syserror.Newf("plugins are not supported for v1beta1/v1 buf.yaml files") + } + bufLockFile, err = bufconfig.NewBufLockFile(fileVersion, depModuleKeys, nil) if err != nil { return err } diff --git a/private/buf/cmd/buf/buf.go b/private/buf/cmd/buf/buf.go index 8094965342..3ace22c11d 100644 --- a/private/buf/cmd/buf/buf.go +++ b/private/buf/cmd/buf/buf.go @@ -35,8 +35,8 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/bufpluginv2" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/lsp" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/price" - "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/plugin/plugindelete" - "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/plugin/pluginpush" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/plugin/betaplugindelete" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/plugin/betapluginpush" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/webhook/webhookcreate" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/webhook/webhookdelete" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/webhook/webhooklist" @@ -62,6 +62,7 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modlsbreakingrules" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modlslintrules" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modopen" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/plugin/pluginpush" "github.com/bufbuild/buf/private/buf/cmd/buf/command/push" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/commit/commitaddlabel" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/commit/commitinfo" @@ -81,6 +82,10 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/organization/organizationdelete" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/organization/organizationinfo" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/organization/organizationupdate" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/plugin/plugincreate" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/plugin/plugindelete" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/plugin/plugininfo" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/plugin/pluginupdate" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/registrycc" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/registrylogin" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/registrylogout" @@ -224,6 +229,16 @@ func NewRootCommand(name string) *appcmd.Command { moduleupdate.NewCommand("update", builder), }, }, + { + Use: "plugin", + Short: "Manage BSR plugins", + SubCommands: []*appcmd.Command{ + plugincreate.NewCommand("create", builder), + plugininfo.NewCommand("info", builder), + plugindelete.NewCommand("delete", builder), + pluginupdate.NewCommand("update", builder), + }, + }, }, }, { @@ -254,8 +269,8 @@ func NewRootCommand(name string) *appcmd.Command { Use: "plugin", Short: "Manage plugins on the Buf Schema Registry", SubCommands: []*appcmd.Command{ - pluginpush.NewCommand("push", builder), - plugindelete.NewCommand("delete", builder), + betapluginpush.NewCommand("push", builder), + betaplugindelete.NewCommand("delete", builder), }, }, }, @@ -285,6 +300,13 @@ func NewRootCommand(name string) *appcmd.Command { }, }, }, + { + Use: "plugin", + Short: "Work with buf plugins", + SubCommands: []*appcmd.Command{ + pluginpush.NewCommand("push", builder), + }, + }, }, } } diff --git a/private/buf/cmd/buf/buf_test.go b/private/buf/cmd/buf/buf_test.go index 18fe6e26c8..5d5ebe951e 100644 --- a/private/buf/cmd/buf/buf_test.go +++ b/private/buf/cmd/buf/buf_test.go @@ -35,6 +35,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" imagev1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/image/v1" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appcmd/appcmdtesting" @@ -1349,7 +1350,7 @@ func TestCheckLsBreakingRulesFromConfigExceptDeprecated(t *testing.T) { t.Run(version.String(), func(t *testing.T) { t.Parallel() // Do not need any custom lint/breaking plugins here. - client, err := bufcheck.NewClient(slogtestext.NewLogger(t), bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime)) + client, err := bufcheck.NewClient(slogtestext.NewLogger(t), bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime, bufplugin.NopPluginKeyProvider, bufplugin.NopPluginDataProvider)) require.NoError(t, err) allRules, err := client.AllRules(context.Background(), check.RuleTypeBreaking, version) require.NoError(t, err) diff --git a/private/buf/cmd/buf/command/beta/lsp/lsp.go b/private/buf/cmd/buf/command/beta/lsp/lsp.go index fabb0c91be..01e74141ba 100644 --- a/private/buf/cmd/buf/command/beta/lsp/lsp.go +++ b/private/buf/cmd/buf/command/beta/lsp/lsp.go @@ -26,6 +26,7 @@ import ( "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/buf/buflsp" "github.com/bufbuild/buf/private/bufpkg/bufcheck" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/command" @@ -107,7 +108,13 @@ func run( }() checkClient, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime), + bufcheck.NewRunnerProvider( + command.NewRunner(), + wasmRuntime, + // TODO(ed) + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), bufcheck.ClientWithStderr(container.Stderr()), ) if err != nil { diff --git a/private/buf/cmd/buf/command/beta/registry/plugin/plugindelete/plugindelete.go b/private/buf/cmd/buf/command/beta/registry/plugin/betaplugindelete/betaplugindelete.go similarity index 99% rename from private/buf/cmd/buf/command/beta/registry/plugin/plugindelete/plugindelete.go rename to private/buf/cmd/buf/command/beta/registry/plugin/betaplugindelete/betaplugindelete.go index 1cb902872b..fba524099e 100644 --- a/private/buf/cmd/buf/command/beta/registry/plugin/plugindelete/plugindelete.go +++ b/private/buf/cmd/buf/command/beta/registry/plugin/betaplugindelete/betaplugindelete.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package plugindelete +package betaplugindelete import ( "context" diff --git a/private/buf/cmd/buf/command/beta/registry/plugin/betaplugindelete/usage.gen.go b/private/buf/cmd/buf/command/beta/registry/plugin/betaplugindelete/usage.gen.go new file mode 100644 index 0000000000..9a2d9d14bd --- /dev/null +++ b/private/buf/cmd/buf/command/beta/registry/plugin/betaplugindelete/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package betaplugindelete + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/cmd/buf/command/beta/registry/plugin/pluginpush/pluginpush.go b/private/buf/cmd/buf/command/beta/registry/plugin/betapluginpush/betapluginpush.go similarity index 99% rename from private/buf/cmd/buf/command/beta/registry/plugin/pluginpush/pluginpush.go rename to private/buf/cmd/buf/command/beta/registry/plugin/betapluginpush/betapluginpush.go index 904768431a..19752e0567 100644 --- a/private/buf/cmd/buf/command/beta/registry/plugin/pluginpush/pluginpush.go +++ b/private/buf/cmd/buf/command/beta/registry/plugin/betapluginpush/betapluginpush.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pluginpush +package betapluginpush import ( "context" diff --git a/private/buf/cmd/buf/command/beta/registry/plugin/betapluginpush/usage.gen.go b/private/buf/cmd/buf/command/beta/registry/plugin/betapluginpush/usage.gen.go new file mode 100644 index 0000000000..4b0b4d8b75 --- /dev/null +++ b/private/buf/cmd/buf/command/beta/registry/plugin/betapluginpush/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package betapluginpush + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/cmd/buf/command/breaking/breaking.go b/private/buf/cmd/buf/command/breaking/breaking.go index 0cfd5c5a01..a7cb8aadc3 100644 --- a/private/buf/cmd/buf/command/breaking/breaking.go +++ b/private/buf/cmd/buf/command/breaking/breaking.go @@ -25,6 +25,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/command" @@ -222,7 +223,13 @@ func run( for i, imageWithConfig := range imageWithConfigs { client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime), + bufcheck.NewRunnerProvider( + command.NewRunner(), + wasmRuntime, + // TODO(ed) + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), bufcheck.ClientWithStderr(container.Stderr()), ) if err != nil { diff --git a/private/buf/cmd/buf/command/config/internal/internal.go b/private/buf/cmd/buf/command/config/internal/internal.go index 959907bd70..1145cf0501 100644 --- a/private/buf/cmd/buf/command/config/internal/internal.go +++ b/private/buf/cmd/buf/command/config/internal/internal.go @@ -24,6 +24,7 @@ import ( "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/command" @@ -198,7 +199,13 @@ func lsRun( }() client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime), + bufcheck.NewRunnerProvider( + command.NewRunner(), + wasmRuntime, + // TODO(ed) + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), bufcheck.ClientWithStderr(container.Stderr()), ) if err != nil { diff --git a/private/buf/cmd/buf/command/dep/depupdate/depupdate.go b/private/buf/cmd/buf/command/dep/depupdate/depupdate.go index 7c536b0334..db5990a5d6 100644 --- a/private/buf/cmd/buf/command/dep/depupdate/depupdate.go +++ b/private/buf/cmd/buf/command/dep/depupdate/depupdate.go @@ -124,13 +124,25 @@ func run( "all deps", slog.Any("deps", slicesext.Map(configuredDepModuleKeys, bufmodule.ModuleKey.String)), ) + configuredRemotePluginRefs, err := workspaceDepManager.ConfiguredRemotePluginRefs(ctx) + if err != nil { + return err + } + configuredRemotePluginKeys, err := internal.PluginKeysForPluginRefs( + ctx, + container, + configuredRemotePluginRefs, + ) + if err != nil { + return err + } // Store the existing buf.lock data. existingDepModuleKeys, err := workspaceDepManager.ExistingBufLockFileDepModuleKeys(ctx) if err != nil { return err } - if configuredDepModuleKeys == nil && existingDepModuleKeys == nil { + if configuredDepModuleKeys == nil && existingDepModuleKeys == nil && configuredRemotePluginKeys == nil { // No new configured deps were found, and no existing buf.lock deps were found, so there // is nothing to update, we can return here. // This ensures we do not create an empty buf.lock when one did not exist in the first @@ -148,11 +160,11 @@ func run( // overlay the new buf.lock file in a union bucket. defer func() { if retErr != nil { - retErr = multierr.Append(retErr, workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys)) + retErr = multierr.Append(retErr, workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, nil /* TODO(ed) */)) } }() // Edit the buf.lock file with the unpruned dependencies. - if err := workspaceDepManager.UpdateBufLockFile(ctx, configuredDepModuleKeys); err != nil { + if err := workspaceDepManager.UpdateBufLockFile(ctx, configuredDepModuleKeys, configuredRemotePluginKeys); err != nil { return err } workspace, err := controller.GetWorkspace(ctx, dirPath, bufctl.WithIgnoreAndDisallowV1BufWorkYAMLs()) diff --git a/private/buf/cmd/buf/command/dep/internal/internal.go b/private/buf/cmd/buf/command/dep/internal/internal.go index b68a2e0b62..362697927e 100644 --- a/private/buf/cmd/buf/command/dep/internal/internal.go +++ b/private/buf/cmd/buf/command/dep/internal/internal.go @@ -23,6 +23,7 @@ import ( "github.com/bufbuild/buf/private/buf/bufctl" "github.com/bufbuild/buf/private/buf/bufworkspace" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/syserror" @@ -51,6 +52,24 @@ func ModuleKeysAndTransitiveDepModuleKeysForModuleRefs( return moduleKeysAndTransitiveDepModuleKeysForModuleKeys(ctx, container, moduleKeys) } +// TODO(ed) +func PluginKeysForPluginRefs( + ctx context.Context, + container appext.Container, + pluginRefs []bufplugin.PluginRef, +) ([]bufplugin.PluginKey, error) { + pluginKeyProvider, err := bufcli.NewPluginKeyProvider(container) + if err != nil { + return nil, err + } + pluginKeys, err := pluginKeyProvider.GetPluginKeysForPluginRefs( + ctx, + pluginRefs, + bufplugin.DigestTypeP1, + ) + return pluginKeys, err +} + // Prune prunes the buf.lock. // // Used by dep/mod prune. @@ -105,7 +124,7 @@ func Prune( if err := validateModuleKeysContains(bufYAMLBasedDepModuleKeys, depModuleKeys); err != nil { return err } - return workspaceDepManager.UpdateBufLockFile(ctx, depModuleKeys) + return workspaceDepManager.UpdateBufLockFile(ctx, depModuleKeys, nil /* TODO(ed) */) } // LogUnusedConfiugredDepsForWorkspace takes a workspace and logs the unused configured diff --git a/private/buf/cmd/buf/command/lint/lint.go b/private/buf/cmd/buf/command/lint/lint.go index 42066f96c1..99f260171a 100644 --- a/private/buf/cmd/buf/command/lint/lint.go +++ b/private/buf/cmd/buf/command/lint/lint.go @@ -147,7 +147,12 @@ func run( for _, imageWithConfig := range imageWithConfigs { client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime), + bufcheck.NewRunnerProvider( + command.NewRunner(), + wasmRuntime, + imageWithConfig.PluginKeyProvider(), + imageWithConfig.PluginDataProvider(), + ), bufcheck.ClientWithStderr(container.Stderr()), ) if err != nil { diff --git a/private/buf/cmd/buf/command/mod/internal/internal.go b/private/buf/cmd/buf/command/mod/internal/internal.go index accfd9aec8..35544b3a14 100644 --- a/private/buf/cmd/buf/command/mod/internal/internal.go +++ b/private/buf/cmd/buf/command/mod/internal/internal.go @@ -24,6 +24,7 @@ import ( "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/command" @@ -176,7 +177,12 @@ func lsRun( // BufYAMLFiles <=v1 never had plugins. client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime), + bufcheck.NewRunnerProvider( + command.NewRunner(), + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), bufcheck.ClientWithStderr(container.Stderr()), ) if err != nil { diff --git a/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go new file mode 100644 index 0000000000..69ae17b8cc --- /dev/null +++ b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go @@ -0,0 +1,614 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pluginpush + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "strings" + + "buf.build/gen/go/bufbuild/registry/connectrpc/go/buf/registry/plugin/v1beta1/pluginv1beta1connect" + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/bufpkg/bufapi" + "github.com/bufbuild/buf/private/bufpkg/bufcas" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin" + "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin/bufremotepluginconfig" + "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin/bufremoteplugindocker" + "github.com/bufbuild/buf/private/pkg/app/appcmd" + "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/netrc" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/storage/storageos" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/uuidutil" + "github.com/bufbuild/buf/private/pkg/wasm" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + pkgv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/klauspost/compress/zstd" + "github.com/spf13/pflag" + "go.uber.org/multierr" +) + +const ( + disableSymlinksFlagName = "disable-symlinks" + labelFlagName = "label" + imageFlagName = "image" + imageConfigFlagName = "image-config" + binaryFlagName = "binary" + sourceControlURLFlagName = "source-control-url" +) + +// NewCommand returns a new Command. +func NewCommand( + name string, + builder appext.SubCommandBuilder, +) *appcmd.Command { + flags := newFlags() + return &appcmd.Command{ + Use: name + " ", + Short: "Push a plugin to a registry", + Long: `The first argument is the plugin full name in the format .`, + Args: appcmd.MaximumNArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appext.Container) error { + return run(ctx, container, flags) + }, + ), + BindFlags: flags.Bind, + } +} + +type flags struct { + DisableSymlinks bool + Labels []string + Image string + ImageConfig string + Binary string + SourceControlURL string +} + +func newFlags() *flags { + return &flags{} +} + +func (f *flags) Bind(flagSet *pflag.FlagSet) { + bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) + flagSet.StringVar( + &f.Image, + imageFlagName, + "", + "Push the plugin docker image to the registry.", + ) + flagSet.StringVar( + &f.ImageConfig, + imageConfigFlagName, + "", + fmt.Sprintf( + "Set the plugin image config. Must be set if --%s is set.", + imageFlagName, + ), + ) + flagSet.StringVar( + &f.Binary, + binaryFlagName, + "", + "Push the plugin binary to the registry.", + ) + flagSet.StringSliceVar( + &f.Labels, + labelFlagName, + nil, + "Associate the label with the plugins pushed. Can be used multiple times.", + ) + flagSet.StringVar( + &f.SourceControlURL, + sourceControlURLFlagName, + "", + "The URL for viewing the source code of the pushed modules (e.g. the specific commit in source control).", + ) +} + +func run( + ctx context.Context, + container appext.Container, + flags *flags, +) (retErr error) { + if err := validateFlags(flags); err != nil { + return err + } + // We parse the plugin full name from the user-provided argument. + pluginFullName, err := bufplugin.ParsePluginFullName(container.Arg(0)) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + + clientConfig, err := bufcli.NewConnectClientConfig(container) + if err != nil { + return err + } + uploadServiceClient := bufapi.NewClientProvider(clientConfig). + PluginV1Beta1UploadServiceClient(pluginFullName.Registry()) + + pluginKey, err := upload(ctx, container, flags, pluginFullName, uploadServiceClient) + if err != nil { + return err + } + // Only one plugin key is returned. + if _, err := fmt.Fprintf(container.Stdout(), "%s\n", pluginKey.String()); err != nil { + return syserror.Wrap(err) + } + return nil +} + +func upload( + ctx context.Context, + container appext.Container, + flags *flags, + pluginFullName bufplugin.PluginFullName, + uploadServiceClient pluginv1beta1connect.UploadServiceClient, +) (_ bufplugin.PluginKey, retErr error) { + // TODO: we should get the plugin here and check the type of upload + // is supported. Also dedupe digest here before doing the push. + + switch { + case flags.Image != "": + return uploadImage(ctx, container, flags, pluginFullName, uploadServiceClient) + case flags.Binary != "": + return uploadBinary(ctx, container, flags, pluginFullName, uploadServiceClient) + default: + // This should never happen because the flags are validated. + return nil, syserror.Newf("either --%s or --%s must be set", imageFlagName, binaryFlagName) + } +} + +func uploadBinary( + ctx context.Context, + container appext.Container, + flags *flags, + pluginFullName bufplugin.PluginFullName, + uploadServiceClient pluginv1beta1connect.UploadServiceClient, +) (_ bufplugin.PluginKey, retErr error) { + wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) + if err != nil { + return nil, err + } + wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir)) + if err != nil { + return nil, err + } + defer func() { + retErr = multierr.Append(retErr, wasmRuntime.Close(ctx)) + }() + + // Load the binary from the `--binary` flag. + wasmBinary, err := os.ReadFile(flags.Binary) + if err != nil { + return nil, fmt.Errorf("could not read binary %q: %w", flags.Binary, err) + } + + // Maybe validate the binary is a valid plugin binary? + _, err = wasmRuntime.Compile(ctx, pluginFullName.Name(), wasmBinary) + if err != nil { + return nil, fmt.Errorf("could not compile binary %q: %w", flags.Binary, err) + } + + // Upload the binary to the registry. + content := &pluginv1beta1.UploadRequest_Content{ + PluginRef: &pluginv1beta1.PluginRef{ + Value: &pluginv1beta1.PluginRef_Name_{ + Name: &pluginv1beta1.PluginRef_Name{ + Owner: pluginFullName.Owner(), + Plugin: pluginFullName.Name(), + }, + }, + }, + CompressionType: pluginv1beta1.CompressionType_COMPRESSION_TYPE_ZSTD, + Content: zstdCompress(wasmBinary), + } + + uploadResponse, err := uploadServiceClient.Upload(ctx, connect.NewRequest(&pluginv1beta1.UploadRequest{ + Contents: []*pluginv1beta1.UploadRequest_Content{content}, + })) + if err != nil { + return nil, err + } + if len(uploadResponse.Msg.Commits) != 1 { + return nil, syserror.Newf("unexpected number of commits returned from server: %d", len(uploadResponse.Msg.Commits)) + } + + protoCommit := uploadResponse.Msg.Commits[0] + + commitID, err := uuidutil.FromDashless(protoCommit.Id) + if err != nil { + return nil, err + } + pluginKey, err := bufplugin.NewPluginKey( + pluginFullName, + commitID, + func() (bufplugin.Digest, error) { + return v1beta1ProtoToDigest(protoCommit.Digest) + }, + ) + if err != nil { + return nil, err + } + return pluginKey, nil +} + +// v1beta1ProtoToDigest converts the given proto Digest to a Digest. +// +// Validation is performed to ensure the DigestType is known, and the value +// is a valid digest value for the given DigestType. +func v1beta1ProtoToDigest(protoDigest *pluginv1beta1.Digest) (bufplugin.Digest, error) { + digestType, err := v1beta1ProtoToDigestType(protoDigest.Type) + if err != nil { + return nil, err + } + bufcasDigest, err := bufcas.NewDigest(protoDigest.Value) + if err != nil { + return nil, err + } + return bufplugin.NewDigest(digestType, bufcasDigest) +} + +var ( + v1beta1ProtoDigestTypeToDigestType = map[pluginv1beta1.DigestType]bufplugin.DigestType{ + pluginv1beta1.DigestType_DIGEST_TYPE_P1: bufplugin.DigestTypeP1, + } +) + +func v1beta1ProtoToDigestType(protoDigestType pluginv1beta1.DigestType) (bufplugin.DigestType, error) { + digestType, ok := v1beta1ProtoDigestTypeToDigestType[protoDigestType] + if !ok { + return 0, fmt.Errorf("unknown pluginv1beta1.DigestType: %v", protoDigestType) + } + return digestType, nil +} + +func uploadImage( + ctx context.Context, + container appext.Container, + flags *flags, + pluginFullName bufplugin.PluginFullName, + uploadServiceClient pluginv1beta1connect.UploadServiceClient, +) (_ bufplugin.PluginKey, retErr error) { + if flags.ImageConfig == "" { + return nil, appcmd.NewInvalidArgumentErrorf("--%s is required", imageConfigFlagName) + } + source, err := bufcli.GetInputValue(container, "" /* The input hashtag is not supported here */, ".") + if err != nil { + return nil, err + } + storageProvider := newStorageosProvider(flags.DisableSymlinks) + sourceBucket, err := storageProvider.NewReadWriteBucket(source) + if err != nil { + return nil, err + } + options := []bufremotepluginconfig.ConfigOption{ + bufremotepluginconfig.WithOverrideRemote(pluginFullName.Registry()), + } + pluginConfig, err := bufremotepluginconfig.GetConfigForBucket(ctx, sourceBucket, options...) + if err != nil { + return nil, err + } + + dockerClient, err := bufremoteplugindocker.NewClient(container.Logger(), bufcli.Version) + if err != nil { + return nil, err + } + defer func() { + retErr = multierr.Append(retErr, dockerClient.Close()) + }() + machine, err := netrc.GetMachineForName(container, pluginFullName.Registry()) + if err != nil { + return nil, err + } + authConfig := &bufremoteplugindocker.RegistryAuthConfig{} + if machine != nil { + authConfig.ServerAddress = machine.Name() + authConfig.Username = machine.Login() + authConfig.Password = machine.Password() + } + // Resolve the image reference. + dockerInspectResponse, err := dockerClient.Inspect(ctx, flags.Image) + if err != nil { + return nil, err + } + imageID := dockerInspectResponse.ImageID + + currentImageDigest := "" + { + // TODO: need to resolve the current image digest. + } + imageDigest, err := findExistingDigestForImageID(ctx, pluginFullName, authConfig, imageID, currentImageDigest) + if err != nil { + return nil, err + } + if imageDigest == "" { + imageDigest, err = pushImage(ctx, dockerClient, authConfig, pluginConfig, imageID) + if err != nil { + return nil, err + } + } + // TODO: log image digest wasn't pushed. + + plugin, err := bufremoteplugin.NewPlugin( + pluginConfig.PluginVersion, + pluginConfig.Dependencies, + pluginConfig.Registry, + imageDigest, + pluginConfig.SourceURL, + pluginConfig.Description, + ) + if err != nil { + return nil, err + } + // TODO: upload the image to the BSR + _ = plugin + content := &pluginv1beta1.UploadImageRequest_Content{ + PluginRef: &pluginv1beta1.PluginRef{ + Value: &pluginv1beta1.PluginRef_Name_{ + Name: &pluginv1beta1.PluginRef_Name{ + Owner: pluginFullName.Owner(), + Plugin: pluginFullName.Name(), + }, + }, + }, + Version: "", + Revision: 0, + LicenseUrl: "", + LicenseSpdxIdentifier: "", + CodeGeneration: &pluginv1beta1.CodeGenerationConfig{}, + } + _ = content + + return nil, fmt.Errorf("not implemented") +} + +func validateFlags(flags *flags) error { + if err := validateLabelFlags(flags); err != nil { + return err + } + if err := validateTypeFlags(flags); err != nil { + return err + } + return nil +} + +func validateLabelFlags(flags *flags) error { + return validateLabelFlagValues(flags) +} + +func validateTypeFlags(flags *flags) error { + var usedFlags []string + if flags.Image != "" { + usedFlags = append(usedFlags, imageFlagName) + } + if flags.Binary != "" { + usedFlags = append(usedFlags, binaryFlagName) + } + if len(usedFlags) > 1 { + usedFlagsErrStr := strings.Join( + slicesext.Map( + usedFlags, + func(flag string) string { return fmt.Sprintf("--%s", flag) }, + ), + ", ", + ) + return appcmd.NewInvalidArgumentErrorf("These flags cannot be used in combination with one another: %s", usedFlagsErrStr) + } + if flags.Image != "" && flags.ImageConfig == "" { + return appcmd.NewInvalidArgumentErrorf( + "--%s is required if --%s is set", + imageConfigFlagName, + imageFlagName, + ) + } + return nil +} + +func validateLabelFlagValues(flags *flags) error { + for _, label := range flags.Labels { + if label == "" { + return appcmd.NewInvalidArgumentErrorf("--%s requires a non-empty string", labelFlagName) + } + } + return nil +} + +var ( + zstdEncoder, _ = zstd.NewWriter(nil) +) + +func zstdCompress(src []byte) []byte { + return zstdEncoder.EncodeAll(src, make([]byte, 0, len(src))) +} + +func newStorageosProvider(disableSymlinks bool) storageos.Provider { + var options []storageos.ProviderOption + if !disableSymlinks { + options = append(options, storageos.ProviderWithSymlinks()) + } + return storageos.NewProvider(options...) +} + +// pushImage pushes the image to the OCI registry. It returns the digest of the +// pushed image. +func pushImage( + ctx context.Context, + dockerClient bufremoteplugindocker.Client, + authConfig *bufremoteplugindocker.RegistryAuthConfig, + pluginConfig *bufremotepluginconfig.Config, + image string, +) (_ string, retErr error) { + tagResponse, err := dockerClient.Tag(ctx, image, pluginConfig) + if err != nil { + return "", err + } + createdImage := tagResponse.Image + // We tag a Docker image using a unique ID label each time. + // After we're done publishing the image, we delete it to not leave a lot of images left behind. + defer func() { + if _, err := dockerClient.Delete(ctx, createdImage); err != nil { + retErr = multierr.Append(retErr, fmt.Errorf("failed to delete image %q", createdImage)) + } + }() + pushResponse, err := dockerClient.Push(ctx, createdImage, authConfig) + if err != nil { + return "", err + } + return pushResponse.Digest, nil +} + +// findExistingDigestForImageID will query the OCI registry to see if the imageID already exists. +// If an image is found with the same imageID, its digest will be returned (and we'll skip pushing to OCI registry). +// +// It performs the following search: +// +// - GET /v2/{owner}/{plugin}/tags/list +// - For each tag: +// - Fetch image: GET /v2/{owner}/{plugin}/manifests/{tag} +// - If image manifest matches imageID, we can use the image digest for the image. +func findExistingDigestForImageID( + ctx context.Context, + pluginFullName bufplugin.PluginFullName, + authConfig *bufremoteplugindocker.RegistryAuthConfig, + imageID string, + currentImageDigest string, +) (string, error) { + repo, err := name.NewRepository(pluginFullName.String()) + if err != nil { + return "", err + } + auth := &authn.Basic{Username: authConfig.Username, Password: authConfig.Password} + remoteOpts := []remote.Option{remote.WithContext(ctx), remote.WithAuth(auth)} + // First attempt to see if the current image digest matches the image ID + if currentImageDigest != "" { + remoteImageID, _, err := getImageIDAndDigestFromReference(ctx, repo.Digest(currentImageDigest), remoteOpts...) + if err != nil { + return "", err + } + if remoteImageID == imageID { + return currentImageDigest, nil + } + } + // List all tags and check for a match + tags, err := remote.List(repo, remoteOpts...) + if err != nil { + structuredErr := new(transport.Error) + if errors.As(err, &structuredErr) { + if structuredErr.StatusCode == http.StatusUnauthorized { + return "", errors.New("you are not authenticated. For details, visit https://buf.build/docs/bsr/authentication") + } + if structuredErr.StatusCode == http.StatusNotFound { + return "", nil + } + } + return "", err + } + for _, tag := range tags { + remoteImageID, imageDigest, err := getImageIDAndDigestFromReference(ctx, repo.Tag(tag), remoteOpts...) + if err != nil { + return "", err + } + if remoteImageID == imageID { + return imageDigest, nil + } + } + return "", nil +} + +// getImageIDAndDigestFromReference takes an image reference and returns 2 resolved digests: +// +// 1. The image config digest (https://github.com/opencontainers/image-spec/blob/v1.1.0/config.md) +// 2. The image manifest digest (https://github.com/opencontainers/image-spec/blob/v1.1.0/manifest.md) +// +// The incoming ref is expected to be either an image manifest digest or an image index digest. +func getImageIDAndDigestFromReference( + ctx context.Context, + ref name.Reference, + options ...remote.Option, +) (string, string, error) { + puller, err := remote.NewPuller(options...) + if err != nil { + return "", "", err + } + desc, err := puller.Get(ctx, ref) + if err != nil { + return "", "", err + } + + switch { + case desc.MediaType.IsIndex(): + imageIndex, err := desc.ImageIndex() + if err != nil { + return "", "", fmt.Errorf("failed to get image index: %w", err) + } + indexManifest, err := imageIndex.IndexManifest() + if err != nil { + return "", "", fmt.Errorf("failed to get image manifests: %w", err) + } + var manifest pkgv1.Descriptor + for _, desc := range indexManifest.Manifests { + if p := desc.Platform; p != nil { + // Drop attestations, which don't have a valid platform set. + if p.OS == "unknown" && p.Architecture == "unknown" { + continue + } + manifest = desc + break + } + } + refNameWithoutDigest, _, ok := strings.Cut(ref.Name(), "@") + if !ok { + return "", "", fmt.Errorf("failed to parse reference name %q", ref) + } + repository, err := name.NewRepository(refNameWithoutDigest) + if err != nil { + return "", "", fmt.Errorf("failed to construct repository %q: %w", refNameWithoutDigest, err) + } + // We resolved the image index to an image manifest digest, we can now call this function + // again to resolve the image manifest digest to an image config digest. + return getImageIDAndDigestFromReference( + ctx, + repository.Digest(manifest.Digest.String()), + options..., + ) + case desc.MediaType.IsImage(): + imageManifest, err := desc.Image() + if err != nil { + return "", "", fmt.Errorf("failed to get image: %w", err) + } + imageManifestDigest, err := imageManifest.Digest() + if err != nil { + return "", "", fmt.Errorf("failed to get image digest for %q: %w", ref, err) + } + manifest, err := imageManifest.Manifest() + if err != nil { + return "", "", fmt.Errorf("failed to get image manifest for %q: %w", ref, err) + } + return manifest.Config.Digest.String(), imageManifestDigest.String(), nil + } + return "", "", fmt.Errorf("unsupported media type: %q", desc.MediaType) +} diff --git a/private/buf/cmd/buf/command/beta/registry/plugin/pluginpush/usage.gen.go b/private/buf/cmd/buf/command/plugin/pluginpush/usage.gen.go similarity index 100% rename from private/buf/cmd/buf/command/beta/registry/plugin/pluginpush/usage.gen.go rename to private/buf/cmd/buf/command/plugin/pluginpush/usage.gen.go diff --git a/private/buf/cmd/buf/command/registry/plugin/plugincreate/plugincreate.go b/private/buf/cmd/buf/command/registry/plugin/plugincreate/plugincreate.go new file mode 100644 index 0000000000..6762326ad5 --- /dev/null +++ b/private/buf/cmd/buf/command/registry/plugin/plugincreate/plugincreate.go @@ -0,0 +1,193 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plugincreate + +import ( + "context" + "fmt" + + ownerv1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/owner/v1" + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/buf/bufprint" + "github.com/bufbuild/buf/private/bufpkg/bufapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/app/appcmd" + "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/stringutil" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/spf13/pflag" +) + +const ( + formatFlagName = "format" + visibilityFlagName = "visibility" + defaultLabeFlagName = "default-label-name" + typeFlagName = "type" + + defaultDefaultLabel = "main" + + pluginTypeProtocCodeGeneration = "protoc_code_generation" + pluginTypeCodeGeneration = "code_generation" + pluginTypeCheck = "check" +) + +var ( + allPluginTypeStrings = []string{ + pluginTypeProtocCodeGeneration, + pluginTypeCodeGeneration, + pluginTypeCheck, + } +) + +// NewCommand returns a new Command +func NewCommand( + name string, + builder appext.SubCommandBuilder, +) *appcmd.Command { + flags := newFlags() + return &appcmd.Command{ + Use: name + " ", + Short: "Create a BSR plugin", + Args: appcmd.ExactArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appext.Container) error { + return run(ctx, container, flags) + }, + ), + BindFlags: flags.Bind, + } +} + +type flags struct { + Format string + Visibility string + DefautlLabel string + Type string +} + +func newFlags() *flags { + return &flags{} +} + +func (f *flags) Bind(flagSet *pflag.FlagSet) { + bufcli.BindVisibility(flagSet, &f.Visibility, visibilityFlagName, false) + flagSet.StringVar( + &f.Format, + formatFlagName, + bufprint.FormatText.String(), + fmt.Sprintf(`The output format to use. Must be one of %s`, bufprint.AllFormatsString), + ) + flagSet.StringVar( + &f.DefautlLabel, + defaultLabeFlagName, + defaultDefaultLabel, + "The default label name of the module", + ) + flagSet.StringVar( + &f.Type, + typeFlagName, + "", + fmt.Sprintf( + "The type of the plugin. Must be one of %s", + stringutil.SliceToString(allPluginTypeStrings), + ), + ) + _ = appcmd.MarkFlagRequired(flagSet, typeFlagName) +} + +func run( + ctx context.Context, + container appext.Container, + flags *flags, +) error { + pluginFullName, err := bufplugin.ParsePluginFullName(container.Arg(0)) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + visibility, err := bufcli.VisibilityFlagToPluginVisibilityAllowUnspecified(flags.Visibility) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + format, err := bufprint.ParseFormat(flags.Format) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + pluginType, err := typeFlagToPluginType(flags.Type) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + + clientConfig, err := bufcli.NewConnectClientConfig(container) + if err != nil { + return err + } + pluginServiceClient := bufapi.NewClientProvider(clientConfig). + PluginV1Beta1PluginServiceClient(pluginFullName.Registry()) + + pluginResponse, err := pluginServiceClient.CreatePlugins(ctx, connect.NewRequest( + &pluginv1beta1.CreatePluginsRequest{ + Values: []*pluginv1beta1.CreatePluginsRequest_Value{ + { + OwnerRef: &ownerv1.OwnerRef{ + Value: &ownerv1.OwnerRef_Name{ + Name: pluginFullName.Owner(), + }, + }, + Name: pluginFullName.Name(), + Visibility: visibility, + Type: pluginType, + }, + }, + }, + )) + if err != nil { + if connect.CodeOf(err) == connect.CodeAlreadyExists { + return bufcli.NewPluginNameAlreadyExistsError(pluginFullName.String()) + } + return err + } + plugins := pluginResponse.Msg.Plugins + if len(plugins) != 1 { + return syserror.Newf("unexpected number of plugins returned from server: %d", len(plugins)) + } + if format == bufprint.FormatText { + _, err = fmt.Fprintf(container.Stdout(), "Created %s.\n", pluginFullName) + if err != nil { + return syserror.Wrap(err) + } + return nil + } + return bufprint.PrintNames( + container.Stdout(), + format, + bufprint.NewPluginEntity(plugins[0], pluginFullName), + ) +} + +// typeFlagToPluginType parses the given string as a pluginv1.PluginType. +func typeFlagToPluginType(pluginType string) (pluginv1beta1.PluginType, error) { + switch pluginType { + case pluginTypeProtocCodeGeneration: + return pluginv1beta1.PluginType_PLUGIN_TYPE_PROTOC_CODE_GENERATION, nil + case pluginTypeCodeGeneration: + return pluginv1beta1.PluginType_PLUGIN_TYPE_CODE_GENERATION, nil + case pluginTypeCheck: + return pluginv1beta1.PluginType_PLUGIN_TYPE_CHECK, nil + default: + return 0, fmt.Errorf("invalid plugin type: %s", pluginType) + } +} diff --git a/private/buf/cmd/buf/command/registry/plugin/plugincreate/usage.gen.go b/private/buf/cmd/buf/command/registry/plugin/plugincreate/usage.gen.go new file mode 100644 index 0000000000..6934f63116 --- /dev/null +++ b/private/buf/cmd/buf/command/registry/plugin/plugincreate/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package plugincreate + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/cmd/buf/command/registry/plugin/plugindelete/plugindelete.go b/private/buf/cmd/buf/command/registry/plugin/plugindelete/plugindelete.go new file mode 100644 index 0000000000..7c46049245 --- /dev/null +++ b/private/buf/cmd/buf/command/registry/plugin/plugindelete/plugindelete.go @@ -0,0 +1,114 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plugindelete + +import ( + "context" + "fmt" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/bufpkg/bufapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/app/appcmd" + "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/spf13/pflag" +) + +const forceFlagName = "force" + +// NewCommand returns a new Command +func NewCommand( + name string, + builder appext.SubCommandBuilder, +) *appcmd.Command { + flags := newFlags() + return &appcmd.Command{ + Use: name + " ", + Short: "Delete a BSR plugin", + Args: appcmd.ExactArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appext.Container) error { + return run(ctx, container, flags) + }, + ), + BindFlags: flags.Bind, + } +} + +type flags struct { + Force bool +} + +func newFlags() *flags { + return &flags{} +} + +func (f *flags) Bind(flagSet *pflag.FlagSet) { + flagSet.BoolVar( + &f.Force, + forceFlagName, + false, + "Force deletion without confirming. Use with caution", + ) +} + +func run( + ctx context.Context, + container appext.Container, + flags *flags, +) error { + pluginFullName, err := bufplugin.ParsePluginFullName(container.Arg(0)) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + if !flags.Force { + if err := bufcli.PromptUserForDelete(container, "entity", pluginFullName.Name()); err != nil { + return err + } + } + clientConfig, err := bufcli.NewConnectClientConfig(container) + if err != nil { + return err + } + pluginServiceClient := bufapi.NewClientProvider(clientConfig). + PluginV1Beta1PluginServiceClient(pluginFullName.Registry()) + + if _, err := pluginServiceClient.DeletePlugins(ctx, connect.NewRequest( + &pluginv1beta1.DeletePluginsRequest{ + PluginRefs: []*pluginv1beta1.PluginRef{ + { + Value: &pluginv1beta1.PluginRef_Name_{ + Name: &pluginv1beta1.PluginRef_Name{ + Owner: pluginFullName.Owner(), + Plugin: pluginFullName.Name(), + }, + }, + }, + }, + }, + )); err != nil { + if connect.CodeOf(err) == connect.CodeNotFound { + return bufcli.NewPluginNotFoundError(container.Arg(0)) + } + return err + } + if _, err := fmt.Fprintf(container.Stdout(), "Deleted %s.\n", pluginFullName); err != nil { + return syserror.Wrap(err) + } + return nil +} diff --git a/private/buf/cmd/buf/command/beta/registry/plugin/plugindelete/usage.gen.go b/private/buf/cmd/buf/command/registry/plugin/plugindelete/usage.gen.go similarity index 100% rename from private/buf/cmd/buf/command/beta/registry/plugin/plugindelete/usage.gen.go rename to private/buf/cmd/buf/command/registry/plugin/plugindelete/usage.gen.go diff --git a/private/buf/cmd/buf/command/registry/plugin/plugininfo/plugininfo.go b/private/buf/cmd/buf/command/registry/plugin/plugininfo/plugininfo.go new file mode 100644 index 0000000000..90a54a025e --- /dev/null +++ b/private/buf/cmd/buf/command/registry/plugin/plugininfo/plugininfo.go @@ -0,0 +1,121 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plugininfo + +import ( + "context" + "fmt" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/buf/bufprint" + "github.com/bufbuild/buf/private/bufpkg/bufapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/app/appcmd" + "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/spf13/pflag" +) + +const formatFlagName = "format" + +// NewCommand returns a new Command +func NewCommand( + name string, + builder appext.SubCommandBuilder, +) *appcmd.Command { + flags := newFlags() + return &appcmd.Command{ + Use: name + " ", + Short: "Get a BSR plugin", + Args: appcmd.ExactArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appext.Container) error { + return run(ctx, container, flags) + }, + ), + BindFlags: flags.Bind, + } +} + +type flags struct { + Format string +} + +func newFlags() *flags { + return &flags{} +} + +func (f *flags) Bind(flagSet *pflag.FlagSet) { + flagSet.StringVar( + &f.Format, + formatFlagName, + bufprint.FormatText.String(), + fmt.Sprintf(`The output format to use. Must be one of %s`, bufprint.AllFormatsString), + ) +} + +func run( + ctx context.Context, + container appext.Container, + flags *flags, +) error { + pluginFullName, err := bufplugin.ParsePluginFullName(container.Arg(0)) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + format, err := bufprint.ParseFormat(flags.Format) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + + clientConfig, err := bufcli.NewConnectClientConfig(container) + if err != nil { + return err + } + pluginServiceClient := bufapi.NewClientProvider(clientConfig). + PluginV1Beta1PluginServiceClient(pluginFullName.Registry()) + + pluginsResponse, err := pluginServiceClient.GetPlugins(ctx, connect.NewRequest( + &pluginv1beta1.GetPluginsRequest{ + PluginRefs: []*pluginv1beta1.PluginRef{ + { + Value: &pluginv1beta1.PluginRef_Name_{ + Name: &pluginv1beta1.PluginRef_Name{ + Owner: pluginFullName.Owner(), + Plugin: pluginFullName.Name(), + }, + }, + }, + }, + }, + )) + if err != nil { + if connect.CodeOf(err) == connect.CodeNotFound { + return bufcli.NewPluginNotFoundError(container.Arg(0)) + } + return err + } + plugins := pluginsResponse.Msg.Plugins + if len(plugins) != 1 { + return syserror.Newf("unexpected number of plugins returned from server: %d", len(plugins)) + } + return bufprint.PrintEntity( + container.Stdout(), + format, + bufprint.NewPluginEntity(plugins[0], pluginFullName), + ) +} diff --git a/private/buf/cmd/buf/command/registry/plugin/plugininfo/usage.gen.go b/private/buf/cmd/buf/command/registry/plugin/plugininfo/usage.gen.go new file mode 100644 index 0000000000..7889165761 --- /dev/null +++ b/private/buf/cmd/buf/command/registry/plugin/plugininfo/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package plugininfo + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/cmd/buf/command/registry/plugin/pluginupdate/pluginupdate.go b/private/buf/cmd/buf/command/registry/plugin/pluginupdate/pluginupdate.go new file mode 100644 index 0000000000..ba91d903bb --- /dev/null +++ b/private/buf/cmd/buf/command/registry/plugin/pluginupdate/pluginupdate.go @@ -0,0 +1,161 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pluginupdate + +import ( + "context" + "fmt" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/buf/bufprint" + "github.com/bufbuild/buf/private/bufpkg/bufapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/app/appcmd" + "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/spf13/pflag" +) + +const ( + formatFlagName = "format" + visibilityFlagName = "visibility" + descriptionFlagName = "description" + urlFlagName = "url" +) + +// NewCommand returns a new Command +func NewCommand(name string, builder appext.SubCommandBuilder) *appcmd.Command { + flags := newFlags() + return &appcmd.Command{ + Use: name + " ", + Short: "Update BSR plugin settings", + Args: appcmd.ExactArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appext.Container) error { + return run(ctx, container, flags) + }, + ), + BindFlags: flags.Bind, + } +} + +type flags struct { + Format string + Visibility string + Description *string + URL *string + DefaultLabel string +} + +func newFlags() *flags { + return &flags{} +} + +func (f *flags) Bind(flagSet *pflag.FlagSet) { + bufcli.BindVisibility(flagSet, &f.Visibility, visibilityFlagName, true) + flagSet.StringVar( + &f.Format, + formatFlagName, + bufprint.FormatText.String(), + fmt.Sprintf(`The output format to use. Must be one of %s`, bufprint.AllFormatsString), + ) + bufcli.BindStringPointer( + flagSet, + descriptionFlagName, + &f.Description, + "The new description for the plugin", + ) + bufcli.BindStringPointer( + flagSet, + urlFlagName, + &f.URL, + "The new URL for the plugin", + ) +} + +func run( + ctx context.Context, + container appext.Container, + flags *flags, +) error { + pluginFullName, err := bufplugin.ParsePluginFullName(container.Arg(0)) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + format, err := bufprint.ParseFormat(flags.Format) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + visibility, err := bufcli.VisibilityFlagToPluginVisibilityAllowUnspecified(flags.Visibility) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + clientConfig, err := bufcli.NewConnectClientConfig(container) + if err != nil { + return err + } + var visibilityUpdate *pluginv1beta1.PluginVisibility + if visibility != pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_UNSPECIFIED { + visibilityUpdate = &visibility + } + + pluginServiceClient := bufapi.NewClientProvider(clientConfig). + PluginV1Beta1PluginServiceClient(pluginFullName.Registry()) + + pluginResponse, err := pluginServiceClient.UpdatePlugins(ctx, connect.NewRequest( + &pluginv1beta1.UpdatePluginsRequest{ + Values: []*pluginv1beta1.UpdatePluginsRequest_Value{ + { + PluginRef: &pluginv1beta1.PluginRef{ + Value: &pluginv1beta1.PluginRef_Name_{ + Name: &pluginv1beta1.PluginRef_Name{ + Owner: pluginFullName.Owner(), + Plugin: pluginFullName.Name(), + }, + }, + }, + Visibility: visibilityUpdate, + }, + }, + }, + )) + if err != nil { + if connect.CodeOf(err) == connect.CodeNotFound { + return bufcli.NewModuleNotFoundError(container.Arg(0)) + } + return err + } + plugins := pluginResponse.Msg.Plugins + if len(plugins) != 1 { + return syserror.Newf("unexpected number of plugins returned from server: %d", len(plugins)) + } + if format == bufprint.FormatText { + _, err = fmt.Fprintf(container.Stdout(), "Updated %s.\n", pluginFullName) + if err != nil { + return syserror.Wrap(err) + } + return nil + } + if _, err := fmt.Fprintln(container.Stdout(), "Plugin updated."); err != nil { + return syserror.Wrap(err) + } + return bufprint.PrintNames( + container.Stdout(), + format, + bufprint.NewPluginEntity(plugins[0], pluginFullName), + ) +} diff --git a/private/buf/cmd/buf/command/registry/plugin/pluginupdate/usage.gen.go b/private/buf/cmd/buf/command/registry/plugin/pluginupdate/usage.gen.go new file mode 100644 index 0000000000..23136ff2a4 --- /dev/null +++ b/private/buf/cmd/buf/command/registry/plugin/pluginupdate/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package pluginupdate + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/cmd/protoc-gen-buf-breaking/breaking.go b/private/buf/cmd/protoc-gen-buf-breaking/breaking.go index ec29b5f6a2..6c007a29b8 100644 --- a/private/buf/cmd/protoc-gen-buf-breaking/breaking.go +++ b/private/buf/cmd/protoc-gen-buf-breaking/breaking.go @@ -28,6 +28,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/protodescriptor" @@ -126,7 +127,12 @@ func handle( // The protoc plugins do not support custom lint/breaking change plugins for now. client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime), + bufcheck.NewRunnerProvider( + command.NewRunner(), + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), bufcheck.ClientWithStderr(pluginEnv.Stderr), ) if err != nil { diff --git a/private/buf/cmd/protoc-gen-buf-lint/lint.go b/private/buf/cmd/protoc-gen-buf-lint/lint.go index 6ee136a4b6..cac8947c20 100644 --- a/private/buf/cmd/protoc-gen-buf-lint/lint.go +++ b/private/buf/cmd/protoc-gen-buf-lint/lint.go @@ -27,6 +27,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/protodescriptor" @@ -101,7 +102,12 @@ func handle( // The protoc plugins do not support custom lint/breaking change plugins for now. client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime), + bufcheck.NewRunnerProvider( + command.NewRunner(), + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), bufcheck.ClientWithStderr(pluginEnv.Stderr), ) if err != nil { diff --git a/private/bufpkg/bufapi/bufapi.go b/private/bufpkg/bufapi/bufapi.go index 61be94c414..80f4f4aa08 100644 --- a/private/bufpkg/bufapi/bufapi.go +++ b/private/bufpkg/bufapi/bufapi.go @@ -18,6 +18,7 @@ import ( "buf.build/gen/go/bufbuild/registry/connectrpc/go/buf/registry/module/v1/modulev1connect" "buf.build/gen/go/bufbuild/registry/connectrpc/go/buf/registry/module/v1beta1/modulev1beta1connect" "buf.build/gen/go/bufbuild/registry/connectrpc/go/buf/registry/owner/v1/ownerv1connect" + "buf.build/gen/go/bufbuild/registry/connectrpc/go/buf/registry/plugin/v1beta1/pluginv1beta1connect" "github.com/bufbuild/buf/private/pkg/connectclient" ) @@ -136,6 +137,31 @@ type V1Beta1UploadServiceClientProvider interface { V1Beta1UploadServiceClient(registry string) modulev1beta1connect.UploadServiceClient } +// PluginV1Beta1CommitServiceClientProvider provides CommitServiceClients for plugins. +type PluginV1Beta1CommitServiceClientProvider interface { + PluginV1Beta1CommitServiceClient(registry string) pluginv1beta1connect.CommitServiceClient +} + +// PluginV1Beta1DownloadServiceClientProvider provides DownloadServiceClients for plugins. +type PluginV1Beta1DownloadServiceClientProvider interface { + PluginV1Beta1DownloadServiceClient(registry string) pluginv1beta1connect.DownloadServiceClient +} + +// PluginV1Beta1LabelServiceClientProvider provides LabelServiceClients for plugins. +type PluginV1Beta1LabelServiceClientProvider interface { + PluginV1Beta1LabelServiceClient(registry string) pluginv1beta1connect.LabelServiceClient +} + +// PluginV1Beta1PluginServiceClientProvider provides PluginServiceClients for plugins. +type PluginV1Beta1PluginServiceClientProvider interface { + PluginV1Beta1PluginServiceClient(registry string) pluginv1beta1connect.PluginServiceClient +} + +// PluginV1Beta1UploadServiceClientProvider provides UploadServiceClients for plugins. +type PluginV1Beta1UploadServiceClientProvider interface { + PluginV1Beta1UploadServiceClient(registry string) pluginv1beta1connect.UploadServiceClient +} + // ClientProvider provides API clients for BSR services. type ClientProvider interface { V1CommitServiceClientProvider @@ -154,6 +180,11 @@ type ClientProvider interface { V1Beta1LabelServiceClientProvider V1Beta1ModuleServiceClientProvider V1Beta1UploadServiceClientProvider + PluginV1Beta1CommitServiceClientProvider + PluginV1Beta1DownloadServiceClientProvider + PluginV1Beta1LabelServiceClientProvider + PluginV1Beta1PluginServiceClientProvider + PluginV1Beta1UploadServiceClientProvider } // NewClientProvider returns a new ClientProvider. @@ -301,6 +332,46 @@ func (c *clientProvider) V1Beta1UploadServiceClient(registry string) modulev1bet ) } +func (c *clientProvider) PluginV1Beta1CommitServiceClient(registry string) pluginv1beta1connect.CommitServiceClient { + return connectclient.Make( + c.clientConfig, + registry, + pluginv1beta1connect.NewCommitServiceClient, + ) +} + +func (c *clientProvider) PluginV1Beta1DownloadServiceClient(registry string) pluginv1beta1connect.DownloadServiceClient { + return connectclient.Make( + c.clientConfig, + registry, + pluginv1beta1connect.NewDownloadServiceClient, + ) +} + +func (c *clientProvider) PluginV1Beta1LabelServiceClient(registry string) pluginv1beta1connect.LabelServiceClient { + return connectclient.Make( + c.clientConfig, + registry, + pluginv1beta1connect.NewLabelServiceClient, + ) +} + +func (c *clientProvider) PluginV1Beta1UploadServiceClient(registry string) pluginv1beta1connect.UploadServiceClient { + return connectclient.Make( + c.clientConfig, + registry, + pluginv1beta1connect.NewUploadServiceClient, + ) +} + +func (c *clientProvider) PluginV1Beta1PluginServiceClient(registry string) pluginv1beta1connect.PluginServiceClient { + return connectclient.Make( + c.clientConfig, + registry, + pluginv1beta1connect.NewPluginServiceClient, + ) +} + type nopClientProvider struct{} func (nopClientProvider) V1CommitServiceClient(registry string) modulev1connect.CommitServiceClient { @@ -366,3 +437,23 @@ func (nopClientProvider) V1Beta1ModuleServiceClient(registry string) modulev1bet func (nopClientProvider) V1Beta1UploadServiceClient(registry string) modulev1beta1connect.UploadServiceClient { return modulev1beta1connect.UnimplementedUploadServiceHandler{} } + +func (nopClientProvider) PluginV1Beta1CommitServiceClient(registry string) pluginv1beta1connect.CommitServiceClient { + return pluginv1beta1connect.UnimplementedCommitServiceHandler{} +} + +func (nopClientProvider) PluginV1Beta1DownloadServiceClient(registry string) pluginv1beta1connect.DownloadServiceClient { + return pluginv1beta1connect.UnimplementedDownloadServiceHandler{} +} + +func (nopClientProvider) PluginV1Beta1LabelServiceClient(registry string) pluginv1beta1connect.LabelServiceClient { + return pluginv1beta1connect.UnimplementedLabelServiceHandler{} +} + +func (nopClientProvider) PluginV1Beta1UploadServiceClient(registry string) pluginv1beta1connect.UploadServiceClient { + return pluginv1beta1connect.UnimplementedUploadServiceHandler{} +} + +func (nopClientProvider) PluginV1Beta1PluginServiceClient(registry string) pluginv1beta1connect.PluginServiceClient { + return pluginv1beta1connect.UnimplementedPluginServiceHandler{} +} diff --git a/private/bufpkg/bufcheck/breaking_test.go b/private/bufpkg/bufcheck/breaking_test.go index c87af2d1e6..af71987418 100644 --- a/private/bufpkg/bufcheck/breaking_test.go +++ b/private/bufpkg/bufcheck/breaking_test.go @@ -30,6 +30,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/slogtestext" "github.com/bufbuild/buf/private/pkg/storage/storageos" @@ -1345,7 +1346,15 @@ func testBreaking( require.NoError(t, err) breakingConfig := workspace.GetBreakingConfigForOpaqueID(opaqueID) require.NotNil(t, breakingConfig) - client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime)) + client, err := bufcheck.NewClient( + logger, + bufcheck.NewRunnerProvider( + command.NewRunner(), + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), + ) require.NoError(t, err) err = client.Breaking( ctx, diff --git a/private/bufpkg/bufcheck/bufcheck.go b/private/bufpkg/bufcheck/bufcheck.go index 0cff78425d..5d8c4e2798 100644 --- a/private/bufpkg/bufcheck/bufcheck.go +++ b/private/bufpkg/bufcheck/bufcheck.go @@ -22,6 +22,7 @@ import ( "buf.build/go/bufplugin/check" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/syserror" @@ -179,10 +180,21 @@ func (r RunnerProviderFunc) NewRunner(pluginConfig bufconfig.PluginConfig) (plug // The supported types are: // - bufconfig.PluginConfigTypeLocal // - bufconfig.PluginConfigTypeLocalWasm +// - bufconfig.PluginConfigTypeRemote // // If the PluginConfigType is not supported, an error is returned. -func NewRunnerProvider(commandRunner command.Runner, wasmRuntime wasm.Runtime) RunnerProvider { - return newRunnerProvider(commandRunner, wasmRuntime) +func NewRunnerProvider( + commandRunner command.Runner, + wasmRuntime wasm.Runtime, + pluginKeyProvider bufplugin.PluginKeyProvider, + pluginDataProvider bufplugin.PluginDataProvider, +) RunnerProvider { + return newRunnerProvider( + commandRunner, + wasmRuntime, + pluginKeyProvider, + pluginDataProvider, + ) } // NewClient returns a new Client. diff --git a/private/bufpkg/bufcheck/lint_test.go b/private/bufpkg/bufcheck/lint_test.go index 1ff56ab7f3..27abd61bed 100644 --- a/private/bufpkg/bufcheck/lint_test.go +++ b/private/bufpkg/bufcheck/lint_test.go @@ -27,6 +27,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/slogtestext" "github.com/bufbuild/buf/private/pkg/storage/storageos" @@ -1356,7 +1357,12 @@ func testLintWithOptions( }) client, err := bufcheck.NewClient( logger, - bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime), + bufcheck.NewRunnerProvider( + command.NewRunner(), + wasmRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), ) require.NoError(t, err) err = client.Lint( diff --git a/private/bufpkg/bufcheck/multi_client_test.go b/private/bufpkg/bufcheck/multi_client_test.go index 810887cd9a..8c7c7c63ef 100644 --- a/private/bufpkg/bufcheck/multi_client_test.go +++ b/private/bufpkg/bufcheck/multi_client_test.go @@ -24,6 +24,7 @@ import ( "buf.build/go/bufplugin/check/checkutil" "buf.build/go/bufplugin/option" "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/slogtestext" @@ -183,7 +184,12 @@ func TestMultiClientCannotHaveOverlappingRulesWithBuiltIn(t *testing.T) { client, err := newClient( slogtestext.NewLogger(t), - NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime), + NewRunnerProvider( + command.NewRunner(), + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), ) require.NoError(t, err) duplicateBuiltInRulePluginConfig, err := bufconfig.NewLocalPluginConfig( @@ -276,7 +282,12 @@ func TestMultiClientCannotHaveOverlappingCategoriesWithBuiltIn(t *testing.T) { client, err := newClient( slogtestext.NewLogger(t), - NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime), + NewRunnerProvider( + command.NewRunner(), + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), ) require.NoError(t, err) duplicateBuiltInRulePluginConfig, err := bufconfig.NewLocalPluginConfig( diff --git a/private/bufpkg/bufcheck/runner_provider.go b/private/bufpkg/bufcheck/runner_provider.go index e5788c9017..462f4caaf4 100644 --- a/private/bufpkg/bufcheck/runner_provider.go +++ b/private/bufpkg/bufcheck/runner_provider.go @@ -15,7 +15,11 @@ package bufcheck import ( + "context" + "sync" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/pluginrpcutil" "github.com/bufbuild/buf/private/pkg/syserror" @@ -24,14 +28,23 @@ import ( ) type runnerProvider struct { - commandRunner command.Runner - wasmRuntime wasm.Runtime + commandRunner command.Runner + wasmRuntime wasm.Runtime + pluginKeyProvider bufplugin.PluginKeyProvider + pluginDataProvider bufplugin.PluginDataProvider } -func newRunnerProvider(commandRunner command.Runner, wasmRuntime wasm.Runtime) *runnerProvider { +func newRunnerProvider( + commandRunner command.Runner, + wasmRuntime wasm.Runtime, + pluginKeyProvider bufplugin.PluginKeyProvider, + pluginDataProvider bufplugin.PluginDataProvider, +) *runnerProvider { return &runnerProvider{ - commandRunner: commandRunner, - wasmRuntime: wasmRuntime, + commandRunner: commandRunner, + wasmRuntime: wasmRuntime, + pluginKeyProvider: pluginKeyProvider, + pluginDataProvider: pluginDataProvider, } } @@ -53,7 +66,65 @@ func (r *runnerProvider) NewRunner(pluginConfig bufconfig.PluginConfig) (pluginr path[0], path[1:]..., ), nil + case bufconfig.PluginConfigTypeRemote: + var ( + once sync.Once + compiledModule wasm.CompiledModule + compiledModuleErr error + ) + return rpcRunnerFunc(func(ctx context.Context, env pluginrpc.Env) error { + once.Do(func() { + compiledModule, compiledModuleErr = r.loadRemotePlugin(ctx, pluginConfig) + }) + if compiledModuleErr != nil { + return compiledModuleErr + } + return compiledModule.Run(ctx, env) + + }), nil + default: return nil, syserror.Newf("unknown PluginConfigType: %v", pluginConfig.Type()) } } + +type rpcRunnerFunc func(ctx context.Context, env pluginrpc.Env) error + +func (f rpcRunnerFunc) Run(ctx context.Context, env pluginrpc.Env) error { + return f(ctx, env) +} + +func (r *runnerProvider) loadRemotePlugin(ctx context.Context, pluginConfig bufconfig.PluginConfig) (wasm.CompiledModule, error) { + pluginRef := pluginConfig.PluginRef() + if pluginRef == nil { + return nil, syserror.New("pluginRef is required for remote plugins") + } + pluginKeys, err := r.pluginKeyProvider.GetPluginKeysForPluginRefs( + ctx, + []bufplugin.PluginRef{pluginRef}, + bufplugin.DigestTypeP1, + ) + if err != nil { + return nil, err + } + if len(pluginKeys) != 1 { + return nil, syserror.Newf("expected 1 plugin key for %s", pluginRef) + } + pluginDatas, err := r.pluginDataProvider.GetPluginDatasForPluginKeys( + ctx, + []bufplugin.PluginKey{pluginKeys[0]}, + ) + if err != nil { + return nil, err + } + if len(pluginDatas) != 1 { + return nil, syserror.Newf("expected 1 plugin data for %s", pluginRef) + } + pluginData := pluginDatas[0] + + data, err := pluginData.Data() + if err != nil { + return nil, err + } + return r.wasmRuntime.Compile(ctx, pluginConfig.Name(), data) +} diff --git a/private/bufpkg/bufconfig/buf_lock_file.go b/private/bufpkg/bufconfig/buf_lock_file.go index ace54fd8e5..3e683f7724 100644 --- a/private/bufpkg/bufconfig/buf_lock_file.go +++ b/private/bufpkg/bufconfig/buf_lock_file.go @@ -24,6 +24,7 @@ import ( "time" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage" @@ -72,6 +73,9 @@ type BufLockFile interface { // while Files with FileVersionV2 will only have ModuleKeys with Digests of DigestTypeB5. DepModuleKeys() []bufmodule.ModuleKey + // TODO(ed) + PluginKeys() []bufplugin.PluginKey + isBufLockFile() } @@ -79,8 +83,8 @@ type BufLockFile interface { // // Note that digests are lazily-loaded; if you need to ensure that all digests are valid, run // ValidateBufLockFileDigests(). -func NewBufLockFile(fileVersion FileVersion, depModuleKeys []bufmodule.ModuleKey) (BufLockFile, error) { - return newBufLockFile(fileVersion, nil, depModuleKeys) +func NewBufLockFile(fileVersion FileVersion, depModuleKeys []bufmodule.ModuleKey, pluginKeys []bufplugin.PluginKey) (BufLockFile, error) { + return newBufLockFile(fileVersion, nil, depModuleKeys, pluginKeys) } // GetBufLockFileForPrefix gets the buf.lock file at the given bucket prefix. @@ -183,12 +187,14 @@ type bufLockFile struct { fileVersion FileVersion objectData ObjectData depModuleKeys []bufmodule.ModuleKey + pluginKeys []bufplugin.PluginKey } func newBufLockFile( fileVersion FileVersion, objectData ObjectData, depModuleKeys []bufmodule.ModuleKey, + pluginKeys []bufplugin.PluginKey, ) (*bufLockFile, error) { if err := validateNoDuplicateModuleKeysByModuleFullName(depModuleKeys); err != nil { return nil, err @@ -240,6 +246,10 @@ func (l *bufLockFile) DepModuleKeys() []bufmodule.ModuleKey { return l.depModuleKeys } +func (l *bufLockFile) PluginKeys() []bufplugin.PluginKey { + return l.pluginKeys +} + func (*bufLockFile) isBufLockFile() {} func (*bufLockFile) isFile() {} func (*bufLockFile) isFileInfo() {} @@ -314,7 +324,7 @@ func readBufLockFile( } depModuleKeys[i] = depModuleKey } - return newBufLockFile(fileVersion, objectData, depModuleKeys) + return newBufLockFile(fileVersion, objectData, depModuleKeys, nil) case FileVersionV2: var externalBufLockFile externalBufLockFileV2 if err := getUnmarshalStrict(allowJSON)(data, &externalBufLockFile); err != nil { @@ -356,7 +366,39 @@ func readBufLockFile( } depModuleKeys[i] = depModuleKey } - return newBufLockFile(fileVersion, objectData, depModuleKeys) + pluginKeys := make([]bufplugin.PluginKey, len(externalBufLockFile.Plugins)) + for i, plugin := range externalBufLockFile.Plugins { + plugin := plugin + if plugin.Name == "" { + return nil, errors.New("no plugin name specified") + } + pluginFullName, err := bufplugin.ParsePluginFullName(plugin.Name) + if err != nil { + return nil, fmt.Errorf("invalid plugin name: %w", err) + } + if plugin.Commit == "" { + return nil, fmt.Errorf("no commit specified for plugin %s", pluginFullName.String()) + } + if plugin.Digest == "" { + return nil, fmt.Errorf("no digest specified for plugin %s", pluginFullName.String()) + } + commitID, err := uuidutil.FromDashless(plugin.Commit) + if err != nil { + return nil, err + } + pluginKey, err := bufplugin.NewPluginKey( + pluginFullName, + commitID, + func() (bufplugin.Digest, error) { + return bufplugin.ParseDigest(plugin.Digest) + }, + ) + if err != nil { + return nil, err + } + pluginKeys[i] = pluginKey + } + return newBufLockFile(fileVersion, objectData, depModuleKeys, pluginKeys) default: // This is a system error since we've already parsed. return nil, syserror.Newf("unknown FileVersion: %v", fileVersion) @@ -373,6 +415,9 @@ func writeBufLockFile( switch fileVersion := bufLockFile.FileVersion(); fileVersion { case FileVersionV1Beta1, FileVersionV1: depModuleKeys := bufLockFile.DepModuleKeys() + if len(bufLockFile.PluginKeys()) > 0 { + return syserror.Newf("v1beta1 and v1 buf.lock files do not support plugins") + } externalBufLockFile := externalBufLockFileV1Beta1V1{ Version: fileVersion.String(), Deps: make([]externalBufLockFileDepV1Beta1V1, len(depModuleKeys)), @@ -399,9 +444,11 @@ func writeBufLockFile( return err case FileVersionV2: depModuleKeys := bufLockFile.DepModuleKeys() + depPluginKeys := bufLockFile.PluginKeys() externalBufLockFile := externalBufLockFileV2{ Version: fileVersion.String(), Deps: make([]externalBufLockFileDepV2, len(depModuleKeys)), + Plugins: make([]externalBufLockFileDepV2, len(depPluginKeys)), } for i, depModuleKey := range depModuleKeys { digest, err := depModuleKey.Digest() @@ -414,6 +461,17 @@ func writeBufLockFile( Digest: digest.String(), } } + for i, depPluginKey := range depPluginKeys { + digest, err := depPluginKey.Digest() + if err != nil { + return err + } + externalBufLockFile.Plugins[i] = externalBufLockFileDepV2{ + Name: depPluginKey.PluginFullName().String(), + Commit: uuidutil.ToDashless(depPluginKey.CommitID()), + Digest: digest.String(), + } + } // No need to sort - depModuleKeys is already sorted by ModuleFullName data, err := encoding.MarshalYAML(&externalBufLockFile) if err != nil { @@ -523,6 +581,7 @@ type externalBufLockFileDepV1Beta1V1 struct { type externalBufLockFileV2 struct { Version string `json:"version,omitempty" yaml:"version,omitempty"` Deps []externalBufLockFileDepV2 `json:"deps,omitempty" yaml:"deps,omitempty"` + Plugins []externalBufLockFileDepV2 `json:"plugins,omitempty" yaml:"plugins,omitempty"` } // externalBufLockFileDepV2 represents a single dep within a v2 buf.lock file. diff --git a/private/bufpkg/bufconfig/plugin_config.go b/private/bufpkg/bufconfig/plugin_config.go index 388bb15b50..3e2ceb815a 100644 --- a/private/bufpkg/bufconfig/plugin_config.go +++ b/private/bufpkg/bufconfig/plugin_config.go @@ -20,6 +20,7 @@ import ( "path/filepath" "strings" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/syserror" ) @@ -29,6 +30,8 @@ const ( PluginConfigTypeLocal PluginConfigType = iota + 1 // PluginConfigTypeLocalWasm is the local Wasm plugin config type. PluginConfigTypeLocalWasm + // PluginConfigTypeRemote is the remote plugin config type. + PluginConfigTypeRemote ) // PluginConfigType is a generate plugin configuration type. @@ -49,6 +52,10 @@ type PluginConfig interface { // // This is not empty only when the plugin is local. Path() []string + // PluginRef returns the plugin reference. + // + // This is only non-nil when the plugin is remote. + PluginRef() bufplugin.PluginRef isPluginConfig() } @@ -90,6 +97,7 @@ type pluginConfig struct { name string options map[string]any path []string + pluginRef bufplugin.PluginRef } func newPluginConfigForExternalV2( @@ -106,8 +114,7 @@ func newPluginConfigForExternalV2( } options[key] = value } - // TODO: differentiate between local and remote in the future - // Use the same heuristic that we do for dir vs module in buffetch + // Plugins are specified as a path, remote reference, or Wasm file. path, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Plugin) if err != nil { return nil, err @@ -115,6 +122,13 @@ func newPluginConfigForExternalV2( if len(path) == 0 { return nil, errors.New("must specify a path to the plugin") } + // Remote plugins are specified as plugin references. + if pluginRef, err := bufplugin.ParsePluginRef(path[0]); err == nil { + return newRemotePluginConfig( + pluginRef, + options, + ) + } // Wasm plugins are suffixed with .wasm. Otherwise, it's a binary. if filepath.Ext(path[0]) == ".wasm" { return newLocalWasmPluginConfig( @@ -165,6 +179,17 @@ func newLocalWasmPluginConfig( }, nil } +func newRemotePluginConfig( + pluginRef bufplugin.PluginRef, + options map[string]any, +) (*pluginConfig, error) { + return &pluginConfig{ + pluginConfigType: PluginConfigTypeRemote, + options: options, + pluginRef: pluginRef, + }, nil +} + func (p *pluginConfig) Type() PluginConfigType { return p.pluginConfigType } @@ -181,6 +206,10 @@ func (p *pluginConfig) Path() []string { return p.path } +func (p *pluginConfig) PluginRef() bufplugin.PluginRef { + return p.pluginRef +} + func (p *pluginConfig) isPluginConfig() {} func newExternalV2ForPluginConfig( diff --git a/private/bufpkg/bufmodule/module_ref.go b/private/bufpkg/bufmodule/module_ref.go index 8ef19f4824..d088b2179b 100644 --- a/private/bufpkg/bufmodule/module_ref.go +++ b/private/bufpkg/bufmodule/module_ref.go @@ -38,7 +38,7 @@ type ModuleRef interface { isModuleRef() } -// NewModuleRef returns a new ModuleRef for the given compoonents. +// NewModuleRef returns a new ModuleRef for the given components. func NewModuleRef( registry string, owner string, diff --git a/private/bufpkg/bufparse/bufparse.go b/private/bufpkg/bufparse/bufparse.go new file mode 100644 index 0000000000..bd0d7468b5 --- /dev/null +++ b/private/bufpkg/bufparse/bufparse.go @@ -0,0 +1,15 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufparse diff --git a/private/bufpkg/bufparse/errors.go b/private/bufpkg/bufparse/errors.go new file mode 100644 index 0000000000..2058afbd07 --- /dev/null +++ b/private/bufpkg/bufparse/errors.go @@ -0,0 +1,85 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufparse + +import "strings" + +// ParseError is an error that occurred during parsing. +// +// This is returned by all Parse.* functions in this package. +type ParseError struct { + // typeString is the user-consumable string representing of the type that was attempted to be parsed. + // + // Users cannot rely on this data being structured. + // Examples: "digest", "digest type". + typeString string + // input is the input string that was attempted to be parsed. + input string + // err is the underlying error. + // + // Err may be a *ParseError itself. + // + // This is an error we may give back to the user, use pretty strings that should + // be read. + err error +} + +// NewParseError returns a new ParseError. +func NewParseError(typeString string, input string, err error) *ParseError { + return &ParseError{ + typeString: typeString, + input: input, + err: err, + } +} + +// Error implements the error interface. +func (p *ParseError) Error() string { + if p == nil { + return "" + } + var builder strings.Builder + _, _ = builder.WriteString(`could not parse`) + if p.typeString != "" { + _, _ = builder.WriteString(` `) + _, _ = builder.WriteString(p.typeString) + } + if p.input != "" { + _, _ = builder.WriteString(` "`) + _, _ = builder.WriteString(p.input) + _, _ = builder.WriteString(`"`) + } + if p.err != nil { + _, _ = builder.WriteString(`: `) + _, _ = builder.WriteString(p.err.Error()) + } + return builder.String() +} + +// Unwrap returns the underlying error. +func (p *ParseError) Unwrap() error { + if p == nil { + return nil + } + return p.err +} + +// Input returns the input string that was attempted to be parsed. +func (p *ParseError) Input() string { + if p == nil { + return "" + } + return p.input +} diff --git a/private/bufpkg/bufparse/parse.go b/private/bufpkg/bufparse/parse.go new file mode 100644 index 0000000000..8df3d5c113 --- /dev/null +++ b/private/bufpkg/bufparse/parse.go @@ -0,0 +1,83 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufparse + +import ( + "errors" + "strings" +) + +func ParseFullNameComponents(path string) (registry string, owner string, name string, err error) { + slashSplit := strings.Split(path, "/") + if len(slashSplit) != 3 { + return "", "", "", newInvalidFullNameStringError(path) + } + registry = strings.TrimSpace(slashSplit[0]) + if registry == "" { + return "", "", "", newInvalidFullNameStringError(path) + } + owner = strings.TrimSpace(slashSplit[1]) + if owner == "" { + return "", "", "", newInvalidFullNameStringError(path) + } + name = strings.TrimSpace(slashSplit[2]) + if name == "" { + return "", "", "", newInvalidFullNameStringError(path) + } + return registry, owner, name, nil +} + +func ParseRefComponents(path string) (registry string, owner string, name string, ref string, err error) { + // split by the first "/" to separate the registry and remaining part + slashSplit := strings.SplitN(path, "/", 2) + if len(slashSplit) != 2 { + return "", "", "", "", newInvalidRefStringError(path) + } + registry, rest := slashSplit[0], slashSplit[1] + // split the remaining part by ":" to separate the reference + colonSplit := strings.Split(rest, ":") + switch len(colonSplit) { + case 1: + // path excluding registry has no colon, no need to handle its ref + case 2: + ref = strings.TrimSpace(colonSplit[1]) + if ref == "" { + return "", "", "", "", newInvalidRefStringError(path) + } + default: + return "", "", "", "", newInvalidRefStringError(path) + } + registry, owner, name, err = ParseFullNameComponents(registry + "/" + colonSplit[0]) + if err != nil { + return "", "", "", "", newInvalidRefStringError(path) + } + return registry, owner, name, ref, nil +} + +func newInvalidFullNameStringError(s string) error { + return NewParseError( + "name", + s, + errors.New("must be in the form registry/owner/name"), + ) +} + +func newInvalidRefStringError(s string) error { + return NewParseError( + "reference", + s, + errors.New("must be in the form registry/owner/name[:ref]"), + ) +} diff --git a/private/bufpkg/bufparse/usage.gen.go b/private/bufpkg/bufparse/usage.gen.go new file mode 100644 index 0000000000..f0a3423dc8 --- /dev/null +++ b/private/bufpkg/bufparse/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufparse + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufplugin/bufplugin.go b/private/bufpkg/bufplugin/bufplugin.go new file mode 100644 index 0000000000..3e18be9213 --- /dev/null +++ b/private/bufpkg/bufplugin/bufplugin.go @@ -0,0 +1,16 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bufplugin contains the core primitives for working with Buf plugins. +package bufplugin diff --git a/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go b/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go new file mode 100644 index 0000000000..816d5770ad --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go @@ -0,0 +1,15 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi diff --git a/private/bufpkg/bufplugin/bufpluginapi/convert.go b/private/bufpkg/bufplugin/bufpluginapi/convert.go new file mode 100644 index 0000000000..ae2130659c --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/convert.go @@ -0,0 +1,53 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi + +import ( + "fmt" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "github.com/bufbuild/buf/private/bufpkg/bufcas" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" +) + +var ( + v1beta1ProtoDigestTypeToDigestType = map[pluginv1beta1.DigestType]bufplugin.DigestType{ + pluginv1beta1.DigestType_DIGEST_TYPE_P1: bufplugin.DigestTypeP1, + } +) + +// V1Beta1ProtoToDigest converts the given proto Digest to a Digest. +// +// Validation is performed to ensure the DigestType is known, and the value +// is a valid digest value for the given DigestType. +func V1Beta1ProtoToDigest(protoDigest *pluginv1beta1.Digest) (bufplugin.Digest, error) { + digestType, err := v1beta1ProtoToDigestType(protoDigest.Type) + if err != nil { + return nil, err + } + bufcasDigest, err := bufcas.NewDigest(protoDigest.Value) + if err != nil { + return nil, err + } + return bufplugin.NewDigest(digestType, bufcasDigest) +} + +func v1beta1ProtoToDigestType(protoDigestType pluginv1beta1.DigestType) (bufplugin.DigestType, error) { + digestType, ok := v1beta1ProtoDigestTypeToDigestType[protoDigestType] + if !ok { + return 0, fmt.Errorf("unknown pluginv1beta1.DigestType: %v", protoDigestType) + } + return digestType, nil +} diff --git a/private/bufpkg/bufplugin/bufpluginapi/plugin_data_provider.go b/private/bufpkg/bufplugin/bufpluginapi/plugin_data_provider.go new file mode 100644 index 0000000000..2b25bf4d2f --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/plugin_data_provider.go @@ -0,0 +1,174 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi + +import ( + "context" + "log/slog" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/bufpkg/bufapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/google/uuid" +) + +// NewPluginDataProvider returns a new PluginDataProvider for the given API client. +// +// A warning is printed to the logger if a given Plugin is deprecated. +func NewPluingDataProvider( + logger *slog.Logger, + clientProvider interface { + bufapi.PluginV1Beta1DownloadServiceClientProvider + bufapi.PluginV1Beta1PluginServiceClientProvider + }, +) bufplugin.PluginDataProvider { + return newPluginDataProvider(logger, clientProvider) +} + +// *** PRIVATE *** + +type pluginDataProvider struct { + logger *slog.Logger + clientProvider interface { + bufapi.PluginV1Beta1DownloadServiceClientProvider + bufapi.PluginV1Beta1PluginServiceClientProvider + } +} + +func newPluginDataProvider( + logger *slog.Logger, + clientProvider interface { + bufapi.PluginV1Beta1DownloadServiceClientProvider + bufapi.PluginV1Beta1PluginServiceClientProvider + }, +) *pluginDataProvider { + return &pluginDataProvider{ + logger: logger, + clientProvider: clientProvider, + } +} + +func (p *pluginDataProvider) GetPluginDatasForPluginKeys( + ctx context.Context, + pluginKeys []bufplugin.PluginKey, +) ([]bufplugin.PluginData, error) { + if len(pluginKeys) == 0 { + return nil, nil + } + // TODO(ed): check unique digests. + // TODO(ed): check unique full names. + + registryToIndexedPluginKeys := slicesext.ToIndexedValuesMap( + pluginKeys, + func(pluginKey bufplugin.PluginKey) string { + return pluginKey.PluginFullName().Registry() + }, + ) + indexedPluginDatas := make([]slicesext.Indexed[bufplugin.PluginData], 0, len(pluginKeys)) + for registry, indexedPluginKeys := range registryToIndexedPluginKeys { + indexedRegistryPluginDatas, err := p.getIndexedPluginDatasForRegistryAndIndexedPluginKeys( + ctx, + registry, + indexedPluginKeys, + ) + if err != nil { + return nil, err + } + indexedPluginDatas = append(indexedPluginDatas, indexedRegistryPluginDatas...) + } + return slicesext.IndexedToSortedValues(indexedPluginDatas), nil +} + +func (p *pluginDataProvider) getIndexedPluginDatasForRegistryAndIndexedPluginKeys( + ctx context.Context, + registry string, + indexedPluginKeys []slicesext.Indexed[bufplugin.PluginKey], +) ([]slicesext.Indexed[bufplugin.PluginData], error) { + values := slicesext.Map(indexedPluginKeys, func(indexedPluginKey slicesext.Indexed[bufplugin.PluginKey]) *pluginv1beta1.DownloadRequest_Value { + resourceRefName := &pluginv1beta1.ResourceRef_Name{ + Owner: indexedPluginKey.Value.PluginFullName().Owner(), + Plugin: indexedPluginKey.Value.PluginFullName().Name(), + Child: &pluginv1beta1.ResourceRef_Name_Ref{ + Ref: indexedPluginKey.Value.CommitID().String(), + }, + } + return &pluginv1beta1.DownloadRequest_Value{ + ResourceRef: &pluginv1beta1.ResourceRef{ + Value: &pluginv1beta1.ResourceRef_Name_{ + Name: resourceRefName, + }, + }, + } + }) + + pluginResponse, err := p.clientProvider.PluginV1Beta1DownloadServiceClient(registry).Download( + ctx, + connect.NewRequest(&pluginv1beta1.DownloadRequest{ + Values: values, + }), + ) + if err != nil { + return nil, err + } + pluginContents := pluginResponse.Msg.Contents + if len(pluginContents) != len(indexedPluginKeys) { + return nil, syserror.New("did not get the expected number of plugin datas") + } + + commitIDToIndexedPluginKeys, err := slicesext.ToUniqueValuesMapError( + indexedPluginKeys, + func(indexedPluginKey slicesext.Indexed[bufplugin.PluginKey]) (uuid.UUID, error) { + return indexedPluginKey.Value.CommitID(), nil + }, + ) + if err != nil { + return nil, err + } + + indexedPluginDatas := make([]slicesext.Indexed[bufplugin.PluginData], 0, len(indexedPluginKeys)) + for _, pluginContent := range pluginContents { + commitID, err := uuid.Parse(pluginContent.Commit.Id) + if err != nil { + return nil, err + } + indexedPluginKey, ok := commitIDToIndexedPluginKeys[commitID] + if !ok { + return nil, syserror.Newf("did not get plugin key from store with commitID %q", commitID) + } + // TODO(ed): handle compression. + data := pluginContent.Content + + pluginData, err := bufplugin.NewPluginData( + ctx, indexedPluginKey.Value, func() ([]byte, error) { + // TODO: handle compression here? + return data, nil + }, + ) + if err != nil { + return nil, err + } + indexedPluginDatas = append( + indexedPluginDatas, + slicesext.Indexed[bufplugin.PluginData]{ + Value: pluginData, + Index: indexedPluginKey.Index, + }, + ) + } + return indexedPluginDatas, nil +} diff --git a/private/bufpkg/bufplugin/bufpluginapi/plugin_key_provider.go b/private/bufpkg/bufplugin/bufpluginapi/plugin_key_provider.go new file mode 100644 index 0000000000..82da443339 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/plugin_key_provider.go @@ -0,0 +1,167 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi + +import ( + "context" + "log/slog" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/bufpkg/bufapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/uuidutil" +) + +// NewPluginKeyProvider returns a new PluginKeyProvider for the given API clients. +func NewPluginKeyProvider( + logger *slog.Logger, + clientProvider interface { + bufapi.PluginV1Beta1CommitServiceClientProvider + bufapi.PluginV1Beta1PluginServiceClientProvider + }, +) bufplugin.PluginKeyProvider { + return newPluginKeyProvider(logger, clientProvider) +} + +// *** PRIVATE *** + +type pluginKeyProvider struct { + logger *slog.Logger + clientProvider interface { + bufapi.PluginV1Beta1CommitServiceClientProvider + bufapi.PluginV1Beta1PluginServiceClientProvider + } +} + +func newPluginKeyProvider( + logger *slog.Logger, + clientProvider interface { + bufapi.PluginV1Beta1CommitServiceClientProvider + bufapi.PluginV1Beta1PluginServiceClientProvider + }, +) *pluginKeyProvider { + return &pluginKeyProvider{ + logger: logger, + clientProvider: clientProvider, + } +} + +func (p *pluginKeyProvider) GetPluginKeysForPluginRefs( + ctx context.Context, + pluginRefs []bufplugin.PluginRef, + digestType bufplugin.DigestType, +) ([]bufplugin.PluginKey, error) { + if len(pluginRefs) == 0 { + return nil, nil + } + // Check unique pluginRefs. + if _, err := slicesext.ToUniqueValuesMapError( + pluginRefs, + func(pluginRef bufplugin.PluginRef) (string, error) { + return pluginRef.String(), nil + }, + ); err != nil { + return nil, err + } + registryToIndexedPluginRefs := slicesext.ToIndexedValuesMap( + pluginRefs, + func(pluginRef bufplugin.PluginRef) string { + return pluginRef.PluginFullName().Registry() + }, + ) + indexedPluginKeys := make([]slicesext.Indexed[bufplugin.PluginKey], 0, len(pluginRefs)) + for registry, indexedPluginRefs := range registryToIndexedPluginRefs { + indexedRegistryPluginKeys, err := p.getIndexedPluginKeysForRegistryAndIndexedPluginRefs( + ctx, + registry, + indexedPluginRefs, + digestType, + ) + if err != nil { + return nil, err + } + indexedPluginKeys = append(indexedPluginKeys, indexedRegistryPluginKeys...) + } + return slicesext.IndexedToSortedValues(indexedPluginKeys), nil +} + +func (p *pluginKeyProvider) getIndexedPluginKeysForRegistryAndIndexedPluginRefs( + ctx context.Context, + registry string, + indexedPluginRefs []slicesext.Indexed[bufplugin.PluginRef], + digestType bufplugin.DigestType, +) ([]slicesext.Indexed[bufplugin.PluginKey], error) { + resourceRefs := slicesext.Map(indexedPluginRefs, func(indexedPluginRef slicesext.Indexed[bufplugin.PluginRef]) *pluginv1beta1.ResourceRef { + resourceRefName := &pluginv1beta1.ResourceRef_Name{ + Owner: indexedPluginRef.Value.PluginFullName().Owner(), + Plugin: indexedPluginRef.Value.PluginFullName().Name(), + } + if ref := indexedPluginRef.Value.Ref(); ref != "" { + resourceRefName.Child = &pluginv1beta1.ResourceRef_Name_Ref{ + Ref: ref, + } + } + return &pluginv1beta1.ResourceRef{ + Value: &pluginv1beta1.ResourceRef_Name_{ + Name: resourceRefName, + }, + } + }) + + pluginResponse, err := p.clientProvider.PluginV1Beta1CommitServiceClient(registry).GetCommits( + ctx, + connect.NewRequest(&pluginv1beta1.GetCommitsRequest{ + ResourceRefs: resourceRefs, + }), + ) + if err != nil { + return nil, err + } + commits := pluginResponse.Msg.Commits + if len(commits) != len(indexedPluginRefs) { + return nil, syserror.New("did not get the expected number of plugin datas") + } + + indexedPluginKeys := make([]slicesext.Indexed[bufplugin.PluginKey], len(commits)) + for i, commit := range commits { + commitID, err := uuidutil.FromDashless(commit.Id) + if err != nil { + return nil, err + } + digest, err := V1Beta1ProtoToDigest(commit.Digest) + if err != nil { + return nil, err + } + pluginKey, err := bufplugin.NewPluginKey( + // Note we don't have to resolve owner_name and plugin_name since we already have them. + indexedPluginRefs[i].Value.PluginFullName(), + commitID, + func() (bufplugin.Digest, error) { + return digest, nil + }, + ) + if err != nil { + return nil, err + } + indexedPluginKeys[i] = slicesext.Indexed[bufplugin.PluginKey]{ + Value: pluginKey, + Index: indexedPluginRefs[i].Index, + } + } + return indexedPluginKeys, nil +} diff --git a/private/bufpkg/bufplugin/bufplugincache/bufplugincache.go b/private/bufpkg/bufplugin/bufplugincache/bufplugincache.go new file mode 100644 index 0000000000..7c0fa4c721 --- /dev/null +++ b/private/bufpkg/bufplugin/bufplugincache/bufplugincache.go @@ -0,0 +1,15 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugincache diff --git a/private/bufpkg/bufplugin/bufplugincache/plugin_data_provider.go b/private/bufpkg/bufplugin/bufplugincache/plugin_data_provider.go new file mode 100644 index 0000000000..e252a2dca2 --- /dev/null +++ b/private/bufpkg/bufplugin/bufplugincache/plugin_data_provider.go @@ -0,0 +1,111 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugincache + +import ( + "context" + "log/slog" + "sync/atomic" + + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginstore" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/uuidutil" + "github.com/google/uuid" +) + +// NewPluginDataProvider returns a new PluginDataProvider that caches the results of the delegate. +// +// The PluginDataStore is used as a cache. +func NewPluginDataProvider( + logger *slog.Logger, + delegate bufplugin.PluginDataProvider, + store bufpluginstore.PluginDataStore, +) bufplugin.PluginDataProvider { + return newPluginDataProvider(logger, delegate, store) +} + +/// *** PRIVATE *** + +type pluginDataProvider struct { + logger *slog.Logger + delegate bufplugin.PluginDataProvider + store bufpluginstore.PluginDataStore + + keysRetrieved atomic.Int64 + keysHit atomic.Int64 +} + +func newPluginDataProvider( + logger *slog.Logger, + delegate bufplugin.PluginDataProvider, + store bufpluginstore.PluginDataStore, +) *pluginDataProvider { + return &pluginDataProvider{ + logger: logger, + delegate: delegate, + store: store, + } +} + +func (p *pluginDataProvider) GetPluginDatasForPluginKeys( + ctx context.Context, + pluginKeys []bufplugin.PluginKey, +) ([]bufplugin.PluginData, error) { + foundValues, notFoundKeys, err := p.store.GetPluginDatasForPluginKeys(ctx, pluginKeys) + if err != nil { + return nil, err + } + + delegateValues, err := p.delegate.GetPluginDatasForPluginKeys(ctx, notFoundKeys) + if err != nil { + return nil, err + } + if err := p.store.PutPluginDatas(ctx, delegateValues); err != nil { + return nil, err + } + + p.keysRetrieved.Add(int64(len(pluginKeys))) + p.keysHit.Add(int64(len(foundValues))) + + commitIDToIndexedKey, err := slicesext.ToUniqueIndexedValuesMap( + pluginKeys, + func(pluginKey bufplugin.PluginKey) uuid.UUID { + return pluginKey.CommitID() + }, + ) + if err != nil { + return nil, err + } + indexedValues, err := slicesext.MapError( + append(foundValues, delegateValues...), + func(value bufplugin.PluginData) (slicesext.Indexed[bufplugin.PluginData], error) { + commitID := value.PluginKey().CommitID() + indexedKey, ok := commitIDToIndexedKey[commitID] + if !ok { + return slicesext.Indexed[bufplugin.PluginData]{}, syserror.Newf("did not get value from store with commitID %q", uuidutil.ToDashless(commitID)) + } + return slicesext.Indexed[bufplugin.PluginData]{ + Value: value, + Index: indexedKey.Index, + }, nil + }, + ) + if err != nil { + return nil, err + } + return slicesext.IndexedToSortedValues(indexedValues), nil +} diff --git a/private/bufpkg/bufplugin/bufplugincache/usage.gen.go b/private/bufpkg/bufplugin/bufplugincache/usage.gen.go new file mode 100644 index 0000000000..4f54051370 --- /dev/null +++ b/private/bufpkg/bufplugin/bufplugincache/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufplugincache + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufplugin/bufpluginstore/bufpluginstore.go b/private/bufpkg/bufplugin/bufpluginstore/bufpluginstore.go new file mode 100644 index 0000000000..a8755d2de3 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginstore/bufpluginstore.go @@ -0,0 +1,15 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginstore diff --git a/private/bufpkg/bufplugin/bufpluginstore/module_data_store.go b/private/bufpkg/bufplugin/bufpluginstore/module_data_store.go new file mode 100644 index 0000000000..02b1edb3cb --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginstore/module_data_store.go @@ -0,0 +1,167 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginstore + +import ( + "context" + "errors" + "io/fs" + "log/slog" + + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/normalpath" + "github.com/bufbuild/buf/private/pkg/storage" + "github.com/bufbuild/buf/private/pkg/uuidutil" +) + +// PluginStore reads and writes PluginsDatas. +type PluginDataStore interface { + // GetPluginDatasForPluginKey gets the PluginDatas from the store for the PluginKeys. + // + // Returns the found PluginDatas, and the input PluginKeys that were not found, each + // ordered by the order of the input PluginKeys. + GetPluginDatasForPluginKeys(context.Context, []bufplugin.PluginKey) ( + foundPluginDatas []bufplugin.PluginData, + notFoundPluginKeys []bufplugin.PluginKey, + err error, + ) + // Put puts the PluginDatas to the store. + PutPluginDatas(ctx context.Context, moduleDatas []bufplugin.PluginData) error +} + +// NewPluginDataStore returns a new PluginDataStore for the given bucket. +// +// It is assumed that the PluginDataStore has complete control of the bucket. +// +// This is typically used to interact with a cache directory. +func NewPluginDataStore( + logger *slog.Logger, + bucket storage.ReadWriteBucket, +) PluginDataStore { + return newPluginDataStore(logger, bucket) +} + +/// *** PRIVATE *** + +type pluginDataStore struct { + logger *slog.Logger + bucket storage.ReadWriteBucket + + tar bool +} + +func newPluginDataStore( + logger *slog.Logger, + bucket storage.ReadWriteBucket, +) *pluginDataStore { + return &pluginDataStore{ + logger: logger, + bucket: bucket, + } +} + +func (p *pluginDataStore) GetPluginDatasForPluginKeys( + ctx context.Context, + pluginKeys []bufplugin.PluginKey, +) ([]bufplugin.PluginData, []bufplugin.PluginKey, error) { + var foundPluginDatas []bufplugin.PluginData + var notFoundPluginKeys []bufplugin.PluginKey + for _, pluginKey := range pluginKeys { + pluginData, err := p.getPluginDataForPluginKey(ctx, pluginKey) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, nil, err + } + notFoundPluginKeys = append(notFoundPluginKeys, pluginKey) + } else { + foundPluginDatas = append(foundPluginDatas, pluginData) + } + } + return foundPluginDatas, notFoundPluginKeys, nil +} + +func (p *pluginDataStore) PutPluginDatas( + ctx context.Context, + pluginDatas []bufplugin.PluginData, +) error { + for _, pluginData := range pluginDatas { + if err := p.putPluginData(ctx, pluginData); err != nil { + return err + } + } + return nil +} + +// getPluginDataForPluginKey reads the plugin data for the plugin key from the cache. +func (p *pluginDataStore) getPluginDataForPluginKey( + ctx context.Context, + pluginKey bufplugin.PluginKey, +) (bufplugin.PluginData, error) { + pluginDataStorePath, err := getPluginDataStorePath(pluginKey) + if err != nil { + return nil, err + } + if exists, err := storage.Exists(ctx, p.bucket, pluginDataStorePath); err != nil { + return nil, err + } else if !exists { + return nil, fs.ErrNotExist + } + return bufplugin.NewPluginData( + ctx, + pluginKey, + func() ([]byte, error) { + // TODO(ed): handle compression. + return storage.ReadPath(ctx, p.bucket, pluginDataStorePath) + }, + ) +} + +// putPluginData puts the plugin data into the plugin cache. +func (p *pluginDataStore) putPluginData( + ctx context.Context, + pluginData bufplugin.PluginData, +) error { + pluginKey := pluginData.PluginKey() + pluginDataStorePath, err := getPluginDataStorePath(pluginKey) + if err != nil { + return err + } + data, err := pluginData.Data() + if err != nil { + return err + } + // TODO(ed): handle compression. + return storage.PutPath(ctx, p.bucket, pluginDataStorePath, data) +} + +// getPluginDataStorePath returns the path for the plugin data store for the plugin key. +// +// This is "digestType/registry/owner/name/dashlessCommitID", e.g. the plugin +// "buf.build/acme/check-plugin" with commit "12345-abcde" and digest type "p1" +// will return "p1/buf.build/acme/check-plugin/12345abcde". +func getPluginDataStorePath(pluginKey bufplugin.PluginKey) (string, error) { + digest, err := pluginKey.Digest() + if err != nil { + return "", err + } + fullName := pluginKey.PluginFullName() + return normalpath.Join( + digest.Type().String(), + fullName.Registry(), + fullName.Owner(), + fullName.Name(), + uuidutil.ToDashless(pluginKey.CommitID())+".wasm", + ), nil +} diff --git a/private/bufpkg/bufplugin/bufpluginstore/usage.gen.go b/private/bufpkg/bufplugin/bufpluginstore/usage.gen.go new file mode 100644 index 0000000000..3b402991cf --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginstore/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufpluginstore + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufplugin/commit.go b/private/bufpkg/bufplugin/commit.go new file mode 100644 index 0000000000..fa1daccddb --- /dev/null +++ b/private/bufpkg/bufplugin/commit.go @@ -0,0 +1,72 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "sync" + "time" +) + +// Commit represents a Commit on the BSR. +type Commit interface { + // PluginKey returns the PluginKey for the Commit. + PluginKey() PluginKey + // CreateTime returns the time the Commit was created on the BSR. + CreateTime() (time.Time, error) + + isCommit() +} + +// NewCommit returns a new Commit. +func NewCommit( + pluginKey PluginKey, + getCreateTime func() (time.Time, error), +) Commit { + return newCommit( + pluginKey, + getCreateTime, + ) +} + +// *** PRIVATE *** + +type commit struct { + pluginKey PluginKey + getCreateTime func() (time.Time, error) +} + +func newCommit( + pluginKey PluginKey, + getCreateTime func() (time.Time, error), +) *commit { + return &commit{ + pluginKey: pluginKey, + getCreateTime: sync.OnceValues(getCreateTime), + } +} + +func (c *commit) PluginKey() PluginKey { + return c.pluginKey +} + +func (c *commit) CreateTime() (time.Time, error) { + // This may invoke tamper-proofing per newCommit construction. + if _, err := c.pluginKey.Digest(); err != nil { + return time.Time{}, err + } + return c.getCreateTime() +} + +func (*commit) isCommit() {} diff --git a/private/bufpkg/bufplugin/digest.go b/private/bufpkg/bufplugin/digest.go new file mode 100644 index 0000000000..9c623b5147 --- /dev/null +++ b/private/bufpkg/bufplugin/digest.go @@ -0,0 +1,211 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/bufbuild/buf/private/bufpkg/bufcas" + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/pkg/syserror" +) + +const ( + // DigestTypeP1 represents the p4 plugin digest type. + // + // This is the newest digest type, and should generally be used. The string value + // of this is "p1". + DigestTypeP1 = iota + 1 +) + +var ( + // AllDigestTypes are all known DigestTypes. + AllDigestTypes = []DigestType{ + DigestTypeP1, + } + digestTypeToString = map[DigestType]string{ + DigestTypeP1: "p1", + } + stringToDigestType = map[string]DigestType{ + "p1": DigestTypeP1, + } +) + +// DigestType is a type of digest. +type DigestType int + +// ParseDigestType parses a DigestType from its string representation. +// +// This reverses DigestType.String(). +// +// Returns an error of type *bufparse.ParseError if thie string could not be parsed. +func ParseDigestType(s string) (DigestType, error) { + d, ok := stringToDigestType[s] + if !ok { + return 0, bufparse.NewParseError( + "plugin digest type", + s, + fmt.Errorf("unknown type: %q", s), + ) + } + return d, nil +} + +// String prints the string representation of the DigestType. +func (d DigestType) String() string { + s, ok := digestTypeToString[d] + if !ok { + return strconv.Itoa(int(d)) + } + return s +} + +// Digest is a digest of some content. +// +// It consists of a DigestType and a digest value. +type Digest interface { + // String() prints typeString:hexValue. + fmt.Stringer + + // Type returns the type of digest. + // Always a valid value. + Type() DigestType + // Value returns the digest value. + // + // Always non-empty. + Value() []byte + + isDigest() +} + +// NewDigest creates a new Digest. +func NewDigest(digestType DigestType, bufcasDigest bufcas.Digest) (Digest, error) { + switch digestType { + case DigestTypeP1: + if bufcasDigest.Type() != bufcas.DigestTypeShake256 { + return nil, syserror.Newf( + "trying to create a %v Digest for a cas Digest of type %v", + digestType, + bufcasDigest.Type(), + ) + } + return newDigest(digestType, bufcasDigest), nil + default: + // This is a system error. + return nil, syserror.Newf("unknown DigestType: %v", digestType) + } +} + +// DigestEqual returns true if the given Digests are considered equal. +// +// If both Digests are nil, this returns true. +// +// This checks both the DigestType and Digest value. +func DigestEqual(a Digest, b Digest) bool { + if (a == nil) != (b == nil) { + return false + } + if a == nil { + return true + } + if a.Type() != b.Type() { + return false + } + return bytes.Equal(a.Value(), b.Value()) +} + +// ParseDigest parses a Digest from its string representation. +// +// A Digest string is of the form typeString:hexValue. +// The string is expected to be non-empty, If not, an error is treutned. +// +// This reverses Digest.String(). +func ParseDigest(s string) (Digest, error) { + if s == "" { + // This should be considered a system error. + return nil, errors.New("empty string passed to ParseDigest") + } + digestTypeString, hexValue, ok := strings.Cut(s, ":") + if !ok { + return nil, bufparse.NewParseError( + "plugin digest", s, + errors.New(`must be in the form "digest_type:digest_hex_value"`), + ) + } + digestType, err := ParseDigestType(digestTypeString) + if err != nil { + return nil, bufparse.NewParseError( + "plugin digest", + digestTypeString, + err, + ) + } + value, err := hex.DecodeString(hexValue) + if err != nil { + return nil, bufparse.NewParseError( + "plugin digest", + s, + errors.New(`could not parse hex: must in the form "digest_type:digest_hex_value"`), + ) + } + switch digestType { + case DigestTypeP1: + bufcasDigest, err := bufcas.NewDigest(value) + if err != nil { + return nil, err + } + return NewDigest(digestType, bufcasDigest) + default: + return nil, syserror.Newf("unknown DigestType: %v", digestType) + } +} + +/// *** PRIVATE *** + +type digest struct { + digestType DigestType + bufcasDigest bufcas.Digest + // Cache as we call String pretty often. + // We could do this lazily but not worth it. + stringValue string +} + +// validation should occur outside of this function. +func newDigest(digestType DigestType, bufcasDigest bufcas.Digest) *digest { + return &digest{ + digestType: digestType, + bufcasDigest: bufcasDigest, + stringValue: digestType.String() + ":" + hex.EncodeToString(bufcasDigest.Value()), + } +} + +func (d *digest) Type() DigestType { + return d.digestType +} + +func (d *digest) Value() []byte { + return d.bufcasDigest.Value() +} + +func (d *digest) String() string { + return d.stringValue +} + +func (*digest) isDigest() {} diff --git a/private/bufpkg/bufplugin/errors.go b/private/bufpkg/bufplugin/errors.go new file mode 100644 index 0000000000..8543df2fd0 --- /dev/null +++ b/private/bufpkg/bufplugin/errors.go @@ -0,0 +1,69 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "strings" + + "github.com/bufbuild/buf/private/pkg/uuidutil" + "github.com/google/uuid" +) + +// DigestMismatchError is the error returned if the Digest of a downloaded Module or Commit +// does not match the expected digest in a buf.lock file. +type DigestMismatchError struct { + PluginFullName PluginFullName + CommitID uuid.UUID + ExpectedDigest Digest + ActualDigest Digest +} + +// Error implements the error interface. +func (m *DigestMismatchError) Error() string { + if m == nil { + return "" + } + var builder strings.Builder + _, _ = builder.WriteString(`*** Digest verification failed`) + if m.PluginFullName != nil { + _, _ = builder.WriteString(` for "`) + _, _ = builder.WriteString(m.PluginFullName.String()) + if m.CommitID != uuid.Nil { + _, _ = builder.WriteString(`:`) + _, _ = builder.WriteString(uuidutil.ToDashless(m.CommitID)) + } + _, _ = builder.WriteString(`"`) + } + _, _ = builder.WriteString(` ***`) + _, _ = builder.WriteString("\n") + if m.ExpectedDigest != nil && m.ActualDigest != nil { + _, _ = builder.WriteString("\t") + _, _ = builder.WriteString(`Expected digest (from buf.lock): "`) + _, _ = builder.WriteString(m.ExpectedDigest.String()) + _, _ = builder.WriteString(`"`) + _, _ = builder.WriteString("\n") + _, _ = builder.WriteString("\t") + _, _ = builder.WriteString(`Actual digest: "`) + _, _ = builder.WriteString(m.ActualDigest.String()) + _, _ = builder.WriteString(`"`) + _, _ = builder.WriteString("\n") + } + _, _ = builder.WriteString("\t") + _, _ = builder.WriteString(`This may be the result of a hand-edited or corrupted buf.lock file, a corrupted local cache, and/or an attack.`) + _, _ = builder.WriteString("\n") + _, _ = builder.WriteString("\t") + _, _ = builder.WriteString(`To clear your local cache, run "buf registry cc".`) + return builder.String() +} diff --git a/private/bufpkg/bufplugin/plugin_data.go b/private/bufpkg/bufplugin/plugin_data.go new file mode 100644 index 0000000000..96a46e634c --- /dev/null +++ b/private/bufpkg/bufplugin/plugin_data.go @@ -0,0 +1,113 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "bytes" + "context" + "sync" + + "github.com/bufbuild/buf/private/bufpkg/bufcas" +) + +// TODO(ed): comment. +type PluginData interface { + // PluginKey used to downoad this PluginData. + // + // The Digest from this PluginKey is used for tamper-proofing. + PluginKey() PluginKey + // Data returns the raw bytes for the plugin data. + Data() ([]byte, error) + + isPluginData() +} + +// NewPluginData returns a new PluginData. +func NewPluginData( + ctx context.Context, + pluginKey PluginKey, + getData func() ([]byte, error), +) (PluginData, error) { + return newPluginData( + ctx, + pluginKey, + getData, + ) +} + +// *** PRIVATE *** + +type pluginData struct { + pluginKey PluginKey + getData func() ([]byte, error) + + checkDigest func() error +} + +func newPluginData( + ctx context.Context, + pluginKey PluginKey, + getData func() ([]byte, error), +) (*pluginData, error) { + + pluginData := &pluginData{ + pluginKey: pluginKey, + getData: getData, + } + pluginData.checkDigest = sync.OnceValue(func() error { + pluginData, err := pluginData.getData() + if err != nil { + return err + } + bufcasDigest, err := bufcas.NewDigestForContent( + bytes.NewReader(pluginData), + ) + if err != nil { + return err + } + actualDigest, err := NewDigest(DigestTypeP1, bufcasDigest) + if err != nil { + return err + } + expectedDigest, err := pluginKey.Digest() + if err != nil { + return err + } + if !DigestEqual(actualDigest, expectedDigest) { + return &DigestMismatchError{ + PluginFullName: pluginKey.PluginFullName(), + CommitID: pluginKey.CommitID(), + ExpectedDigest: expectedDigest, + ActualDigest: actualDigest, + } + + } + return nil + }) + return pluginData, nil +} + +func (p *pluginData) PluginKey() PluginKey { + return p.pluginKey +} + +func (p *pluginData) Data() ([]byte, error) { + if err := p.checkDigest(); err != nil { + return nil, err + } + return p.getData() +} + +func (*pluginData) isPluginData() {} diff --git a/private/bufpkg/bufplugin/plugin_data_provider.go b/private/bufpkg/bufplugin/plugin_data_provider.go new file mode 100644 index 0000000000..a12b50a2a3 --- /dev/null +++ b/private/bufpkg/bufplugin/plugin_data_provider.go @@ -0,0 +1,57 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "context" + "io/fs" +) + +var ( + // NopPluginDataProvider is a no-op PluginDataProvider. + NopPluginDataProvider PluginDataProvider = nopPluginDataProvider{} +) + +// PluginDataProvider provides PluginsDatas. +type PluginDataProvider interface { + // GetPluginDatasForPluginKeys gets the PluginDatas for the PluginKeys. + // + // Returned PluginDatas will be in the same order as the input PluginKeys. + // + // The input PluginKeys are expected to be unique by PluginFullName. The implementation + // may error if this is not the case. + // + // The input PluginKeys are expected to have the same DigestType. The implementation + // may error if this is not the case. + // + // If there is no error, the length of the PluginDatas returned will match the length of the PluginKeys. + // If there is an error, no PluginDatas will be returned. + // If any PluginKey is not found, an error with fs.ErrNotExist will be returned. + GetPluginDatasForPluginKeys( + context.Context, + []PluginKey, + ) ([]PluginData, error) +} + +// *** PRIVATE *** + +type nopPluginDataProvider struct{} + +func (nopPluginDataProvider) GetPluginDatasForPluginKeys( + context.Context, + []PluginKey, +) ([]PluginData, error) { + return nil, fs.ErrNotExist +} diff --git a/private/bufpkg/bufplugin/plugin_full_name.go b/private/bufpkg/bufplugin/plugin_full_name.go new file mode 100644 index 0000000000..4b078fafff --- /dev/null +++ b/private/bufpkg/bufplugin/plugin_full_name.go @@ -0,0 +1,124 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "errors" + "fmt" + "strings" + + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/pkg/netext" +) + +// PluginFullName represents the full name of the Plugin, including its registry, owner, and name. +type PluginFullName interface { + // String returns "registry/owner/name". + fmt.Stringer + + // Registry returns the hostname of the BSR instance that this Plugin is contained within. + Registry() string + // Owner returns the name of the user or organization that owns this Plugin. + Owner() string + // Name returns the name of the Plugin. + Name() string + + isPluginFullName() +} + +// NewPluginFullName returns a new PluginFullName for the given components. +func NewPluginFullName( + registry string, + owner string, + name string, +) (PluginFullName, error) { + return newPluginFullName(registry, owner, name) +} + +// ParsePluginFullName parses a PluginFullName from a string in the form "registry/owner/name". +func ParsePluginFullName(pluginFullNameString string) (PluginFullName, error) { + registry, owner, name, err := bufparse.ParseFullNameComponents(pluginFullNameString) + if err != nil { + return nil, err + } + return newPluginFullName(registry, owner, name) +} + +// *** PRIVATE *** + +type pluginFullName struct { + registry string + owner string + name string +} + +func newPluginFullName( + registry string, + owner string, + name string, +) (*pluginFullName, error) { + if err := validateModuleFullNameParameters(registry, owner, name); err != nil { + return nil, err + } + return &pluginFullName{ + registry: registry, + owner: owner, + name: name, + }, nil +} + +func (p *pluginFullName) Registry() string { + return p.registry +} + +func (p *pluginFullName) Owner() string { + return p.owner +} + +func (p *pluginFullName) Name() string { + return p.name +} + +func (p *pluginFullName) String() string { + return p.registry + "/" + p.owner + "/" + p.name +} + +func (*pluginFullName) isPluginFullName() {} + +func validateModuleFullNameParameters( + registry string, + owner string, + name string, +) error { + if registry == "" { + return errors.New("registry is empty") + } + if _, err := netext.ValidateHostname(registry); err != nil { + return fmt.Errorf("registry %q is not a valid hostname: %w", registry, err) + } + if owner == "" { + return errors.New("owner is empty") + } + if strings.Contains(owner, "/") { + return fmt.Errorf("owner %q cannot contain slashes", owner) + } + if name == "" { + return errors.New("name is empty") + } + if strings.Contains(name, "/") { + return fmt.Errorf("name %q cannot contain slashes", name) + } + return nil +} diff --git a/private/bufpkg/bufplugin/plugin_key.go b/private/bufpkg/bufplugin/plugin_key.go new file mode 100644 index 0000000000..3177d9d61f --- /dev/null +++ b/private/bufpkg/bufplugin/plugin_key.go @@ -0,0 +1,106 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "errors" + "fmt" + "sync" + + "github.com/bufbuild/buf/private/pkg/uuidutil" + "github.com/google/uuid" +) + +// PluginKey provides identifying information for a Plugin. +// +// TODO(emcfarlane) +type PluginKey interface { + // String returns "registry/owner/name:dashlessCommitID". + fmt.Stringer + + // PluginFullName returns the full name of the Plugin. + // + // Always present. + PluginFullName() PluginFullName + // CommitID returns the ID of the Commit. + // + // It is up to the caller to convert this to a dashless ID when necessary. + // + // Always present, that is CommitID() == uuid.Nil will always be false. + CommitID() uuid.UUID + // Digest returns the Plugin digest. + // + // TODO(emcfarlane) + Digest() (Digest, error) + + isPluginKey() +} + +func NewPluginKey( + pluginFullName PluginFullName, + commitID uuid.UUID, + getDigest func() (Digest, error), +) (PluginKey, error) { + return newPluginKey( + pluginFullName, + commitID, + getDigest, + ) +} + +// ** PRIVATE ** + +type pluginKey struct { + pluginFullName PluginFullName + commitID uuid.UUID + + getDigest func() (Digest, error) +} + +func newPluginKey( + pluginFullName PluginFullName, + commitID uuid.UUID, + getDigest func() (Digest, error), +) (*pluginKey, error) { + if pluginFullName == nil { + return nil, errors.New("nil PluginFullName when constructing PluginKey") + } + if commitID == uuid.Nil { + return nil, errors.New("empty commitID when constructing PluginKey") + } + return &pluginKey{ + pluginFullName: pluginFullName, + commitID: commitID, + getDigest: sync.OnceValues(getDigest), + }, nil +} + +func (p *pluginKey) PluginFullName() PluginFullName { + return p.pluginFullName +} + +func (p *pluginKey) CommitID() uuid.UUID { + return p.commitID +} + +func (p *pluginKey) Digest() (Digest, error) { + return p.getDigest() +} + +func (p *pluginKey) String() string { + return p.pluginFullName.String() + ":" + uuidutil.ToDashless(p.commitID) +} + +func (*pluginKey) isPluginKey() {} diff --git a/private/bufpkg/bufplugin/plugin_key_provider.go b/private/bufpkg/bufplugin/plugin_key_provider.go new file mode 100644 index 0000000000..01c647340a --- /dev/null +++ b/private/bufpkg/bufplugin/plugin_key_provider.go @@ -0,0 +1,43 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "context" + "io/fs" +) + +var ( + // NopPluginKeyProvider is a no-op PluginKeyProvider. + NopPluginKeyProvider PluginKeyProvider = nopPluginKeyProvider{} +) + +// PluginKeyProvider provides PluginKeys for PluginRefs. +type PluginKeyProvider interface { + // TODO(ed) + GetPluginKeysForPluginRefs(context.Context, []PluginRef, DigestType) ([]PluginKey, error) +} + +// *** PRIVATE *** + +type nopPluginKeyProvider struct{} + +func (nopPluginKeyProvider) GetPluginKeysForPluginRefs( + context.Context, + []PluginRef, + DigestType, +) ([]PluginKey, error) { + return nil, fs.ErrNotExist +} diff --git a/private/bufpkg/bufplugin/plugin_ref.go b/private/bufpkg/bufplugin/plugin_ref.go new file mode 100644 index 0000000000..0a30f542b4 --- /dev/null +++ b/private/bufpkg/bufplugin/plugin_ref.go @@ -0,0 +1,98 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "errors" + "fmt" + + "github.com/bufbuild/buf/private/bufpkg/bufparse" +) + +// PluginRef is an unresolved reference to a Plugin. +type PluginRef interface { + // String returns "registry/owner/name[:ref]". + fmt.Stringer + + // PluginFullName returns the full name of the Plugin. + // + // Always present. + PluginFullName() PluginFullName + // Ref returns the reference within the Plugin. + // + // May be a label or dashless commitID. + // + // May be empty, in which case this references the commit of the default label of the Plugin. + Ref() string + + isPluginRef() +} + +// NewPluginRef returns a new PluginRef for the given compoonents. +func NewPluginRef( + registry string, + owner string, + name string, + ref string, +) (PluginRef, error) { + pluginFullName, err := NewPluginFullName(registry, owner, name) + if err != nil { + return nil, err + } + return newPluginRef(pluginFullName, ref) +} + +func ParsePluginRef(pluginRefString string) (PluginRef, error) { + registry, owner, name, ref, err := bufparse.ParseRefComponents(pluginRefString) + if err != nil { + return nil, err + } + // We don't rely on constructors for ParseErrors. + return NewPluginRef(registry, owner, name, ref) +} + +// *** PRIVATE *** + +type pluginRef struct { + pluginFullName PluginFullName + ref string +} + +func newPluginRef( + pluginFullName PluginFullName, + ref string, +) (*pluginRef, error) { + if pluginFullName == nil { + return nil, errors.New("pluginFullName is required") + } + return &pluginRef{ + pluginFullName: pluginFullName, + ref: ref, + }, nil +} + +func (m *pluginRef) PluginFullName() PluginFullName { + return m.pluginFullName +} + +func (m *pluginRef) Ref() string { + return m.ref +} + +func (m *pluginRef) String() string { + return m.pluginFullName.String() + ":" + m.ref +} + +func (*pluginRef) isPluginRef() {} diff --git a/private/bufpkg/bufplugin/usage.gen.go b/private/bufpkg/bufplugin/usage.gen.go new file mode 100644 index 0000000000..6ffd8bf974 --- /dev/null +++ b/private/bufpkg/bufplugin/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufplugin + +import _ "github.com/bufbuild/buf/private/usage"