feat!: Make registry: cloudquery the default registry instead of `r…

…egistry: github` (#15409) Uses ~cloudquery/plugin-pb-go#170 cloudquery/plugin-pb-go#172 (now tagged as 1.14.2) Part of cloudquery/cloudquery-issues#924 (Internal issue)
cloudquery · Nov 22, 2023 · 6d9b2a9 · 6d9b2a9
1 parent 3c2d536
commit 6d9b2a9
Show file tree

Hide file tree

Showing 21 changed files with 363 additions and 95 deletions.
diff --git a/cli/cmd/infer_test.go b/cli/cmd/infer_test.go
@@ -0,0 +1,159 @@
+//go:build !windows
+
+package cmd
+
+import (
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestInferRegistry(t *testing.T) {
+	configs := []struct {
+		name               string
+		config             string
+		outContains        string
+		wantErrContains    string
+		wantErrNotContains string
+	}{
+		{
+			name: "infer registry success",
+			config: `
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  version: v10.0.0
+  table_concurrency: 10
+  destinations: [dummydest]
+  tables: [test]
+---
+kind: destination
+spec:
+  name: dummydest
+  path: localhost:7777
+  registry: grpc
+`,
+			outContains:        "Downloading https://storage.googleapis.com/cq-cloud-releases/cloudquery/source/gcp/v10.0.0/",
+			wantErrContains:    "",
+			wantErrNotContains: "",
+		},
+		{
+			name: "infer registry fail: recommend github",
+			config: `
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  version: v99.999.0
+  table_concurrency: 10
+  destinations: [dummydest]
+  tables: [test]
+---
+kind: destination
+spec:
+  name: dummydest
+  path: localhost:7777
+  registry: grpc
+`,
+			wantErrContains:    "Hint",
+			wantErrNotContains: "",
+		},
+		{
+			name: "no infer registry fail: don't recommend github",
+			config: `
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  registry: cloudquery
+  version: v99.999.0
+  table_concurrency: 10
+  destinations: [dummydest]
+  tables: [test]
+---
+kind: destination
+spec:
+  name: dummydest
+  path: localhost:7777
+  registry: grpc
+`,
+			wantErrContains:    "",
+			wantErrNotContains: "Hint",
+		},
+		{
+			name: "no infer registry fail: don't recommend github for github",
+			config: `
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  registry: github
+  version: v99.999.0
+  table_concurrency: 10
+  destinations: [dummydest]
+  tables: [test]
+---
+kind: destination
+spec:
+  name: dummydest
+  path: localhost:7777
+  registry: grpc
+`,
+			wantErrContains:    "",
+			wantErrNotContains: "Hint",
+		},
+	}
+
+	for _, tc := range configs {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			cqDir := t.TempDir()
+			t.Cleanup(func() {
+				os.RemoveAll(cqDir)
+			})
+			testConfig := filepath.Join(cqDir, "config.yml")
+			logFileName := path.Join(cqDir, "cloudquery.log")
+			err := os.WriteFile(testConfig, []byte(tc.config), 0644)
+			require.NoError(t, err)
+
+			oldStdout := os.Stdout
+			r, w, _ := os.Pipe()
+			os.Stdout = w
+			defer func() {
+				os.Stdout = oldStdout
+			}()
+
+			cmd := NewCmdRoot()
+			cmd.SetArgs([]string{"plugin", "install", testConfig, "--cq-dir", cqDir, "--log-file-name", logFileName})
+			err = cmd.Execute()
+
+			_ = w.Close()
+			out, _ := io.ReadAll(r)
+			if tc.outContains != "" {
+				require.Contains(t, string(out), tc.outContains)
+			}
+
+			if tc.wantErrContains == "" && tc.wantErrNotContains == "" {
+				require.NoError(t, err)
+			}
+			if tc.wantErrContains != "" {
+				require.ErrorContains(t, err, tc.wantErrContains)
+			}
+			if tc.wantErrNotContains != "" {
+				require.Error(t, err)
+				require.NotContains(t, err.Error(), tc.wantErrNotContains)
+			}
+
+			// check that log was written and contains some lines
+			b, logFileError := os.ReadFile(path.Join(cqDir, "cloudquery.log"))
+			logContent := string(b)
+			require.NoError(t, logFileError, "failed to read cloudquery.log")
+			require.NotEmpty(t, logContent, "cloudquery.log empty; expected some logs")
+		})
+	}
+}
diff --git a/cli/cmd/install.go b/cli/cmd/install.go
@@ -72,31 +72,55 @@ func installPlugin(cmd *cobra.Command, args []string) error {
 		opts = append(opts, managedplugin.WithNoSentry())
 	}
 
-	sourcePluginConfigs := make([]managedplugin.Config, 0, len(sources))
-	for _, source := range sources {
-		sourcePluginConfigs = append(sourcePluginConfigs, managedplugin.Config{
+	sourcePluginConfigs := make([]managedplugin.Config, len(sources))
+	sourceRegInferred := make([]bool, len(sources))
+	for i, source := range sources {
+		sourcePluginConfigs[i] = managedplugin.Config{
 			Name:     source.Name,
 			Version:  source.Version,
 			Path:     source.Path,
 			Registry: SpecRegistryToPlugin(source.Registry),
-		})
+		}
+		sourceRegInferred[i] = source.RegistryInferred()
 	}
-	destinationPluginConfigs := make([]managedplugin.Config, 0, len(destinations))
-	for _, destination := range destinations {
-		destinationPluginConfigs = append(destinationPluginConfigs, managedplugin.Config{
+	destinationPluginConfigs := make([]managedplugin.Config, len(destinations))
+	destinationRegInferred := make([]bool, len(destinations))
+	for i, destination := range destinations {
+		destinationPluginConfigs[i] = managedplugin.Config{
 			Name:     destination.Name,
 			Version:  destination.Version,
 			Path:     destination.Path,
 			Registry: SpecRegistryToPlugin(destination.Registry),
-		})
+		}
+		destinationRegInferred[i] = destination.RegistryInferred()
 	}
+	if clist, err := managedplugin.NewClients(ctx, managedplugin.PluginSource, sourcePluginConfigs, opts...); err != nil {
+		return enrichClientError(clist, sourceRegInferred, err)
+	}
+	if clist, err := managedplugin.NewClients(ctx, managedplugin.PluginDestination, destinationPluginConfigs, opts...); err != nil {
+		return enrichClientError(clist, destinationRegInferred, err)
+	}
+
+	return nil
+}
 
-	if _, err := managedplugin.NewClients(ctx, managedplugin.PluginSource, sourcePluginConfigs, opts...); err != nil {
+// enrichClientError gets the index of the failed client (which is one more than the last one on the list) and checks if the registry was inferred.
+// If so, adds a hint to the error message.
+func enrichClientError(clientsList managedplugin.Clients, inferredList []bool, err error) error {
+	if err == nil {
+		return nil
+	}
+	if !strings.Contains(err.Error(), "not found") {
 		return err
 	}
-	if _, err := managedplugin.NewClients(ctx, managedplugin.PluginDestination, destinationPluginConfigs, opts...); err != nil {
+	l := len(clientsList)
+	il := len(inferredList)
+	if l > il {
+		return err // shouldn't happen
+	}
+	if !inferredList[l] {
 		return err
 	}
 
-	return nil
+	return fmt.Errorf("%w. Hint: make sure to use the latest plugin version from hub.cloudquery.io or to keep using an outdated version add `registry: github` to your configuration", err)
 }
diff --git a/cli/cmd/migrate.go b/cli/cmd/migrate.go
@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"strings"
-
 	"slices"
+	"strings"
 
 	"github.com/cloudquery/cloudquery/cli/internal/auth"
 	"github.com/cloudquery/cloudquery/cli/internal/specs/v0"
@@ -69,28 +68,32 @@ func migrate(cmd *cobra.Command, args []string) error {
 	if disableSentry {
 		opts = append(opts, managedplugin.WithNoSentry())
 	}
-	sourcePluginConfigs := make([]managedplugin.Config, 0, len(sources))
-	for _, source := range sources {
-		sourcePluginConfigs = append(sourcePluginConfigs, managedplugin.Config{
+	sourcePluginConfigs := make([]managedplugin.Config, len(sources))
+	sourceRegInferred := make([]bool, len(sources))
+	for i, source := range sources {
+		sourcePluginConfigs[i] = managedplugin.Config{
 			Name:     source.Name,
 			Version:  source.Version,
 			Path:     source.Path,
 			Registry: SpecRegistryToPlugin(source.Registry),
-		})
+		}
+		sourceRegInferred[i] = source.RegistryInferred()
 	}
-	destinationPluginConfigs := make([]managedplugin.Config, 0, len(destinations))
-	for _, destination := range destinations {
-		destinationPluginConfigs = append(destinationPluginConfigs, managedplugin.Config{
+	destinationPluginConfigs := make([]managedplugin.Config, len(destinations))
+	destinationRegInferred := make([]bool, len(destinations))
+	for i, destination := range destinations {
+		destinationPluginConfigs[i] = managedplugin.Config{
 			Name:     destination.Name,
 			Version:  destination.Version,
 			Path:     destination.Path,
 			Registry: SpecRegistryToPlugin(destination.Registry),
-		})
+		}
+		destinationRegInferred[i] = destination.RegistryInferred()
 	}
 
 	managedSourceClients, err := managedplugin.NewClients(ctx, managedplugin.PluginSource, sourcePluginConfigs, opts...)
 	if err != nil {
-		return err
+		return enrichClientError(managedSourceClients, sourceRegInferred, err)
 	}
 	defer func() {
 		if err := managedSourceClients.Terminate(); err != nil {
@@ -99,7 +102,7 @@ func migrate(cmd *cobra.Command, args []string) error {
 	}()
 	destinationPluginClients, err := managedplugin.NewClients(ctx, managedplugin.PluginDestination, destinationPluginConfigs, opts...)
 	if err != nil {
-		return err
+		return enrichClientError(destinationPluginClients, destinationRegInferred, err)
 	}
 	defer func() {
 		if err := destinationPluginClients.Terminate(); err != nil {

diff --git a/cli/cmd/registry.go b/cli/cmd/registry.go
@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"fmt"
+
 	"github.com/cloudquery/cloudquery/cli/internal/specs/v0"
 	"github.com/cloudquery/plugin-pb-go/managedplugin"
 )
@@ -18,6 +20,6 @@ func SpecRegistryToPlugin(registry specs.Registry) managedplugin.Registry {
 	case specs.RegistryCloudQuery:
 		return managedplugin.RegistryCloudQuery
 	default:
-		panic("unknown registry " + registry.String())
+		panic(fmt.Sprintf("unknown registry %q", registry.String()))
 	}
 }
diff --git a/cli/cmd/specs.go b/cli/cmd/specs.go
@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"fmt"
+
 	"github.com/cloudquery/cloudquery/cli/internal/specs/v0"
 	pbSpecs "github.com/cloudquery/plugin-pb-go/specs"
 )
@@ -12,7 +14,7 @@ func CLIBackendToPbBackend(backend specs.Backend) pbSpecs.Backend {
 	case specs.BackendNone:
 		return pbSpecs.BackendNone
 	default:
-		panic("unknown backend " + backend.String())
+		panic(fmt.Sprintf("unknown backend %q", backend.String()))
 	}
 }
 
@@ -27,7 +29,7 @@ func CLIRegistryToPbRegistry(registry specs.Registry) pbSpecs.Registry {
 	case specs.RegistryCloudQuery:
 		return pbSpecs.RegistryCloudQuery
 	default:
-		panic("unknown registry " + registry.String())
+		panic(fmt.Sprintf("unknown registry %q", registry.String()))
 	}
 }
 
@@ -38,7 +40,7 @@ func CLISchedulerToPbScheduler(scheduler specs.Scheduler) pbSpecs.Scheduler {
 	case specs.SchedulerRoundRobin:
 		return pbSpecs.SchedulerRoundRobin
 	default:
-		panic("unknown scheduler " + scheduler.String())
+		panic(fmt.Sprintf("unknown scheduler %q", scheduler.String()))
 	}
 }
 
@@ -76,7 +78,7 @@ func CLIWriteModeToPbWriteMode(writeMode specs.WriteMode) pbSpecs.WriteMode {
 	case specs.WriteModeOverwriteDeleteStale:
 		return pbSpecs.WriteModeOverwriteDeleteStale
 	default:
-		panic("unknown write mode " + writeMode.String())
+		panic(fmt.Sprintf("unknown write mode %q", writeMode.String()))
 	}
 }
 
@@ -87,7 +89,7 @@ func CLIMigrateModeToPbMigrateMode(migrateMode specs.MigrateMode) pbSpecs.Migrat
 	case specs.MigrateModeForced:
 		return pbSpecs.MigrateModeForced
 	default:
-		panic("unknown migrate mode " + migrateMode.String())
+		panic(fmt.Sprintf("unknown migrate mode %q", migrateMode.String()))
 	}
 }
 
@@ -98,7 +100,7 @@ func CLIPkModeToPbPKMode(pkMode specs.PKMode) pbSpecs.PKMode {
 	case specs.PKModeDefaultKeys:
 		return pbSpecs.PKModeDefaultKeys
 	default:
-		panic("unknown pk mode " + pkMode.String())
+		panic(fmt.Sprintf("unknown pk mode %q", pkMode.String()))
 	}
 }
 

diff --git a/cli/cmd/sync.go b/cli/cmd/sync.go
@@ -3,9 +3,8 @@ package cmd
 import (
 	"fmt"
 	"math"
-	"strings"
-
 	"slices"
+	"strings"
 
 	"github.com/google/uuid"
 
@@ -138,7 +137,7 @@ func sync(cmd *cobra.Command, args []string) error {
 		}
 		sourcePluginClient, err := managedplugin.NewClient(ctx, managedplugin.PluginSource, cfg, opts...)
 		if err != nil {
-			return err
+			return enrichClientError(managedplugin.Clients{}, []bool{source.RegistryInferred()}, err)
 		}
 		sourcePluginClients = append(sourcePluginClients, sourcePluginClient)
 	}
@@ -168,7 +167,7 @@ func sync(cmd *cobra.Command, args []string) error {
 		}
 		destPluginClient, err := managedplugin.NewClient(ctx, managedplugin.PluginDestination, cfg, opts...)
 		if err != nil {
-			return err
+			return enrichClientError(managedplugin.Clients{}, []bool{destination.RegistryInferred()}, err)
 		}
 		destinationPluginClients = append(destinationPluginClients, destPluginClient)
 	}